feat(usage): rename metrics → tokens_embedded/pages_embedded + export token cost to Prometheus

Billing product model finalized (Deck #281): bill pages externally, record
tokens internally. Rename the data-plane metric literals to match the now-
canonical contract (Deck #284) — the control plane's METRIC_EVENT_NAMES is
already renamed, so the old names would be unmapped and never sync to Stripe.

Rename (values unchanged):
- embeddings_queries → tokens_embedded (value = real token count, already
  emitted by this PR; the unit upstream providers bill on).
- pages_chunks → pages_embedded (value kept as len(chunk_texts) interim;
  TODO(#282): real normalized "pages indexed" count — real pages for paginated
  types, chars/tokens-per-page constant otherwise — is deferred to the
  instrumentation card, this only lands the name/contract).
- All literals, log strings, docstrings, comments, the migration comment, and
  tests renamed; grep confirms zero old strings remain.

Observability (new): export embedding token cost to Prometheus as
astrolabe_embedding_tokens_total{provider,operation} (operation = index|query)
so the billed cost unit is visible in Grafana, not just the per-tenant billing
DB. Dedicated counter (doesn't inflate the existing chunk/request metrics) and
always-on (independent of USAGE_METERING_ENABLED, so OSS/self-host gets it).
Wired on both the indexing batch embed and the search query embed (query inside
the per-request cache-miss branch, so reused embeddings aren't double-counted).

Note: the rename orphans any pre-existing embeddings_queries/pages_chunks rows
in tenant app DBs (CP no longer maps them) — acceptable; pipeline is inert with
throwaway dev/sandbox data.

Deck #284 (folded into PR #875).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

nextcloud_mcp_server/alembic/versions/20260607_1200_007_add_usage_events.py

@@ -55,7 +55,7 @@ def upgrade() -> None:
             postgresql.TIMESTAMP(timezone=True) if is_pg else sa.TIMESTAMP(),
             nullable=False,
         ),
-        # Catalog metric: 'embeddings_queries' or 'pages_chunks'. Deliberately
+        # Catalog metric: 'tokens_embedded' or 'pages_embedded'. Deliberately
         # an unconstrained Text (no CHECK/enum) — the metric catalog lives in
         # control-plane config, not the app-DB schema. If a third metric is
         # ever added, the CP-side catalog must learn it too, or its rollup will

nextcloud_mcp_server/observability/metrics.py

@@ -338,6 +338,16 @@ embedding_chars_total = Counter(
     ["kind", "provider"],
 )
 
+# Token consumption — the billed cost unit (mirrors the tokens_embedded billing
+# measure, Deck #67). On a dedicated counter (not folded into the chunk/request
+# metrics above) so query embeds don't inflate indexing dashboards; labelled by
+# operation = index | query. Always emitted, independent of USAGE_METERING_ENABLED.
+embedding_tokens_total = Counter(
+    "astrolabe_embedding_tokens_total",
+    "Total embedding tokens consumed (provider-reported or estimated)",
+    ["provider", "operation"],  # operation: index | query
+)
+
 # --- Chunking & indexed-by-type -----------------------------------------------
 
 document_chunks_total = Counter(
@@ -727,6 +737,25 @@ def record_embedding(
             embedding_chars_total.labels(kind=kind, provider=provider).inc(chars)
 
 
+def record_embedding_tokens(provider: str, operation: str, tokens: int) -> None:
+    """Export embedding token consumption to Prometheus.
+
+    Mirrors the ``tokens_embedded`` billing measure (Deck #67) as an always-on
+    observability signal — emitted regardless of ``USAGE_METERING_ENABLED`` so
+    OSS/self-host deployments still see token cost in Grafana.
+
+    Args:
+        provider: Provider family (mistral | openai | bedrock | ollama | simple).
+        operation: ``"index"`` (chunk-batch embedding) or ``"query"`` (search
+            query embedding).
+        tokens: Token count for this embedding request (no-op when ``<= 0``).
+    """
+    if tokens > 0:
+        embedding_tokens_total.labels(provider=provider, operation=operation).inc(
+            tokens
+        )
+
+
 def record_document_chunks(doc_type: str, count: int) -> None:
     """
     Record the number of chunks produced for a document.

nextcloud_mcp_server/providers/base.py

@@ -73,7 +73,7 @@ class Provider(ABC):
         Returns ``(embedding, token_count)``. The default delegates to
         :meth:`embed` and estimates the tokens; providers that surface real
         usage from their embedding response override this. Used by the
-        usage-metering hooks (Deck #67) to bill ``embeddings_queries`` by
+        usage-metering hooks (Deck #67) to bill ``tokens_embedded`` by
         tokens rather than by operation count.
 
         IMPORTANT (recursion invariant): this default calls ``self.embed``. A

nextcloud_mcp_server/providers/mistral.py

@@ -143,7 +143,7 @@ class MistralProvider(Provider):
         Returns ``(embeddings, total_tokens)`` where ``total_tokens`` is the
         sum of ``response.usage.total_tokens`` across the ``BATCH_SIZE`` sub-
         requests (the unit Mistral bills on). Used by the usage-metering hooks
-        to record ``embeddings_queries`` by tokens (Deck #67).
+        to record ``tokens_embedded`` by tokens (Deck #67).
         """
         if not self.supports_embeddings:
             raise NotImplementedError(_NO_EMBEDDING_MODEL_MSG)

nextcloud_mcp_server/search/algorithms.py

@@ -288,7 +288,7 @@ class SearchAlgorithm(ABC):
         query_token_count: Token count of the query embedding request from the
             last search (provider-reported, or estimated). Set by algorithms
             that embed the query so the usage-metering hook can bill
-            ``embeddings_queries`` by tokens (Deck #67). The instance is
+            ``tokens_embedded`` by tokens (Deck #67). The instance is
             per-request, so this side-channel is concurrency-safe.
     """
 

nextcloud_mcp_server/search/bm25_hybrid.py

@@ -9,7 +9,10 @@ from qdrant_client.models import Filter
 
 from nextcloud_mcp_server.config import get_settings
 from nextcloud_mcp_server.embedding import get_bm25_service, get_embedding_service
-from nextcloud_mcp_server.observability.metrics import record_qdrant_operation
+from nextcloud_mcp_server.observability.metrics import (
+    record_embedding_tokens,
+    record_qdrant_operation,
+)
 from nextcloud_mcp_server.observability.tracing import trace_operation
 from nextcloud_mcp_server.search.access_filter import build_base_filter_conditions
 from nextcloud_mcp_server.search.algorithms import (
@@ -158,6 +161,13 @@ class BM25HybridSearchAlgorithm(SearchAlgorithm):
                 self.query_embedding = dense_embedding
                 self.query_token_count = query_tokens
                 self._embedded_query = query
+                # Export query-embedding token cost to Prometheus
+                # (operation=query), mirroring the per-search billing record in
+                # server/semantic.py. Inside the cache-miss branch so a reused
+                # embedding isn't double-counted.
+                record_embedding_tokens(
+                    settings.get_embedding_provider_family(), "query", query_tokens
+                )
         logger.debug("Generated dense embedding (dimension=%s)", len(dense_embedding))
 
         # Generate sparse embedding for BM25 keyword search

nextcloud_mcp_server/server/semantic.py

@@ -64,7 +64,7 @@ async def record_search_usage(
     doc_types: list[str] | None,
     token_count: int | None,
 ) -> None:
-    """Record the billable ``embeddings_queries`` event for one semantic search.
+    """Record the billable ``tokens_embedded`` event for one semantic search.
 
     The value is the query embedding's token count (provider-reported or
     estimated) — the unit upstream providers bill on, and the same metric the
@@ -89,7 +89,7 @@ async def record_search_usage(
     try:
         store = await UsageEventStore.shared()
         await store.record_usage_event(
-            metric="embeddings_queries",
+            metric="tokens_embedded",
             value=token_count or 0,
             metadata={
                 "user_id": user_id,
@@ -111,9 +111,7 @@ async def record_search_usage(
         # (record_usage_event swallows its own write failures). Metering is on,
         # so warn — a silent DEBUG line would hide "operator enabled metering
         # but gets no data".
-        logger.warning(
-            "usage metering hook (embeddings_queries) skipped", exc_info=True
-        )
+        logger.warning("usage metering hook (tokens_embedded) skipped", exc_info=True)
 
 
 def configure_semantic_tools(mcp: FastMCP):
@@ -588,7 +586,7 @@ def configure_semantic_tools(mcp: FastMCP):
             logger.info("Returning %d results from BM25 hybrid search", len(results))
 
             # Usage metering (Deck #67): record the query embedding's token
-            # count as a billable 'embeddings_queries' event. query_token_count
+            # count as a billable 'tokens_embedded' event. query_token_count
             # is set by BM25HybridSearchAlgorithm during the search() above; the
             # doc_types loop reuses one search_algo instance for the same query
             # and the algorithm caches the dense embedding per query, so the

nextcloud_mcp_server/usage/store.py

@@ -102,8 +102,8 @@ class UsageEventStore:
         logged and swallowed — this must never break the caller's operation.
 
         Args:
-            metric: Catalog metric, e.g. ``"embeddings_queries"`` or
-                ``"pages_chunks"``.
+            metric: Catalog metric, e.g. ``"tokens_embedded"`` or
+                ``"pages_embedded"``.
             value: Count/quantity for this event.
             occurred_at: Operation completion time; defaults to now (UTC).
             metadata: Optional rawest-unit context (provider, model, tokens,

nextcloud_mcp_server/vector/processor.py

@@ -23,6 +23,7 @@ from nextcloud_mcp_server.observability.metrics import (
     record_document_chunks,
     record_document_parse_failed,
     record_embedding,
+    record_embedding_tokens,
     record_qdrant_operation,
     record_vector_sync_processing,
     update_vector_sync_queue_size,
@@ -125,10 +126,16 @@ async def record_indexing_usage(
 ) -> None:
     """Record the two billable usage events for one embedded document.
 
-    ``pages_chunks`` is the volume (chunks embedded); ``embeddings_queries`` is
-    the embedding request's token count — the same metric search records, so the
-    meter bills embedding tokens whether they were incurred indexing a document
-    or embedding a query (Deck #67).
+    ``pages_embedded`` is the buyer-facing "pages indexed" dimension;
+    ``tokens_embedded`` is the embedding request's token count — the same metric
+    search records, so the meter bills embedding tokens whether they were
+    incurred indexing a document or embedding a query (Deck #67).
+
+    TODO(#282): ``pages_embedded`` currently carries the raw chunk count
+    (``len(chunk_texts)``) as an interim value. The real normalized "pages
+    indexed" count — real pages for paginated types (PDF/DOCX/PPT), a fixed
+    chars/tokens-per-page constant otherwise — is deferred to instrumentation
+    card #282; this code (card #284) only lands the metric name/contract.
 
     Best-effort and flag-gated: a metering failure is logged and never breaks
     indexing. No-op when metering is disabled or the document produced no chunks
@@ -154,14 +161,19 @@ async def record_indexing_usage(
         # enabled=True: the guard above already confirmed the flag, so the store
         # skips a second uncached Settings build per record (ADR-024).
         # record_usage_event swallows its own write failures, so the two records
-        # are independent; if pages_chunks somehow raised mid-way, embeddings_-
-        # queries would be skipped, leaving an unmatched pages_chunks row —
-        # acceptable under the (day, metric) SUM-aggregation billing model.
+        # are independent; if pages_embedded somehow raised mid-way,
+        # tokens_embedded would be skipped, leaving an unmatched pages_embedded
+        # row — acceptable under the (day, metric) SUM-aggregation billing model.
         await store.record_usage_event(
-            metric="pages_chunks", value=chunk_count, metadata=metadata, enabled=True
+            # TODO(#282): value is the interim chunk count; switch to normalized
+            # real-page count when the per-page constant lands.
+            metric="pages_embedded",
+            value=chunk_count,
+            metadata=metadata,
+            enabled=True,
         )
         await store.record_usage_event(
-            metric="embeddings_queries",
+            metric="tokens_embedded",
             value=token_count,
             metadata=metadata,
             enabled=True,
@@ -662,7 +674,7 @@ async def _index_document(
                     user_id=doc_task.user_id,
                 )
                 # No embedding ran, so no usage is recorded here — stated
-                # explicitly so a "fewer embeddings_queries rows than expected"
+                # explicitly so a "fewer tokens_embedded rows than expected"
                 # audit lands on the dedup path rather than reconstructing it
                 # from Qdrant claim logs.
                 logger.info(
@@ -892,6 +904,9 @@ async def _index_document(
                 chunks=len(chunk_texts),
                 chars=total_chars,
             )
+            # Export token consumption to Prometheus (always-on, independent of
+            # the billing flag) so Grafana sees indexing token cost.
+            record_embedding_tokens(provider, "index", embed_tokens)
             # Usage metering (Deck #67): record the chunk volume +
             # embedding-token count for this document. Best-effort and
             # flag-gated; placed after the embedding succeeds so it can never

tests/unit/providers/test_provider_usage.py

@@ -1,6 +1,6 @@
 """Token-usage surfacing: the Provider ABC estimate default + SimpleProvider.
 
-The usage-metering hooks (Deck #67) bill ``embeddings_queries`` by tokens. Real
+The usage-metering hooks (Deck #67) bill ``tokens_embedded`` by tokens. Real
 providers report exact counts from their API response; providers without a token
 field (Simple, and the ABC default) fall back to a char-based estimate so the
 billable value stays non-zero and monotone with input size.

tests/unit/search/test_bm25_hybrid.py

@@ -85,6 +85,7 @@ def patched_search(monkeypatch):
 
     settings = MagicMock()
     settings.get_collection_name.return_value = "test_collection"
+    settings.get_embedding_provider_family.return_value = "mistral"
     monkeypatch.setattr(
         "nextcloud_mcp_server.search.bm25_hybrid.get_settings", lambda: settings
     )

tests/unit/server/test_semantic_metering.py

@@ -1,6 +1,6 @@
 """Unit tests for the search-path usage-metering helper (Deck #67).
 
-``record_search_usage`` records the billable ``embeddings_queries`` event for a
+``record_search_usage`` records the billable ``tokens_embedded`` event for a
 semantic search. These pin the value mapping (query token count), the flag-off
 no-op, the doc_types metadata bounding, and the best-effort failure path —
 covering the server-tool metering wiring without standing up the full
@@ -38,7 +38,7 @@ async def test_records_query_token_count(store_spy):
 
     store_spy.record_usage_event.assert_awaited_once()
     kwargs = store_spy.record_usage_event.await_args.kwargs
-    assert kwargs["metric"] == "embeddings_queries"
+    assert kwargs["metric"] == "tokens_embedded"
     assert kwargs["value"] == 42
     assert kwargs["enabled"] is True
     assert kwargs["metadata"]["user_id"] == "alice"

tests/unit/test_embedding_metrics.py

@@ -12,7 +12,10 @@ from __future__ import annotations
 import pytest
 
 from nextcloud_mcp_server.config import Settings
-from nextcloud_mcp_server.observability.metrics import record_embedding
+from nextcloud_mcp_server.observability.metrics import (
+    record_embedding,
+    record_embedding_tokens,
+)
 
 pytestmark = pytest.mark.unit
 
@@ -120,3 +123,31 @@ class TestRecordEmbedding:
         assert metric_sample(
             "astrolabe_embedding_requests_total", {**labels, "status": "error"}
         ) == pytest.approx(1.0)
+
+
+class TestRecordEmbeddingTokens:
+    """astrolabe_embedding_tokens_total — token cost split by index/query."""
+
+    def test_index_increments_by_token_count(self, metric_sample):
+        labels = {"provider": "tok-prov", "operation": "index"}
+        before = metric_sample("astrolabe_embedding_tokens_total", labels)
+        record_embedding_tokens("tok-prov", "index", 4242)
+        assert metric_sample(
+            "astrolabe_embedding_tokens_total", labels
+        ) == pytest.approx(before + 4242)
+
+    def test_query_operation_is_separate_series(self, metric_sample):
+        labels = {"provider": "tok-prov", "operation": "query"}
+        before = metric_sample("astrolabe_embedding_tokens_total", labels)
+        record_embedding_tokens("tok-prov", "query", 7)
+        assert metric_sample(
+            "astrolabe_embedding_tokens_total", labels
+        ) == pytest.approx(before + 7)
+
+    def test_zero_or_negative_is_noop(self, metric_sample):
+        labels = {"provider": "tok-noop", "operation": "index"}
+        record_embedding_tokens("tok-noop", "index", 0)
+        record_embedding_tokens("tok-noop", "index", -3)
+        assert metric_sample(
+            "astrolabe_embedding_tokens_total", labels
+        ) == pytest.approx(0.0)

tests/unit/test_processor_metering.py

@@ -1,7 +1,7 @@
 """Unit tests for the indexing-path usage-metering helper (Deck #67).
 
-``record_indexing_usage`` records the two billable events (``pages_chunks`` +
-``embeddings_queries``) after a document's chunks are embedded. These cover the
+``record_indexing_usage`` records the two billable events (``pages_embedded`` +
+``tokens_embedded``) after a document's chunks are embedded. These cover the
 value mapping, the flag/zero-chunk no-ops, and the best-effort failure path
 without standing up the full document pipeline.
 """
@@ -25,8 +25,8 @@ def store_spy(monkeypatch):
 
 
 @pytest.mark.unit
-async def test_records_pages_chunks_and_token_count(store_spy):
-    """Both events fire: pages_chunks = chunk count, embeddings_queries = tokens."""
+async def test_records_pages_embedded_and_token_count(store_spy):
+    """Both events fire: pages_embedded = chunk count, tokens_embedded = tokens."""
     await processor.record_indexing_usage(
         enabled=True,
         provider="mistral",
@@ -40,7 +40,7 @@ async def test_records_pages_chunks_and_token_count(store_spy):
 
     calls = store_spy.record_usage_event.await_args_list
     by_metric = {c.kwargs["metric"]: c.kwargs["value"] for c in calls}
-    assert by_metric == {"pages_chunks": 110, "embeddings_queries": 4242}
+    assert by_metric == {"pages_embedded": 110, "tokens_embedded": 4242}
     for c in calls:
         # Hot-path fast-gate + tenant-local attribution metadata.
         assert c.kwargs["enabled"] is True

tests/unit/test_usage_store.py

@@ -95,7 +95,7 @@ async def test_flag_off_is_noop(storage, monkeypatch):
     """With metering disabled, nothing is written (zero DB work)."""
     _set_metering(monkeypatch, False)
     store = UsageEventStore(storage)
-    await store.record_usage_event(metric="pages_chunks", value=5)
+    await store.record_usage_event(metric="pages_embedded", value=5)
     assert await _count(storage) == 0
 
 
@@ -115,12 +115,12 @@ async def test_enabled_param_short_circuits_without_reading_settings(
     monkeypatch.setattr(store_module, "get_settings", _boom)
     store = UsageEventStore(storage)
 
-    await store.record_usage_event(metric="pages_chunks", value=1, enabled=False)
+    await store.record_usage_event(metric="pages_embedded", value=1, enabled=False)
     assert await _count(storage) == 0
 
     eid = str(uuid.uuid4())
     await store.record_usage_event(
-        metric="pages_chunks", value=1, event_id=eid, enabled=True
+        metric="pages_embedded", value=1, event_id=eid, enabled=True
     )
     assert await _count(storage) == 1
 
@@ -131,7 +131,7 @@ async def test_insert_roundtrip(storage, monkeypatch):
     store = UsageEventStore(storage)
     eid = str(uuid.uuid4())
     await store.record_usage_event(
-        metric="pages_chunks",
+        metric="pages_embedded",
         value=7,
         event_id=eid,
         metadata={"provider": "gateway"},
@@ -140,7 +140,7 @@ async def test_insert_roundtrip(storage, monkeypatch):
     assert row is not None
     # Postgres returns event_id as a uuid.UUID; normalize to str for compare.
     assert str(row[0]) == eid
-    assert row[2] == "pages_chunks"
+    assert row[2] == "pages_embedded"
     assert row[3] == 7
 
 
@@ -149,11 +149,11 @@ async def test_on_conflict_dedup(storage, monkeypatch):
     _set_metering(monkeypatch, True)
     store = UsageEventStore(storage)
     eid = str(uuid.uuid4())
-    await store.record_usage_event(metric="pages_chunks", value=1, event_id=eid)
-    await store.record_usage_event(metric="embeddings_queries", value=99, event_id=eid)
+    await store.record_usage_event(metric="pages_embedded", value=1, event_id=eid)
+    await store.record_usage_event(metric="tokens_embedded", value=99, event_id=eid)
     assert await _count(storage) == 1
     row = await _fetch(storage, eid)
-    assert row[2] == "pages_chunks"  # DO NOTHING, not DO UPDATE
+    assert row[2] == "pages_embedded"  # DO NOTHING, not DO UPDATE
     assert row[3] == 1
 
 
@@ -164,7 +164,7 @@ async def test_metadata_json_roundtrip(storage, monkeypatch):
     eid = str(uuid.uuid4())
     meta = {"provider": "gateway", "model": "titan", "nested": {"chunks": 3}}
     await store.record_usage_event(
-        metric="pages_chunks", value=3, event_id=eid, metadata=meta
+        metric="pages_embedded", value=3, event_id=eid, metadata=meta
     )
     row = await _fetch(storage, eid)
     raw = row[4]
@@ -187,7 +187,7 @@ async def test_occurred_at_roundtrip(storage, monkeypatch):
     eid = str(uuid.uuid4())
     when = datetime(2026, 1, 15, 12, 0, 0, tzinfo=timezone.utc)
     await store.record_usage_event(
-        metric="pages_chunks", value=1, event_id=eid, occurred_at=when
+        metric="pages_embedded", value=1, event_id=eid, occurred_at=when
     )
     row = await _fetch(storage, eid)
     stored = row[1]
@@ -207,7 +207,7 @@ async def test_metadata_none_is_null(storage, monkeypatch):
     store = UsageEventStore(storage)
     eid = str(uuid.uuid4())
     await store.record_usage_event(
-        metric="embeddings_queries", value=1, event_id=eid, metadata=None
+        metric="tokens_embedded", value=1, event_id=eid, metadata=None
     )
     row = await _fetch(storage, eid)
     assert row[4] is None
@@ -233,7 +233,7 @@ async def test_best_effort_swallows_db_errors(storage, monkeypatch, caplog):
 
     # Must not raise.
     with caplog.at_level(logging.WARNING, logger="nextcloud_mcp_server.usage.store"):
-        await store.record_usage_event(metric="pages_chunks", value=1)
+        await store.record_usage_event(metric="pages_embedded", value=1)
 
     assert recorded, "record_db_operation should be called on the error path"
     assert recorded[-1][3] == "error"
@@ -262,7 +262,7 @@ async def test_best_effort_swallows_unserializable_metadata(
     # Must not raise.
     with caplog.at_level(logging.WARNING, logger="nextcloud_mcp_server.usage.store"):
         await store.record_usage_event(
-            metric="pages_chunks", value=1, metadata=bad_metadata
+            metric="pages_embedded", value=1, metadata=bad_metadata
         )
 
     # Nothing was written — the encode failed before the insert.