feat(usage): rename metrics → tokens_embedded/pages_embedded + export token cost to Prometheus
Billing product model finalized (Deck #281): bill pages externally, record tokens internally. Rename the data-plane metric literals to match the now- canonical contract (Deck #284) — the control plane's METRIC_EVENT_NAMES is already renamed, so the old names would be unmapped and never sync to Stripe. Rename (values unchanged): - embeddings_queries → tokens_embedded (value = real token count, already emitted by this PR; the unit upstream providers bill on). - pages_chunks → pages_embedded (value kept as len(chunk_texts) interim; TODO(#282): real normalized "pages indexed" count — real pages for paginated types, chars/tokens-per-page constant otherwise — is deferred to the instrumentation card, this only lands the name/contract). - All literals, log strings, docstrings, comments, the migration comment, and tests renamed; grep confirms zero old strings remain. Observability (new): export embedding token cost to Prometheus as astrolabe_embedding_tokens_total{provider,operation} (operation = index|query) so the billed cost unit is visible in Grafana, not just the per-tenant billing DB. Dedicated counter (doesn't inflate the existing chunk/request metrics) and always-on (independent of USAGE_METERING_ENABLED, so OSS/self-host gets it). Wired on both the indexing batch embed and the search query embed (query inside the per-request cache-miss branch, so reused embeddings aren't double-counted). Note: the rename orphans any pre-existing embeddings_queries/pages_chunks rows in tenant app DBs (CP no longer maps them) — acceptable; pipeline is inert with throwaway dev/sandbox data. Deck #284 (folded into PR #875). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
ddefb03701
commit
973f80e7b9
@@ -55,7 +55,7 @@ def upgrade() -> None:
|
||||
postgresql.TIMESTAMP(timezone=True) if is_pg else sa.TIMESTAMP(),
|
||||
nullable=False,
|
||||
),
|
||||
# Catalog metric: 'embeddings_queries' or 'pages_chunks'. Deliberately
|
||||
# Catalog metric: 'tokens_embedded' or 'pages_embedded'. Deliberately
|
||||
# an unconstrained Text (no CHECK/enum) — the metric catalog lives in
|
||||
# control-plane config, not the app-DB schema. If a third metric is
|
||||
# ever added, the CP-side catalog must learn it too, or its rollup will
|
||||
|
||||
@@ -338,6 +338,16 @@ embedding_chars_total = Counter(
|
||||
["kind", "provider"],
|
||||
)
|
||||
|
||||
# Token consumption — the billed cost unit (mirrors the tokens_embedded billing
|
||||
# measure, Deck #67). On a dedicated counter (not folded into the chunk/request
|
||||
# metrics above) so query embeds don't inflate indexing dashboards; labelled by
|
||||
# operation = index | query. Always emitted, independent of USAGE_METERING_ENABLED.
|
||||
embedding_tokens_total = Counter(
|
||||
"astrolabe_embedding_tokens_total",
|
||||
"Total embedding tokens consumed (provider-reported or estimated)",
|
||||
["provider", "operation"], # operation: index | query
|
||||
)
|
||||
|
||||
# --- Chunking & indexed-by-type -----------------------------------------------
|
||||
|
||||
document_chunks_total = Counter(
|
||||
@@ -727,6 +737,25 @@ def record_embedding(
|
||||
embedding_chars_total.labels(kind=kind, provider=provider).inc(chars)
|
||||
|
||||
|
||||
def record_embedding_tokens(provider: str, operation: str, tokens: int) -> None:
|
||||
"""Export embedding token consumption to Prometheus.
|
||||
|
||||
Mirrors the ``tokens_embedded`` billing measure (Deck #67) as an always-on
|
||||
observability signal — emitted regardless of ``USAGE_METERING_ENABLED`` so
|
||||
OSS/self-host deployments still see token cost in Grafana.
|
||||
|
||||
Args:
|
||||
provider: Provider family (mistral | openai | bedrock | ollama | simple).
|
||||
operation: ``"index"`` (chunk-batch embedding) or ``"query"`` (search
|
||||
query embedding).
|
||||
tokens: Token count for this embedding request (no-op when ``<= 0``).
|
||||
"""
|
||||
if tokens > 0:
|
||||
embedding_tokens_total.labels(provider=provider, operation=operation).inc(
|
||||
tokens
|
||||
)
|
||||
|
||||
|
||||
def record_document_chunks(doc_type: str, count: int) -> None:
|
||||
"""
|
||||
Record the number of chunks produced for a document.
|
||||
|
||||
@@ -73,7 +73,7 @@ class Provider(ABC):
|
||||
Returns ``(embedding, token_count)``. The default delegates to
|
||||
:meth:`embed` and estimates the tokens; providers that surface real
|
||||
usage from their embedding response override this. Used by the
|
||||
usage-metering hooks (Deck #67) to bill ``embeddings_queries`` by
|
||||
usage-metering hooks (Deck #67) to bill ``tokens_embedded`` by
|
||||
tokens rather than by operation count.
|
||||
|
||||
IMPORTANT (recursion invariant): this default calls ``self.embed``. A
|
||||
|
||||
@@ -143,7 +143,7 @@ class MistralProvider(Provider):
|
||||
Returns ``(embeddings, total_tokens)`` where ``total_tokens`` is the
|
||||
sum of ``response.usage.total_tokens`` across the ``BATCH_SIZE`` sub-
|
||||
requests (the unit Mistral bills on). Used by the usage-metering hooks
|
||||
to record ``embeddings_queries`` by tokens (Deck #67).
|
||||
to record ``tokens_embedded`` by tokens (Deck #67).
|
||||
"""
|
||||
if not self.supports_embeddings:
|
||||
raise NotImplementedError(_NO_EMBEDDING_MODEL_MSG)
|
||||
|
||||
@@ -288,7 +288,7 @@ class SearchAlgorithm(ABC):
|
||||
query_token_count: Token count of the query embedding request from the
|
||||
last search (provider-reported, or estimated). Set by algorithms
|
||||
that embed the query so the usage-metering hook can bill
|
||||
``embeddings_queries`` by tokens (Deck #67). The instance is
|
||||
``tokens_embedded`` by tokens (Deck #67). The instance is
|
||||
per-request, so this side-channel is concurrency-safe.
|
||||
"""
|
||||
|
||||
|
||||
@@ -9,7 +9,10 @@ from qdrant_client.models import Filter
|
||||
|
||||
from nextcloud_mcp_server.config import get_settings
|
||||
from nextcloud_mcp_server.embedding import get_bm25_service, get_embedding_service
|
||||
from nextcloud_mcp_server.observability.metrics import record_qdrant_operation
|
||||
from nextcloud_mcp_server.observability.metrics import (
|
||||
record_embedding_tokens,
|
||||
record_qdrant_operation,
|
||||
)
|
||||
from nextcloud_mcp_server.observability.tracing import trace_operation
|
||||
from nextcloud_mcp_server.search.access_filter import build_base_filter_conditions
|
||||
from nextcloud_mcp_server.search.algorithms import (
|
||||
@@ -158,6 +161,13 @@ class BM25HybridSearchAlgorithm(SearchAlgorithm):
|
||||
self.query_embedding = dense_embedding
|
||||
self.query_token_count = query_tokens
|
||||
self._embedded_query = query
|
||||
# Export query-embedding token cost to Prometheus
|
||||
# (operation=query), mirroring the per-search billing record in
|
||||
# server/semantic.py. Inside the cache-miss branch so a reused
|
||||
# embedding isn't double-counted.
|
||||
record_embedding_tokens(
|
||||
settings.get_embedding_provider_family(), "query", query_tokens
|
||||
)
|
||||
logger.debug("Generated dense embedding (dimension=%s)", len(dense_embedding))
|
||||
|
||||
# Generate sparse embedding for BM25 keyword search
|
||||
|
||||
@@ -64,7 +64,7 @@ async def record_search_usage(
|
||||
doc_types: list[str] | None,
|
||||
token_count: int | None,
|
||||
) -> None:
|
||||
"""Record the billable ``embeddings_queries`` event for one semantic search.
|
||||
"""Record the billable ``tokens_embedded`` event for one semantic search.
|
||||
|
||||
The value is the query embedding's token count (provider-reported or
|
||||
estimated) — the unit upstream providers bill on, and the same metric the
|
||||
@@ -89,7 +89,7 @@ async def record_search_usage(
|
||||
try:
|
||||
store = await UsageEventStore.shared()
|
||||
await store.record_usage_event(
|
||||
metric="embeddings_queries",
|
||||
metric="tokens_embedded",
|
||||
value=token_count or 0,
|
||||
metadata={
|
||||
"user_id": user_id,
|
||||
@@ -111,9 +111,7 @@ async def record_search_usage(
|
||||
# (record_usage_event swallows its own write failures). Metering is on,
|
||||
# so warn — a silent DEBUG line would hide "operator enabled metering
|
||||
# but gets no data".
|
||||
logger.warning(
|
||||
"usage metering hook (embeddings_queries) skipped", exc_info=True
|
||||
)
|
||||
logger.warning("usage metering hook (tokens_embedded) skipped", exc_info=True)
|
||||
|
||||
|
||||
def configure_semantic_tools(mcp: FastMCP):
|
||||
@@ -588,7 +586,7 @@ def configure_semantic_tools(mcp: FastMCP):
|
||||
logger.info("Returning %d results from BM25 hybrid search", len(results))
|
||||
|
||||
# Usage metering (Deck #67): record the query embedding's token
|
||||
# count as a billable 'embeddings_queries' event. query_token_count
|
||||
# count as a billable 'tokens_embedded' event. query_token_count
|
||||
# is set by BM25HybridSearchAlgorithm during the search() above; the
|
||||
# doc_types loop reuses one search_algo instance for the same query
|
||||
# and the algorithm caches the dense embedding per query, so the
|
||||
|
||||
@@ -102,8 +102,8 @@ class UsageEventStore:
|
||||
logged and swallowed — this must never break the caller's operation.
|
||||
|
||||
Args:
|
||||
metric: Catalog metric, e.g. ``"embeddings_queries"`` or
|
||||
``"pages_chunks"``.
|
||||
metric: Catalog metric, e.g. ``"tokens_embedded"`` or
|
||||
``"pages_embedded"``.
|
||||
value: Count/quantity for this event.
|
||||
occurred_at: Operation completion time; defaults to now (UTC).
|
||||
metadata: Optional rawest-unit context (provider, model, tokens,
|
||||
|
||||
@@ -23,6 +23,7 @@ from nextcloud_mcp_server.observability.metrics import (
|
||||
record_document_chunks,
|
||||
record_document_parse_failed,
|
||||
record_embedding,
|
||||
record_embedding_tokens,
|
||||
record_qdrant_operation,
|
||||
record_vector_sync_processing,
|
||||
update_vector_sync_queue_size,
|
||||
@@ -125,10 +126,16 @@ async def record_indexing_usage(
|
||||
) -> None:
|
||||
"""Record the two billable usage events for one embedded document.
|
||||
|
||||
``pages_chunks`` is the volume (chunks embedded); ``embeddings_queries`` is
|
||||
the embedding request's token count — the same metric search records, so the
|
||||
meter bills embedding tokens whether they were incurred indexing a document
|
||||
or embedding a query (Deck #67).
|
||||
``pages_embedded`` is the buyer-facing "pages indexed" dimension;
|
||||
``tokens_embedded`` is the embedding request's token count — the same metric
|
||||
search records, so the meter bills embedding tokens whether they were
|
||||
incurred indexing a document or embedding a query (Deck #67).
|
||||
|
||||
TODO(#282): ``pages_embedded`` currently carries the raw chunk count
|
||||
(``len(chunk_texts)``) as an interim value. The real normalized "pages
|
||||
indexed" count — real pages for paginated types (PDF/DOCX/PPT), a fixed
|
||||
chars/tokens-per-page constant otherwise — is deferred to instrumentation
|
||||
card #282; this code (card #284) only lands the metric name/contract.
|
||||
|
||||
Best-effort and flag-gated: a metering failure is logged and never breaks
|
||||
indexing. No-op when metering is disabled or the document produced no chunks
|
||||
@@ -154,14 +161,19 @@ async def record_indexing_usage(
|
||||
# enabled=True: the guard above already confirmed the flag, so the store
|
||||
# skips a second uncached Settings build per record (ADR-024).
|
||||
# record_usage_event swallows its own write failures, so the two records
|
||||
# are independent; if pages_chunks somehow raised mid-way, embeddings_-
|
||||
# queries would be skipped, leaving an unmatched pages_chunks row —
|
||||
# acceptable under the (day, metric) SUM-aggregation billing model.
|
||||
# are independent; if pages_embedded somehow raised mid-way,
|
||||
# tokens_embedded would be skipped, leaving an unmatched pages_embedded
|
||||
# row — acceptable under the (day, metric) SUM-aggregation billing model.
|
||||
await store.record_usage_event(
|
||||
metric="pages_chunks", value=chunk_count, metadata=metadata, enabled=True
|
||||
# TODO(#282): value is the interim chunk count; switch to normalized
|
||||
# real-page count when the per-page constant lands.
|
||||
metric="pages_embedded",
|
||||
value=chunk_count,
|
||||
metadata=metadata,
|
||||
enabled=True,
|
||||
)
|
||||
await store.record_usage_event(
|
||||
metric="embeddings_queries",
|
||||
metric="tokens_embedded",
|
||||
value=token_count,
|
||||
metadata=metadata,
|
||||
enabled=True,
|
||||
@@ -662,7 +674,7 @@ async def _index_document(
|
||||
user_id=doc_task.user_id,
|
||||
)
|
||||
# No embedding ran, so no usage is recorded here — stated
|
||||
# explicitly so a "fewer embeddings_queries rows than expected"
|
||||
# explicitly so a "fewer tokens_embedded rows than expected"
|
||||
# audit lands on the dedup path rather than reconstructing it
|
||||
# from Qdrant claim logs.
|
||||
logger.info(
|
||||
@@ -892,6 +904,9 @@ async def _index_document(
|
||||
chunks=len(chunk_texts),
|
||||
chars=total_chars,
|
||||
)
|
||||
# Export token consumption to Prometheus (always-on, independent of
|
||||
# the billing flag) so Grafana sees indexing token cost.
|
||||
record_embedding_tokens(provider, "index", embed_tokens)
|
||||
# Usage metering (Deck #67): record the chunk volume +
|
||||
# embedding-token count for this document. Best-effort and
|
||||
# flag-gated; placed after the embedding succeeds so it can never
|
||||
|
||||
Reference in New Issue
Block a user