feat(usage): meter embedding tokens as embeddings_queries on both paths

embeddings_queries now records the embedding request's token count (the unit
upstream providers bill on) instead of an operation count, and fires on the
indexing path too. Previously only semantic search recorded it (value=1), so a
re-indexing run produced no embeddings_queries events at all — only pages_chunks.

- Provider layer: additive embed_with_usage / embed_batch_with_usage surface the
  per-request token count (Mistral/OpenAI usage.total_tokens, Bedrock Titan
  inputTextTokenCount, Ollama prompt_eval_count); a char-based estimate is the
  fallback (Simple, and any provider/response without a token field). Gateway and
  EmbeddingService forward through. The count travels as a return value / a
  per-request SearchAlgorithm attribute — never on the singleton — so concurrent
  indexing + search can't mis-attribute bills.
- Indexing (vector/processor.py): records embeddings_queries (value=batch tokens)
  alongside the existing pages_chunks event.
- Search (server/semantic.py): value is now the query embedding's token count,
  relayed from BM25HybridSearchAlgorithm via query_token_count.

The astrolabe_embeddings_queries Stripe meter (sum aggregation) now sums tokens
with no CP/Terraform change. The meter "queries"->tokens naming/unit
clarification (homelab-terraform #254) + CP rollup/portal copy is a follow-up.

Deck #67.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

nextcloud_mcp_server/providers/base.py

@@ -1,5 +1,6 @@
 """Unified provider interface for embeddings and text generation."""
 
+import math
 from abc import ABC, abstractmethod
 
 
@@ -55,6 +56,40 @@ class Provider(ABC):
         """
         pass
 
+    @staticmethod
+    def _estimate_tokens(texts: list[str]) -> int:
+        """Best-effort token estimate when a provider returns no usage data.
+
+        Uses a coarse ~4-chars-per-token heuristic so the billable token
+        value stays non-zero and monotone with input size for local/dev
+        providers (Simple, Ollama without ``prompt_eval_count``). Real
+        providers override ``*_with_usage`` to report exact counts.
+        """
+        return math.ceil(sum(len(t) for t in texts) / 4)
+
+    async def embed_with_usage(self, text: str) -> tuple[list[float], int]:
+        """Embed one text and report the request's token count.
+
+        Returns ``(embedding, token_count)``. The default delegates to
+        :meth:`embed` and estimates the tokens; providers that surface real
+        usage from their embedding response override this. Used by the
+        usage-metering hooks (Deck #67) to bill ``embeddings_queries`` by
+        tokens rather than by operation count.
+        """
+        embedding = await self.embed(text)
+        return embedding, self._estimate_tokens([text])
+
+    async def embed_batch_with_usage(
+        self, texts: list[str]
+    ) -> tuple[list[list[float]], int]:
+        """Embed multiple texts and report the total token count.
+
+        Returns ``(embeddings, token_count)``; the default estimates. See
+        :meth:`embed_with_usage`.
+        """
+        embeddings = await self.embed_batch(texts)
+        return embeddings, self._estimate_tokens(texts)
+
     @abstractmethod
     def get_dimension(self) -> int:
         """

nextcloud_mcp_server/providers/bedrock.py

@@ -164,6 +164,16 @@ class BedrockProvider(Provider):
             NotImplementedError: If embeddings not enabled (no embedding_model)
             ClientError: If Bedrock API call fails
         """
+        embedding, _ = await self.embed_with_usage(text)
+        return embedding
+
+    async def embed_with_usage(self, text: str) -> tuple[list[float], int]:
+        """Embed one text, reporting the request's token count.
+
+        Titan Embed responses carry ``inputTextTokenCount``; for Cohere /
+        unknown models (no token field) this falls back to a char-based
+        estimate. Used by the usage-metering hooks (Deck #67).
+        """
         if not self.supports_embeddings:
             raise NotImplementedError(
                 "Embedding not supported - no embedding_model configured"
@@ -182,7 +192,13 @@ class BedrockProvider(Provider):
             response_body = json.loads(response["body"].read())
             embedding = self._parse_embedding_response(response_body)
 
-            return embedding
+            token_count = response_body.get("inputTextTokenCount")
+            tokens = (
+                int(token_count)
+                if isinstance(token_count, (int, float))
+                else self._estimate_tokens([text])
+            )
+            return embedding, tokens
 
         except (BotoCoreError, ClientError) as e:
             logger.error("Bedrock embedding error: %s", e)
@@ -205,16 +221,30 @@ class BedrockProvider(Provider):
             NotImplementedError: If embeddings not enabled (no embedding_model)
             ClientError: If Bedrock API call fails
         """
+        embeddings, _ = await self.embed_batch_with_usage(texts)
+        return embeddings
+
+    async def embed_batch_with_usage(
+        self, texts: list[str]
+    ) -> tuple[list[list[float]], int]:
+        """Embed multiple texts, summing the per-call token counts.
+
+        Bedrock has no batch embedding API, so requests run sequentially and
+        the token total is the sum of each call's ``inputTextTokenCount``
+        (Titan) or estimate (Cohere/unknown).
+        """
         if not self.supports_embeddings:
             raise NotImplementedError(
                 "Embedding not supported - no embedding_model configured"
             )
 
-        embeddings = []
+        embeddings: list[list[float]] = []
+        total_tokens = 0
         for text in texts:
-            embedding = await self.embed(text)
+            embedding, tokens = await self.embed_with_usage(text)
             embeddings.append(embedding)
-        return embeddings
+            total_tokens += tokens
+        return embeddings, total_tokens
 
     async def _detect_dimension(self):
         """

nextcloud_mcp_server/providers/mistral.py

@@ -122,17 +122,42 @@ class MistralProvider(Provider):
 
     async def embed_batch(self, texts: list[str]) -> list[list[float]]:
         """Generate embeddings for multiple texts, chunking by ``BATCH_SIZE``."""
+        embeddings, _ = await self.embed_batch_with_usage(texts)
+        return embeddings
+
+    async def embed_with_usage(self, text: str) -> tuple[list[float], int]:
+        """Embed one text, reporting the Mistral request's token count."""
+        embeddings, tokens = await self.embed_batch_with_usage([text])
+        if not embeddings:
+            raise RuntimeError(
+                f"Mistral embeddings API returned no embedding for model "
+                f"{self.embedding_model}"
+            )
+        return embeddings[0], tokens
+
+    async def embed_batch_with_usage(
+        self, texts: list[str]
+    ) -> tuple[list[list[float]], int]:
+        """Embed multiple texts, summing the Mistral-reported token usage.
+
+        Returns ``(embeddings, total_tokens)`` where ``total_tokens`` is the
+        sum of ``response.usage.total_tokens`` across the ``BATCH_SIZE`` sub-
+        requests (the unit Mistral bills on). Used by the usage-metering hooks
+        to record ``embeddings_queries`` by tokens (Deck #67).
+        """
         if not self.supports_embeddings:
             raise NotImplementedError(_NO_EMBEDDING_MODEL_MSG)
 
         if not texts:
-            return []
+            return [], 0
 
         all_embeddings: list[list[float]] = []
+        total_tokens = 0
         for i in range(0, len(texts), BATCH_SIZE):
             batch = texts[i : i + BATCH_SIZE]
-            batch_embeddings = await self._embed_batch_request(batch)
+            batch_embeddings, batch_tokens = await self._embed_batch_request(batch)
             all_embeddings.extend(batch_embeddings)
+            total_tokens += batch_tokens
 
             if self._dimension is None and batch_embeddings:
                 self._dimension = len(batch_embeddings[0])
@@ -142,11 +167,18 @@ class MistralProvider(Provider):
                     self.embedding_model,
                 )
 
-        return all_embeddings
+        return all_embeddings, total_tokens
 
     @_retry_429
-    async def _embed_batch_request(self, batch: list[str]) -> list[list[float]]:
-        """Single batch request with rate-limit retry."""
+    async def _embed_batch_request(
+        self, batch: list[str]
+    ) -> tuple[list[list[float]], int]:
+        """Single batch request with rate-limit retry.
+
+        Returns ``(embeddings, token_count)``; ``token_count`` comes from the
+        response's ``usage.total_tokens`` and falls back to a char-based
+        estimate if the API omits usage.
+        """
         assert self.embedding_model is not None
         response = await self.client.embeddings.create_async(
             model=self.embedding_model,
@@ -170,7 +202,18 @@ class MistralProvider(Provider):
                 f"Mistral embeddings API returned {len(result)} embeddings "
                 f"for {len(batch)} inputs"
             )
-        return result
+
+        usage = getattr(response, "usage", None)
+        total_tokens = getattr(usage, "total_tokens", None) if usage else None
+        # Guard on numeric type (not just ``is not None``): a real response
+        # gives an int, but test doubles / partial responses can surface a
+        # non-numeric attribute — fall back to the estimate there.
+        tokens = (
+            int(total_tokens)
+            if isinstance(total_tokens, (int, float))
+            else self._estimate_tokens(batch)
+        )
+        return result, tokens
 
     def get_dimension(self) -> int:
         if not self.supports_embeddings:

nextcloud_mcp_server/providers/ollama.py

@@ -116,12 +116,44 @@ class OllamaProvider(Provider):
         Raises:
             NotImplementedError: If embeddings not enabled (no embedding_model)
         """
+        embeddings, _ = await self.embed_batch_with_usage(texts, batch_size=batch_size)
+        return embeddings
+
+    async def embed_with_usage(self, text: str) -> tuple[list[float], int]:
+        """Embed one text, reporting the request's token count.
+
+        Routes through ``/api/embed`` (which carries ``prompt_eval_count``)
+        rather than the legacy ``/api/embeddings`` so a token count is
+        available; falls back to a char-based estimate when the field is
+        absent. Used by the usage-metering hooks (Deck #67).
+        """
+        embeddings, tokens = await self.embed_batch_with_usage([text])
+        if not embeddings:
+            raise RuntimeError(
+                "Ollama embeddings API returned no embedding for model "
+                f"{self.embedding_model}"
+            )
+        return embeddings[0], tokens
+
+    async def embed_batch_with_usage(
+        self, texts: list[str], batch_size: int = 32
+    ) -> tuple[list[list[float]], int]:
+        """Embed multiple texts, summing ``prompt_eval_count`` token usage.
+
+        Returns ``(embeddings, total_tokens)``. Ollama's ``/api/embed`` may
+        omit ``prompt_eval_count`` (older versions); a char-based estimate is
+        used per batch when it does.
+        """
         if not self.supports_embeddings:
             raise NotImplementedError(
                 "Embedding not supported - no embedding_model configured"
             )
 
-        all_embeddings = []
+        if not texts:
+            return [], 0
+
+        all_embeddings: list[list[float]] = []
+        total_tokens = 0
         for i in range(0, len(texts), batch_size):
             batch = texts[i : i + batch_size]
             response = await self.client.post(
@@ -129,9 +161,17 @@ class OllamaProvider(Provider):
                 json={"model": self.embedding_model, "input": batch},
             )
             response.raise_for_status()
-            all_embeddings.extend(response.json()["embeddings"])
+            data = response.json()
+            all_embeddings.extend(data["embeddings"])
 
-        return all_embeddings
+            prompt_eval = data.get("prompt_eval_count")
+            total_tokens += (
+                int(prompt_eval)
+                if isinstance(prompt_eval, (int, float))
+                else self._estimate_tokens(batch)
+            )
+
+        return all_embeddings, total_tokens
 
     async def _detect_dimension(self):
         """

nextcloud_mcp_server/providers/openai.py

@@ -153,19 +153,49 @@ class OpenAIProvider(Provider):
                 "Embedding not supported - no embedding_model configured"
             )
 
+        embeddings, _ = await self.embed_batch_with_usage(texts)
+        return embeddings
+
+    async def embed_with_usage(self, text: str) -> tuple[list[float], int]:
+        """Embed one text, reporting the request's token count."""
+        embeddings, tokens = await self.embed_batch_with_usage([text])
+        if not embeddings:
+            raise RuntimeError(
+                "OpenAI embeddings API returned no embedding for model "
+                f"{self.embedding_model}"
+            )
+        return embeddings[0], tokens
+
+    async def embed_batch_with_usage(
+        self, texts: list[str]
+    ) -> tuple[list[list[float]], int]:
+        """Embed multiple texts, summing the API-reported token usage.
+
+        Returns ``(embeddings, total_tokens)`` where ``total_tokens`` sums
+        ``response.usage.total_tokens`` across the sub-requests (the unit the
+        provider bills on). Used by the usage-metering hooks (Deck #67). Also
+        serves the gateway path via :class:`GatewayProvider`.
+        """
+        if not self.supports_embeddings:
+            raise NotImplementedError(
+                "Embedding not supported - no embedding_model configured"
+            )
+
         if not texts:
-            return []
+            return [], 0
 
         # OpenAI supports batches up to 2048, but use smaller batches for safety
         batch_size = 100
         all_embeddings: list[list[float]] = []
+        total_tokens = 0
 
         for i in range(0, len(texts), batch_size):
             batch = texts[i : i + batch_size]
 
             # Use helper method with retry logic for each batch
-            batch_embeddings = await self._embed_batch_request(batch)
+            batch_embeddings, batch_tokens = await self._embed_batch_request(batch)
             all_embeddings.extend(batch_embeddings)
+            total_tokens += batch_tokens
 
             # Update dimension if not set
             if self._dimension is None and batch_embeddings:
@@ -176,11 +206,18 @@ class OpenAIProvider(Provider):
                     self.embedding_model,
                 )
 
-        return all_embeddings
+        return all_embeddings, total_tokens
 
     @_retry_429
-    async def _embed_batch_request(self, batch: list[str]) -> list[list[float]]:
-        """Make a single batch embedding request with retry logic."""
+    async def _embed_batch_request(
+        self, batch: list[str]
+    ) -> tuple[list[list[float]], int]:
+        """Make a single batch embedding request with retry logic.
+
+        Returns ``(embeddings, token_count)``; ``token_count`` comes from the
+        response's ``usage.total_tokens`` and falls back to a char-based
+        estimate if the API omits usage.
+        """
         assert self.embedding_model is not None  # Type narrowing
         response = await self.client.embeddings.create(
             input=batch,
@@ -188,7 +225,19 @@ class OpenAIProvider(Provider):
         )
         # Sort by index to maintain order
         sorted_data = sorted(response.data, key=lambda x: x.index)
-        return [item.embedding for item in sorted_data]
+        embeddings = [item.embedding for item in sorted_data]
+
+        usage = getattr(response, "usage", None)
+        total_tokens = getattr(usage, "total_tokens", None) if usage else None
+        # Guard on numeric type (not just ``is not None``): a real response
+        # gives an int, but test doubles / partial responses can surface a
+        # non-numeric attribute — fall back to the estimate there.
+        tokens = (
+            int(total_tokens)
+            if isinstance(total_tokens, (int, float))
+            else self._estimate_tokens(batch)
+        )
+        return embeddings, tokens
 
     def get_dimension(self) -> int:
         """