feat(usage): meter embedding tokens as embeddings_queries on both paths
embeddings_queries now records the embedding request's token count (the unit upstream providers bill on) instead of an operation count, and fires on the indexing path too. Previously only semantic search recorded it (value=1), so a re-indexing run produced no embeddings_queries events at all — only pages_chunks. - Provider layer: additive embed_with_usage / embed_batch_with_usage surface the per-request token count (Mistral/OpenAI usage.total_tokens, Bedrock Titan inputTextTokenCount, Ollama prompt_eval_count); a char-based estimate is the fallback (Simple, and any provider/response without a token field). Gateway and EmbeddingService forward through. The count travels as a return value / a per-request SearchAlgorithm attribute — never on the singleton — so concurrent indexing + search can't mis-attribute bills. - Indexing (vector/processor.py): records embeddings_queries (value=batch tokens) alongside the existing pages_chunks event. - Search (server/semantic.py): value is now the query embedding's token count, relayed from BM25HybridSearchAlgorithm via query_token_count. The astrolabe_embeddings_queries Stripe meter (sum aggregation) now sums tokens with no CP/Terraform change. The meter "queries"->tokens naming/unit clarification (homelab-terraform #254) + CP rollup/portal copy is a follow-up. Deck #67. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
fe17994c4d
commit
64318f0b25
@@ -255,6 +255,73 @@ async def test_bedrock_dimension_detection(mock_bedrock_client):
|
||||
assert provider.get_dimension() == 1536
|
||||
|
||||
|
||||
def _titan_body(embedding, token_count=None):
|
||||
payload = {"embedding": embedding}
|
||||
if token_count is not None:
|
||||
payload["inputTextTokenCount"] = token_count
|
||||
return {
|
||||
"body": MagicMock(read=MagicMock(return_value=json.dumps(payload).encode()))
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_bedrock_embed_with_usage_reports_titan_tokens(mock_bedrock_client):
|
||||
"""Titan's inputTextTokenCount is surfaced as the token count."""
|
||||
mock_bedrock_client.invoke_model.return_value = _titan_body(
|
||||
[0.1, 0.2], token_count=6
|
||||
)
|
||||
|
||||
provider = BedrockProvider(
|
||||
region_name="us-east-1",
|
||||
embedding_model="amazon.titan-embed-text-v2:0",
|
||||
generation_model=None,
|
||||
)
|
||||
embedding, tokens = await provider.embed_with_usage("test text")
|
||||
|
||||
assert embedding == [0.1, 0.2]
|
||||
assert tokens == 6
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_bedrock_embed_batch_with_usage_sums_token_counts(mock_bedrock_client):
|
||||
"""Sequential per-text calls sum their inputTextTokenCount values."""
|
||||
mock_bedrock_client.invoke_model.return_value = _titan_body(
|
||||
[0.1, 0.2], token_count=4
|
||||
)
|
||||
|
||||
provider = BedrockProvider(
|
||||
region_name="us-east-1",
|
||||
embedding_model="amazon.titan-embed-text-v2:0",
|
||||
generation_model=None,
|
||||
)
|
||||
embeddings, tokens = await provider.embed_batch_with_usage(["t1", "t2", "t3"])
|
||||
|
||||
assert len(embeddings) == 3
|
||||
assert tokens == 12 # 4 tokens per call × 3 calls
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_bedrock_with_usage_estimates_when_token_count_absent(
|
||||
mock_bedrock_client,
|
||||
):
|
||||
"""Cohere returns no inputTextTokenCount → char-based estimate."""
|
||||
mock_bedrock_client.invoke_model.return_value = {
|
||||
"body": MagicMock(
|
||||
read=MagicMock(
|
||||
return_value=json.dumps({"embeddings": [[0.1, 0.2]]}).encode()
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
provider = BedrockProvider(
|
||||
region_name="us-east-1",
|
||||
embedding_model="cohere.embed-english-v3",
|
||||
)
|
||||
_, tokens = await provider.embed_with_usage("abcdefgh") # 8 chars → 2 tokens
|
||||
|
||||
assert tokens == 2
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_bedrock_cohere_embedding(mock_bedrock_client):
|
||||
"""Test Bedrock with Cohere embedding model."""
|
||||
|
||||
@@ -6,6 +6,7 @@ tenant realm); creds are all-or-nothing.
|
||||
"""
|
||||
|
||||
import time
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
@@ -327,6 +328,69 @@ def test_trailing_slash_base_url_normalized():
|
||||
assert not base.endswith("/v1/v1")
|
||||
|
||||
|
||||
async def test_gateway_embed_with_usage_forwards_after_bearer(monkeypatch):
|
||||
"""embed_with_usage refreshes the bearer, then returns the (embedding,
|
||||
token_count) from the inherited OpenAI implementation."""
|
||||
provider = GatewayProvider(
|
||||
base_url="http://gw:8083/v1", embedding_model="mistral/mistral-embed"
|
||||
)
|
||||
|
||||
order: list[str] = []
|
||||
|
||||
async def _ensure_bearer():
|
||||
order.append("bearer")
|
||||
|
||||
monkeypatch.setattr(provider, "_ensure_bearer", _ensure_bearer)
|
||||
|
||||
item = MagicMock()
|
||||
item.embedding = [0.1, 0.2]
|
||||
item.index = 0
|
||||
response = MagicMock()
|
||||
response.data = [item]
|
||||
response.usage = MagicMock(total_tokens=8)
|
||||
|
||||
async def _create(**_kwargs):
|
||||
order.append("embed")
|
||||
return response
|
||||
|
||||
monkeypatch.setattr(provider.client.embeddings, "create", _create)
|
||||
|
||||
embedding, tokens = await provider.embed_with_usage("hello")
|
||||
|
||||
assert embedding == [0.1, 0.2]
|
||||
assert tokens == 8
|
||||
assert order == ["bearer", "embed"] # bearer refreshed before the embed call
|
||||
|
||||
|
||||
async def test_gateway_embed_batch_with_usage_forwards_after_bearer(monkeypatch):
|
||||
"""embed_batch_with_usage also refreshes the bearer before delegating."""
|
||||
provider = GatewayProvider(
|
||||
base_url="http://gw:8083/v1", embedding_model="mistral/mistral-embed"
|
||||
)
|
||||
ensured = {"n": 0}
|
||||
|
||||
async def _ensure_bearer():
|
||||
ensured["n"] += 1
|
||||
|
||||
monkeypatch.setattr(provider, "_ensure_bearer", _ensure_bearer)
|
||||
|
||||
item = MagicMock()
|
||||
item.embedding = [0.3, 0.4]
|
||||
item.index = 0
|
||||
response = MagicMock()
|
||||
response.data = [item]
|
||||
response.usage = MagicMock(total_tokens=5)
|
||||
monkeypatch.setattr(
|
||||
provider.client.embeddings, "create", AsyncMock(return_value=response)
|
||||
)
|
||||
|
||||
embeddings, tokens = await provider.embed_batch_with_usage(["x"])
|
||||
|
||||
assert embeddings == [[0.3, 0.4]]
|
||||
assert tokens == 5
|
||||
assert ensured["n"] == 1
|
||||
|
||||
|
||||
async def test_detect_dimension_with_bare_base_url_hits_v1_models(monkeypatch):
|
||||
"""End-to-end of the fix: a bare-origin base_url still resolves the
|
||||
dimension because discovery lands on /v1/models."""
|
||||
|
||||
@@ -255,6 +255,68 @@ async def test_mistral_batch_raises_on_count_mismatch(mock_mistral_client):
|
||||
await provider.embed_batch(["a", "b"])
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_mistral_embed_batch_with_usage_reports_tokens(mock_mistral_client):
|
||||
"""embed_batch_with_usage returns the provider-reported total_tokens."""
|
||||
response = _make_response([[0.1, 0.2], [0.3, 0.4]])
|
||||
response.usage = MagicMock(total_tokens=11)
|
||||
mock_mistral_client.embeddings.create_async = AsyncMock(return_value=response)
|
||||
|
||||
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
|
||||
embeddings, tokens = await provider.embed_batch_with_usage(["a", "b"])
|
||||
|
||||
assert embeddings == [[0.1, 0.2], [0.3, 0.4]]
|
||||
assert tokens == 11
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_mistral_embed_batch_with_usage_sums_across_chunks(mock_mistral_client):
|
||||
"""Token counts sum across the BATCH_SIZE sub-requests (1 token/input here)."""
|
||||
|
||||
def _side_effect(*, model, inputs, **_kwargs):
|
||||
resp = _make_response([[float(i)] for i in range(len(inputs))])
|
||||
resp.usage = MagicMock(total_tokens=len(inputs))
|
||||
return resp
|
||||
|
||||
mock_mistral_client.embeddings.create_async = AsyncMock(side_effect=_side_effect)
|
||||
|
||||
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
|
||||
total = BATCH_SIZE * 2 + 5 # three chunks
|
||||
embeddings, tokens = await provider.embed_batch_with_usage(
|
||||
[f"t-{i}" for i in range(total)]
|
||||
)
|
||||
|
||||
assert len(embeddings) == total
|
||||
assert tokens == total # summed across all three chunks
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_mistral_with_usage_estimates_when_usage_absent(mock_mistral_client):
|
||||
"""Missing usage falls back to the char-based estimate, not a crash."""
|
||||
response = _make_response([[0.1, 0.2]])
|
||||
response.usage = None
|
||||
mock_mistral_client.embeddings.create_async = AsyncMock(return_value=response)
|
||||
|
||||
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
|
||||
_, tokens = await provider.embed_batch_with_usage(["abcd"]) # 4 chars → 1 token
|
||||
|
||||
assert tokens == 1
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_mistral_embed_with_usage_single(mock_mistral_client):
|
||||
"""embed_with_usage returns the single embedding plus its token count."""
|
||||
response = _make_response([[0.5, 0.6]])
|
||||
response.usage = MagicMock(total_tokens=3)
|
||||
mock_mistral_client.embeddings.create_async = AsyncMock(return_value=response)
|
||||
|
||||
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
|
||||
embedding, tokens = await provider.embed_with_usage("hello")
|
||||
|
||||
assert embedding == [0.5, 0.6]
|
||||
assert tokens == 3
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_mistral_is_rate_limit_predicate():
|
||||
"""_is_rate_limit returns True only for SDKErrors with status_code == 429."""
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Unit tests for Ollama provider token-usage surfacing.
|
||||
|
||||
The provider has no other unit coverage; these focus on the ``*_with_usage``
|
||||
methods added for usage metering (Deck #67) — provider-reported
|
||||
``prompt_eval_count`` and the char-based estimate fallback when it's absent.
|
||||
"""
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from nextcloud_mcp_server.providers.ollama import OllamaProvider
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ollama_provider():
|
||||
# Construct with no models so __init__ skips _check_model_is_loaded (no
|
||||
# network call), then enable embeddings post-construction.
|
||||
provider = OllamaProvider(base_url="http://ollama:11434")
|
||||
provider.embedding_model = "nomic-embed-text"
|
||||
return provider
|
||||
|
||||
|
||||
def _embed_response(embeddings, prompt_eval_count=None):
|
||||
payload = {"embeddings": embeddings}
|
||||
if prompt_eval_count is not None:
|
||||
payload["prompt_eval_count"] = prompt_eval_count
|
||||
resp = MagicMock()
|
||||
resp.json = MagicMock(return_value=payload)
|
||||
resp.raise_for_status = MagicMock()
|
||||
return resp
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_ollama_embed_batch_with_usage_reports_prompt_eval_count(ollama_provider):
|
||||
"""prompt_eval_count from /api/embed is surfaced as the token count."""
|
||||
ollama_provider.client.post = AsyncMock(
|
||||
return_value=_embed_response([[0.1, 0.2], [0.3, 0.4]], prompt_eval_count=7)
|
||||
)
|
||||
|
||||
embeddings, tokens = await ollama_provider.embed_batch_with_usage(["a", "b"])
|
||||
|
||||
assert embeddings == [[0.1, 0.2], [0.3, 0.4]]
|
||||
assert tokens == 7
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_ollama_with_usage_estimates_when_count_absent(ollama_provider):
|
||||
"""Older Ollama omits prompt_eval_count → char-based estimate."""
|
||||
ollama_provider.client.post = AsyncMock(
|
||||
return_value=_embed_response([[0.1]], prompt_eval_count=None)
|
||||
)
|
||||
|
||||
_, tokens = await ollama_provider.embed_with_usage("abcdefgh") # 8 chars → 2
|
||||
|
||||
assert tokens == 2
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_ollama_empty_batch_with_usage(ollama_provider):
|
||||
"""Empty batch returns no embeddings, zero tokens, and makes no request."""
|
||||
ollama_provider.client.post = AsyncMock()
|
||||
|
||||
embeddings, tokens = await ollama_provider.embed_batch_with_usage([])
|
||||
|
||||
assert embeddings == []
|
||||
assert tokens == 0
|
||||
ollama_provider.client.post.assert_not_called()
|
||||
@@ -280,6 +280,46 @@ async def test_openai_empty_batch():
|
||||
assert embeddings == []
|
||||
|
||||
|
||||
def _embed_item(embedding, index):
|
||||
item = MagicMock()
|
||||
item.embedding = embedding
|
||||
item.index = index
|
||||
return item
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_openai_embed_batch_with_usage_reports_tokens(mock_openai_client):
|
||||
"""embed_batch_with_usage returns the response's total_tokens."""
|
||||
response = MagicMock()
|
||||
response.data = [_embed_item([0.1, 0.2], 0), _embed_item([0.3, 0.4], 1)]
|
||||
response.usage = MagicMock(total_tokens=9)
|
||||
mock_openai_client.embeddings.create = AsyncMock(return_value=response)
|
||||
|
||||
provider = OpenAIProvider(
|
||||
api_key="test-key", embedding_model="text-embedding-3-small"
|
||||
)
|
||||
embeddings, tokens = await provider.embed_batch_with_usage(["a", "b"])
|
||||
|
||||
assert embeddings == [[0.1, 0.2], [0.3, 0.4]]
|
||||
assert tokens == 9
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_openai_with_usage_estimates_when_usage_absent(mock_openai_client):
|
||||
"""Missing usage falls back to the char-based estimate."""
|
||||
response = MagicMock()
|
||||
response.data = [_embed_item([0.1], 0)]
|
||||
response.usage = None
|
||||
mock_openai_client.embeddings.create = AsyncMock(return_value=response)
|
||||
|
||||
provider = OpenAIProvider(
|
||||
api_key="test-key", embedding_model="text-embedding-3-small"
|
||||
)
|
||||
_, tokens = await provider.embed_with_usage("abcdefgh") # 8 chars → 2 tokens
|
||||
|
||||
assert tokens == 2
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_openai_close(mock_openai_client):
|
||||
"""Test OpenAI client close."""
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Token-usage surfacing: the Provider ABC estimate default + SimpleProvider.
|
||||
|
||||
The usage-metering hooks (Deck #67) bill ``embeddings_queries`` by tokens. Real
|
||||
providers report exact counts from their API response; providers without a token
|
||||
field (Simple, and the ABC default) fall back to a char-based estimate so the
|
||||
billable value stays non-zero and monotone with input size.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from nextcloud_mcp_server.providers.base import Provider
|
||||
from nextcloud_mcp_server.providers.simple import SimpleProvider
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_estimate_tokens_is_char_based():
|
||||
"""~4-chars-per-token, ceil-rounded, summed across inputs."""
|
||||
assert Provider._estimate_tokens(["abcd"]) == 1 # 4 chars
|
||||
assert Provider._estimate_tokens(["abcde"]) == 2 # 5 chars → ceil(5/4)
|
||||
assert Provider._estimate_tokens(["ab", "cd"]) == 1 # 4 chars total
|
||||
assert Provider._estimate_tokens([]) == 0
|
||||
assert Provider._estimate_tokens([""]) == 0
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_simple_provider_embed_with_usage_estimates():
|
||||
"""SimpleProvider has no real usage → estimate path via the ABC default."""
|
||||
provider = SimpleProvider(dimension=8)
|
||||
embedding, tokens = await provider.embed_with_usage("abcdefgh") # 8 chars → 2
|
||||
|
||||
assert len(embedding) == 8
|
||||
assert tokens == 2
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_simple_provider_embed_batch_with_usage_estimates():
|
||||
"""Batch estimate sums character counts across all inputs."""
|
||||
provider = SimpleProvider(dimension=8)
|
||||
embeddings, tokens = await provider.embed_batch_with_usage(["abcd", "efgh"])
|
||||
|
||||
assert len(embeddings) == 2
|
||||
assert tokens == 2 # 8 chars total → 2 tokens
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_simple_provider_empty_batch_with_usage():
|
||||
"""Empty batch returns no embeddings and zero tokens."""
|
||||
provider = SimpleProvider(dimension=8)
|
||||
embeddings, tokens = await provider.embed_batch_with_usage([])
|
||||
|
||||
assert embeddings == []
|
||||
assert tokens == 0
|
||||
Reference in New Issue
Block a user