feat(metering): pages_embedded = real parsed page count
`pages_embedded` carried an interim chunk count (`len(chunk_texts)`, TODO #282). Reframe it as a charge for *parsing* (PDF page extraction / OCR) rather than a normalized content size: - Parsed files (PDFs) record `pages_embedded` = real `page_count` from the document processor metadata. - Text content (notes, deck cards, news items) is never parsed, carries no `page_count`, and records no `pages_embedded` row — only `tokens_embedded`. There is deliberately no chars/tokens-per-page constant; pages map 1:1 to parsed document pages. `record_indexing_usage` now takes `page_count` and records the two dimensions independently, gating `pages_embedded` on a truthy page count (not the doc_type) so a future non-PDF parsed type stays correct. Stays flag-gated + best-effort. Tests cover parsed-file, text-only, and zero-page cases. Deck #282 (board 8). Billing-model ADR corrected in astrolabe-cloud-website docs/control-plane/usage-metering.md. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
20ce2a1845
commit
9c89a58a07
@@ -1,9 +1,12 @@
|
||||
"""Unit tests for the indexing-path usage-metering helper (Deck #67).
|
||||
|
||||
``record_indexing_usage`` records the two billable events (``pages_embedded`` +
|
||||
``tokens_embedded``) after a document's chunks are embedded. These cover the
|
||||
value mapping, the flag/zero-chunk no-ops, and the best-effort failure path
|
||||
without standing up the full document pipeline.
|
||||
``record_indexing_usage`` records the billable events after a document's chunks
|
||||
are embedded: ``tokens_embedded`` for every document, and ``pages_embedded``
|
||||
only for parsed files (real ``page_count``). Text content (no ``page_count``)
|
||||
meters tokens only — ``pages_embedded`` is a charge for parsing, not content
|
||||
size (card #282). These cover the value mapping, the flag/zero-chunk no-ops, the
|
||||
text-only path, and the best-effort failure path without standing up the full
|
||||
document pipeline.
|
||||
"""
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
@@ -25,8 +28,8 @@ def store_spy(monkeypatch):
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_records_pages_embedded_and_token_count(store_spy):
|
||||
"""Both events fire: pages_embedded = chunk count, tokens_embedded = tokens."""
|
||||
async def test_parsed_file_records_pages_and_tokens(store_spy):
|
||||
"""A parsed PDF fires both events: pages_embedded = real page count."""
|
||||
await processor.record_indexing_usage(
|
||||
enabled=True,
|
||||
provider="mistral",
|
||||
@@ -36,11 +39,13 @@ async def test_records_pages_embedded_and_token_count(store_spy):
|
||||
chunk_count=110,
|
||||
token_count=4242,
|
||||
total_chars=170826,
|
||||
page_count=12,
|
||||
)
|
||||
|
||||
calls = store_spy.record_usage_event.await_args_list
|
||||
by_metric = {c.kwargs["metric"]: c.kwargs["value"] for c in calls}
|
||||
assert by_metric == {"pages_embedded": 110, "tokens_embedded": 4242}
|
||||
# pages_embedded is the real parsed-page count, NOT the chunk count.
|
||||
assert by_metric == {"pages_embedded": 12, "tokens_embedded": 4242}
|
||||
for c in calls:
|
||||
# Hot-path fast-gate + tenant-local attribution metadata.
|
||||
assert c.kwargs["enabled"] is True
|
||||
@@ -50,6 +55,47 @@ async def test_records_pages_embedded_and_token_count(store_spy):
|
||||
assert c.kwargs["metadata"]["doc_type"] == "file"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_text_doc_records_tokens_only(store_spy):
|
||||
"""Unparsed text content (no page_count) meters tokens, never pages."""
|
||||
await processor.record_indexing_usage(
|
||||
enabled=True,
|
||||
provider="mistral",
|
||||
model="mistral-embed",
|
||||
doc_type="note",
|
||||
user_id="alice",
|
||||
chunk_count=4,
|
||||
token_count=512,
|
||||
total_chars=7000,
|
||||
page_count=None,
|
||||
)
|
||||
|
||||
calls = store_spy.record_usage_event.await_args_list
|
||||
by_metric = {c.kwargs["metric"]: c.kwargs["value"] for c in calls}
|
||||
assert by_metric == {"tokens_embedded": 512}
|
||||
assert "pages_embedded" not in by_metric
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_zero_pages_skips_pages(store_spy):
|
||||
"""page_count=0 (e.g. an empty/corrupt PDF) records tokens but no pages."""
|
||||
await processor.record_indexing_usage(
|
||||
enabled=True,
|
||||
provider="mistral",
|
||||
model="mistral-embed",
|
||||
doc_type="file",
|
||||
user_id="alice",
|
||||
chunk_count=4,
|
||||
token_count=99,
|
||||
total_chars=1000,
|
||||
page_count=0,
|
||||
)
|
||||
|
||||
calls = store_spy.record_usage_event.await_args_list
|
||||
by_metric = {c.kwargs["metric"]: c.kwargs["value"] for c in calls}
|
||||
assert by_metric == {"tokens_embedded": 99}
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_disabled_is_noop(store_spy):
|
||||
"""Flag off → no store access, no events."""
|
||||
@@ -62,6 +108,7 @@ async def test_disabled_is_noop(store_spy):
|
||||
chunk_count=10,
|
||||
token_count=20,
|
||||
total_chars=5,
|
||||
page_count=3,
|
||||
)
|
||||
store_spy.record_usage_event.assert_not_awaited()
|
||||
|
||||
@@ -78,6 +125,7 @@ async def test_zero_chunks_is_noop(store_spy):
|
||||
chunk_count=0,
|
||||
token_count=0,
|
||||
total_chars=0,
|
||||
page_count=3,
|
||||
)
|
||||
store_spy.record_usage_event.assert_not_awaited()
|
||||
|
||||
@@ -101,4 +149,5 @@ async def test_store_failure_is_swallowed(monkeypatch):
|
||||
chunk_count=3,
|
||||
token_count=7,
|
||||
total_chars=9,
|
||||
page_count=2,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user