feat(metering): pages_embedded = real parsed page count

`pages_embedded` carried an interim chunk count (`len(chunk_texts)`,
TODO #282). Reframe it as a charge for *parsing* (PDF page extraction /
OCR) rather than a normalized content size:

- Parsed files (PDFs) record `pages_embedded` = real `page_count` from
  the document processor metadata.
- Text content (notes, deck cards, news items) is never parsed, carries
  no `page_count`, and records no `pages_embedded` row — only
  `tokens_embedded`. There is deliberately no chars/tokens-per-page
  constant; pages map 1:1 to parsed document pages.

`record_indexing_usage` now takes `page_count` and records the two
dimensions independently, gating `pages_embedded` on a truthy page count
(not the doc_type) so a future non-PDF parsed type stays correct. Stays
flag-gated + best-effort. Tests cover parsed-file, text-only, and
zero-page cases.

Deck #282 (board 8). Billing-model ADR corrected in
astrolabe-cloud-website docs/control-plane/usage-metering.md.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
2026-06-08 16:58:31 +02:00
co-authored by Claude Opus 4.8
parent 20ce2a1845
commit 9c89a58a07
2 changed files with 102 additions and 33 deletions
+56 -7
View File
@@ -1,9 +1,12 @@
"""Unit tests for the indexing-path usage-metering helper (Deck #67).
``record_indexing_usage`` records the two billable events (``pages_embedded`` +
``tokens_embedded``) after a document's chunks are embedded. These cover the
value mapping, the flag/zero-chunk no-ops, and the best-effort failure path
without standing up the full document pipeline.
``record_indexing_usage`` records the billable events after a document's chunks
are embedded: ``tokens_embedded`` for every document, and ``pages_embedded``
only for parsed files (real ``page_count``). Text content (no ``page_count``)
meters tokens only — ``pages_embedded`` is a charge for parsing, not content
size (card #282). These cover the value mapping, the flag/zero-chunk no-ops, the
text-only path, and the best-effort failure path without standing up the full
document pipeline.
"""
from unittest.mock import AsyncMock, MagicMock
@@ -25,8 +28,8 @@ def store_spy(monkeypatch):
@pytest.mark.unit
async def test_records_pages_embedded_and_token_count(store_spy):
"""Both events fire: pages_embedded = chunk count, tokens_embedded = tokens."""
async def test_parsed_file_records_pages_and_tokens(store_spy):
"""A parsed PDF fires both events: pages_embedded = real page count."""
await processor.record_indexing_usage(
enabled=True,
provider="mistral",
@@ -36,11 +39,13 @@ async def test_records_pages_embedded_and_token_count(store_spy):
chunk_count=110,
token_count=4242,
total_chars=170826,
page_count=12,
)
calls = store_spy.record_usage_event.await_args_list
by_metric = {c.kwargs["metric"]: c.kwargs["value"] for c in calls}
assert by_metric == {"pages_embedded": 110, "tokens_embedded": 4242}
# pages_embedded is the real parsed-page count, NOT the chunk count.
assert by_metric == {"pages_embedded": 12, "tokens_embedded": 4242}
for c in calls:
# Hot-path fast-gate + tenant-local attribution metadata.
assert c.kwargs["enabled"] is True
@@ -50,6 +55,47 @@ async def test_records_pages_embedded_and_token_count(store_spy):
assert c.kwargs["metadata"]["doc_type"] == "file"
@pytest.mark.unit
async def test_text_doc_records_tokens_only(store_spy):
"""Unparsed text content (no page_count) meters tokens, never pages."""
await processor.record_indexing_usage(
enabled=True,
provider="mistral",
model="mistral-embed",
doc_type="note",
user_id="alice",
chunk_count=4,
token_count=512,
total_chars=7000,
page_count=None,
)
calls = store_spy.record_usage_event.await_args_list
by_metric = {c.kwargs["metric"]: c.kwargs["value"] for c in calls}
assert by_metric == {"tokens_embedded": 512}
assert "pages_embedded" not in by_metric
@pytest.mark.unit
async def test_zero_pages_skips_pages(store_spy):
"""page_count=0 (e.g. an empty/corrupt PDF) records tokens but no pages."""
await processor.record_indexing_usage(
enabled=True,
provider="mistral",
model="mistral-embed",
doc_type="file",
user_id="alice",
chunk_count=4,
token_count=99,
total_chars=1000,
page_count=0,
)
calls = store_spy.record_usage_event.await_args_list
by_metric = {c.kwargs["metric"]: c.kwargs["value"] for c in calls}
assert by_metric == {"tokens_embedded": 99}
@pytest.mark.unit
async def test_disabled_is_noop(store_spy):
"""Flag off → no store access, no events."""
@@ -62,6 +108,7 @@ async def test_disabled_is_noop(store_spy):
chunk_count=10,
token_count=20,
total_chars=5,
page_count=3,
)
store_spy.record_usage_event.assert_not_awaited()
@@ -78,6 +125,7 @@ async def test_zero_chunks_is_noop(store_spy):
chunk_count=0,
token_count=0,
total_chars=0,
page_count=3,
)
store_spy.record_usage_event.assert_not_awaited()
@@ -101,4 +149,5 @@ async def test_store_failure_is_swallowed(monkeypatch):
chunk_count=3,
token_count=7,
total_chars=9,
page_count=2,
)