docs(ingest): refresh BatchPending + classifier-vocab docs; add 4-rung suppressed test

Address claude-review round 5 on #922 (verdict: good to merge after the docstring):
- BatchPending docstring: the deferred job stays on its own `(ocr-upstream)` tier
  queue, not the retired `(ocr)` — batch mode is the upstream Mistral path only.
- classifier.py: clarify that `recommended_tier == "ocr"` is the classifier's
  COARSE vocabulary ("needs OCR"), resolved to a concrete rung
  (ocr-incluster -> ocr-upstream) by the registry — NOT a TIER_LADDER tier name.
- Added test_evaluate_escalation_suppressed_targets_incluster_four_rung: with both
  OCR rungs registered but both flags off, the suppressed what-if-OCR signal names
  the cheapest ideal rung (ocr-incluster), not ocr-upstream.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
2026-06-18 01:02:32 +02:00
co-authored by Claude Opus 4.8
parent 0614709960
commit d05cbc0dc1
3 changed files with 31 additions and 2 deletions
@@ -89,7 +89,10 @@ class DocClassification:
total_chars: int
mean_text_quality: float
ocr_page_fraction: float # fraction of sampled pages flagged needs_ocr
recommended_tier: str # "fast" | "structured" | "ocr"
# Classifier vocabulary (coarse): "fast" | "structured" | "ocr". "ocr" means
# "needs OCR" — the registry resolves it to a concrete rung (ocr-incluster ->
# ocr-upstream) via next_available_tier; it is NOT the TIER_LADDER tier name.
recommended_tier: str
mean_control_ratio: float = 0.0 # doc-level C0-control-char ratio (glyph-leak)
flags: set[str] = field(
default_factory=set
@@ -144,7 +144,8 @@ class BatchPending(Exception):
``except Exception`` on the indexing path, never counted as a drop/parse
error, and never marks the placeholder failed (the doc isn't done yet).
Unlike ``EscalateError`` it does NOT change queue — the job stays on its own
(``ocr``) tier queue and is simply deferred.
(``ocr-upstream``) tier queue and is simply deferred (batch mode is the
upstream Mistral path only; the in-cluster rung is synchronous).
"""
def __init__(self, *, retry_in: int) -> None:
+25
View File
@@ -659,6 +659,31 @@ def test_evaluate_escalation_suppressed_when_ocr_disabled(monkeypatch):
assert decision == EscalationDecision("suppressed", "ocr-upstream", "empty_text")
def test_evaluate_escalation_suppressed_targets_incluster_four_rung(monkeypatch):
"""Four-rung registry, BOTH OCR flags off: the suppressed what-if-OCR signal
names the *cheapest* ideal rung (ocr-incluster), not ocr-upstream, since the
ideal-target walk (ignore_ocr_enabled) picks the cheapest registered OCR rung."""
monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
r = _registry(
(_Fake("fast", "fast"), 20),
(_Fake("structured", "structured"), 10),
(_Fake("ocr-incluster", "ocr-incluster"), 6),
(_Fake("ocr-upstream", "ocr-upstream"), 5),
)
res = ProcessingResult(
text="",
metadata={
"page_count": 1,
"page_boundaries": [{"page": 1, "start_offset": 0, "end_offset": 0}],
},
processor="fast",
)
decision = r.evaluate_escalation(
res, b"%PDF", "fast", _Settings(ocr=False, ocr_incluster=False)
)
assert decision == EscalationDecision("suppressed", "ocr-incluster", "empty_text")
def test_evaluate_escalation_lowconf_suppressed_when_only_ocr_disabled(monkeypatch):
"""fast+ocr only, OCR off, junk text: the next rung is the disabled ocr, so
the would-be hop is suppressed (not a structured hop, which isn't registered)."""