docs(ingest): refresh BatchPending + classifier-vocab docs; add 4-rung suppressed test
Address claude-review round 5 on #922 (verdict: good to merge after the docstring): - BatchPending docstring: the deferred job stays on its own `(ocr-upstream)` tier queue, not the retired `(ocr)` — batch mode is the upstream Mistral path only. - classifier.py: clarify that `recommended_tier == "ocr"` is the classifier's COARSE vocabulary ("needs OCR"), resolved to a concrete rung (ocr-incluster -> ocr-upstream) by the registry — NOT a TIER_LADDER tier name. - Added test_evaluate_escalation_suppressed_targets_incluster_four_rung: with both OCR rungs registered but both flags off, the suppressed what-if-OCR signal names the cheapest ideal rung (ocr-incluster), not ocr-upstream. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
0614709960
commit
d05cbc0dc1
@@ -89,7 +89,10 @@ class DocClassification:
|
||||
total_chars: int
|
||||
mean_text_quality: float
|
||||
ocr_page_fraction: float # fraction of sampled pages flagged needs_ocr
|
||||
recommended_tier: str # "fast" | "structured" | "ocr"
|
||||
# Classifier vocabulary (coarse): "fast" | "structured" | "ocr". "ocr" means
|
||||
# "needs OCR" — the registry resolves it to a concrete rung (ocr-incluster ->
|
||||
# ocr-upstream) via next_available_tier; it is NOT the TIER_LADDER tier name.
|
||||
recommended_tier: str
|
||||
mean_control_ratio: float = 0.0 # doc-level C0-control-char ratio (glyph-leak)
|
||||
flags: set[str] = field(
|
||||
default_factory=set
|
||||
|
||||
@@ -144,7 +144,8 @@ class BatchPending(Exception):
|
||||
``except Exception`` on the indexing path, never counted as a drop/parse
|
||||
error, and never marks the placeholder failed (the doc isn't done yet).
|
||||
Unlike ``EscalateError`` it does NOT change queue — the job stays on its own
|
||||
(``ocr``) tier queue and is simply deferred.
|
||||
(``ocr-upstream``) tier queue and is simply deferred (batch mode is the
|
||||
upstream Mistral path only; the in-cluster rung is synchronous).
|
||||
"""
|
||||
|
||||
def __init__(self, *, retry_in: int) -> None:
|
||||
|
||||
@@ -659,6 +659,31 @@ def test_evaluate_escalation_suppressed_when_ocr_disabled(monkeypatch):
|
||||
assert decision == EscalationDecision("suppressed", "ocr-upstream", "empty_text")
|
||||
|
||||
|
||||
def test_evaluate_escalation_suppressed_targets_incluster_four_rung(monkeypatch):
|
||||
"""Four-rung registry, BOTH OCR flags off: the suppressed what-if-OCR signal
|
||||
names the *cheapest* ideal rung (ocr-incluster), not ocr-upstream, since the
|
||||
ideal-target walk (ignore_ocr_enabled) picks the cheapest registered OCR rung."""
|
||||
monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
|
||||
r = _registry(
|
||||
(_Fake("fast", "fast"), 20),
|
||||
(_Fake("structured", "structured"), 10),
|
||||
(_Fake("ocr-incluster", "ocr-incluster"), 6),
|
||||
(_Fake("ocr-upstream", "ocr-upstream"), 5),
|
||||
)
|
||||
res = ProcessingResult(
|
||||
text="",
|
||||
metadata={
|
||||
"page_count": 1,
|
||||
"page_boundaries": [{"page": 1, "start_offset": 0, "end_offset": 0}],
|
||||
},
|
||||
processor="fast",
|
||||
)
|
||||
decision = r.evaluate_escalation(
|
||||
res, b"%PDF", "fast", _Settings(ocr=False, ocr_incluster=False)
|
||||
)
|
||||
assert decision == EscalationDecision("suppressed", "ocr-incluster", "empty_text")
|
||||
|
||||
|
||||
def test_evaluate_escalation_lowconf_suppressed_when_only_ocr_disabled(monkeypatch):
|
||||
"""fast+ocr only, OCR off, junk text: the next rung is the disabled ocr, so
|
||||
the would-be hop is suppressed (not a structured hop, which isn't registered)."""
|
||||
|
||||
Reference in New Issue
Block a user