fix(document-processors): inline/external parity when structured tier is absent
Address round-3 review on PR #914: - When a glyph-corrupt doc's structured rung is NOT registered, the inline path now falls through to OCR (with reason corrupt_glyphs), mirroring the external next_available_tier instead of silently keeping the fast result. A structured parse FAILURE remains terminal (tracked via structured_failed), matching the external path which does not escalate a failure. Added a debug log for the unregistered case and "(OCR not attempted)" to the failure warning. - Tests: inline + external glyph-corrupt fallthrough to OCR when structured is unregistered; glyph-corrupt + junk-quality both-flags precedence (structured wins over the bad_text_layer/ocr route). - Note the total_chars>0 mutual-exclusion with the scanned branch in _route_from_signals. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
24c22f19d6
commit
33aadbcf80
@@ -395,6 +395,21 @@ def test_empty_doc_routes_ocr_not_structured():
|
||||
assert "corrupt_glyphs" not in c.flags
|
||||
|
||||
|
||||
def test_glyph_corrupt_takes_precedence_over_junk_text_layer():
|
||||
# A layer that is BOTH glyph-corrupt (high control ratio) AND junk-quality
|
||||
# (mashed, no whitespace -> low text_quality): both flags fire, but
|
||||
# glyph-corrupt wins the route (structured, not ocr) -- the structured
|
||||
# re-extract is the cheaper correct fix, and re-classification catches any
|
||||
# residual junk afterwards.
|
||||
text = "WKHTXLFNEURZQIRAMXPSV\x0f\x10\x11\x0f\x10" * 3
|
||||
c = clf.classify_from_text(
|
||||
text, [{"page": 1, "start_offset": 0, "end_offset": len(text)}]
|
||||
)
|
||||
assert c.recommended_tier == "structured"
|
||||
assert "corrupt_glyphs" in c.flags
|
||||
assert "bad_text_layer" in c.flags
|
||||
|
||||
|
||||
def test_classify_pdf_glyph_corrupt_routes_structured():
|
||||
# Symmetry with the classify_from_text routing on the standalone/diagnostic
|
||||
# classify_pdf path (which re-opens the PDF and samples pages).
|
||||
|
||||
@@ -273,13 +273,29 @@ async def test_glyph_corrupt_escalates_fast_to_structured(monkeypatch):
|
||||
|
||||
|
||||
async def test_glyph_corrupt_no_structured_stays_fast(monkeypatch):
|
||||
# No structured processor registered -> nothing to escalate to; keep fast.
|
||||
# No structured processor registered AND OCR off -> nothing to escalate to;
|
||||
# keep fast (the inline counterpart of the external "suppressed" outcome).
|
||||
monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=False))
|
||||
r = _registry((_Fake("fast", "fast", text=_GLYPH), 20))
|
||||
res = await r.process(b"%PDF-1.7", "application/pdf")
|
||||
assert res.processor == "fast"
|
||||
|
||||
|
||||
async def test_glyph_corrupt_no_structured_falls_through_to_ocr(monkeypatch):
|
||||
# Parity with the external path: structured unregistered but OCR enabled ->
|
||||
# the glyph-corrupt doc falls through to OCR (not silently kept at fast).
|
||||
monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=True))
|
||||
esc = MagicMock()
|
||||
monkeypatch.setattr(reg_mod, "record_document_escalation", esc)
|
||||
r = _registry(
|
||||
(_Fake("fast", "fast", text=_GLYPH), 20),
|
||||
(_Fake("ocr", "ocr", text="ocr recovered text"), 5),
|
||||
) # no structured registered
|
||||
res = await r.process(b"%PDF-1.7", "application/pdf")
|
||||
assert res.processor == "ocr"
|
||||
esc.assert_called_once_with("fast", "ocr", "corrupt_glyphs")
|
||||
|
||||
|
||||
async def test_inline_lowconf_tries_structured_before_ocr(monkeypatch):
|
||||
# Full-ladder parity with the external path: a junk-but-non-empty fast layer
|
||||
# tries structured (fast->structured) BEFORE any OCR, even with OCR enabled.
|
||||
@@ -355,6 +371,30 @@ def test_evaluate_escalation_glyph_corrupt_goes_structured(monkeypatch):
|
||||
assert decision == EscalationDecision("hop", "structured", "corrupt_glyphs")
|
||||
|
||||
|
||||
def test_evaluate_escalation_glyph_corrupt_no_structured_falls_through_to_ocr(
|
||||
monkeypatch,
|
||||
):
|
||||
# External path with structured unregistered: next_available_tier skips the
|
||||
# missing rung and lands on OCR, keeping the corrupt_glyphs reason.
|
||||
monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
|
||||
r = _registry(
|
||||
(_Fake("fast", "fast"), 20),
|
||||
(_Fake("ocr", "ocr"), 5),
|
||||
) # no structured registered
|
||||
res = ProcessingResult(
|
||||
text=_GLYPH,
|
||||
metadata={
|
||||
"page_count": 1,
|
||||
"page_boundaries": [
|
||||
{"page": 1, "start_offset": 0, "end_offset": len(_GLYPH)}
|
||||
],
|
||||
},
|
||||
processor="fast",
|
||||
)
|
||||
decision = r.evaluate_escalation(res, b"%PDF", "fast", _Settings(ocr=True))
|
||||
assert decision == EscalationDecision("hop", "ocr", "corrupt_glyphs")
|
||||
|
||||
|
||||
# --- Per-tier external path (Deck #323) -------------------------------------
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user