fix(document-processors): inline/external parity when structured tier is absent

Address round-3 review on PR #914:
- When a glyph-corrupt doc's structured rung is NOT registered, the inline path
  now falls through to OCR (with reason corrupt_glyphs), mirroring the external
  next_available_tier instead of silently keeping the fast result. A structured
  parse FAILURE remains terminal (tracked via structured_failed), matching the
  external path which does not escalate a failure. Added a debug log for the
  unregistered case and "(OCR not attempted)" to the failure warning.
- Tests: inline + external glyph-corrupt fallthrough to OCR when structured is
  unregistered; glyph-corrupt + junk-quality both-flags precedence (structured
  wins over the bad_text_layer/ocr route).
- Note the total_chars>0 mutual-exclusion with the scanned branch in
  _route_from_signals.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
2026-06-16 20:34:07 +02:00
co-authored by Claude Opus 4.8
parent 24c22f19d6
commit 33aadbcf80
4 changed files with 91 additions and 12 deletions
+15
View File
@@ -395,6 +395,21 @@ def test_empty_doc_routes_ocr_not_structured():
assert "corrupt_glyphs" not in c.flags
def test_glyph_corrupt_takes_precedence_over_junk_text_layer():
# A layer that is BOTH glyph-corrupt (high control ratio) AND junk-quality
# (mashed, no whitespace -> low text_quality): both flags fire, but
# glyph-corrupt wins the route (structured, not ocr) -- the structured
# re-extract is the cheaper correct fix, and re-classification catches any
# residual junk afterwards.
text = "WKHTXLFNEURZQIRAMXPSV\x0f\x10\x11\x0f\x10" * 3
c = clf.classify_from_text(
text, [{"page": 1, "start_offset": 0, "end_offset": len(text)}]
)
assert c.recommended_tier == "structured"
assert "corrupt_glyphs" in c.flags
assert "bad_text_layer" in c.flags
def test_classify_pdf_glyph_corrupt_routes_structured():
# Symmetry with the classify_from_text routing on the standalone/diagnostic
# classify_pdf path (which re-opens the PDF and samples pages).
+41 -1
View File
@@ -273,13 +273,29 @@ async def test_glyph_corrupt_escalates_fast_to_structured(monkeypatch):
async def test_glyph_corrupt_no_structured_stays_fast(monkeypatch):
# No structured processor registered -> nothing to escalate to; keep fast.
# No structured processor registered AND OCR off -> nothing to escalate to;
# keep fast (the inline counterpart of the external "suppressed" outcome).
monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=False))
r = _registry((_Fake("fast", "fast", text=_GLYPH), 20))
res = await r.process(b"%PDF-1.7", "application/pdf")
assert res.processor == "fast"
async def test_glyph_corrupt_no_structured_falls_through_to_ocr(monkeypatch):
# Parity with the external path: structured unregistered but OCR enabled ->
# the glyph-corrupt doc falls through to OCR (not silently kept at fast).
monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=True))
esc = MagicMock()
monkeypatch.setattr(reg_mod, "record_document_escalation", esc)
r = _registry(
(_Fake("fast", "fast", text=_GLYPH), 20),
(_Fake("ocr", "ocr", text="ocr recovered text"), 5),
) # no structured registered
res = await r.process(b"%PDF-1.7", "application/pdf")
assert res.processor == "ocr"
esc.assert_called_once_with("fast", "ocr", "corrupt_glyphs")
async def test_inline_lowconf_tries_structured_before_ocr(monkeypatch):
# Full-ladder parity with the external path: a junk-but-non-empty fast layer
# tries structured (fast->structured) BEFORE any OCR, even with OCR enabled.
@@ -355,6 +371,30 @@ def test_evaluate_escalation_glyph_corrupt_goes_structured(monkeypatch):
assert decision == EscalationDecision("hop", "structured", "corrupt_glyphs")
def test_evaluate_escalation_glyph_corrupt_no_structured_falls_through_to_ocr(
monkeypatch,
):
# External path with structured unregistered: next_available_tier skips the
# missing rung and lands on OCR, keeping the corrupt_glyphs reason.
monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
r = _registry(
(_Fake("fast", "fast"), 20),
(_Fake("ocr", "ocr"), 5),
) # no structured registered
res = ProcessingResult(
text=_GLYPH,
metadata={
"page_count": 1,
"page_boundaries": [
{"page": 1, "start_offset": 0, "end_offset": len(_GLYPH)}
],
},
processor="fast",
)
decision = r.evaluate_escalation(res, b"%PDF", "fast", _Settings(ocr=True))
assert decision == EscalationDecision("hop", "ocr", "corrupt_glyphs")
# --- Per-tier external path (Deck #323) -------------------------------------