fix(document-processors): make glyph-corruption ratio of 0 disable the signal

Address round-4 review on PR #914:
- glyph_corruption_ratio <= 0 now disables the signal (previously `control_ratio
  > 0` fired on any single C0 control byte), matching the "0 disables" convention
  used elsewhere (document_max_pdf_size_mb) and the config comment. Add a
  zero-disables test.
- Correct the document_escalation_suppressed_total comment: corrupt_glyphs CAN
  appear there in the narrow case where structured is unregistered and OCR is
  registered-but-disabled (evaluate_escalation follows minimum="structured" past
  the missing rung to a gated-off OCR). Add a test for that suppressed decision.
- Add a test for the double-corruption edge: a structured re-extract that is also
  glyph-corrupt escalates structured->ocr with reason corrupt_glyphs.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
2026-06-16 20:43:25 +02:00
co-authored by Claude Opus 4.8
parent 33aadbcf80
commit 4af7c7104b
4 changed files with 67 additions and 6 deletions
@@ -203,9 +203,15 @@ def _route_from_signals(
Flags are diagnostic and independent of the verdict (e.g. ``image_heavy`` Flags are diagnostic and independent of the verdict (e.g. ``image_heavy``
fires on ANY image-heavy page; the OCR route needs a page FRACTION). fires on ANY image-heavy page; the OCR route needs a page FRACTION).
""" """
# total_chars > 0 also guarantees this never overlaps the scanned branch # glyph_corruption_ratio <= 0 disables the signal (a ratio of 0 would otherwise
# (total_chars == 0), so a doc is never both glyph-corrupt and "scanned". # fire on any single C0 control byte). total_chars > 0 also guarantees this
glyph_corrupt = total_chars > 0 and control_ratio > glyph_corruption_ratio # never overlaps the scanned branch (total_chars == 0), so a doc is never both
# glyph-corrupt and "scanned".
glyph_corrupt = (
glyph_corruption_ratio > 0
and total_chars > 0
and control_ratio > glyph_corruption_ratio
)
flags: set[str] = set() flags: set[str] = set()
if ocr_frac >= page_fraction and total_chars == 0: if ocr_frac >= page_fraction and total_chars == 0:
@@ -287,9 +287,10 @@ document_escalation_total = Counter(
document_escalation_suppressed_total = Counter( document_escalation_suppressed_total = Counter(
"astrolabe_document_escalation_suppressed_total", "astrolabe_document_escalation_suppressed_total",
"Would-be tier escalations suppressed because the target tier is disabled", "Would-be tier escalations suppressed because the target tier is disabled",
# reason: low_confidence | empty_text. (corrupt_glyphs never appears here: it # reason: low_confidence | empty_text | corrupt_glyphs. (corrupt_glyphs lands
# targets the structured tier, which has no enabled-gate -- if registered it # here only in the narrow case where the structured tier is unregistered AND
# runs, else there is nothing to suppress -- so it only ever hops or returns.) # OCR is registered-but-disabled: evaluate_escalation follows minimum="structured"
# past the missing rung to OCR, which is gated off -> suppressed{to_tier="ocr"}.)
["from_tier", "to_tier", "reason"], ["from_tier", "to_tier", "reason"],
) )
+9
View File
@@ -387,6 +387,15 @@ def test_glyph_corruption_ratio_override_disables_trigger():
assert "corrupt_glyphs" not in c.flags assert "corrupt_glyphs" not in c.flags
def test_glyph_corruption_ratio_zero_disables_trigger():
full = _GLYPH_CORRUPT
bounds = [{"page": 1, "start_offset": 0, "end_offset": len(full)}]
# 0 disables the signal (rather than firing on any single control byte).
c = clf.classify_from_text(full, bounds, glyph_corruption_ratio=0.0)
assert c.recommended_tier == "fast"
assert "corrupt_glyphs" not in c.flags
def test_empty_doc_routes_ocr_not_structured(): def test_empty_doc_routes_ocr_not_structured():
# Precedence: a scanned/empty doc (no text layer) has no control chars to leak, # Precedence: a scanned/empty doc (no text layer) has no control chars to leak,
# so it must stay an OCR case, never structured. # so it must stay an OCR case, never structured.
+45
View File
@@ -349,6 +349,26 @@ async def test_inline_fast_structured_ocr_cascade(monkeypatch):
] ]
async def test_inline_structured_still_corrupt_escalates_to_ocr(monkeypatch):
# Edge: the structured re-extract is ALSO glyph-corrupt (pymupdf also failed to
# decode). Re-classification stays "structured", so the OCR gate fires --
# attributed from_tier="structured" with reason corrupt_glyphs.
monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=True))
esc = MagicMock()
monkeypatch.setattr(reg_mod, "record_document_escalation", esc)
r = _registry(
(_Fake("fast", "fast", text=_GLYPH), 20),
(_Fake("structured", "structured", text=_GLYPH), 10), # still corrupt
(_Fake("ocr", "ocr", text="ocr recovered text"), 5),
)
res = await r.process(b"%PDF-1.7", "application/pdf")
assert res.processor == "ocr"
assert esc.call_args_list == [
call("fast", "structured", "corrupt_glyphs"),
call("structured", "ocr", "corrupt_glyphs"),
]
def test_evaluate_escalation_glyph_corrupt_goes_structured(monkeypatch): def test_evaluate_escalation_glyph_corrupt_goes_structured(monkeypatch):
# External path mirrors the inline path: glyph-corrupt -> structured, never OCR. # External path mirrors the inline path: glyph-corrupt -> structured, never OCR.
monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock()) monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
@@ -395,6 +415,31 @@ def test_evaluate_escalation_glyph_corrupt_no_structured_falls_through_to_ocr(
assert decision == EscalationDecision("hop", "ocr", "corrupt_glyphs") assert decision == EscalationDecision("hop", "ocr", "corrupt_glyphs")
def test_evaluate_escalation_glyph_corrupt_no_structured_ocr_disabled_suppressed(
monkeypatch,
):
# Structured unregistered AND OCR registered-but-disabled: the would-be OCR
# fallthrough is suppressed, and it carries the corrupt_glyphs reason (so the
# "what-if OCR" counter can show latent glyph-corruption demand).
monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
r = _registry(
(_Fake("fast", "fast"), 20),
(_Fake("ocr", "ocr"), 5),
) # structured not registered; ocr registered but disabled below
res = ProcessingResult(
text=_GLYPH,
metadata={
"page_count": 1,
"page_boundaries": [
{"page": 1, "start_offset": 0, "end_offset": len(_GLYPH)}
],
},
processor="fast",
)
decision = r.evaluate_escalation(res, b"%PDF", "fast", _Settings(ocr=False))
assert decision == EscalationDecision("suppressed", "ocr", "corrupt_glyphs")
# --- Per-tier external path (Deck #323) ------------------------------------- # --- Per-tier external path (Deck #323) -------------------------------------