fix(document-processors): make glyph-corruption ratio of 0 disable the signal
Address round-4 review on PR #914: - glyph_corruption_ratio <= 0 now disables the signal (previously `control_ratio > 0` fired on any single C0 control byte), matching the "0 disables" convention used elsewhere (document_max_pdf_size_mb) and the config comment. Add a zero-disables test. - Correct the document_escalation_suppressed_total comment: corrupt_glyphs CAN appear there in the narrow case where structured is unregistered and OCR is registered-but-disabled (evaluate_escalation follows minimum="structured" past the missing rung to a gated-off OCR). Add a test for that suppressed decision. - Add a test for the double-corruption edge: a structured re-extract that is also glyph-corrupt escalates structured->ocr with reason corrupt_glyphs. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
33aadbcf80
commit
4af7c7104b
@@ -203,9 +203,15 @@ def _route_from_signals(
|
|||||||
Flags are diagnostic and independent of the verdict (e.g. ``image_heavy``
|
Flags are diagnostic and independent of the verdict (e.g. ``image_heavy``
|
||||||
fires on ANY image-heavy page; the OCR route needs a page FRACTION).
|
fires on ANY image-heavy page; the OCR route needs a page FRACTION).
|
||||||
"""
|
"""
|
||||||
# total_chars > 0 also guarantees this never overlaps the scanned branch
|
# glyph_corruption_ratio <= 0 disables the signal (a ratio of 0 would otherwise
|
||||||
# (total_chars == 0), so a doc is never both glyph-corrupt and "scanned".
|
# fire on any single C0 control byte). total_chars > 0 also guarantees this
|
||||||
glyph_corrupt = total_chars > 0 and control_ratio > glyph_corruption_ratio
|
# never overlaps the scanned branch (total_chars == 0), so a doc is never both
|
||||||
|
# glyph-corrupt and "scanned".
|
||||||
|
glyph_corrupt = (
|
||||||
|
glyph_corruption_ratio > 0
|
||||||
|
and total_chars > 0
|
||||||
|
and control_ratio > glyph_corruption_ratio
|
||||||
|
)
|
||||||
|
|
||||||
flags: set[str] = set()
|
flags: set[str] = set()
|
||||||
if ocr_frac >= page_fraction and total_chars == 0:
|
if ocr_frac >= page_fraction and total_chars == 0:
|
||||||
|
|||||||
@@ -287,9 +287,10 @@ document_escalation_total = Counter(
|
|||||||
document_escalation_suppressed_total = Counter(
|
document_escalation_suppressed_total = Counter(
|
||||||
"astrolabe_document_escalation_suppressed_total",
|
"astrolabe_document_escalation_suppressed_total",
|
||||||
"Would-be tier escalations suppressed because the target tier is disabled",
|
"Would-be tier escalations suppressed because the target tier is disabled",
|
||||||
# reason: low_confidence | empty_text. (corrupt_glyphs never appears here: it
|
# reason: low_confidence | empty_text | corrupt_glyphs. (corrupt_glyphs lands
|
||||||
# targets the structured tier, which has no enabled-gate -- if registered it
|
# here only in the narrow case where the structured tier is unregistered AND
|
||||||
# runs, else there is nothing to suppress -- so it only ever hops or returns.)
|
# OCR is registered-but-disabled: evaluate_escalation follows minimum="structured"
|
||||||
|
# past the missing rung to OCR, which is gated off -> suppressed{to_tier="ocr"}.)
|
||||||
["from_tier", "to_tier", "reason"],
|
["from_tier", "to_tier", "reason"],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -387,6 +387,15 @@ def test_glyph_corruption_ratio_override_disables_trigger():
|
|||||||
assert "corrupt_glyphs" not in c.flags
|
assert "corrupt_glyphs" not in c.flags
|
||||||
|
|
||||||
|
|
||||||
|
def test_glyph_corruption_ratio_zero_disables_trigger():
|
||||||
|
full = _GLYPH_CORRUPT
|
||||||
|
bounds = [{"page": 1, "start_offset": 0, "end_offset": len(full)}]
|
||||||
|
# 0 disables the signal (rather than firing on any single control byte).
|
||||||
|
c = clf.classify_from_text(full, bounds, glyph_corruption_ratio=0.0)
|
||||||
|
assert c.recommended_tier == "fast"
|
||||||
|
assert "corrupt_glyphs" not in c.flags
|
||||||
|
|
||||||
|
|
||||||
def test_empty_doc_routes_ocr_not_structured():
|
def test_empty_doc_routes_ocr_not_structured():
|
||||||
# Precedence: a scanned/empty doc (no text layer) has no control chars to leak,
|
# Precedence: a scanned/empty doc (no text layer) has no control chars to leak,
|
||||||
# so it must stay an OCR case, never structured.
|
# so it must stay an OCR case, never structured.
|
||||||
|
|||||||
@@ -349,6 +349,26 @@ async def test_inline_fast_structured_ocr_cascade(monkeypatch):
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_inline_structured_still_corrupt_escalates_to_ocr(monkeypatch):
|
||||||
|
# Edge: the structured re-extract is ALSO glyph-corrupt (pymupdf also failed to
|
||||||
|
# decode). Re-classification stays "structured", so the OCR gate fires --
|
||||||
|
# attributed from_tier="structured" with reason corrupt_glyphs.
|
||||||
|
monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=True))
|
||||||
|
esc = MagicMock()
|
||||||
|
monkeypatch.setattr(reg_mod, "record_document_escalation", esc)
|
||||||
|
r = _registry(
|
||||||
|
(_Fake("fast", "fast", text=_GLYPH), 20),
|
||||||
|
(_Fake("structured", "structured", text=_GLYPH), 10), # still corrupt
|
||||||
|
(_Fake("ocr", "ocr", text="ocr recovered text"), 5),
|
||||||
|
)
|
||||||
|
res = await r.process(b"%PDF-1.7", "application/pdf")
|
||||||
|
assert res.processor == "ocr"
|
||||||
|
assert esc.call_args_list == [
|
||||||
|
call("fast", "structured", "corrupt_glyphs"),
|
||||||
|
call("structured", "ocr", "corrupt_glyphs"),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def test_evaluate_escalation_glyph_corrupt_goes_structured(monkeypatch):
|
def test_evaluate_escalation_glyph_corrupt_goes_structured(monkeypatch):
|
||||||
# External path mirrors the inline path: glyph-corrupt -> structured, never OCR.
|
# External path mirrors the inline path: glyph-corrupt -> structured, never OCR.
|
||||||
monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
|
monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
|
||||||
@@ -395,6 +415,31 @@ def test_evaluate_escalation_glyph_corrupt_no_structured_falls_through_to_ocr(
|
|||||||
assert decision == EscalationDecision("hop", "ocr", "corrupt_glyphs")
|
assert decision == EscalationDecision("hop", "ocr", "corrupt_glyphs")
|
||||||
|
|
||||||
|
|
||||||
|
def test_evaluate_escalation_glyph_corrupt_no_structured_ocr_disabled_suppressed(
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
# Structured unregistered AND OCR registered-but-disabled: the would-be OCR
|
||||||
|
# fallthrough is suppressed, and it carries the corrupt_glyphs reason (so the
|
||||||
|
# "what-if OCR" counter can show latent glyph-corruption demand).
|
||||||
|
monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
|
||||||
|
r = _registry(
|
||||||
|
(_Fake("fast", "fast"), 20),
|
||||||
|
(_Fake("ocr", "ocr"), 5),
|
||||||
|
) # structured not registered; ocr registered but disabled below
|
||||||
|
res = ProcessingResult(
|
||||||
|
text=_GLYPH,
|
||||||
|
metadata={
|
||||||
|
"page_count": 1,
|
||||||
|
"page_boundaries": [
|
||||||
|
{"page": 1, "start_offset": 0, "end_offset": len(_GLYPH)}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
processor="fast",
|
||||||
|
)
|
||||||
|
decision = r.evaluate_escalation(res, b"%PDF", "fast", _Settings(ocr=False))
|
||||||
|
assert decision == EscalationDecision("suppressed", "ocr", "corrupt_glyphs")
|
||||||
|
|
||||||
|
|
||||||
# --- Per-tier external path (Deck #323) -------------------------------------
|
# --- Per-tier external path (Deck #323) -------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user