fix(document-processors): escalate glyph-corrupt PDFs to the structured tier

The fast (pypdfium2) extractor can leak raw glyph codes on subset fonts with a
broken /ToUnicode CMap. The result scores high on the existing text-quality
heuristic -- a uniform glyph/Caesar offset preserves whitespace and token
lengths -- yet is unsearchable. The structured (pymupdf) tier extracts the same
pages correctly.

Add a language-agnostic C0-control-character-ratio signal to the tier-0
classifier that detects this corruption and routes the document to a new
`structured` recommended_tier. Wire the fast->structured hop on the inline path
and generalise it so a low-quality-but-non-empty layer also tries structured
before OCR -- the inline and external ingest modes now follow the full
fast->structured->ocr ladder identically. A scanned / no-text-layer document
(total_chars == 0) still shortcuts straight to OCR, since a text extractor
cannot recover a pure raster.

New per-tenant tunable DOCUMENT_GLYPH_CORRUPTION_RATIO (default 0.02); escalation
metrics gain a `corrupt_glyphs` reason label.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

tests/unit/test_config.py

@@ -266,6 +266,30 @@ class TestChunkConfigValidation:
             _reload_config()
             assert get_settings().document_max_pdf_size_mb == pytest.approx(12.5)
 
+    def test_glyph_corruption_ratio_default_and_env_override(self):
+        """document_glyph_corruption_ratio defaults to 0.02 and reads its env var.
+
+        Guards the _DEFAULTS-key-must-match-env-var footgun.
+        """
+        assert Settings().document_glyph_corruption_ratio == pytest.approx(0.02)
+        with patch.dict(
+            os.environ, {"DOCUMENT_GLYPH_CORRUPTION_RATIO": "0.05"}, clear=True
+        ):
+            _reload_config()
+            assert get_settings().document_glyph_corruption_ratio == pytest.approx(0.05)
+
+    @patch.dict(
+        os.environ,
+        {"DOCUMENT_GLYPH_CORRUPTION_RATIO": "1.5"},
+        clear=True,
+    )
+    def test_glyph_corruption_ratio_out_of_range_raises_error(self):
+        """The ratio must be within [0, 1]."""
+        from dynaconf import ValidationError
+
+        with pytest.raises(ValidationError, match="DOCUMENT_GLYPH_CORRUPTION_RATIO"):
+            _reload_config()
+
     def test_valid_chunk_settings(self):
         """Test valid chunk size and overlap configuration."""
         settings = Settings(

tests/unit/test_doc_classifier.py

@@ -320,3 +320,63 @@ def test_scan_coverage_shorter_than_pages_aligns_without_crash():
     assert all(p.needs_ocr is False for p in c.pages)  # coverage no longer routes
     assert "image_heavy" in c.flags  # but page 0 still flags image_heavy
     assert c.recommended_tier == "fast"
+
+
+# --- glyph-corruption signal (broken /ToUnicode -> structured escalation) -----
+
+# pypdfium2-style leak: a uniform glyph/Caesar offset turns clean prose into
+# alphabetic-but-wrong tokens (normal spacing + token length => HIGH text_quality)
+# while digits/punctuation map to C0 control bytes. The control-char ratio is the
+# only signal that catches this; _text_quality scores it ~1.0.
+_GLYPH_CORRUPT = "WKH \x0f TXLFN \x10 EURZQ \x11 IRA MXPSV \x0f RYHU \x10 GRJ " * 6
+
+
+def test_control_char_ratio_clean_is_zero():
+    assert clf._control_char_ratio("the quick brown fox") == 0.0
+    # legitimate whitespace controls (tab/newline/CR/form-feed/vtab) don't count
+    assert clf._control_char_ratio("a\tb\nc\r\nd\f\ve") == 0.0
+
+
+def test_control_char_ratio_detects_glyph_leak():
+    assert clf._control_char_ratio(_GLYPH_CORRUPT) > clf.GLYPH_CORRUPTION_RATIO
+
+
+def test_clean_text_not_flagged_corrupt():
+    txt = "the quick brown fox jumps over the lazy dog " * 3
+    c = clf.classify_from_text(
+        txt, [{"page": 1, "start_offset": 0, "end_offset": len(txt)}]
+    )
+    assert "corrupt_glyphs" not in c.flags
+    assert c.mean_control_ratio == pytest.approx(0.0)
+    assert c.recommended_tier == "fast"
+
+
+def test_glyph_corrupt_routes_structured_not_ocr():
+    full = _GLYPH_CORRUPT
+    c = clf.classify_from_text(
+        full, [{"page": 1, "start_offset": 0, "end_offset": len(full)}]
+    )
+    assert c.recommended_tier == "structured"
+    assert "corrupt_glyphs" in c.flags
+    # The point: it is NOT a low-quality signal -- the cipher scores high, so only
+    # the control-char ratio diverts it (to structured, the free pymupdf re-parse).
+    assert c.mean_text_quality >= clf.MIN_TEXT_QUALITY
+    assert c.mean_control_ratio > clf.GLYPH_CORRUPTION_RATIO
+
+
+def test_glyph_corruption_ratio_override_disables_trigger():
+    full = _GLYPH_CORRUPT
+    bounds = [{"page": 1, "start_offset": 0, "end_offset": len(full)}]
+    # A threshold of 1.0 can never be exceeded => not treated as corrupt => the
+    # other (high-quality) signals win => fast.
+    c = clf.classify_from_text(full, bounds, glyph_corruption_ratio=1.0)
+    assert c.recommended_tier == "fast"
+    assert "corrupt_glyphs" not in c.flags
+
+
+def test_empty_doc_routes_ocr_not_structured():
+    # Precedence: a scanned/empty doc (no text layer) has no control chars to leak,
+    # so it must stay an OCR case, never structured.
+    c = clf.classify_from_text("", [{"page": 1, "start_offset": 0, "end_offset": 0}])
+    assert c.recommended_tier == "ocr"
+    assert "corrupt_glyphs" not in c.flags

tests/unit/test_registry_tiering.py

@@ -24,7 +24,9 @@ class _Fake(DocumentProcessor):
         self,
         name: str,
         tier: str,
-        text: str = "clean text here",
+        # >= MIN_PAGE_CHARS of clean, whitespace-separated prose so the default
+        # classifies "fast" (a shorter string trips the near-empty OCR signal).
+        text: str = "this is clean readable prose text",
         success=True,
         pages: int = 1,
     ):
@@ -75,6 +77,7 @@ class _Settings:
         page_fraction=0.5,
         min_page_chars=16,
         detect_scanned=False,
+        glyph_corruption_ratio=0.02,
         # Guard off by default so existing tiering tests are unaffected; tests
         # that exercise the size guard pass an explicit cap.
         max_pdf_size_mb=0.0,
@@ -86,6 +89,7 @@ class _Settings:
         self.document_ocr_page_fraction = page_fraction
         self.document_ocr_min_page_chars = min_page_chars
         self.document_ocr_detect_scanned = detect_scanned
+        self.document_glyph_corruption_ratio = glyph_corruption_ratio
         self.document_max_pdf_size_mb = max_pdf_size_mb
 
 
@@ -244,6 +248,91 @@ async def test_no_ocr_escalation_when_disabled(monkeypatch):
     assert res.processor == "fast"
 
 
+# --- glyph-corruption escalation + full-ladder parity ------------------------
+
+# A fast-tier text layer that looks like words (normal spacing/token lengths ->
+# HIGH text_quality) but leaks C0 control chars: the broken-/ToUnicode signature
+# the control-char ratio catches. Decodes to a pangram under a -3 shift.
+_GLYPH = "WKH \x0f TXLFN \x10 EURZQ \x11 IRA MXPSV \x0f RYHU \x10 GRJ " * 6
+
+
+async def test_glyph_corrupt_escalates_fast_to_structured(monkeypatch):
+    # Not gated on OCR: structured is free + in-cluster, so a glyph-corrupt layer
+    # escalates fast->structured even with OCR disabled.
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=False))
+    esc = MagicMock()
+    monkeypatch.setattr(reg_mod, "record_document_escalation", esc)
+    r = _registry(
+        (_Fake("fast", "fast", text=_GLYPH), 20),
+        (_Fake("structured", "structured", text="clean recovered prose text"), 10),
+    )
+    res = await r.process(b"%PDF-1.7", "application/pdf")
+    assert res.processor == "structured"
+    esc.assert_called_once_with("fast", "structured", "corrupt_glyphs")
+
+
+async def test_glyph_corrupt_no_structured_stays_fast(monkeypatch):
+    # No structured processor registered -> nothing to escalate to; keep fast.
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=False))
+    r = _registry((_Fake("fast", "fast", text=_GLYPH), 20))
+    res = await r.process(b"%PDF-1.7", "application/pdf")
+    assert res.processor == "fast"
+
+
+async def test_inline_lowconf_tries_structured_before_ocr(monkeypatch):
+    # Full-ladder parity with the external path: a junk-but-non-empty fast layer
+    # tries structured (fast->structured) BEFORE any OCR, even with OCR enabled.
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=True))
+    esc = MagicMock()
+    monkeypatch.setattr(reg_mod, "record_document_escalation", esc)
+    r = _registry(
+        (_Fake("fast", "fast", text="x" * 40), 20),  # one long token -> quality ~0
+        (_Fake("structured", "structured", text="clean recovered prose text here"), 10),
+        (_Fake("ocr", "ocr", text="ocr text"), 5),
+    )
+    res = await r.process(b"%PDF-1.7", "application/pdf")
+    assert res.processor == "structured"
+    esc.assert_called_once_with("fast", "structured", "low_confidence")
+
+
+async def test_inline_empty_skips_structured_straight_to_ocr(monkeypatch):
+    # The one intended shortcut: a scanned/no-text-layer doc (total_chars == 0)
+    # skips structured (it cannot extract text from a raster) and goes to OCR.
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=True))
+    esc = MagicMock()
+    monkeypatch.setattr(reg_mod, "record_document_escalation", esc)
+    r = _registry(
+        (_Fake("fast", "fast", text=""), 20),
+        (_Fake("structured", "structured", text="should not run"), 10),
+        (_Fake("ocr", "ocr", text="ocr text"), 5),
+    )
+    res = await r.process(b"%PDF-1.7", "application/pdf")
+    assert res.processor == "ocr"
+    esc.assert_called_once_with("fast", "ocr", "empty_text")
+
+
+def test_evaluate_escalation_glyph_corrupt_goes_structured(monkeypatch):
+    # External path mirrors the inline path: glyph-corrupt -> structured, never OCR.
+    monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
+    r = _registry(
+        (_Fake("fast", "fast"), 20),
+        (_Fake("structured", "structured"), 10),
+        (_Fake("ocr", "ocr"), 5),
+    )
+    res = ProcessingResult(
+        text=_GLYPH,
+        metadata={
+            "page_count": 1,
+            "page_boundaries": [
+                {"page": 1, "start_offset": 0, "end_offset": len(_GLYPH)}
+            ],
+        },
+        processor="fast",
+    )
+    decision = r.evaluate_escalation(res, b"%PDF", "fast", _Settings(ocr=True))
+    assert decision == EscalationDecision("hop", "structured", "corrupt_glyphs")
+
+
 # --- Per-tier external path (Deck #323) -------------------------------------