fix(document-processors): escalate glyph-corrupt PDFs to the structured tier

The fast (pypdfium2) extractor can leak raw glyph codes on subset fonts with a
broken /ToUnicode CMap. The result scores high on the existing text-quality
heuristic -- a uniform glyph/Caesar offset preserves whitespace and token
lengths -- yet is unsearchable. The structured (pymupdf) tier extracts the same
pages correctly.

Add a language-agnostic C0-control-character-ratio signal to the tier-0
classifier that detects this corruption and routes the document to a new
`structured` recommended_tier. Wire the fast->structured hop on the inline path
and generalise it so a low-quality-but-non-empty layer also tries structured
before OCR -- the inline and external ingest modes now follow the full
fast->structured->ocr ladder identically. A scanned / no-text-layer document
(total_chars == 0) still shortcuts straight to OCR, since a text extractor
cannot recover a pure raster.

New per-tenant tunable DOCUMENT_GLYPH_CORRUPTION_RATIO (default 0.02); escalation
metrics gain a `corrupt_glyphs` reason label.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

tests/unit/test_registry_tiering.py

@@ -24,7 +24,9 @@ class _Fake(DocumentProcessor):
         self,
         name: str,
         tier: str,
-        text: str = "clean text here",
+        # >= MIN_PAGE_CHARS of clean, whitespace-separated prose so the default
+        # classifies "fast" (a shorter string trips the near-empty OCR signal).
+        text: str = "this is clean readable prose text",
         success=True,
         pages: int = 1,
     ):
@@ -75,6 +77,7 @@ class _Settings:
         page_fraction=0.5,
         min_page_chars=16,
         detect_scanned=False,
+        glyph_corruption_ratio=0.02,
         # Guard off by default so existing tiering tests are unaffected; tests
         # that exercise the size guard pass an explicit cap.
         max_pdf_size_mb=0.0,
@@ -86,6 +89,7 @@ class _Settings:
         self.document_ocr_page_fraction = page_fraction
         self.document_ocr_min_page_chars = min_page_chars
         self.document_ocr_detect_scanned = detect_scanned
+        self.document_glyph_corruption_ratio = glyph_corruption_ratio
         self.document_max_pdf_size_mb = max_pdf_size_mb
 
 
@@ -244,6 +248,91 @@ async def test_no_ocr_escalation_when_disabled(monkeypatch):
     assert res.processor == "fast"
 
 
+# --- glyph-corruption escalation + full-ladder parity ------------------------
+
+# A fast-tier text layer that looks like words (normal spacing/token lengths ->
+# HIGH text_quality) but leaks C0 control chars: the broken-/ToUnicode signature
+# the control-char ratio catches. Decodes to a pangram under a -3 shift.
+_GLYPH = "WKH \x0f TXLFN \x10 EURZQ \x11 IRA MXPSV \x0f RYHU \x10 GRJ " * 6
+
+
+async def test_glyph_corrupt_escalates_fast_to_structured(monkeypatch):
+    # Not gated on OCR: structured is free + in-cluster, so a glyph-corrupt layer
+    # escalates fast->structured even with OCR disabled.
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=False))
+    esc = MagicMock()
+    monkeypatch.setattr(reg_mod, "record_document_escalation", esc)
+    r = _registry(
+        (_Fake("fast", "fast", text=_GLYPH), 20),
+        (_Fake("structured", "structured", text="clean recovered prose text"), 10),
+    )
+    res = await r.process(b"%PDF-1.7", "application/pdf")
+    assert res.processor == "structured"
+    esc.assert_called_once_with("fast", "structured", "corrupt_glyphs")
+
+
+async def test_glyph_corrupt_no_structured_stays_fast(monkeypatch):
+    # No structured processor registered -> nothing to escalate to; keep fast.
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=False))
+    r = _registry((_Fake("fast", "fast", text=_GLYPH), 20))
+    res = await r.process(b"%PDF-1.7", "application/pdf")
+    assert res.processor == "fast"
+
+
+async def test_inline_lowconf_tries_structured_before_ocr(monkeypatch):
+    # Full-ladder parity with the external path: a junk-but-non-empty fast layer
+    # tries structured (fast->structured) BEFORE any OCR, even with OCR enabled.
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=True))
+    esc = MagicMock()
+    monkeypatch.setattr(reg_mod, "record_document_escalation", esc)
+    r = _registry(
+        (_Fake("fast", "fast", text="x" * 40), 20),  # one long token -> quality ~0
+        (_Fake("structured", "structured", text="clean recovered prose text here"), 10),
+        (_Fake("ocr", "ocr", text="ocr text"), 5),
+    )
+    res = await r.process(b"%PDF-1.7", "application/pdf")
+    assert res.processor == "structured"
+    esc.assert_called_once_with("fast", "structured", "low_confidence")
+
+
+async def test_inline_empty_skips_structured_straight_to_ocr(monkeypatch):
+    # The one intended shortcut: a scanned/no-text-layer doc (total_chars == 0)
+    # skips structured (it cannot extract text from a raster) and goes to OCR.
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=True))
+    esc = MagicMock()
+    monkeypatch.setattr(reg_mod, "record_document_escalation", esc)
+    r = _registry(
+        (_Fake("fast", "fast", text=""), 20),
+        (_Fake("structured", "structured", text="should not run"), 10),
+        (_Fake("ocr", "ocr", text="ocr text"), 5),
+    )
+    res = await r.process(b"%PDF-1.7", "application/pdf")
+    assert res.processor == "ocr"
+    esc.assert_called_once_with("fast", "ocr", "empty_text")
+
+
+def test_evaluate_escalation_glyph_corrupt_goes_structured(monkeypatch):
+    # External path mirrors the inline path: glyph-corrupt -> structured, never OCR.
+    monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
+    r = _registry(
+        (_Fake("fast", "fast"), 20),
+        (_Fake("structured", "structured"), 10),
+        (_Fake("ocr", "ocr"), 5),
+    )
+    res = ProcessingResult(
+        text=_GLYPH,
+        metadata={
+            "page_count": 1,
+            "page_boundaries": [
+                {"page": 1, "start_offset": 0, "end_offset": len(_GLYPH)}
+            ],
+        },
+        processor="fast",
+    )
+    decision = r.evaluate_escalation(res, b"%PDF", "fast", _Settings(ocr=True))
+    assert decision == EscalationDecision("hop", "structured", "corrupt_glyphs")
+
+
 # --- Per-tier external path (Deck #323) -------------------------------------