Merge pull request #914 from cbcoutinho/fix/glyph-corruption-structured-escalation

fix(document-processors): escalate glyph-corrupt PDFs to the structured tier

tests/fixtures/glyph_corruption.py

@@ -0,0 +1,11 @@
+"""Shared test data for the tier-0 glyph-corruption signal.
+
+A fast-tier text layer that looks like words -- normal spacing and token lengths,
+so it scores HIGH on ``_text_quality`` -- but leaks C0 control characters: the
+broken-/ToUnicode signature that ``classifier._control_char_ratio`` catches. The
+alphabetic tokens decode to a pangram under a -3 (Caesar) shift.
+
+Kept in one place so the classifier and registry tiering tests can't diverge.
+"""
+
+GLYPH_CORRUPT_TEXT = "WKH \x0f TXLFN \x10 EURZQ \x11 IRA MXPSV \x0f RYHU \x10 GRJ " * 6

tests/unit/test_config.py

@@ -266,6 +266,30 @@ class TestChunkConfigValidation:
             _reload_config()
             assert get_settings().document_max_pdf_size_mb == pytest.approx(12.5)
 
+    def test_glyph_corruption_ratio_default_and_env_override(self):
+        """document_glyph_corruption_ratio defaults to 0.02 and reads its env var.
+
+        Guards the _DEFAULTS-key-must-match-env-var footgun.
+        """
+        assert Settings().document_glyph_corruption_ratio == pytest.approx(0.02)
+        with patch.dict(
+            os.environ, {"DOCUMENT_GLYPH_CORRUPTION_RATIO": "0.05"}, clear=True
+        ):
+            _reload_config()
+            assert get_settings().document_glyph_corruption_ratio == pytest.approx(0.05)
+
+    @patch.dict(
+        os.environ,
+        {"DOCUMENT_GLYPH_CORRUPTION_RATIO": "1.5"},
+        clear=True,
+    )
+    def test_glyph_corruption_ratio_out_of_range_raises_error(self):
+        """The ratio must be within [0, 1]."""
+        from dynaconf import ValidationError
+
+        with pytest.raises(ValidationError, match="DOCUMENT_GLYPH_CORRUPTION_RATIO"):
+            _reload_config()
+
     def test_valid_chunk_settings(self):
         """Test valid chunk size and overlap configuration."""
         settings = Settings(

tests/unit/test_doc_classifier.py

@@ -14,6 +14,7 @@ import pymupdf
 import pytest
 
 from nextcloud_mcp_server.document_processors import classifier as clf
+from tests.fixtures.glyph_corruption import GLYPH_CORRUPT_TEXT
 
 pytestmark = pytest.mark.unit
 
@@ -30,6 +31,18 @@ def _digital_pdf(
     return data
 
 
+def _glyph_corrupt_pdf(pages: int = 2) -> bytes:
+    # A born-digital PDF whose text layer carries the glyph-leak control chars,
+    # for the classify_pdf (diagnostic) path. pymupdf round-trips the C0 controls.
+    doc = pymupdf.open()
+    for _ in range(pages):
+        page = doc.new_page(width=595, height=842)
+        page.insert_text((50, 60), GLYPH_CORRUPT_TEXT)
+    data: bytes = doc.tobytes()
+    doc.close()
+    return data
+
+
 def _full_page_image_pdf(pages: int = 2) -> bytes:
     # A page whose entire area is a raster image -> looks scanned.
     doc = pymupdf.open()
@@ -320,3 +333,97 @@ def test_scan_coverage_shorter_than_pages_aligns_without_crash():
     assert all(p.needs_ocr is False for p in c.pages)  # coverage no longer routes
     assert "image_heavy" in c.flags  # but page 0 still flags image_heavy
     assert c.recommended_tier == "fast"
+
+
+# --- glyph-corruption signal (broken /ToUnicode -> structured escalation) -----
+
+# A uniform glyph/Caesar offset turns clean prose into alphabetic-but-wrong tokens
+# (normal spacing + token length => HIGH text_quality) while digits/punctuation map
+# to C0 control bytes. The control-char ratio is the only signal that catches this;
+# _text_quality scores it ~1.0. Shared with the registry tiering tests.
+_GLYPH_CORRUPT = GLYPH_CORRUPT_TEXT
+
+
+def test_control_char_ratio_clean_is_zero():
+    assert clf._control_char_ratio("the quick brown fox") == pytest.approx(0.0)
+    # legitimate whitespace controls (tab/newline/CR/form-feed/vtab) don't count
+    assert clf._control_char_ratio("a\tb\nc\r\nd\f\ve") == pytest.approx(0.0)
+
+
+def test_control_char_ratio_detects_glyph_leak():
+    assert clf._control_char_ratio(_GLYPH_CORRUPT) > clf.GLYPH_CORRUPTION_RATIO
+
+
+def test_clean_text_not_flagged_corrupt():
+    txt = "the quick brown fox jumps over the lazy dog " * 3
+    c = clf.classify_from_text(
+        txt, [{"page": 1, "start_offset": 0, "end_offset": len(txt)}]
+    )
+    assert "corrupt_glyphs" not in c.flags
+    assert c.mean_control_ratio == pytest.approx(0.0)
+    assert c.recommended_tier == "fast"
+
+
+def test_glyph_corrupt_routes_structured_not_ocr():
+    full = _GLYPH_CORRUPT
+    c = clf.classify_from_text(
+        full, [{"page": 1, "start_offset": 0, "end_offset": len(full)}]
+    )
+    assert c.recommended_tier == "structured"
+    assert "corrupt_glyphs" in c.flags
+    # The point: it is NOT a low-quality signal -- the cipher scores high, so only
+    # the control-char ratio diverts it (to structured, the free pymupdf re-parse).
+    assert c.mean_text_quality >= clf.MIN_TEXT_QUALITY
+    assert c.mean_control_ratio > clf.GLYPH_CORRUPTION_RATIO
+
+
+def test_glyph_corruption_ratio_override_disables_trigger():
+    full = _GLYPH_CORRUPT
+    bounds = [{"page": 1, "start_offset": 0, "end_offset": len(full)}]
+    # A threshold of 1.0 can never be exceeded => not treated as corrupt => the
+    # other (high-quality) signals win => fast.
+    c = clf.classify_from_text(full, bounds, glyph_corruption_ratio=1.0)
+    assert c.recommended_tier == "fast"
+    assert "corrupt_glyphs" not in c.flags
+
+
+def test_glyph_corruption_ratio_zero_disables_trigger():
+    full = _GLYPH_CORRUPT
+    bounds = [{"page": 1, "start_offset": 0, "end_offset": len(full)}]
+    # 0 disables the signal (rather than firing on any single control byte).
+    c = clf.classify_from_text(full, bounds, glyph_corruption_ratio=0.0)
+    assert c.recommended_tier == "fast"
+    assert "corrupt_glyphs" not in c.flags
+
+
+def test_empty_doc_routes_ocr_not_structured():
+    # Precedence: a scanned/empty doc (no text layer) has no control chars to leak,
+    # so it must stay an OCR case, never structured.
+    c = clf.classify_from_text("", [{"page": 1, "start_offset": 0, "end_offset": 0}])
+    assert c.recommended_tier == "ocr"
+    assert "corrupt_glyphs" not in c.flags
+
+
+def test_glyph_corrupt_takes_precedence_over_junk_text_layer():
+    # A layer that is BOTH glyph-corrupt (high control ratio) AND junk-quality
+    # (mashed, no whitespace -> low text_quality): both flags fire, but
+    # glyph-corrupt wins the route (structured, not ocr) -- the structured
+    # re-extract is the cheaper correct fix, and re-classification catches any
+    # residual junk afterwards.
+    text = "WKHTXLFNEURZQIRAMXPSV\x0f\x10\x11\x0f\x10" * 3
+    c = clf.classify_from_text(
+        text, [{"page": 1, "start_offset": 0, "end_offset": len(text)}]
+    )
+    assert c.recommended_tier == "structured"
+    assert "corrupt_glyphs" in c.flags
+    assert "bad_text_layer" in c.flags
+
+
+def test_classify_pdf_glyph_corrupt_routes_structured():
+    # Symmetry with the classify_from_text routing on the standalone/diagnostic
+    # classify_pdf path (which re-opens the PDF and samples pages).
+    c = clf.classify_pdf(_glyph_corrupt_pdf())
+    assert c.recommended_tier == "structured"
+    assert "corrupt_glyphs" in c.flags
+    assert c.mean_control_ratio > clf.GLYPH_CORRUPTION_RATIO
+    assert c.mean_text_quality >= clf.MIN_TEXT_QUALITY  # control signal, not quality

tests/unit/test_registry_tiering.py

@@ -4,7 +4,7 @@ Covers: default fast-tier routing, the pymupdf rollback toggle, classification
 recording derived from the extraction, and OCR escalation (on/off).
 """
 
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, call
 
 import pytest
 
@@ -15,6 +15,7 @@ from nextcloud_mcp_server.document_processors.base import (
 )
 from nextcloud_mcp_server.document_processors.escalation import EscalationDecision
 from nextcloud_mcp_server.document_processors.registry import ProcessorRegistry
+from tests.fixtures.glyph_corruption import GLYPH_CORRUPT_TEXT
 
 pytestmark = pytest.mark.unit
 
@@ -24,7 +25,9 @@ class _Fake(DocumentProcessor):
         self,
         name: str,
         tier: str,
-        text: str = "clean text here",
+        # >= MIN_PAGE_CHARS of clean, whitespace-separated prose so the default
+        # classifies "fast" (a shorter string trips the near-empty OCR signal).
+        text: str = "this is clean readable prose text",
         success=True,
         pages: int = 1,
     ):
@@ -75,6 +78,7 @@ class _Settings:
         page_fraction=0.5,
         min_page_chars=16,
         detect_scanned=False,
+        glyph_corruption_ratio=0.02,
         # Guard off by default so existing tiering tests are unaffected; tests
         # that exercise the size guard pass an explicit cap.
         max_pdf_size_mb=0.0,
@@ -86,6 +90,7 @@ class _Settings:
         self.document_ocr_page_fraction = page_fraction
         self.document_ocr_min_page_chars = min_page_chars
         self.document_ocr_detect_scanned = detect_scanned
+        self.document_glyph_corruption_ratio = glyph_corruption_ratio
         self.document_max_pdf_size_mb = max_pdf_size_mb
 
 
@@ -244,6 +249,197 @@ async def test_no_ocr_escalation_when_disabled(monkeypatch):
     assert res.processor == "fast"
 
 
+# --- glyph-corruption escalation + full-ladder parity ------------------------
+
+# A fast-tier text layer that looks like words (HIGH text_quality) but leaks C0
+# control chars -- the broken-/ToUnicode signature the control-char ratio catches.
+# Shared with the classifier tests so the two can't diverge.
+_GLYPH = GLYPH_CORRUPT_TEXT
+
+
+async def test_glyph_corrupt_escalates_fast_to_structured(monkeypatch):
+    # Not gated on OCR: structured is free + in-cluster, so a glyph-corrupt layer
+    # escalates fast->structured even with OCR disabled.
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=False))
+    esc = MagicMock()
+    monkeypatch.setattr(reg_mod, "record_document_escalation", esc)
+    r = _registry(
+        (_Fake("fast", "fast", text=_GLYPH), 20),
+        (_Fake("structured", "structured", text="clean recovered prose text"), 10),
+    )
+    res = await r.process(b"%PDF-1.7", "application/pdf")
+    assert res.processor == "structured"
+    esc.assert_called_once_with("fast", "structured", "corrupt_glyphs")
+
+
+async def test_glyph_corrupt_no_structured_stays_fast(monkeypatch):
+    # No structured processor registered AND OCR off -> nothing to escalate to;
+    # keep fast (the inline counterpart of the external "suppressed" outcome).
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=False))
+    r = _registry((_Fake("fast", "fast", text=_GLYPH), 20))
+    res = await r.process(b"%PDF-1.7", "application/pdf")
+    assert res.processor == "fast"
+
+
+async def test_glyph_corrupt_no_structured_falls_through_to_ocr(monkeypatch):
+    # Parity with the external path: structured unregistered but OCR enabled ->
+    # the glyph-corrupt doc falls through to OCR (not silently kept at fast).
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=True))
+    esc = MagicMock()
+    monkeypatch.setattr(reg_mod, "record_document_escalation", esc)
+    r = _registry(
+        (_Fake("fast", "fast", text=_GLYPH), 20),
+        (_Fake("ocr", "ocr", text="ocr recovered text"), 5),
+    )  # no structured registered
+    res = await r.process(b"%PDF-1.7", "application/pdf")
+    assert res.processor == "ocr"
+    esc.assert_called_once_with("fast", "ocr", "corrupt_glyphs")
+
+
+async def test_inline_lowconf_tries_structured_before_ocr(monkeypatch):
+    # Full-ladder parity with the external path: a junk-but-non-empty fast layer
+    # tries structured (fast->structured) BEFORE any OCR, even with OCR enabled.
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=True))
+    esc = MagicMock()
+    monkeypatch.setattr(reg_mod, "record_document_escalation", esc)
+    r = _registry(
+        (_Fake("fast", "fast", text="x" * 40), 20),  # one long token -> quality ~0
+        (_Fake("structured", "structured", text="clean recovered prose text here"), 10),
+        (_Fake("ocr", "ocr", text="ocr text"), 5),
+    )
+    res = await r.process(b"%PDF-1.7", "application/pdf")
+    assert res.processor == "structured"
+    esc.assert_called_once_with("fast", "structured", "low_confidence")
+
+
+async def test_inline_empty_skips_structured_straight_to_ocr(monkeypatch):
+    # The one intended shortcut: a scanned/no-text-layer doc (total_chars == 0)
+    # skips structured (it cannot extract text from a raster) and goes to OCR.
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=True))
+    esc = MagicMock()
+    monkeypatch.setattr(reg_mod, "record_document_escalation", esc)
+    r = _registry(
+        (_Fake("fast", "fast", text=""), 20),
+        (_Fake("structured", "structured", text="should not run"), 10),
+        (_Fake("ocr", "ocr", text="ocr text"), 5),
+    )
+    res = await r.process(b"%PDF-1.7", "application/pdf")
+    assert res.processor == "ocr"
+    esc.assert_called_once_with("fast", "ocr", "empty_text")
+
+
+async def test_inline_fast_structured_ocr_cascade(monkeypatch):
+    # Full cascade: a junk-but-non-empty fast layer hops to structured, the
+    # structured re-extract is empty (a doc that was ALSO scanned), so it then
+    # hops to OCR. The second hop must be attributed from_tier="structured",
+    # NOT a second "fast" escalation.
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=True))
+    esc = MagicMock()
+    monkeypatch.setattr(reg_mod, "record_document_escalation", esc)
+    r = _registry(
+        (_Fake("fast", "fast", text="x" * 40), 20),  # quality ~0, non-empty
+        (_Fake("structured", "structured", text=""), 10),  # re-extract empty
+        (_Fake("ocr", "ocr", text="ocr recovered text"), 5),
+    )
+    res = await r.process(b"%PDF-1.7", "application/pdf")
+    assert res.processor == "ocr"
+    assert esc.call_args_list == [
+        call("fast", "structured", "low_confidence"),
+        call("structured", "ocr", "empty_text"),
+    ]
+
+
+async def test_inline_structured_still_corrupt_escalates_to_ocr(monkeypatch):
+    # Edge: the structured re-extract is ALSO glyph-corrupt (pymupdf also failed to
+    # decode). Re-classification stays "structured", so the OCR gate fires --
+    # attributed from_tier="structured" with reason corrupt_glyphs.
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=True))
+    esc = MagicMock()
+    monkeypatch.setattr(reg_mod, "record_document_escalation", esc)
+    r = _registry(
+        (_Fake("fast", "fast", text=_GLYPH), 20),
+        (_Fake("structured", "structured", text=_GLYPH), 10),  # still corrupt
+        (_Fake("ocr", "ocr", text="ocr recovered text"), 5),
+    )
+    res = await r.process(b"%PDF-1.7", "application/pdf")
+    assert res.processor == "ocr"
+    assert esc.call_args_list == [
+        call("fast", "structured", "corrupt_glyphs"),
+        call("structured", "ocr", "corrupt_glyphs"),
+    ]
+
+
+def test_evaluate_escalation_glyph_corrupt_goes_structured(monkeypatch):
+    # External path mirrors the inline path: glyph-corrupt -> structured, never OCR.
+    monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
+    r = _registry(
+        (_Fake("fast", "fast"), 20),
+        (_Fake("structured", "structured"), 10),
+        (_Fake("ocr", "ocr"), 5),
+    )
+    res = ProcessingResult(
+        text=_GLYPH,
+        metadata={
+            "page_count": 1,
+            "page_boundaries": [
+                {"page": 1, "start_offset": 0, "end_offset": len(_GLYPH)}
+            ],
+        },
+        processor="fast",
+    )
+    decision = r.evaluate_escalation(res, b"%PDF", "fast", _Settings(ocr=True))
+    assert decision == EscalationDecision("hop", "structured", "corrupt_glyphs")
+
+
+def test_evaluate_escalation_glyph_corrupt_no_structured_falls_through_to_ocr(
+    monkeypatch,
+):
+    # External path with structured unregistered: next_available_tier skips the
+    # missing rung and lands on OCR, keeping the corrupt_glyphs reason.
+    monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
+    r = _registry(
+        (_Fake("fast", "fast"), 20),
+        (_Fake("ocr", "ocr"), 5),
+    )  # no structured registered
+    res = ProcessingResult(
+        text=_GLYPH,
+        metadata={
+            "page_count": 1,
+            "page_boundaries": [
+                {"page": 1, "start_offset": 0, "end_offset": len(_GLYPH)}
+            ],
+        },
+        processor="fast",
+    )
+    decision = r.evaluate_escalation(res, b"%PDF", "fast", _Settings(ocr=True))
+    assert decision == EscalationDecision("hop", "ocr", "corrupt_glyphs")
+
+
+def test_evaluate_escalation_glyph_corrupt_no_structured_ocr_disabled_suppressed(
+    monkeypatch,
+):
+    # Structured unregistered AND OCR registered-but-disabled: the would-be OCR
+    # fallthrough is suppressed, and it carries the corrupt_glyphs reason (so the
+    # "what-if OCR" counter can show latent glyph-corruption demand).
+    monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
+    r = _registry(
+        (_Fake("fast", "fast"), 20),
+        (_Fake("ocr", "ocr"), 5),
+    )  # structured not registered; ocr registered but disabled below
+    res = ProcessingResult(
+        text=_GLYPH,
+        metadata={
+            "page_count": 1,
+            "page_boundaries": [
+                {"page": 1, "start_offset": 0, "end_offset": len(_GLYPH)}
+            ],
+        },
+        processor="fast",
+    )
+    decision = r.evaluate_escalation(res, b"%PDF", "fast", _Settings(ocr=False))
+    assert decision == EscalationDecision("suppressed", "ocr", "corrupt_glyphs")
+
+
 # --- Per-tier external path (Deck #323) -------------------------------------