feat: tiered PDF processor with pypdfium2 fast path (deprecate pymupdf4llm)

Replaces single-engine pymupdf4llm extraction with a tiered pipeline (Deck #205,
follows the tier-0 classifier #855). pypdfium2 becomes the default and only
hot-path PDF extractor; pymupdf4llm is deprecated to a rollback toggle.

Why: pymupdf4llm's O(n^2) find_tables drove the OOM (#852) and the form-PDF
parse timeouts (#856), carries AGPL/commercial licensing liability, and -- per
the benchmarks -- recovers near-zero usable tables on the real corpus. pypdfium2
(Apache/BSD) extracts the same text far faster (Student 1a.pdf: 120s timeout ->
0.2s) with no table-detection bomb.

- document_processors/pypdfium2_fast.py: tier-1 "fast" processor emitting text +
  exact page_boundaries (the pdf_highlighter contract). pymupdf processor is now
  tier "structured" (the rollback engine), registered but not default.
- registry: tiered routing in ProcessorRegistry. tier-1 fast extracts, then
  classification is DERIVED from that text (classifier.classify_from_text -- no
  PDF re-open), records the classification metrics, and escalates scanned /
  no-text-layer docs to the "ocr" tier when document_ocr_enabled (default off;
  no provider yet, so fast is terminal). Wires record_document_escalation + the
  real "escalated" span attribute (was hardcoded False).
- Removes the separate _shadow_classify pass from vector/processor.py -- it
  re-opened every PDF and re-extracted text (~0.5-1.3s/doc of pure duplicated
  CPU that lowered throughput); classification now rides the tier-1 extraction.
- Settings: document_tier1_engine ("pypdfium2" default | "pymupdf" rollback,
  enum-validated), document_ocr_enabled (default false).

Tests: pypdfium2 extractor, registry tiering (fast routing, rollback, classify
recording, OCR escalation on/off), classify_from_text. Full unit suite green.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

tests/unit/test_doc_classifier.py

@@ -156,3 +156,23 @@ def test_image_heavy_flag_without_ocr_routing():
     assert "image_heavy" in c.flags
     assert c.recommended_tier == "fast"
     assert c.ocr_page_fraction < clf.OCR_PAGE_FRACTION
+
+
+# --- classify_from_text (hot-path, derived from tier-1 extraction) -----------
+
+
+def test_classify_from_text_clean_routes_fast():
+    txt = "the quick brown fox jumps over the lazy dog " * 3
+    c = clf.classify_from_text(
+        txt, [{"page": 1, "start_offset": 0, "end_offset": len(txt)}]
+    )
+    assert c.recommended_tier == "fast"
+    assert c.mean_text_quality > 0.8
+    assert c.flags == set()
+
+
+def test_classify_from_text_empty_routes_ocr():
+    c = clf.classify_from_text("", [{"page": 1, "start_offset": 0, "end_offset": 0}])
+    assert c.recommended_tier == "ocr"
+    assert "no_text_layer" in c.flags
+    assert c.total_chars == 0

tests/unit/test_pypdfium2_fast.py

@@ -0,0 +1,59 @@
+"""Unit tests for the tier-1 pypdfium2 fast PDF extractor."""
+
+import pymupdf
+import pytest
+
+from nextcloud_mcp_server.document_processors.pypdfium2_fast import (
+    Pypdfium2FastProcessor,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def _digital_pdf(
+    pages: int = 3, body: str = "Hello world this is clean text. "
+) -> bytes:
+    doc = pymupdf.open()
+    for _ in range(pages):
+        page = doc.new_page(width=595, height=842)
+        page.insert_text((50, 60), body * 8)
+    data: bytes = doc.tobytes()
+    doc.close()
+    return data
+
+
+def test_processor_identity():
+    p = Pypdfium2FastProcessor()
+    assert p.name == "pypdfium2_fast"
+    assert p.tier == "fast"
+    assert "application/pdf" in p.supported_mime_types
+
+
+async def test_extract_text_and_exact_page_boundaries():
+    p = Pypdfium2FastProcessor()
+    result = await p.process(_digital_pdf(pages=3), "application/pdf", filename="t.pdf")
+
+    assert result.success is True
+    assert "Hello world" in result.text
+    assert result.metadata["page_count"] == 3
+
+    boundaries = result.metadata["page_boundaries"]
+    assert len(boundaries) == 3
+    assert boundaries[0]["start_offset"] == 0
+    # Offsets must index exactly into the returned text (pdf_highlighter contract).
+    assert boundaries[-1]["end_offset"] == len(result.text)
+    for prev, nxt in zip(boundaries, boundaries[1:]):
+        assert prev["end_offset"] == nxt["start_offset"]
+
+
+async def test_malformed_pdf_returns_success_false():
+    p = Pypdfium2FastProcessor()
+    result = await p.process(b"not a pdf at all", "application/pdf", filename="bad.pdf")
+
+    assert result.success is False
+    assert result.text == ""
+    assert result.metadata["parse_failed_reason"] == "error"
+
+
+async def test_health_check():
+    assert await Pypdfium2FastProcessor().health_check() is True

tests/unit/test_registry_tiering.py

@@ -0,0 +1,128 @@
+"""Unit tests for the tiered PDF routing in ProcessorRegistry.
+
+Covers: default fast-tier routing, the pymupdf rollback toggle, classification
+recording derived from the extraction, and OCR escalation (on/off).
+"""
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from nextcloud_mcp_server.document_processors import registry as reg_mod
+from nextcloud_mcp_server.document_processors.base import (
+    DocumentProcessor,
+    ProcessingResult,
+)
+from nextcloud_mcp_server.document_processors.registry import ProcessorRegistry
+
+pytestmark = pytest.mark.unit
+
+
+class _Fake(DocumentProcessor):
+    def __init__(
+        self, name: str, tier: str, text: str = "clean text here", success=True
+    ):
+        self._name = name
+        self._tier = tier
+        self._text = text
+        self._success = success
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def tier(self) -> str:
+        return self._tier
+
+    @property
+    def supported_mime_types(self) -> set[str]:
+        return {"application/pdf"}
+
+    async def process(
+        self, content, content_type, filename=None, options=None, progress_callback=None
+    ):
+        return ProcessingResult(
+            text=self._text,
+            metadata={
+                "page_count": 1,
+                "page_boundaries": [
+                    {"page": 1, "start_offset": 0, "end_offset": len(self._text)}
+                ],
+            },
+            processor=self._name,
+            success=self._success,
+        )
+
+    async def health_check(self) -> bool:
+        return True
+
+
+class _Settings:
+    def __init__(self, engine="pypdfium2", classify=True, ocr=False):
+        self.document_tier1_engine = engine
+        self.document_classify_enabled = classify
+        self.document_ocr_enabled = ocr
+
+
+def _registry(*procs: tuple[DocumentProcessor, int]) -> ProcessorRegistry:
+    r = ProcessorRegistry()
+    for proc, prio in procs:
+        r.register(proc, priority=prio)
+    return r
+
+
+async def test_pdf_routes_to_fast_tier(monkeypatch):
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings())
+    r = _registry((_Fake("fast", "fast"), 20), (_Fake("structured", "structured"), 10))
+    res = await r.process(b"%PDF-1.7", "application/pdf")
+    assert res.processor == "fast"
+
+
+async def test_engine_rollback_uses_structured(monkeypatch):
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(engine="pymupdf"))
+    r = _registry((_Fake("fast", "fast"), 20), (_Fake("structured", "structured"), 10))
+    res = await r.process(b"%PDF-1.7", "application/pdf")
+    assert res.processor == "structured"
+
+
+async def test_records_classification(monkeypatch):
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings())
+    rec = MagicMock()
+    monkeypatch.setattr(reg_mod, "record_document_classification", rec)
+    r = _registry((_Fake("fast", "fast"), 20))
+    await r.process(b"%PDF-1.7", "application/pdf")
+    rec.assert_called_once()
+
+
+async def test_classify_disabled_skips_recording(monkeypatch):
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(classify=False))
+    rec = MagicMock()
+    monkeypatch.setattr(reg_mod, "record_document_classification", rec)
+    r = _registry((_Fake("fast", "fast"), 20))
+    await r.process(b"%PDF-1.7", "application/pdf")
+    rec.assert_not_called()
+
+
+async def test_ocr_escalation_on_empty_text(monkeypatch):
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=True))
+    esc = MagicMock()
+    monkeypatch.setattr(reg_mod, "record_document_escalation", esc)
+    r = _registry(
+        (_Fake("fast", "fast", text=""), 20),
+        (_Fake("ocr", "ocr", text="ocr text"), 5),
+    )
+    res = await r.process(b"%PDF-1.7", "application/pdf")
+    assert res.processor == "ocr"
+    esc.assert_called_once()
+
+
+async def test_no_ocr_escalation_when_disabled(monkeypatch):
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=False))
+    r = _registry(
+        (_Fake("fast", "fast", text=""), 20),
+        (_Fake("ocr", "ocr"), 5),
+    )
+    res = await r.process(b"%PDF-1.7", "application/pdf")
+    # Fast tier is terminal when OCR is disabled.
+    assert res.processor == "fast"

tests/unit/test_shadow_classify.py

@@ -1,62 +0,0 @@
-"""Tests for the tier-0 shadow-classification wiring in the processor.
-
-Shadow mode = observability only: it emits classification metrics but must never
-block or fail indexing, and only applies to PDFs.
-"""
-
-from unittest.mock import MagicMock
-
-import pytest
-
-from nextcloud_mcp_server.document_processors.classifier import DocClassification
-from nextcloud_mcp_server.vector import processor as proc
-
-pytestmark = pytest.mark.unit
-
-
-def _classification() -> DocClassification:
-    return DocClassification(
-        page_count=2,
-        sampled_pages=2,
-        total_chars=100,
-        mean_text_quality=0.9,
-        ocr_page_fraction=0.0,
-        recommended_tier="fast",
-        flags={"image_heavy"},
-    )
-
-
-async def test_shadow_classify_records_metrics(monkeypatch):
-    monkeypatch.setattr(proc, "classify_pdf", lambda content: _classification())
-    rec = MagicMock()
-    monkeypatch.setattr(proc, "record_document_classification", rec)
-
-    await proc._shadow_classify(b"%PDF-1.7", "application/pdf", "f.pdf")
-
-    rec.assert_called_once_with("fast", {"image_heavy"}, 0.9)
-
-
-async def test_shadow_classify_skips_non_pdf(monkeypatch):
-    called = MagicMock()
-    monkeypatch.setattr(proc, "classify_pdf", called)
-    rec = MagicMock()
-    monkeypatch.setattr(proc, "record_document_classification", rec)
-
-    await proc._shadow_classify(b"plain", "text/plain", "f.txt")
-
-    called.assert_not_called()
-    rec.assert_not_called()
-
-
-async def test_shadow_classify_swallows_errors(monkeypatch):
-    def boom(content):
-        raise ValueError("bad pdf")
-
-    monkeypatch.setattr(proc, "classify_pdf", boom)
-    rec = MagicMock()
-    monkeypatch.setattr(proc, "record_document_classification", rec)
-
-    # Must not raise -- shadow classification is best-effort, off the index path.
-    await proc._shadow_classify(b"%PDF-1.7", "application/pdf", "f.pdf")
-
-    rec.assert_not_called()