Merge pull request #858 from cbcoutinho/feat/tiered-doc-processor-b2-tier1

feat: tiered PDF processor — pypdfium2 fast path (deprecate pymupdf4llm)

tests/unit/test_decomposition_config.py

@@ -29,9 +29,13 @@ class TestDecompositionDefaults:
         s = Settings(
             collection_metadata_source=" QDRANT ",
             mcp_role=" API ",
+            document_tier1_engine=" PyPDFium2 ",
+            document_ocr_provider=" Gateway ",
         )
         assert s.collection_metadata_source == "qdrant"
         assert s.mcp_role == "api"
+        assert s.document_tier1_engine == "pypdfium2"
+        assert s.document_ocr_provider == "gateway"
 
 
 class TestEnumValidation:
@@ -41,6 +45,8 @@ class TestEnumValidation:
             ("embedding_provider", "openai"),
             ("mcp_role", "leader"),
             ("collection_metadata_source", "redis"),
+            ("document_tier1_engine", "mupdf"),
+            ("document_ocr_provider", "gatway"),
         ],
     )
     def test_invalid_enum_rejected(self, field, value):

tests/unit/test_doc_classifier.py

@@ -156,3 +156,32 @@ def test_image_heavy_flag_without_ocr_routing():
     assert "image_heavy" in c.flags
     assert c.recommended_tier == "fast"
     assert c.ocr_page_fraction < clf.OCR_PAGE_FRACTION
+
+
+# --- classify_from_text (hot-path, derived from tier-1 extraction) -----------
+
+
+def test_classify_from_text_clean_routes_fast():
+    txt = "the quick brown fox jumps over the lazy dog " * 3
+    c = clf.classify_from_text(
+        txt, [{"page": 1, "start_offset": 0, "end_offset": len(txt)}]
+    )
+    assert c.recommended_tier == "fast"
+    assert c.mean_text_quality > 0.8
+    assert c.flags == set()
+
+
+def test_classify_from_text_empty_routes_ocr():
+    c = clf.classify_from_text("", [{"page": 1, "start_offset": 0, "end_offset": 0}])
+    assert c.recommended_tier == "ocr"
+    assert "no_text_layer" in c.flags
+    assert c.total_chars == 0
+
+
+def test_classify_from_text_no_pages_routes_fast():
+    # An empty/corrupt PDF (no page boundaries) is not OCR evidence -> "fast",
+    # so the recorded classification metric isn't a misleading "ocr".
+    c = clf.classify_from_text("", [])
+    assert c.recommended_tier == "fast"
+    assert c.ocr_page_fraction == pytest.approx(0.0)
+    assert c.flags == set()

tests/unit/test_ocr_processor.py

@@ -0,0 +1,117 @@
+"""Unit tests for the tier-3 OCR processor + backend selection."""
+
+from types import SimpleNamespace
+from typing import Any
+
+import pytest
+
+from nextcloud_mcp_server.document_processors import ocr
+
+pytestmark = pytest.mark.unit
+
+
+def _settings(**kw) -> Any:  # a Settings stand-in (only the read fields matter)
+    base = dict(
+        document_ocr_provider="auto",
+        document_ocr_model="mistral/mistral-ocr-latest",
+        embedding_gateway_url=None,
+        embedding_gateway_client_id=None,
+        embedding_gateway_client_secret=None,
+        embedding_gateway_token_url=None,
+        embedding_gateway_scope=None,
+        mistral_api_key=None,
+        mistral_base_url=None,
+    )
+    base.update(kw)
+    return SimpleNamespace(**base)
+
+
+# --- _pages_to_text ----------------------------------------------------------
+
+
+def test_pages_to_text_orders_and_exact_boundaries():
+    text, boundaries = ocr._pages_to_text([(1, "B"), (0, "A")])  # out of order
+    assert text == "A\n\nB"
+    assert boundaries[0] == {"page": 1, "start_offset": 0, "end_offset": 1}
+    assert boundaries[1]["page"] == 2
+    # contiguous + offsets index exactly into the text
+    assert boundaries[0]["end_offset"] <= boundaries[1]["start_offset"]
+    assert boundaries[-1]["end_offset"] == len(text)
+
+
+# --- backend selection -------------------------------------------------------
+
+
+def test_build_backend_none():
+    assert ocr.build_ocr_backend(_settings(document_ocr_provider="none")) is None
+
+
+def test_build_backend_gateway():
+    b = ocr.build_ocr_backend(
+        _settings(document_ocr_provider="gateway", embedding_gateway_url="http://gw")
+    )
+    assert isinstance(b, ocr._GatewayOcrBackend)
+
+
+def test_build_backend_mistral():
+    b = ocr.build_ocr_backend(
+        _settings(document_ocr_provider="mistral", mistral_api_key="k")
+    )
+    assert isinstance(b, ocr._MistralOcrBackend)
+
+
+def test_build_backend_auto_prefers_gateway():
+    b = ocr.build_ocr_backend(
+        _settings(embedding_gateway_url="http://gw", mistral_api_key="k")
+    )
+    assert isinstance(b, ocr._GatewayOcrBackend)
+
+
+def test_build_backend_auto_none_configured():
+    assert ocr.build_ocr_backend(_settings()) is None
+
+
+def test_gateway_backend_url_normalization():
+    b = ocr._GatewayOcrBackend("http://gw", "mistral/mistral-ocr-latest")
+    assert b._url == "http://gw/v1/ocr"
+    b2 = ocr._GatewayOcrBackend("http://gw/v1/", "m")
+    assert b2._url == "http://gw/v1/ocr"
+
+
+# --- OcrProcessor ------------------------------------------------------------
+
+
+async def test_processor_unsupported_when_no_backend(monkeypatch):
+    monkeypatch.setattr(
+        ocr, "get_settings", lambda: _settings(document_ocr_provider="none")
+    )
+    monkeypatch.setattr(ocr, "build_ocr_backend", lambda s: None)
+    r = await ocr.OcrProcessor().process(b"%PDF-1.7", "application/pdf")
+    assert r.success is False
+    assert r.metadata["parse_failed_reason"] == "unsupported"
+
+
+async def test_processor_success(monkeypatch):
+    class _FakeBackend:
+        async def ocr(self, content, mime_type):
+            return "hello world", [{"page": 1, "start_offset": 0, "end_offset": 11}]
+
+    monkeypatch.setattr(ocr, "get_settings", lambda: _settings())
+    monkeypatch.setattr(ocr, "build_ocr_backend", lambda s: _FakeBackend())
+    r = await ocr.OcrProcessor().process(b"%PDF-1.7", "application/pdf")
+    assert r.success is True
+    assert r.text == "hello world"
+    assert r.metadata["page_count"] == 1
+    assert r.processor == "ocr"
+
+
+async def test_processor_backend_error_returns_success_false(monkeypatch):
+    class _BoomBackend:
+        async def ocr(self, content, mime_type):
+            raise RuntimeError("api down")
+
+    monkeypatch.setattr(ocr, "get_settings", lambda: _settings())
+    monkeypatch.setattr(ocr, "build_ocr_backend", lambda s: _BoomBackend())
+    r = await ocr.OcrProcessor().process(b"%PDF-1.7", "application/pdf")
+    assert r.success is False
+    assert r.metadata["parse_failed_reason"] == "error"

tests/unit/test_pypdfium2_fast.py

@@ -0,0 +1,59 @@
+"""Unit tests for the tier-1 pypdfium2 fast PDF extractor."""
+
+import pymupdf
+import pytest
+
+from nextcloud_mcp_server.document_processors.pypdfium2_fast import (
+    Pypdfium2FastProcessor,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def _digital_pdf(
+    pages: int = 3, body: str = "Hello world this is clean text. "
+) -> bytes:
+    doc = pymupdf.open()
+    for _ in range(pages):
+        page = doc.new_page(width=595, height=842)
+        page.insert_text((50, 60), body * 8)
+    data: bytes = doc.tobytes()
+    doc.close()
+    return data
+
+
+def test_processor_identity():
+    p = Pypdfium2FastProcessor()
+    assert p.name == "pypdfium2_fast"
+    assert p.tier == "fast"
+    assert "application/pdf" in p.supported_mime_types
+
+
+async def test_extract_text_and_exact_page_boundaries():
+    p = Pypdfium2FastProcessor()
+    result = await p.process(_digital_pdf(pages=3), "application/pdf", filename="t.pdf")
+
+    assert result.success is True
+    assert "Hello world" in result.text
+    assert result.metadata["page_count"] == 3
+
+    boundaries = result.metadata["page_boundaries"]
+    assert len(boundaries) == 3
+    assert boundaries[0]["start_offset"] == 0
+    # Offsets must index exactly into the returned text (pdf_highlighter contract).
+    assert boundaries[-1]["end_offset"] == len(result.text)
+    for prev, nxt in zip(boundaries, boundaries[1:]):
+        assert prev["end_offset"] == nxt["start_offset"]
+
+
+async def test_malformed_pdf_returns_success_false():
+    p = Pypdfium2FastProcessor()
+    result = await p.process(b"not a pdf at all", "application/pdf", filename="bad.pdf")
+
+    assert result.success is False
+    assert result.text == ""
+    assert result.metadata["parse_failed_reason"] == "error"
+
+
+async def test_health_check():
+    assert await Pypdfium2FastProcessor().health_check() is True

tests/unit/test_registry_tiering.py

@@ -0,0 +1,184 @@
+"""Unit tests for the tiered PDF routing in ProcessorRegistry.
+
+Covers: default fast-tier routing, the pymupdf rollback toggle, classification
+recording derived from the extraction, and OCR escalation (on/off).
+"""
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from nextcloud_mcp_server.document_processors import registry as reg_mod
+from nextcloud_mcp_server.document_processors.base import (
+    DocumentProcessor,
+    ProcessingResult,
+)
+from nextcloud_mcp_server.document_processors.registry import ProcessorRegistry
+
+pytestmark = pytest.mark.unit
+
+
+class _Fake(DocumentProcessor):
+    def __init__(
+        self,
+        name: str,
+        tier: str,
+        text: str = "clean text here",
+        success=True,
+        pages: int = 1,
+    ):
+        self._name = name
+        self._tier = tier
+        self._text = text
+        self._success = success
+        self._pages = pages
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def tier(self) -> str:
+        return self._tier
+
+    @property
+    def supported_mime_types(self) -> set[str]:
+        return {"application/pdf"}
+
+    async def process(
+        self, content, content_type, filename=None, options=None, progress_callback=None
+    ):
+        boundaries = (
+            [{"page": 1, "start_offset": 0, "end_offset": len(self._text)}]
+            if self._pages
+            else []
+        )
+        return ProcessingResult(
+            text=self._text,
+            metadata={"page_count": self._pages, "page_boundaries": boundaries},
+            processor=self._name,
+            success=self._success,
+        )
+
+    async def health_check(self) -> bool:
+        return True
+
+
+class _Settings:
+    def __init__(self, engine="pypdfium2", classify=True, ocr=False):
+        self.document_tier1_engine = engine
+        self.document_classify_enabled = classify
+        self.document_ocr_enabled = ocr
+
+
+def _registry(*procs: tuple[DocumentProcessor, int]) -> ProcessorRegistry:
+    r = ProcessorRegistry()
+    for proc, prio in procs:
+        r.register(proc, priority=prio)
+    return r
+
+
+async def test_pdf_routes_to_fast_tier(monkeypatch):
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings())
+    r = _registry((_Fake("fast", "fast"), 20), (_Fake("structured", "structured"), 10))
+    res = await r.process(b"%PDF-1.7", "application/pdf")
+    assert res.processor == "fast"
+
+
+async def test_engine_rollback_uses_structured(monkeypatch):
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(engine="pymupdf"))
+    r = _registry((_Fake("fast", "fast"), 20), (_Fake("structured", "structured"), 10))
+    res = await r.process(b"%PDF-1.7", "application/pdf")
+    assert res.processor == "structured"
+
+
+async def test_engine_rollback_warns_when_no_structured(monkeypatch, caplog):
+    # pymupdf rollback with no structured processor registered: it falls back to
+    # the fast processor but must warn (it silently used what the user opted out
+    # of otherwise).
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(engine="pymupdf"))
+    r = _registry((_Fake("fast", "fast"), 20))
+    with caplog.at_level(
+        "WARNING", logger="nextcloud_mcp_server.document_processors.registry"
+    ):
+        res = await r.process(b"%PDF-1.7", "application/pdf")
+    assert res.processor == "fast"
+    assert any("no 'structured' processor" in rec.message for rec in caplog.records)
+
+
+async def test_records_classification(monkeypatch):
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings())
+    rec = MagicMock()
+    monkeypatch.setattr(reg_mod, "record_document_classification", rec)
+    r = _registry((_Fake("fast", "fast"), 20))
+    await r.process(b"%PDF-1.7", "application/pdf")
+    rec.assert_called_once()
+
+
+async def test_classify_disabled_skips_recording(monkeypatch):
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(classify=False))
+    rec = MagicMock()
+    monkeypatch.setattr(reg_mod, "record_document_classification", rec)
+    r = _registry((_Fake("fast", "fast"), 20))
+    await r.process(b"%PDF-1.7", "application/pdf")
+    rec.assert_not_called()
+
+
+async def test_ocr_escalation_on_empty_text(monkeypatch):
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=True))
+    esc = MagicMock()
+    monkeypatch.setattr(reg_mod, "record_document_escalation", esc)
+    r = _registry(
+        (_Fake("fast", "fast", text=""), 20),
+        (_Fake("ocr", "ocr", text="ocr text"), 5),
+    )
+    res = await r.process(b"%PDF-1.7", "application/pdf")
+    assert res.processor == "ocr"
+    esc.assert_called_once()
+
+
+async def test_zero_page_pdf_does_not_escalate(monkeypatch):
+    # An empty/corrupt PDF (no pages) classifies "ocr" but must NOT escalate --
+    # OCR can't help and it would be wasteful.
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=True))
+    esc = MagicMock()
+    monkeypatch.setattr(reg_mod, "record_document_escalation", esc)
+    r = _registry(
+        (_Fake("fast", "fast", text="", pages=0), 20),
+        (_Fake("ocr", "ocr"), 5),
+    )
+    res = await r.process(b"%PDF-1.7", "application/pdf")
+    assert res.processor == "fast"
+    esc.assert_not_called()
+
+
+async def test_pipeline_tier_stamped_on_metadata(monkeypatch):
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings())
+    r = _registry((_Fake("fast", "fast"), 20))
+    res = await r.process(b"%PDF-1.7", "application/pdf")
+    assert res.metadata["pipeline_tier"] == "fast"
+
+
+async def test_ocr_failure_falls_back_to_fast(monkeypatch):
+    # OCR enabled but the backend can't run (no creds / API down) -> keep the
+    # tier-1 result instead of failing the document.
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=True))
+    monkeypatch.setattr(reg_mod, "record_document_escalation", MagicMock())
+    r = _registry(
+        (_Fake("fast", "fast", text=""), 20),
+        (_Fake("ocr", "ocr", text="", success=False), 5),
+    )
+    res = await r.process(b"%PDF-1.7", "application/pdf")
+    assert res.processor == "fast"
+    assert res.success is True
+
+
+async def test_no_ocr_escalation_when_disabled(monkeypatch):
+    monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=False))
+    r = _registry(
+        (_Fake("fast", "fast", text=""), 20),
+        (_Fake("ocr", "ocr"), 5),
+    )
+    res = await r.process(b"%PDF-1.7", "application/pdf")
+    # Fast tier is terminal when OCR is disabled.
+    assert res.processor == "fast"

tests/unit/test_shadow_classify.py

@@ -1,62 +0,0 @@
-"""Tests for the tier-0 shadow-classification wiring in the processor.
-
-Shadow mode = observability only: it emits classification metrics but must never
-block or fail indexing, and only applies to PDFs.
-"""
-
-from unittest.mock import MagicMock
-
-import pytest
-
-from nextcloud_mcp_server.document_processors.classifier import DocClassification
-from nextcloud_mcp_server.vector import processor as proc
-
-pytestmark = pytest.mark.unit
-
-
-def _classification() -> DocClassification:
-    return DocClassification(
-        page_count=2,
-        sampled_pages=2,
-        total_chars=100,
-        mean_text_quality=0.9,
-        ocr_page_fraction=0.0,
-        recommended_tier="fast",
-        flags={"image_heavy"},
-    )
-
-
-async def test_shadow_classify_records_metrics(monkeypatch):
-    monkeypatch.setattr(proc, "classify_pdf", lambda content: _classification())
-    rec = MagicMock()
-    monkeypatch.setattr(proc, "record_document_classification", rec)
-
-    await proc._shadow_classify(b"%PDF-1.7", "application/pdf", "f.pdf")
-
-    rec.assert_called_once_with("fast", {"image_heavy"}, 0.9)
-
-
-async def test_shadow_classify_skips_non_pdf(monkeypatch):
-    called = MagicMock()
-    monkeypatch.setattr(proc, "classify_pdf", called)
-    rec = MagicMock()
-    monkeypatch.setattr(proc, "record_document_classification", rec)
-
-    await proc._shadow_classify(b"plain", "text/plain", "f.txt")
-
-    called.assert_not_called()
-    rec.assert_not_called()
-
-
-async def test_shadow_classify_swallows_errors(monkeypatch):
-    def boom(content):
-        raise ValueError("bad pdf")
-
-    monkeypatch.setattr(proc, "classify_pdf", boom)
-    rec = MagicMock()
-    monkeypatch.setattr(proc, "record_document_classification", rec)
-
-    # Must not raise -- shadow classification is best-effort, off the index path.
-    await proc._shadow_classify(b"%PDF-1.7", "application/pdf", "f.pdf")
-
-    rec.assert_not_called()