Chris CoutinhoandClaude Opus 4.8
c48a797896
Replaces single-engine pymupdf4llm extraction with a tiered pipeline (Deck #205, follows the tier-0 classifier #855). pypdfium2 becomes the default and only hot-path PDF extractor; pymupdf4llm is deprecated to a rollback toggle. Why: pymupdf4llm's O(n^2) find_tables drove the OOM (#852) and the form-PDF parse timeouts (#856), carries AGPL/commercial licensing liability, and -- per the benchmarks -- recovers near-zero usable tables on the real corpus. pypdfium2 (Apache/BSD) extracts the same text far faster (Student 1a.pdf: 120s timeout -> 0.2s) with no table-detection bomb. - document_processors/pypdfium2_fast.py: tier-1 "fast" processor emitting text + exact page_boundaries (the pdf_highlighter contract). pymupdf processor is now tier "structured" (the rollback engine), registered but not default. - registry: tiered routing in ProcessorRegistry. tier-1 fast extracts, then classification is DERIVED from that text (classifier.classify_from_text -- no PDF re-open), records the classification metrics, and escalates scanned / no-text-layer docs to the "ocr" tier when document_ocr_enabled (default off; no provider yet, so fast is terminal). Wires record_document_escalation + the real "escalated" span attribute (was hardcoded False). - Removes the separate _shadow_classify pass from vector/processor.py -- it re-opened every PDF and re-extracted text (~0.5-1.3s/doc of pure duplicated CPU that lowered throughput); classification now rides the tier-1 extraction. - Settings: document_tier1_engine ("pypdfium2" default | "pymupdf" rollback, enum-validated), document_ocr_enabled (default false). Tests: pypdfium2 extractor, registry tiering (fast routing, rollback, classify recording, OCR escalation on/off), classify_from_text. Full unit suite green. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
129 lines
4.1 KiB
Python
129 lines
4.1 KiB
Python
"""Unit tests for the tiered PDF routing in ProcessorRegistry.
|
|
|
|
Covers: default fast-tier routing, the pymupdf rollback toggle, classification
|
|
recording derived from the extraction, and OCR escalation (on/off).
|
|
"""
|
|
|
|
from unittest.mock import MagicMock
|
|
|
|
import pytest
|
|
|
|
from nextcloud_mcp_server.document_processors import registry as reg_mod
|
|
from nextcloud_mcp_server.document_processors.base import (
|
|
DocumentProcessor,
|
|
ProcessingResult,
|
|
)
|
|
from nextcloud_mcp_server.document_processors.registry import ProcessorRegistry
|
|
|
|
pytestmark = pytest.mark.unit
|
|
|
|
|
|
class _Fake(DocumentProcessor):
|
|
def __init__(
|
|
self, name: str, tier: str, text: str = "clean text here", success=True
|
|
):
|
|
self._name = name
|
|
self._tier = tier
|
|
self._text = text
|
|
self._success = success
|
|
|
|
@property
|
|
def name(self) -> str:
|
|
return self._name
|
|
|
|
@property
|
|
def tier(self) -> str:
|
|
return self._tier
|
|
|
|
@property
|
|
def supported_mime_types(self) -> set[str]:
|
|
return {"application/pdf"}
|
|
|
|
async def process(
|
|
self, content, content_type, filename=None, options=None, progress_callback=None
|
|
):
|
|
return ProcessingResult(
|
|
text=self._text,
|
|
metadata={
|
|
"page_count": 1,
|
|
"page_boundaries": [
|
|
{"page": 1, "start_offset": 0, "end_offset": len(self._text)}
|
|
],
|
|
},
|
|
processor=self._name,
|
|
success=self._success,
|
|
)
|
|
|
|
async def health_check(self) -> bool:
|
|
return True
|
|
|
|
|
|
class _Settings:
|
|
def __init__(self, engine="pypdfium2", classify=True, ocr=False):
|
|
self.document_tier1_engine = engine
|
|
self.document_classify_enabled = classify
|
|
self.document_ocr_enabled = ocr
|
|
|
|
|
|
def _registry(*procs: tuple[DocumentProcessor, int]) -> ProcessorRegistry:
|
|
r = ProcessorRegistry()
|
|
for proc, prio in procs:
|
|
r.register(proc, priority=prio)
|
|
return r
|
|
|
|
|
|
async def test_pdf_routes_to_fast_tier(monkeypatch):
|
|
monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings())
|
|
r = _registry((_Fake("fast", "fast"), 20), (_Fake("structured", "structured"), 10))
|
|
res = await r.process(b"%PDF-1.7", "application/pdf")
|
|
assert res.processor == "fast"
|
|
|
|
|
|
async def test_engine_rollback_uses_structured(monkeypatch):
|
|
monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(engine="pymupdf"))
|
|
r = _registry((_Fake("fast", "fast"), 20), (_Fake("structured", "structured"), 10))
|
|
res = await r.process(b"%PDF-1.7", "application/pdf")
|
|
assert res.processor == "structured"
|
|
|
|
|
|
async def test_records_classification(monkeypatch):
|
|
monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings())
|
|
rec = MagicMock()
|
|
monkeypatch.setattr(reg_mod, "record_document_classification", rec)
|
|
r = _registry((_Fake("fast", "fast"), 20))
|
|
await r.process(b"%PDF-1.7", "application/pdf")
|
|
rec.assert_called_once()
|
|
|
|
|
|
async def test_classify_disabled_skips_recording(monkeypatch):
|
|
monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(classify=False))
|
|
rec = MagicMock()
|
|
monkeypatch.setattr(reg_mod, "record_document_classification", rec)
|
|
r = _registry((_Fake("fast", "fast"), 20))
|
|
await r.process(b"%PDF-1.7", "application/pdf")
|
|
rec.assert_not_called()
|
|
|
|
|
|
async def test_ocr_escalation_on_empty_text(monkeypatch):
|
|
monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=True))
|
|
esc = MagicMock()
|
|
monkeypatch.setattr(reg_mod, "record_document_escalation", esc)
|
|
r = _registry(
|
|
(_Fake("fast", "fast", text=""), 20),
|
|
(_Fake("ocr", "ocr", text="ocr text"), 5),
|
|
)
|
|
res = await r.process(b"%PDF-1.7", "application/pdf")
|
|
assert res.processor == "ocr"
|
|
esc.assert_called_once()
|
|
|
|
|
|
async def test_no_ocr_escalation_when_disabled(monkeypatch):
|
|
monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=False))
|
|
r = _registry(
|
|
(_Fake("fast", "fast", text=""), 20),
|
|
(_Fake("ocr", "ocr"), 5),
|
|
)
|
|
res = await r.process(b"%PDF-1.7", "application/pdf")
|
|
# Fast tier is terminal when OCR is disabled.
|
|
assert res.processor == "fast"
|