feat: tier-3 OCR processor (gateway or direct Mistral)
Adds the OCR escalation target the tiered registry already routes to. Scanned /
no-text-layer PDFs (the tier-0 "ocr" verdict) escalate here when
document_ocr_enabled (default off).
Two interchangeable backends, selected by document_ocr_provider
(auto | gateway | mistral | none):
- gateway: POST to the Astrolabe Cloud model gateway's /v1/ocr -- the same
M2M-authenticated gateway as embeddings, so NO provider keys live in the pod
(the platform default; reuses EMBEDDING_GATEWAY_URL + the M2M creds).
- mistral: call the Mistral OCR API directly from the pod (MISTRAL_API_KEY), for
self-hosters / deployments without the gateway.
"auto" prefers the gateway, then direct Mistral.
Both return per-page markdown joined into text + exact page_boundaries (the
pdf_highlighter contract; bbox re-derived from the PDF bytes as for other tiers).
Validated end-to-end via direct Mistral on the scanned Student 147.pdf:
success, 15 pages, 22k chars, offsets exact, ~4s.
Settings: document_ocr_provider (enum-validated), document_ocr_model
("mistral/mistral-ocr-latest" -- gateway routes on the prefix, the direct mistral
backend strips it). OcrProcessor registered at lowest priority so it is never the
non-tiered default.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
c48a797896
commit
3bd1b46d9c
@@ -143,6 +143,12 @@ _DEFAULTS: dict[str, Any] = {
|
||||
# escalation target and is off by default (no provider wired yet).
|
||||
"document_tier1_engine": "pypdfium2",
|
||||
"document_ocr_enabled": False,
|
||||
# OCR backend: "auto" picks gateway (if EMBEDDING_GATEWAY_URL) else mistral
|
||||
# (if MISTRAL_API_KEY); "gateway"/"mistral" force one; "none" disables.
|
||||
"document_ocr_provider": "auto",
|
||||
# Provider-namespaced OCR model id (gateway routes on the prefix; the direct
|
||||
# mistral backend strips it).
|
||||
"document_ocr_model": "mistral/mistral-ocr-latest",
|
||||
# Observability
|
||||
"metrics_enabled": True,
|
||||
"metrics_port": 9090,
|
||||
@@ -298,6 +304,9 @@ _dynaconf = Dynaconf(
|
||||
# Enum constraints
|
||||
Validator("LOG_FORMAT", is_in=["text", "json"]),
|
||||
Validator("DOCUMENT_TIER1_ENGINE", is_in=["pypdfium2", "pymupdf"]),
|
||||
Validator(
|
||||
"DOCUMENT_OCR_PROVIDER", is_in=["auto", "gateway", "mistral", "none"]
|
||||
),
|
||||
Validator(
|
||||
"LOG_LEVEL",
|
||||
is_in=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
|
||||
@@ -742,9 +751,15 @@ class Settings:
|
||||
# permissive license, no find_tables) is the hot path; "pymupdf" is a
|
||||
# deprecated rollback to pymupdf4llm (AGPL, graphics-limited) for one corpus.
|
||||
document_tier1_engine: str = "pypdfium2"
|
||||
# Route scanned/no-text-layer PDFs to the tier-3 OCR provider. Off until an
|
||||
# OCR backend is wired; when off, the fast tier is terminal.
|
||||
# Route scanned/no-text-layer PDFs to the tier-3 OCR provider. Off by
|
||||
# default; when off, the fast tier is terminal.
|
||||
document_ocr_enabled: bool = False
|
||||
# OCR backend selection: "auto" | "gateway" | "mistral" | "none".
|
||||
document_ocr_provider: str = "auto"
|
||||
# Provider-namespaced OCR model id (e.g. "mistral/mistral-ocr-latest"). The
|
||||
# gateway routes on the "<provider>/" prefix; the direct mistral backend
|
||||
# strips it.
|
||||
document_ocr_model: str = "mistral/mistral-ocr-latest"
|
||||
|
||||
# Observability settings
|
||||
metrics_enabled: bool = True
|
||||
@@ -1360,6 +1375,8 @@ def get_settings() -> Settings:
|
||||
"document_classify_enabled": "DOCUMENT_CLASSIFY_ENABLED",
|
||||
"document_tier1_engine": "DOCUMENT_TIER1_ENGINE",
|
||||
"document_ocr_enabled": "DOCUMENT_OCR_ENABLED",
|
||||
"document_ocr_provider": "DOCUMENT_OCR_PROVIDER",
|
||||
"document_ocr_model": "DOCUMENT_OCR_MODEL",
|
||||
# Observability settings
|
||||
"metrics_enabled": "METRICS_ENABLED",
|
||||
"metrics_port": "METRICS_PORT",
|
||||
|
||||
@@ -1,17 +1,20 @@
|
||||
"""Document processing plugins for extracting text from various file formats."""
|
||||
|
||||
from .base import DocumentProcessor, ProcessingResult, ProcessorError
|
||||
from .ocr import OcrProcessor
|
||||
from .pymupdf import PyMuPDFProcessor
|
||||
from .pypdfium2_fast import Pypdfium2FastProcessor
|
||||
from .registry import ProcessorRegistry, get_registry
|
||||
|
||||
# Register processors at module initialization. The tiered PDF pipeline selects
|
||||
# by tier (not priority): Pypdfium2FastProcessor is the ``fast`` tier and
|
||||
# PyMuPDFProcessor the ``structured`` escalation target. Priority still orders
|
||||
# the non-tiered fallback path and other MIME types.
|
||||
# by tier (not priority): Pypdfium2FastProcessor is the ``fast`` tier,
|
||||
# PyMuPDFProcessor the ``structured`` rollback, and OcrProcessor the ``ocr``
|
||||
# escalation target (reached only when document_ocr_enabled). OcrProcessor gets
|
||||
# the lowest priority so it is never the non-tiered default for PDFs.
|
||||
_registry = get_registry()
|
||||
_registry.register(Pypdfium2FastProcessor(), priority=20)
|
||||
_registry.register(PyMuPDFProcessor(), priority=10)
|
||||
_registry.register(OcrProcessor(), priority=1)
|
||||
|
||||
__all__ = [
|
||||
"DocumentProcessor",
|
||||
@@ -21,4 +24,5 @@ __all__ = [
|
||||
"get_registry",
|
||||
"PyMuPDFProcessor",
|
||||
"Pypdfium2FastProcessor",
|
||||
"OcrProcessor",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
"""Tier-3 OCR processor.
|
||||
|
||||
Routes scanned / no-text-layer PDFs (the tier-0 classifier's ``ocr`` verdict) to
|
||||
an OCR backend that returns per-page markdown. Two interchangeable backends,
|
||||
selected by ``document_ocr_provider``:
|
||||
|
||||
* ``gateway`` -- POST the document to the Astrolabe Cloud model gateway's
|
||||
``POST /v1/ocr`` (the same M2M-authenticated gateway as embeddings; NO
|
||||
provider keys in the pod). The platform default.
|
||||
* ``mistral`` -- call the Mistral OCR API directly from the pod
|
||||
(``MISTRAL_API_KEY``), for self-hosters / deployments without the gateway.
|
||||
|
||||
``auto`` prefers the gateway (if ``EMBEDDING_GATEWAY_URL`` is set) then direct
|
||||
Mistral (if ``MISTRAL_API_KEY``). Both return GitHub-flavoured markdown + exact
|
||||
``page_boundaries``; bbox is re-derived from the PDF bytes + boundaries by
|
||||
``search/pdf_highlighter``, as for the other tiers.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import logging
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from nextcloud_mcp_server.config import Settings, get_settings
|
||||
|
||||
from .base import DocumentProcessor, ProcessingResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_OCR_TIMEOUT_SECONDS = 180.0
|
||||
|
||||
|
||||
def _pages_to_text(
|
||||
pages: list[tuple[int, str]],
|
||||
) -> tuple[str, list[dict[str, Any]]]:
|
||||
"""Join per-page markdown (ordered by index) into one string + boundaries.
|
||||
|
||||
Pages are joined with a blank line; each page owns its leading separator so
|
||||
``page_boundaries`` stay contiguous and index exactly into the returned text
|
||||
(the ``search/pdf_highlighter`` contract).
|
||||
"""
|
||||
sep = "\n\n"
|
||||
parts: list[str] = []
|
||||
boundaries: list[dict[str, Any]] = []
|
||||
offset = 0
|
||||
for i, (index, markdown) in enumerate(sorted(pages, key=lambda p: p[0])):
|
||||
chunk = markdown if i == 0 else sep + markdown
|
||||
start = offset
|
||||
offset += len(chunk)
|
||||
parts.append(chunk)
|
||||
boundaries.append(
|
||||
{"page": index + 1, "start_offset": start, "end_offset": offset}
|
||||
)
|
||||
return "".join(parts), boundaries
|
||||
|
||||
|
||||
class _OcrBackend(ABC):
|
||||
@abstractmethod
|
||||
async def ocr(
|
||||
self, content: bytes, mime_type: str
|
||||
) -> tuple[str, list[dict[str, Any]]]: ...
|
||||
|
||||
|
||||
class _GatewayOcrBackend(_OcrBackend):
|
||||
"""Calls the model gateway's ``POST /v1/ocr`` (key-isolated, M2M-authed)."""
|
||||
|
||||
def __init__(self, base_url: str, model: str, token_provider: Any = None):
|
||||
base = base_url.rstrip("/")
|
||||
if not base.endswith("/v1"):
|
||||
base = f"{base}/v1"
|
||||
self._url = f"{base}/ocr"
|
||||
self._model = model
|
||||
self._token_provider = token_provider
|
||||
|
||||
async def ocr(
|
||||
self, content: bytes, mime_type: str
|
||||
) -> tuple[str, list[dict[str, Any]]]:
|
||||
headers: dict[str, str] = {}
|
||||
if self._token_provider is not None:
|
||||
headers["Authorization"] = (
|
||||
f"Bearer {await self._token_provider.get_token()}"
|
||||
)
|
||||
payload = {
|
||||
"model": self._model,
|
||||
"document_b64": base64.b64encode(content).decode("ascii"),
|
||||
"mime_type": mime_type,
|
||||
}
|
||||
async with httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(_OCR_TIMEOUT_SECONDS, connect=10.0)
|
||||
) as client:
|
||||
resp = await client.post(self._url, json=payload, headers=headers)
|
||||
resp.raise_for_status()
|
||||
body = resp.json()
|
||||
pages = [(p["index"], p.get("markdown", "")) for p in body.get("pages", [])]
|
||||
return _pages_to_text(pages)
|
||||
|
||||
|
||||
class _MistralOcrBackend(_OcrBackend):
|
||||
"""Calls the Mistral OCR API directly (provider key lives in the pod)."""
|
||||
|
||||
def __init__(self, api_key: str, model: str, base_url: str | None = None):
|
||||
from mistralai.client import Mistral # noqa: PLC0415 -- lazy SDK import
|
||||
|
||||
self._client = Mistral(api_key=api_key, server_url=base_url)
|
||||
# The gateway-namespaced "mistral/<model>" id strips down to the bare
|
||||
# upstream model the SDK expects.
|
||||
self._model = model.split("/", 1)[-1]
|
||||
|
||||
async def ocr(
|
||||
self, content: bytes, mime_type: str
|
||||
) -> tuple[str, list[dict[str, Any]]]:
|
||||
data_url = (
|
||||
f"data:{mime_type};base64,{base64.b64encode(content).decode('ascii')}"
|
||||
)
|
||||
resp = await self._client.ocr.process_async(
|
||||
model=self._model,
|
||||
document={"type": "document_url", "document_url": data_url},
|
||||
)
|
||||
pages = [(p.index, p.markdown or "") for p in (resp.pages or [])]
|
||||
return _pages_to_text(pages)
|
||||
|
||||
|
||||
def build_ocr_backend(settings: Settings) -> _OcrBackend | None:
|
||||
"""Select an OCR backend from settings, or None when none is available."""
|
||||
provider = settings.document_ocr_provider
|
||||
if provider == "none":
|
||||
return None
|
||||
|
||||
if provider in ("gateway", "auto") and settings.embedding_gateway_url:
|
||||
token_provider = None
|
||||
if settings.embedding_gateway_client_id:
|
||||
# Lazy import avoids a document_processors -> embedding cycle at load.
|
||||
from ..embedding.gateway_client import ( # noqa: PLC0415
|
||||
GatewayTokenProvider,
|
||||
)
|
||||
|
||||
assert settings.embedding_gateway_token_url is not None
|
||||
assert settings.embedding_gateway_client_secret is not None
|
||||
token_provider = GatewayTokenProvider(
|
||||
token_url=settings.embedding_gateway_token_url,
|
||||
client_id=settings.embedding_gateway_client_id,
|
||||
client_secret=settings.embedding_gateway_client_secret,
|
||||
scope=settings.embedding_gateway_scope,
|
||||
)
|
||||
return _GatewayOcrBackend(
|
||||
settings.embedding_gateway_url, settings.document_ocr_model, token_provider
|
||||
)
|
||||
|
||||
if provider in ("mistral", "auto") and settings.mistral_api_key:
|
||||
return _MistralOcrBackend(
|
||||
settings.mistral_api_key,
|
||||
settings.document_ocr_model,
|
||||
settings.mistral_base_url,
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
class OcrProcessor(DocumentProcessor):
|
||||
"""Tier-3 OCR processor (gateway or direct Mistral backend)."""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "ocr"
|
||||
|
||||
@property
|
||||
def tier(self) -> str:
|
||||
return "ocr"
|
||||
|
||||
@property
|
||||
def supported_mime_types(self) -> set[str]:
|
||||
return {"application/pdf"}
|
||||
|
||||
async def process(
|
||||
self,
|
||||
content: bytes,
|
||||
content_type: str,
|
||||
filename: str | None = None,
|
||||
options: dict[str, Any] | None = None,
|
||||
progress_callback: (
|
||||
Callable[[float, float | None, str | None], Awaitable[None]] | None
|
||||
) = None,
|
||||
) -> ProcessingResult:
|
||||
settings = get_settings()
|
||||
backend = build_ocr_backend(settings)
|
||||
if backend is None:
|
||||
logger.warning(
|
||||
"OCR requested for %s but no backend is configured (provider=%s)",
|
||||
filename or "<bytes>",
|
||||
settings.document_ocr_provider,
|
||||
)
|
||||
return ProcessingResult(
|
||||
text="",
|
||||
metadata={"parse_failed_reason": "unsupported"},
|
||||
processor=self.name,
|
||||
success=False,
|
||||
error="no OCR backend configured",
|
||||
)
|
||||
try:
|
||||
text, boundaries = await backend.ocr(
|
||||
content, content_type.split(";")[0].strip().lower()
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("OCR failed for %s: %s", filename or "<bytes>", e)
|
||||
return ProcessingResult(
|
||||
text="",
|
||||
metadata={"parse_failed_reason": "error"},
|
||||
processor=self.name,
|
||||
success=False,
|
||||
error=f"{type(e).__name__}: {e}",
|
||||
)
|
||||
return ProcessingResult(
|
||||
text=text,
|
||||
metadata={
|
||||
"page_count": len(boundaries),
|
||||
"page_boundaries": boundaries,
|
||||
"file_size": len(content),
|
||||
},
|
||||
processor=self.name,
|
||||
)
|
||||
|
||||
async def health_check(self) -> bool:
|
||||
return True
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Unit tests for the tier-3 OCR processor + backend selection."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from nextcloud_mcp_server.document_processors import ocr
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def _settings(**kw) -> Any: # a Settings stand-in (only the read fields matter)
|
||||
base = dict(
|
||||
document_ocr_provider="auto",
|
||||
document_ocr_model="mistral/mistral-ocr-latest",
|
||||
embedding_gateway_url=None,
|
||||
embedding_gateway_client_id=None,
|
||||
embedding_gateway_client_secret=None,
|
||||
embedding_gateway_token_url=None,
|
||||
embedding_gateway_scope=None,
|
||||
mistral_api_key=None,
|
||||
mistral_base_url=None,
|
||||
)
|
||||
base.update(kw)
|
||||
return SimpleNamespace(**base)
|
||||
|
||||
|
||||
# --- _pages_to_text ----------------------------------------------------------
|
||||
|
||||
|
||||
def test_pages_to_text_orders_and_exact_boundaries():
|
||||
text, boundaries = ocr._pages_to_text([(1, "B"), (0, "A")]) # out of order
|
||||
assert text == "A\n\nB"
|
||||
assert boundaries[0] == {"page": 1, "start_offset": 0, "end_offset": 1}
|
||||
assert boundaries[1]["page"] == 2
|
||||
# contiguous + offsets index exactly into the text
|
||||
assert boundaries[0]["end_offset"] <= boundaries[1]["start_offset"]
|
||||
assert boundaries[-1]["end_offset"] == len(text)
|
||||
|
||||
|
||||
# --- backend selection -------------------------------------------------------
|
||||
|
||||
|
||||
def test_build_backend_none():
|
||||
assert ocr.build_ocr_backend(_settings(document_ocr_provider="none")) is None
|
||||
|
||||
|
||||
def test_build_backend_gateway():
|
||||
b = ocr.build_ocr_backend(
|
||||
_settings(document_ocr_provider="gateway", embedding_gateway_url="http://gw")
|
||||
)
|
||||
assert isinstance(b, ocr._GatewayOcrBackend)
|
||||
|
||||
|
||||
def test_build_backend_mistral():
|
||||
b = ocr.build_ocr_backend(
|
||||
_settings(document_ocr_provider="mistral", mistral_api_key="k")
|
||||
)
|
||||
assert isinstance(b, ocr._MistralOcrBackend)
|
||||
|
||||
|
||||
def test_build_backend_auto_prefers_gateway():
|
||||
b = ocr.build_ocr_backend(
|
||||
_settings(embedding_gateway_url="http://gw", mistral_api_key="k")
|
||||
)
|
||||
assert isinstance(b, ocr._GatewayOcrBackend)
|
||||
|
||||
|
||||
def test_build_backend_auto_none_configured():
|
||||
assert ocr.build_ocr_backend(_settings()) is None
|
||||
|
||||
|
||||
def test_gateway_backend_url_normalization():
|
||||
b = ocr._GatewayOcrBackend("http://gw", "mistral/mistral-ocr-latest")
|
||||
assert b._url == "http://gw/v1/ocr"
|
||||
b2 = ocr._GatewayOcrBackend("http://gw/v1/", "m")
|
||||
assert b2._url == "http://gw/v1/ocr"
|
||||
|
||||
|
||||
# --- OcrProcessor ------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_processor_unsupported_when_no_backend(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
ocr, "get_settings", lambda: _settings(document_ocr_provider="none")
|
||||
)
|
||||
monkeypatch.setattr(ocr, "build_ocr_backend", lambda s: None)
|
||||
r = await ocr.OcrProcessor().process(b"%PDF-1.7", "application/pdf")
|
||||
assert r.success is False
|
||||
assert r.metadata["parse_failed_reason"] == "unsupported"
|
||||
|
||||
|
||||
async def test_processor_success(monkeypatch):
|
||||
class _FakeBackend:
|
||||
async def ocr(self, content, mime_type):
|
||||
return "hello world", [{"page": 1, "start_offset": 0, "end_offset": 11}]
|
||||
|
||||
monkeypatch.setattr(ocr, "get_settings", lambda: _settings())
|
||||
monkeypatch.setattr(ocr, "build_ocr_backend", lambda s: _FakeBackend())
|
||||
r = await ocr.OcrProcessor().process(b"%PDF-1.7", "application/pdf")
|
||||
assert r.success is True
|
||||
assert r.text == "hello world"
|
||||
assert r.metadata["page_count"] == 1
|
||||
assert r.processor == "ocr"
|
||||
|
||||
|
||||
async def test_processor_backend_error_returns_success_false(monkeypatch):
|
||||
class _BoomBackend:
|
||||
async def ocr(self, content, mime_type):
|
||||
raise RuntimeError("api down")
|
||||
|
||||
monkeypatch.setattr(ocr, "get_settings", lambda: _settings())
|
||||
monkeypatch.setattr(ocr, "build_ocr_backend", lambda s: _BoomBackend())
|
||||
r = await ocr.OcrProcessor().process(b"%PDF-1.7", "application/pdf")
|
||||
assert r.success is False
|
||||
assert r.metadata["parse_failed_reason"] == "error"
|
||||
Reference in New Issue
Block a user