admin/mcp-nextcloud
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #892 from cbcoutinho/feat/309-ocr-timeout-pdf-size-guard

Browse Source 
feat(document): configurable OCR timeout and fail-fast PDF size guard
This commit is contained in:
Chris Coutinho
2026-06-11 10:40:15 +02:00
committed by GitHub
parent 0c73fb7bff ebd0b469f5
commit 0672e4a05c
7 changed files with 268 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nextcloud_mcp_server/config.py
+22
View File
@@ -146,6 +146,10 @@ _DEFAULTS: dict[str, Any] = {
"document_pdf_graphics_limit": 1000,
"document_parse_timeout_seconds": 120.0,
"document_parse_mem_limit_mb": 1536,
# Pre-parse size cap (MB): PDFs larger than this fail fast with reason
# "oversize" instead of burning the OCR timeout to 0 chars on a pathological
# file. 0 disables the guard.
"document_max_pdf_size_mb": 50.0,
# Tier-0 classifier (records classification metrics on the tiered path)
"document_classify_enabled": True,
# Tiered PDF pipeline: pypdfium2 is the default/only hot-path extractor;
@@ -168,6 +172,10 @@ _DEFAULTS: dict[str, Any] = {
"document_ocr_page_fraction": 0.5,
"document_ocr_min_page_chars": 16,
"document_ocr_detect_scanned": True,
# OCR backend request timeout (seconds). Slow scanned newspapers can take
# 20-60s; raise/lower per tenant. Configurable so a tenant isn't stuck with
# the 180s default when its gateway has its own shorter ceiling.
"document_ocr_timeout_seconds": 180.0,
# Observability
"metrics_enabled": True,
"metrics_port": 9090,
@@ -319,7 +327,10 @@ _dynaconf = Dynaconf(
Validator("VERIFICATION_CONCURRENCY", gte=1),
Validator("DOCUMENT_CHUNK_SIZE", gte=1),
Validator("DOCUMENT_PARSE_TIMEOUT_SECONDS", gte=1),
Validator("DOCUMENT_OCR_TIMEOUT_SECONDS", gte=1),
Validator("DOCUMENT_PARSE_MEM_LIMIT_MB", gte=128),
# 0 disables the pre-parse PDF size cap; otherwise it must be positive.
Validator("DOCUMENT_MAX_PDF_SIZE_MB", gte=0),
# >=1: pymupdf4llm treats graphics_limit=0 as "no cap", which would
# re-expose the OOM this guards against.
Validator("DOCUMENT_PDF_GRAPHICS_LIMIT", gte=1),
@@ -782,6 +793,11 @@ class Settings:
# float so a fractional DOCUMENT_PARSE_TIMEOUT_SECONDS is honoured, matching
# anyio.move_on_after's float seconds.
document_parse_timeout_seconds: float = 120.0
# Pre-parse PDF size cap (MB). A PDF larger than this fails fast with
# parse_failed_reason="oversize" (placeholder marked "failed") rather than
# being handed to the fast/OCR tiers, where a pathological large file burns
# the OCR timeout for 0 chars. 0 disables the guard.
document_max_pdf_size_mb: float = 50.0
# RLIMIT_AS in the parse subprocess (below the pod limit). Applied once per
# worker for its lifetime, so changing it needs a pod restart.
document_parse_mem_limit_mb: int = 1536
@@ -801,6 +817,10 @@ class Settings:
# gateway routes on the "<provider>/" prefix; the direct mistral backend
# strips it.
document_ocr_model: str = "mistral/mistral-ocr-latest"
# OCR backend HTTP request timeout (seconds). float for parity with the
# parse timeout / httpx.Timeout; per-tenant tunable so a gateway with a
# shorter ceiling isn't masked by the 180s default.
document_ocr_timeout_seconds: float = 180.0
# OCR escalation triggers (tier-0), per-tenant tunable. A page is OCR-worthy
# if near-empty (< min_page_chars) OR low text-quality (< min_text_quality)
# OR (when detect_scanned, image-analysis only runs when OCR is enabled)
@@ -1431,12 +1451,14 @@ def get_settings() -> Settings:
"document_chunk_page_aware": "DOCUMENT_CHUNK_PAGE_AWARE",
"document_pdf_graphics_limit": "DOCUMENT_PDF_GRAPHICS_LIMIT",
"document_parse_timeout_seconds": "DOCUMENT_PARSE_TIMEOUT_SECONDS",
"document_max_pdf_size_mb": "DOCUMENT_MAX_PDF_SIZE_MB",
"document_parse_mem_limit_mb": "DOCUMENT_PARSE_MEM_LIMIT_MB",
"document_classify_enabled": "DOCUMENT_CLASSIFY_ENABLED",
"document_tier1_engine": "DOCUMENT_TIER1_ENGINE",
"document_ocr_enabled": "DOCUMENT_OCR_ENABLED",
"document_ocr_provider": "DOCUMENT_OCR_PROVIDER",
"document_ocr_model": "DOCUMENT_OCR_MODEL",
"document_ocr_timeout_seconds": "DOCUMENT_OCR_TIMEOUT_SECONDS",
"document_ocr_min_text_quality": "DOCUMENT_OCR_MIN_TEXT_QUALITY",
"document_ocr_page_fraction": "DOCUMENT_OCR_PAGE_FRACTION",
"document_ocr_min_page_chars": "DOCUMENT_OCR_MIN_PAGE_CHARS",
nextcloud_mcp_server/document_processors/ocr.py
+37 -6
View File
@@ -31,7 +31,9 @@ from .base import DocumentProcessor, ProcessingResult
logger = logging.getLogger(__name__)
_OCR_TIMEOUT_SECONDS = 180.0
# Connect timeout for the OCR backend request. The overall (read) timeout is
# configurable via DOCUMENT_OCR_TIMEOUT_SECONDS and resolved per call.
_OCR_CONNECT_TIMEOUT_SECONDS = 10.0
def _pages_to_text(
@@ -93,8 +95,12 @@ class _GatewayOcrBackend(_OcrBackend):
"document_b64": base64.b64encode(content).decode("ascii"),
"mime_type": mime_type,
}
# Resolved per call (get_settings builds fresh) so test monkeypatching is
# honoured; a live tenant change still needs a restart because the backend
# instance itself is cached for the pod's lifetime.
ocr_timeout = get_settings().document_ocr_timeout_seconds
async with httpx.AsyncClient(
timeout=httpx.Timeout(_OCR_TIMEOUT_SECONDS, connect=10.0)
timeout=httpx.Timeout(ocr_timeout, connect=_OCR_CONNECT_TIMEOUT_SECONDS)
) as client:
resp = await client.post(self._url, json=payload, headers=headers)
resp.raise_for_status()
@@ -120,10 +126,17 @@ class _MistralOcrBackend(_OcrBackend):
data_url = (
f"data:{mime_type};base64,{base64.b64encode(content).decode('ascii')}"
)
resp = await self._client.ocr.process_async(
model=self._model,
document={"type": "document_url", "document_url": data_url},
)
# Apply DOCUMENT_OCR_TIMEOUT_SECONDS uniformly with the gateway backend.
# The Mistral SDK manages its own httpx client, so wrap the call in an
# anyio cancel-scope timeout rather than threading a per-request timeout
# through the SDK; on expiry this raises TimeoutError, which the
# OcrProcessor turns into a clean parse failure.
ocr_timeout = get_settings().document_ocr_timeout_seconds
with anyio.fail_after(ocr_timeout):
resp = await self._client.ocr.process_async(
model=self._model,
document={"type": "document_url", "document_url": data_url},
)
pages = [(p.index, p.markdown or "") for p in (resp.pages or [])]
return _pages_to_text(pages)
@@ -251,6 +264,24 @@ class OcrProcessor(DocumentProcessor):
text, boundaries = await backend.ocr(
content, content_type.split(";")[0].strip().lower()
)
except (TimeoutError, httpx.TimeoutException):
# Two timeout shapes reach here: the Mistral backend's
# anyio.fail_after raises the builtin TimeoutError, while the gateway
# backend's httpx.Timeout raises httpx.ReadTimeout (a
# httpx.TimeoutException, NOT a TimeoutError). Catch both so a
# too-low DOCUMENT_OCR_TIMEOUT_SECONDS lands in its own reason bucket
# rather than being conflated with provider errors.
timeout = settings.document_ocr_timeout_seconds
logger.warning(
"OCR timed out for %s after %.1fs", filename or "<bytes>", timeout
)
return ProcessingResult(
text="",
metadata={"parse_failed_reason": "timeout"},
processor=self.name,
success=False,
error=f"OCR timed out after {timeout:.1f}s",
)
except Exception as e:
logger.warning("OCR failed for %s: %s", filename or "<bytes>", e)
return ProcessingResult(
nextcloud_mcp_server/document_processors/registry.py
+27
View File
@@ -202,6 +202,33 @@ class ProcessorRegistry:
"""
settings = get_settings()
# Pre-parse size guard: a pathologically large PDF (e.g. a 42 MB scanned
# DUDE) burns the OCR timeout for 0 chars. Fail fast with an explicit
# reason so the caller marks the placeholder "failed" instead of
# retrying. 0 disables the cap. This lives on the auto-tiered path only:
# an explicit processor_name="ocr" override (registry.process) bypasses
# _process_pdf entirely and is intentionally not size-gated (power-user
# escape hatch). Returning here also skips _run_processor, so the
# rejection is counted on astrolabe_document_parse_failed_total{oversize}
# (via vector/processor.py) but deliberately not on the parse-duration
# histogram -- there is no parse to time.
max_pdf_mb = settings.document_max_pdf_size_mb
if max_pdf_mb > 0 and len(content) > max_pdf_mb * 1024 * 1024:
size_mb = len(content) / (1024 * 1024)
logger.warning(
"PDF %s is %.1f MB (> %.1f MB cap); failing fast as oversize",
filename or "<bytes>",
size_mb,
max_pdf_mb,
)
return ProcessingResult(
text="",
metadata={"parse_failed_reason": "oversize"},
processor="size_guard",
success=False,
error=(f"PDF exceeds size cap: {size_mb:.1f} MB > {max_pdf_mb:.1f} MB"),
)
if settings.document_tier1_engine == "pymupdf":
processor = self._pdf_processor_for_tier("structured")
if processor is None: