feat(document): configurable OCR timeout and fail-fast PDF size guard
Two ingest-robustness fixes from card 309 (OHR-Bench smoke-test triage).
The OCR backend timeout was a hardcoded 180s module constant, so a tenant
whose gateway has its own shorter ceiling couldn't tune it. Promote it to
DOCUMENT_OCR_TIMEOUT_SECONDS (default 180), resolved per call via get_settings
so an override applies without a restart.
Large, awkward PDFs (e.g. a 42 MB scanned DUDE) were handed straight to the
fast/OCR tiers, where they burned the full OCR timeout for zero recovered
text. Add a pre-parse size guard in the tiered PDF pipeline: a PDF over
DOCUMENT_MAX_PDF_SIZE_MB (default 50, 0 disables) fails fast with
parse_failed_reason="oversize" before any tier runs, so the existing
permanent-failure path marks the placeholder failed and records
astrolabe_document_parse_failed_total{reason="oversize"} instead of retrying.
Both knobs go through Settings + dynaconf validators (env-var keys verified by
regression tests) and are documented under Background Indexing Configuration.
Refs: Deck board 12 card 309 (AC #3 OCR timeout + size guard).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
457c115ef4
commit
523e4cb7b5
@@ -74,6 +74,7 @@ class _Settings:
|
||||
page_fraction=0.5,
|
||||
min_page_chars=16,
|
||||
detect_scanned=False,
|
||||
max_pdf_size_mb=0.0,
|
||||
):
|
||||
self.document_tier1_engine = engine
|
||||
self.document_classify_enabled = classify
|
||||
@@ -82,6 +83,7 @@ class _Settings:
|
||||
self.document_ocr_page_fraction = page_fraction
|
||||
self.document_ocr_min_page_chars = min_page_chars
|
||||
self.document_ocr_detect_scanned = detect_scanned
|
||||
self.document_max_pdf_size_mb = max_pdf_size_mb
|
||||
|
||||
|
||||
def _registry(*procs: tuple[DocumentProcessor, int]) -> ProcessorRegistry:
|
||||
@@ -98,6 +100,43 @@ async def test_pdf_routes_to_fast_tier(monkeypatch):
|
||||
assert res.processor == "fast"
|
||||
|
||||
|
||||
async def test_oversize_pdf_fails_fast_without_parsing(monkeypatch):
|
||||
"""A PDF over the size cap must fail fast as 'oversize' before any tier runs."""
|
||||
monkeypatch.setattr(
|
||||
reg_mod, "get_settings", lambda: _Settings(max_pdf_size_mb=0.001)
|
||||
)
|
||||
fast = _Fake("fast", "fast")
|
||||
ran = False
|
||||
orig = fast.process
|
||||
|
||||
async def _tracking(*a, **k):
|
||||
nonlocal ran
|
||||
ran = True
|
||||
return await orig(*a, **k)
|
||||
|
||||
fast.process = _tracking # type: ignore[method-assign]
|
||||
r = _registry((fast, 20))
|
||||
|
||||
# ~2 KB > 0.001 MB (~1 KB) cap.
|
||||
res = await r.process(b"%PDF-1.7" + b"0" * 2048, "application/pdf", "big.pdf")
|
||||
|
||||
assert res.success is False
|
||||
assert res.metadata["parse_failed_reason"] == "oversize"
|
||||
assert res.processor == "size_guard"
|
||||
assert ran is False, "size guard must short-circuit before the fast tier runs"
|
||||
|
||||
|
||||
async def test_under_cap_pdf_still_parses(monkeypatch):
|
||||
"""A PDF under the cap is unaffected by the guard."""
|
||||
monkeypatch.setattr(
|
||||
reg_mod, "get_settings", lambda: _Settings(max_pdf_size_mb=10.0)
|
||||
)
|
||||
r = _registry((_Fake("fast", "fast"), 20))
|
||||
res = await r.process(b"%PDF-1.7", "application/pdf")
|
||||
assert res.success is True
|
||||
assert res.processor == "fast"
|
||||
|
||||
|
||||
async def test_engine_rollback_uses_structured(monkeypatch):
|
||||
monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(engine="pymupdf"))
|
||||
r = _registry((_Fake("fast", "fast"), 20), (_Fake("structured", "structured"), 10))
|
||||
|
||||
Reference in New Issue
Block a user