fix(document): apply OCR timeout to Mistral backend + review/Sonar fixes (#892)
Round-1 review on PR #892: - Wire DOCUMENT_OCR_TIMEOUT_SECONDS into _MistralOcrBackend too (was gateway-only): wrap process_async in anyio.fail_after so the SDK-managed client honours the setting; on expiry it fails fast as a clean parse error. Test added. - Tighten the misleading "honoured without a restart" comment — per-call get_settings() is for test monkeypatching; a live change still needs a restart since the backend is cached for the pod lifetime. - Comment the size guard's two intentional gaps: an explicit processor_name override bypasses it, and the early return skips the parse-duration histogram. SonarCloud (new-code smells in the added tests): - S1244 float-equality asserts → pytest.approx (test_config.py, test_ocr_processor.py). - S1186/S7503: rewrite the gateway-timeout test with mocker AsyncMock/MagicMock instead of a hand-rolled fake client (no empty method, no async-without-await). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
523e4cb7b5
commit
64ea5c8631
@@ -95,8 +95,9 @@ class _GatewayOcrBackend(_OcrBackend):
|
||||
"document_b64": base64.b64encode(content).decode("ascii"),
|
||||
"mime_type": mime_type,
|
||||
}
|
||||
# Resolve the timeout per call (get_settings builds fresh, so a test or
|
||||
# tenant override is honoured without a restart).
|
||||
# Resolved per call (get_settings builds fresh) so test monkeypatching is
|
||||
# honoured; a live tenant change still needs a restart because the backend
|
||||
# instance itself is cached for the pod's lifetime.
|
||||
ocr_timeout = get_settings().document_ocr_timeout_seconds
|
||||
async with httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(ocr_timeout, connect=_OCR_CONNECT_TIMEOUT_SECONDS)
|
||||
@@ -125,10 +126,17 @@ class _MistralOcrBackend(_OcrBackend):
|
||||
data_url = (
|
||||
f"data:{mime_type};base64,{base64.b64encode(content).decode('ascii')}"
|
||||
)
|
||||
resp = await self._client.ocr.process_async(
|
||||
model=self._model,
|
||||
document={"type": "document_url", "document_url": data_url},
|
||||
)
|
||||
# Apply DOCUMENT_OCR_TIMEOUT_SECONDS uniformly with the gateway backend.
|
||||
# The Mistral SDK manages its own httpx client, so wrap the call in an
|
||||
# anyio cancel-scope timeout rather than threading a per-request timeout
|
||||
# through the SDK; on expiry this raises TimeoutError, which the
|
||||
# OcrProcessor turns into a clean parse failure.
|
||||
ocr_timeout = get_settings().document_ocr_timeout_seconds
|
||||
with anyio.fail_after(ocr_timeout):
|
||||
resp = await self._client.ocr.process_async(
|
||||
model=self._model,
|
||||
document={"type": "document_url", "document_url": data_url},
|
||||
)
|
||||
pages = [(p.index, p.markdown or "") for p in (resp.pages or [])]
|
||||
return _pages_to_text(pages)
|
||||
|
||||
|
||||
@@ -205,7 +205,13 @@ class ProcessorRegistry:
|
||||
# Pre-parse size guard: a pathologically large PDF (e.g. a 42 MB scanned
|
||||
# DUDE) burns the OCR timeout for 0 chars. Fail fast with an explicit
|
||||
# reason so the caller marks the placeholder "failed" instead of
|
||||
# retrying. 0 disables the cap.
|
||||
# retrying. 0 disables the cap. This lives on the auto-tiered path only:
|
||||
# an explicit processor_name="ocr" override (registry.process) bypasses
|
||||
# _process_pdf entirely and is intentionally not size-gated (power-user
|
||||
# escape hatch). Returning here also skips _run_processor, so the
|
||||
# rejection is counted on astrolabe_document_parse_failed_total{oversize}
|
||||
# (via vector/processor.py) but deliberately not on the parse-duration
|
||||
# histogram -- there is no parse to time.
|
||||
max_pdf_mb = settings.document_max_pdf_size_mb
|
||||
if max_pdf_mb > 0 and len(content) > max_pdf_mb * 1024 * 1024:
|
||||
size_mb = len(content) / (1024 * 1024)
|
||||
|
||||
Reference in New Issue
Block a user