Add DOCUMENT_OCR_MODE=sync|batch (default sync). In batch mode the tier-3 OCR
processor submits documents to the embedding gateway's async Batch OCR routes
(POST /v1/ocr/batch + GET /v1/ocr/batch/{job_id}, astrolabe-cloud-website#372)
for ~50% cheaper large-corpus backfill. The direct Mistral OCR path is left
untouched. Tracked on Deck #332.
Batch jobs run minutes-hours, so the OCR tier cannot block (the procrastinate
worker reclaims jobs in `doing` after INGEST_STALLED_JOB_SECONDS). Instead it
submits, records the gateway job id in a new per-tenant `batch_ocr_jobs` table
(procrastinate args are immutable across retries), and raises a BatchPending
signal that TieredEscalationStrategy turns into a same-queue deferred re-poll —
releasing the worker slot between polls. On completion the per-page markdown is
indexed like the sync path; a failure or a job past
DOCUMENT_OCR_BATCH_MAX_WAIT_SECONDS marks the document parse-failed.
Batch is opt-in and gateway-only: with the direct mistral backend, no gateway
URL, or the inline/memory pipeline (which can't defer), it falls back to sync.
One batch job per document (coalescing N docs/job is a follow-up).
- embedding/gateway_batch_client.py: submit/poll client (reuses GatewayTokenProvider).
- vector/batch_ocr_store.py + migration 008: job tracking (portable SQLite+PG).
- document_processors/escalation.py: BatchPending control-flow signal.
- document_processors/ocr.py: batch state machine + sync fallback.
- vector/processor.py: thread doc identity to the OCR tier; raise BatchPending
from the pending sentinel; propagate it as control flow (not a failure).
- vector/queue/procrastinate.py: BatchPending -> same-queue retry_in, exempt
from the transient cap (bounded by the processor's deadline).
- config + docs; tests across client/store/processor/strategy/parse-tier.
1653 unit tests pass; ruff + ty green.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
137 lines
5.4 KiB
Python
137 lines
5.4 KiB
Python
"""Unit tests for the per-tier PDF parse + escalation gate (Deck #323/#324).
|
|
|
|
``processor._parse_pdf_tier`` runs one tier and either indexes the result,
|
|
raises ``EscalateError`` (a real queue-hop), or — when the ideal next tier is
|
|
disabled (OCR off) — records a suppressed escalation and indexes as terminal.
|
|
These exercise the decision without standing up the full ingest pipeline.
|
|
"""
|
|
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
|
|
import pytest
|
|
|
|
from nextcloud_mcp_server.document_processors.base import ProcessingResult
|
|
from nextcloud_mcp_server.document_processors.escalation import (
|
|
EscalateError,
|
|
EscalationDecision,
|
|
)
|
|
from nextcloud_mcp_server.vector import processor
|
|
|
|
pytestmark = pytest.mark.unit
|
|
|
|
|
|
def _registry(result: ProcessingResult, decision):
|
|
reg = MagicMock()
|
|
reg.process_tier = AsyncMock(return_value=result)
|
|
reg.evaluate_escalation = MagicMock(return_value=decision)
|
|
return reg
|
|
|
|
|
|
async def test_good_parse_returns_result(monkeypatch):
|
|
rec = MagicMock()
|
|
sup = MagicMock()
|
|
monkeypatch.setattr(processor, "record_document_escalation", rec)
|
|
monkeypatch.setattr(processor, "record_document_escalation_suppressed", sup)
|
|
result = ProcessingResult(text="clean", metadata={}, processor="fast")
|
|
reg = _registry(result, decision=None)
|
|
out = await processor._parse_pdf_tier(
|
|
reg, b"%PDF", "application/pdf", "f.pdf", "fast", settings=object()
|
|
)
|
|
assert out is result
|
|
rec.assert_not_called()
|
|
sup.assert_not_called()
|
|
|
|
|
|
async def test_low_quality_parse_raises_escalate(monkeypatch):
|
|
rec = MagicMock()
|
|
monkeypatch.setattr(processor, "record_document_escalation", rec)
|
|
result = ProcessingResult(text="", metadata={}, processor="fast")
|
|
reg = _registry(result, decision=EscalationDecision("hop", "ocr", "empty_text"))
|
|
with pytest.raises(EscalateError) as ei:
|
|
await processor._parse_pdf_tier(
|
|
reg, b"%PDF", "application/pdf", "f.pdf", "fast", settings=object()
|
|
)
|
|
assert ei.value.from_tier == "fast"
|
|
assert ei.value.to_tier == "ocr"
|
|
assert ei.value.reason == "empty_text"
|
|
# The escalation is recorded at the decision point.
|
|
rec.assert_called_once_with("fast", "ocr", "empty_text")
|
|
|
|
|
|
async def test_suppressed_decision_indexes_without_hop(monkeypatch):
|
|
"""OCR-off (suppressed): index this tier's result, record the would-be hop,
|
|
do NOT raise EscalateError."""
|
|
rec = MagicMock()
|
|
sup = MagicMock()
|
|
monkeypatch.setattr(processor, "record_document_escalation", rec)
|
|
monkeypatch.setattr(processor, "record_document_escalation_suppressed", sup)
|
|
result = ProcessingResult(text="junk", metadata={}, processor="fast")
|
|
reg = _registry(
|
|
result, decision=EscalationDecision("suppressed", "ocr", "empty_text")
|
|
)
|
|
out = await processor._parse_pdf_tier(
|
|
reg, b"%PDF", "application/pdf", "f.pdf", "fast", settings=object()
|
|
)
|
|
assert out is result # indexed as terminal, no hop
|
|
sup.assert_called_once_with("fast", "ocr", "empty_text")
|
|
rec.assert_not_called()
|
|
|
|
|
|
async def test_hard_failure_returns_result_without_escalating(monkeypatch):
|
|
rec = MagicMock()
|
|
sup = MagicMock()
|
|
monkeypatch.setattr(processor, "record_document_escalation", rec)
|
|
monkeypatch.setattr(processor, "record_document_escalation_suppressed", sup)
|
|
result = ProcessingResult(
|
|
text="",
|
|
metadata={"parse_failed_reason": "oversize"},
|
|
processor="size_guard",
|
|
success=False,
|
|
)
|
|
reg = _registry(result, decision=EscalationDecision("hop", "ocr", "empty_text"))
|
|
out = await processor._parse_pdf_tier(
|
|
reg, b"%PDF", "application/pdf", "big.pdf", "fast", settings=object()
|
|
)
|
|
# success=False short-circuits: the gate is never consulted, no escalation.
|
|
assert out is result
|
|
reg.evaluate_escalation.assert_not_called()
|
|
rec.assert_not_called()
|
|
sup.assert_not_called()
|
|
|
|
|
|
async def test_ocr_batch_pending_sentinel_raises_batch_pending():
|
|
"""Batch OCR (Deck #332): the OCR tier's pending sentinel result is turned
|
|
into a BatchPending raise (same decision point as EscalateError), carrying
|
|
the processor's retry_in, and the escalation gate is never consulted."""
|
|
from nextcloud_mcp_server.document_processors.escalation import BatchPending
|
|
from nextcloud_mcp_server.document_processors.ocr import (
|
|
OCR_BATCH_PENDING_KEY,
|
|
OCR_BATCH_RETRY_IN_KEY,
|
|
)
|
|
|
|
result = ProcessingResult(
|
|
text="",
|
|
metadata={OCR_BATCH_PENDING_KEY: True, OCR_BATCH_RETRY_IN_KEY: 90},
|
|
processor="ocr",
|
|
success=False,
|
|
)
|
|
reg = _registry(result, decision=None)
|
|
with pytest.raises(BatchPending) as ei:
|
|
await processor._parse_pdf_tier(
|
|
reg, b"%PDF", "application/pdf", "scan.pdf", "ocr", settings=object()
|
|
)
|
|
assert ei.value.retry_in == 90
|
|
reg.evaluate_escalation.assert_not_called()
|
|
|
|
|
|
async def test_options_threaded_to_process_tier():
|
|
"""The OCR identity options are forwarded to process_tier (batch needs them)."""
|
|
result = ProcessingResult(text="clean", metadata={}, processor="ocr")
|
|
reg = _registry(result, decision=None)
|
|
opts = {"user_id": "u", "doc_id": "d", "doc_type": "file", "etag": "v"}
|
|
await processor._parse_pdf_tier(
|
|
reg, b"%PDF", "application/pdf", "f.pdf", "ocr", settings=object(), options=opts
|
|
)
|
|
# process_tier(content, content_type, filename, tier, options=...)
|
|
assert reg.process_tier.await_args.kwargs["options"] == opts
|