Add DOCUMENT_OCR_MODE=sync|batch (default sync). In batch mode the tier-3 OCR
processor submits documents to the embedding gateway's async Batch OCR routes
(POST /v1/ocr/batch + GET /v1/ocr/batch/{job_id}, astrolabe-cloud-website#372)
for ~50% cheaper large-corpus backfill. The direct Mistral OCR path is left
untouched. Tracked on Deck #332.
Batch jobs run minutes-hours, so the OCR tier cannot block (the procrastinate
worker reclaims jobs in `doing` after INGEST_STALLED_JOB_SECONDS). Instead it
submits, records the gateway job id in a new per-tenant `batch_ocr_jobs` table
(procrastinate args are immutable across retries), and raises a BatchPending
signal that TieredEscalationStrategy turns into a same-queue deferred re-poll —
releasing the worker slot between polls. On completion the per-page markdown is
indexed like the sync path; a failure or a job past
DOCUMENT_OCR_BATCH_MAX_WAIT_SECONDS marks the document parse-failed.
Batch is opt-in and gateway-only: with the direct mistral backend, no gateway
URL, or the inline/memory pipeline (which can't defer), it falls back to sync.
One batch job per document (coalescing N docs/job is a follow-up).
- embedding/gateway_batch_client.py: submit/poll client (reuses GatewayTokenProvider).
- vector/batch_ocr_store.py + migration 008: job tracking (portable SQLite+PG).
- document_processors/escalation.py: BatchPending control-flow signal.
- document_processors/ocr.py: batch state machine + sync fallback.
- vector/processor.py: thread doc identity to the OCR tier; raise BatchPending
from the pending sentinel; propagate it as control flow (not a failure).
- vector/queue/procrastinate.py: BatchPending -> same-queue retry_in, exempt
from the transient cap (bounded by the processor's deadline).
- config + docs; tests across client/store/processor/strategy/parse-tier.
1653 unit tests pass; ruff + ty green.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
142 lines
5.0 KiB
Python
142 lines
5.0 KiB
Python
"""Unit tests for the gateway batch OCR client (Deck #332).
|
|
|
|
HTTP is exercised via an ``httpx.MockTransport`` injected by monkeypatching
|
|
``httpx.AsyncClient`` (the repo has no respx dependency).
|
|
"""
|
|
|
|
from typing import Any, cast
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from nextcloud_mcp_server.embedding import gateway_batch_client as gbc
|
|
|
|
pytestmark = pytest.mark.unit
|
|
|
|
|
|
def _patch_transport(monkeypatch, handler) -> list[httpx.Request]:
|
|
"""Route the client's httpx calls through ``handler``; return a list that
|
|
captures each issued request for assertions."""
|
|
seen: list[httpx.Request] = []
|
|
real = httpx.AsyncClient
|
|
|
|
def factory(*args: Any, **kwargs: Any) -> httpx.AsyncClient:
|
|
def _wrapped(request: httpx.Request) -> httpx.Response:
|
|
seen.append(request)
|
|
return handler(request)
|
|
|
|
kwargs["transport"] = httpx.MockTransport(_wrapped)
|
|
return real(*args, **kwargs)
|
|
|
|
monkeypatch.setattr(httpx, "AsyncClient", factory)
|
|
return seen
|
|
|
|
|
|
def test_base_url_normalization():
|
|
assert gbc.GatewayBatchOcrClient("http://gw", "m")._base == "http://gw/v1"
|
|
assert gbc.GatewayBatchOcrClient("http://gw/v1/", "m")._base == "http://gw/v1"
|
|
|
|
|
|
async def test_submit_posts_one_document_and_returns_job_id(monkeypatch):
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(
|
|
202, json={"job_id": "mistral/job-1", "status": "pending"}
|
|
)
|
|
|
|
seen = _patch_transport(monkeypatch, handler)
|
|
client = gbc.GatewayBatchOcrClient("http://gw", "mistral/mistral-ocr-latest")
|
|
|
|
job_id = await client.submit(b"%PDF-1.7", "application/pdf", custom_id="doc-9")
|
|
|
|
assert job_id == "mistral/job-1"
|
|
req = seen[0]
|
|
assert req.method == "POST" and req.url.path == "/v1/ocr/batch"
|
|
import json
|
|
|
|
body = json.loads(req.content)
|
|
assert body["model"] == "mistral/mistral-ocr-latest"
|
|
assert len(body["documents"]) == 1
|
|
assert body["documents"][0]["custom_id"] == "doc-9"
|
|
assert body["documents"][0]["mime_type"] == "application/pdf"
|
|
assert body["documents"][0]["document_b64"] # base64 present
|
|
|
|
|
|
async def test_submit_sends_bearer_when_token_provider(monkeypatch):
|
|
class _Tok:
|
|
async def get_token(self) -> str:
|
|
return "tok-abc"
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(202, json={"job_id": "mistral/j", "status": "pending"})
|
|
|
|
seen = _patch_transport(monkeypatch, handler)
|
|
# _Tok duck-types get_token; cast for the type checker (the client only awaits
|
|
# get_token()).
|
|
client = gbc.GatewayBatchOcrClient(
|
|
"http://gw", "m", token_provider=cast(Any, _Tok())
|
|
)
|
|
await client.submit(b"x", "application/pdf", custom_id="d")
|
|
assert seen[0].headers["Authorization"] == "Bearer tok-abc"
|
|
|
|
|
|
async def test_poll_pending(monkeypatch):
|
|
_patch_transport(
|
|
monkeypatch,
|
|
lambda r: httpx.Response(200, json={"status": "pending", "total": 1}),
|
|
)
|
|
result = await gbc.GatewayBatchOcrClient("http://gw", "m").poll("mistral/j")
|
|
assert result.is_pending and result.pages == []
|
|
|
|
|
|
async def test_poll_succeeded_maps_pages(monkeypatch):
|
|
body = {
|
|
"status": "succeeded",
|
|
"results": [
|
|
{
|
|
"custom_id": "d",
|
|
"pages": [
|
|
{"index": 1, "markdown": "two"},
|
|
{"index": 0, "markdown": "one"},
|
|
],
|
|
}
|
|
],
|
|
}
|
|
_patch_transport(monkeypatch, lambda r: httpx.Response(200, json=body))
|
|
result = await gbc.GatewayBatchOcrClient("http://gw", "m").poll("mistral/j")
|
|
assert result.is_succeeded
|
|
# Order is preserved as returned; _pages_to_text sorts downstream.
|
|
assert result.pages == [(1, "two"), (0, "one")]
|
|
|
|
|
|
async def test_poll_failed_surfaces_error(monkeypatch):
|
|
_patch_transport(
|
|
monkeypatch,
|
|
lambda r: httpx.Response(200, json={"status": "failed", "error": "quota"}),
|
|
)
|
|
result = await gbc.GatewayBatchOcrClient("http://gw", "m").poll("mistral/j")
|
|
assert result.is_failed and result.error == "quota"
|
|
|
|
|
|
async def test_poll_succeeded_with_per_document_error_is_failed(monkeypatch):
|
|
body = {"status": "succeeded", "results": [{"custom_id": "d", "error": "bad page"}]}
|
|
_patch_transport(monkeypatch, lambda r: httpx.Response(200, json=body))
|
|
result = await gbc.GatewayBatchOcrClient("http://gw", "m").poll("mistral/j")
|
|
assert result.is_failed and result.error == "bad page"
|
|
|
|
|
|
async def test_poll_succeeded_no_results_is_failed(monkeypatch):
|
|
_patch_transport(
|
|
monkeypatch,
|
|
lambda r: httpx.Response(200, json={"status": "succeeded", "results": []}),
|
|
)
|
|
result = await gbc.GatewayBatchOcrClient("http://gw", "m").poll("mistral/j")
|
|
assert result.is_failed
|
|
|
|
|
|
async def test_poll_raises_on_http_error(monkeypatch):
|
|
_patch_transport(
|
|
monkeypatch, lambda r: httpx.Response(503, json={"detail": "down"})
|
|
)
|
|
with pytest.raises(httpx.HTTPStatusError):
|
|
await gbc.GatewayBatchOcrClient("http://gw", "m").poll("mistral/j")
|