feat(ocr): opt-in batch OCR mode via the gateway's async batch routes

Add DOCUMENT_OCR_MODE=sync|batch (default sync). In batch mode the tier-3 OCR
processor submits documents to the embedding gateway's async Batch OCR routes
(POST /v1/ocr/batch + GET /v1/ocr/batch/{job_id}, astrolabe-cloud-website#372)
for ~50% cheaper large-corpus backfill. The direct Mistral OCR path is left
untouched. Tracked on Deck #332.

Batch jobs run minutes-hours, so the OCR tier cannot block (the procrastinate
worker reclaims jobs in `doing` after INGEST_STALLED_JOB_SECONDS). Instead it
submits, records the gateway job id in a new per-tenant `batch_ocr_jobs` table
(procrastinate args are immutable across retries), and raises a BatchPending
signal that TieredEscalationStrategy turns into a same-queue deferred re-poll —
releasing the worker slot between polls. On completion the per-page markdown is
indexed like the sync path; a failure or a job past
DOCUMENT_OCR_BATCH_MAX_WAIT_SECONDS marks the document parse-failed.

Batch is opt-in and gateway-only: with the direct mistral backend, no gateway
URL, or the inline/memory pipeline (which can't defer), it falls back to sync.
One batch job per document (coalescing N docs/job is a follow-up).

- embedding/gateway_batch_client.py: submit/poll client (reuses GatewayTokenProvider).
- vector/batch_ocr_store.py + migration 008: job tracking (portable SQLite+PG).
- document_processors/escalation.py: BatchPending control-flow signal.
- document_processors/ocr.py: batch state machine + sync fallback.
- vector/processor.py: thread doc identity to the OCR tier; raise BatchPending
  from the pending sentinel; propagate it as control flow (not a failure).
- vector/queue/procrastinate.py: BatchPending -> same-queue retry_in, exempt
  from the transient cap (bounded by the processor's deadline).
- config + docs; tests across client/store/processor/strategy/parse-tier.

1653 unit tests pass; ruff + ty green.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

tests/unit/test_batch_ocr_store.py

@@ -0,0 +1,78 @@
+"""Unit tests for the batch OCR job-tracking store (Deck #332).
+
+Runs against a real temp-SQLite ``RefreshTokenStorage`` (its ``initialize()``
+applies the migrations, incl. ``batch_ocr_jobs``).
+"""
+
+import tempfile
+from pathlib import Path
+
+import pytest
+
+from nextcloud_mcp_server.auth.storage import RefreshTokenStorage
+from nextcloud_mcp_server.vector.batch_ocr_store import BatchOcrJobStore
+
+pytestmark = pytest.mark.unit
+
+
+@pytest.fixture
+async def store():
+    with tempfile.TemporaryDirectory() as tmp:
+        storage = RefreshTokenStorage(db_path=str(Path(tmp) / "batch.db"))
+        await storage.initialize()
+        yield BatchOcrJobStore(storage)
+
+
+_DOC = dict(user_id="u1", doc_id="d1", doc_type="file", etag="v1")
+
+
+async def test_get_missing_returns_none(store):
+    assert await store.get(**_DOC) is None
+
+
+async def test_insert_then_get(store):
+    await store.insert_pending(**_DOC, job_id="mistral/j1")
+    job = await store.get(**_DOC)
+    assert job is not None
+    assert job.job_id == "mistral/j1"
+    assert job.status == "pending"
+    assert job.submitted_at > 0
+
+
+async def test_insert_is_idempotent_on_conflict(store):
+    await store.insert_pending(**_DOC, job_id="mistral/j1", submitted_at=100)
+    # A racing re-submit must not overwrite the first row's job id.
+    await store.insert_pending(**_DOC, job_id="mistral/j2", submitted_at=200)
+    job = await store.get(**_DOC)
+    assert job.job_id == "mistral/j1"
+    assert job.submitted_at == 100
+
+
+async def test_delete(store):
+    await store.insert_pending(**_DOC, job_id="mistral/j1")
+    await store.delete(**_DOC)
+    assert await store.get(**_DOC) is None
+
+
+async def test_delete_stale_for_doc_keeps_current_etag(store):
+    await store.insert_pending(
+        user_id="u1", doc_id="d1", doc_type="file", etag="old", job_id="mistral/old"
+    )
+    await store.insert_pending(
+        user_id="u1", doc_id="d1", doc_type="file", etag="new", job_id="mistral/new"
+    )
+    await store.delete_stale_for_doc(
+        user_id="u1", doc_id="d1", doc_type="file", keep_etag="new"
+    )
+    # Old version row gone; current one kept.
+    assert (
+        await store.get(user_id="u1", doc_id="d1", doc_type="file", etag="old")
+    ) is None
+    kept = await store.get(user_id="u1", doc_id="d1", doc_type="file", etag="new")
+    assert kept is not None and kept.job_id == "mistral/new"
+
+
+async def test_rows_are_scoped_per_document(store):
+    await store.insert_pending(**_DOC, job_id="mistral/j1")
+    other = await store.get(user_id="u1", doc_id="d2", doc_type="file", etag="v1")
+    assert other is None  # different doc_id

tests/unit/test_gateway_batch_client.py

@@ -0,0 +1,141 @@
+"""Unit tests for the gateway batch OCR client (Deck #332).
+
+HTTP is exercised via an ``httpx.MockTransport`` injected by monkeypatching
+``httpx.AsyncClient`` (the repo has no respx dependency).
+"""
+
+from typing import Any, cast
+
+import httpx
+import pytest
+
+from nextcloud_mcp_server.embedding import gateway_batch_client as gbc
+
+pytestmark = pytest.mark.unit
+
+
+def _patch_transport(monkeypatch, handler) -> list[httpx.Request]:
+    """Route the client's httpx calls through ``handler``; return a list that
+    captures each issued request for assertions."""
+    seen: list[httpx.Request] = []
+    real = httpx.AsyncClient
+
+    def factory(*args: Any, **kwargs: Any) -> httpx.AsyncClient:
+        def _wrapped(request: httpx.Request) -> httpx.Response:
+            seen.append(request)
+            return handler(request)
+
+        kwargs["transport"] = httpx.MockTransport(_wrapped)
+        return real(*args, **kwargs)
+
+    monkeypatch.setattr(httpx, "AsyncClient", factory)
+    return seen
+
+
+def test_base_url_normalization():
+    assert gbc.GatewayBatchOcrClient("http://gw", "m")._base == "http://gw/v1"
+    assert gbc.GatewayBatchOcrClient("http://gw/v1/", "m")._base == "http://gw/v1"
+
+
+async def test_submit_posts_one_document_and_returns_job_id(monkeypatch):
+    def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(
+            202, json={"job_id": "mistral/job-1", "status": "pending"}
+        )
+
+    seen = _patch_transport(monkeypatch, handler)
+    client = gbc.GatewayBatchOcrClient("http://gw", "mistral/mistral-ocr-latest")
+
+    job_id = await client.submit(b"%PDF-1.7", "application/pdf", custom_id="doc-9")
+
+    assert job_id == "mistral/job-1"
+    req = seen[0]
+    assert req.method == "POST" and req.url.path == "/v1/ocr/batch"
+    import json
+
+    body = json.loads(req.content)
+    assert body["model"] == "mistral/mistral-ocr-latest"
+    assert len(body["documents"]) == 1
+    assert body["documents"][0]["custom_id"] == "doc-9"
+    assert body["documents"][0]["mime_type"] == "application/pdf"
+    assert body["documents"][0]["document_b64"]  # base64 present
+
+
+async def test_submit_sends_bearer_when_token_provider(monkeypatch):
+    class _Tok:
+        async def get_token(self) -> str:
+            return "tok-abc"
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(202, json={"job_id": "mistral/j", "status": "pending"})
+
+    seen = _patch_transport(monkeypatch, handler)
+    # _Tok duck-types get_token; cast for the type checker (the client only awaits
+    # get_token()).
+    client = gbc.GatewayBatchOcrClient(
+        "http://gw", "m", token_provider=cast(Any, _Tok())
+    )
+    await client.submit(b"x", "application/pdf", custom_id="d")
+    assert seen[0].headers["Authorization"] == "Bearer tok-abc"
+
+
+async def test_poll_pending(monkeypatch):
+    _patch_transport(
+        monkeypatch,
+        lambda r: httpx.Response(200, json={"status": "pending", "total": 1}),
+    )
+    result = await gbc.GatewayBatchOcrClient("http://gw", "m").poll("mistral/j")
+    assert result.is_pending and result.pages == []
+
+
+async def test_poll_succeeded_maps_pages(monkeypatch):
+    body = {
+        "status": "succeeded",
+        "results": [
+            {
+                "custom_id": "d",
+                "pages": [
+                    {"index": 1, "markdown": "two"},
+                    {"index": 0, "markdown": "one"},
+                ],
+            }
+        ],
+    }
+    _patch_transport(monkeypatch, lambda r: httpx.Response(200, json=body))
+    result = await gbc.GatewayBatchOcrClient("http://gw", "m").poll("mistral/j")
+    assert result.is_succeeded
+    # Order is preserved as returned; _pages_to_text sorts downstream.
+    assert result.pages == [(1, "two"), (0, "one")]
+
+
+async def test_poll_failed_surfaces_error(monkeypatch):
+    _patch_transport(
+        monkeypatch,
+        lambda r: httpx.Response(200, json={"status": "failed", "error": "quota"}),
+    )
+    result = await gbc.GatewayBatchOcrClient("http://gw", "m").poll("mistral/j")
+    assert result.is_failed and result.error == "quota"
+
+
+async def test_poll_succeeded_with_per_document_error_is_failed(monkeypatch):
+    body = {"status": "succeeded", "results": [{"custom_id": "d", "error": "bad page"}]}
+    _patch_transport(monkeypatch, lambda r: httpx.Response(200, json=body))
+    result = await gbc.GatewayBatchOcrClient("http://gw", "m").poll("mistral/j")
+    assert result.is_failed and result.error == "bad page"
+
+
+async def test_poll_succeeded_no_results_is_failed(monkeypatch):
+    _patch_transport(
+        monkeypatch,
+        lambda r: httpx.Response(200, json={"status": "succeeded", "results": []}),
+    )
+    result = await gbc.GatewayBatchOcrClient("http://gw", "m").poll("mistral/j")
+    assert result.is_failed
+
+
+async def test_poll_raises_on_http_error(monkeypatch):
+    _patch_transport(
+        monkeypatch, lambda r: httpx.Response(503, json={"detail": "down"})
+    )
+    with pytest.raises(httpx.HTTPStatusError):
+        await gbc.GatewayBatchOcrClient("http://gw", "m").poll("mistral/j")

tests/unit/test_ocr_processor.py

@@ -16,6 +16,9 @@ def _settings(**kw) -> Any:  # a Settings stand-in (only the read fields matter)
         document_ocr_provider="auto",
         document_ocr_model="mistral/mistral-ocr-latest",
         document_ocr_timeout_seconds=180.0,
+        document_ocr_mode="sync",
+        document_ocr_batch_poll_seconds=120,
+        document_ocr_batch_max_wait_seconds=86400,
         embedding_gateway_url=None,
         embedding_gateway_client_id=None,
         embedding_gateway_client_secret=None,
@@ -146,7 +149,7 @@ async def test_processor_timeout_returns_timeout_reason(monkeypatch):
     r = await ocr.OcrProcessor().process(b"%PDF-1.7", "application/pdf")
     assert r.success is False
     assert r.metadata["parse_failed_reason"] == "timeout"
-    assert "timed out" in r.error
+    assert "timed out" in (r.error or "")
 
 
 async def test_gateway_httpx_timeout_maps_to_timeout_reason(monkeypatch):
@@ -165,7 +168,7 @@ async def test_gateway_httpx_timeout_maps_to_timeout_reason(monkeypatch):
     r = await ocr.OcrProcessor().process(b"%PDF-1.7", "application/pdf")
     assert r.success is False
     assert r.metadata["parse_failed_reason"] == "timeout"
-    assert "timed out" in r.error
+    assert "timed out" in (r.error or "")
 
 
 async def test_gateway_backend_uses_configured_timeout(mocker, monkeypatch):
@@ -218,3 +221,196 @@ async def test_mistral_backend_applies_timeout(mocker, monkeypatch):
 
     with pytest.raises(TimeoutError):
         await backend.ocr(b"%PDF-1.7", "application/pdf")
+
+
+# --- batch mode (Deck #332) --------------------------------------------------
+
+from nextcloud_mcp_server.embedding.gateway_batch_client import (  # noqa: E402
+    BatchPollResult,
+)
+from nextcloud_mcp_server.vector import batch_ocr_store as _bos  # noqa: E402
+
+_IDENTITY = {"user_id": "u1", "doc_id": "d1", "doc_type": "file", "etag": "v1"}
+
+
+class _FakeStore:
+    """In-memory stand-in for BatchOcrJobStore keyed like the real table."""
+
+    def __init__(self, preset=None):
+        self.rows: dict[tuple, Any] = {}
+        self.deleted: list[tuple] = []
+        self.stale_swept: list[tuple] = []
+        if preset is not None:
+            self.rows[("u1", "d1", "file", "v1")] = preset
+
+    async def get(self, *, user_id, doc_id, doc_type, etag):
+        return self.rows.get((user_id, doc_id, doc_type, etag))
+
+    async def insert_pending(
+        self, *, user_id, doc_id, doc_type, etag, job_id, submitted_at=None
+    ):
+        self.rows[(user_id, doc_id, doc_type, etag)] = SimpleNamespace(
+            job_id=job_id, status="pending", submitted_at=submitted_at or 1000
+        )
+
+    async def delete(self, *, user_id, doc_id, doc_type, etag):
+        self.deleted.append((user_id, doc_id, doc_type, etag))
+        self.rows.pop((user_id, doc_id, doc_type, etag), None)
+
+    async def delete_stale_for_doc(self, *, user_id, doc_id, doc_type, keep_etag):
+        self.stale_swept.append((user_id, doc_id, doc_type, keep_etag))
+
+
+class _FakeBatchClient:
+    def __init__(self, *, submit_job="mistral/job-1", poll=None):
+        self._submit_job = submit_job
+        self._poll = poll or BatchPollResult(status="pending", pages=[])
+        self.submitted: list[tuple] = []
+        self.polled: list[str] = []
+
+    async def submit(self, content, mime_type, custom_id):
+        self.submitted.append((content, mime_type, custom_id))
+        return self._submit_job
+
+    async def poll(self, job_id):
+        self.polled.append(job_id)
+        return self._poll
+
+
+def _wire_batch(monkeypatch, *, client, store, settings=None):
+    settings = settings or _settings(
+        document_ocr_mode="batch",
+        document_ocr_provider="gateway",
+        embedding_gateway_url="http://gw",
+    )
+    monkeypatch.setattr(ocr, "get_settings", lambda: settings)
+    monkeypatch.setattr(ocr, "build_gateway_batch_client", lambda s: client)
+
+    async def _shared(cls):
+        return store
+
+    monkeypatch.setattr(_bos.BatchOcrJobStore, "shared", classmethod(_shared))
+
+
+async def test_batch_first_run_submits_and_returns_pending_sentinel(monkeypatch):
+    client = _FakeBatchClient()
+    store = _FakeStore()
+    _wire_batch(monkeypatch, client=client, store=store)
+
+    r = await ocr.OcrProcessor().process(
+        b"%PDF-1.7", "application/pdf", options=dict(_IDENTITY)
+    )
+
+    assert r.success is False
+    assert r.metadata[ocr.OCR_BATCH_PENDING_KEY] is True
+    assert r.metadata[ocr.OCR_BATCH_RETRY_IN_KEY] == 120
+    # submitted with the doc id as custom_id, recorded a pending row, swept stale
+    assert client.submitted and client.submitted[0][2] == "d1"
+    assert store.rows[("u1", "d1", "file", "v1")].job_id == "mistral/job-1"
+    assert store.stale_swept == [("u1", "d1", "file", "v1")]
+
+
+async def test_batch_existing_pending_polls_and_defers(monkeypatch):
+    preset = SimpleNamespace(job_id="mistral/j", status="pending", submitted_at=1000)
+    client = _FakeBatchClient(poll=BatchPollResult(status="pending", pages=[]))
+    store = _FakeStore(preset=preset)
+    # submitted just now -> deadline not reached
+    monkeypatch.setattr(ocr.time, "time", lambda: 1000.0)
+    _wire_batch(monkeypatch, client=client, store=store)
+
+    r = await ocr.OcrProcessor().process(
+        b"%PDF", "application/pdf", options=dict(_IDENTITY)
+    )
+
+    assert client.polled == ["mistral/j"]
+    assert r.metadata[ocr.OCR_BATCH_PENDING_KEY] is True
+    assert client.submitted == []  # did NOT resubmit
+
+
+async def test_batch_succeeded_returns_indexed_result(monkeypatch):
+    preset = SimpleNamespace(job_id="mistral/j", status="pending", submitted_at=1000)
+    client = _FakeBatchClient(
+        poll=BatchPollResult(status="succeeded", pages=[(0, "# One"), (1, "## Two")])
+    )
+    store = _FakeStore(preset=preset)
+    _wire_batch(monkeypatch, client=client, store=store)
+
+    r = await ocr.OcrProcessor().process(
+        b"%PDF", "application/pdf", options=dict(_IDENTITY)
+    )
+
+    assert r.success is True
+    assert r.text == "# One\n\n## Two"
+    assert r.metadata["page_count"] == 2
+    assert ("u1", "d1", "file", "v1") in store.deleted  # row cleaned up
+
+
+async def test_batch_failed_marks_parse_error(monkeypatch):
+    preset = SimpleNamespace(job_id="mistral/j", status="pending", submitted_at=1000)
+    client = _FakeBatchClient(
+        poll=BatchPollResult(status="failed", pages=[], error="x")
+    )
+    store = _FakeStore(preset=preset)
+    _wire_batch(monkeypatch, client=client, store=store)
+
+    r = await ocr.OcrProcessor().process(
+        b"%PDF", "application/pdf", options=dict(_IDENTITY)
+    )
+
+    assert r.success is False
+    assert r.metadata["parse_failed_reason"] == "error"
+    assert ("u1", "d1", "file", "v1") in store.deleted
+
+
+async def test_batch_deadline_exceeded_marks_timeout(monkeypatch):
+    preset = SimpleNamespace(job_id="mistral/j", status="pending", submitted_at=1000)
+    client = _FakeBatchClient(poll=BatchPollResult(status="pending", pages=[]))
+    store = _FakeStore(preset=preset)
+    # now far past submitted_at + max_wait (86400)
+    monkeypatch.setattr(ocr.time, "time", lambda: 1000.0 + 90000)
+    _wire_batch(monkeypatch, client=client, store=store)
+
+    r = await ocr.OcrProcessor().process(
+        b"%PDF", "application/pdf", options=dict(_IDENTITY)
+    )
+
+    assert r.success is False
+    assert r.metadata["parse_failed_reason"] == "timeout"
+    assert ("u1", "d1", "file", "v1") in store.deleted
+
+
+async def test_batch_falls_back_to_sync_when_no_gateway(monkeypatch):
+    class _FakeBackend:
+        async def ocr(self, content, mime_type):
+            return "sync text", [{"page": 1, "start_offset": 0, "end_offset": 9}]
+
+    settings = _settings(document_ocr_mode="batch", document_ocr_provider="mistral")
+    monkeypatch.setattr(ocr, "get_settings", lambda: settings)
+    monkeypatch.setattr(ocr, "build_gateway_batch_client", lambda s: None)
+    monkeypatch.setattr(ocr, "build_ocr_backend", lambda s: _FakeBackend())
+
+    r = await ocr.OcrProcessor().process(
+        b"%PDF", "application/pdf", options=dict(_IDENTITY)
+    )
+    assert r.success is True and r.text == "sync text"
+
+
+async def test_batch_falls_back_to_sync_when_no_identity(monkeypatch):
+    class _FakeBackend:
+        async def ocr(self, content, mime_type):
+            return "sync text", [{"page": 1, "start_offset": 0, "end_offset": 9}]
+
+    client = _FakeBatchClient()
+    settings = _settings(
+        document_ocr_mode="batch",
+        document_ocr_provider="gateway",
+        embedding_gateway_url="http://gw",
+    )
+    monkeypatch.setattr(ocr, "get_settings", lambda: settings)
+    monkeypatch.setattr(ocr, "build_gateway_batch_client", lambda s: client)
+    monkeypatch.setattr(ocr, "build_ocr_backend", lambda s: _FakeBackend())
+
+    # No options -> inline path -> batch inapplicable -> sync fallback.
+    r = await ocr.OcrProcessor().process(b"%PDF", "application/pdf", options=None)
+    assert r.success is True and r.text == "sync text"
+    assert client.submitted == []  # never attempted batch

tests/unit/vector/test_parse_pdf_tier.py

@@ -97,3 +97,40 @@ async def test_hard_failure_returns_result_without_escalating(monkeypatch):
     reg.evaluate_escalation.assert_not_called()
     rec.assert_not_called()
     sup.assert_not_called()
+
+
+async def test_ocr_batch_pending_sentinel_raises_batch_pending():
+    """Batch OCR (Deck #332): the OCR tier's pending sentinel result is turned
+    into a BatchPending raise (same decision point as EscalateError), carrying
+    the processor's retry_in, and the escalation gate is never consulted."""
+    from nextcloud_mcp_server.document_processors.escalation import BatchPending
+    from nextcloud_mcp_server.document_processors.ocr import (
+        OCR_BATCH_PENDING_KEY,
+        OCR_BATCH_RETRY_IN_KEY,
+    )
+
+    result = ProcessingResult(
+        text="",
+        metadata={OCR_BATCH_PENDING_KEY: True, OCR_BATCH_RETRY_IN_KEY: 90},
+        processor="ocr",
+        success=False,
+    )
+    reg = _registry(result, decision=None)
+    with pytest.raises(BatchPending) as ei:
+        await processor._parse_pdf_tier(
+            reg, b"%PDF", "application/pdf", "scan.pdf", "ocr", settings=object()
+        )
+    assert ei.value.retry_in == 90
+    reg.evaluate_escalation.assert_not_called()
+
+
+async def test_options_threaded_to_process_tier():
+    """The OCR identity options are forwarded to process_tier (batch needs them)."""
+    result = ProcessingResult(text="clean", metadata={}, processor="ocr")
+    reg = _registry(result, decision=None)
+    opts = {"user_id": "u", "doc_id": "d", "doc_type": "file", "etag": "v"}
+    await processor._parse_pdf_tier(
+        reg, b"%PDF", "application/pdf", "f.pdf", "ocr", settings=object(), options=opts
+    )
+    # process_tier(content, content_type, filename, tier, options=...)
+    assert reg.process_tier.await_args.kwargs["options"] == opts

tests/unit/vector/test_tiered_escalation_strategy.py

@@ -14,6 +14,7 @@ from procrastinate.jobs import Job
 import nextcloud_mcp_server.vector.queue.procrastinate as pq
 from nextcloud_mcp_server.document_processors.escalation import (
     TIER_LADDER,
+    BatchPending,
     EscalateError,
     next_tier,
 )
@@ -120,3 +121,30 @@ class TestTieredEscalationStrategy:
             exception=ValueError("permanent"), job=_job(attempts=1)
         )
         assert decision is None
+
+    def test_batch_pending_defers_same_queue(self):
+        # Batch OCR re-poll (Deck #332): same-queue deferral after retry_in.
+        before = datetime.now(timezone.utc)
+        decision = self._strategy().get_retry_decision(
+            exception=BatchPending(retry_in=120), job=_job(queue=pq.INGEST_QUEUE_OCR)
+        )
+        after = datetime.now(timezone.utc)
+        assert decision is not None
+        assert decision.queue is None  # stays on its own tier queue
+        assert decision.retry_at is not None
+        lo = (decision.retry_at - after).total_seconds()
+        hi = (decision.retry_at - before).total_seconds()
+        assert lo <= 120 <= hi
+
+    def test_batch_pending_exempt_from_transient_cap(self):
+        # A batch can take hours -> many polls; the transient cap must NOT stop it
+        # (the OCR processor's own deadline terminates a stuck job instead).
+        decision = self._strategy(max_transient=5).get_retry_decision(
+            exception=BatchPending(retry_in=60), job=_job(attempts=999)
+        )
+        assert decision is not None and decision.retry_at is not None
+
+    def test_batch_pending_unwraps_exception_group(self):
+        group = ExceptionGroup("wrapped", [BatchPending(retry_in=60)])
+        decision = self._strategy().get_retry_decision(exception=group, job=_job())
+        assert decision is not None and decision.retry_at is not None