feat(ingest): per-tier escalation via procrastinate queue-hop

Split external (procrastinate) document processing into per-tier queues so a
document is attempted at most once per tier and requeued to the next tier's
queue on a low-quality parse, using procrastinate's native retry.

- escalation.py: TIER_LADDER (fast->structured->ocr) + EscalateError signal
- registry: process_tier (one tier) + evaluate_escalation post-parse gate
  (reuses classify_from_text) + next_available_tier; shared _classify_result
  and _oversize_result with the inline pipeline
- processor: process_document(tier=...) runs one tier and raises EscalateError
  before embed (junk text never indexed); inline memory path unchanged
- queue/procrastinate: ingest-fast|structured|ocr queues; TieredEscalationStrategy
  (queue-hop on EscalateError, bounded same-tier transient retry); queue-aware
  task; producer defers to ingest-fast; per-queue counts + all-queue reclaim
- cli: worker --tier {fast,structured,ocr}
- billing: pages_ocr usage event + pipeline_tier metadata (paid OCR billed apart)
- observability: astrolabe_ingest_queue_depth{queue,status} gauge + per-queue
  counts in nc_get_vector_sync_status / management status endpoint
- config: INGEST_ESCALATION_ENABLED (default true), INGEST_TRANSIENT_MAX_ATTEMPTS

INGEST_ESCALATION_ENABLED=false and INGEST_QUEUE=memory preserve prior behaviour.

Deck #323.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

tests/unit/vector/test_parse_pdf_tier.py

@@ -0,0 +1,70 @@
+"""Unit tests for the per-tier PDF parse + escalation gate (Deck #323).
+
+``processor._parse_pdf_tier`` runs one tier and either returns the result to
+index or raises ``EscalateError`` (a queue-hop). These exercise the decision
+without standing up the full ingest pipeline.
+"""
+
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from nextcloud_mcp_server.document_processors.base import ProcessingResult
+from nextcloud_mcp_server.document_processors.escalation import EscalateError
+from nextcloud_mcp_server.vector import processor
+
+pytestmark = pytest.mark.unit
+
+
+def _registry(result: ProcessingResult, decision):
+    reg = MagicMock()
+    reg.process_tier = AsyncMock(return_value=result)
+    reg.evaluate_escalation = MagicMock(return_value=decision)
+    return reg
+
+
+async def test_good_parse_returns_result(monkeypatch):
+    rec = MagicMock()
+    monkeypatch.setattr(processor, "record_document_escalation", rec)
+    result = ProcessingResult(text="clean", metadata={}, processor="fast")
+    reg = _registry(result, decision=None)
+    out = await processor._parse_pdf_tier(
+        reg, b"%PDF", "application/pdf", "f.pdf", "fast", settings=object()
+    )
+    assert out is result
+    rec.assert_not_called()
+
+
+async def test_low_quality_parse_raises_escalate(monkeypatch):
+    rec = MagicMock()
+    monkeypatch.setattr(processor, "record_document_escalation", rec)
+    result = ProcessingResult(text="", metadata={}, processor="fast")
+    reg = _registry(result, decision=("ocr", "empty_text"))
+    with pytest.raises(EscalateError) as ei:
+        await processor._parse_pdf_tier(
+            reg, b"%PDF", "application/pdf", "f.pdf", "fast", settings=object()
+        )
+    assert ei.value.from_tier == "fast"
+    assert ei.value.to_tier == "ocr"
+    assert ei.value.reason == "empty_text"
+    # The escalation is recorded at the decision point.
+    rec.assert_called_once_with("fast", "ocr", "empty_text")
+
+
+async def test_hard_failure_returns_result_without_escalating(monkeypatch):
+    rec = MagicMock()
+    monkeypatch.setattr(processor, "record_document_escalation", rec)
+    result = ProcessingResult(
+        text="",
+        metadata={"parse_failed_reason": "oversize"},
+        processor="size_guard",
+        success=False,
+    )
+    reg = _registry(result, decision=("ocr", "empty_text"))
+    out = await processor._parse_pdf_tier(
+        reg, b"%PDF", "application/pdf", "big.pdf", "fast", settings=object()
+    )
+    # success=False short-circuits: the gate is never consulted, no escalation.
+    assert out is result
+    reg.evaluate_escalation.assert_not_called()
+    rec.assert_not_called()

tests/unit/vector/test_procrastinate_producer.py

@@ -3,6 +3,7 @@
 Uses procrastinate's in-memory connector so no live Postgres is required.
 """
 
+from types import SimpleNamespace
 from typing import cast
 from unittest.mock import AsyncMock
 
@@ -15,6 +16,11 @@ from nextcloud_mcp_server.vector.scanner import DocumentTask
 pytestmark = pytest.mark.unit
 
 
+def _ctx(queue: str = pq.INGEST_QUEUE_FAST) -> JobContext:
+    """Minimal JobContext stand-in: the task only reads ``context.job.queue``."""
+    return cast(JobContext, SimpleNamespace(job=SimpleNamespace(queue=queue)))
+
+
 @pytest.fixture
 def app():
     """An App bound to the in-memory connector with the ingest tasks."""
@@ -94,18 +100,21 @@ class TestProcessDocumentTask:
             captured["user_id"] = user_id
             return fake_client
 
-        async def fake_process(task, nc_client, *, max_retries):
+        async def fake_process(task, nc_client, *, max_retries, tier):
             captured["task"] = task
             captured["nc_client"] = nc_client
             captured["max_retries"] = max_retries
+            captured["tier"] = tier
 
         monkeypatch.setattr(pq, "_resolve_client", fake_resolve)
         monkeypatch.setattr(
             "nextcloud_mcp_server.vector.processor.process_document", fake_process
         )
 
-        # Calling the Task runs its wrapped function in-process.
+        # Calling the Task runs its wrapped function in-process. The job is on the
+        # ocr queue, so the queue-aware task must derive tier="ocr".
         await pq.process_document_task(
+            _ctx(pq.INGEST_QUEUE_OCR),
             user_id="alice",
             doc_id="42",
             doc_type="note",
@@ -120,6 +129,8 @@ class TestProcessDocumentTask:
         assert captured["task"].etag == "e1"
         # Worker disables the in-process retry loop; durable retry is the queue's.
         assert captured["max_retries"] == 1
+        # Tier is derived from the job's queue (escalation enabled by default).
+        assert captured["tier"] == "ocr"
         fake_client.close.assert_awaited_once()
 
     async def test_pipeline_error_propagates_and_closes_client(self, monkeypatch):
@@ -130,7 +141,7 @@ class TestProcessDocumentTask:
         async def fake_resolve(user_id):
             return fake_client
 
-        async def fake_process(task, nc_client, *, max_retries):
+        async def fake_process(task, nc_client, *, max_retries, tier):
             raise RuntimeError("transient qdrant failure")
 
         monkeypatch.setattr(pq, "_resolve_client", fake_resolve)
@@ -140,6 +151,7 @@ class TestProcessDocumentTask:
 
         with pytest.raises(RuntimeError, match="transient qdrant failure"):
             await pq.process_document_task(
+                _ctx(),
                 user_id="alice",
                 doc_id="42",
                 doc_type="note",
@@ -167,6 +179,7 @@ class TestProcessDocumentTask:
 
         # Returns cleanly (job succeeds as a no-op); pipeline never runs.
         await pq.process_document_task(
+            _ctx(),
             user_id="ghost",
             doc_id="9",
             doc_type="note",
@@ -188,7 +201,8 @@ class TestReclaimStalledJobs:
 
         class FakeManager:
             async def get_stalled_jobs(self, queue=None, seconds_since_heartbeat=0):
-                assert queue == pq.INGEST_QUEUE_NAME
+                # Reclaim sweeps EVERY queue (Deck #323), so no queue filter.
+                assert queue is None
                 return [Job(1), Job(2), Job(None)]  # None id is skipped
 
             async def retry_job_by_id_async(self, job_id, retry_at):
@@ -208,27 +222,55 @@ class TestReclaimStalledJobs:
 class TestGetIngestJobCounts:
     async def test_aggregates_stats_rows(self):
         class FakeManager:
-            async def list_queues_async(self, queue=None):
-                assert queue == pq.INGEST_QUEUE_NAME
+            async def list_queues_async(self, queue=None, **kwargs):
+                # Counts now aggregate across all managed queues (Deck #323), so
+                # the helper lists every queue and filters by name itself.
+                assert queue is None
                 # procrastinate flattens per-status stats into top-level keys.
                 return [
                     {
-                        "name": "ingest",
-                        "jobs_count": 6,
+                        "name": "ingest-fast",
+                        "jobs_count": 4,
                         "todo": 3,
                         "doing": 1,
                         "succeeded": 0,
+                        "failed": 0,
+                        "cancelled": 0,
+                        "aborted": 0,
+                    },
+                    {
+                        "name": "ingest-ocr",
+                        "jobs_count": 2,
+                        "todo": 0,
+                        "doing": 0,
+                        "succeeded": 0,
                         "failed": 2,
                         "cancelled": 0,
                         "aborted": 0,
-                    }
+                    },
+                    {
+                        # An unmanaged queue must NOT pollute ingest counts.
+                        "name": "some-other-queue",
+                        "jobs_count": 9,
+                        "todo": 9,
+                        "doing": 0,
+                        "succeeded": 0,
+                        "failed": 0,
+                        "cancelled": 0,
+                        "aborted": 0,
+                    },
                 ]
 
         class FakeApp:
             job_manager = FakeManager()
 
         counts = await pq.get_ingest_job_counts(cast(App, FakeApp()))
-        assert counts["todo"] == 3
+        assert counts["todo"] == 3  # only ingest-* queues, not some-other-queue
         assert counts["doing"] == 1
         assert counts["failed"] == 2
         assert counts["succeeded"] == 0
+
+        by_queue = await pq.get_ingest_job_counts_by_queue(cast(App, FakeApp()))
+        assert set(by_queue) == {"ingest-fast", "ingest-ocr"}
+        assert by_queue["ingest-fast"]["todo"] == 3
+        assert by_queue["ingest-ocr"]["failed"] == 2

tests/unit/vector/test_tiered_escalation_strategy.py

@@ -0,0 +1,100 @@
+"""Unit tests for the per-tier escalation primitives (Deck #323).
+
+Covers the tier-ladder helpers + EscalateError (document_processors.escalation)
+and the procrastinate TieredEscalationStrategy that turns a raised exception
+into a queue-hop / same-tier retry / give-up decision.
+"""
+
+import httpx
+import pytest
+from procrastinate.jobs import Job
+
+import nextcloud_mcp_server.vector.queue.procrastinate as pq
+from nextcloud_mcp_server.document_processors.escalation import (
+    TIER_LADDER,
+    EscalateError,
+    next_tier,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def _job(queue: str = pq.INGEST_QUEUE_FAST, attempts: int = 1) -> Job:
+    return Job(
+        id=1,
+        queue=queue,
+        task_name=pq.INGEST_TASK_NAME,
+        lock=None,
+        queueing_lock=None,
+        attempts=attempts,
+    )
+
+
+class TestLadder:
+    def test_next_tier_ordering(self):
+        assert next_tier("fast") == "structured"
+        assert next_tier("structured") == "ocr"
+        assert next_tier("ocr") is None  # terminal
+        assert next_tier("unknown") is None
+
+    def test_ladder_is_cheapest_first(self):
+        assert TIER_LADDER == ("fast", "structured", "ocr")
+
+    def test_tier_for_queue(self):
+        assert pq.tier_for_queue(pq.INGEST_QUEUE_OCR) == "ocr"
+        assert pq.tier_for_queue(pq.INGEST_QUEUE_STRUCTURED) == "structured"
+        # Legacy / unknown / None all fall back to the cheapest tier.
+        assert pq.tier_for_queue(pq.LEGACY_INGEST_QUEUE) == "fast"
+        assert pq.tier_for_queue(None) == "fast"
+
+
+class TestTieredEscalationStrategy:
+    def _strategy(self, max_transient: int = 5):
+        return pq.TieredEscalationStrategy(max_transient_attempts=max_transient)
+
+    def test_escalate_hops_to_target_queue(self):
+        exc = EscalateError(from_tier="fast", to_tier="ocr", reason="empty_text")
+        decision = self._strategy().get_retry_decision(exception=exc, job=_job())
+        assert decision is not None
+        assert decision.queue == pq.INGEST_QUEUE_OCR
+
+    def test_escalate_to_structured(self):
+        exc = EscalateError(
+            from_tier="fast", to_tier="structured", reason="low_confidence"
+        )
+        decision = self._strategy().get_retry_decision(exception=exc, job=_job())
+        assert decision is not None
+        assert decision.queue == pq.INGEST_QUEUE_STRUCTURED
+
+    def test_escalate_unknown_tier_gives_up(self):
+        exc = EscalateError(from_tier="ocr", to_tier="bogus", reason="low_confidence")
+        decision = self._strategy().get_retry_decision(exception=exc, job=_job())
+        assert decision is None
+
+    def test_escalate_unwraps_exception_group(self):
+        exc = EscalateError(from_tier="fast", to_tier="ocr", reason="empty_text")
+        group = ExceptionGroup("wrapped", [exc])
+        decision = self._strategy().get_retry_decision(exception=group, job=_job())
+        assert decision is not None
+        assert decision.queue == pq.INGEST_QUEUE_OCR
+
+    def test_transient_retries_same_queue_under_cap(self):
+        decision = self._strategy(max_transient=5).get_retry_decision(
+            exception=httpx.ConnectError("refused"), job=_job(attempts=1)
+        )
+        assert decision is not None
+        # Same-tier retry: no queue override (stays on its current queue).
+        assert decision.queue is None
+        assert decision.retry_at is not None
+
+    def test_transient_gives_up_over_cap(self):
+        decision = self._strategy(max_transient=5).get_retry_decision(
+            exception=httpx.ConnectError("refused"), job=_job(attempts=5)
+        )
+        assert decision is None
+
+    def test_non_transient_error_gives_up(self):
+        decision = self._strategy().get_retry_decision(
+            exception=ValueError("permanent"), job=_job(attempts=1)
+        )
+        assert decision is None