feat(ingest): per-tier escalation via procrastinate queue-hop

Split external (procrastinate) document processing into per-tier queues so a
document is attempted at most once per tier and requeued to the next tier's
queue on a low-quality parse, using procrastinate's native retry.

- escalation.py: TIER_LADDER (fast->structured->ocr) + EscalateError signal
- registry: process_tier (one tier) + evaluate_escalation post-parse gate
  (reuses classify_from_text) + next_available_tier; shared _classify_result
  and _oversize_result with the inline pipeline
- processor: process_document(tier=...) runs one tier and raises EscalateError
  before embed (junk text never indexed); inline memory path unchanged
- queue/procrastinate: ingest-fast|structured|ocr queues; TieredEscalationStrategy
  (queue-hop on EscalateError, bounded same-tier transient retry); queue-aware
  task; producer defers to ingest-fast; per-queue counts + all-queue reclaim
- cli: worker --tier {fast,structured,ocr}
- billing: pages_ocr usage event + pipeline_tier metadata (paid OCR billed apart)
- observability: astrolabe_ingest_queue_depth{queue,status} gauge + per-queue
  counts in nc_get_vector_sync_status / management status endpoint
- config: INGEST_ESCALATION_ENABLED (default true), INGEST_TRANSIENT_MAX_ATTEMPTS

INGEST_ESCALATION_ENABLED=false and INGEST_QUEUE=memory preserve prior behaviour.

Deck #323.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

tests/unit/vector/test_procrastinate_producer.py

@@ -3,6 +3,7 @@
 Uses procrastinate's in-memory connector so no live Postgres is required.
 """
 
+from types import SimpleNamespace
 from typing import cast
 from unittest.mock import AsyncMock
 
@@ -15,6 +16,11 @@ from nextcloud_mcp_server.vector.scanner import DocumentTask
 pytestmark = pytest.mark.unit
 
 
+def _ctx(queue: str = pq.INGEST_QUEUE_FAST) -> JobContext:
+    """Minimal JobContext stand-in: the task only reads ``context.job.queue``."""
+    return cast(JobContext, SimpleNamespace(job=SimpleNamespace(queue=queue)))
+
+
 @pytest.fixture
 def app():
     """An App bound to the in-memory connector with the ingest tasks."""
@@ -94,18 +100,21 @@ class TestProcessDocumentTask:
             captured["user_id"] = user_id
             return fake_client
 
-        async def fake_process(task, nc_client, *, max_retries):
+        async def fake_process(task, nc_client, *, max_retries, tier):
             captured["task"] = task
             captured["nc_client"] = nc_client
             captured["max_retries"] = max_retries
+            captured["tier"] = tier
 
         monkeypatch.setattr(pq, "_resolve_client", fake_resolve)
         monkeypatch.setattr(
             "nextcloud_mcp_server.vector.processor.process_document", fake_process
         )
 
-        # Calling the Task runs its wrapped function in-process.
+        # Calling the Task runs its wrapped function in-process. The job is on the
+        # ocr queue, so the queue-aware task must derive tier="ocr".
         await pq.process_document_task(
+            _ctx(pq.INGEST_QUEUE_OCR),
             user_id="alice",
             doc_id="42",
             doc_type="note",
@@ -120,6 +129,8 @@ class TestProcessDocumentTask:
         assert captured["task"].etag == "e1"
         # Worker disables the in-process retry loop; durable retry is the queue's.
         assert captured["max_retries"] == 1
+        # Tier is derived from the job's queue (escalation enabled by default).
+        assert captured["tier"] == "ocr"
         fake_client.close.assert_awaited_once()
 
     async def test_pipeline_error_propagates_and_closes_client(self, monkeypatch):
@@ -130,7 +141,7 @@ class TestProcessDocumentTask:
         async def fake_resolve(user_id):
             return fake_client
 
-        async def fake_process(task, nc_client, *, max_retries):
+        async def fake_process(task, nc_client, *, max_retries, tier):
             raise RuntimeError("transient qdrant failure")
 
         monkeypatch.setattr(pq, "_resolve_client", fake_resolve)
@@ -140,6 +151,7 @@ class TestProcessDocumentTask:
 
         with pytest.raises(RuntimeError, match="transient qdrant failure"):
             await pq.process_document_task(
+                _ctx(),
                 user_id="alice",
                 doc_id="42",
                 doc_type="note",
@@ -167,6 +179,7 @@ class TestProcessDocumentTask:
 
         # Returns cleanly (job succeeds as a no-op); pipeline never runs.
         await pq.process_document_task(
+            _ctx(),
             user_id="ghost",
             doc_id="9",
             doc_type="note",
@@ -188,7 +201,8 @@ class TestReclaimStalledJobs:
 
         class FakeManager:
             async def get_stalled_jobs(self, queue=None, seconds_since_heartbeat=0):
-                assert queue == pq.INGEST_QUEUE_NAME
+                # Reclaim sweeps EVERY queue (Deck #323), so no queue filter.
+                assert queue is None
                 return [Job(1), Job(2), Job(None)]  # None id is skipped
 
             async def retry_job_by_id_async(self, job_id, retry_at):
@@ -208,27 +222,55 @@ class TestReclaimStalledJobs:
 class TestGetIngestJobCounts:
     async def test_aggregates_stats_rows(self):
         class FakeManager:
-            async def list_queues_async(self, queue=None):
-                assert queue == pq.INGEST_QUEUE_NAME
+            async def list_queues_async(self, queue=None, **kwargs):
+                # Counts now aggregate across all managed queues (Deck #323), so
+                # the helper lists every queue and filters by name itself.
+                assert queue is None
                 # procrastinate flattens per-status stats into top-level keys.
                 return [
                     {
-                        "name": "ingest",
-                        "jobs_count": 6,
+                        "name": "ingest-fast",
+                        "jobs_count": 4,
                         "todo": 3,
                         "doing": 1,
                         "succeeded": 0,
+                        "failed": 0,
+                        "cancelled": 0,
+                        "aborted": 0,
+                    },
+                    {
+                        "name": "ingest-ocr",
+                        "jobs_count": 2,
+                        "todo": 0,
+                        "doing": 0,
+                        "succeeded": 0,
                         "failed": 2,
                         "cancelled": 0,
                         "aborted": 0,
-                    }
+                    },
+                    {
+                        # An unmanaged queue must NOT pollute ingest counts.
+                        "name": "some-other-queue",
+                        "jobs_count": 9,
+                        "todo": 9,
+                        "doing": 0,
+                        "succeeded": 0,
+                        "failed": 0,
+                        "cancelled": 0,
+                        "aborted": 0,
+                    },
                 ]
 
         class FakeApp:
             job_manager = FakeManager()
 
         counts = await pq.get_ingest_job_counts(cast(App, FakeApp()))
-        assert counts["todo"] == 3
+        assert counts["todo"] == 3  # only ingest-* queues, not some-other-queue
         assert counts["doing"] == 1
         assert counts["failed"] == 2
         assert counts["succeeded"] == 0
+
+        by_queue = await pq.get_ingest_job_counts_by_queue(cast(App, FakeApp()))
+        assert set(by_queue) == {"ingest-fast", "ingest-ocr"}
+        assert by_queue["ingest-fast"]["todo"] == 3
+        assert by_queue["ingest-ocr"]["failed"] == 2