Chris CoutinhoandClaude Opus 4.8
9676bb3106
Split external (procrastinate) document processing into per-tier queues so a
document is attempted at most once per tier and requeued to the next tier's
queue on a low-quality parse, using procrastinate's native retry.
- escalation.py: TIER_LADDER (fast->structured->ocr) + EscalateError signal
- registry: process_tier (one tier) + evaluate_escalation post-parse gate
(reuses classify_from_text) + next_available_tier; shared _classify_result
and _oversize_result with the inline pipeline
- processor: process_document(tier=...) runs one tier and raises EscalateError
before embed (junk text never indexed); inline memory path unchanged
- queue/procrastinate: ingest-fast|structured|ocr queues; TieredEscalationStrategy
(queue-hop on EscalateError, bounded same-tier transient retry); queue-aware
task; producer defers to ingest-fast; per-queue counts + all-queue reclaim
- cli: worker --tier {fast,structured,ocr}
- billing: pages_ocr usage event + pipeline_tier metadata (paid OCR billed apart)
- observability: astrolabe_ingest_queue_depth{queue,status} gauge + per-queue
counts in nc_get_vector_sync_status / management status endpoint
- config: INGEST_ESCALATION_ENABLED (default true), INGEST_TRANSIENT_MAX_ATTEMPTS
INGEST_ESCALATION_ENABLED=false and INGEST_QUEUE=memory preserve prior behaviour.
Deck #323.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
101 lines
3.7 KiB
Python
101 lines
3.7 KiB
Python
"""Unit tests for the per-tier escalation primitives (Deck #323).
|
|
|
|
Covers the tier-ladder helpers + EscalateError (document_processors.escalation)
|
|
and the procrastinate TieredEscalationStrategy that turns a raised exception
|
|
into a queue-hop / same-tier retry / give-up decision.
|
|
"""
|
|
|
|
import httpx
|
|
import pytest
|
|
from procrastinate.jobs import Job
|
|
|
|
import nextcloud_mcp_server.vector.queue.procrastinate as pq
|
|
from nextcloud_mcp_server.document_processors.escalation import (
|
|
TIER_LADDER,
|
|
EscalateError,
|
|
next_tier,
|
|
)
|
|
|
|
pytestmark = pytest.mark.unit
|
|
|
|
|
|
def _job(queue: str = pq.INGEST_QUEUE_FAST, attempts: int = 1) -> Job:
|
|
return Job(
|
|
id=1,
|
|
queue=queue,
|
|
task_name=pq.INGEST_TASK_NAME,
|
|
lock=None,
|
|
queueing_lock=None,
|
|
attempts=attempts,
|
|
)
|
|
|
|
|
|
class TestLadder:
|
|
def test_next_tier_ordering(self):
|
|
assert next_tier("fast") == "structured"
|
|
assert next_tier("structured") == "ocr"
|
|
assert next_tier("ocr") is None # terminal
|
|
assert next_tier("unknown") is None
|
|
|
|
def test_ladder_is_cheapest_first(self):
|
|
assert TIER_LADDER == ("fast", "structured", "ocr")
|
|
|
|
def test_tier_for_queue(self):
|
|
assert pq.tier_for_queue(pq.INGEST_QUEUE_OCR) == "ocr"
|
|
assert pq.tier_for_queue(pq.INGEST_QUEUE_STRUCTURED) == "structured"
|
|
# Legacy / unknown / None all fall back to the cheapest tier.
|
|
assert pq.tier_for_queue(pq.LEGACY_INGEST_QUEUE) == "fast"
|
|
assert pq.tier_for_queue(None) == "fast"
|
|
|
|
|
|
class TestTieredEscalationStrategy:
|
|
def _strategy(self, max_transient: int = 5):
|
|
return pq.TieredEscalationStrategy(max_transient_attempts=max_transient)
|
|
|
|
def test_escalate_hops_to_target_queue(self):
|
|
exc = EscalateError(from_tier="fast", to_tier="ocr", reason="empty_text")
|
|
decision = self._strategy().get_retry_decision(exception=exc, job=_job())
|
|
assert decision is not None
|
|
assert decision.queue == pq.INGEST_QUEUE_OCR
|
|
|
|
def test_escalate_to_structured(self):
|
|
exc = EscalateError(
|
|
from_tier="fast", to_tier="structured", reason="low_confidence"
|
|
)
|
|
decision = self._strategy().get_retry_decision(exception=exc, job=_job())
|
|
assert decision is not None
|
|
assert decision.queue == pq.INGEST_QUEUE_STRUCTURED
|
|
|
|
def test_escalate_unknown_tier_gives_up(self):
|
|
exc = EscalateError(from_tier="ocr", to_tier="bogus", reason="low_confidence")
|
|
decision = self._strategy().get_retry_decision(exception=exc, job=_job())
|
|
assert decision is None
|
|
|
|
def test_escalate_unwraps_exception_group(self):
|
|
exc = EscalateError(from_tier="fast", to_tier="ocr", reason="empty_text")
|
|
group = ExceptionGroup("wrapped", [exc])
|
|
decision = self._strategy().get_retry_decision(exception=group, job=_job())
|
|
assert decision is not None
|
|
assert decision.queue == pq.INGEST_QUEUE_OCR
|
|
|
|
def test_transient_retries_same_queue_under_cap(self):
|
|
decision = self._strategy(max_transient=5).get_retry_decision(
|
|
exception=httpx.ConnectError("refused"), job=_job(attempts=1)
|
|
)
|
|
assert decision is not None
|
|
# Same-tier retry: no queue override (stays on its current queue).
|
|
assert decision.queue is None
|
|
assert decision.retry_at is not None
|
|
|
|
def test_transient_gives_up_over_cap(self):
|
|
decision = self._strategy(max_transient=5).get_retry_decision(
|
|
exception=httpx.ConnectError("refused"), job=_job(attempts=5)
|
|
)
|
|
assert decision is None
|
|
|
|
def test_non_transient_error_gives_up(self):
|
|
decision = self._strategy().get_retry_decision(
|
|
exception=ValueError("permanent"), job=_job(attempts=1)
|
|
)
|
|
assert decision is None
|