feat(ingest): per-tier escalation via procrastinate queue-hop
Split external (procrastinate) document processing into per-tier queues so a
document is attempted at most once per tier and requeued to the next tier's
queue on a low-quality parse, using procrastinate's native retry.
- escalation.py: TIER_LADDER (fast->structured->ocr) + EscalateError signal
- registry: process_tier (one tier) + evaluate_escalation post-parse gate
(reuses classify_from_text) + next_available_tier; shared _classify_result
and _oversize_result with the inline pipeline
- processor: process_document(tier=...) runs one tier and raises EscalateError
before embed (junk text never indexed); inline memory path unchanged
- queue/procrastinate: ingest-fast|structured|ocr queues; TieredEscalationStrategy
(queue-hop on EscalateError, bounded same-tier transient retry); queue-aware
task; producer defers to ingest-fast; per-queue counts + all-queue reclaim
- cli: worker --tier {fast,structured,ocr}
- billing: pages_ocr usage event + pipeline_tier metadata (paid OCR billed apart)
- observability: astrolabe_ingest_queue_depth{queue,status} gauge + per-queue
counts in nc_get_vector_sync_status / management status endpoint
- config: INGEST_ESCALATION_ENABLED (default true), INGEST_TRANSIENT_MAX_ATTEMPTS
INGEST_ESCALATION_ENABLED=false and INGEST_QUEUE=memory preserve prior behaviour.
Deck #323.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
6fab0e2ae3
commit
9676bb3106
@@ -176,3 +176,53 @@ async def test_store_failure_is_swallowed(monkeypatch):
|
||||
total_chars=9,
|
||||
page_count=2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_ocr_tier_records_pages_ocr(store_spy):
|
||||
"""OCR-tier pages are metered as a separate pages_ocr line (Deck #323)."""
|
||||
await processor.record_indexing_usage(
|
||||
enabled=True,
|
||||
provider="mistral",
|
||||
model="mistral-embed",
|
||||
doc_type="file",
|
||||
user_id="alice",
|
||||
chunk_count=20,
|
||||
token_count=900,
|
||||
total_chars=40000,
|
||||
page_count=8,
|
||||
pipeline_tier="ocr",
|
||||
)
|
||||
by_metric = {
|
||||
c.kwargs["metric"]: c.kwargs["value"]
|
||||
for c in store_spy.record_usage_event.await_args_list
|
||||
}
|
||||
# pages_ocr fires IN ADDITION to pages_embedded for OCR-tier pages.
|
||||
assert by_metric == {
|
||||
"tokens_embedded": 900,
|
||||
"pages_embedded": 8,
|
||||
"pages_ocr": 8,
|
||||
}
|
||||
# pipeline_tier is threaded into the billing metadata for CP attribution.
|
||||
for c in store_spy.record_usage_event.await_args_list:
|
||||
assert c.kwargs["metadata"]["pipeline_tier"] == "ocr"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_fast_tier_does_not_record_pages_ocr(store_spy):
|
||||
"""A CPU-cheap fast-tier parse must NOT incur the paid pages_ocr line."""
|
||||
await processor.record_indexing_usage(
|
||||
enabled=True,
|
||||
provider="mistral",
|
||||
model="mistral-embed",
|
||||
doc_type="file",
|
||||
user_id="alice",
|
||||
chunk_count=10,
|
||||
token_count=500,
|
||||
total_chars=20000,
|
||||
page_count=4,
|
||||
pipeline_tier="fast",
|
||||
)
|
||||
metrics = {c.kwargs["metric"] for c in store_spy.record_usage_event.await_args_list}
|
||||
assert "pages_ocr" not in metrics
|
||||
assert metrics == {"tokens_embedded", "pages_embedded"}
|
||||
|
||||
@@ -241,3 +241,150 @@ async def test_no_ocr_escalation_when_disabled(monkeypatch):
|
||||
res = await r.process(b"%PDF-1.7", "application/pdf")
|
||||
# Fast tier is terminal when OCR is disabled.
|
||||
assert res.processor == "fast"
|
||||
|
||||
|
||||
# --- Per-tier external path (Deck #323) -------------------------------------
|
||||
|
||||
|
||||
async def test_process_tier_runs_named_tier(monkeypatch):
|
||||
"""process_tier runs exactly the requested tier's processor, not priority."""
|
||||
monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings())
|
||||
r = _registry(
|
||||
(_Fake("fast", "fast"), 20),
|
||||
(_Fake("structured", "structured"), 10),
|
||||
(_Fake("ocr", "ocr"), 5),
|
||||
)
|
||||
res = await r.process_tier(b"%PDF-1.7", "application/pdf", "f.pdf", "structured")
|
||||
assert res.processor == "structured"
|
||||
|
||||
|
||||
async def test_process_tier_unknown_tier_raises(monkeypatch):
|
||||
from nextcloud_mcp_server.document_processors.base import ProcessorError
|
||||
|
||||
monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings())
|
||||
r = _registry((_Fake("fast", "fast"), 20))
|
||||
with pytest.raises(ProcessorError, match="structured"):
|
||||
await r.process_tier(b"%PDF-1.7", "application/pdf", "f.pdf", "structured")
|
||||
|
||||
|
||||
async def test_process_tier_oversize_fails_fast(monkeypatch):
|
||||
"""The size guard applies on the per-tier path too (before any parse)."""
|
||||
monkeypatch.setattr(
|
||||
reg_mod, "get_settings", lambda: _Settings(max_pdf_size_mb=0.001)
|
||||
)
|
||||
r = _registry((_Fake("ocr", "ocr"), 5))
|
||||
res = await r.process_tier(b"x" * 4096, "application/pdf", "big.pdf", "ocr")
|
||||
assert res.success is False
|
||||
assert res.metadata["parse_failed_reason"] == "oversize"
|
||||
|
||||
|
||||
def test_next_available_tier_walks_ladder():
|
||||
r = _registry(
|
||||
(_Fake("fast", "fast"), 20),
|
||||
(_Fake("structured", "structured"), 10),
|
||||
(_Fake("ocr", "ocr"), 5),
|
||||
)
|
||||
# ocr disabled -> structured is the only target above fast.
|
||||
s = _Settings(ocr=False)
|
||||
assert r.next_available_tier("fast", s) == "structured"
|
||||
assert r.next_available_tier("structured", s) is None # ocr gated off
|
||||
# ocr enabled -> reachable; minimum skips the structured rung.
|
||||
s_ocr = _Settings(ocr=True)
|
||||
assert r.next_available_tier("structured", s_ocr) == "ocr"
|
||||
assert r.next_available_tier("fast", s_ocr, minimum="ocr") == "ocr"
|
||||
|
||||
|
||||
def test_next_available_tier_skips_unregistered():
|
||||
# No structured processor -> fast escalates straight to ocr.
|
||||
r = _registry((_Fake("fast", "fast"), 20), (_Fake("ocr", "ocr"), 5))
|
||||
assert r.next_available_tier("fast", _Settings(ocr=True)) == "ocr"
|
||||
|
||||
|
||||
def test_evaluate_escalation_good_text_indexes(monkeypatch):
|
||||
monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
|
||||
r = _registry(
|
||||
(_Fake("fast", "fast", text="This is clean readable prose text."), 20),
|
||||
(_Fake("ocr", "ocr"), 5),
|
||||
)
|
||||
res = ProcessingResult(
|
||||
text="This is clean readable prose text.",
|
||||
metadata={
|
||||
"page_count": 1,
|
||||
"page_boundaries": [{"page": 1, "start_offset": 0, "end_offset": 34}],
|
||||
},
|
||||
processor="fast",
|
||||
)
|
||||
assert r.evaluate_escalation(res, b"%PDF", "fast", _Settings(ocr=True)) is None
|
||||
|
||||
|
||||
def test_evaluate_escalation_empty_jumps_to_ocr(monkeypatch):
|
||||
"""A scanned (no-text-layer) result targets ocr directly, skipping structured."""
|
||||
monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
|
||||
r = _registry(
|
||||
(_Fake("fast", "fast"), 20),
|
||||
(_Fake("structured", "structured"), 10),
|
||||
(_Fake("ocr", "ocr"), 5),
|
||||
)
|
||||
res = ProcessingResult(
|
||||
text="",
|
||||
metadata={
|
||||
"page_count": 1,
|
||||
"page_boundaries": [{"page": 1, "start_offset": 0, "end_offset": 0}],
|
||||
},
|
||||
processor="fast",
|
||||
)
|
||||
decision = r.evaluate_escalation(res, b"%PDF", "fast", _Settings(ocr=True))
|
||||
assert decision == ("ocr", "empty_text")
|
||||
|
||||
|
||||
def test_evaluate_escalation_lowconf_goes_to_structured(monkeypatch):
|
||||
"""A junk-but-non-empty layer escalates to the next rung (structured)."""
|
||||
monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
|
||||
junk = "x" * 40 # one long token, no whitespace -> quality ~0
|
||||
r = _registry(
|
||||
(_Fake("fast", "fast"), 20),
|
||||
(_Fake("structured", "structured"), 10),
|
||||
(_Fake("ocr", "ocr"), 5),
|
||||
)
|
||||
res = ProcessingResult(
|
||||
text=junk,
|
||||
metadata={
|
||||
"page_count": 1,
|
||||
"page_boundaries": [
|
||||
{"page": 1, "start_offset": 0, "end_offset": len(junk)}
|
||||
],
|
||||
},
|
||||
processor="fast",
|
||||
)
|
||||
decision = r.evaluate_escalation(res, b"%PDF", "fast", _Settings(ocr=True))
|
||||
assert decision == ("structured", "low_confidence")
|
||||
|
||||
|
||||
def test_evaluate_escalation_failure_not_escalated(monkeypatch):
|
||||
monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
|
||||
r = _registry((_Fake("fast", "fast"), 20), (_Fake("ocr", "ocr"), 5))
|
||||
res = ProcessingResult(
|
||||
text="",
|
||||
metadata={"parse_failed_reason": "error"},
|
||||
processor="fast",
|
||||
success=False,
|
||||
)
|
||||
assert r.evaluate_escalation(res, b"%PDF", "fast", _Settings(ocr=True)) is None
|
||||
|
||||
|
||||
def test_evaluate_escalation_terminal_when_no_higher_tier(monkeypatch):
|
||||
monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
|
||||
# Only fast registered -> nowhere to escalate even on junk text.
|
||||
r = _registry((_Fake("fast", "fast"), 20))
|
||||
junk = "y" * 40
|
||||
res = ProcessingResult(
|
||||
text=junk,
|
||||
metadata={
|
||||
"page_count": 1,
|
||||
"page_boundaries": [
|
||||
{"page": 1, "start_offset": 0, "end_offset": len(junk)}
|
||||
],
|
||||
},
|
||||
processor="fast",
|
||||
)
|
||||
assert r.evaluate_escalation(res, b"%PDF", "fast", _Settings(ocr=True)) is None
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Unit tests for the per-tier PDF parse + escalation gate (Deck #323).
|
||||
|
||||
``processor._parse_pdf_tier`` runs one tier and either returns the result to
|
||||
index or raises ``EscalateError`` (a queue-hop). These exercise the decision
|
||||
without standing up the full ingest pipeline.
|
||||
"""
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from nextcloud_mcp_server.document_processors.base import ProcessingResult
|
||||
from nextcloud_mcp_server.document_processors.escalation import EscalateError
|
||||
from nextcloud_mcp_server.vector import processor
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def _registry(result: ProcessingResult, decision):
|
||||
reg = MagicMock()
|
||||
reg.process_tier = AsyncMock(return_value=result)
|
||||
reg.evaluate_escalation = MagicMock(return_value=decision)
|
||||
return reg
|
||||
|
||||
|
||||
async def test_good_parse_returns_result(monkeypatch):
|
||||
rec = MagicMock()
|
||||
monkeypatch.setattr(processor, "record_document_escalation", rec)
|
||||
result = ProcessingResult(text="clean", metadata={}, processor="fast")
|
||||
reg = _registry(result, decision=None)
|
||||
out = await processor._parse_pdf_tier(
|
||||
reg, b"%PDF", "application/pdf", "f.pdf", "fast", settings=object()
|
||||
)
|
||||
assert out is result
|
||||
rec.assert_not_called()
|
||||
|
||||
|
||||
async def test_low_quality_parse_raises_escalate(monkeypatch):
|
||||
rec = MagicMock()
|
||||
monkeypatch.setattr(processor, "record_document_escalation", rec)
|
||||
result = ProcessingResult(text="", metadata={}, processor="fast")
|
||||
reg = _registry(result, decision=("ocr", "empty_text"))
|
||||
with pytest.raises(EscalateError) as ei:
|
||||
await processor._parse_pdf_tier(
|
||||
reg, b"%PDF", "application/pdf", "f.pdf", "fast", settings=object()
|
||||
)
|
||||
assert ei.value.from_tier == "fast"
|
||||
assert ei.value.to_tier == "ocr"
|
||||
assert ei.value.reason == "empty_text"
|
||||
# The escalation is recorded at the decision point.
|
||||
rec.assert_called_once_with("fast", "ocr", "empty_text")
|
||||
|
||||
|
||||
async def test_hard_failure_returns_result_without_escalating(monkeypatch):
|
||||
rec = MagicMock()
|
||||
monkeypatch.setattr(processor, "record_document_escalation", rec)
|
||||
result = ProcessingResult(
|
||||
text="",
|
||||
metadata={"parse_failed_reason": "oversize"},
|
||||
processor="size_guard",
|
||||
success=False,
|
||||
)
|
||||
reg = _registry(result, decision=("ocr", "empty_text"))
|
||||
out = await processor._parse_pdf_tier(
|
||||
reg, b"%PDF", "application/pdf", "big.pdf", "fast", settings=object()
|
||||
)
|
||||
# success=False short-circuits: the gate is never consulted, no escalation.
|
||||
assert out is result
|
||||
reg.evaluate_escalation.assert_not_called()
|
||||
rec.assert_not_called()
|
||||
@@ -3,6 +3,7 @@
|
||||
Uses procrastinate's in-memory connector so no live Postgres is required.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import cast
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
@@ -15,6 +16,11 @@ from nextcloud_mcp_server.vector.scanner import DocumentTask
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def _ctx(queue: str = pq.INGEST_QUEUE_FAST) -> JobContext:
|
||||
"""Minimal JobContext stand-in: the task only reads ``context.job.queue``."""
|
||||
return cast(JobContext, SimpleNamespace(job=SimpleNamespace(queue=queue)))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app():
|
||||
"""An App bound to the in-memory connector with the ingest tasks."""
|
||||
@@ -94,18 +100,21 @@ class TestProcessDocumentTask:
|
||||
captured["user_id"] = user_id
|
||||
return fake_client
|
||||
|
||||
async def fake_process(task, nc_client, *, max_retries):
|
||||
async def fake_process(task, nc_client, *, max_retries, tier):
|
||||
captured["task"] = task
|
||||
captured["nc_client"] = nc_client
|
||||
captured["max_retries"] = max_retries
|
||||
captured["tier"] = tier
|
||||
|
||||
monkeypatch.setattr(pq, "_resolve_client", fake_resolve)
|
||||
monkeypatch.setattr(
|
||||
"nextcloud_mcp_server.vector.processor.process_document", fake_process
|
||||
)
|
||||
|
||||
# Calling the Task runs its wrapped function in-process.
|
||||
# Calling the Task runs its wrapped function in-process. The job is on the
|
||||
# ocr queue, so the queue-aware task must derive tier="ocr".
|
||||
await pq.process_document_task(
|
||||
_ctx(pq.INGEST_QUEUE_OCR),
|
||||
user_id="alice",
|
||||
doc_id="42",
|
||||
doc_type="note",
|
||||
@@ -120,6 +129,8 @@ class TestProcessDocumentTask:
|
||||
assert captured["task"].etag == "e1"
|
||||
# Worker disables the in-process retry loop; durable retry is the queue's.
|
||||
assert captured["max_retries"] == 1
|
||||
# Tier is derived from the job's queue (escalation enabled by default).
|
||||
assert captured["tier"] == "ocr"
|
||||
fake_client.close.assert_awaited_once()
|
||||
|
||||
async def test_pipeline_error_propagates_and_closes_client(self, monkeypatch):
|
||||
@@ -130,7 +141,7 @@ class TestProcessDocumentTask:
|
||||
async def fake_resolve(user_id):
|
||||
return fake_client
|
||||
|
||||
async def fake_process(task, nc_client, *, max_retries):
|
||||
async def fake_process(task, nc_client, *, max_retries, tier):
|
||||
raise RuntimeError("transient qdrant failure")
|
||||
|
||||
monkeypatch.setattr(pq, "_resolve_client", fake_resolve)
|
||||
@@ -140,6 +151,7 @@ class TestProcessDocumentTask:
|
||||
|
||||
with pytest.raises(RuntimeError, match="transient qdrant failure"):
|
||||
await pq.process_document_task(
|
||||
_ctx(),
|
||||
user_id="alice",
|
||||
doc_id="42",
|
||||
doc_type="note",
|
||||
@@ -167,6 +179,7 @@ class TestProcessDocumentTask:
|
||||
|
||||
# Returns cleanly (job succeeds as a no-op); pipeline never runs.
|
||||
await pq.process_document_task(
|
||||
_ctx(),
|
||||
user_id="ghost",
|
||||
doc_id="9",
|
||||
doc_type="note",
|
||||
@@ -188,7 +201,8 @@ class TestReclaimStalledJobs:
|
||||
|
||||
class FakeManager:
|
||||
async def get_stalled_jobs(self, queue=None, seconds_since_heartbeat=0):
|
||||
assert queue == pq.INGEST_QUEUE_NAME
|
||||
# Reclaim sweeps EVERY queue (Deck #323), so no queue filter.
|
||||
assert queue is None
|
||||
return [Job(1), Job(2), Job(None)] # None id is skipped
|
||||
|
||||
async def retry_job_by_id_async(self, job_id, retry_at):
|
||||
@@ -208,27 +222,55 @@ class TestReclaimStalledJobs:
|
||||
class TestGetIngestJobCounts:
|
||||
async def test_aggregates_stats_rows(self):
|
||||
class FakeManager:
|
||||
async def list_queues_async(self, queue=None):
|
||||
assert queue == pq.INGEST_QUEUE_NAME
|
||||
async def list_queues_async(self, queue=None, **kwargs):
|
||||
# Counts now aggregate across all managed queues (Deck #323), so
|
||||
# the helper lists every queue and filters by name itself.
|
||||
assert queue is None
|
||||
# procrastinate flattens per-status stats into top-level keys.
|
||||
return [
|
||||
{
|
||||
"name": "ingest",
|
||||
"jobs_count": 6,
|
||||
"name": "ingest-fast",
|
||||
"jobs_count": 4,
|
||||
"todo": 3,
|
||||
"doing": 1,
|
||||
"succeeded": 0,
|
||||
"failed": 0,
|
||||
"cancelled": 0,
|
||||
"aborted": 0,
|
||||
},
|
||||
{
|
||||
"name": "ingest-ocr",
|
||||
"jobs_count": 2,
|
||||
"todo": 0,
|
||||
"doing": 0,
|
||||
"succeeded": 0,
|
||||
"failed": 2,
|
||||
"cancelled": 0,
|
||||
"aborted": 0,
|
||||
}
|
||||
},
|
||||
{
|
||||
# An unmanaged queue must NOT pollute ingest counts.
|
||||
"name": "some-other-queue",
|
||||
"jobs_count": 9,
|
||||
"todo": 9,
|
||||
"doing": 0,
|
||||
"succeeded": 0,
|
||||
"failed": 0,
|
||||
"cancelled": 0,
|
||||
"aborted": 0,
|
||||
},
|
||||
]
|
||||
|
||||
class FakeApp:
|
||||
job_manager = FakeManager()
|
||||
|
||||
counts = await pq.get_ingest_job_counts(cast(App, FakeApp()))
|
||||
assert counts["todo"] == 3
|
||||
assert counts["todo"] == 3 # only ingest-* queues, not some-other-queue
|
||||
assert counts["doing"] == 1
|
||||
assert counts["failed"] == 2
|
||||
assert counts["succeeded"] == 0
|
||||
|
||||
by_queue = await pq.get_ingest_job_counts_by_queue(cast(App, FakeApp()))
|
||||
assert set(by_queue) == {"ingest-fast", "ingest-ocr"}
|
||||
assert by_queue["ingest-fast"]["todo"] == 3
|
||||
assert by_queue["ingest-ocr"]["failed"] == 2
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Unit tests for the per-tier escalation primitives (Deck #323).
|
||||
|
||||
Covers the tier-ladder helpers + EscalateError (document_processors.escalation)
|
||||
and the procrastinate TieredEscalationStrategy that turns a raised exception
|
||||
into a queue-hop / same-tier retry / give-up decision.
|
||||
"""
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from procrastinate.jobs import Job
|
||||
|
||||
import nextcloud_mcp_server.vector.queue.procrastinate as pq
|
||||
from nextcloud_mcp_server.document_processors.escalation import (
|
||||
TIER_LADDER,
|
||||
EscalateError,
|
||||
next_tier,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def _job(queue: str = pq.INGEST_QUEUE_FAST, attempts: int = 1) -> Job:
|
||||
return Job(
|
||||
id=1,
|
||||
queue=queue,
|
||||
task_name=pq.INGEST_TASK_NAME,
|
||||
lock=None,
|
||||
queueing_lock=None,
|
||||
attempts=attempts,
|
||||
)
|
||||
|
||||
|
||||
class TestLadder:
|
||||
def test_next_tier_ordering(self):
|
||||
assert next_tier("fast") == "structured"
|
||||
assert next_tier("structured") == "ocr"
|
||||
assert next_tier("ocr") is None # terminal
|
||||
assert next_tier("unknown") is None
|
||||
|
||||
def test_ladder_is_cheapest_first(self):
|
||||
assert TIER_LADDER == ("fast", "structured", "ocr")
|
||||
|
||||
def test_tier_for_queue(self):
|
||||
assert pq.tier_for_queue(pq.INGEST_QUEUE_OCR) == "ocr"
|
||||
assert pq.tier_for_queue(pq.INGEST_QUEUE_STRUCTURED) == "structured"
|
||||
# Legacy / unknown / None all fall back to the cheapest tier.
|
||||
assert pq.tier_for_queue(pq.LEGACY_INGEST_QUEUE) == "fast"
|
||||
assert pq.tier_for_queue(None) == "fast"
|
||||
|
||||
|
||||
class TestTieredEscalationStrategy:
|
||||
def _strategy(self, max_transient: int = 5):
|
||||
return pq.TieredEscalationStrategy(max_transient_attempts=max_transient)
|
||||
|
||||
def test_escalate_hops_to_target_queue(self):
|
||||
exc = EscalateError(from_tier="fast", to_tier="ocr", reason="empty_text")
|
||||
decision = self._strategy().get_retry_decision(exception=exc, job=_job())
|
||||
assert decision is not None
|
||||
assert decision.queue == pq.INGEST_QUEUE_OCR
|
||||
|
||||
def test_escalate_to_structured(self):
|
||||
exc = EscalateError(
|
||||
from_tier="fast", to_tier="structured", reason="low_confidence"
|
||||
)
|
||||
decision = self._strategy().get_retry_decision(exception=exc, job=_job())
|
||||
assert decision is not None
|
||||
assert decision.queue == pq.INGEST_QUEUE_STRUCTURED
|
||||
|
||||
def test_escalate_unknown_tier_gives_up(self):
|
||||
exc = EscalateError(from_tier="ocr", to_tier="bogus", reason="low_confidence")
|
||||
decision = self._strategy().get_retry_decision(exception=exc, job=_job())
|
||||
assert decision is None
|
||||
|
||||
def test_escalate_unwraps_exception_group(self):
|
||||
exc = EscalateError(from_tier="fast", to_tier="ocr", reason="empty_text")
|
||||
group = ExceptionGroup("wrapped", [exc])
|
||||
decision = self._strategy().get_retry_decision(exception=group, job=_job())
|
||||
assert decision is not None
|
||||
assert decision.queue == pq.INGEST_QUEUE_OCR
|
||||
|
||||
def test_transient_retries_same_queue_under_cap(self):
|
||||
decision = self._strategy(max_transient=5).get_retry_decision(
|
||||
exception=httpx.ConnectError("refused"), job=_job(attempts=1)
|
||||
)
|
||||
assert decision is not None
|
||||
# Same-tier retry: no queue override (stays on its current queue).
|
||||
assert decision.queue is None
|
||||
assert decision.retry_at is not None
|
||||
|
||||
def test_transient_gives_up_over_cap(self):
|
||||
decision = self._strategy(max_transient=5).get_retry_decision(
|
||||
exception=httpx.ConnectError("refused"), job=_job(attempts=5)
|
||||
)
|
||||
assert decision is None
|
||||
|
||||
def test_non_transient_error_gives_up(self):
|
||||
decision = self._strategy().get_retry_decision(
|
||||
exception=ValueError("permanent"), job=_job(attempts=1)
|
||||
)
|
||||
assert decision is None
|
||||
Reference in New Issue
Block a user