fix(ingest): address review round 3 + SonarCloud reliability gate
- tests: make the transient-backoff progression assertion load-independent by bracketing the get_retry_decision call with before/after timestamps instead of measuring against a second datetime.now() (no freezegun dependency). - tests: use pytest.approx for the ingest-queue-depth gauge assertions — SonarCloud python:S1244 (float == ) was a MAJOR reliability finding that tripped the new_reliability_rating quality gate. - processor: tighten the EscalateError lazy-bind comment (file processing already imports the document stack via get_registry; the gating only spares the delete / text-doc paths and module-load time). Deck #323. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
e7c0c23486
commit
44f72839ed
@@ -507,10 +507,11 @@ async def process_document(
|
|||||||
one outer attempt (~30s) and defers. Don't stack a third retry layer here.
|
one outer attempt (~30s) and defers. Don't stack a third retry layer here.
|
||||||
"""
|
"""
|
||||||
# EscalateError is a control-flow signal that arises ONLY on the per-tier
|
# EscalateError is a control-flow signal that arises ONLY on the per-tier
|
||||||
# external path (tier set). Bind the class lazily there so the in-process /
|
# external path (tier set). Bind the class lazily here, and only when a tier
|
||||||
# memory path never pulls the document stack at call time (mirrors the lazy
|
# is set, so the document stack is never imported at *module load* (the #877
|
||||||
# get_registry import; see #877). When tier is None it can't be raised, so
|
# invariant) nor on the delete / text-doc call paths (file processing already
|
||||||
# the guards below stay inert.
|
# imports it via get_registry regardless). When tier is None it can't be
|
||||||
|
# raised, so the guards below stay inert.
|
||||||
escalate_error_cls: type[BaseException] | None = None
|
escalate_error_cls: type[BaseException] | None = None
|
||||||
if tier is not None:
|
if tier is not None:
|
||||||
from nextcloud_mcp_server.document_processors.escalation import ( # noqa: PLC0415
|
from nextcloud_mcp_server.document_processors.escalation import ( # noqa: PLC0415
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ procrastinate's ``list_queues_async``) must read 0, not its last non-zero value.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
from pytest import approx
|
||||||
|
|
||||||
from nextcloud_mcp_server.observability.metrics import update_ingest_queue_depth
|
from nextcloud_mcp_server.observability.metrics import update_ingest_queue_depth
|
||||||
|
|
||||||
@@ -14,19 +15,27 @@ _METRIC = "astrolabe_ingest_queue_depth"
|
|||||||
|
|
||||||
|
|
||||||
def test_drained_queue_zeroes_not_stale(metric_sample):
|
def test_drained_queue_zeroes_not_stale(metric_sample):
|
||||||
# ocr has a backlog this tick.
|
# ocr has a backlog this tick. (pytest.approx: the gauge sample is a float.)
|
||||||
update_ingest_queue_depth({"ingest-ocr": {"todo": 4}})
|
update_ingest_queue_depth({"ingest-ocr": {"todo": 4}})
|
||||||
assert metric_sample(_METRIC, {"queue": "ingest-ocr", "status": "todo"}) == 4.0
|
assert metric_sample(_METRIC, {"queue": "ingest-ocr", "status": "todo"}) == approx(
|
||||||
|
4
|
||||||
|
)
|
||||||
|
|
||||||
# Next tick ocr has drained → procrastinate omits it from by_queue entirely.
|
# Next tick ocr has drained → procrastinate omits it from by_queue entirely.
|
||||||
update_ingest_queue_depth({"ingest-fast": {"todo": 1}})
|
update_ingest_queue_depth({"ingest-fast": {"todo": 1}})
|
||||||
# The gauge must read 0 for the drained queue, not the stale 4.
|
# The gauge must read 0 for the drained queue, not the stale 4.
|
||||||
assert metric_sample(_METRIC, {"queue": "ingest-ocr", "status": "todo"}) == 0.0
|
assert metric_sample(_METRIC, {"queue": "ingest-ocr", "status": "todo"}) == approx(
|
||||||
assert metric_sample(_METRIC, {"queue": "ingest-fast", "status": "todo"}) == 1.0
|
0
|
||||||
|
)
|
||||||
|
assert metric_sample(_METRIC, {"queue": "ingest-fast", "status": "todo"}) == approx(
|
||||||
|
1
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_none_is_noop(metric_sample):
|
def test_none_is_noop(metric_sample):
|
||||||
update_ingest_queue_depth({"ingest-fast": {"doing": 2}})
|
update_ingest_queue_depth({"ingest-fast": {"doing": 2}})
|
||||||
# Memory backend passes None → must not wipe the last published values.
|
# Memory backend passes None → must not wipe the last published values.
|
||||||
update_ingest_queue_depth(None)
|
update_ingest_queue_depth(None)
|
||||||
assert metric_sample(_METRIC, {"queue": "ingest-fast", "status": "doing"}) == 2.0
|
assert metric_sample(
|
||||||
|
_METRIC, {"queue": "ingest-fast", "status": "doing"}
|
||||||
|
) == approx(2)
|
||||||
|
|||||||
@@ -91,16 +91,22 @@ class TestTieredEscalationStrategy:
|
|||||||
|
|
||||||
def test_transient_backoff_progression(self):
|
def test_transient_backoff_progression(self):
|
||||||
# min(4 * 2**(attempts-1), 300): 4, 8, 16, ... capped at 300s.
|
# min(4 * 2**(attempts-1), 300): 4, 8, 16, ... capped at 300s.
|
||||||
|
# procrastinate sets retry_at = utcnow() + wait at call time. Bracketing
|
||||||
|
# the call with before/after makes the assertion exact and independent of
|
||||||
|
# runner load: with before <= call_now <= after, we have
|
||||||
|
# (retry_at - after) <= wait <= (retry_at - before).
|
||||||
strat = self._strategy(max_transient=100)
|
strat = self._strategy(max_transient=100)
|
||||||
for attempts, expected in [(1, 4), (2, 8), (3, 16), (4, 32), (20, 300)]:
|
for attempts, expected in [(1, 4), (2, 8), (3, 16), (4, 32), (20, 300)]:
|
||||||
|
before = datetime.now(timezone.utc)
|
||||||
decision = strat.get_retry_decision(
|
decision = strat.get_retry_decision(
|
||||||
exception=httpx.ConnectError("x"), job=_job(attempts=attempts)
|
exception=httpx.ConnectError("x"), job=_job(attempts=attempts)
|
||||||
)
|
)
|
||||||
|
after = datetime.now(timezone.utc)
|
||||||
assert decision is not None and decision.retry_at is not None
|
assert decision is not None and decision.retry_at is not None
|
||||||
delta = (decision.retry_at - datetime.now(timezone.utc)).total_seconds()
|
lo = (decision.retry_at - after).total_seconds()
|
||||||
# retry_at = now + wait; allow a small window for execution time.
|
hi = (decision.retry_at - before).total_seconds()
|
||||||
assert expected - 2 <= delta <= expected + 1, (
|
assert lo <= expected <= hi, (
|
||||||
f"attempts={attempts}: delta={delta:.2f}s, expected≈{expected}s"
|
f"attempts={attempts}: expected={expected}s not in [{lo:.3f}, {hi:.3f}]"
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_transient_gives_up_over_cap(self):
|
def test_transient_gives_up_over_cap(self):
|
||||||
|
|||||||
Reference in New Issue
Block a user