docs(vector): document oversize dead-letter reason and failure-mode comments
Round-4 review nits on PR #920 (none blocking): - record_document_dead_lettered: enumerate the oversize reason (added this PR) alongside timeout/oom/error in the docstring + counter comment. - Note the clear-dead-letter-before-upsert ordering implication (a transient upsert failure re-parses once, never a silent drop). - Clarify the orphan sweep's kept counter for tenant-wide dead-letter markers. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
a7f7461716
commit
114af7bf12
@@ -316,7 +316,7 @@ document_parse_failed_total = Counter(
|
||||
document_dead_lettered_total = Counter(
|
||||
"astrolabe_document_dead_lettered_total",
|
||||
"Documents dead-lettered after a terminal parse failure (no escalation tier)",
|
||||
["reason"], # reason: timeout | oom | error
|
||||
["reason"], # reason: timeout | oom | error | oversize
|
||||
)
|
||||
|
||||
# Documents dropped after exhausting in-process indexing retries (the scanner
|
||||
@@ -806,7 +806,8 @@ def record_document_dead_lettered(reason: str) -> None:
|
||||
|
||||
Args:
|
||||
reason: ``timeout`` | ``oom`` | ``error`` (the terminal parse failure
|
||||
reason carried from the isolated worker).
|
||||
reason carried from the isolated worker) or ``oversize`` (rejected by
|
||||
the pre-parse size guard, which no tier can ever parse).
|
||||
"""
|
||||
document_dead_lettered_total.labels(reason=reason).inc()
|
||||
|
||||
|
||||
@@ -402,6 +402,8 @@ async def sweep_orphan_placeholders(
|
||||
# in-flight placeholders -- they carry no/foreign instance_id and must
|
||||
# survive a Pod restart, so never sweep them as orphans.
|
||||
if payload.get("dead_letter") is True:
|
||||
# Tenant-wide and always kept (not Pod-scoped); counted under
|
||||
# ``kept`` only because the sweep's tally has no separate bucket.
|
||||
kept += 1
|
||||
continue
|
||||
point_instance = payload.get("instance_id")
|
||||
|
||||
@@ -1615,6 +1615,9 @@ async def _index_document(
|
||||
# escalation tier finally parsed it) so it isn't left behind. Only files are
|
||||
# ever dead-lettered, and only with a non-empty etag (is_dead_lettered
|
||||
# early-returns without one), so skip the extra Qdrant round-trip otherwise.
|
||||
# Cleared before the real-chunk upsert below: if that upsert then fails
|
||||
# transiently, the document is re-queued and re-parses once on the next scan
|
||||
# (an extra parse, never a silent drop) -- the safe ordering.
|
||||
if doc_task.doc_type == "file" and doc_task.etag:
|
||||
await clear_dead_letter(doc_task.doc_id, doc_task.doc_type)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user