fix(vector): dead-letter terminally-failed documents to stop multi-user re-queue loop
A pathological PDF (a 206-page ChronoScan scan with ~3400 JBIG2/JPX images)
jammed a tenant's structured ingest worker in an infinite reprocess loop,
re-burning a 120s pymupdf4llm parse (and occasionally OOM-racing the 2Gi pod)
every few minutes.
Root cause: the per-user placeholder "failed" mark could not stop the loop. The
placeholder point ID is user-agnostic (uuid5("file:<doc_id>:placeholder")) but
the scanner's freshness gate, query, and status update all filter by user_id.
For a file visible to several users the single shared placeholder's user_id is
overwritten by whoever scanned last, so every other user's scan sees "no record"
and re-queues -- an N-user ping-pong that never honours the failed status.
Fix: when a parse fails terminally (no higher escalation tier available, e.g.
structured with OCR off) record a durable, content-addressed, user-agnostic
dead-letter marker (mirrors vector/sharing_state.py). The scanner consults it
tenant-wide for every user and skips re-queuing until the content (etag) OR the
escalation-tier set (tiers_sig -- e.g. OCR enabled) changes, so the document is
attempted once per content-version instead of forever.
- new vector/dead_letter.py: mark/is/clear, content-addressed marker carrying
is_placeholder=True (inherits search exclusion) + dead_letter=True
- escalation.escalation_tiers_signature(settings): retry-on-tier-change key
- processor: dead-letter terminal failures, clear on successful (re-)index
- scanner: user-agnostic is_dead_lettered skip beside claim_existing_index
- placeholder: exempt dead_letter markers from the orphan sweep (durability)
- metrics: astrolabe_document_dead_lettered_total{reason}
Deck #349.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
d969526613
commit
8c9339501e
@@ -22,13 +22,36 @@ mapping lives in the queue layer, which imports :class:`EscalateError` from here
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal
|
||||
from typing import Any, Literal
|
||||
|
||||
# Cheapest-first. ``llm`` is reserved (see base.DocumentProcessor.tier) and not
|
||||
# wired yet, so it is intentionally absent from the live ladder.
|
||||
TIER_LADDER: tuple[str, ...] = ("fast", "structured", "ocr")
|
||||
|
||||
|
||||
def escalation_tiers_signature(settings: Any) -> str:
|
||||
"""A stable string fingerprint of the runtime escalation-tier configuration.
|
||||
|
||||
Used by the document dead-letter marker (``vector/dead_letter.py``) as part of
|
||||
its content key: a document that fails its terminal tier is dead-lettered until
|
||||
either its content (etag) OR this signature changes. The signature therefore
|
||||
captures every setting that can make a *new* escalation tier become available
|
||||
at runtime — flip it and previously dead-lettered documents become retryable.
|
||||
|
||||
Derived purely from settings (not the live ``ProcessorRegistry``) so it is
|
||||
identical across the API/scanner and worker roles: the *registered* processor
|
||||
set is build-constant, so the only runtime variables are the OCR-enabled gate
|
||||
(``document_ocr_enabled`` — the single tier toggle today) and the tier-1 engine
|
||||
pin (``document_tier1_engine``). Enabling OCR changes the signature, so the
|
||||
pathological-but-OCR-recoverable documents dead-lettered while OCR was off are
|
||||
re-attempted automatically.
|
||||
"""
|
||||
return (
|
||||
f"ocr={int(bool(settings.document_ocr_enabled))};"
|
||||
f"t1={settings.document_tier1_engine}"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EscalationDecision:
|
||||
"""Outcome of the post-parse quality gate (``ProcessorRegistry.evaluate_escalation``).
|
||||
|
||||
@@ -305,6 +305,20 @@ document_parse_failed_total = Counter(
|
||||
["reason"], # reason: timeout | oom | error
|
||||
)
|
||||
|
||||
# Documents dead-lettered after a terminal parse failure: the failing tier had
|
||||
# no higher escalation tier available (e.g. structured timed out with OCR off),
|
||||
# so the document is recorded as permanently failed for this content-version and
|
||||
# stops being re-queued (vector/dead_letter.py). Distinct from
|
||||
# ``document_parse_failed_total`` (which counts every failed parse attempt,
|
||||
# including the ones that will be retried) -- this fires once when a document
|
||||
# is given up on, and clears implicitly when its etag or the escalation-tier set
|
||||
# changes and it is re-attempted.
|
||||
document_dead_lettered_total = Counter(
|
||||
"astrolabe_document_dead_lettered_total",
|
||||
"Documents dead-lettered after a terminal parse failure (no escalation tier)",
|
||||
["reason"], # reason: timeout | oom | error
|
||||
)
|
||||
|
||||
# Documents dropped after exhausting in-process indexing retries (the scanner
|
||||
# re-picks them on a later full scan, so this is "dropped for this cycle", not
|
||||
# "lost forever"). Labelled by classified cause so the embed-drop rate from a
|
||||
@@ -787,6 +801,16 @@ def record_document_parse_failed(reason: str) -> None:
|
||||
document_parse_failed_total.labels(reason=reason).inc()
|
||||
|
||||
|
||||
def record_document_dead_lettered(reason: str) -> None:
|
||||
"""Record a document dead-lettered after a terminal parse failure.
|
||||
|
||||
Args:
|
||||
reason: ``timeout`` | ``oom`` | ``error`` (the terminal parse failure
|
||||
reason carried from the isolated worker).
|
||||
"""
|
||||
document_dead_lettered_total.labels(reason=reason).inc()
|
||||
|
||||
|
||||
def record_ingest_dropped(reason: str) -> None:
|
||||
"""Record a document dropped after exhausting in-process indexing retries.
|
||||
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
"""Content-addressed dead-letter markers for terminally-failed documents.
|
||||
|
||||
A document that fails its terminal extraction tier (a hard parse failure with no
|
||||
higher tier to escalate to — e.g. the ``structured`` tier timing out while OCR is
|
||||
disabled) must not be retried forever. The per-user placeholder ``status="failed"``
|
||||
mark cannot stop the loop on its own: placeholder point IDs are user-agnostic
|
||||
(``uuid5("file:<doc_id>:placeholder")``) but the scanner's freshness gate filters
|
||||
by ``user_id``, so for a file visible to *several* users the single shared
|
||||
placeholder's ``user_id`` is overwritten by whoever scanned last and every other
|
||||
user's scan sees "no record → re-queue", re-burning the (failing) parse on a loop.
|
||||
|
||||
This module records a **durable, content-addressed, user-agnostic** dead-letter
|
||||
marker instead. One marker point per document (a distinct deterministic ID, kept
|
||||
separate from the in-flight placeholder), carrying the ``etag`` and an escalation
|
||||
``tiers_sig`` (see ``document_processors.escalation.escalation_tiers_signature``).
|
||||
The scanner consults it tenant-wide — for every user — and skips re-queuing while
|
||||
BOTH still match, so the document is attempted once per content-version and never
|
||||
loops. A content change (new ``etag``) or a config change that adds an escalation
|
||||
tier (e.g. enabling OCR — new ``tiers_sig``) makes the marker stale and the
|
||||
document retryable again.
|
||||
|
||||
The marker carries ``is_placeholder=True`` so the existing search exclusion
|
||||
(``get_placeholder_filter``) keeps it out of user-facing results with no extra
|
||||
filter, plus ``dead_letter=True`` so the orphan-placeholder sweep and the scanner
|
||||
can tell it apart from a volatile in-flight placeholder. Mirrors the fail-safe
|
||||
philosophy of ``sharing_state``: a Qdrant error never aborts ingest — a failed
|
||||
lookup degrades to "process normally", a failed write is logged, not raised.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from qdrant_client import models
|
||||
from qdrant_client.models import FieldCondition, Filter, MatchValue, PointStruct
|
||||
|
||||
from nextcloud_mcp_server.config import get_settings
|
||||
from nextcloud_mcp_server.embedding import get_embedding_service
|
||||
from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Payload flag distinguishing a durable dead-letter marker from an in-flight
|
||||
# placeholder (both carry is_placeholder=True to inherit the search exclusion).
|
||||
DEAD_LETTER_KEY = "dead_letter"
|
||||
|
||||
|
||||
def _generate_dead_letter_id(doc_type: str, doc_id: str) -> str:
|
||||
"""Deterministic, user-agnostic point ID for a document's dead-letter marker.
|
||||
|
||||
Distinct from the in-flight placeholder ID (``…:placeholder``) so the two can
|
||||
coexist briefly and never collide; one marker per ``(doc_type, doc_id)``, so
|
||||
a re-failure upserts in place rather than accumulating.
|
||||
"""
|
||||
return str(uuid.uuid5(uuid.NAMESPACE_DNS, f"{doc_type}:{doc_id}:deadletter"))
|
||||
|
||||
|
||||
def _dead_letter_filter(doc_id: str, doc_type: str) -> Filter:
|
||||
"""Match the dead-letter marker for one document (tenant-wide, no user_id)."""
|
||||
return Filter(
|
||||
must=[
|
||||
FieldCondition(key="doc_id", match=MatchValue(value=doc_id)),
|
||||
FieldCondition(key="doc_type", match=MatchValue(value=doc_type)),
|
||||
FieldCondition(key=DEAD_LETTER_KEY, match=MatchValue(value=True)),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
async def mark_dead_letter(
|
||||
doc_id: str,
|
||||
doc_type: str,
|
||||
etag: str,
|
||||
tiers_sig: str,
|
||||
reason: str,
|
||||
*,
|
||||
file_path: str | None = None,
|
||||
) -> None:
|
||||
"""Upsert a durable dead-letter marker for a terminally-failed document.
|
||||
|
||||
Keyed by content (``etag``) + escalation config (``tiers_sig``); the scanner
|
||||
skips re-queuing while both match. ``reason`` is the parse failure reason
|
||||
(``timeout`` | ``oom`` | ``error``). Fail-safe: a Qdrant error is logged, not
|
||||
raised — a missed mark just means the document is retried (the bounded prior
|
||||
behaviour), never a crash.
|
||||
"""
|
||||
try:
|
||||
qdrant_client = await get_qdrant_client()
|
||||
settings = get_settings()
|
||||
dimension = get_embedding_service().get_dimension()
|
||||
|
||||
payload: dict = {
|
||||
"doc_id": doc_id,
|
||||
"doc_type": doc_type,
|
||||
"is_placeholder": True,
|
||||
DEAD_LETTER_KEY: True,
|
||||
"etag": etag,
|
||||
"tiers_sig": tiers_sig,
|
||||
"reason": reason,
|
||||
"failed_at": int(time.time()),
|
||||
}
|
||||
if doc_type == "file" and file_path:
|
||||
payload["file_path"] = file_path
|
||||
|
||||
point = PointStruct(
|
||||
id=_generate_dead_letter_id(doc_type, doc_id),
|
||||
vector={
|
||||
"dense": [0.0] * dimension,
|
||||
"sparse": models.SparseVector(indices=[], values=[]),
|
||||
},
|
||||
payload=payload,
|
||||
)
|
||||
await qdrant_client.upsert(
|
||||
collection_name=settings.get_collection_name(),
|
||||
points=[point],
|
||||
wait=True,
|
||||
)
|
||||
logger.info(
|
||||
"Dead-lettered %s_%s (reason=%s, etag=%s)",
|
||||
doc_type,
|
||||
doc_id,
|
||||
reason,
|
||||
etag,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Failed to write dead-letter marker for %s_%s: %s", doc_type, doc_id, e
|
||||
)
|
||||
# Don't raise — dead-lettering is best-effort; a miss just retries.
|
||||
|
||||
|
||||
async def is_dead_lettered(
|
||||
doc_id: str,
|
||||
doc_type: str,
|
||||
etag: str,
|
||||
tiers_sig: str,
|
||||
) -> bool:
|
||||
"""Whether this exact content is currently dead-lettered (skip re-queuing).
|
||||
|
||||
Returns True only when a marker exists for ``(doc_id, doc_type)`` whose stored
|
||||
``etag`` AND ``tiers_sig`` both match the current values — so a content change
|
||||
or a new escalation tier (e.g. OCR enabled) makes the document retryable
|
||||
again. An empty ``etag`` is never dead-lettered (we cannot content-address it).
|
||||
Fail-safe: a Qdrant error degrades to False (process normally), mirroring
|
||||
``sharing_state.claim_existing_index``.
|
||||
"""
|
||||
if not etag:
|
||||
return False
|
||||
try:
|
||||
qdrant_client = await get_qdrant_client()
|
||||
settings = get_settings()
|
||||
points, _ = await qdrant_client.scroll(
|
||||
collection_name=settings.get_collection_name(),
|
||||
scroll_filter=_dead_letter_filter(doc_id, doc_type),
|
||||
limit=1,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Dead-letter lookup failed for %s_%s (%s); processing normally",
|
||||
doc_type,
|
||||
doc_id,
|
||||
e,
|
||||
)
|
||||
return False
|
||||
if not points:
|
||||
return False
|
||||
payload = dict(points[0].payload or {})
|
||||
return payload.get("etag") == etag and payload.get("tiers_sig") == tiers_sig
|
||||
|
||||
|
||||
async def clear_dead_letter(doc_id: str, doc_type: str) -> None:
|
||||
"""Delete a document's dead-letter marker (on successful index / release).
|
||||
|
||||
Idempotent and fail-safe: deleting a non-existent marker is a Qdrant no-op,
|
||||
and an error is logged rather than raised so it never breaks the indexing
|
||||
path that calls it.
|
||||
"""
|
||||
try:
|
||||
qdrant_client = await get_qdrant_client()
|
||||
settings = get_settings()
|
||||
await qdrant_client.delete(
|
||||
collection_name=settings.get_collection_name(),
|
||||
points_selector=_dead_letter_filter(doc_id, doc_type),
|
||||
)
|
||||
logger.debug("Cleared dead-letter marker for %s_%s", doc_type, doc_id)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Failed to clear dead-letter marker for %s_%s: %s", doc_type, doc_id, e
|
||||
)
|
||||
@@ -397,6 +397,13 @@ async def sweep_orphan_placeholders(
|
||||
orphan_ids = []
|
||||
for point in points:
|
||||
payload = point.payload or {}
|
||||
# Dead-letter markers (vector/dead_letter.py) reuse is_placeholder=True
|
||||
# for the search exclusion but are DURABLE terminal-state records, not
|
||||
# in-flight placeholders -- they carry no/foreign instance_id and must
|
||||
# survive a Pod restart, so never sweep them as orphans.
|
||||
if payload.get("dead_letter") is True:
|
||||
kept += 1
|
||||
continue
|
||||
point_instance = payload.get("instance_id")
|
||||
if point_instance == _INSTANCE_ID:
|
||||
kept += 1
|
||||
|
||||
@@ -28,6 +28,7 @@ from nextcloud_mcp_server.embedding import get_bm25_service, get_embedding_servi
|
||||
from nextcloud_mcp_server.models.deck import DeckCard
|
||||
from nextcloud_mcp_server.observability.metrics import (
|
||||
record_document_chunks,
|
||||
record_document_dead_lettered,
|
||||
record_document_escalation,
|
||||
record_document_escalation_suppressed,
|
||||
record_document_parse_failed,
|
||||
@@ -43,6 +44,10 @@ from nextcloud_mcp_server.search.pdf_highlighter import PDFHighlighter
|
||||
from nextcloud_mcp_server.usage import UsageEventStore
|
||||
from nextcloud_mcp_server.vector import payload_keys
|
||||
from nextcloud_mcp_server.vector._errors import format_exception_group
|
||||
from nextcloud_mcp_server.vector.dead_letter import (
|
||||
clear_dead_letter,
|
||||
mark_dead_letter,
|
||||
)
|
||||
from nextcloud_mcp_server.vector.document_chunker import (
|
||||
DocumentChunker,
|
||||
PageAwareChunker,
|
||||
@@ -993,8 +998,10 @@ async def _index_document(
|
||||
get_registry,
|
||||
)
|
||||
from nextcloud_mcp_server.document_processors.escalation import ( # noqa: PLC0415
|
||||
TIER_LADDER,
|
||||
BatchPending,
|
||||
EscalateError,
|
||||
escalation_tiers_signature,
|
||||
)
|
||||
|
||||
registry = get_registry()
|
||||
@@ -1035,34 +1042,89 @@ async def _index_document(
|
||||
# A permanent parse failure (e.g. an isolated-worker OOM/timeout
|
||||
# on a pathological PDF) returns success=False rather than
|
||||
# raising -- there is nothing to index and retrying would just
|
||||
# fail again. Mark the placeholder "failed" so the scanner stops
|
||||
# re-queuing it (until the file changes) and return False so the
|
||||
# caller skips the success metrics (it was not indexed).
|
||||
# fail again.
|
||||
if not result.success:
|
||||
reason = result.metadata.get("parse_failed_reason", "error")
|
||||
record_document_parse_failed(reason)
|
||||
logger.warning(
|
||||
"Permanent parse failure for %s (reason=%s); marking "
|
||||
"failed and skipping index",
|
||||
file_path,
|
||||
reason,
|
||||
# The tier that produced this failed result: the worker's own
|
||||
# tier on the per-tier path, else the deepest tier the inline
|
||||
# pipeline reached (recorded as ``pipeline_tier``).
|
||||
failing_tier = tier or result.metadata.get(
|
||||
"pipeline_tier", TIER_LADDER[0]
|
||||
)
|
||||
try:
|
||||
await update_placeholder_status(
|
||||
doc_id=doc_task.doc_id,
|
||||
doc_type=doc_task.doc_type,
|
||||
user_id=doc_task.user_id,
|
||||
status="failed",
|
||||
)
|
||||
except Exception:
|
||||
# Best-effort: a transient Qdrant error here only means
|
||||
# the placeholder isn't marked, so the scanner retries
|
||||
# the (still un-indexable) file later -- not fatal.
|
||||
logger.debug(
|
||||
"Could not mark placeholder failed for %s",
|
||||
terminal = (
|
||||
registry.next_available_tier(failing_tier, settings) is None
|
||||
)
|
||||
if terminal:
|
||||
# No higher tier can run (e.g. structured timed out with
|
||||
# OCR off), so retrying just re-burns the same failing
|
||||
# parse. Dead-letter the document tenant-wide
|
||||
# (content-addressed, user-agnostic) so EVERY user's scan
|
||||
# stops re-queuing it until its content (etag) or the
|
||||
# escalation-tier set (e.g. OCR enabled -> new tiers_sig)
|
||||
# changes. This fixes the multi-user placeholder
|
||||
# ping-pong the per-user "failed" mark could not: a file
|
||||
# shared by N users has ONE user-agnostic placeholder
|
||||
# whose user_id is overwritten by the last scanner, so
|
||||
# every other user re-queued it forever.
|
||||
await mark_dead_letter(
|
||||
doc_task.doc_id,
|
||||
exc_info=True,
|
||||
doc_task.doc_type,
|
||||
doc_task.etag or "",
|
||||
escalation_tiers_signature(settings),
|
||||
reason,
|
||||
file_path=file_path,
|
||||
)
|
||||
record_document_dead_lettered(reason)
|
||||
logger.warning(
|
||||
"Permanent parse failure for %s (reason=%s); "
|
||||
"dead-lettered (terminal tier=%s, no escalation) and "
|
||||
"skipping index",
|
||||
file_path,
|
||||
reason,
|
||||
failing_tier,
|
||||
)
|
||||
# Drop the volatile in-flight placeholder; the durable
|
||||
# marker is now the document's terminal-state record.
|
||||
try:
|
||||
await delete_placeholder_point(
|
||||
doc_id=doc_task.doc_id,
|
||||
doc_type=doc_task.doc_type,
|
||||
user_id=doc_task.user_id,
|
||||
)
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"Could not delete placeholder for dead-lettered %s",
|
||||
doc_task.doc_id,
|
||||
exc_info=True,
|
||||
)
|
||||
else:
|
||||
# A higher tier exists; parse failures don't escalate to
|
||||
# it today, so keep the legacy per-user "failed"
|
||||
# placeholder mark (not dead-lettered -- a future change
|
||||
# may route the failure to that tier).
|
||||
logger.warning(
|
||||
"Permanent parse failure for %s (reason=%s); marking "
|
||||
"failed and skipping index",
|
||||
file_path,
|
||||
reason,
|
||||
)
|
||||
try:
|
||||
await update_placeholder_status(
|
||||
doc_id=doc_task.doc_id,
|
||||
doc_type=doc_task.doc_type,
|
||||
user_id=doc_task.user_id,
|
||||
status="failed",
|
||||
)
|
||||
except Exception:
|
||||
# Best-effort: a transient Qdrant error here only
|
||||
# means the placeholder isn't marked, so the scanner
|
||||
# retries the (still un-indexable) file later.
|
||||
logger.debug(
|
||||
"Could not mark placeholder failed for %s",
|
||||
doc_task.doc_id,
|
||||
exc_info=True,
|
||||
)
|
||||
return False
|
||||
|
||||
content = result.text
|
||||
@@ -1529,6 +1591,14 @@ async def _index_document(
|
||||
)
|
||||
)
|
||||
|
||||
# A successful (re-)index supersedes any prior terminal failure: clear a
|
||||
# stale dead-letter marker (e.g. the file was fixed/replaced, or a new
|
||||
# escalation tier finally parsed it) so it isn't left behind. Only files are
|
||||
# ever dead-lettered (the mark lives in the file branch), so skip the extra
|
||||
# Qdrant round-trip for the other doc types on the hot indexing path.
|
||||
if doc_task.doc_type == "file":
|
||||
await clear_dead_letter(doc_task.doc_id, doc_task.doc_type)
|
||||
|
||||
# Delete placeholder before writing real vectors
|
||||
# This prevents duplicates and cleans up the placeholder state
|
||||
try:
|
||||
|
||||
@@ -27,6 +27,7 @@ from nextcloud_mcp_server.server.tag_exclusion import (
|
||||
get_excluded_file_paths,
|
||||
is_path_excluded,
|
||||
)
|
||||
from nextcloud_mcp_server.vector.dead_letter import is_dead_lettered
|
||||
from nextcloud_mcp_server.vector.placeholder import (
|
||||
query_document_metadata,
|
||||
write_placeholder_point,
|
||||
@@ -664,6 +665,16 @@ async def scan_user_documents(
|
||||
skipped,
|
||||
)
|
||||
|
||||
# Escalation-tier fingerprint for the dead-letter skip below, computed
|
||||
# once per scan. Lazy import: document_processors.__init__ pulls the
|
||||
# heavy parse stack (pymupdf/_isolation, Unix-only ``resource``; #877),
|
||||
# which the scanner (API role) must not load at module import.
|
||||
from nextcloud_mcp_server.document_processors.escalation import ( # noqa: PLC0415
|
||||
escalation_tiers_signature,
|
||||
)
|
||||
|
||||
tiers_sig = escalation_tiers_signature(get_settings())
|
||||
|
||||
for file_info in tagged_files:
|
||||
# Files are already filtered by MIME type in find_files_by_tag()
|
||||
file_count += 1
|
||||
@@ -703,6 +714,24 @@ async def scan_user_documents(
|
||||
)
|
||||
continue
|
||||
|
||||
# Tenant-wide dead-letter skip: a document that terminally failed
|
||||
# parsing (no escalation tier) is recorded user-agnostically, so
|
||||
# EVERY user's scan skips re-queuing it until its content (etag) or
|
||||
# the escalation-tier set (tiers_sig, e.g. OCR enabled) changes.
|
||||
# Unlike the per-user placeholder "failed" mark this is not
|
||||
# defeated by a file shared across users -- whose single
|
||||
# user-agnostic placeholder's user_id is overwritten by the last
|
||||
# scanner, so every other user re-queued it on a loop.
|
||||
if etag and await is_dead_lettered(file_id, "file", etag, tiers_sig):
|
||||
_potentially_deleted.pop((user_id, file_id), None)
|
||||
logger.debug(
|
||||
"Skipping dead-lettered file %s (ID: %s) until content/"
|
||||
"tier change",
|
||||
file_path,
|
||||
file_id,
|
||||
)
|
||||
continue
|
||||
|
||||
if initial_sync:
|
||||
# Send everything on first sync - write placeholder first
|
||||
await write_placeholder_point(
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Unit tests for the escalation-tier signature used by dead-letter keying.
|
||||
|
||||
``escalation_tiers_signature`` fingerprints the runtime escalation config so a
|
||||
dead-lettered document becomes retryable when a new tier appears (e.g. an
|
||||
operator enables OCR). It must be settings-derived (role-independent) and must
|
||||
change when OCR is toggled.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from nextcloud_mcp_server.document_processors.escalation import (
|
||||
escalation_tiers_signature,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def _settings(*, ocr: bool, engine: str = "pypdfium2") -> SimpleNamespace:
|
||||
return SimpleNamespace(document_ocr_enabled=ocr, document_tier1_engine=engine)
|
||||
|
||||
|
||||
def test_signature_is_stable_for_same_config() -> None:
|
||||
assert escalation_tiers_signature(
|
||||
_settings(ocr=False)
|
||||
) == escalation_tiers_signature(_settings(ocr=False))
|
||||
|
||||
|
||||
def test_enabling_ocr_changes_signature() -> None:
|
||||
# Enabling OCR adds an escalation tier -> previously dead-lettered docs retry.
|
||||
assert escalation_tiers_signature(
|
||||
_settings(ocr=False)
|
||||
) != escalation_tiers_signature(_settings(ocr=True))
|
||||
|
||||
|
||||
def test_tier1_engine_change_changes_signature() -> None:
|
||||
assert escalation_tiers_signature(
|
||||
_settings(ocr=False, engine="pypdfium2")
|
||||
) != escalation_tiers_signature(_settings(ocr=False, engine="pymupdf"))
|
||||
@@ -0,0 +1,170 @@
|
||||
"""Unit tests for content-addressed dead-letter markers.
|
||||
|
||||
Covers vector/dead_letter.py: the durable, user-agnostic terminal-failure marker
|
||||
that stops a multi-user shared file (whose single placeholder's user_id is
|
||||
overwritten by the last scanner) from being re-queued forever. A marker is keyed
|
||||
by content (``etag``) + escalation config (``tiers_sig``); a scan skips only while
|
||||
both still match, so a content change or a newly-available tier (e.g. OCR enabled)
|
||||
makes the document retryable again.
|
||||
|
||||
Qdrant is reached via ``get_qdrant_client``/``get_settings``/``get_embedding_service``,
|
||||
all monkeypatched here so the logic runs without a live Qdrant.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from nextcloud_mcp_server.vector import dead_letter as dl
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
_COLLECTION = "test_collection"
|
||||
|
||||
|
||||
class _Settings:
|
||||
def get_collection_name(self) -> str:
|
||||
return _COLLECTION
|
||||
|
||||
|
||||
def _point(payload: dict) -> SimpleNamespace:
|
||||
"""Stand-in for a qdrant_client Record (only id/payload are read)."""
|
||||
return SimpleNamespace(id="pt", payload=payload)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(monkeypatch) -> AsyncMock:
|
||||
"""An AsyncMock Qdrant client wired into dead_letter, plus stub deps.
|
||||
|
||||
``scroll`` defaults to "no points"; individual tests override
|
||||
``client.scroll.return_value``/``side_effect``.
|
||||
"""
|
||||
qc = AsyncMock()
|
||||
qc.scroll.return_value = ([], None)
|
||||
monkeypatch.setattr(dl, "get_qdrant_client", AsyncMock(return_value=qc))
|
||||
monkeypatch.setattr(dl, "get_settings", lambda: _Settings())
|
||||
monkeypatch.setattr(
|
||||
dl, "get_embedding_service", lambda: SimpleNamespace(get_dimension=lambda: 4)
|
||||
)
|
||||
return qc
|
||||
|
||||
|
||||
def _must_conditions(flt) -> dict:
|
||||
"""Map FieldCondition key -> matched value for a Filter's ``must`` clause."""
|
||||
out = {}
|
||||
for c in flt.must or []:
|
||||
key = getattr(c, "key", None)
|
||||
match = getattr(c, "match", None)
|
||||
out[key] = getattr(match, "value", None)
|
||||
return out
|
||||
|
||||
|
||||
class TestDeadLetterId:
|
||||
def test_user_agnostic_and_distinct_from_placeholder(self) -> None:
|
||||
from nextcloud_mcp_server.vector.placeholder import _generate_placeholder_id
|
||||
|
||||
dl_id = dl._generate_dead_letter_id("file", "520189")
|
||||
# Deterministic + user-agnostic (depends only on doc_type:doc_id).
|
||||
assert dl_id == dl._generate_dead_letter_id("file", "520189")
|
||||
# Never collides with the in-flight placeholder for the same document.
|
||||
assert dl_id != _generate_placeholder_id("file", "520189")
|
||||
|
||||
|
||||
class TestMarkDeadLetter:
|
||||
async def test_upserts_content_addressed_marker(self, client) -> None:
|
||||
await dl.mark_dead_letter(
|
||||
"520189",
|
||||
"file",
|
||||
"etag-1",
|
||||
"ocr=0;t1=pypdfium2",
|
||||
"timeout",
|
||||
file_path="/Plans/big.pdf",
|
||||
)
|
||||
client.upsert.assert_awaited_once()
|
||||
point = client.upsert.await_args.kwargs["points"][0]
|
||||
assert point.id == dl._generate_dead_letter_id("file", "520189")
|
||||
payload = point.payload
|
||||
assert payload["is_placeholder"] is True
|
||||
assert payload[dl.DEAD_LETTER_KEY] is True
|
||||
assert payload["etag"] == "etag-1"
|
||||
assert payload["tiers_sig"] == "ocr=0;t1=pypdfium2"
|
||||
assert payload["reason"] == "timeout"
|
||||
assert payload["doc_id"] == "520189"
|
||||
assert payload["file_path"] == "/Plans/big.pdf"
|
||||
|
||||
async def test_failure_is_swallowed(self, client) -> None:
|
||||
client.upsert.side_effect = RuntimeError("qdrant down")
|
||||
# Best-effort: a write failure must not propagate (would crash the worker).
|
||||
await dl.mark_dead_letter("1", "file", "e", "sig", "oom")
|
||||
|
||||
|
||||
class TestIsDeadLettered:
|
||||
def _marker(self, etag: str, tiers_sig: str) -> SimpleNamespace:
|
||||
return _point(
|
||||
{
|
||||
"doc_id": "520189",
|
||||
dl.DEAD_LETTER_KEY: True,
|
||||
"etag": etag,
|
||||
"tiers_sig": tiers_sig,
|
||||
}
|
||||
)
|
||||
|
||||
async def test_true_when_etag_and_sig_match(self, client) -> None:
|
||||
client.scroll.return_value = ([self._marker("e1", "sig1")], None)
|
||||
assert await dl.is_dead_lettered("520189", "file", "e1", "sig1") is True
|
||||
|
||||
async def test_false_on_etag_change(self, client) -> None:
|
||||
client.scroll.return_value = ([self._marker("e1", "sig1")], None)
|
||||
# File content changed -> retryable.
|
||||
assert await dl.is_dead_lettered("520189", "file", "e2", "sig1") is False
|
||||
|
||||
async def test_false_on_tiers_sig_change(self, client) -> None:
|
||||
client.scroll.return_value = ([self._marker("e1", "ocr=0;t1=pypdfium2")], None)
|
||||
# OCR just enabled -> a new escalation tier exists -> retryable.
|
||||
assert (
|
||||
await dl.is_dead_lettered("520189", "file", "e1", "ocr=1;t1=pypdfium2")
|
||||
is False
|
||||
)
|
||||
|
||||
async def test_false_when_no_marker(self, client) -> None:
|
||||
client.scroll.return_value = ([], None)
|
||||
assert await dl.is_dead_lettered("520189", "file", "e1", "sig1") is False
|
||||
|
||||
async def test_false_on_empty_etag(self, client) -> None:
|
||||
# Cannot content-address without an etag; never dead-lettered.
|
||||
assert await dl.is_dead_lettered("520189", "file", "", "sig1") is False
|
||||
client.scroll.assert_not_awaited()
|
||||
|
||||
async def test_false_on_qdrant_error(self, client) -> None:
|
||||
client.scroll.side_effect = RuntimeError("qdrant down")
|
||||
# Degrade to "process normally" rather than aborting the scan.
|
||||
assert await dl.is_dead_lettered("520189", "file", "e1", "sig1") is False
|
||||
|
||||
async def test_filter_is_user_agnostic(self, client) -> None:
|
||||
client.scroll.return_value = ([self._marker("e1", "sig1")], None)
|
||||
await dl.is_dead_lettered("520189", "file", "e1", "sig1")
|
||||
flt = client.scroll.await_args.kwargs["scroll_filter"]
|
||||
conds = _must_conditions(flt)
|
||||
assert conds == {
|
||||
"doc_id": "520189",
|
||||
"doc_type": "file",
|
||||
dl.DEAD_LETTER_KEY: True,
|
||||
}
|
||||
assert "user_id" not in conds
|
||||
|
||||
|
||||
class TestClearDeadLetter:
|
||||
async def test_deletes_by_marker_filter(self, client) -> None:
|
||||
await dl.clear_dead_letter("520189", "file")
|
||||
client.delete.assert_awaited_once()
|
||||
flt = client.delete.await_args.kwargs["points_selector"]
|
||||
conds = _must_conditions(flt)
|
||||
assert conds[dl.DEAD_LETTER_KEY] is True
|
||||
assert conds["doc_id"] == "520189"
|
||||
|
||||
async def test_failure_is_swallowed(self, client) -> None:
|
||||
client.delete.side_effect = RuntimeError("qdrant down")
|
||||
await dl.clear_dead_letter("520189", "file")
|
||||
@@ -110,6 +110,26 @@ async def test_keeps_own_pod_placeholders(monkeypatch):
|
||||
client.delete.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_keeps_dead_letter_markers(monkeypatch):
|
||||
"""Dead-letter markers (vector/dead_letter.py) reuse is_placeholder=True for
|
||||
the search exclusion but are DURABLE terminal-state records, not in-flight
|
||||
placeholders. They carry a foreign/absent instance_id, so without the
|
||||
carve-out the sweep would delete them on every Pod restart and the
|
||||
dead-lettered document would loop again. They MUST be kept."""
|
||||
monkeypatch.setattr(placeholder_module, "_INSTANCE_ID", "pod-new")
|
||||
marker = SimpleNamespace(
|
||||
id="dl-1",
|
||||
payload={"is_placeholder": True, "dead_letter": True, "instance_id": "pod-old"},
|
||||
)
|
||||
client = _make_client([([marker], None)])
|
||||
|
||||
swept, kept = await sweep_orphan_placeholders(client, "nextcloud_content")
|
||||
|
||||
assert (swept, kept) == (0, 1)
|
||||
client.delete.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_noop_when_no_placeholders_exist(monkeypatch):
|
||||
"""Cold-boot tenant with an empty collection — sweep does NOT
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Unit tests for the processor's terminal-parse-failure dead-lettering.
|
||||
|
||||
When a PDF parse fails permanently (isolated-worker timeout/OOM) and the failing
|
||||
tier has NO higher escalation tier available (e.g. ``structured`` with OCR off),
|
||||
``_index_document`` records a durable, content-addressed dead-letter marker
|
||||
instead of the per-user ``status="failed"`` placeholder mark — the latter could
|
||||
not stop the multi-user re-queue loop. A failure that still has a higher tier
|
||||
keeps the legacy per-user mark.
|
||||
|
||||
The real ``ProcessorRegistry`` singleton is used so the terminal decision
|
||||
(``next_available_tier``) is exercised faithfully; only the parse itself, the
|
||||
content fetch, and the Qdrant side-effects are mocked.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from nextcloud_mcp_server.document_processors.base import ProcessingResult
|
||||
from nextcloud_mcp_server.vector import processor
|
||||
from nextcloud_mcp_server.vector.scanner import DocumentTask
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def _settings(*, ocr_enabled: bool) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
document_ocr_enabled=ocr_enabled,
|
||||
document_tier1_engine="pypdfium2",
|
||||
get_collection_name=lambda: "c",
|
||||
)
|
||||
|
||||
|
||||
def _file_task() -> DocumentTask:
|
||||
return DocumentTask(
|
||||
user_id="Demo-User",
|
||||
doc_id="520189",
|
||||
doc_type="file",
|
||||
operation="index",
|
||||
modified_at=0,
|
||||
file_path="/Plans/big.pdf",
|
||||
etag="etag-1",
|
||||
)
|
||||
|
||||
|
||||
def _nc_client() -> MagicMock:
|
||||
# MagicMock (typed Any) keeps the pre-commit ty-check happy where the real
|
||||
# signature wants a NextcloudClient -- matching the other processor tests.
|
||||
nc = MagicMock()
|
||||
nc.webdav.read_file = AsyncMock(return_value=(b"%PDF-1.4", "application/pdf"))
|
||||
return nc
|
||||
|
||||
|
||||
def _patch_common(mocker, *, ocr_enabled: bool):
|
||||
"""Patch the shared seams; returns the spies for assertions."""
|
||||
mocker.patch.object(
|
||||
processor, "get_settings", lambda: _settings(ocr_enabled=ocr_enabled)
|
||||
)
|
||||
# Never a tenant-wide dedup hit (file was never indexed).
|
||||
mocker.patch.object(
|
||||
processor, "claim_existing_index", AsyncMock(return_value=False)
|
||||
)
|
||||
spies = SimpleNamespace(
|
||||
mark=mocker.patch.object(processor, "mark_dead_letter", AsyncMock()),
|
||||
dead_metric=mocker.patch.object(processor, "record_document_dead_lettered"),
|
||||
delete_ph=mocker.patch.object(
|
||||
processor, "delete_placeholder_point", AsyncMock()
|
||||
),
|
||||
update_ph=mocker.patch.object(
|
||||
processor, "update_placeholder_status", AsyncMock()
|
||||
),
|
||||
)
|
||||
return spies
|
||||
|
||||
|
||||
async def test_terminal_failure_dead_letters(mocker):
|
||||
"""structured tier fails + OCR off (no higher tier) -> dead-letter, not mark."""
|
||||
spies = _patch_common(mocker, ocr_enabled=False)
|
||||
# The per-tier worker runs the structured tier and the parse times out.
|
||||
mocker.patch.object(
|
||||
processor,
|
||||
"_parse_pdf_tier",
|
||||
AsyncMock(
|
||||
return_value=ProcessingResult(
|
||||
text="",
|
||||
metadata={
|
||||
"parse_failed_reason": "timeout",
|
||||
"pipeline_tier": "structured",
|
||||
},
|
||||
processor="pymupdf",
|
||||
success=False,
|
||||
error="isolated parse failed (timeout)",
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
result = await processor._index_document(
|
||||
_file_task(), _nc_client(), MagicMock(), tier="structured"
|
||||
)
|
||||
|
||||
assert result is False
|
||||
spies.mark.assert_awaited_once()
|
||||
# Marker is content-addressed with this etag + the OCR-off tiers signature.
|
||||
args = spies.mark.await_args.args
|
||||
assert args[0] == "520189" and args[1] == "file"
|
||||
assert args[2] == "etag-1" # etag
|
||||
assert "ocr=0" in args[3] # tiers_sig
|
||||
assert args[4] == "timeout" # reason
|
||||
spies.dead_metric.assert_called_once_with("timeout")
|
||||
spies.delete_ph.assert_awaited_once() # volatile placeholder dropped
|
||||
spies.update_ph.assert_not_awaited() # NOT the legacy per-user failed mark
|
||||
|
||||
|
||||
async def test_non_terminal_failure_keeps_legacy_mark(mocker):
|
||||
"""fast tier fails while structured is still available -> legacy failed mark."""
|
||||
spies = _patch_common(mocker, ocr_enabled=False)
|
||||
mocker.patch.object(
|
||||
processor,
|
||||
"_parse_pdf_tier",
|
||||
AsyncMock(
|
||||
return_value=ProcessingResult(
|
||||
text="",
|
||||
metadata={"parse_failed_reason": "error", "pipeline_tier": "fast"},
|
||||
processor="pypdfium2",
|
||||
success=False,
|
||||
error="isolated parse failed (error)",
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
result = await processor._index_document(
|
||||
_file_task(), _nc_client(), MagicMock(), tier="fast"
|
||||
)
|
||||
|
||||
assert result is False
|
||||
spies.update_ph.assert_awaited_once() # legacy per-user failed mark
|
||||
spies.mark.assert_not_awaited() # NOT dead-lettered (structured can still run)
|
||||
spies.dead_metric.assert_not_called()
|
||||
Reference in New Issue
Block a user