Merge pull request #920 from cbcoutinho/fix/dead-letter-terminal-parse-failures

fix(vector): dead-letter terminally-failed documents to stop multi-user re-queue loop

nextcloud_mcp_server/document_processors/escalation.py

@@ -22,13 +22,43 @@ mapping lives in the queue layer, which imports :class:`EscalateError` from here
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import Literal
+from typing import Any, Literal
 
 # Cheapest-first. ``llm`` is reserved (see base.DocumentProcessor.tier) and not
 # wired yet, so it is intentionally absent from the live ladder.
 TIER_LADDER: tuple[str, ...] = ("fast", "structured", "ocr")
 
 
+def escalation_tiers_signature(settings: Any) -> str:
+    """A stable string fingerprint of the runtime escalation-tier configuration.
+
+    Used by the document dead-letter marker (``vector/dead_letter.py``) as part of
+    its content key: a document that fails its terminal tier is dead-lettered until
+    either its content (etag) OR this signature changes. The signature therefore
+    captures every setting that can make a *new* escalation tier become available
+    at runtime — flip it and previously dead-lettered documents become retryable.
+
+    Derived purely from settings (not the live ``ProcessorRegistry``) so it is
+    identical across the API/scanner and worker roles: the *registered* processor
+    set is build-constant, so the only runtime variables are the OCR-enabled gate
+    (``document_ocr_enabled`` — the single tier toggle today) and the tier-1 engine
+    pin (``document_tier1_engine``). Enabling OCR changes the signature, so the
+    pathological-but-OCR-recoverable documents dead-lettered while OCR was off are
+    re-attempted automatically.
+
+    TODO: when a future setting can make a previously-terminal document parseable,
+    fold it in here so raising it auto-retries existing dead-letters. Two known
+    candidates: a new escalation tier becoming toggleable (e.g. the reserved
+    ``llm`` rung in ``TIER_LADDER``), and a raised oversize cap (an oversize PDF is
+    always-terminal, so without the cap in this signature it stays dead-lettered
+    until its etag changes even after an operator allows bigger files).
+    """
+    return (
+        f"ocr={int(bool(settings.document_ocr_enabled))};"
+        f"t1={settings.document_tier1_engine}"
+    )
+
+
 @dataclass(frozen=True)
 class EscalationDecision:
     """Outcome of the post-parse quality gate (``ProcessorRegistry.evaluate_escalation``).

nextcloud_mcp_server/observability/metrics.py

@@ -305,6 +305,20 @@ document_parse_failed_total = Counter(
     ["reason"],  # reason: timeout | oom | error
 )
 
+# Documents dead-lettered after a terminal parse failure: the failing tier had
+# no higher escalation tier available (e.g. structured timed out with OCR off),
+# so the document is recorded as permanently failed for this content-version and
+# stops being re-queued (vector/dead_letter.py). Distinct from
+# ``document_parse_failed_total`` (which counts every failed parse attempt,
+# including the ones that will be retried) -- this fires once when a document
+# is given up on, and clears implicitly when its etag or the escalation-tier set
+# changes and it is re-attempted.
+document_dead_lettered_total = Counter(
+    "astrolabe_document_dead_lettered_total",
+    "Documents dead-lettered after a terminal parse failure (no escalation tier)",
+    ["reason"],  # reason: timeout | oom | error | oversize
+)
+
 # Documents dropped after exhausting in-process indexing retries (the scanner
 # re-picks them on a later full scan, so this is "dropped for this cycle", not
 # "lost forever"). Labelled by classified cause so the embed-drop rate from a
@@ -787,6 +801,22 @@ def record_document_parse_failed(reason: str) -> None:
     document_parse_failed_total.labels(reason=reason).inc()
 
 
+def record_document_dead_lettered(reason: str) -> None:
+    """Record a document dead-lettered after a terminal parse failure.
+
+    Counts the dead-letter *attempt*: it is incremented alongside the
+    ``mark_dead_letter`` call, which is fail-safe (a Qdrant write error is logged,
+    not raised), so a transient write failure can leave this counter marginally
+    above the live marker count in Qdrant.
+
+    Args:
+        reason: ``timeout`` | ``oom`` | ``error`` (the terminal parse failure
+            reason carried from the isolated worker) or ``oversize`` (rejected by
+            the pre-parse size guard, which no tier can ever parse).
+    """
+    document_dead_lettered_total.labels(reason=reason).inc()
+
+
 def record_ingest_dropped(reason: str) -> None:
     """Record a document dropped after exhausting in-process indexing retries.
 

nextcloud_mcp_server/vector/dead_letter.py

@@ -0,0 +1,207 @@
+"""Content-addressed dead-letter markers for terminally-failed documents.
+
+A document that fails its terminal extraction tier (a hard parse failure with no
+higher tier to escalate to — e.g. the ``structured`` tier timing out while OCR is
+disabled) must not be retried forever. The per-user placeholder ``status="failed"``
+mark cannot stop the loop on its own: placeholder point IDs are user-agnostic
+(``uuid5("file:<doc_id>:placeholder")``) but the scanner's freshness gate filters
+by ``user_id``, so for a file visible to *several* users the single shared
+placeholder's ``user_id`` is overwritten by whoever scanned last and every other
+user's scan sees "no record → re-queue", re-burning the (failing) parse on a loop.
+
+This module records a **durable, content-addressed, user-agnostic** dead-letter
+marker instead. One marker point per document (a distinct deterministic ID, kept
+separate from the in-flight placeholder), carrying the ``etag`` and an escalation
+``tiers_sig`` (see ``document_processors.escalation.escalation_tiers_signature``).
+The scanner consults it tenant-wide — for every user — and skips re-queuing while
+BOTH still match, so the document is attempted once per content-version and never
+loops. A content change (new ``etag``) or a config change that adds an escalation
+tier (e.g. enabling OCR — new ``tiers_sig``) makes the marker stale and the
+document retryable again.
+
+The marker carries ``is_placeholder=True`` so the existing search exclusion
+(``get_placeholder_filter``) keeps it out of user-facing results with no extra
+filter, plus ``dead_letter=True`` so the orphan-placeholder sweep and the scanner
+can tell it apart from a volatile in-flight placeholder. Mirrors the fail-safe
+philosophy of ``sharing_state``: a Qdrant error never aborts ingest — a failed
+lookup degrades to "process normally", a failed write is logged, not raised.
+
+TODO(deck-349): a marker for a file that is dead-lettered and *then* deleted from
+Nextcloud can be orphaned. The processor's delete path clears it, but the
+scanner's grace-period deletion tracking only sees a file via its real indexed
+points (filtered by ``user_id``); a dead-lettered file has only this
+user-agnostic marker, so its disappearance enqueues no delete task and the marker
+is never reached. Not a correctness issue (search excludes ``is_placeholder=True``
+and the etag check means a stale marker never blocks new content), but it
+accumulates. A dedicated marker sweep or a TTL payload field would close it —
+tracked as a follow-up, not done here.
+"""
+
+import logging
+import time
+import uuid
+from typing import Any
+
+from qdrant_client import models
+from qdrant_client.models import FieldCondition, Filter, MatchValue, PointStruct
+
+from nextcloud_mcp_server.config import get_settings
+from nextcloud_mcp_server.embedding import get_embedding_service
+from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client
+
+logger = logging.getLogger(__name__)
+
+# Payload flag distinguishing a durable dead-letter marker from an in-flight
+# placeholder (both carry is_placeholder=True to inherit the search exclusion).
+DEAD_LETTER_KEY = "dead_letter"
+
+
+def _generate_dead_letter_id(doc_type: str, doc_id: str) -> str:
+    """Deterministic, user-agnostic point ID for a document's dead-letter marker.
+
+    Distinct from the in-flight placeholder ID (``…:placeholder``) so the two can
+    coexist briefly and never collide; one marker per ``(doc_type, doc_id)``, so
+    a re-failure upserts in place rather than accumulating.
+    """
+    return str(uuid.uuid5(uuid.NAMESPACE_DNS, f"{doc_type}:{doc_id}:deadletter"))
+
+
+def _dead_letter_filter(doc_id: str, doc_type: str) -> Filter:
+    """Match the dead-letter marker for one document (tenant-wide, no user_id).
+
+    Includes ``is_placeholder=True`` (redundant with ``dead_letter=True``, which
+    nothing else sets) so Qdrant can lean on the existing ``is_placeholder``
+    payload index.
+    """
+    return Filter(
+        must=[
+            FieldCondition(key="doc_id", match=MatchValue(value=doc_id)),
+            FieldCondition(key="doc_type", match=MatchValue(value=doc_type)),
+            FieldCondition(key="is_placeholder", match=MatchValue(value=True)),
+            FieldCondition(key=DEAD_LETTER_KEY, match=MatchValue(value=True)),
+        ]
+    )
+
+
+async def mark_dead_letter(
+    doc_id: str,
+    doc_type: str,
+    etag: str,
+    tiers_sig: str,
+    reason: str,
+    *,
+    file_path: str | None = None,
+) -> None:
+    """Upsert a durable dead-letter marker for a terminally-failed document.
+
+    Keyed by content (``etag``) + escalation config (``tiers_sig``); the scanner
+    skips re-queuing while both match. ``reason`` is the parse failure reason
+    (``timeout`` | ``oom`` | ``error``). Fail-safe: a Qdrant error is logged, not
+    raised — a missed mark just means the document is retried (the bounded prior
+    behaviour), never a crash.
+    """
+    try:
+        qdrant_client = await get_qdrant_client()
+        settings = get_settings()
+        dimension = get_embedding_service().get_dimension()
+
+        payload: dict[str, Any] = {
+            "doc_id": doc_id,
+            "doc_type": doc_type,
+            "is_placeholder": True,
+            DEAD_LETTER_KEY: True,
+            "etag": etag,
+            "tiers_sig": tiers_sig,
+            "reason": reason,
+            "failed_at": int(time.time()),
+        }
+        if doc_type == "file" and file_path:
+            payload["file_path"] = file_path
+
+        point = PointStruct(
+            id=_generate_dead_letter_id(doc_type, doc_id),
+            vector={
+                "dense": [0.0] * dimension,
+                "sparse": models.SparseVector(indices=[], values=[]),
+            },
+            payload=payload,
+        )
+        await qdrant_client.upsert(
+            collection_name=settings.get_collection_name(),
+            points=[point],
+            wait=True,
+        )
+        logger.info(
+            "Dead-lettered %s_%s (reason=%s, etag=%s)",
+            doc_type,
+            doc_id,
+            reason,
+            etag,
+        )
+    except Exception as e:
+        logger.warning(
+            "Failed to write dead-letter marker for %s_%s: %s", doc_type, doc_id, e
+        )
+        # Don't raise — dead-lettering is best-effort; a miss just retries.
+
+
+async def is_dead_lettered(
+    doc_id: str,
+    doc_type: str,
+    etag: str,
+    tiers_sig: str,
+) -> bool:
+    """Whether this exact content is currently dead-lettered (skip re-queuing).
+
+    Returns True only when a marker exists for ``(doc_id, doc_type)`` whose stored
+    ``etag`` AND ``tiers_sig`` both match the current values — so a content change
+    or a new escalation tier (e.g. OCR enabled) makes the document retryable
+    again. An empty ``etag`` is never dead-lettered (we cannot content-address it).
+    Fail-safe: a Qdrant error degrades to False (process normally), mirroring
+    ``sharing_state.claim_existing_index``.
+    """
+    if not etag:
+        return False
+    try:
+        qdrant_client = await get_qdrant_client()
+        settings = get_settings()
+        points, _ = await qdrant_client.scroll(
+            collection_name=settings.get_collection_name(),
+            scroll_filter=_dead_letter_filter(doc_id, doc_type),
+            limit=1,
+            with_payload=True,
+            with_vectors=False,
+        )
+    except Exception as e:
+        logger.warning(
+            "Dead-letter lookup failed for %s_%s (%s); processing normally",
+            doc_type,
+            doc_id,
+            e,
+        )
+        return False
+    if not points:
+        return False
+    payload = dict(points[0].payload or {})
+    return payload.get("etag") == etag and payload.get("tiers_sig") == tiers_sig
+
+
+async def clear_dead_letter(doc_id: str, doc_type: str) -> None:
+    """Delete a document's dead-letter marker (on successful index / release).
+
+    Idempotent and fail-safe: deleting a non-existent marker is a Qdrant no-op,
+    and an error is logged rather than raised so it never breaks the indexing
+    path that calls it.
+    """
+    try:
+        qdrant_client = await get_qdrant_client()
+        settings = get_settings()
+        await qdrant_client.delete(
+            collection_name=settings.get_collection_name(),
+            points_selector=_dead_letter_filter(doc_id, doc_type),
+        )
+        logger.debug("Cleared dead-letter marker for %s_%s", doc_type, doc_id)
+    except Exception as e:
+        logger.warning(
+            "Failed to clear dead-letter marker for %s_%s: %s", doc_type, doc_id, e
+        )

nextcloud_mcp_server/vector/placeholder.py

@@ -397,6 +397,15 @@ async def sweep_orphan_placeholders(
         orphan_ids = []
         for point in points:
             payload = point.payload or {}
+            # Dead-letter markers (vector/dead_letter.py) reuse is_placeholder=True
+            # for the search exclusion but are DURABLE terminal-state records, not
+            # in-flight placeholders -- they carry no/foreign instance_id and must
+            # survive a Pod restart, so never sweep them as orphans.
+            if payload.get("dead_letter") is True:
+                # Tenant-wide and always kept (not Pod-scoped); counted under
+                # ``kept`` only because the sweep's tally has no separate bucket.
+                kept += 1
+                continue
             point_instance = payload.get("instance_id")
             if point_instance == _INSTANCE_ID:
                 kept += 1

nextcloud_mcp_server/vector/processor.py

@@ -28,6 +28,7 @@ from nextcloud_mcp_server.embedding import get_bm25_service, get_embedding_servi
 from nextcloud_mcp_server.models.deck import DeckCard
 from nextcloud_mcp_server.observability.metrics import (
     record_document_chunks,
+    record_document_dead_lettered,
     record_document_escalation,
     record_document_escalation_suppressed,
     record_document_parse_failed,
@@ -43,6 +44,10 @@ from nextcloud_mcp_server.search.pdf_highlighter import PDFHighlighter
 from nextcloud_mcp_server.usage import UsageEventStore
 from nextcloud_mcp_server.vector import payload_keys
 from nextcloud_mcp_server.vector._errors import format_exception_group
+from nextcloud_mcp_server.vector.dead_letter import (
+    clear_dead_letter,
+    mark_dead_letter,
+)
 from nextcloud_mcp_server.vector.document_chunker import (
     DocumentChunker,
     PageAwareChunker,
@@ -621,6 +626,13 @@ async def process_document(
                 await release_document_for_user(
                     doc_task.doc_id, doc_task.doc_type, doc_task.user_id
                 )
+                # Drop any dead-letter marker for the file too: release only
+                # removes it when the last reader leaves (its filter misses the
+                # user-agnostic, principal-less marker), so without this a
+                # dead-lettered-then-deleted file would leave an orphan marker
+                # accumulating in Qdrant. Only files are ever dead-lettered.
+                if doc_task.doc_type == "file":
+                    await clear_dead_letter(doc_task.doc_id, doc_task.doc_type)
                 logger.info(
                     "Deleted %s_%s for %s",
                     doc_task.doc_type,
@@ -993,8 +1005,10 @@ async def _index_document(
                 get_registry,
             )
             from nextcloud_mcp_server.document_processors.escalation import (  # noqa: PLC0415
+                TIER_LADDER,
                 BatchPending,
                 EscalateError,
+                escalation_tiers_signature,
             )
 
             registry = get_registry()
@@ -1035,34 +1049,101 @@ async def _index_document(
                 # A permanent parse failure (e.g. an isolated-worker OOM/timeout
                 # on a pathological PDF) returns success=False rather than
                 # raising -- there is nothing to index and retrying would just
-                # fail again. Mark the placeholder "failed" so the scanner stops
+                # fail again.
-                # re-queuing it (until the file changes) and return False so the
-                # caller skips the success metrics (it was not indexed).
                 if not result.success:
                     reason = result.metadata.get("parse_failed_reason", "error")
                     record_document_parse_failed(reason)
-                    logger.warning(
+                    # The tier that produced this failed result: the worker's own
-                        "Permanent parse failure for %s (reason=%s); marking "
+                    # tier on the per-tier path, else the deepest tier the inline
-                        "failed and skipping index",
+                    # pipeline reached (recorded as ``pipeline_tier``).
-                        file_path,
+                    failing_tier = tier or result.metadata.get(
-                        reason,
+                        "pipeline_tier", TIER_LADDER[0]
                     )
-                    try:
+                    # An oversize PDF is rejected by the pre-parse size guard
-                        await update_placeholder_status(
+                    # before any tier runs (no pipeline_tier stamped on the inline
-                            doc_id=doc_task.doc_id,
+                    # path) and no tier can ever parse it, so it is terminal
-                            doc_type=doc_task.doc_type,
+                    # regardless of failing_tier. Otherwise, terminal == no higher
-                            user_id=doc_task.user_id,
+                    # tier can run.
-                            status="failed",
+                    terminal = (
-                        )
+                        reason == "oversize"
-                    except Exception:
+                        or registry.next_available_tier(failing_tier, settings) is None
-                        # Best-effort: a transient Qdrant error here only means
+                    )
-                        # the placeholder isn't marked, so the scanner retries
+                    if terminal and doc_task.etag:
-                        # the (still un-indexable) file later -- not fatal.
+                        # No higher tier can run (e.g. structured timed out with
-                        logger.debug(
+                        # OCR off), so retrying just re-burns the same failing
-                            "Could not mark placeholder failed for %s",
+                        # parse. Dead-letter the document tenant-wide
+                        # (content-addressed, user-agnostic) so EVERY user's scan
+                        # stops re-queuing it until its content (etag) or the
+                        # escalation-tier set (e.g. OCR enabled -> new tiers_sig)
+                        # changes. This fixes the multi-user placeholder
+                        # ping-pong the per-user "failed" mark could not: a file
+                        # shared by N users has ONE user-agnostic placeholder
+                        # whose user_id is overwritten by the last scanner, so
+                        # every other user re-queued it forever. Requires an etag
+                        # to content-address the marker; without one (rare) we
+                        # fall back to the legacy per-user mark below.
+                        await mark_dead_letter(
                             doc_task.doc_id,
-                            exc_info=True,
+                            doc_task.doc_type,
+                            doc_task.etag,
+                            escalation_tiers_signature(settings),
+                            reason,
+                            file_path=file_path,
                         )
+                        record_document_dead_lettered(reason)
+                        logger.warning(
+                            "Permanent parse failure for %s (reason=%s); "
+                            "dead-lettered (terminal tier=%s, no escalation) and "
+                            "skipping index",
+                            file_path,
+                            reason,
+                            failing_tier,
+                        )
+                        # Drop the volatile in-flight placeholder; the durable
+                        # marker is now the document's terminal-state record.
+                        try:
+                            await delete_placeholder_point(
+                                doc_id=doc_task.doc_id,
+                                doc_type=doc_task.doc_type,
+                                user_id=doc_task.user_id,
+                            )
+                        except Exception:
+                            # A real Qdrant I/O failure (not control-flow): warn so
+                            # it's observable. Non-fatal -- the durable dead-letter
+                            # marker is already written, so the leftover volatile
+                            # placeholder is merely redundant.
+                            logger.warning(
+                                "Could not delete placeholder for dead-lettered %s",
+                                doc_task.doc_id,
+                                exc_info=True,
+                            )
+                    else:
+                        # Either a higher tier exists (parse failures don't
+                        # escalate to it today) or there's no etag to
+                        # content-address a dead-letter marker. Keep the legacy
+                        # per-user "failed" placeholder mark.
+                        logger.warning(
+                            "Permanent parse failure for %s (reason=%s); marking "
+                            "failed and skipping index",
+                            file_path,
+                            reason,
+                        )
+                        try:
+                            await update_placeholder_status(
+                                doc_id=doc_task.doc_id,
+                                doc_type=doc_task.doc_type,
+                                user_id=doc_task.user_id,
+                                status="failed",
+                            )
+                        except Exception:
+                            # Best-effort: a transient Qdrant error here only
+                            # means the placeholder isn't marked, so the scanner
+                            # retries the (still un-indexable) file later.
+                            logger.debug(
+                                "Could not mark placeholder failed for %s",
+                                doc_task.doc_id,
+                                exc_info=True,
+                            )
                     return False
 
                 content = result.text
@@ -1529,6 +1610,17 @@ async def _index_document(
             )
         )
 
+    # A successful (re-)index supersedes any prior terminal failure: clear a
+    # stale dead-letter marker (e.g. the file was fixed/replaced, or a new
+    # escalation tier finally parsed it) so it isn't left behind. Only files are
+    # ever dead-lettered, and only with a non-empty etag (is_dead_lettered
+    # early-returns without one), so skip the extra Qdrant round-trip otherwise.
+    # Cleared before the real-chunk upsert below: if that upsert then fails
+    # transiently, the document is re-queued and re-parses once on the next scan
+    # (an extra parse, never a silent drop) -- the safe ordering.
+    if doc_task.doc_type == "file" and doc_task.etag:
+        await clear_dead_letter(doc_task.doc_id, doc_task.doc_type)
+
     # Delete placeholder before writing real vectors
     # This prevents duplicates and cleans up the placeholder state
     try:

nextcloud_mcp_server/vector/scanner.py

@@ -27,6 +27,7 @@ from nextcloud_mcp_server.server.tag_exclusion import (
     get_excluded_file_paths,
     is_path_excluded,
 )
+from nextcloud_mcp_server.vector.dead_letter import is_dead_lettered
 from nextcloud_mcp_server.vector.placeholder import (
     query_document_metadata,
     write_placeholder_point,
@@ -664,6 +665,16 @@ async def scan_user_documents(
                         skipped,
                     )
 
+            # Escalation-tier fingerprint for the dead-letter skip below, computed
+            # once per scan. Lazy import: document_processors.__init__ pulls the
+            # heavy parse stack (pymupdf/_isolation, Unix-only ``resource``; #877),
+            # which the scanner (API role) must not load at module import.
+            from nextcloud_mcp_server.document_processors.escalation import (  # noqa: PLC0415
+                escalation_tiers_signature,
+            )
+
+            tiers_sig = escalation_tiers_signature(get_settings())
+
             for file_info in tagged_files:
                 # Files are already filtered by MIME type in find_files_by_tag()
                 file_count += 1
@@ -703,6 +714,24 @@ async def scan_user_documents(
                     )
                     continue
 
+                # Tenant-wide dead-letter skip: a document that terminally failed
+                # parsing (no escalation tier) is recorded user-agnostically, so
+                # EVERY user's scan skips re-queuing it until its content (etag) or
+                # the escalation-tier set (tiers_sig, e.g. OCR enabled) changes.
+                # Unlike the per-user placeholder "failed" mark this is not
+                # defeated by a file shared across users -- whose single
+                # user-agnostic placeholder's user_id is overwritten by the last
+                # scanner, so every other user re-queued it on a loop.
+                if etag and await is_dead_lettered(file_id, "file", etag, tiers_sig):
+                    _potentially_deleted.pop((user_id, file_id), None)
+                    logger.debug(
+                        "Skipping dead-lettered file %s (ID: %s) until content/"
+                        "tier change",
+                        file_path,
+                        file_id,
+                    )
+                    continue
+
                 if initial_sync:
                     # Send everything on first sync - write placeholder first
                     await write_placeholder_point(

tests/unit/test_escalation_signature.py

@@ -0,0 +1,42 @@
+"""Unit tests for the escalation-tier signature used by dead-letter keying.
+
+``escalation_tiers_signature`` fingerprints the runtime escalation config so a
+dead-lettered document becomes retryable when a new tier appears (e.g. an
+operator enables OCR). It must be settings-derived (role-independent) and must
+change when OCR is toggled.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+from nextcloud_mcp_server.document_processors.escalation import (
+    escalation_tiers_signature,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def _settings(*, ocr: bool, engine: str = "pypdfium2") -> SimpleNamespace:
+    return SimpleNamespace(document_ocr_enabled=ocr, document_tier1_engine=engine)
+
+
+def test_signature_is_stable_for_same_config() -> None:
+    assert escalation_tiers_signature(
+        _settings(ocr=False)
+    ) == escalation_tiers_signature(_settings(ocr=False))
+
+
+def test_enabling_ocr_changes_signature() -> None:
+    # Enabling OCR adds an escalation tier -> previously dead-lettered docs retry.
+    assert escalation_tiers_signature(
+        _settings(ocr=False)
+    ) != escalation_tiers_signature(_settings(ocr=True))
+
+
+def test_tier1_engine_change_changes_signature() -> None:
+    assert escalation_tiers_signature(
+        _settings(ocr=False, engine="pypdfium2")
+    ) != escalation_tiers_signature(_settings(ocr=False, engine="pymupdf"))

tests/unit/vector/test_dead_letter.py

@@ -0,0 +1,171 @@
+"""Unit tests for content-addressed dead-letter markers.
+
+Covers vector/dead_letter.py: the durable, user-agnostic terminal-failure marker
+that stops a multi-user shared file (whose single placeholder's user_id is
+overwritten by the last scanner) from being re-queued forever. A marker is keyed
+by content (``etag``) + escalation config (``tiers_sig``); a scan skips only while
+both still match, so a content change or a newly-available tier (e.g. OCR enabled)
+makes the document retryable again.
+
+Qdrant is reached via ``get_qdrant_client``/``get_settings``/``get_embedding_service``,
+all monkeypatched here so the logic runs without a live Qdrant.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from nextcloud_mcp_server.vector import dead_letter as dl
+
+pytestmark = pytest.mark.unit
+
+_COLLECTION = "test_collection"
+
+
+class _Settings:
+    def get_collection_name(self) -> str:
+        return _COLLECTION
+
+
+def _point(payload: dict) -> SimpleNamespace:
+    """Stand-in for a qdrant_client Record (only id/payload are read)."""
+    return SimpleNamespace(id="pt", payload=payload)
+
+
+@pytest.fixture
+def client(monkeypatch) -> AsyncMock:
+    """An AsyncMock Qdrant client wired into dead_letter, plus stub deps.
+
+    ``scroll`` defaults to "no points"; individual tests override
+    ``client.scroll.return_value``/``side_effect``.
+    """
+    qc = AsyncMock()
+    qc.scroll.return_value = ([], None)
+    monkeypatch.setattr(dl, "get_qdrant_client", AsyncMock(return_value=qc))
+    monkeypatch.setattr(dl, "get_settings", lambda: _Settings())
+    monkeypatch.setattr(
+        dl, "get_embedding_service", lambda: SimpleNamespace(get_dimension=lambda: 4)
+    )
+    return qc
+
+
+def _must_conditions(flt) -> dict:
+    """Map FieldCondition key -> matched value for a Filter's ``must`` clause."""
+    out = {}
+    for c in flt.must or []:
+        key = getattr(c, "key", None)
+        match = getattr(c, "match", None)
+        out[key] = getattr(match, "value", None)
+    return out
+
+
+class TestDeadLetterId:
+    def test_user_agnostic_and_distinct_from_placeholder(self) -> None:
+        from nextcloud_mcp_server.vector.placeholder import _generate_placeholder_id
+
+        dl_id = dl._generate_dead_letter_id("file", "520189")
+        # Deterministic + user-agnostic (depends only on doc_type:doc_id).
+        assert dl_id == dl._generate_dead_letter_id("file", "520189")
+        # Never collides with the in-flight placeholder for the same document.
+        assert dl_id != _generate_placeholder_id("file", "520189")
+
+
+class TestMarkDeadLetter:
+    async def test_upserts_content_addressed_marker(self, client) -> None:
+        await dl.mark_dead_letter(
+            "520189",
+            "file",
+            "etag-1",
+            "ocr=0;t1=pypdfium2",
+            "timeout",
+            file_path="/Plans/big.pdf",
+        )
+        client.upsert.assert_awaited_once()
+        point = client.upsert.await_args.kwargs["points"][0]
+        assert point.id == dl._generate_dead_letter_id("file", "520189")
+        payload = point.payload
+        assert payload["is_placeholder"] is True
+        assert payload[dl.DEAD_LETTER_KEY] is True
+        assert payload["etag"] == "etag-1"
+        assert payload["tiers_sig"] == "ocr=0;t1=pypdfium2"
+        assert payload["reason"] == "timeout"
+        assert payload["doc_id"] == "520189"
+        assert payload["file_path"] == "/Plans/big.pdf"
+
+    async def test_failure_is_swallowed(self, client) -> None:
+        client.upsert.side_effect = RuntimeError("qdrant down")
+        # Best-effort: a write failure must not propagate (would crash the worker).
+        await dl.mark_dead_letter("1", "file", "e", "sig", "oom")
+
+
+class TestIsDeadLettered:
+    def _marker(self, etag: str, tiers_sig: str) -> SimpleNamespace:
+        return _point(
+            {
+                "doc_id": "520189",
+                dl.DEAD_LETTER_KEY: True,
+                "etag": etag,
+                "tiers_sig": tiers_sig,
+            }
+        )
+
+    async def test_true_when_etag_and_sig_match(self, client) -> None:
+        client.scroll.return_value = ([self._marker("e1", "sig1")], None)
+        assert await dl.is_dead_lettered("520189", "file", "e1", "sig1") is True
+
+    async def test_false_on_etag_change(self, client) -> None:
+        client.scroll.return_value = ([self._marker("e1", "sig1")], None)
+        # File content changed -> retryable.
+        assert await dl.is_dead_lettered("520189", "file", "e2", "sig1") is False
+
+    async def test_false_on_tiers_sig_change(self, client) -> None:
+        client.scroll.return_value = ([self._marker("e1", "ocr=0;t1=pypdfium2")], None)
+        # OCR just enabled -> a new escalation tier exists -> retryable.
+        assert (
+            await dl.is_dead_lettered("520189", "file", "e1", "ocr=1;t1=pypdfium2")
+            is False
+        )
+
+    async def test_false_when_no_marker(self, client) -> None:
+        client.scroll.return_value = ([], None)
+        assert await dl.is_dead_lettered("520189", "file", "e1", "sig1") is False
+
+    async def test_false_on_empty_etag(self, client) -> None:
+        # Cannot content-address without an etag; never dead-lettered.
+        assert await dl.is_dead_lettered("520189", "file", "", "sig1") is False
+        client.scroll.assert_not_awaited()
+
+    async def test_false_on_qdrant_error(self, client) -> None:
+        client.scroll.side_effect = RuntimeError("qdrant down")
+        # Degrade to "process normally" rather than aborting the scan.
+        assert await dl.is_dead_lettered("520189", "file", "e1", "sig1") is False
+
+    async def test_filter_is_user_agnostic(self, client) -> None:
+        client.scroll.return_value = ([self._marker("e1", "sig1")], None)
+        await dl.is_dead_lettered("520189", "file", "e1", "sig1")
+        flt = client.scroll.await_args.kwargs["scroll_filter"]
+        conds = _must_conditions(flt)
+        assert conds == {
+            "doc_id": "520189",
+            "doc_type": "file",
+            "is_placeholder": True,
+            dl.DEAD_LETTER_KEY: True,
+        }
+        assert "user_id" not in conds
+
+
+class TestClearDeadLetter:
+    async def test_deletes_by_marker_filter(self, client) -> None:
+        await dl.clear_dead_letter("520189", "file")
+        client.delete.assert_awaited_once()
+        flt = client.delete.await_args.kwargs["points_selector"]
+        conds = _must_conditions(flt)
+        assert conds[dl.DEAD_LETTER_KEY] is True
+        assert conds["doc_id"] == "520189"
+
+    async def test_failure_is_swallowed(self, client) -> None:
+        client.delete.side_effect = RuntimeError("qdrant down")
+        await dl.clear_dead_letter("520189", "file")

tests/unit/vector/test_placeholder.py

@@ -110,6 +110,26 @@ async def test_keeps_own_pod_placeholders(monkeypatch):
     client.delete.assert_not_awaited()
 
 
+@pytest.mark.unit
+async def test_keeps_dead_letter_markers(monkeypatch):
+    """Dead-letter markers (vector/dead_letter.py) reuse is_placeholder=True for
+    the search exclusion but are DURABLE terminal-state records, not in-flight
+    placeholders. They carry a foreign/absent instance_id, so without the
+    carve-out the sweep would delete them on every Pod restart and the
+    dead-lettered document would loop again. They MUST be kept."""
+    monkeypatch.setattr(placeholder_module, "_INSTANCE_ID", "pod-new")
+    marker = SimpleNamespace(
+        id="dl-1",
+        payload={"is_placeholder": True, "dead_letter": True, "instance_id": "pod-old"},
+    )
+    client = _make_client([([marker], None)])
+
+    swept, kept = await sweep_orphan_placeholders(client, "nextcloud_content")
+
+    assert (swept, kept) == (0, 1)
+    client.delete.assert_not_awaited()
+
+
 @pytest.mark.unit
 async def test_noop_when_no_placeholders_exist(monkeypatch):
     """Cold-boot tenant with an empty collection — sweep does NOT

tests/unit/vector/test_processor_dead_letter.py

@@ -0,0 +1,224 @@
+"""Unit tests for the processor's terminal-parse-failure dead-lettering.
+
+When a PDF parse fails permanently (isolated-worker timeout/OOM) and the failing
+tier has NO higher escalation tier available (e.g. ``structured`` with OCR off),
+``_index_document`` records a durable, content-addressed dead-letter marker
+instead of the per-user ``status="failed"`` placeholder mark — the latter could
+not stop the multi-user re-queue loop. A failure that still has a higher tier
+keeps the legacy per-user mark.
+
+The real ``ProcessorRegistry`` singleton is used so the terminal decision
+(``next_available_tier``) is exercised faithfully; only the parse itself, the
+content fetch, and the Qdrant side-effects are mocked.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from nextcloud_mcp_server.document_processors.base import ProcessingResult
+from nextcloud_mcp_server.vector import processor
+from nextcloud_mcp_server.vector.scanner import DocumentTask
+
+pytestmark = pytest.mark.unit
+
+
+def _settings(*, ocr_enabled: bool) -> SimpleNamespace:
+    return SimpleNamespace(
+        document_ocr_enabled=ocr_enabled,
+        document_tier1_engine="pypdfium2",
+        get_collection_name=lambda: "c",
+    )
+
+
+def _file_task() -> DocumentTask:
+    return DocumentTask(
+        user_id="Demo-User",
+        doc_id="520189",
+        doc_type="file",
+        operation="index",
+        modified_at=0,
+        file_path="/Plans/big.pdf",
+        etag="etag-1",
+    )
+
+
+def _nc_client() -> MagicMock:
+    # MagicMock (typed Any) keeps the pre-commit ty-check happy where the real
+    # signature wants a NextcloudClient -- matching the other processor tests.
+    nc = MagicMock()
+    nc.webdav.read_file = AsyncMock(return_value=(b"%PDF-1.4", "application/pdf"))
+    return nc
+
+
+def _patch_common(mocker, *, ocr_enabled: bool):
+    """Patch the shared seams; returns the spies for assertions."""
+    mocker.patch.object(
+        processor, "get_settings", lambda: _settings(ocr_enabled=ocr_enabled)
+    )
+    # Never a tenant-wide dedup hit (file was never indexed).
+    mocker.patch.object(
+        processor, "claim_existing_index", AsyncMock(return_value=False)
+    )
+    spies = SimpleNamespace(
+        mark=mocker.patch.object(processor, "mark_dead_letter", AsyncMock()),
+        dead_metric=mocker.patch.object(processor, "record_document_dead_lettered"),
+        delete_ph=mocker.patch.object(
+            processor, "delete_placeholder_point", AsyncMock()
+        ),
+        update_ph=mocker.patch.object(
+            processor, "update_placeholder_status", AsyncMock()
+        ),
+    )
+    return spies
+
+
+async def test_terminal_failure_dead_letters(mocker):
+    """structured tier fails + OCR off (no higher tier) -> dead-letter, not mark."""
+    spies = _patch_common(mocker, ocr_enabled=False)
+    # The per-tier worker runs the structured tier and the parse times out.
+    mocker.patch.object(
+        processor,
+        "_parse_pdf_tier",
+        AsyncMock(
+            return_value=ProcessingResult(
+                text="",
+                metadata={
+                    "parse_failed_reason": "timeout",
+                    "pipeline_tier": "structured",
+                },
+                processor="pymupdf",
+                success=False,
+                error="isolated parse failed (timeout)",
+            )
+        ),
+    )
+
+    result = await processor._index_document(
+        _file_task(), _nc_client(), MagicMock(), tier="structured"
+    )
+
+    assert result is False
+    spies.mark.assert_awaited_once()
+    # Marker is content-addressed with this etag + the OCR-off tiers signature.
+    args = spies.mark.await_args.args
+    assert args[0] == "520189" and args[1] == "file"
+    assert args[2] == "etag-1"  # etag
+    assert "ocr=0" in args[3]  # tiers_sig
+    assert args[4] == "timeout"  # reason
+    spies.dead_metric.assert_called_once_with("timeout")
+    spies.delete_ph.assert_awaited_once()  # volatile placeholder dropped
+    spies.update_ph.assert_not_awaited()  # NOT the legacy per-user failed mark
+
+
+async def test_non_terminal_failure_keeps_legacy_mark(mocker):
+    """fast tier fails while structured is still available -> legacy failed mark."""
+    spies = _patch_common(mocker, ocr_enabled=False)
+    mocker.patch.object(
+        processor,
+        "_parse_pdf_tier",
+        AsyncMock(
+            return_value=ProcessingResult(
+                text="",
+                metadata={"parse_failed_reason": "error", "pipeline_tier": "fast"},
+                processor="pypdfium2",
+                success=False,
+                error="isolated parse failed (error)",
+            )
+        ),
+    )
+
+    result = await processor._index_document(
+        _file_task(), _nc_client(), MagicMock(), tier="fast"
+    )
+
+    assert result is False
+    spies.update_ph.assert_awaited_once()  # legacy per-user failed mark
+    spies.mark.assert_not_awaited()  # NOT dead-lettered (structured can still run)
+    spies.dead_metric.assert_not_called()
+
+
+async def test_oversize_failure_dead_letters_regardless_of_tier(mocker):
+    """An oversize PDF is terminal at any tier (no tier can parse it) -> dead-letter
+    even though a higher tier (structured) is nominally available above 'fast'."""
+    spies = _patch_common(mocker, ocr_enabled=False)
+    mocker.patch.object(
+        processor,
+        "_parse_pdf_tier",
+        AsyncMock(
+            return_value=ProcessingResult(
+                text="",
+                metadata={"parse_failed_reason": "oversize"},
+                processor="size_guard",
+                success=False,
+                error="PDF exceeds size cap",
+            )
+        ),
+    )
+
+    result = await processor._index_document(
+        _file_task(), _nc_client(), MagicMock(), tier="fast"
+    )
+
+    assert result is False
+    spies.mark.assert_awaited_once()
+    assert spies.mark.await_args.args[4] == "oversize"  # reason
+    spies.dead_metric.assert_called_once_with("oversize")
+    spies.update_ph.assert_not_awaited()
+
+
+async def test_terminal_failure_without_etag_uses_legacy_mark(mocker):
+    """A terminal failure with no etag can't be content-addressed, so fall back to
+    the legacy per-user placeholder mark instead of writing an unmatchable marker."""
+    spies = _patch_common(mocker, ocr_enabled=False)
+    mocker.patch.object(
+        processor,
+        "_parse_pdf_tier",
+        AsyncMock(
+            return_value=ProcessingResult(
+                text="",
+                metadata={
+                    "parse_failed_reason": "timeout",
+                    "pipeline_tier": "structured",
+                },
+                processor="pymupdf",
+                success=False,
+                error="isolated parse failed (timeout)",
+            )
+        ),
+    )
+    task = _file_task()
+    task.etag = None  # no content key
+
+    result = await processor._index_document(
+        task, _nc_client(), MagicMock(), tier="structured"
+    )
+
+    assert result is False
+    spies.mark.assert_not_awaited()  # no unmatchable marker written
+    spies.update_ph.assert_awaited_once()  # legacy fallback
+
+
+async def test_delete_clears_dead_letter_marker(mocker):
+    """Deleting a file must also drop its dead-letter marker, else a
+    dead-lettered-then-deleted file leaves an orphan accumulating in Qdrant
+    (release_document_for_user's filter misses the user-agnostic marker)."""
+    mocker.patch.object(processor, "get_qdrant_client", AsyncMock())
+    mocker.patch.object(processor, "release_document_for_user", AsyncMock())
+    clear = mocker.patch.object(processor, "clear_dead_letter", AsyncMock())
+
+    task = DocumentTask(
+        user_id="Demo-User",
+        doc_id="520189",
+        doc_type="file",
+        operation="delete",
+        modified_at=0,
+        file_path="/Plans/big.pdf",
+        etag="etag-1",
+    )
+    await processor.process_document(task, MagicMock(), max_retries=1)
+
+    clear.assert_awaited_once_with("520189", "file")