mcp-nextcloud/nextcloud_mcp_server/vector/dead_letter.py

"""Content-addressed dead-letter markers for terminally-failed documents.

A document that fails its terminal extraction tier (a hard parse failure with no
higher tier to escalate to — e.g. the ``structured`` tier timing out while OCR is
disabled) must not be retried forever. The per-user placeholder ``status="failed"``
mark cannot stop the loop on its own: placeholder point IDs are user-agnostic
(``uuid5("file:<doc_id>:placeholder")``) but the scanner's freshness gate filters
by ``user_id``, so for a file visible to *several* users the single shared
placeholder's ``user_id`` is overwritten by whoever scanned last and every other
user's scan sees "no record → re-queue", re-burning the (failing) parse on a loop.

This module records a **durable, content-addressed, user-agnostic** dead-letter
marker instead. One marker point per document (a distinct deterministic ID, kept
separate from the in-flight placeholder), carrying the ``etag`` and an escalation
``tiers_sig`` (see ``document_processors.escalation.escalation_tiers_signature``).
The scanner consults it tenant-wide — for every user — and skips re-queuing while
BOTH still match, so the document is attempted once per content-version and never
loops. A content change (new ``etag``) or a config change that adds an escalation
tier (e.g. enabling OCR — new ``tiers_sig``) makes the marker stale and the
document retryable again.

The marker carries ``is_placeholder=True`` so the existing search exclusion
(``get_placeholder_filter``) keeps it out of user-facing results with no extra
filter, plus ``dead_letter=True`` so the orphan-placeholder sweep and the scanner
can tell it apart from a volatile in-flight placeholder. Mirrors the fail-safe
philosophy of ``sharing_state``: a Qdrant error never aborts ingest — a failed
lookup degrades to "process normally", a failed write is logged, not raised.
"""

import logging
import time
import uuid
from typing import Any

from qdrant_client import models
from qdrant_client.models import FieldCondition, Filter, MatchValue, PointStruct

from nextcloud_mcp_server.config import get_settings
from nextcloud_mcp_server.embedding import get_embedding_service
from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client

logger = logging.getLogger(__name__)

# Payload flag distinguishing a durable dead-letter marker from an in-flight
# placeholder (both carry is_placeholder=True to inherit the search exclusion).
DEAD_LETTER_KEY = "dead_letter"


def _generate_dead_letter_id(doc_type: str, doc_id: str) -> str:
    """Deterministic, user-agnostic point ID for a document's dead-letter marker.

    Distinct from the in-flight placeholder ID (``…:placeholder``) so the two can
    coexist briefly and never collide; one marker per ``(doc_type, doc_id)``, so
    a re-failure upserts in place rather than accumulating.
    """
    return str(uuid.uuid5(uuid.NAMESPACE_DNS, f"{doc_type}:{doc_id}:deadletter"))


def _dead_letter_filter(doc_id: str, doc_type: str) -> Filter:
    """Match the dead-letter marker for one document (tenant-wide, no user_id)."""
    return Filter(
        must=[
            FieldCondition(key="doc_id", match=MatchValue(value=doc_id)),
            FieldCondition(key="doc_type", match=MatchValue(value=doc_type)),
            FieldCondition(key=DEAD_LETTER_KEY, match=MatchValue(value=True)),
        ]
    )


async def mark_dead_letter(
    doc_id: str,
    doc_type: str,
    etag: str,
    tiers_sig: str,
    reason: str,
    *,
    file_path: str | None = None,
) -> None:
    """Upsert a durable dead-letter marker for a terminally-failed document.

    Keyed by content (``etag``) + escalation config (``tiers_sig``); the scanner
    skips re-queuing while both match. ``reason`` is the parse failure reason
    (``timeout`` | ``oom`` | ``error``). Fail-safe: a Qdrant error is logged, not
    raised — a missed mark just means the document is retried (the bounded prior
    behaviour), never a crash.
    """
    try:
        qdrant_client = await get_qdrant_client()
        settings = get_settings()
        dimension = get_embedding_service().get_dimension()

        payload: dict[str, Any] = {
            "doc_id": doc_id,
            "doc_type": doc_type,
            "is_placeholder": True,
            DEAD_LETTER_KEY: True,
            "etag": etag,
            "tiers_sig": tiers_sig,
            "reason": reason,
            "failed_at": int(time.time()),
        }
        if doc_type == "file" and file_path:
            payload["file_path"] = file_path

        point = PointStruct(
            id=_generate_dead_letter_id(doc_type, doc_id),
            vector={
                "dense": [0.0] * dimension,
                "sparse": models.SparseVector(indices=[], values=[]),
            },
            payload=payload,
        )
        await qdrant_client.upsert(
            collection_name=settings.get_collection_name(),
            points=[point],
            wait=True,
        )
        logger.info(
            "Dead-lettered %s_%s (reason=%s, etag=%s)",
            doc_type,
            doc_id,
            reason,
            etag,
        )
    except Exception as e:
        logger.warning(
            "Failed to write dead-letter marker for %s_%s: %s", doc_type, doc_id, e
        )
        # Don't raise — dead-lettering is best-effort; a miss just retries.


async def is_dead_lettered(
    doc_id: str,
    doc_type: str,
    etag: str,
    tiers_sig: str,
) -> bool:
    """Whether this exact content is currently dead-lettered (skip re-queuing).

    Returns True only when a marker exists for ``(doc_id, doc_type)`` whose stored
    ``etag`` AND ``tiers_sig`` both match the current values — so a content change
    or a new escalation tier (e.g. OCR enabled) makes the document retryable
    again. An empty ``etag`` is never dead-lettered (we cannot content-address it).
    Fail-safe: a Qdrant error degrades to False (process normally), mirroring
    ``sharing_state.claim_existing_index``.
    """
    if not etag:
        return False
    try:
        qdrant_client = await get_qdrant_client()
        settings = get_settings()
        points, _ = await qdrant_client.scroll(
            collection_name=settings.get_collection_name(),
            scroll_filter=_dead_letter_filter(doc_id, doc_type),
            limit=1,
            with_payload=True,
            with_vectors=False,
        )
    except Exception as e:
        logger.warning(
            "Dead-letter lookup failed for %s_%s (%s); processing normally",
            doc_type,
            doc_id,
            e,
        )
        return False
    if not points:
        return False
    payload = dict(points[0].payload or {})
    return payload.get("etag") == etag and payload.get("tiers_sig") == tiers_sig


async def clear_dead_letter(doc_id: str, doc_type: str) -> None:
    """Delete a document's dead-letter marker (on successful index / release).

    Idempotent and fail-safe: deleting a non-existent marker is a Qdrant no-op,
    and an error is logged rather than raised so it never breaks the indexing
    path that calls it.
    """
    try:
        qdrant_client = await get_qdrant_client()
        settings = get_settings()
        await qdrant_client.delete(
            collection_name=settings.get_collection_name(),
            points_selector=_dead_letter_filter(doc_id, doc_type),
        )
        logger.debug("Cleared dead-letter marker for %s_%s", doc_type, doc_id)
    except Exception as e:
        logger.warning(
            "Failed to clear dead-letter marker for %s_%s: %s", doc_type, doc_id, e
        )