Files
mcp-nextcloud/nextcloud_mcp_server/models/semantic.py
T
Chris CoutinhoandClaude Opus 4.7 719b3b5034 fix(vector): normalize doc_id to str + add Qdrant keyword payload indexes
Production was logging two cascading classes of Qdrant errors against the
welcomed-malamute deployment:

1. HTTP 400 — "Bad request: Index required but not found for \"doc_id\" of
   one of the following types: [keyword]". The collection was created via
   create_collection() with no payload indexes, so any FieldCondition
   filter on doc_id failed at the Qdrant layer (placeholder writes/reads,
   eviction, search context lookups).

2. Compounding the missing index, producers wrote a mix of int and str
   doc_ids: webhook_parser stringified node_id, scanner stringified note
   IDs, news IDs, and deck card IDs — but the file scanner passed the
   numeric file_id through unchanged. A keyword index would not have
   covered both kinds even if it had existed.

This change:

- Normalizes doc_id to str at every producer site (scanner.py:459,
  DocumentTask.doc_id, indexed_*_ids reads from Qdrant).
- Tightens str|int annotations to str across placeholder.py,
  eviction.py, search/verification.py, search/context.py,
  SearchResult.id, and the auth/api visualization endpoints.
- Defensive str() coercion on doc_id reads in semantic.py /
  bm25_hybrid.py / vector/visualization.py for the transition window
  before the backfill runs.
- Adds an idempotent startup migration in get_qdrant_client():
  - _ensure_keyword_payload_indexes creates KEYWORD indexes for
    doc_id, user_id, and doc_type (tolerates "already exists" 400s).
  - _backfill_doc_id_to_string scrolls the collection once and rewrites
    int doc_ids to str. Skipped after a quick sample shows no legacy
    int payloads.
- Public API preserved: SemanticSearchResult.id stays int via explicit
  int(r.id) narrowing in server/semantic.py — surfaces a TypeError with
  actionable context if a future doc_type ships non-numeric ids.
- Documents the startup migration in docs/configuration.md.

Tests: 11 new unit tests in tests/unit/vector/test_qdrant_client.py
covering happy path / already-exists / unrelated-400 for the index
helpers, and sample-skip / mixed-batch rewrite / payload=None edge cases
for the backfill. 889 unit tests pass.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-08 19:30:51 +02:00

181 lines
7.2 KiB
Python

"""Pydantic models for semantic search responses."""
from pydantic import BaseModel, Field
from .base import BaseResponse
class SemanticSearchResult(BaseModel):
"""Model for semantic search results with additional metadata."""
id: int = Field(
description=(
"Document ID. Numeric for all currently indexed types (notes, files, "
"deck cards, news items). The internal SearchResult.id is stringified "
"for Qdrant's keyword-indexed doc_id payload; the MCP response narrows "
"back to int via int(r.id). A future doc_type with non-numeric ids "
"would surface here as a TypeError at the narrowing boundary, "
"forcing a deliberate widening of this field rather than a silent "
"API change."
)
)
doc_type: str = Field(
description="Document type (note, calendar_event, deck_card, etc.)"
)
title: str = Field(description="Document title")
category: str = Field(
default="", description="Document category (notes) or location (calendar)"
)
excerpt: str = Field(description="Excerpt from matching chunk")
score: float = Field(
description=(
"Relevance score (≥ 0.0, higher is better). "
"Score range depends on fusion method: "
"RRF produces scores in [0.0, 1.0], "
"DBSF can exceed 1.0 (sum of normalized scores from multiple systems)"
)
)
chunk_index: int = Field(description="Index of matching chunk in document")
total_chunks: int = Field(description="Total number of chunks in document")
chunk_start_offset: int | None = Field(
default=None, description="Character position where chunk starts in document"
)
chunk_end_offset: int | None = Field(
default=None, description="Character position where chunk ends in document"
)
page_number: int | None = Field(
default=None, description="Page number for PDF documents"
)
page_count: int | None = Field(
default=None, description="Total number of pages in PDF document"
)
# Context expansion fields (optional, populated when include_context=True)
has_context_expansion: bool = Field(
default=False, description="Whether context expansion was performed"
)
marked_text: str | None = Field(
default=None,
description="Full text with position markers around matched chunk",
)
before_context: str | None = Field(
default=None, description="Text before the matched chunk"
)
after_context: str | None = Field(
default=None, description="Text after the matched chunk"
)
has_before_truncation: bool | None = Field(
default=None, description="Whether before_context was truncated"
)
has_after_truncation: bool | None = Field(
default=None, description="Whether after_context was truncated"
)
class SemanticSearchResponse(BaseResponse):
"""Response model for semantic search across all indexed Nextcloud apps."""
results: list[SemanticSearchResult] = Field(
description="Semantic search results with similarity scores"
)
query: str = Field(description="The search query used")
total_found: int = Field(description="Total number of documents found")
search_method: str = Field(
default="semantic", description="Search method used (semantic or hybrid)"
)
verified_chunk_count: int = Field(
default=0,
description=(
"Number of search result chunks that passed verify-on-read "
"access checks (ADR-019). Equals len(verified_results) before "
"trimming to limit. Sized in chunks (result rows), NOT in "
"unique documents — see dropped_document_count for the "
"per-document counterpart."
),
)
dropped_document_count: int = Field(
default=0,
description=(
"Number of unique (doc_id, doc_type) pairs dropped as ghost "
"records during verify-on-read (ADR-019). A short result page "
"(len(results) < limit) combined with a non-zero "
"dropped_document_count indicates ghost density rather than "
"scarcity of relevant content. Note: this counter is sized in "
"unique documents while verified_chunk_count is sized in "
"chunks — a single document can contribute multiple chunks, "
"so subtracting dropped_document_count from "
"verified_chunk_count is NOT a meaningful operation."
),
)
class SamplingSearchResponse(BaseResponse):
"""Response from semantic search with LLM-generated answer via MCP sampling.
This response includes both a generated natural language answer (created by
the MCP client's LLM via sampling) and the source documents used to generate
that answer. Users can read the answer for quick information and review
sources for verification and deeper exploration.
Attributes:
query: The original user query
generated_answer: Natural language answer generated by client's LLM
sources: List of semantic search results used as context
total_found: Total number of matching documents found
search_method: Always "semantic_sampling" for this response type
model_used: Name of model that generated the answer (e.g., "claude-3-5-sonnet")
stop_reason: Why generation stopped ("endTurn", "maxTokens", etc.)
"""
query: str = Field(..., description="Original user query")
generated_answer: str = Field(
..., description="LLM-generated answer based on retrieved documents"
)
sources: list[SemanticSearchResult] = Field(
default_factory=list,
description="Source documents with excerpts and relevance scores",
)
total_found: int = Field(..., description="Total matching documents")
search_method: str = Field(
default="semantic_sampling", description="Search method used"
)
model_used: str | None = Field(
default=None, description="Model that generated the answer"
)
stop_reason: str | None = Field(
default=None, description="Reason generation stopped"
)
class VectorSyncStatusResponse(BaseResponse):
"""Response for vector sync status.
Provides information about the current state of vector sync,
including how many documents are indexed and how many are pending.
Attributes:
indexed_count: Number of documents in Qdrant vector database
pending_count: Number of documents in processing queue
status: Current sync status ("idle" or "syncing")
enabled: Whether vector sync is enabled
"""
indexed_count: int = Field(
default=0, description="Number of documents indexed in vector database"
)
pending_count: int = Field(
default=0, description="Number of documents pending processing"
)
status: str = Field(
default="disabled",
description='Sync status: "idle", "syncing", or "disabled"',
)
enabled: bool = Field(default=False, description="Whether vector sync is enabled")
__all__ = [
"SemanticSearchResult",
"SemanticSearchResponse",
"SamplingSearchResponse",
"VectorSyncStatusResponse",
]