From 719b3b5034f944cc7a80b4f71a4f5eca8190ca63 Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Fri, 8 May 2026 19:30:51 +0200 Subject: [PATCH 01/27] fix(vector): normalize doc_id to str + add Qdrant keyword payload indexes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Production was logging two cascading classes of Qdrant errors against the welcomed-malamute deployment: 1. HTTP 400 — "Bad request: Index required but not found for \"doc_id\" of one of the following types: [keyword]". The collection was created via create_collection() with no payload indexes, so any FieldCondition filter on doc_id failed at the Qdrant layer (placeholder writes/reads, eviction, search context lookups). 2. Compounding the missing index, producers wrote a mix of int and str doc_ids: webhook_parser stringified node_id, scanner stringified note IDs, news IDs, and deck card IDs — but the file scanner passed the numeric file_id through unchanged. A keyword index would not have covered both kinds even if it had existed. This change: - Normalizes doc_id to str at every producer site (scanner.py:459, DocumentTask.doc_id, indexed_*_ids reads from Qdrant). - Tightens str|int annotations to str across placeholder.py, eviction.py, search/verification.py, search/context.py, SearchResult.id, and the auth/api visualization endpoints. - Defensive str() coercion on doc_id reads in semantic.py / bm25_hybrid.py / vector/visualization.py for the transition window before the backfill runs. - Adds an idempotent startup migration in get_qdrant_client(): - _ensure_keyword_payload_indexes creates KEYWORD indexes for doc_id, user_id, and doc_type (tolerates "already exists" 400s). - _backfill_doc_id_to_string scrolls the collection once and rewrites int doc_ids to str. Skipped after a quick sample shows no legacy int payloads. - Public API preserved: SemanticSearchResult.id stays int via explicit int(r.id) narrowing in server/semantic.py — surfaces a TypeError with actionable context if a future doc_type ships non-numeric ids. - Documents the startup migration in docs/configuration.md. Tests: 11 new unit tests in tests/unit/vector/test_qdrant_client.py covering happy path / already-exists / unrelated-400 for the index helpers, and sample-skip / mixed-batch rewrite / payload=None edge cases for the backfill. 889 unit tests pass. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/configuration.md | 16 ++ nextcloud_mcp_server/api/visualization.py | 8 +- nextcloud_mcp_server/auth/viz_routes.py | 13 +- nextcloud_mcp_server/models/semantic.py | 10 +- nextcloud_mcp_server/search/algorithms.py | 11 +- nextcloud_mcp_server/search/bm25_hybrid.py | 5 +- nextcloud_mcp_server/search/context.py | 36 ++- nextcloud_mcp_server/search/semantic.py | 5 +- nextcloud_mcp_server/search/verification.py | 2 +- nextcloud_mcp_server/server/semantic.py | 16 +- nextcloud_mcp_server/vector/eviction.py | 4 +- nextcloud_mcp_server/vector/placeholder.py | 12 +- nextcloud_mcp_server/vector/qdrant_client.py | 137 +++++++++- nextcloud_mcp_server/vector/scanner.py | 14 +- nextcloud_mcp_server/vector/visualization.py | 5 +- tests/unit/vector/__init__.py | 0 tests/unit/vector/test_qdrant_client.py | 265 +++++++++++++++++++ 17 files changed, 493 insertions(+), 66 deletions(-) create mode 100644 tests/unit/vector/__init__.py create mode 100644 tests/unit/vector/test_qdrant_client.py diff --git a/docs/configuration.md b/docs/configuration.md index f6f5d8e0..0214bf50 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -329,6 +329,22 @@ OLLAMA_EMBEDDING_MODEL=all-minilm - **Switching models requires re-embedding** all documents (may take time for large note collections) - **Old collection remains** in Qdrant and can be deleted manually if no longer needed +#### Startup migrations on existing collections + +On the first call to `get_qdrant_client()` against an existing collection, the +server runs two idempotent migrations: + +1. **Payload-index creation** — adds `KEYWORD` payload indexes for `doc_id`, + `user_id`, and `doc_type`. Required by Qdrant for any `FieldCondition` + filter. Cheap; runs even on healthy collections. +2. **`doc_id` backfill** — rewrites any legacy integer `doc_id` payloads to + strings so they match the keyword index. Skipped after a quick sample + shows the collection is already clean. On a dirty collection, the full + scroll runs once and writes are emitted point-by-point — expect a delay + proportional to point count on the first startup after the upgrade. + +Both steps emit INFO-level log lines so operators can track progress. + #### Explicit Override Set `QDRANT_COLLECTION` to use a specific collection name: diff --git a/nextcloud_mcp_server/api/visualization.py b/nextcloud_mcp_server/api/visualization.py index 242c7c13..18bcbd4f 100644 --- a/nextcloud_mcp_server/api/visualization.py +++ b/nextcloud_mcp_server/api/visualization.py @@ -511,8 +511,8 @@ async def get_chunk_context(request: Request) -> JSONResponse: raise ValueError("end must be greater than start") except ValueError as e: return JSONResponse({"success": False, "error": str(e)}, status_code=400) - # Convert doc_id to int if possible (most IDs are int) - doc_id_val: str | int = int(doc_id) if doc_id.isdigit() else doc_id + # doc_id is keyword-indexed in Qdrant as str — pass through verbatim + # (no int coercion; producers always stringify on write). # Get Nextcloud host from OAuth context oauth_ctx = request.app.state.oauth_context @@ -537,7 +537,7 @@ async def get_chunk_context(request: Request) -> JSONResponse: chunk_context = await get_chunk_with_context( nc_client=nc_client, user_id=user_id, - doc_id=doc_id_val, + doc_id=doc_id, doc_type=doc_type, chunk_start=start, chunk_end=end, @@ -570,7 +570,7 @@ async def get_chunk_context(request: Request) -> JSONResponse: must=[ get_placeholder_filter(), FieldCondition( - key="doc_id", match=MatchValue(value=doc_id_val) + key="doc_id", match=MatchValue(value=doc_id) ), FieldCondition( key="user_id", match=MatchValue(value=user_id) diff --git a/nextcloud_mcp_server/auth/viz_routes.py b/nextcloud_mcp_server/auth/viz_routes.py index d072c373..01b8a917 100644 --- a/nextcloud_mcp_server/auth/viz_routes.py +++ b/nextcloud_mcp_server/auth/viz_routes.py @@ -285,7 +285,11 @@ async def vector_visualization_search(request: Request) -> JSONResponse: vector = point.vector if vector is not None and point.payload: - doc_id = point.payload.get("doc_id") + # SearchResult.id is str; coerce payload doc_id to match + # so the tuple lookup below succeeds even on legacy + # int-typed payloads written before normalization. + raw_doc_id = point.payload.get("doc_id") + doc_id = None if raw_doc_id is None else str(raw_doc_id) chunk_start = point.payload.get("chunk_start_offset") chunk_end = point.payload.get("chunk_end_offset") chunk_key = (doc_id, chunk_start, chunk_end) @@ -555,8 +559,7 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse: start = int(start_str) end = int(end_str) - # Convert doc_id to int (all document types use int IDs) - doc_id_int = int(doc_id) + # doc_id is keyword-indexed in Qdrant as str — pass through verbatim. user_id = request.user.display_name settings = get_settings() @@ -580,7 +583,7 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse: chunk_context = await get_chunk_with_context( nc_client=nc_client, user_id=user_id, - doc_id=doc_id_int, + doc_id=doc_id, doc_type=doc_type, chunk_start=start, chunk_end=end, @@ -620,7 +623,7 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse: must=[ get_placeholder_filter(), FieldCondition( - key="doc_id", match=MatchValue(value=doc_id_int) + key="doc_id", match=MatchValue(value=doc_id) ), FieldCondition( key="user_id", match=MatchValue(value=username) diff --git a/nextcloud_mcp_server/models/semantic.py b/nextcloud_mcp_server/models/semantic.py index 4612d26d..bb0091dc 100644 --- a/nextcloud_mcp_server/models/semantic.py +++ b/nextcloud_mcp_server/models/semantic.py @@ -11,10 +11,12 @@ class SemanticSearchResult(BaseModel): id: int = Field( description=( "Document ID. Numeric for all currently indexed types (notes, files, " - "deck cards, news items). The internal SearchResult.id is typed as " - "int|str to leave room for future doc types with string identifiers; " - "the MCP response narrows to int and a future widening here would be " - "a deliberate, breaking-by-design API change." + "deck cards, news items). The internal SearchResult.id is stringified " + "for Qdrant's keyword-indexed doc_id payload; the MCP response narrows " + "back to int via int(r.id). A future doc_type with non-numeric ids " + "would surface here as a TypeError at the narrowing boundary, " + "forcing a deliberate widening of this field rather than a silent " + "API change." ) ) doc_type: str = Field( diff --git a/nextcloud_mcp_server/search/algorithms.py b/nextcloud_mcp_server/search/algorithms.py index 2657bb45..1d14fdf7 100644 --- a/nextcloud_mcp_server/search/algorithms.py +++ b/nextcloud_mcp_server/search/algorithms.py @@ -132,9 +132,12 @@ class SearchResult: """A single search result with metadata and score. Attributes: - id: Document ID. Numeric for indexed types today (notes, files, - deck cards, news items), but typed as ``int | str`` to allow - future doc types that use string identifiers (e.g., file paths). + id: Document ID — always a string. Producers stringify their native + ID before writing to Qdrant so the keyword payload index on + ``doc_id`` matches every point regardless of source doc_type. + Public response models (e.g. ``SemanticSearchResult``) re-narrow + this back to ``int`` at the MCP boundary via ``int(r.id)`` — + see ``server/semantic.py`` for the narrowing site. doc_type: Document type (note, file, calendar, contact, etc.) title: Document title excerpt: Content excerpt showing match context @@ -151,7 +154,7 @@ class SearchResult: point_id: Qdrant point ID for batch vector retrieval (None if not from Qdrant) """ - id: int | str + id: str doc_type: str title: str excerpt: str diff --git a/nextcloud_mcp_server/search/bm25_hybrid.py b/nextcloud_mcp_server/search/bm25_hybrid.py index c5848450..c6f8dfda 100644 --- a/nextcloud_mcp_server/search/bm25_hybrid.py +++ b/nextcloud_mcp_server/search/bm25_hybrid.py @@ -208,8 +208,9 @@ class BM25HybridSearchAlgorithm(SearchAlgorithm): for result in search_response.points: if result.payload is None: continue - # doc_id can be int (files) or str (notes/news_items/deck_cards) — see scanner.py - doc_id = result.payload["doc_id"] + # doc_id is always str post-normalization, but defensively coerce + # legacy int payloads on read until the backfill has run everywhere. + doc_id = str(result.payload["doc_id"]) doc_type = result.payload.get("doc_type", "note") chunk_start = result.payload.get("chunk_start_offset") chunk_end = result.payload.get("chunk_end_offset") diff --git a/nextcloud_mcp_server/search/context.py b/nextcloud_mcp_server/search/context.py index cff7d1cc..c249bb73 100644 --- a/nextcloud_mcp_server/search/context.py +++ b/nextcloud_mcp_server/search/context.py @@ -20,7 +20,7 @@ logger = logging.getLogger(__name__) async def _get_chunk_from_qdrant( - user_id: str, doc_id: int, doc_type: str, chunk_start: int, chunk_end: int + user_id: str, doc_id: str, doc_type: str, chunk_start: int, chunk_end: int ) -> str | None: """Retrieve full chunk text from Qdrant payload. @@ -87,7 +87,7 @@ async def _get_chunk_from_qdrant( async def _get_chunk_by_index_from_qdrant( - user_id: str, doc_id: int, doc_type: str, chunk_index: int + user_id: str, doc_id: str, doc_type: str, chunk_index: int ) -> str | None: """Retrieve chunk text by chunk_index from Qdrant payload. @@ -145,7 +145,7 @@ async def _get_chunk_by_index_from_qdrant( async def _get_file_path_from_qdrant( - user_id: str, file_id: int, chunk_start: int, chunk_end: int + user_id: str, file_id: str, chunk_start: int, chunk_end: int ) -> str | None: """Resolve file_id to file_path by querying Qdrant payload. @@ -202,7 +202,7 @@ async def _get_file_path_from_qdrant( async def _get_deck_metadata_from_qdrant( - user_id: str, card_id: int + user_id: str, card_id: str ) -> dict[str, int] | None: """Retrieve board_id and stack_id for a deck card from Qdrant payload. @@ -288,7 +288,7 @@ class ChunkContext: async def get_chunk_with_context( nc_client: NextcloudClient, user_id: str, - doc_id: str | int, + doc_id: str, doc_type: str, chunk_start: int, chunk_end: int, @@ -306,7 +306,7 @@ async def get_chunk_with_context( Args: nc_client: Authenticated Nextcloud client user_id: User ID who owns the document - doc_id: Document ID (int for notes/files) + doc_id: Document ID (str — keyword-indexed in Qdrant payload) doc_type: Type of document ("note", "file", etc.) chunk_start: Character offset where chunk starts chunk_end: Character offset where chunk ends @@ -319,17 +319,10 @@ async def get_chunk_with_context( ChunkContext with expanded context and markers, or None if document cannot be retrieved """ - # Convert doc_id to int for Qdrant query - doc_id_int = ( - int(doc_id) - if isinstance(doc_id, str) and doc_id.isdigit() - else (doc_id if isinstance(doc_id, int) else None) - ) - # Try to get chunk from Qdrant first (fast path) - if doc_id_int is not None: + if doc_id: chunk_text = await _get_chunk_from_qdrant( - user_id, doc_id_int, doc_type, chunk_start, chunk_end + user_id, doc_id, doc_type, chunk_start, chunk_end ) if chunk_text: logger.info( @@ -350,7 +343,7 @@ async def get_chunk_with_context( # Fetch previous chunk if not first chunk if chunk_index > 0: before_chunk = await _get_chunk_by_index_from_qdrant( - user_id, doc_id_int, doc_type, chunk_index - 1 + user_id, doc_id, doc_type, chunk_index - 1 ) if before_chunk: # Remove overlap: the last chunk_overlap chars of previous chunk @@ -371,7 +364,7 @@ async def get_chunk_with_context( # Fetch next chunk if not last chunk if chunk_index < total_chunks - 1: after_chunk = await _get_chunk_by_index_from_qdrant( - user_id, doc_id_int, doc_type, chunk_index + 1 + user_id, doc_id, doc_type, chunk_index + 1 ) if after_chunk: # Remove overlap: the first chunk_overlap chars of next chunk @@ -422,9 +415,10 @@ async def get_chunk_with_context( f"(Qdrant cache miss, possibly legacy data)" ) - # For files, retrieve file_path from Qdrant payload + # For files, the doc_id is the numeric file ID (as a string) — resolve it + # to a WebDAV path so _fetch_document_text can retrieve the binary content. resolved_doc_id = doc_id - if doc_type == "file" and isinstance(doc_id, int): + if doc_type == "file": file_path = await _get_file_path_from_qdrant( user_id, doc_id, chunk_start, chunk_end ) @@ -498,7 +492,7 @@ async def get_chunk_with_context( async def _fetch_document_text( - nc_client: NextcloudClient, doc_id: str | int, doc_type: str, user_id: str + nc_client: NextcloudClient, doc_id: str, doc_type: str, user_id: str ) -> str | None: """Fetch full text content of a document. @@ -590,7 +584,7 @@ async def _fetch_document_text( # Try to get board_id/stack_id from Qdrant metadata (O(1) lookup) # Otherwise fall back to iteration (legacy data) card = None - deck_metadata = await _get_deck_metadata_from_qdrant(user_id, int(doc_id)) + deck_metadata = await _get_deck_metadata_from_qdrant(user_id, doc_id) if deck_metadata: # Fast path: Direct lookup with known board_id/stack_id diff --git a/nextcloud_mcp_server/search/semantic.py b/nextcloud_mcp_server/search/semantic.py index c01b0a37..d391b653 100644 --- a/nextcloud_mcp_server/search/semantic.py +++ b/nextcloud_mcp_server/search/semantic.py @@ -140,8 +140,9 @@ class SemanticSearchAlgorithm(SearchAlgorithm): for result in search_response.points: if result.payload is None: continue - # doc_id can be int (notes) or str (files - file paths) - doc_id = result.payload["doc_id"] + # doc_id is always str post-normalization, but defensively coerce + # legacy int payloads on read until the backfill has run everywhere. + doc_id = str(result.payload["doc_id"]) doc_type = result.payload.get("doc_type", "note") chunk_start = result.payload.get("chunk_start_offset") chunk_end = result.payload.get("chunk_end_offset") diff --git a/nextcloud_mcp_server/search/verification.py b/nextcloud_mcp_server/search/verification.py index bb3fb678..e5665f3e 100644 --- a/nextcloud_mcp_server/search/verification.py +++ b/nextcloud_mcp_server/search/verification.py @@ -565,7 +565,7 @@ async def verify_search_results( # complete by the time `verify_search_results` returns. if evict_on_missing and inaccessible: - async def evict(doc_id: int | str, doc_type: str) -> None: + async def evict(doc_id: str, doc_type: str) -> None: try: await delete_document_points(doc_id, doc_type, user_id) except Exception as e: diff --git a/nextcloud_mcp_server/server/semantic.py b/nextcloud_mcp_server/server/semantic.py index c1beb208..e8d1a396 100644 --- a/nextcloud_mcp_server/server/semantic.py +++ b/nextcloud_mcp_server/server/semantic.py @@ -224,12 +224,11 @@ def configure_semantic_tools(mcp: FastMCP): search_results = verified_results[:limit] # Convert SearchResult objects to SemanticSearchResult for response. - # SearchResult.id is typed `int | str` for forward-compat with future - # doc_types, but every currently indexed type uses numeric ids and - # the MCP response model narrows to `int`. Casting here makes the - # narrowing explicit and surfaces any future string-id type as a - # loud failure at the boundary instead of silently widening the - # public API. + # SearchResult.id is `str` (Qdrant keyword-indexed payload), but + # every currently indexed type uses numeric ids and the MCP response + # model narrows to `int`. Casting here makes the narrowing explicit + # and surfaces any future non-numeric-id type as a loud failure at + # the boundary instead of silently widening the public API. results = [] for r in search_results: try: @@ -304,7 +303,10 @@ def configure_semantic_tools(mcp: FastMCP): chunk_context = await get_chunk_with_context( nc_client=client, user_id=username, - doc_id=result.id, + # SemanticSearchResult.id is the int-narrowed + # public form; get_chunk_with_context queries + # Qdrant where doc_id is keyword-indexed as str. + doc_id=str(result.id), doc_type=result.doc_type, chunk_start=result.chunk_start_offset, chunk_end=result.chunk_end_offset, diff --git a/nextcloud_mcp_server/vector/eviction.py b/nextcloud_mcp_server/vector/eviction.py index 44874e08..8c7555a7 100644 --- a/nextcloud_mcp_server/vector/eviction.py +++ b/nextcloud_mcp_server/vector/eviction.py @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) async def delete_document_points( - doc_id: str | int, + doc_id: str, doc_type: str, user_id: str, ) -> None: @@ -29,7 +29,7 @@ async def delete_document_points( not present — Qdrant returns successfully with zero points affected. Args: - doc_id: Document ID (int for notes/files/cards/news, str otherwise) + doc_id: Document ID (str — keyword-indexed in Qdrant payload) doc_type: Document type (note, file, deck_card, news_item) user_id: Owner of the points being evicted diff --git a/nextcloud_mcp_server/vector/placeholder.py b/nextcloud_mcp_server/vector/placeholder.py index fcd12bca..4950c2fc 100644 --- a/nextcloud_mcp_server/vector/placeholder.py +++ b/nextcloud_mcp_server/vector/placeholder.py @@ -31,7 +31,7 @@ from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client logger = logging.getLogger(__name__) -def _generate_placeholder_id(doc_type: str, doc_id: str | int) -> str: +def _generate_placeholder_id(doc_type: str, doc_id: str) -> str: """Generate deterministic UUID for placeholder point. Args: @@ -46,7 +46,7 @@ def _generate_placeholder_id(doc_type: str, doc_id: str | int) -> str: async def write_placeholder_point( - doc_id: str | int, + doc_id: str, doc_type: str, user_id: str, modified_at: int, @@ -60,7 +60,7 @@ async def write_placeholder_point( processing completes. Args: - doc_id: Document ID (int for notes/files) + doc_id: Document ID (always str — see DocumentTask) doc_type: Document type (note, file, etc.) user_id: User ID who owns the document modified_at: Document modification timestamp @@ -135,7 +135,7 @@ async def write_placeholder_point( async def query_document_metadata( - doc_id: str | int, + doc_id: str, doc_type: str, user_id: str, ) -> dict | None: @@ -185,7 +185,7 @@ async def query_document_metadata( async def delete_placeholder_point( - doc_id: str | int, + doc_id: str, doc_type: str, user_id: str, ) -> None: @@ -230,7 +230,7 @@ async def delete_placeholder_point( async def update_placeholder_status( - doc_id: str | int, + doc_id: str, doc_type: str, user_id: str, status: str, diff --git a/nextcloud_mcp_server/vector/qdrant_client.py b/nextcloud_mcp_server/vector/qdrant_client.py index b2aa0e65..a192eeec 100644 --- a/nextcloud_mcp_server/vector/qdrant_client.py +++ b/nextcloud_mcp_server/vector/qdrant_client.py @@ -3,7 +3,8 @@ import logging from qdrant_client import AsyncQdrantClient, models -from qdrant_client.models import Distance, VectorParams +from qdrant_client.http.exceptions import UnexpectedResponse +from qdrant_client.models import Distance, PayloadSchemaType, VectorParams from nextcloud_mcp_server.config import get_settings from nextcloud_mcp_server.embedding import get_embedding_service @@ -11,10 +12,137 @@ from nextcloud_mcp_server.embedding import get_embedding_service logger = logging.getLogger(__name__) +# Payload fields filtered by exact-match in scanner/processor/placeholder/eviction. +# Qdrant requires a payload index for any field used in a FieldCondition; without +# one, queries fail with HTTP 400 ("Index required but not found"). All three +# carry string values after producer normalization, so a KEYWORD index is the +# correct schema (see ADR notes in commit message). +_KEYWORD_PAYLOAD_FIELDS: tuple[str, ...] = ("doc_id", "user_id", "doc_type") + # Singleton instance _qdrant_client: AsyncQdrantClient | None = None +async def _ensure_keyword_payload_indexes( + client: AsyncQdrantClient, collection_name: str +) -> None: + """Create KEYWORD payload indexes for fields used in exact-match filters. + + Idempotent: tolerates 'already exists' errors so it can run on every + startup against existing collections. + """ + for field in _KEYWORD_PAYLOAD_FIELDS: + try: + await client.create_payload_index( + collection_name=collection_name, + field_name=field, + field_schema=PayloadSchemaType.KEYWORD, + wait=True, + ) + logger.info("Created KEYWORD payload index on '%s'", field) + except UnexpectedResponse as e: + # Qdrant returns 400 if the index already exists with a different + # schema, or simply succeeds if it already matches. Treat + # already-exists as benign; surface schema conflicts loudly. + body = getattr(e, "content", b"") or b"" + body_text = body.decode("utf-8", errors="replace") + if "already exists" in body_text.lower(): + logger.debug("Payload index on '%s' already exists", field) + else: + logger.warning( + "Failed to create payload index on '%s': %s", field, body_text + ) + + +async def _has_int_doc_id_sample( + client: AsyncQdrantClient, collection_name: str, sample_size: int = 256 +) -> bool: + """Quick sample to decide whether the full backfill scroll is needed. + + Reading the first batch is cheap; if all sampled doc_ids are already str + (the steady-state on healthy collections), we skip the full pass. + """ + points, _ = await client.scroll( + collection_name=collection_name, + limit=sample_size, + with_payload=["doc_id"], + with_vectors=False, + ) + for point in points: + payload = point.payload or {} + value = payload.get("doc_id") + if value is not None and not isinstance(value, str): + return True + return False + + +async def _backfill_doc_id_to_string( + client: AsyncQdrantClient, collection_name: str +) -> None: + """Rewrite legacy integer doc_id payloads to strings. + + Producers now uniformly write str(doc_id), but historical points may carry + int values from before normalization. A KEYWORD index does not match int + payloads, so any leftover int doc_ids would be silently invisible to + filters. Scroll all points and convert in-place. Idempotent. + + Skipped when the first sample batch already contains only str doc_ids. + """ + if not await _has_int_doc_id_sample(client, collection_name): + logger.debug( + "doc_id backfill: sample shows no legacy int payloads; skipping full scan" + ) + return + + logger.info( + "Running doc_id backfill on '%s' (this may take a moment for large collections)", + collection_name, + ) + + rewritten = 0 + scanned = 0 + # Qdrant scroll returns next_offset as PointId | None — keep it untyped here + # so the qdrant client's full union (UUID/int/str/PointId) flows through. + next_offset = None + batch_size = 256 + + while True: + points, next_offset = await client.scroll( + collection_name=collection_name, + limit=batch_size, + offset=next_offset, + with_payload=["doc_id"], + with_vectors=False, + ) + if not points: + break + + for point in points: + scanned += 1 + # Qdrant client typing allows None payload even when with_payload + # was requested; defensive default so the type checker is happy. + payload = point.payload or {} + value = payload.get("doc_id") + if value is None or isinstance(value, str): + continue + await client.set_payload( + collection_name=collection_name, + payload={"doc_id": str(value)}, + points=[point.id], + wait=False, + ) + rewritten += 1 + + if next_offset is None: + break + + logger.info( + "doc_id backfill complete: rewrote %d/%d payloads from int to str", + rewritten, + scanned, + ) + + async def get_qdrant_client() -> AsyncQdrantClient: """ Get singleton Qdrant client instance. @@ -110,6 +238,12 @@ async def get_qdrant_client() -> AsyncQdrantClient: f"(dimension={actual_dimension}, model={settings.get_embedding_model_name()})" ) + # Existing collections may pre-date the doc_id normalization / + # payload-index work. Backfill before creating the index so the + # index covers every point. + await _backfill_doc_id_to_string(_qdrant_client, collection_name) + await _ensure_keyword_payload_indexes(_qdrant_client, collection_name) + else: # Collection doesn't exist - create it embedding_model = settings.get_embedding_model_name() @@ -141,5 +275,6 @@ async def get_qdrant_client() -> AsyncQdrantClient: f" Distance: COSINE\n" f"Background sync will index all documents with dense + sparse vectors." ) + await _ensure_keyword_payload_indexes(_qdrant_client, collection_name) return _qdrant_client diff --git a/nextcloud_mcp_server/vector/scanner.py b/nextcloud_mcp_server/vector/scanner.py index 4f4edf3f..6a5a3758 100644 --- a/nextcloud_mcp_server/vector/scanner.py +++ b/nextcloud_mcp_server/vector/scanner.py @@ -48,7 +48,7 @@ class DocumentTask: """Document task for processing queue.""" user_id: str - doc_id: int | str # int for files/notes, str for legacy + doc_id: str # Always str — see vector/qdrant_client.py keyword index doc_type: str # "note", "file", "calendar" operation: str # "index" or "delete" modified_at: int @@ -228,7 +228,7 @@ async def scan_user_documents( ) indexed_doc_ids = { - point.payload["doc_id"] + str(point.payload["doc_id"]) for point in (scroll_result[0] or []) if point.payload is not None } @@ -401,7 +401,7 @@ async def scan_user_documents( ) indexed_file_ids = { - point.payload["doc_id"] + str(point.payload["doc_id"]) for point in (file_scroll_result[0] or []) if point.payload is not None } @@ -456,7 +456,9 @@ async def scan_user_documents( for file_info in tagged_files: # Files are already filtered by MIME type in find_files_by_tag() file_count += 1 - file_id = file_info["id"] # Use numeric file ID, not path + # Normalize file ID to str — Qdrant doc_id payload is keyword-indexed + # and producers across doc_types must agree on a single type. + file_id = str(file_info["id"]) file_path = file_info["path"] # Keep path for logging nextcloud_file_ids.add(file_id) @@ -679,7 +681,7 @@ async def scan_news_items( limit=10000, ) indexed_item_ids = { - point.payload["doc_id"] + str(point.payload["doc_id"]) for point in (scroll_result[0] or []) if point.payload is not None } @@ -858,7 +860,7 @@ async def scan_deck_cards( limit=10000, ) indexed_card_ids = { - point.payload["doc_id"] + str(point.payload["doc_id"]) for point in (scroll_result[0] or []) if point.payload is not None } diff --git a/nextcloud_mcp_server/vector/visualization.py b/nextcloud_mcp_server/vector/visualization.py index 5ffb4bd1..3da8d2e6 100644 --- a/nextcloud_mcp_server/vector/visualization.py +++ b/nextcloud_mcp_server/vector/visualization.py @@ -70,7 +70,10 @@ async def compute_pca_coordinates( vector = point.vector if vector is not None and point.payload: - doc_id = point.payload.get("doc_id") + # SearchResult.id is str; coerce payload doc_id to match so the + # tuple lookup below succeeds even on legacy int-typed payloads. + raw_doc_id = point.payload.get("doc_id") + doc_id = None if raw_doc_id is None else str(raw_doc_id) chunk_start = point.payload.get("chunk_start_offset") chunk_end = point.payload.get("chunk_end_offset") chunk_key = (doc_id, chunk_start, chunk_end) diff --git a/tests/unit/vector/__init__.py b/tests/unit/vector/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/vector/test_qdrant_client.py b/tests/unit/vector/test_qdrant_client.py new file mode 100644 index 00000000..db232ddd --- /dev/null +++ b/tests/unit/vector/test_qdrant_client.py @@ -0,0 +1,265 @@ +"""Unit tests for Qdrant payload-index helpers and doc_id backfill. + +These cover the startup-time migrations added to ``vector/qdrant_client.py`` +after production HTTP 400 errors revealed that: + +1. The collection had no payload index for ``doc_id``, so any + ``FieldCondition(key="doc_id", ...)`` filter failed at the Qdrant layer. +2. Producers wrote a mix of ``int`` and ``str`` values for ``doc_id``, so a + single keyword index could not have covered both kinds even if it had + existed. + +The fix has three coordinated parts; this module covers the two helpers that +run at startup. Producer-side normalization is exercised by the existing +scanner tests. +""" + +from types import SimpleNamespace +from unittest.mock import call + +import httpx +import pytest +from qdrant_client.http.exceptions import UnexpectedResponse +from qdrant_client.models import PayloadSchemaType + +from nextcloud_mcp_server.vector.qdrant_client import ( + _KEYWORD_PAYLOAD_FIELDS, + _backfill_doc_id_to_string, + _ensure_keyword_payload_indexes, + _has_int_doc_id_sample, +) + + +def _make_unexpected(status_code: int, body: bytes) -> UnexpectedResponse: + """Build a real UnexpectedResponse for raise_for_status-style branches.""" + return UnexpectedResponse( + status_code=status_code, + reason_phrase="Bad Request", + content=body, + headers=httpx.Headers(), + ) + + +def _record(point_id: int | str, doc_id: int | str | None) -> SimpleNamespace: + """Stand-in for qdrant_client.http.models.Record. + + Tests don't need full Pydantic validation — only ``id`` and ``payload`` + are read by the helpers under test. + """ + payload: dict | None = {"doc_id": doc_id} if doc_id is not None else None + return SimpleNamespace(id=point_id, payload=payload) + + +# --------------------------------------------------------------------------- +# _ensure_keyword_payload_indexes +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +async def test_ensure_keyword_payload_indexes_creates_each_field(mocker): + """Happy path: every field in _KEYWORD_PAYLOAD_FIELDS gets a KEYWORD index.""" + client = mocker.AsyncMock() + + await _ensure_keyword_payload_indexes(client, "test-collection") + + assert client.create_payload_index.await_count == len(_KEYWORD_PAYLOAD_FIELDS) + expected_calls = [ + call( + collection_name="test-collection", + field_name=field, + field_schema=PayloadSchemaType.KEYWORD, + wait=True, + ) + for field in _KEYWORD_PAYLOAD_FIELDS + ] + client.create_payload_index.assert_has_awaits(expected_calls, any_order=False) + + +@pytest.mark.unit +async def test_ensure_keyword_payload_indexes_swallows_already_exists(mocker, caplog): + """Idempotent: 'already exists' 400 is logged at debug, not raised.""" + client = mocker.AsyncMock() + # First call succeeds, second raises "already exists", third succeeds — + # exercises the per-field exception handling. + client.create_payload_index.side_effect = [ + None, + _make_unexpected( + 400, b'{"status":{"error":"Index for \\"user_id\\" already exists"}}' + ), + None, + ] + + with caplog.at_level("DEBUG", logger="nextcloud_mcp_server.vector.qdrant_client"): + await _ensure_keyword_payload_indexes(client, "test-collection") + + assert client.create_payload_index.await_count == len(_KEYWORD_PAYLOAD_FIELDS) + # The "already exists" branch logs at DEBUG; nothing reaches WARNING. + assert not any(record.levelname == "WARNING" for record in caplog.records) + + +@pytest.mark.unit +async def test_ensure_keyword_payload_indexes_logs_unrelated_400_as_warning( + mocker, caplog +): + """Schema conflicts and other 400s are surfaced as warnings, not silenced.""" + client = mocker.AsyncMock() + client.create_payload_index.side_effect = [ + _make_unexpected( + 400, + b'{"status":{"error":"field \\"doc_id\\" indexed with different schema"}}', + ), + None, + None, + ] + + with caplog.at_level("WARNING", logger="nextcloud_mcp_server.vector.qdrant_client"): + await _ensure_keyword_payload_indexes(client, "test-collection") + + # Loop continued past the failing field; all three were attempted. + assert client.create_payload_index.await_count == len(_KEYWORD_PAYLOAD_FIELDS) + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert len(warnings) == 1 + assert "different schema" in warnings[0].getMessage() + + +# --------------------------------------------------------------------------- +# _has_int_doc_id_sample +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +async def test_has_int_doc_id_sample_returns_true_when_int_present(mocker): + """Sample finds an int — caller should run the full backfill.""" + client = mocker.AsyncMock() + client.scroll.return_value = ( + [_record(1, "abc"), _record(2, 42), _record(3, "xyz")], + None, + ) + + assert await _has_int_doc_id_sample(client, "c") is True + client.scroll.assert_awaited_once() + + +@pytest.mark.unit +async def test_has_int_doc_id_sample_returns_false_when_all_str(mocker): + """Sample is clean — caller should skip the full scroll.""" + client = mocker.AsyncMock() + client.scroll.return_value = ( + [_record(1, "abc"), _record(2, "def")], + None, + ) + + assert await _has_int_doc_id_sample(client, "c") is False + + +@pytest.mark.unit +async def test_has_int_doc_id_sample_handles_empty_collection(mocker): + """Empty collection — nothing to backfill, return False.""" + client = mocker.AsyncMock() + client.scroll.return_value = ([], None) + + assert await _has_int_doc_id_sample(client, "c") is False + + +@pytest.mark.unit +async def test_has_int_doc_id_sample_ignores_missing_payload(mocker): + """Records with no payload don't count as int doc_ids.""" + client = mocker.AsyncMock() + client.scroll.return_value = ( + [_record(1, None), _record(2, "abc")], + None, + ) + + assert await _has_int_doc_id_sample(client, "c") is False + + +# --------------------------------------------------------------------------- +# _backfill_doc_id_to_string +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +async def test_backfill_skips_when_sample_is_clean(mocker): + """Short-circuit: clean sample → no full scroll, no set_payload calls.""" + client = mocker.AsyncMock() + # Sample call returns only str payloads → backfill should not proceed. + client.scroll.return_value = ([_record(1, "abc"), _record(2, "def")], None) + + await _backfill_doc_id_to_string(client, "test-collection") + + # Exactly one scroll (the sample) and zero rewrites. + assert client.scroll.await_count == 1 + client.set_payload.assert_not_awaited() + + +@pytest.mark.unit +async def test_backfill_rewrites_int_doc_ids_to_str(mocker): + """Mixed int/str payload across two scroll pages: only ints get rewritten.""" + client = mocker.AsyncMock() + # Three scroll calls: + # 1. sample → finds an int, triggers the full pass + # 2. first batch of full scroll → mixed int/str + # 3. second batch → all str, with next_offset=None to terminate + client.scroll.side_effect = [ + ([_record(1, 100), _record(2, "abc")], None), # sample + ([_record(1, 100), _record(2, "abc")], "next-offset-123"), # batch 1 + ([_record(3, 200), _record(4, "def")], None), # batch 2 (terminal) + ] + + await _backfill_doc_id_to_string(client, "test-collection") + + # Two rewrites: point 1 (int 100) in batch 1, point 3 (int 200) in batch 2. + assert client.set_payload.await_count == 2 + client.set_payload.assert_any_await( + collection_name="test-collection", + payload={"doc_id": "100"}, + points=[1], + wait=False, + ) + client.set_payload.assert_any_await( + collection_name="test-collection", + payload={"doc_id": "200"}, + points=[3], + wait=False, + ) + + +@pytest.mark.unit +async def test_backfill_emits_completion_log(mocker, caplog): + """Backfill logs final rewritten/scanned counts at INFO.""" + client = mocker.AsyncMock() + client.scroll.side_effect = [ + ([_record(1, 7)], None), # sample triggers full pass + ([_record(1, 7), _record(2, "x")], None), # single batch, terminal + ] + + with caplog.at_level("INFO", logger="nextcloud_mcp_server.vector.qdrant_client"): + await _backfill_doc_id_to_string(client, "test-collection") + + completion_logs = [ + r.getMessage() for r in caplog.records if "backfill complete" in r.getMessage() + ] + assert completion_logs, "expected an INFO log line for backfill completion" + msg = completion_logs[0] + assert "1/2" in msg, f"expected '1/2' rewritten/scanned in {msg!r}" + + +@pytest.mark.unit +async def test_backfill_handles_none_payload(mocker): + """A point with payload=None is skipped without crashing.""" + client = mocker.AsyncMock() + client.scroll.side_effect = [ + ([_record(1, 99)], None), # sample triggers full pass + ([_record(1, None), _record(2, 99)], None), # batch with one None payload + ] + + await _backfill_doc_id_to_string(client, "test-collection") + + # Only the int doc_id at point 2 was rewritten; the None-payload point was skipped. + assert client.set_payload.await_count == 1 + client.set_payload.assert_awaited_with( + collection_name="test-collection", + payload={"doc_id": "99"}, + points=[2], + wait=False, + ) From 6aba589a6e102beac0c7c29942fed6a68170ab22 Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Fri, 8 May 2026 21:14:28 +0200 Subject: [PATCH 02/27] =?UTF-8?q?fix(vector):=20address=20PR=20review=20?= =?UTF-8?q?=E2=80=94=20wait=3DTrue=20backfill,=20batched=20writes,=20searc?= =?UTF-8?q?h=20helper?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses reviewer feedback on PR #773: - Backfill set_payload now uses wait=True to avoid a race where _ensure_keyword_payload_indexes builds the KEYWORD index before fire-and-forget writes have committed, leaving int payloads invisible to filters. - Batch points sharing the same int doc_id into a single set_payload call (one document → many chunks → one round-trip instead of N). - Drop _has_int_doc_id_sample short-circuit. The sample's false-negative window (clean first 256 results, ints further in) is gone; full scroll is the dominant cost on first run anyway. - Simplify _ensure_keyword_payload_indexes: the "already exists" 400 branch was dead code (Qdrant returns 200 on identical re-create); any 400 now logs a warning and continues. - search/context.py: comment the broadened file-type guard. Add explicit not doc_id.isdigit() checks at the top of note/news_item/deck_card branches in _fetch_document_text so malformed payloads surface as warnings instead of being swallowed by the broad except. Also extracts build_search_result_from_point into search/algorithms.py to deduplicate the 71-line payload-extraction loop shared by SemanticSearchAlgorithm and BM25HybridSearchAlgorithm. This fixes SonarQube's quality-gate failure (4.0% new-code duplication, max 3%). Test coverage: - 7 new unit tests for build_search_result_from_point covering missing payload, note/file/deck_card metadata, int doc_id coercion, and metadata_extras merging. - Replace _has_int_doc_id_sample tests with clean-collection no-op and per-batch grouping tests. - Update set_payload assertions from wait=False to wait=True. Co-Authored-By: Claude Opus 4.7 (1M context) --- nextcloud_mcp_server/search/algorithms.py | 66 +++++++- nextcloud_mcp_server/search/bm25_hybrid.py | 76 +++------ nextcloud_mcp_server/search/context.py | 38 ++++- nextcloud_mcp_server/search/semantic.py | 65 ++------ nextcloud_mcp_server/vector/qdrant_client.py | 76 ++++----- tests/unit/search/test_search_result.py | 158 ++++++++++++++++++- tests/unit/vector/test_qdrant_client.py | 158 ++++++++----------- 7 files changed, 385 insertions(+), 252 deletions(-) diff --git a/nextcloud_mcp_server/search/algorithms.py b/nextcloud_mcp_server/search/algorithms.py index 1d14fdf7..7b120393 100644 --- a/nextcloud_mcp_server/search/algorithms.py +++ b/nextcloud_mcp_server/search/algorithms.py @@ -5,7 +5,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from typing import Any, Protocol, runtime_checkable -from qdrant_client.models import FieldCondition, Filter, MatchValue +from qdrant_client.models import FieldCondition, Filter, MatchValue, ScoredPoint from nextcloud_mcp_server.config import get_settings from nextcloud_mcp_server.vector.placeholder import get_placeholder_filter @@ -181,6 +181,70 @@ class SearchResult: raise ValueError(f"Score must be non-negative, got {self.score}") +def build_search_result_from_point( + point: ScoredPoint, + *, + metadata_extras: dict[str, Any] | None = None, +) -> SearchResult | None: + """Construct a SearchResult from a Qdrant ScoredPoint payload. + + Returns ``None`` when the payload is missing — callers should skip the + point. The defensive ``str()`` coercion on ``doc_id`` covers legacy int + payloads until the startup backfill has run everywhere (see + ``vector/qdrant_client.py:_backfill_doc_id_to_string``). + + Args: + point: A Qdrant ``ScoredPoint`` from a search response. + metadata_extras: Algorithm-specific metadata merged into the result's + ``metadata`` dict (e.g., ``{"search_method": "bm25_hybrid_rrf"}``). + + Returns: + A populated ``SearchResult``, or ``None`` if ``point.payload`` is + missing. + """ + if point.payload is None: + return None + + doc_id = str(point.payload["doc_id"]) + doc_type = point.payload.get("doc_type", "note") + + metadata: dict[str, Any] = { + "chunk_index": point.payload.get("chunk_index"), + "total_chunks": point.payload.get("total_chunks"), + } + if metadata_extras: + metadata.update(metadata_extras) + + # File-specific metadata for PDF viewer + if doc_type == "file" and (path := point.payload.get("file_path")): + metadata["path"] = path + + # Deck-card metadata for frontend URL construction and verify-on-read + # (ADR-019) — both board_id and stack_id are required to call + # deck.get_card without an O(boards × stacks) iteration fallback. + if doc_type == "deck_card": + if board_id := point.payload.get("board_id"): + metadata["board_id"] = board_id + if stack_id := point.payload.get("stack_id"): + metadata["stack_id"] = stack_id + + return SearchResult( + id=doc_id, + doc_type=doc_type, + title=point.payload.get("title", "Untitled"), + excerpt=point.payload.get("excerpt", ""), + score=point.score, + metadata=metadata, + chunk_start_offset=point.payload.get("chunk_start_offset"), + chunk_end_offset=point.payload.get("chunk_end_offset"), + page_number=point.payload.get("page_number"), + page_count=point.payload.get("page_count"), + chunk_index=point.payload.get("chunk_index", 0), + total_chunks=point.payload.get("total_chunks", 1), + point_id=str(point.id), + ) + + class SearchAlgorithm(ABC): """Abstract base class for search algorithms. diff --git a/nextcloud_mcp_server/search/bm25_hybrid.py b/nextcloud_mcp_server/search/bm25_hybrid.py index c6f8dfda..a400b957 100644 --- a/nextcloud_mcp_server/search/bm25_hybrid.py +++ b/nextcloud_mcp_server/search/bm25_hybrid.py @@ -10,7 +10,11 @@ from nextcloud_mcp_server.config import get_settings from nextcloud_mcp_server.embedding import get_bm25_service, get_embedding_service from nextcloud_mcp_server.observability.metrics import record_qdrant_operation from nextcloud_mcp_server.observability.tracing import trace_operation -from nextcloud_mcp_server.search.algorithms import SearchAlgorithm, SearchResult +from nextcloud_mcp_server.search.algorithms import ( + SearchAlgorithm, + SearchResult, + build_search_result_from_point, +) from nextcloud_mcp_server.vector.placeholder import get_placeholder_filter from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client @@ -202,66 +206,30 @@ class BM25HybridSearchAlgorithm(SearchAlgorithm): "search.deduplicate", attributes={"dedupe.num_points": len(search_response.points)}, ): - seen_chunks = set() - results = [] + seen_chunks: set[tuple[str, str, Any, Any]] = set() + results: list[SearchResult] = [] + metadata_extras = { + "search_method": f"bm25_hybrid_{self.fusion_name}", + } - for result in search_response.points: - if result.payload is None: + for point in search_response.points: + sr = build_search_result_from_point( + point, metadata_extras=metadata_extras + ) + if sr is None: continue - # doc_id is always str post-normalization, but defensively coerce - # legacy int payloads on read until the backfill has run everywhere. - doc_id = str(result.payload["doc_id"]) - doc_type = result.payload.get("doc_type", "note") - chunk_start = result.payload.get("chunk_start_offset") - chunk_end = result.payload.get("chunk_end_offset") - chunk_key = (doc_id, doc_type, chunk_start, chunk_end) - # Skip if we've already seen this exact chunk + chunk_key = ( + sr.id, + sr.doc_type, + sr.chunk_start_offset, + sr.chunk_end_offset, + ) if chunk_key in seen_chunks: continue - seen_chunks.add(chunk_key) - # Build metadata dict with common fields - metadata = { - "chunk_index": result.payload.get("chunk_index"), - "total_chunks": result.payload.get("total_chunks"), - "search_method": f"bm25_hybrid_{self.fusion_name}", - } - - # Add file-specific metadata for PDF viewer - if doc_type == "file" and (path := result.payload.get("file_path")): - metadata["path"] = path - - # Add deck_card-specific metadata for frontend URL construction - # and verify-on-read (ADR-019) — both board_id and stack_id are - # required to call deck.get_card without an O(boards × stacks) - # iteration fallback. - if doc_type == "deck_card": - if board_id := result.payload.get("board_id"): - metadata["board_id"] = board_id - if stack_id := result.payload.get("stack_id"): - metadata["stack_id"] = stack_id - - # Return unverified results (verification happens at output stage) - results.append( - SearchResult( - id=doc_id, - doc_type=doc_type, - title=result.payload.get("title", "Untitled"), - excerpt=result.payload.get("excerpt", ""), - score=result.score, # Fusion score (RRF or DBSF) - metadata=metadata, - chunk_start_offset=result.payload.get("chunk_start_offset"), - chunk_end_offset=result.payload.get("chunk_end_offset"), - page_number=result.payload.get("page_number"), - page_count=result.payload.get("page_count"), - chunk_index=result.payload.get("chunk_index", 0), - total_chunks=result.payload.get("total_chunks", 1), - point_id=str(result.id), # Qdrant point ID for batch retrieval - ) - ) - + results.append(sr) if len(results) >= limit: break diff --git a/nextcloud_mcp_server/search/context.py b/nextcloud_mcp_server/search/context.py index c249bb73..c3d4827a 100644 --- a/nextcloud_mcp_server/search/context.py +++ b/nextcloud_mcp_server/search/context.py @@ -415,8 +415,13 @@ async def get_chunk_with_context( f"(Qdrant cache miss, possibly legacy data)" ) - # For files, the doc_id is the numeric file ID (as a string) — resolve it - # to a WebDAV path so _fetch_document_text can retrieve the binary content. + # For files, doc_id is always the stringified numeric file ID after + # producer normalization — resolve it to a WebDAV path so + # _fetch_document_text can retrieve the binary content. The previous + # `isinstance(doc_id, int)` guard is no longer needed: file producers + # write str(file_id) and the startup backfill rewrites legacy int + # payloads. If lookup fails (e.g. truly malformed legacy data), the + # caller logs and returns None below — a re-index is the recovery path. resolved_doc_id = doc_id if doc_type == "file": file_path = await _get_file_path_from_qdrant( @@ -506,6 +511,15 @@ async def _fetch_document_text( """ try: if doc_type == "note": + # Note IDs are integers in the Nextcloud API; reject non-numeric + # doc_ids explicitly so a malformed payload surfaces in logs + # rather than getting silently swallowed by `except Exception`. + if not doc_id.isdigit(): + logger.warning( + "Expected numeric note doc_id, got %r — skipping document fetch", + doc_id, + ) + return None # Fetch note by ID note = await nc_client.notes.get_note(note_id=int(doc_id)) # Reconstruct full content as indexed: title + "\n\n" + content @@ -562,6 +576,15 @@ async def _fetch_document_text( ) return None elif doc_type == "news_item": + # News item IDs are integers in the Nextcloud News API; reject + # non-numeric doc_ids explicitly so malformed payloads surface + # rather than getting swallowed by the broad except below. + if not doc_id.isdigit(): + logger.warning( + "Expected numeric news_item doc_id, got %r — skipping document fetch", + doc_id, + ) + return None # Fetch news item by ID item = await nc_client.news.get_item(int(doc_id)) # Reconstruct full content as indexed: title + source + URL + body @@ -580,6 +603,17 @@ async def _fetch_document_text( content_parts.append(body_markdown) return "\n".join(content_parts) elif doc_type == "deck_card": + # Deck card IDs are integers in the Nextcloud Deck API; reject + # non-numeric doc_ids explicitly so malformed payloads surface + # rather than getting swallowed by the broad except below. The + # numeric check covers both the metadata-fast-path (line ~600) + # and the iteration fallback (line ~635). + if not doc_id.isdigit(): + logger.warning( + "Expected numeric deck_card doc_id, got %r — skipping document fetch", + doc_id, + ) + return None # Fetch card from Deck API # Try to get board_id/stack_id from Qdrant metadata (O(1) lookup) # Otherwise fall back to iteration (legacy data) diff --git a/nextcloud_mcp_server/search/semantic.py b/nextcloud_mcp_server/search/semantic.py index d391b653..cb15eade 100644 --- a/nextcloud_mcp_server/search/semantic.py +++ b/nextcloud_mcp_server/search/semantic.py @@ -8,7 +8,11 @@ from qdrant_client.models import FieldCondition, Filter, MatchValue from nextcloud_mcp_server.config import get_settings from nextcloud_mcp_server.embedding import get_embedding_service from nextcloud_mcp_server.observability.metrics import record_qdrant_operation -from nextcloud_mcp_server.search.algorithms import SearchAlgorithm, SearchResult +from nextcloud_mcp_server.search.algorithms import ( + SearchAlgorithm, + SearchResult, + build_search_result_from_point, +) from nextcloud_mcp_server.vector.placeholder import get_placeholder_filter from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client @@ -134,65 +138,20 @@ class SemanticSearchAlgorithm(SearchAlgorithm): # Deduplicate by (doc_id, doc_type, chunk_start, chunk_end) # This allows multiple chunks from same doc, but removes duplicate chunks - seen_chunks = set() - results = [] + seen_chunks: set[tuple[str, str, Any, Any]] = set() + results: list[SearchResult] = [] - for result in search_response.points: - if result.payload is None: + for point in search_response.points: + sr = build_search_result_from_point(point) + if sr is None: continue - # doc_id is always str post-normalization, but defensively coerce - # legacy int payloads on read until the backfill has run everywhere. - doc_id = str(result.payload["doc_id"]) - doc_type = result.payload.get("doc_type", "note") - chunk_start = result.payload.get("chunk_start_offset") - chunk_end = result.payload.get("chunk_end_offset") - chunk_key = (doc_id, doc_type, chunk_start, chunk_end) - # Skip if we've already seen this exact chunk + chunk_key = (sr.id, sr.doc_type, sr.chunk_start_offset, sr.chunk_end_offset) if chunk_key in seen_chunks: continue - seen_chunks.add(chunk_key) - # Build metadata dict with common fields - metadata = { - "chunk_index": result.payload.get("chunk_index"), - "total_chunks": result.payload.get("total_chunks"), - } - - # Add file-specific metadata for PDF viewer - if doc_type == "file" and (path := result.payload.get("file_path")): - metadata["path"] = path - - # Add deck_card-specific metadata for frontend URL construction - # and verify-on-read (ADR-019) — both board_id and stack_id are - # required to call deck.get_card without an O(boards × stacks) - # iteration fallback. - if doc_type == "deck_card": - if board_id := result.payload.get("board_id"): - metadata["board_id"] = board_id - if stack_id := result.payload.get("stack_id"): - metadata["stack_id"] = stack_id - - # Return unverified results (verification happens at output stage) - results.append( - SearchResult( - id=doc_id, - doc_type=doc_type, - title=result.payload.get("title", "Untitled"), - excerpt=result.payload.get("excerpt", ""), - score=result.score, - metadata=metadata, - chunk_start_offset=result.payload.get("chunk_start_offset"), - chunk_end_offset=result.payload.get("chunk_end_offset"), - page_number=result.payload.get("page_number"), - page_count=result.payload.get("page_count"), - chunk_index=result.payload.get("chunk_index", 0), - total_chunks=result.payload.get("total_chunks", 1), - point_id=str(result.id), # Qdrant point ID for batch retrieval - ) - ) - + results.append(sr) if len(results) >= limit: break diff --git a/nextcloud_mcp_server/vector/qdrant_client.py b/nextcloud_mcp_server/vector/qdrant_client.py index a192eeec..34b14b3a 100644 --- a/nextcloud_mcp_server/vector/qdrant_client.py +++ b/nextcloud_mcp_server/vector/qdrant_client.py @@ -1,6 +1,7 @@ """Qdrant client wrapper.""" import logging +from typing import Any from qdrant_client import AsyncQdrantClient, models from qdrant_client.http.exceptions import UnexpectedResponse @@ -28,8 +29,10 @@ async def _ensure_keyword_payload_indexes( ) -> None: """Create KEYWORD payload indexes for fields used in exact-match filters. - Idempotent: tolerates 'already exists' errors so it can run on every - startup against existing collections. + Idempotent at the Qdrant layer: re-creating an identical index returns + 200, so this can run on every startup. Schema conflicts (a pre-existing + index with a different type) surface as a 400 — log loudly so operators + can intervene, but keep going so the remaining fields still get indexed. """ for field in _KEYWORD_PAYLOAD_FIELDS: try: @@ -41,39 +44,11 @@ async def _ensure_keyword_payload_indexes( ) logger.info("Created KEYWORD payload index on '%s'", field) except UnexpectedResponse as e: - # Qdrant returns 400 if the index already exists with a different - # schema, or simply succeeds if it already matches. Treat - # already-exists as benign; surface schema conflicts loudly. body = getattr(e, "content", b"") or b"" body_text = body.decode("utf-8", errors="replace") - if "already exists" in body_text.lower(): - logger.debug("Payload index on '%s' already exists", field) - else: - logger.warning( - "Failed to create payload index on '%s': %s", field, body_text - ) - - -async def _has_int_doc_id_sample( - client: AsyncQdrantClient, collection_name: str, sample_size: int = 256 -) -> bool: - """Quick sample to decide whether the full backfill scroll is needed. - - Reading the first batch is cheap; if all sampled doc_ids are already str - (the steady-state on healthy collections), we skip the full pass. - """ - points, _ = await client.scroll( - collection_name=collection_name, - limit=sample_size, - with_payload=["doc_id"], - with_vectors=False, - ) - for point in points: - payload = point.payload or {} - value = payload.get("doc_id") - if value is not None and not isinstance(value, str): - return True - return False + logger.warning( + "Failed to create payload index on '%s': %s", field, body_text + ) async def _backfill_doc_id_to_string( @@ -84,18 +59,15 @@ async def _backfill_doc_id_to_string( Producers now uniformly write str(doc_id), but historical points may carry int values from before normalization. A KEYWORD index does not match int payloads, so any leftover int doc_ids would be silently invisible to - filters. Scroll all points and convert in-place. Idempotent. + filters. Scrolls all points once and converts in-place; idempotent (a + second pass over the same collection performs zero writes). - Skipped when the first sample batch already contains only str doc_ids. + Within each scroll batch, points sharing the same int doc_id are batched + into a single ``set_payload`` call to minimize Qdrant round-trips. """ - if not await _has_int_doc_id_sample(client, collection_name): - logger.debug( - "doc_id backfill: sample shows no legacy int payloads; skipping full scan" - ) - return - logger.info( - "Running doc_id backfill on '%s' (this may take a moment for large collections)", + "Scanning '%s' for legacy int doc_id payloads (this is a one-time " + "migration on first start after upgrade)", collection_name, ) @@ -117,6 +89,11 @@ async def _backfill_doc_id_to_string( if not points: break + # Group by stringified value so points sharing a doc_id (one document + # → many chunks) collapse into a single set_payload call. Point IDs + # can be int/str/UUID, so widen the value type to satisfy the qdrant + # client's PointsSelector signature without re-spelling the union. + by_value: dict[str, list[Any]] = {} for point in points: scanned += 1 # Qdrant client typing allows None payload even when with_payload @@ -125,13 +102,20 @@ async def _backfill_doc_id_to_string( value = payload.get("doc_id") if value is None or isinstance(value, str): continue + by_value.setdefault(str(value), []).append(point.id) + + for str_val, point_ids in by_value.items(): + # wait=True is required: _ensure_keyword_payload_indexes runs + # immediately after this function and only indexes committed + # data — fire-and-forget writes would leave int payloads + # invisible to KEYWORD filters. await client.set_payload( collection_name=collection_name, - payload={"doc_id": str(value)}, - points=[point.id], - wait=False, + payload={"doc_id": str_val}, + points=point_ids, + wait=True, ) - rewritten += 1 + rewritten += len(point_ids) if next_offset is None: break diff --git a/tests/unit/search/test_search_result.py b/tests/unit/search/test_search_result.py index c9dbf0b1..52bf15c0 100644 --- a/tests/unit/search/test_search_result.py +++ b/tests/unit/search/test_search_result.py @@ -1,8 +1,22 @@ """Unit tests for SearchResult validation.""" +from types import SimpleNamespace + import pytest -from nextcloud_mcp_server.search.algorithms import SearchResult +from nextcloud_mcp_server.search.algorithms import ( + SearchResult, + build_search_result_from_point, +) + + +def _make_point(point_id, payload, score=0.5): + """Stand-in for qdrant_client.models.ScoredPoint. + + The helper only reads ``id``, ``payload``, and ``score`` — full Pydantic + validation isn't required for unit tests. + """ + return SimpleNamespace(id=point_id, payload=payload, score=score) @pytest.mark.unit @@ -133,3 +147,145 @@ def test_search_result_with_chunk_offsets(): assert result.chunk_start_offset == 100 assert result.chunk_end_offset == 500 + + +# --------------------------------------------------------------------------- +# build_search_result_from_point +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_build_search_result_from_point_returns_none_when_payload_missing(): + """Helper signals the caller to skip the point by returning None.""" + point = _make_point(point_id="p1", payload=None) + + assert build_search_result_from_point(point) is None + + +@pytest.mark.unit +def test_build_search_result_from_point_note_payload(): + """Note-type payload populates the SearchResult fields without metadata extras.""" + point = _make_point( + point_id="p-1", + payload={ + "doc_id": "42", + "doc_type": "note", + "title": "Hello", + "excerpt": "world", + "chunk_start_offset": 0, + "chunk_end_offset": 100, + "chunk_index": 0, + "total_chunks": 2, + }, + score=0.91, + ) + + sr = build_search_result_from_point(point) + + assert sr is not None + assert sr.id == "42" + assert sr.doc_type == "note" + assert sr.title == "Hello" + assert sr.excerpt == "world" + assert sr.score == 0.91 + assert sr.chunk_start_offset == 0 + assert sr.chunk_end_offset == 100 + assert sr.chunk_index == 0 + assert sr.total_chunks == 2 + assert sr.point_id == "p-1" + assert sr.metadata == {"chunk_index": 0, "total_chunks": 2} + + +@pytest.mark.unit +def test_build_search_result_from_point_coerces_int_doc_id_to_str(): + """Legacy int doc_id payloads are stringified defensively.""" + point = _make_point( + point_id=1, + payload={"doc_id": 7, "doc_type": "note"}, + score=0.5, + ) + + sr = build_search_result_from_point(point) + + assert sr is not None + assert sr.id == "7" + + +@pytest.mark.unit +def test_build_search_result_from_point_file_metadata_includes_path(): + """File-type payloads with a file_path attach it under metadata['path'].""" + point = _make_point( + point_id="p-2", + payload={ + "doc_id": "100", + "doc_type": "file", + "file_path": "/Documents/report.pdf", + "page_number": 3, + "page_count": 12, + }, + ) + + sr = build_search_result_from_point(point) + + assert sr is not None + assert sr.doc_type == "file" + assert sr.metadata["path"] == "/Documents/report.pdf" + assert sr.page_number == 3 + assert sr.page_count == 12 + + +@pytest.mark.unit +def test_build_search_result_from_point_deck_card_metadata(): + """Deck-card payloads carry board_id/stack_id forward for verify-on-read.""" + point = _make_point( + point_id="p-3", + payload={ + "doc_id": "55", + "doc_type": "deck_card", + "board_id": 7, + "stack_id": 12, + "title": "Card", + }, + ) + + sr = build_search_result_from_point(point) + + assert sr is not None + assert sr.metadata["board_id"] == 7 + assert sr.metadata["stack_id"] == 12 + + +@pytest.mark.unit +def test_build_search_result_from_point_merges_metadata_extras(): + """metadata_extras override/augment the helper's computed metadata dict.""" + point = _make_point( + point_id="p-4", + payload={"doc_id": "1", "doc_type": "note"}, + ) + + sr = build_search_result_from_point( + point, metadata_extras={"search_method": "bm25_hybrid_rrf"} + ) + + assert sr is not None + assert sr.metadata["search_method"] == "bm25_hybrid_rrf" + # Common fields still present + assert "chunk_index" in sr.metadata + assert "total_chunks" in sr.metadata + + +@pytest.mark.unit +def test_build_search_result_from_point_defaults_when_optional_fields_missing(): + """Missing optional payload keys fall back to documented defaults.""" + point = _make_point(point_id="p-5", payload={"doc_id": "1"}) + + sr = build_search_result_from_point(point) + + assert sr is not None + assert sr.doc_type == "note" # default doc_type + assert sr.title == "Untitled" + assert sr.excerpt == "" + assert sr.chunk_index == 0 + assert sr.total_chunks == 1 + assert sr.chunk_start_offset is None + assert sr.chunk_end_offset is None diff --git a/tests/unit/vector/test_qdrant_client.py b/tests/unit/vector/test_qdrant_client.py index db232ddd..ab5e38c7 100644 --- a/tests/unit/vector/test_qdrant_client.py +++ b/tests/unit/vector/test_qdrant_client.py @@ -26,7 +26,6 @@ from nextcloud_mcp_server.vector.qdrant_client import ( _KEYWORD_PAYLOAD_FIELDS, _backfill_doc_id_to_string, _ensure_keyword_payload_indexes, - _has_int_doc_id_sample, ) @@ -76,32 +75,14 @@ async def test_ensure_keyword_payload_indexes_creates_each_field(mocker): @pytest.mark.unit -async def test_ensure_keyword_payload_indexes_swallows_already_exists(mocker, caplog): - """Idempotent: 'already exists' 400 is logged at debug, not raised.""" - client = mocker.AsyncMock() - # First call succeeds, second raises "already exists", third succeeds — - # exercises the per-field exception handling. - client.create_payload_index.side_effect = [ - None, - _make_unexpected( - 400, b'{"status":{"error":"Index for \\"user_id\\" already exists"}}' - ), - None, - ] +async def test_ensure_keyword_payload_indexes_logs_400_as_warning(mocker, caplog): + """Any 400 from create_payload_index is logged at WARNING and skipped. - with caplog.at_level("DEBUG", logger="nextcloud_mcp_server.vector.qdrant_client"): - await _ensure_keyword_payload_indexes(client, "test-collection") - - assert client.create_payload_index.await_count == len(_KEYWORD_PAYLOAD_FIELDS) - # The "already exists" branch logs at DEBUG; nothing reaches WARNING. - assert not any(record.levelname == "WARNING" for record in caplog.records) - - -@pytest.mark.unit -async def test_ensure_keyword_payload_indexes_logs_unrelated_400_as_warning( - mocker, caplog -): - """Schema conflicts and other 400s are surfaced as warnings, not silenced.""" + Real Qdrant returns 200 when the index already exists with a matching + schema, so 400s indicate a genuine problem (e.g., schema conflict on a + pre-existing index). The loop continues past the failure so the + remaining fields still get indexed. + """ client = mocker.AsyncMock() client.create_payload_index.side_effect = [ _make_unexpected( @@ -123,104 +104,93 @@ async def test_ensure_keyword_payload_indexes_logs_unrelated_400_as_warning( # --------------------------------------------------------------------------- -# _has_int_doc_id_sample +# _backfill_doc_id_to_string # --------------------------------------------------------------------------- @pytest.mark.unit -async def test_has_int_doc_id_sample_returns_true_when_int_present(mocker): - """Sample finds an int — caller should run the full backfill.""" - client = mocker.AsyncMock() - client.scroll.return_value = ( - [_record(1, "abc"), _record(2, 42), _record(3, "xyz")], - None, - ) +async def test_backfill_clean_collection_makes_no_writes(mocker, caplog): + """A collection with only str doc_ids triggers zero set_payload calls. - assert await _has_int_doc_id_sample(client, "c") is True - client.scroll.assert_awaited_once() - - -@pytest.mark.unit -async def test_has_int_doc_id_sample_returns_false_when_all_str(mocker): - """Sample is clean — caller should skip the full scroll.""" + Verifies idempotency: a second pass over an already-migrated collection + is a no-op modulo the read. + """ client = mocker.AsyncMock() client.scroll.return_value = ( [_record(1, "abc"), _record(2, "def")], None, ) - assert await _has_int_doc_id_sample(client, "c") is False + with caplog.at_level("INFO", logger="nextcloud_mcp_server.vector.qdrant_client"): + await _backfill_doc_id_to_string(client, "test-collection") - -@pytest.mark.unit -async def test_has_int_doc_id_sample_handles_empty_collection(mocker): - """Empty collection — nothing to backfill, return False.""" - client = mocker.AsyncMock() - client.scroll.return_value = ([], None) - - assert await _has_int_doc_id_sample(client, "c") is False - - -@pytest.mark.unit -async def test_has_int_doc_id_sample_ignores_missing_payload(mocker): - """Records with no payload don't count as int doc_ids.""" - client = mocker.AsyncMock() - client.scroll.return_value = ( - [_record(1, None), _record(2, "abc")], - None, - ) - - assert await _has_int_doc_id_sample(client, "c") is False - - -# --------------------------------------------------------------------------- -# _backfill_doc_id_to_string -# --------------------------------------------------------------------------- - - -@pytest.mark.unit -async def test_backfill_skips_when_sample_is_clean(mocker): - """Short-circuit: clean sample → no full scroll, no set_payload calls.""" - client = mocker.AsyncMock() - # Sample call returns only str payloads → backfill should not proceed. - client.scroll.return_value = ([_record(1, "abc"), _record(2, "def")], None) - - await _backfill_doc_id_to_string(client, "test-collection") - - # Exactly one scroll (the sample) and zero rewrites. - assert client.scroll.await_count == 1 client.set_payload.assert_not_awaited() + completion_logs = [ + r.getMessage() for r in caplog.records if "backfill complete" in r.getMessage() + ] + assert completion_logs, "expected an INFO log line for backfill completion" + assert "0/2" in completion_logs[0] @pytest.mark.unit async def test_backfill_rewrites_int_doc_ids_to_str(mocker): """Mixed int/str payload across two scroll pages: only ints get rewritten.""" client = mocker.AsyncMock() - # Three scroll calls: - # 1. sample → finds an int, triggers the full pass - # 2. first batch of full scroll → mixed int/str - # 3. second batch → all str, with next_offset=None to terminate + # Two scroll calls: batch 1 is mixed and reports a next_offset; batch 2 + # is mixed with next_offset=None to terminate. client.scroll.side_effect = [ - ([_record(1, 100), _record(2, "abc")], None), # sample - ([_record(1, 100), _record(2, "abc")], "next-offset-123"), # batch 1 - ([_record(3, 200), _record(4, "def")], None), # batch 2 (terminal) + ([_record(1, 100), _record(2, "abc")], "next-offset-123"), + ([_record(3, 200), _record(4, "def")], None), ] await _backfill_doc_id_to_string(client, "test-collection") - # Two rewrites: point 1 (int 100) in batch 1, point 3 (int 200) in batch 2. + # One set_payload per *unique* int value — point 1 (100) and point 3 + # (200) are in different batches with different values, so two calls. assert client.set_payload.await_count == 2 client.set_payload.assert_any_await( collection_name="test-collection", payload={"doc_id": "100"}, points=[1], - wait=False, + wait=True, ) client.set_payload.assert_any_await( collection_name="test-collection", payload={"doc_id": "200"}, points=[3], - wait=False, + wait=True, + ) + + +@pytest.mark.unit +async def test_backfill_batches_points_with_same_doc_id(mocker): + """Multiple points sharing the same int doc_id collapse to one set_payload. + + A single document indexed as multiple chunks all share its doc_id; the + backfill should issue one set_payload call covering the chunk batch. + """ + client = mocker.AsyncMock() + client.scroll.side_effect = [ + ( + [ + _record(10, 42), + _record(11, 42), + _record(12, 42), + _record(13, "already-str"), + ], + None, + ), + ] + + await _backfill_doc_id_to_string(client, "test-collection") + + # All three int-payload points share doc_id=42, so a single call covers them. + assert client.set_payload.await_count == 1 + client.set_payload.assert_awaited_with( + collection_name="test-collection", + payload={"doc_id": "42"}, + points=[10, 11, 12], + wait=True, ) @@ -229,8 +199,7 @@ async def test_backfill_emits_completion_log(mocker, caplog): """Backfill logs final rewritten/scanned counts at INFO.""" client = mocker.AsyncMock() client.scroll.side_effect = [ - ([_record(1, 7)], None), # sample triggers full pass - ([_record(1, 7), _record(2, "x")], None), # single batch, terminal + ([_record(1, 7), _record(2, "x")], None), ] with caplog.at_level("INFO", logger="nextcloud_mcp_server.vector.qdrant_client"): @@ -249,8 +218,7 @@ async def test_backfill_handles_none_payload(mocker): """A point with payload=None is skipped without crashing.""" client = mocker.AsyncMock() client.scroll.side_effect = [ - ([_record(1, 99)], None), # sample triggers full pass - ([_record(1, None), _record(2, 99)], None), # batch with one None payload + ([_record(1, None), _record(2, 99)], None), ] await _backfill_doc_id_to_string(client, "test-collection") @@ -261,5 +229,5 @@ async def test_backfill_handles_none_payload(mocker): collection_name="test-collection", payload={"doc_id": "99"}, points=[2], - wait=False, + wait=True, ) From b5b4025bb4511c7aaf98d31e87ec05f6810ad5c0 Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Fri, 8 May 2026 21:37:10 +0200 Subject: [PATCH 03/27] =?UTF-8?q?fix(vector):=20address=20PR=20review=20ro?= =?UTF-8?q?und=202=20=E2=80=94=20status=20branching,=20doc=5Fid=20guard,?= =?UTF-8?q?=20doc=20restore?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - _ensure_keyword_payload_indexes: distinguish 400 (schema conflict, warning) from other status codes (5xx/network, error) so a transient outage doesn't silently leave the collection unindexed. - build_search_result_from_point: use .get("doc_id") + return None on missing instead of KeyError-crashing the search; reverse metadata merge order so payload-derived chunk_index/total_chunks win over caller-supplied extras. - docs/configuration.md: restore the OpenAI/Mistral/Bedrock/Simple provider sections + reference-table rows that were dropped in the rebase. Reword the "Startup migrations" bullet to describe what the code actually does (no sampling — full scroll, zero writes when clean). Add operator note about the SemanticSearchResult.id TypeError path. - tests: pytest.approx for float equality (Sonar python:S1244); coverage for non-400 → ERROR, payload={doc_id: None}, and missing doc_id key. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/configuration.md | 90 ++++++++++++++++++-- nextcloud_mcp_server/search/algorithms.py | 21 +++-- nextcloud_mcp_server/vector/qdrant_client.py | 18 +++- tests/unit/search/test_search_result.py | 35 ++++++-- tests/unit/vector/test_qdrant_client.py | 50 +++++++++++ 5 files changed, 188 insertions(+), 26 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 0214bf50..bd7dd87f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -337,14 +337,24 @@ server runs two idempotent migrations: 1. **Payload-index creation** — adds `KEYWORD` payload indexes for `doc_id`, `user_id`, and `doc_type`. Required by Qdrant for any `FieldCondition` filter. Cheap; runs even on healthy collections. -2. **`doc_id` backfill** — rewrites any legacy integer `doc_id` payloads to - strings so they match the keyword index. Skipped after a quick sample - shows the collection is already clean. On a dirty collection, the full - scroll runs once and writes are emitted point-by-point — expect a delay - proportional to point count on the first startup after the upgrade. +2. **`doc_id` backfill** — scans the collection once and rewrites any + legacy integer `doc_id` payloads to strings so they match the keyword + index. Idempotent: on a clean collection (all `doc_id` values already + `str`), the scroll runs but emits zero writes. On the first start after + the upgrade, expect a delay proportional to point count while writes + are issued. Both steps emit INFO-level log lines so operators can track progress. +> **Operator note:** if the server logs `TypeError: SemanticSearchResult.id +> must be int-convertible` after upgrading, this indicates a `doc_type` +> with non-numeric ids has been indexed but the public response model +> (`SemanticSearchResult.id: int`) has not been widened to accept strings. +> Semantic search itself is not broken — the boundary cast in +> `server/semantic.py` is failing loudly on purpose so the discrepancy is +> caught early. Either widen the public model's `id` field or convert the +> id at the verifier layer. + #### Explicit Override Set `QDRANT_COLLECTION` to use a specific collection name: @@ -426,9 +436,16 @@ DOCUMENT_CHUNK_OVERLAP=50 # Overlapping words between chunks (defaul ### Embedding Service Configuration -The server uses an embedding service to generate vector representations. Two options are available: +The server picks an embedding provider via auto-detection. Priority order +(see `nextcloud_mcp_server/providers/registry.py`): -#### Ollama (Recommended) +1. **Bedrock** — if `AWS_REGION` or `BEDROCK_EMBEDDING_MODEL` is set +2. **OpenAI** — if `OPENAI_API_KEY` is set +3. **Mistral** — if `MISTRAL_API_KEY` is set +4. **Ollama** — if `OLLAMA_BASE_URL` is set +5. **Simple** — fallback when nothing else is configured + +#### Ollama (Recommended for self-hosted) Use a local Ollama instance for embeddings: @@ -438,9 +455,52 @@ OLLAMA_EMBEDDING_MODEL=nomic-embed-text # Default model OLLAMA_VERIFY_SSL=true # Verify SSL certificates ``` +#### OpenAI + +Hosted OpenAI embeddings (or any OpenAI-compatible API via `OPENAI_BASE_URL`): + +```dotenv +OPENAI_API_KEY=sk-... +OPENAI_EMBEDDING_MODEL=text-embedding-3-small # default +# OPENAI_BASE_URL=https://models.github.ai/inference # optional +``` + +#### Mistral + +Hosted Mistral embeddings. Requires a Mistral API key from +[console.mistral.ai](https://console.mistral.ai). Currently embeddings only +(no text generation). + +```dotenv +MISTRAL_API_KEY=... +MISTRAL_EMBEDDING_MODEL=mistral-embed # default; produces 1024-dim vectors +# MISTRAL_BASE_URL=https://api.mistral.ai # optional override (proxies, on-prem) +``` + +Switching to or from Mistral forces a new Qdrant collection because the +collection name encodes the model (see "Qdrant Collection Naming" above). + +#### Amazon Bedrock + +Bedrock provides hosted embedding models (Titan, Cohere) and uses the AWS +credential chain (env vars, profiles, or IAM role): + +```dotenv +AWS_REGION=us-east-1 +BEDROCK_EMBEDDING_MODEL=amazon.titan-embed-text-v2:0 +# AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY are optional — boto3 will use +# the standard credential chain if not set. +``` + #### Simple Embedding Provider (Fallback) -If `OLLAMA_BASE_URL` is not set, the server uses a simple random embedding provider for testing. This is **not suitable for production** as it generates random embeddings with no semantic meaning. +If no provider env var is set, the server falls back to a simple deterministic +embedding provider for testing. This is **not suitable for production** as +its embeddings have no semantic meaning. + +```dotenv +SIMPLE_EMBEDDING_DIMENSION=384 # optional; default 384 +``` ### Document Chunking Configuration @@ -549,7 +609,21 @@ equivalent.** Operators who need a runtime toggle should open an issue. | `VECTOR_SYNC_QUEUE_MAX_SIZE` | ⚠️ Optional | `10000` | Max queued documents | | `OLLAMA_BASE_URL` | ⚠️ Optional | - | Ollama API endpoint for embeddings | | `OLLAMA_EMBEDDING_MODEL` | ⚠️ Optional | `nomic-embed-text` | Embedding model to use | +| `OLLAMA_GENERATION_MODEL` | ⚠️ Optional | - | Ollama model for text generation | | `OLLAMA_VERIFY_SSL` | ⚠️ Optional | `true` | Verify SSL certificates | +| `OPENAI_API_KEY` | ⚠️ Optional | - | OpenAI API key (selects OpenAI provider) | +| `OPENAI_BASE_URL` | ⚠️ Optional | - | OpenAI base URL override (for compatible APIs) | +| `OPENAI_EMBEDDING_MODEL` | ⚠️ Optional | `text-embedding-3-small` | OpenAI embedding model | +| `OPENAI_GENERATION_MODEL` | ⚠️ Optional | - | OpenAI model for text generation | +| `MISTRAL_API_KEY` | ⚠️ Optional | - | Mistral API key (selects Mistral provider) | +| `MISTRAL_EMBEDDING_MODEL` | ⚠️ Optional | `mistral-embed` | Mistral embedding model (1024-dim) | +| `MISTRAL_BASE_URL` | ⚠️ Optional | - | Mistral base URL override (proxies, on-prem) | +| `AWS_REGION` | ⚠️ Optional | - | AWS region (selects Bedrock provider) | +| `AWS_ACCESS_KEY_ID` | ⚠️ Optional | - | AWS access key (boto3 credential chain fallback) | +| `AWS_SECRET_ACCESS_KEY` | ⚠️ Optional | - | AWS secret key (boto3 credential chain fallback) | +| `BEDROCK_EMBEDDING_MODEL` | ⚠️ Optional | - | Bedrock embedding model ID | +| `BEDROCK_GENERATION_MODEL` | ⚠️ Optional | - | Bedrock generation model ID | +| `SIMPLE_EMBEDDING_DIMENSION` | ⚠️ Optional | `384` | Dimension for the fallback Simple provider | | `DOCUMENT_CHUNK_SIZE` | ⚠️ Optional | `512` | Words per chunk for document embedding | | `DOCUMENT_CHUNK_OVERLAP` | ⚠️ Optional | `50` | Overlapping words between chunks (must be < chunk size) | diff --git a/nextcloud_mcp_server/search/algorithms.py b/nextcloud_mcp_server/search/algorithms.py index 7b120393..f89a8933 100644 --- a/nextcloud_mcp_server/search/algorithms.py +++ b/nextcloud_mcp_server/search/algorithms.py @@ -11,6 +11,8 @@ from nextcloud_mcp_server.config import get_settings from nextcloud_mcp_server.vector.placeholder import get_placeholder_filter from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client +logger = logging.getLogger(__name__) + @runtime_checkable class NextcloudClientProtocol(Protocol): @@ -91,7 +93,6 @@ async def get_indexed_doc_types(user_id: str) -> set[str]: ... # Search notes """ - logger = logging.getLogger(__name__) settings = get_settings() qdrant_client = await get_qdrant_client() @@ -205,15 +206,19 @@ def build_search_result_from_point( if point.payload is None: return None - doc_id = str(point.payload["doc_id"]) + raw_doc_id = point.payload.get("doc_id") + if raw_doc_id is None: + logger.warning("Skipping point %s: missing doc_id in payload", point.id) + return None + doc_id = str(raw_doc_id) doc_type = point.payload.get("doc_type", "note") - metadata: dict[str, Any] = { - "chunk_index": point.payload.get("chunk_index"), - "total_chunks": point.payload.get("total_chunks"), - } - if metadata_extras: - metadata.update(metadata_extras) + # Caller-supplied metadata is merged first; payload-derived common fields + # (chunk_index, total_chunks) win in case of key collisions so they always + # reflect the actual point. + metadata: dict[str, Any] = dict(metadata_extras) if metadata_extras else {} + metadata["chunk_index"] = point.payload.get("chunk_index") + metadata["total_chunks"] = point.payload.get("total_chunks") # File-specific metadata for PDF viewer if doc_type == "file" and (path := point.payload.get("file_path")): diff --git a/nextcloud_mcp_server/vector/qdrant_client.py b/nextcloud_mcp_server/vector/qdrant_client.py index 34b14b3a..9c3808f7 100644 --- a/nextcloud_mcp_server/vector/qdrant_client.py +++ b/nextcloud_mcp_server/vector/qdrant_client.py @@ -46,9 +46,21 @@ async def _ensure_keyword_payload_indexes( except UnexpectedResponse as e: body = getattr(e, "content", b"") or b"" body_text = body.decode("utf-8", errors="replace") - logger.warning( - "Failed to create payload index on '%s': %s", field, body_text - ) + # 400 is the expected schema-conflict path (index already exists + # with a different type). 5xx / network-shaped errors should not + # be silently downgraded — keep the loop going so the remaining + # fields still get attempted, but log at error so operators see it. + if e.status_code == 400: + logger.warning( + "Schema conflict on payload index '%s': %s", field, body_text + ) + else: + logger.error( + "Unexpected error creating payload index on '%s' (status %s): %s", + field, + e.status_code, + body_text, + ) async def _backfill_doc_id_to_string( diff --git a/tests/unit/search/test_search_result.py b/tests/unit/search/test_search_result.py index 52bf15c0..e29ed496 100644 --- a/tests/unit/search/test_search_result.py +++ b/tests/unit/search/test_search_result.py @@ -162,6 +162,14 @@ def test_build_search_result_from_point_returns_none_when_payload_missing(): assert build_search_result_from_point(point) is None +@pytest.mark.unit +def test_build_search_result_from_point_returns_none_when_doc_id_missing(): + """A payload without a doc_id key is skipped instead of raising KeyError.""" + point = _make_point(point_id="p-bad", payload={"doc_type": "note"}) + + assert build_search_result_from_point(point) is None + + @pytest.mark.unit def test_build_search_result_from_point_note_payload(): """Note-type payload populates the SearchResult fields without metadata extras.""" @@ -187,7 +195,7 @@ def test_build_search_result_from_point_note_payload(): assert sr.doc_type == "note" assert sr.title == "Hello" assert sr.excerpt == "world" - assert sr.score == 0.91 + assert sr.score == pytest.approx(0.91) assert sr.chunk_start_offset == 0 assert sr.chunk_end_offset == 100 assert sr.chunk_index == 0 @@ -257,21 +265,34 @@ def test_build_search_result_from_point_deck_card_metadata(): @pytest.mark.unit def test_build_search_result_from_point_merges_metadata_extras(): - """metadata_extras override/augment the helper's computed metadata dict.""" + """metadata_extras augment the helper's computed metadata dict. + + Common fields (chunk_index, total_chunks) win over caller-supplied + extras to keep them tied to the actual point. + """ point = _make_point( point_id="p-4", - payload={"doc_id": "1", "doc_type": "note"}, + payload={ + "doc_id": "1", + "doc_type": "note", + "chunk_index": 3, + "total_chunks": 9, + }, ) sr = build_search_result_from_point( - point, metadata_extras={"search_method": "bm25_hybrid_rrf"} + point, + metadata_extras={ + "search_method": "bm25_hybrid_rrf", + # Caller tries to override a common field — should be ignored. + "chunk_index": "should-be-overwritten", + }, ) assert sr is not None assert sr.metadata["search_method"] == "bm25_hybrid_rrf" - # Common fields still present - assert "chunk_index" in sr.metadata - assert "total_chunks" in sr.metadata + assert sr.metadata["chunk_index"] == 3 + assert sr.metadata["total_chunks"] == 9 @pytest.mark.unit diff --git a/tests/unit/vector/test_qdrant_client.py b/tests/unit/vector/test_qdrant_client.py index ab5e38c7..2ec93ec9 100644 --- a/tests/unit/vector/test_qdrant_client.py +++ b/tests/unit/vector/test_qdrant_client.py @@ -103,6 +103,32 @@ async def test_ensure_keyword_payload_indexes_logs_400_as_warning(mocker, caplog assert "different schema" in warnings[0].getMessage() +@pytest.mark.unit +async def test_ensure_keyword_payload_indexes_logs_non_400_as_error(mocker, caplog): + """A non-400 status from create_payload_index escalates to ERROR. + + A 5xx response (e.g., Qdrant temporarily unavailable) should not be + silently downgraded to a warning the way a 400 schema-conflict is. + The loop still continues so the remaining fields get attempted. + """ + client = mocker.AsyncMock() + client.create_payload_index.side_effect = [ + _make_unexpected(500, b'{"status":{"error":"internal server error"}}'), + None, + None, + ] + + with caplog.at_level("ERROR", logger="nextcloud_mcp_server.vector.qdrant_client"): + await _ensure_keyword_payload_indexes(client, "test-collection") + + assert client.create_payload_index.await_count == len(_KEYWORD_PAYLOAD_FIELDS) + errors = [r for r in caplog.records if r.levelname == "ERROR"] + assert len(errors) == 1 + msg = errors[0].getMessage() + assert "500" in msg + assert "internal server error" in msg + + # --------------------------------------------------------------------------- # _backfill_doc_id_to_string # --------------------------------------------------------------------------- @@ -231,3 +257,27 @@ async def test_backfill_handles_none_payload(mocker): points=[2], wait=True, ) + + +@pytest.mark.unit +async def test_backfill_handles_payload_with_explicit_none_doc_id(mocker): + """A payload of {doc_id: None, ...} is skipped just like payload=None.""" + client = mocker.AsyncMock() + # Build the record manually to distinguish payload=None from payload={"doc_id": None}. + point_with_explicit_none = SimpleNamespace( + id=1, payload={"doc_id": None, "doc_type": "file"} + ) + client.scroll.side_effect = [ + ([point_with_explicit_none, _record(2, 99)], None), + ] + + await _backfill_doc_id_to_string(client, "test-collection") + + # Only the int doc_id at point 2 was rewritten; the explicit-None payload was skipped. + assert client.set_payload.await_count == 1 + client.set_payload.assert_awaited_with( + collection_name="test-collection", + payload={"doc_id": "99"}, + points=[2], + wait=True, + ) From 92b2d50cd743dc344ec4349bf042260004d7fd50 Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Fri, 8 May 2026 22:59:46 +0200 Subject: [PATCH 04/27] =?UTF-8?q?fix(vector):=20address=20PR=20review=20ro?= =?UTF-8?q?und=203=20=E2=80=94=20sentinel=20guard,=20skip=20indexed=20fiel?= =?UTF-8?q?ds,=20narrow=20types?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add a fixed-UUID sentinel point written after a successful doc_id backfill so subsequent restarts retrieve it and short-circuit the O(N) scroll. Sentinel has no user_id/doc_id/doc_type payload so production search filters never see it. - Pre-fetch payload_schema in _ensure_keyword_payload_indexes and silently skip fields that are already indexed; the "Created KEYWORD payload index" INFO log fires only on actual creation. - Narrow stale `int | str` doc_id annotations to `str` across search/verification.py (BatchVerifier return type, per-verifier accessible sets, by_type / accessible_by_type / inaccessible collections); drop the now-redundant `type(d).__name__` prefix in the dropped-docs log. - Align the backfill log message with the PR description's "Running doc_id backfill" promise; add a caller cross-reference to the wait=True comment. - Fix _get_file_path_from_qdrant docstring (file_id is str, not numeric). - Convert legacy `id=1` to `id="1"` in test_search_result.py to match the SearchResult.id: str annotation. Three new unit tests cover sentinel-found, sentinel-written, and skip-existing-index branches; existing backfill tests pass dimension and explicit retrieve.return_value=[] for the no-sentinel path. Co-Authored-By: Claude Opus 4.7 (1M context) --- nextcloud_mcp_server/search/context.py | 3 +- nextcloud_mcp_server/search/verification.py | 28 ++--- nextcloud_mcp_server/vector/qdrant_client.py | 101 ++++++++++++--- tests/unit/search/test_search_result.py | 16 +-- tests/unit/vector/test_qdrant_client.py | 122 +++++++++++++++++-- 5 files changed, 223 insertions(+), 47 deletions(-) diff --git a/nextcloud_mcp_server/search/context.py b/nextcloud_mcp_server/search/context.py index c3d4827a..1771c087 100644 --- a/nextcloud_mcp_server/search/context.py +++ b/nextcloud_mcp_server/search/context.py @@ -151,7 +151,8 @@ async def _get_file_path_from_qdrant( Args: user_id: User ID who owns the file - file_id: Numeric file ID + file_id: Stringified file ID (Qdrant payload value, post-doc_id + normalization — see vector/qdrant_client.py) chunk_start: Character offset where chunk starts chunk_end: Character offset where chunk ends diff --git a/nextcloud_mcp_server/search/verification.py b/nextcloud_mcp_server/search/verification.py index e5665f3e..a4b0a0d6 100644 --- a/nextcloud_mcp_server/search/verification.py +++ b/nextcloud_mcp_server/search/verification.py @@ -49,7 +49,7 @@ logger = logging.getLogger(__name__) BatchVerifier = Callable[ [NextcloudClientProtocol, list[SearchResult], anyio.Semaphore], - Awaitable[set[int | str]], + Awaitable[set[str]], ] """(client, results, semaphore) -> set of doc_ids accessible to the user.""" @@ -75,9 +75,9 @@ async def _verify_notes( client: NextcloudClientProtocol, results: list[SearchResult], semaphore: anyio.Semaphore, -) -> set[int | str]: +) -> set[str]: # safe: cooperative concurrency, no lock needed (see verify_search_results) - accessible: set[int | str] = set() + accessible: set[str] = set() async def check(result: SearchResult) -> None: doc_id = result.id @@ -130,9 +130,9 @@ async def _verify_files( client: NextcloudClientProtocol, results: list[SearchResult], semaphore: anyio.Semaphore, -) -> set[int | str]: +) -> set[str]: # safe: cooperative concurrency, no lock needed (see verify_search_results) - accessible: set[int | str] = set() + accessible: set[str] = set() async def check(result: SearchResult) -> None: doc_id = result.id @@ -201,9 +201,9 @@ async def _verify_deck_cards( client: NextcloudClientProtocol, results: list[SearchResult], semaphore: anyio.Semaphore, -) -> set[int | str]: +) -> set[str]: # safe: cooperative concurrency, no lock needed (see verify_search_results) - accessible: set[int | str] = set() + accessible: set[str] = set() async def check(result: SearchResult) -> None: doc_id = result.id @@ -283,7 +283,7 @@ async def _verify_news_items( client: NextcloudClientProtocol, results: list[SearchResult], semaphore: anyio.Semaphore, -) -> set[int | str]: +) -> set[str]: """Batch-verify news items with a single fetch. The Nextcloud News API has no per-item endpoint, so ``news.get_item`` is @@ -386,7 +386,7 @@ async def _verify_news_items( # for THAT item only — not the whole batch. Mirrors the per-item # shape of the notes/files/deck verifiers. See the granularity note # above for why this is narrower than the API-response failure path. - accessible: set[int | str] = set() + accessible: set[str] = set() for d in doc_ids: try: if int(d) in present_ids: @@ -474,7 +474,7 @@ async def verify_search_results( # deduplicated batch. We pick one SearchResult per (id, doc_type) to carry # metadata (path, board_id/stack_id) into the verifier — chunks of the # same document share these fields, so any chunk works. - by_type: dict[str, dict[int | str, SearchResult]] = {} + by_type: dict[str, dict[str, SearchResult]] = {} for r in results: by_type.setdefault(r.doc_type, {}).setdefault(r.id, r) @@ -494,7 +494,7 @@ async def verify_search_results( # same write. Adding a lock would be dead weight; using ``anyio.Lock`` # here would force serialization on a path that is intentionally # parallel. - accessible_by_type: dict[str, set[int | str]] = {} + accessible_by_type: dict[str, set[str]] = {} async def run_verifier(doc_type: str, unique_results: list[SearchResult]) -> None: verifier = _VERIFIERS.get(doc_type) @@ -526,7 +526,7 @@ async def verify_search_results( tg.start_soon(run_verifier, doc_type, list(id_to_result.values())) # Compute (doc_id, doc_type) pairs that failed verification - inaccessible: set[tuple[int | str, str]] = set() + inaccessible: set[tuple[str, str]] = set() for doc_type, id_to_result in by_type.items(): # The .get() default is defensive only — run_verifier always populates # accessible_by_type[doc_type], either with the verifier's result or @@ -537,12 +537,10 @@ async def verify_search_results( inaccessible.add((doc_id, doc_type)) if inaccessible: - # Tag ids with their type (int vs str) so ghost-record logs are - # unambiguous: int 42 and str "42" both render as "42" otherwise. logger.info( "Verification dropped %d inaccessible document(s): %s", len(inaccessible), - sorted((f"{type(d).__name__}:{d}", t) for d, t in inaccessible), + sorted(inaccessible), ) # Filter results, preserving order. All chunks of an inaccessible document diff --git a/nextcloud_mcp_server/vector/qdrant_client.py b/nextcloud_mcp_server/vector/qdrant_client.py index 9c3808f7..bf676923 100644 --- a/nextcloud_mcp_server/vector/qdrant_client.py +++ b/nextcloud_mcp_server/vector/qdrant_client.py @@ -5,7 +5,12 @@ from typing import Any from qdrant_client import AsyncQdrantClient, models from qdrant_client.http.exceptions import UnexpectedResponse -from qdrant_client.models import Distance, PayloadSchemaType, VectorParams +from qdrant_client.models import ( + Distance, + PayloadSchemaType, + PointStruct, + VectorParams, +) from nextcloud_mcp_server.config import get_settings from nextcloud_mcp_server.embedding import get_embedding_service @@ -20,6 +25,14 @@ logger = logging.getLogger(__name__) # correct schema (see ADR notes in commit message). _KEYWORD_PAYLOAD_FIELDS: tuple[str, ...] = ("doc_id", "user_id", "doc_type") +# Sentinel point that records "this collection has been backfilled to str +# doc_id". Written after a successful pass of _backfill_doc_id_to_string so +# subsequent restarts can short-circuit the O(N) scroll. Carries no +# user_id/doc_id/doc_type, so production search filters (which always +# require user_id) never see it. +_DOC_ID_BACKFILL_SENTINEL_ID: str = "00000000-0000-0000-0000-d0c1d0d1d0c1" +_DOC_ID_BACKFILL_SENTINEL_PAYLOAD: dict[str, str] = {"_migration_marker": "doc_id_v1"} + # Singleton instance _qdrant_client: AsyncQdrantClient | None = None @@ -29,12 +42,22 @@ async def _ensure_keyword_payload_indexes( ) -> None: """Create KEYWORD payload indexes for fields used in exact-match filters. - Idempotent at the Qdrant layer: re-creating an identical index returns - 200, so this can run on every startup. Schema conflicts (a pre-existing - index with a different type) surface as a 400 — log loudly so operators - can intervene, but keep going so the remaining fields still get indexed. + Pre-fetches the existing payload schema and skips fields that are + already indexed, so routine restarts make no Qdrant write round-trips + and emit no INFO log lines. Schema conflicts (a pre-existing index + with a different type) still surface as a 400 — log loudly so + operators can intervene, but keep going so the remaining fields still + get indexed. """ + collection_info = await client.get_collection(collection_name) + existing_schema = collection_info.payload_schema or {} + for field in _KEYWORD_PAYLOAD_FIELDS: + if field in existing_schema: + # Index already present — silent skip. Logging here on every + # restart would be noise that hides the genuinely interesting + # "first-time creation" line below. + continue try: await client.create_payload_index( collection_name=collection_name, @@ -64,22 +87,48 @@ async def _ensure_keyword_payload_indexes( async def _backfill_doc_id_to_string( - client: AsyncQdrantClient, collection_name: str + client: AsyncQdrantClient, collection_name: str, dimension: int ) -> None: """Rewrite legacy integer doc_id payloads to strings. Producers now uniformly write str(doc_id), but historical points may carry int values from before normalization. A KEYWORD index does not match int payloads, so any leftover int doc_ids would be silently invisible to - filters. Scrolls all points once and converts in-place; idempotent (a - second pass over the same collection performs zero writes). + filters. Scrolls all points once, converts in-place, and writes a + sentinel point on success; subsequent restarts retrieve the sentinel + and skip the scroll entirely. Idempotent in both directions (a second + pass on a migrated collection short-circuits via the sentinel; a + second pass with the sentinel manually deleted is the same zero-write + scroll the first pass would do on an already-clean collection). Within each scroll batch, points sharing the same int doc_id are batched into a single ``set_payload`` call to minimize Qdrant round-trips. + + Args: + client: Qdrant client instance. + collection_name: Target collection. + dimension: Dense-vector dimension for the sentinel point's vector + (forwarded by ``get_qdrant_client`` from the embedding model). """ + # Sentinel guard: if the migration ran successfully against this + # collection on a previous start, retrieve() returns the marker point + # and we skip the scroll. Cheap single-key lookup vs. an O(N) scroll. + sentinel = await client.retrieve( + collection_name=collection_name, + ids=[_DOC_ID_BACKFILL_SENTINEL_ID], + with_payload=False, + with_vectors=False, + ) + if sentinel: + logger.debug( + "doc_id backfill sentinel found on '%s'; skipping scroll", + collection_name, + ) + return + logger.info( - "Scanning '%s' for legacy int doc_id payloads (this is a one-time " - "migration on first start after upgrade)", + "Running doc_id backfill on '%s' (one-time migration on first " + "start after upgrade; subsequent restarts skip via sentinel)", collection_name, ) @@ -117,10 +166,11 @@ async def _backfill_doc_id_to_string( by_value.setdefault(str(value), []).append(point.id) for str_val, point_ids in by_value.items(): - # wait=True is required: _ensure_keyword_payload_indexes runs - # immediately after this function and only indexes committed - # data — fire-and-forget writes would leave int payloads - # invisible to KEYWORD filters. + # wait=True is required because _ensure_keyword_payload_indexes + # runs immediately after this function (see get_qdrant_client + # near the call site) and only indexes committed data — + # fire-and-forget writes would leave int payloads invisible + # to KEYWORD filters. await client.set_payload( collection_name=collection_name, payload={"doc_id": str_val}, @@ -132,6 +182,25 @@ async def _backfill_doc_id_to_string( if next_offset is None: break + # Write the sentinel after a successful scroll so a future restart can + # short-circuit. Empty sparse vector mirrors the placeholder.py + # convention (vector/placeholder.py); zero dense vector is fine + # because the sentinel never participates in a search (no user_id / + # doc_id / doc_type payload to match). + sentinel_point = PointStruct( + id=_DOC_ID_BACKFILL_SENTINEL_ID, + vector={ + "dense": [0.0] * dimension, + "sparse": models.SparseVector(indices=[], values=[]), + }, + payload=dict(_DOC_ID_BACKFILL_SENTINEL_PAYLOAD), + ) + await client.upsert( + collection_name=collection_name, + points=[sentinel_point], + wait=True, + ) + logger.info( "doc_id backfill complete: rewrote %d/%d payloads from int to str", rewritten, @@ -237,7 +306,9 @@ async def get_qdrant_client() -> AsyncQdrantClient: # Existing collections may pre-date the doc_id normalization / # payload-index work. Backfill before creating the index so the # index covers every point. - await _backfill_doc_id_to_string(_qdrant_client, collection_name) + await _backfill_doc_id_to_string( + _qdrant_client, collection_name, expected_dimension + ) await _ensure_keyword_payload_indexes(_qdrant_client, collection_name) else: diff --git a/tests/unit/search/test_search_result.py b/tests/unit/search/test_search_result.py index e29ed496..95eece1e 100644 --- a/tests/unit/search/test_search_result.py +++ b/tests/unit/search/test_search_result.py @@ -23,7 +23,7 @@ def _make_point(point_id, payload, score=0.5): def test_search_result_rrf_score_in_range(): """Test SearchResult accepts RRF scores in [0.0, 1.0] range.""" result = SearchResult( - id=1, + id="1", doc_type="note", title="Test Note", excerpt="Test excerpt", @@ -37,7 +37,7 @@ def test_search_result_rrf_score_in_range(): def test_search_result_rrf_score_at_lower_bound(): """Test SearchResult accepts RRF score at lower bound (0.0).""" result = SearchResult( - id=1, + id="1", doc_type="note", title="Test Note", excerpt="Test excerpt", @@ -51,7 +51,7 @@ def test_search_result_rrf_score_at_lower_bound(): def test_search_result_rrf_score_at_upper_bound(): """Test SearchResult accepts RRF score at upper bound (1.0).""" result = SearchResult( - id=1, + id="1", doc_type="note", title="Test Note", excerpt="Test excerpt", @@ -71,7 +71,7 @@ def test_search_result_dbsf_score_above_one(): """ # Typical DBSF score when both systems agree result = SearchResult( - id=1, + id="1", doc_type="note", title="Highly Relevant Note", excerpt="Contains keywords and is semantically similar", @@ -88,7 +88,7 @@ def test_search_result_dbsf_score_edge_case(): Maximum DBSF score with 2 systems: 1.0 (dense) + 1.0 (sparse) = 2.0 """ result = SearchResult( - id=1, + id="1", doc_type="note", title="Perfect Match", excerpt="Perfect semantic and keyword match", @@ -103,7 +103,7 @@ def test_search_result_negative_score_raises_error(): """Test SearchResult rejects negative scores.""" with pytest.raises(ValueError) as exc_info: SearchResult( - id=1, + id="1", doc_type="note", title="Test Note", excerpt="Test excerpt", @@ -118,7 +118,7 @@ def test_search_result_negative_score_raises_error(): def test_search_result_with_metadata(): """Test SearchResult with optional metadata field.""" result = SearchResult( - id=1, + id="1", doc_type="note", title="Test Note", excerpt="Test excerpt", @@ -136,7 +136,7 @@ def test_search_result_with_metadata(): def test_search_result_with_chunk_offsets(): """Test SearchResult with chunk offset information.""" result = SearchResult( - id=1, + id="1", doc_type="note", title="Test Note", excerpt="matching chunk text", diff --git a/tests/unit/vector/test_qdrant_client.py b/tests/unit/vector/test_qdrant_client.py index 2ec93ec9..92314034 100644 --- a/tests/unit/vector/test_qdrant_client.py +++ b/tests/unit/vector/test_qdrant_client.py @@ -23,12 +23,32 @@ from qdrant_client.http.exceptions import UnexpectedResponse from qdrant_client.models import PayloadSchemaType from nextcloud_mcp_server.vector.qdrant_client import ( + _DOC_ID_BACKFILL_SENTINEL_ID, _KEYWORD_PAYLOAD_FIELDS, _backfill_doc_id_to_string, _ensure_keyword_payload_indexes, ) +def _empty_collection_info() -> SimpleNamespace: + """Stand-in for a CollectionInfo with no payload indexes yet. + + Tests for _ensure_keyword_payload_indexes only read ``payload_schema`` + off the result. None / empty dict both signal "no indexes" — use {} + here to match the production-code default. + """ + return SimpleNamespace(payload_schema={}) + + +def _backfill_dimension() -> int: + """Vector dimension for sentinel writes in backfill tests. + + Any positive int is fine — the sentinel point is never read by the + test bodies, only the upsert call site is asserted. + """ + return 4 + + def _make_unexpected(status_code: int, body: bytes) -> UnexpectedResponse: """Build a real UnexpectedResponse for raise_for_status-style branches.""" return UnexpectedResponse( @@ -58,6 +78,7 @@ def _record(point_id: int | str, doc_id: int | str | None) -> SimpleNamespace: async def test_ensure_keyword_payload_indexes_creates_each_field(mocker): """Happy path: every field in _KEYWORD_PAYLOAD_FIELDS gets a KEYWORD index.""" client = mocker.AsyncMock() + client.get_collection.return_value = _empty_collection_info() await _ensure_keyword_payload_indexes(client, "test-collection") @@ -74,6 +95,37 @@ async def test_ensure_keyword_payload_indexes_creates_each_field(mocker): client.create_payload_index.assert_has_awaits(expected_calls, any_order=False) +@pytest.mark.unit +async def test_ensure_keyword_payload_indexes_skips_fields_already_indexed( + mocker, caplog +): + """Routine restart path: existing payload indexes are silently skipped. + + Without the pre-fetch, every restart logs `Created KEYWORD payload + index on ''` for every field — noise that hides genuinely + interesting first-time-creation lines. With the pre-fetch, no log + fires and no Qdrant write round-trip happens for already-indexed + fields. + """ + client = mocker.AsyncMock() + client.get_collection.return_value = SimpleNamespace( + payload_schema={"doc_id": object()} + ) + + with caplog.at_level("INFO", logger="nextcloud_mcp_server.vector.qdrant_client"): + await _ensure_keyword_payload_indexes(client, "test-collection") + + # Only the two missing fields are created. + assert client.create_payload_index.await_count == 2 + created_fields = { + c.kwargs["field_name"] for c in client.create_payload_index.await_args_list + } + assert created_fields == {"user_id", "doc_type"} + # No INFO log fires for the already-indexed field. + info_messages = [r.getMessage() for r in caplog.records if r.levelname == "INFO"] + assert not any("doc_id" in m for m in info_messages), info_messages + + @pytest.mark.unit async def test_ensure_keyword_payload_indexes_logs_400_as_warning(mocker, caplog): """Any 400 from create_payload_index is logged at WARNING and skipped. @@ -84,6 +136,7 @@ async def test_ensure_keyword_payload_indexes_logs_400_as_warning(mocker, caplog remaining fields still get indexed. """ client = mocker.AsyncMock() + client.get_collection.return_value = _empty_collection_info() client.create_payload_index.side_effect = [ _make_unexpected( 400, @@ -112,6 +165,7 @@ async def test_ensure_keyword_payload_indexes_logs_non_400_as_error(mocker, capl The loop still continues so the remaining fields get attempted. """ client = mocker.AsyncMock() + client.get_collection.return_value = _empty_collection_info() client.create_payload_index.side_effect = [ _make_unexpected(500, b'{"status":{"error":"internal server error"}}'), None, @@ -138,17 +192,20 @@ async def test_ensure_keyword_payload_indexes_logs_non_400_as_error(mocker, capl async def test_backfill_clean_collection_makes_no_writes(mocker, caplog): """A collection with only str doc_ids triggers zero set_payload calls. - Verifies idempotency: a second pass over an already-migrated collection - is a no-op modulo the read. + Verifies the no-write path: scroll runs, no payloads need rewriting, + and a sentinel is written so subsequent restarts can short-circuit. """ client = mocker.AsyncMock() + client.retrieve.return_value = [] # No sentinel — backfill must run client.scroll.return_value = ( [_record(1, "abc"), _record(2, "def")], None, ) with caplog.at_level("INFO", logger="nextcloud_mcp_server.vector.qdrant_client"): - await _backfill_doc_id_to_string(client, "test-collection") + await _backfill_doc_id_to_string( + client, "test-collection", _backfill_dimension() + ) client.set_payload.assert_not_awaited() completion_logs = [ @@ -158,10 +215,53 @@ async def test_backfill_clean_collection_makes_no_writes(mocker, caplog): assert "0/2" in completion_logs[0] +@pytest.mark.unit +async def test_backfill_skips_when_sentinel_present(mocker, caplog): + """If the sentinel exists, retrieve() returns it and the scroll is skipped. + + This is the routine-restart fast path: the migration already ran on a + previous start, so we avoid the O(N) scroll entirely. + """ + client = mocker.AsyncMock() + client.retrieve.return_value = [SimpleNamespace(id=_DOC_ID_BACKFILL_SENTINEL_ID)] + + with caplog.at_level("DEBUG", logger="nextcloud_mcp_server.vector.qdrant_client"): + await _backfill_doc_id_to_string( + client, "test-collection", _backfill_dimension() + ) + + client.scroll.assert_not_awaited() + client.set_payload.assert_not_awaited() + client.upsert.assert_not_awaited() + debug_msgs = [r.getMessage() for r in caplog.records if r.levelname == "DEBUG"] + assert any("sentinel" in m and "skipping" in m for m in debug_msgs), debug_msgs + + +@pytest.mark.unit +async def test_backfill_writes_sentinel_after_successful_scroll(mocker): + """Successful backfill writes a sentinel point so future restarts skip.""" + client = mocker.AsyncMock() + client.retrieve.return_value = [] # No sentinel — backfill must run + client.scroll.return_value = ([_record(1, "abc")], None) + + await _backfill_doc_id_to_string(client, "test-collection", _backfill_dimension()) + + # Single upsert with the sentinel UUID + migration marker payload. + assert client.upsert.await_count == 1 + upsert_kwargs = client.upsert.await_args.kwargs + assert upsert_kwargs["collection_name"] == "test-collection" + assert upsert_kwargs["wait"] is True + points = upsert_kwargs["points"] + assert len(points) == 1 + assert points[0].id == _DOC_ID_BACKFILL_SENTINEL_ID + assert points[0].payload == {"_migration_marker": "doc_id_v1"} + + @pytest.mark.unit async def test_backfill_rewrites_int_doc_ids_to_str(mocker): """Mixed int/str payload across two scroll pages: only ints get rewritten.""" client = mocker.AsyncMock() + client.retrieve.return_value = [] # Two scroll calls: batch 1 is mixed and reports a next_offset; batch 2 # is mixed with next_offset=None to terminate. client.scroll.side_effect = [ @@ -169,7 +269,7 @@ async def test_backfill_rewrites_int_doc_ids_to_str(mocker): ([_record(3, 200), _record(4, "def")], None), ] - await _backfill_doc_id_to_string(client, "test-collection") + await _backfill_doc_id_to_string(client, "test-collection", _backfill_dimension()) # One set_payload per *unique* int value — point 1 (100) and point 3 # (200) are in different batches with different values, so two calls. @@ -196,6 +296,7 @@ async def test_backfill_batches_points_with_same_doc_id(mocker): backfill should issue one set_payload call covering the chunk batch. """ client = mocker.AsyncMock() + client.retrieve.return_value = [] client.scroll.side_effect = [ ( [ @@ -208,7 +309,7 @@ async def test_backfill_batches_points_with_same_doc_id(mocker): ), ] - await _backfill_doc_id_to_string(client, "test-collection") + await _backfill_doc_id_to_string(client, "test-collection", _backfill_dimension()) # All three int-payload points share doc_id=42, so a single call covers them. assert client.set_payload.await_count == 1 @@ -224,12 +325,15 @@ async def test_backfill_batches_points_with_same_doc_id(mocker): async def test_backfill_emits_completion_log(mocker, caplog): """Backfill logs final rewritten/scanned counts at INFO.""" client = mocker.AsyncMock() + client.retrieve.return_value = [] client.scroll.side_effect = [ ([_record(1, 7), _record(2, "x")], None), ] with caplog.at_level("INFO", logger="nextcloud_mcp_server.vector.qdrant_client"): - await _backfill_doc_id_to_string(client, "test-collection") + await _backfill_doc_id_to_string( + client, "test-collection", _backfill_dimension() + ) completion_logs = [ r.getMessage() for r in caplog.records if "backfill complete" in r.getMessage() @@ -243,11 +347,12 @@ async def test_backfill_emits_completion_log(mocker, caplog): async def test_backfill_handles_none_payload(mocker): """A point with payload=None is skipped without crashing.""" client = mocker.AsyncMock() + client.retrieve.return_value = [] client.scroll.side_effect = [ ([_record(1, None), _record(2, 99)], None), ] - await _backfill_doc_id_to_string(client, "test-collection") + await _backfill_doc_id_to_string(client, "test-collection", _backfill_dimension()) # Only the int doc_id at point 2 was rewritten; the None-payload point was skipped. assert client.set_payload.await_count == 1 @@ -263,6 +368,7 @@ async def test_backfill_handles_none_payload(mocker): async def test_backfill_handles_payload_with_explicit_none_doc_id(mocker): """A payload of {doc_id: None, ...} is skipped just like payload=None.""" client = mocker.AsyncMock() + client.retrieve.return_value = [] # Build the record manually to distinguish payload=None from payload={"doc_id": None}. point_with_explicit_none = SimpleNamespace( id=1, payload={"doc_id": None, "doc_type": "file"} @@ -271,7 +377,7 @@ async def test_backfill_handles_payload_with_explicit_none_doc_id(mocker): ([point_with_explicit_none, _record(2, 99)], None), ] - await _backfill_doc_id_to_string(client, "test-collection") + await _backfill_doc_id_to_string(client, "test-collection", _backfill_dimension()) # Only the int doc_id at point 2 was rewritten; the explicit-None payload was skipped. assert client.set_payload.await_count == 1 From b97ac23228418c68dfef91150817ad1e60c48e3f Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Fri, 8 May 2026 23:39:37 +0200 Subject: [PATCH 05/27] =?UTF-8?q?fix(vector):=20address=20PR=20review=20ro?= =?UTF-8?q?und=204=20=E2=80=94=20backfill=20resilience=20+=20degraded-mode?= =?UTF-8?q?=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove three stale `# Use numeric file ID` / `# Pass file path` comments in scanner.py. file_id is already normalized to str() above each call site, so the inline comments mislead readers. - Wrap `_backfill_doc_id_to_string` scroll loop + sentinel upsert in try/except Exception. The qdrant_client singleton is assigned before this migration runs, so a transient scroll failure was leaving the process holding a usable client with int payloads permanently unbackfilled until the next restart. Catch broadly, log ERROR with exc_info, and return without writing the sentinel — next process restart retries from scratch. - Note `:memory:` mode behavior near the sentinel constants so future readers don't read the every-start scroll as a bug. - Document the two degraded-migration ERROR log signals in docs/configuration.md so operators know when a clean restart is required to recover indexing. - Add unit test asserting scroll-time exceptions are logged and swallowed without writing the sentinel. Closes round-4 review feedback on PR #773. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/configuration.md | 17 +++ nextcloud_mcp_server/vector/qdrant_client.py | 143 +++++++++++-------- nextcloud_mcp_server/vector/scanner.py | 10 +- tests/unit/vector/test_qdrant_client.py | 31 ++++ 4 files changed, 133 insertions(+), 68 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index bd7dd87f..8ca27842 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -355,6 +355,23 @@ Both steps emit INFO-level log lines so operators can track progress. > caught early. Either widen the public model's `id` field or convert the > id at the verifier layer. +> **Degraded-migration signals:** both startup steps swallow non-fatal +> failures so the server still starts, but each leaves a distinct ERROR +> log line that operators should treat as a "restart needed" signal: +> +> - `Unexpected error creating payload index on '' (status 5xx)` — +> the index was not created. Searches filtering on that field will keep +> returning HTTP 400 (`Index required but not found`) until a subsequent +> restart succeeds in creating it. +> - `doc_id backfill failed on ''; will retry on next restart` — +> the migration sentinel was not written. Legacy integer `doc_id` +> payloads remain invisible to the keyword index in the meantime; the +> scroll re-runs from scratch on the next process start. +> +> Neither prevents the server from accepting requests, but both indicate +> that vector search is operating in a degraded state on the affected +> collection until the next clean restart. + #### Explicit Override Set `QDRANT_COLLECTION` to use a specific collection name: diff --git a/nextcloud_mcp_server/vector/qdrant_client.py b/nextcloud_mcp_server/vector/qdrant_client.py index bf676923..6209bc42 100644 --- a/nextcloud_mcp_server/vector/qdrant_client.py +++ b/nextcloud_mcp_server/vector/qdrant_client.py @@ -29,7 +29,9 @@ _KEYWORD_PAYLOAD_FIELDS: tuple[str, ...] = ("doc_id", "user_id", "doc_type") # doc_id". Written after a successful pass of _backfill_doc_id_to_string so # subsequent restarts can short-circuit the O(N) scroll. Carries no # user_id/doc_id/doc_type, so production search filters (which always -# require user_id) never see it. +# require user_id) never see it. In :memory: mode the sentinel does not +# survive a restart — the scroll runs every start, but is a no-op against +# an empty in-memory collection. _DOC_ID_BACKFILL_SENTINEL_ID: str = "00000000-0000-0000-0000-d0c1d0d1d0c1" _DOC_ID_BACKFILL_SENTINEL_PAYLOAD: dict[str, str] = {"_migration_marker": "doc_id_v1"} @@ -139,73 +141,88 @@ async def _backfill_doc_id_to_string( next_offset = None batch_size = 256 - while True: - points, next_offset = await client.scroll( - collection_name=collection_name, - limit=batch_size, - offset=next_offset, - with_payload=["doc_id"], - with_vectors=False, - ) - if not points: - break - - # Group by stringified value so points sharing a doc_id (one document - # → many chunks) collapse into a single set_payload call. Point IDs - # can be int/str/UUID, so widen the value type to satisfy the qdrant - # client's PointsSelector signature without re-spelling the union. - by_value: dict[str, list[Any]] = {} - for point in points: - scanned += 1 - # Qdrant client typing allows None payload even when with_payload - # was requested; defensive default so the type checker is happy. - payload = point.payload or {} - value = payload.get("doc_id") - if value is None or isinstance(value, str): - continue - by_value.setdefault(str(value), []).append(point.id) - - for str_val, point_ids in by_value.items(): - # wait=True is required because _ensure_keyword_payload_indexes - # runs immediately after this function (see get_qdrant_client - # near the call site) and only indexes committed data — - # fire-and-forget writes would leave int payloads invisible - # to KEYWORD filters. - await client.set_payload( + # A transient Qdrant failure mid-scroll (network blip, timeout) must not + # crash startup. The singleton in get_qdrant_client is already assigned + # by the time this runs, so re-raising here would leave the process in + # a half-initialized state where the next call returns the cached + # client and skips this migration entirely. Catch broadly, log with + # exc_info, and return without writing the sentinel — the next process + # restart will retry from scratch. + try: + while True: + points, next_offset = await client.scroll( collection_name=collection_name, - payload={"doc_id": str_val}, - points=point_ids, - wait=True, + limit=batch_size, + offset=next_offset, + with_payload=["doc_id"], + with_vectors=False, ) - rewritten += len(point_ids) + if not points: + break - if next_offset is None: - break + # Group by stringified value so points sharing a doc_id (one document + # → many chunks) collapse into a single set_payload call. Point IDs + # can be int/str/UUID, so widen the value type to satisfy the qdrant + # client's PointsSelector signature without re-spelling the union. + by_value: dict[str, list[Any]] = {} + for point in points: + scanned += 1 + # Qdrant client typing allows None payload even when with_payload + # was requested; defensive default so the type checker is happy. + payload = point.payload or {} + value = payload.get("doc_id") + if value is None or isinstance(value, str): + continue + by_value.setdefault(str(value), []).append(point.id) - # Write the sentinel after a successful scroll so a future restart can - # short-circuit. Empty sparse vector mirrors the placeholder.py - # convention (vector/placeholder.py); zero dense vector is fine - # because the sentinel never participates in a search (no user_id / - # doc_id / doc_type payload to match). - sentinel_point = PointStruct( - id=_DOC_ID_BACKFILL_SENTINEL_ID, - vector={ - "dense": [0.0] * dimension, - "sparse": models.SparseVector(indices=[], values=[]), - }, - payload=dict(_DOC_ID_BACKFILL_SENTINEL_PAYLOAD), - ) - await client.upsert( - collection_name=collection_name, - points=[sentinel_point], - wait=True, - ) + for str_val, point_ids in by_value.items(): + # wait=True is required because _ensure_keyword_payload_indexes + # runs immediately after this function (see get_qdrant_client + # near the call site) and only indexes committed data — + # fire-and-forget writes would leave int payloads invisible + # to KEYWORD filters. + await client.set_payload( + collection_name=collection_name, + payload={"doc_id": str_val}, + points=point_ids, + wait=True, + ) + rewritten += len(point_ids) - logger.info( - "doc_id backfill complete: rewrote %d/%d payloads from int to str", - rewritten, - scanned, - ) + if next_offset is None: + break + + # Write the sentinel after a successful scroll so a future restart can + # short-circuit. Empty sparse vector mirrors the placeholder.py + # convention (vector/placeholder.py); zero dense vector is fine + # because the sentinel never participates in a search (no user_id / + # doc_id / doc_type payload to match). + sentinel_point = PointStruct( + id=_DOC_ID_BACKFILL_SENTINEL_ID, + vector={ + "dense": [0.0] * dimension, + "sparse": models.SparseVector(indices=[], values=[]), + }, + payload=dict(_DOC_ID_BACKFILL_SENTINEL_PAYLOAD), + ) + await client.upsert( + collection_name=collection_name, + points=[sentinel_point], + wait=True, + ) + + logger.info( + "doc_id backfill complete: rewrote %d/%d payloads from int to str", + rewritten, + scanned, + ) + except Exception: + logger.error( + "doc_id backfill failed on '%s'; will retry on next restart", + collection_name, + exc_info=True, + ) + return async def get_qdrant_client() -> AsyncQdrantClient: diff --git a/nextcloud_mcp_server/vector/scanner.py b/nextcloud_mcp_server/vector/scanner.py index 6a5a3758..c483019b 100644 --- a/nextcloud_mcp_server/vector/scanner.py +++ b/nextcloud_mcp_server/vector/scanner.py @@ -484,11 +484,11 @@ async def scan_user_documents( await send_stream.send( DocumentTask( user_id=user_id, - doc_id=file_id, # Use numeric file ID + doc_id=file_id, doc_type="file", operation="index", modified_at=modified_at, - file_path=file_path, # Pass file path for content retrieval + file_path=file_path, ) ) file_queued += 1 @@ -547,11 +547,11 @@ async def scan_user_documents( await send_stream.send( DocumentTask( user_id=user_id, - doc_id=file_id, # Use numeric file ID + doc_id=file_id, doc_type="file", operation="index", modified_at=modified_at, - file_path=file_path, # Pass file path for content retrieval + file_path=file_path, ) ) file_queued += 1 @@ -581,7 +581,7 @@ async def scan_user_documents( await send_stream.send( DocumentTask( user_id=user_id, - doc_id=file_id, # Use numeric file ID + doc_id=file_id, doc_type="file", operation="delete", modified_at=0, diff --git a/tests/unit/vector/test_qdrant_client.py b/tests/unit/vector/test_qdrant_client.py index 92314034..332d0bc6 100644 --- a/tests/unit/vector/test_qdrant_client.py +++ b/tests/unit/vector/test_qdrant_client.py @@ -387,3 +387,34 @@ async def test_backfill_handles_payload_with_explicit_none_doc_id(mocker): points=[2], wait=True, ) + + +@pytest.mark.unit +async def test_backfill_logs_and_returns_when_scroll_raises(mocker, caplog): + """A scroll-time exception is logged and swallowed; sentinel is not written. + + The singleton client in get_qdrant_client is already assigned by the + time _backfill_doc_id_to_string runs, so re-raising here would leave + the process holding a usable client with the migration silently + skipped on every subsequent call. Catching, logging, and returning + without writing the sentinel preserves retry-on-next-restart behavior. + """ + client = mocker.AsyncMock() + client.retrieve.return_value = [] # No sentinel — backfill must run + client.scroll.side_effect = RuntimeError("boom") + + with caplog.at_level("ERROR", logger="nextcloud_mcp_server.vector.qdrant_client"): + await _backfill_doc_id_to_string( + client, "test-collection", _backfill_dimension() + ) + + # No sentinel written — next process restart will retry from scratch. + client.upsert.assert_not_awaited() + client.set_payload.assert_not_awaited() + errors = [r for r in caplog.records if r.levelname == "ERROR"] + assert len(errors) == 1 + assert "doc_id backfill failed" in errors[0].getMessage() + assert "test-collection" in errors[0].getMessage() + # exc_info=True attaches the original exception to the log record. + assert errors[0].exc_info is not None + assert errors[0].exc_info[0] is RuntimeError From c27556c3320ad440833290d40bbbdfbd3c066632 Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Sat, 9 May 2026 00:15:56 +0200 Subject: [PATCH 06/27] =?UTF-8?q?fix(vector):=20address=20PR=20review=20ro?= =?UTF-8?q?und=205=20=E2=80=94=20progress=20logging,=20summary=20visibilit?= =?UTF-8?q?y,=20sentinel=20split?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses three important findings from the latest reviewer comment: - Add progress INFO log every 20 scroll batches (≈5120 points at batch_size=256) in _backfill_doc_id_to_string so a long-running migration on a large collection (50k+ points) doesn't look like a startup hang. The line carries collection name, scanned count, and rewritten count so it doubles as a heartbeat. - Track non-400 failures in _ensure_keyword_payload_indexes and emit a WARNING summary line listing every field that failed to get an index. Per-field ERROR lines are easy to miss in startup noise; the summary makes the partial-failure state visible at a glance. - Split the sentinel upsert out of the data-scroll try/except in _backfill_doc_id_to_string. A scroll-time failure still logs ERROR with the new "scroll failed" wording (data is incomplete). A sentinel-write failure now logs WARNING with "data succeeded but sentinel write failed" wording — data is correct, only the short-circuit marker is missing, and the next restart re-scrolls an already-clean collection (idempotent zero-write) before retrying the upsert. Also fix the RuntimeWarning emitted by test_backfill_logs_and_returns_when_scroll_raises: replace the bare `RuntimeError` side_effect with an async-callable side_effect so AsyncMock awaits the coroutine before the exception propagates. Three new unit tests cover the new branches: test_backfill_emits_progress_log_every_20_batches, test_backfill_logs_warning_when_sentinel_upsert_fails, test_ensure_keyword_payload_indexes_summarises_failed_fields. Co-Authored-By: Claude Opus 4.7 (1M context) --- nextcloud_mcp_server/vector/qdrant_client.py | 90 ++++++++++---- tests/unit/vector/test_qdrant_client.py | 124 ++++++++++++++++++- 2 files changed, 190 insertions(+), 24 deletions(-) diff --git a/nextcloud_mcp_server/vector/qdrant_client.py b/nextcloud_mcp_server/vector/qdrant_client.py index 6209bc42..dda203cf 100644 --- a/nextcloud_mcp_server/vector/qdrant_client.py +++ b/nextcloud_mcp_server/vector/qdrant_client.py @@ -53,6 +53,7 @@ async def _ensure_keyword_payload_indexes( """ collection_info = await client.get_collection(collection_name) existing_schema = collection_info.payload_schema or {} + failed_fields: list[str] = [] for field in _KEYWORD_PAYLOAD_FIELDS: if field in existing_schema: @@ -86,6 +87,20 @@ async def _ensure_keyword_payload_indexes( e.status_code, body_text, ) + failed_fields.append(field) + + # A single per-field ERROR line is easy to miss in startup noise. Surface + # the partial-failure summary at WARNING so operators auditing the log + # for the post-startup state see a single line listing every missing + # index. See docs/configuration.md for the recovery procedure. + if failed_fields: + logger.warning( + "Payload index creation incomplete on '%s' — fields without indexes: %s. " + "Searches filtering on these fields will fail with HTTP 400 " + "(`Index required but not found`) until the next successful restart.", + collection_name, + ", ".join(failed_fields), + ) async def _backfill_doc_id_to_string( @@ -136,10 +151,15 @@ async def _backfill_doc_id_to_string( rewritten = 0 scanned = 0 + batch_num = 0 # Qdrant scroll returns next_offset as PointId | None — keep it untyped here # so the qdrant client's full union (UUID/int/str/PointId) flows through. next_offset = None batch_size = 256 + # Log progress every N batches so a long-running migration on a large + # collection (≥ 50k points) doesn't look like a startup hang. At batch + # size 256, every 20 batches ≈ 5 120 points scanned. + progress_log_every = 20 # A transient Qdrant failure mid-scroll (network blip, timeout) must not # crash startup. The singleton in get_qdrant_client is already assigned @@ -147,7 +167,10 @@ async def _backfill_doc_id_to_string( # a half-initialized state where the next call returns the cached # client and skips this migration entirely. Catch broadly, log with # exc_info, and return without writing the sentinel — the next process - # restart will retry from scratch. + # restart will retry from scratch. The sentinel write is NOT covered by + # this try/except: a failure there means the data migration succeeded + # and only the short-circuit marker is missing, which is a different + # (and milder) condition than a scroll failure. try: while True: points, next_offset = await client.scroll( @@ -160,6 +183,8 @@ async def _backfill_doc_id_to_string( if not points: break + batch_num += 1 + # Group by stringified value so points sharing a doc_id (one document # → many chunks) collapse into a single set_payload call. Point IDs # can be int/str/UUID, so widen the value type to satisfy the qdrant @@ -189,41 +214,62 @@ async def _backfill_doc_id_to_string( ) rewritten += len(point_ids) + if batch_num % progress_log_every == 0: + logger.info( + "doc_id backfill progress on '%s': scanned %d points, " + "rewrote %d so far", + collection_name, + scanned, + rewritten, + ) + if next_offset is None: break - - # Write the sentinel after a successful scroll so a future restart can - # short-circuit. Empty sparse vector mirrors the placeholder.py - # convention (vector/placeholder.py); zero dense vector is fine - # because the sentinel never participates in a search (no user_id / - # doc_id / doc_type payload to match). - sentinel_point = PointStruct( - id=_DOC_ID_BACKFILL_SENTINEL_ID, - vector={ - "dense": [0.0] * dimension, - "sparse": models.SparseVector(indices=[], values=[]), - }, - payload=dict(_DOC_ID_BACKFILL_SENTINEL_PAYLOAD), + except Exception: + logger.error( + "doc_id backfill scroll failed on '%s'; will retry on next restart", + collection_name, + exc_info=True, ) + return + + # Data backfill succeeded — write the sentinel so a future restart can + # short-circuit. Empty sparse vector mirrors the placeholder.py + # convention (vector/placeholder.py); zero dense vector is fine + # because the sentinel never participates in a search (no user_id / + # doc_id / doc_type payload to match). A failure here is non-fatal: + # the data is correct; only the short-circuit marker is missing, so + # the next restart will re-scroll an already-clean collection (idempotent + # zero-write) before retrying the upsert. + sentinel_point = PointStruct( + id=_DOC_ID_BACKFILL_SENTINEL_ID, + vector={ + "dense": [0.0] * dimension, + "sparse": models.SparseVector(indices=[], values=[]), + }, + payload=dict(_DOC_ID_BACKFILL_SENTINEL_PAYLOAD), + ) + try: await client.upsert( collection_name=collection_name, points=[sentinel_point], wait=True, ) - - logger.info( - "doc_id backfill complete: rewrote %d/%d payloads from int to str", - rewritten, - scanned, - ) except Exception: - logger.error( - "doc_id backfill failed on '%s'; will retry on next restart", + logger.warning( + "doc_id backfill data succeeded on '%s' but sentinel write failed; " + "next restart will re-scroll (idempotent zero-write on clean collection)", collection_name, exc_info=True, ) return + logger.info( + "doc_id backfill complete: rewrote %d/%d payloads from int to str", + rewritten, + scanned, + ) + async def get_qdrant_client() -> AsyncQdrantClient: """ diff --git a/tests/unit/vector/test_qdrant_client.py b/tests/unit/vector/test_qdrant_client.py index 332d0bc6..be728d48 100644 --- a/tests/unit/vector/test_qdrant_client.py +++ b/tests/unit/vector/test_qdrant_client.py @@ -401,7 +401,14 @@ async def test_backfill_logs_and_returns_when_scroll_raises(mocker, caplog): """ client = mocker.AsyncMock() client.retrieve.return_value = [] # No sentinel — backfill must run - client.scroll.side_effect = RuntimeError("boom") + + # An async-callable side_effect lets AsyncMock await the coroutine + # before the exception propagates; assigning a bare exception class + # leaks an un-awaited coroutine and trips RuntimeWarning at gc time. + async def _scroll_raises(*args, **kwargs): + raise RuntimeError("boom") + + client.scroll.side_effect = _scroll_raises with caplog.at_level("ERROR", logger="nextcloud_mcp_server.vector.qdrant_client"): await _backfill_doc_id_to_string( @@ -413,8 +420,121 @@ async def test_backfill_logs_and_returns_when_scroll_raises(mocker, caplog): client.set_payload.assert_not_awaited() errors = [r for r in caplog.records if r.levelname == "ERROR"] assert len(errors) == 1 - assert "doc_id backfill failed" in errors[0].getMessage() + assert "doc_id backfill scroll failed" in errors[0].getMessage() assert "test-collection" in errors[0].getMessage() # exc_info=True attaches the original exception to the log record. assert errors[0].exc_info is not None assert errors[0].exc_info[0] is RuntimeError + + +@pytest.mark.unit +async def test_backfill_logs_warning_when_sentinel_upsert_fails(mocker, caplog): + """Sentinel-write failure after a successful scroll logs WARNING, not ERROR. + + A failure here means the data migration succeeded but the + short-circuit marker is missing. The data is correct; only the + marker is absent, so the next restart will re-scroll an + already-clean collection (idempotent zero-write) and retry the + upsert. Differentiating this from a genuine scroll failure prevents + an "ERROR — backfill failed" log line that contradicts the + successful data state. + """ + client = mocker.AsyncMock() + client.retrieve.return_value = [] # No sentinel — backfill must run + client.scroll.return_value = ([], None) # Empty scroll — clean collection + + async def _upsert_raises(*args, **kwargs): + raise RuntimeError("sentinel write blip") + + client.upsert.side_effect = _upsert_raises + + with caplog.at_level("WARNING", logger="nextcloud_mcp_server.vector.qdrant_client"): + await _backfill_doc_id_to_string( + client, "test-collection", _backfill_dimension() + ) + + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert len(warnings) == 1 + assert "sentinel write failed" in warnings[0].getMessage() + assert "test-collection" in warnings[0].getMessage() + assert warnings[0].exc_info is not None + assert warnings[0].exc_info[0] is RuntimeError + # No ERROR — data state is correct, not a backfill failure. + assert not [r for r in caplog.records if r.levelname == "ERROR"] + + +@pytest.mark.unit +async def test_backfill_emits_progress_log_every_20_batches(mocker, caplog): + """Long scrolls emit a progress INFO line every 20 batches. + + Operators auditing a 50k+ point collection's startup migration need + proof the server isn't hung; a single start/end pair leaves a + minutes-long silence in the log. The progress line carries the + collection name, scanned count, and rewritten count so the same + log message also acts as a heartbeat. + """ + client = mocker.AsyncMock() + client.retrieve.return_value = [] + + # Return 21 non-empty batches followed by an empty one to terminate + # the loop; every batch contains points already in str form so no + # set_payload calls happen — the test focuses on the progress log + # cadence, not the rewrite path. + str_point = SimpleNamespace(id=1, payload={"doc_id": "abc"}) + batches: list[tuple[list[SimpleNamespace], int | None]] = [ + ([str_point], 1) for _ in range(21) + ] + [([], None)] + client.scroll.side_effect = batches + + with caplog.at_level("INFO", logger="nextcloud_mcp_server.vector.qdrant_client"): + await _backfill_doc_id_to_string( + client, "test-collection", _backfill_dimension() + ) + + progress_messages = [ + r.getMessage() + for r in caplog.records + if "doc_id backfill progress on" in r.getMessage() + ] + # 21 batches → exactly one progress line at batch 20. + assert len(progress_messages) == 1 + assert "scanned 20 points" in progress_messages[0] + assert "test-collection" in progress_messages[0] + + +@pytest.mark.unit +async def test_ensure_keyword_payload_indexes_summarises_failed_fields(mocker, caplog): + """A non-400 failure surfaces both as ERROR and a WARNING summary. + + Per-field ERROR lines are easy to miss in startup noise; the + WARNING summary at the end of the loop names every field that + didn't get an index, so operators auditing the log can spot the + degraded state at a glance. + """ + client = mocker.AsyncMock() + client.get_collection.return_value = SimpleNamespace(payload_schema={}) + # Two of the three fields fail with 5xx; one succeeds. + call_count = {"n": 0} + + async def _create_index(*args, **kwargs): + call_count["n"] += 1 + if call_count["n"] != 2: + raise _make_unexpected(500, b'{"status":{"error":"boom"}}') + return None + + client.create_payload_index.side_effect = _create_index + + with caplog.at_level("WARNING", logger="nextcloud_mcp_server.vector.qdrant_client"): + await _ensure_keyword_payload_indexes(client, "test-collection") + + summary = [ + r.getMessage() + for r in caplog.records + if "Payload index creation incomplete" in r.getMessage() + ] + assert len(summary) == 1 + # Field order matches _KEYWORD_PAYLOAD_FIELDS = ("doc_id", "user_id", "doc_type") + assert "doc_id" in summary[0] + assert "doc_type" in summary[0] + assert "user_id" not in summary[0] # The one that succeeded. + assert "test-collection" in summary[0] From 60a9882c9239dc2e69f9099ee0853f0195e6d1d2 Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Sat, 9 May 2026 00:30:29 +0200 Subject: [PATCH 07/27] fix(vector): address PR review round 6 + SonarCloud findings Reviewer feedback (3 important + 3 nits): - Wrap _ensure_keyword_payload_indexes' get_collection() call in try/except. The qdrant_client singleton is already assigned by the time this function runs, so a transient timeout/DNS failure propagating out left the process holding a usable client with the migration silently skipped on every subsequent call. Now logs ERROR with exc_info and returns; next process restart retries. - Add `and "doc_id" in point.payload` guard to the four set comprehensions in scanner.py (indexed_doc_ids, indexed_file_ids, indexed_item_ids, indexed_card_ids). Previously a payload missing the doc_id key would raise KeyError and crash the entire scan. - Tighten test_ensure_keyword_payload_indexes_logs_400_as_warning to match the per-field warning prefix exactly (`startswith("Schema conflict on payload index")`), so a future change adding 400s to the partial-failure summary surfaces here as a count mismatch. - Add new-collection vs existing-collection context to the _backfill_doc_id_to_string docstring's `dimension` parameter. - Replace the misleading "rewrote 0/N from int to str" wording when no rewriting was needed with "N points scanned, none required rewriting (collection already in str form)". - Add test_ensure_keyword_payload_indexes_logs_and_returns_when_ get_collection_raises mirroring the scroll-failure test. SonarCloud (1 CRITICAL + 1 MINOR): - Refactor _backfill_doc_id_to_string to bring cognitive complexity under 15 (was 19). Extracted two pure helpers: _group_int_doc_ids (group point IDs by stringified doc_id) and _apply_backfill_writes (apply set_payload calls and return rewritten count). The main function's scroll/loop/sentinel structure is unchanged. - Add `await asyncio.sleep(0)` to the three async test side_effect helpers (_scroll_raises, _upsert_raises, _create_index) so they use an actual async feature (S7503). The async-callable shape is still required to avoid the AsyncMock unawaited-coroutine warning when side_effect raises. Co-Authored-By: Claude Opus 4.7 (1M context) --- nextcloud_mcp_server/vector/qdrant_client.py | 129 +++++++++++++------ nextcloud_mcp_server/vector/scanner.py | 8 +- tests/unit/vector/test_qdrant_client.py | 55 +++++++- 3 files changed, 150 insertions(+), 42 deletions(-) diff --git a/nextcloud_mcp_server/vector/qdrant_client.py b/nextcloud_mcp_server/vector/qdrant_client.py index dda203cf..3937b07a 100644 --- a/nextcloud_mcp_server/vector/qdrant_client.py +++ b/nextcloud_mcp_server/vector/qdrant_client.py @@ -51,7 +51,22 @@ async def _ensure_keyword_payload_indexes( operators can intervene, but keep going so the remaining fields still get indexed. """ - collection_info = await client.get_collection(collection_name) + # Mirror the broad swallow in `_backfill_doc_id_to_string`: the singleton + # in `get_qdrant_client` is already assigned by the time this function + # runs, so a transient `get_collection` failure (timeout, DNS blip) + # propagating out would leave the process holding a usable client with + # the migration silently skipped on every subsequent call. Log ERROR + # with exc_info and return; the next process restart retries from scratch. + try: + collection_info = await client.get_collection(collection_name) + except Exception: + logger.error( + "Failed to fetch collection info for '%s'; payload indexes not " + "created. Will retry on next restart.", + collection_name, + exc_info=True, + ) + return existing_schema = collection_info.payload_schema or {} failed_fields: list[str] = [] @@ -103,6 +118,56 @@ async def _ensure_keyword_payload_indexes( ) +def _group_int_doc_ids(points: list[Any]) -> tuple[dict[str, list[Any]], int]: + """Group point IDs whose payload carries an int doc_id, keyed by str(doc_id). + + Returns ``(by_value, scanned)`` where ``scanned`` is the total number of + points inspected (str / missing payloads count toward scanned but are not + grouped). Pulled out of ``_backfill_doc_id_to_string`` to keep that + function's cognitive complexity within the project's limit. + + Point IDs widen to ``Any`` to satisfy the qdrant client's + ``PointsSelector`` signature (UUID / int / str unions) without re-spelling + the full type union here. + """ + by_value: dict[str, list[Any]] = {} + scanned = 0 + for point in points: + scanned += 1 + # Qdrant client typing allows None payload even when with_payload was + # requested; defensive default so the type checker is happy. + payload = point.payload or {} + value = payload.get("doc_id") + if value is None or isinstance(value, str): + continue + by_value.setdefault(str(value), []).append(point.id) + return by_value, scanned + + +async def _apply_backfill_writes( + client: AsyncQdrantClient, + collection_name: str, + by_value: dict[str, list[Any]], +) -> int: + """Apply one ``set_payload`` per stringified doc_id; return rewritten count. + + ``wait=True`` is required because ``_ensure_keyword_payload_indexes`` runs + immediately after this function (see ``get_qdrant_client`` near the call + site) and only indexes committed data — fire-and-forget writes would + leave int payloads invisible to KEYWORD filters. + """ + rewritten = 0 + for str_val, point_ids in by_value.items(): + await client.set_payload( + collection_name=collection_name, + payload={"doc_id": str_val}, + points=point_ids, + wait=True, + ) + rewritten += len(point_ids) + return rewritten + + async def _backfill_doc_id_to_string( client: AsyncQdrantClient, collection_name: str, dimension: int ) -> None: @@ -121,11 +186,18 @@ async def _backfill_doc_id_to_string( Within each scroll batch, points sharing the same int doc_id are batched into a single ``set_payload`` call to minimize Qdrant round-trips. + Only called for **existing** collections (see the + ``if collection_name in collection_names`` branch in + ``get_qdrant_client``); brand-new collections skip the backfill since + there can be no legacy int payloads in a freshly created collection. + Args: client: Qdrant client instance. collection_name: Target collection. - dimension: Dense-vector dimension for the sentinel point's vector - (forwarded by ``get_qdrant_client`` from the embedding model). + dimension: Dense-vector dimension for the sentinel point's vector, + forwarded by ``get_qdrant_client`` from the embedding model. + Required because the sentinel is upserted into an existing + collection and must match the collection's vector schema. """ # Sentinel guard: if the migration ran successfully against this # collection on a previous start, retrieve() returns the marker point @@ -184,35 +256,9 @@ async def _backfill_doc_id_to_string( break batch_num += 1 - - # Group by stringified value so points sharing a doc_id (one document - # → many chunks) collapse into a single set_payload call. Point IDs - # can be int/str/UUID, so widen the value type to satisfy the qdrant - # client's PointsSelector signature without re-spelling the union. - by_value: dict[str, list[Any]] = {} - for point in points: - scanned += 1 - # Qdrant client typing allows None payload even when with_payload - # was requested; defensive default so the type checker is happy. - payload = point.payload or {} - value = payload.get("doc_id") - if value is None or isinstance(value, str): - continue - by_value.setdefault(str(value), []).append(point.id) - - for str_val, point_ids in by_value.items(): - # wait=True is required because _ensure_keyword_payload_indexes - # runs immediately after this function (see get_qdrant_client - # near the call site) and only indexes committed data — - # fire-and-forget writes would leave int payloads invisible - # to KEYWORD filters. - await client.set_payload( - collection_name=collection_name, - payload={"doc_id": str_val}, - points=point_ids, - wait=True, - ) - rewritten += len(point_ids) + by_value, batch_scanned = _group_int_doc_ids(points) + scanned += batch_scanned + rewritten += await _apply_backfill_writes(client, collection_name, by_value) if batch_num % progress_log_every == 0: logger.info( @@ -264,11 +310,20 @@ async def _backfill_doc_id_to_string( ) return - logger.info( - "doc_id backfill complete: rewrote %d/%d payloads from int to str", - rewritten, - scanned, - ) + if rewritten: + logger.info( + "doc_id backfill complete on '%s': rewrote %d/%d int payloads to str", + collection_name, + rewritten, + scanned, + ) + else: + logger.info( + "doc_id backfill complete on '%s': %d points scanned, none required " + "rewriting (collection already in str form)", + collection_name, + scanned, + ) async def get_qdrant_client() -> AsyncQdrantClient: diff --git a/nextcloud_mcp_server/vector/scanner.py b/nextcloud_mcp_server/vector/scanner.py index c483019b..6322411e 100644 --- a/nextcloud_mcp_server/vector/scanner.py +++ b/nextcloud_mcp_server/vector/scanner.py @@ -230,7 +230,7 @@ async def scan_user_documents( indexed_doc_ids = { str(point.payload["doc_id"]) for point in (scroll_result[0] or []) - if point.payload is not None + if point.payload is not None and "doc_id" in point.payload } logger.debug(f"Found {len(indexed_doc_ids)} indexed documents in Qdrant") @@ -403,7 +403,7 @@ async def scan_user_documents( indexed_file_ids = { str(point.payload["doc_id"]) for point in (file_scroll_result[0] or []) - if point.payload is not None + if point.payload is not None and "doc_id" in point.payload } logger.debug(f"Found {len(indexed_file_ids)} indexed files in Qdrant") @@ -683,7 +683,7 @@ async def scan_news_items( indexed_item_ids = { str(point.payload["doc_id"]) for point in (scroll_result[0] or []) - if point.payload is not None + if point.payload is not None and "doc_id" in point.payload } logger.debug(f"Found {len(indexed_item_ids)} indexed news items in Qdrant") @@ -862,7 +862,7 @@ async def scan_deck_cards( indexed_card_ids = { str(point.payload["doc_id"]) for point in (scroll_result[0] or []) - if point.payload is not None + if point.payload is not None and "doc_id" in point.payload } logger.debug(f"Found {len(indexed_card_ids)} indexed deck cards in Qdrant") diff --git a/tests/unit/vector/test_qdrant_client.py b/tests/unit/vector/test_qdrant_client.py index be728d48..b345d24c 100644 --- a/tests/unit/vector/test_qdrant_client.py +++ b/tests/unit/vector/test_qdrant_client.py @@ -14,6 +14,7 @@ run at startup. Producer-side normalization is exercised by the existing scanner tests. """ +import asyncio from types import SimpleNamespace from unittest.mock import call @@ -152,7 +153,12 @@ async def test_ensure_keyword_payload_indexes_logs_400_as_warning(mocker, caplog # Loop continued past the failing field; all three were attempted. assert client.create_payload_index.await_count == len(_KEYWORD_PAYLOAD_FIELDS) warnings = [r for r in caplog.records if r.levelname == "WARNING"] + # 400s do not contribute to the partial-failure summary (which fires + # only for non-400 errors), so this is the per-field warning, not the + # summary. Match the message prefix exactly so a future change adding + # 400s to the summary would surface here as a count mismatch. assert len(warnings) == 1 + assert warnings[0].getMessage().startswith("Schema conflict on payload index") assert "different schema" in warnings[0].getMessage() @@ -183,6 +189,42 @@ async def test_ensure_keyword_payload_indexes_logs_non_400_as_error(mocker, capl assert "internal server error" in msg +@pytest.mark.unit +async def test_ensure_keyword_payload_indexes_logs_and_returns_when_get_collection_raises( + mocker, caplog +): + """A get_collection failure is logged and swallowed; no indexes are attempted. + + Mirrors the broad swallow in `_backfill_doc_id_to_string`. The + qdrant_client singleton is already assigned by the time this + function runs, so re-raising would leave the process holding a + usable client with the migration silently skipped on every + subsequent call. Catching, logging, and returning preserves the + retry-on-next-restart behavior. + """ + client = mocker.AsyncMock() + + async def _get_collection_raises(*args, **kwargs): + # See _scroll_raises in the backfill section for why this is async. + await asyncio.sleep(0) + raise RuntimeError("connection refused") + + client.get_collection.side_effect = _get_collection_raises + + with caplog.at_level("ERROR", logger="nextcloud_mcp_server.vector.qdrant_client"): + await _ensure_keyword_payload_indexes(client, "test-collection") + + # No index creation was attempted — the function returned early. + client.create_payload_index.assert_not_awaited() + errors = [r for r in caplog.records if r.levelname == "ERROR"] + assert len(errors) == 1 + msg = errors[0].getMessage() + assert "Failed to fetch collection info for 'test-collection'" in msg + assert "Will retry on next restart" in msg + assert errors[0].exc_info is not None + assert errors[0].exc_info[0] is RuntimeError + + # --------------------------------------------------------------------------- # _backfill_doc_id_to_string # --------------------------------------------------------------------------- @@ -212,7 +254,10 @@ async def test_backfill_clean_collection_makes_no_writes(mocker, caplog): r.getMessage() for r in caplog.records if "backfill complete" in r.getMessage() ] assert completion_logs, "expected an INFO log line for backfill completion" - assert "0/2" in completion_logs[0] + # rewritten=0 → human-readable wording instead of the misleading + # "rewrote 0/N from int to str" formula. + assert "2 points scanned" in completion_logs[0] + assert "none required rewriting" in completion_logs[0] @pytest.mark.unit @@ -405,7 +450,11 @@ async def test_backfill_logs_and_returns_when_scroll_raises(mocker, caplog): # An async-callable side_effect lets AsyncMock await the coroutine # before the exception propagates; assigning a bare exception class # leaks an un-awaited coroutine and trips RuntimeWarning at gc time. + # The `await asyncio.sleep(0)` is a no-op event-loop yield that + # satisfies static analysis ("async function uses no async features") + # without changing observable behavior. async def _scroll_raises(*args, **kwargs): + await asyncio.sleep(0) raise RuntimeError("boom") client.scroll.side_effect = _scroll_raises @@ -444,6 +493,8 @@ async def test_backfill_logs_warning_when_sentinel_upsert_fails(mocker, caplog): client.scroll.return_value = ([], None) # Empty scroll — clean collection async def _upsert_raises(*args, **kwargs): + # See _scroll_raises above for why this is async + sleep(0). + await asyncio.sleep(0) raise RuntimeError("sentinel write blip") client.upsert.side_effect = _upsert_raises @@ -517,6 +568,8 @@ async def test_ensure_keyword_payload_indexes_summarises_failed_fields(mocker, c call_count = {"n": 0} async def _create_index(*args, **kwargs): + # See _scroll_raises above for why this is async + sleep(0). + await asyncio.sleep(0) call_count["n"] += 1 if call_count["n"] != 2: raise _make_unexpected(500, b'{"status":{"error":"boom"}}') From d00779ce7947a8f5260877c022dbca3312991e75 Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Sat, 9 May 2026 13:01:05 +0200 Subject: [PATCH 08/27] fix(vector): add BOOL index for is_placeholder + correct wait=True docstring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewer feedback (2 items): - Add a BOOL payload index for `is_placeholder` alongside the three KEYWORD fields. Strict-mode index-required filtering on Qdrant Cloud enforces a payload index on any field used in a `FieldCondition` regardless of value type, so `get_placeholder_filter` and `delete_placeholder_point` would have produced HTTP 400 on Cloud instances even after this PR's KEYWORD fix. Implementation: replace `_KEYWORD_PAYLOAD_FIELDS: tuple` with `_PAYLOAD_INDEX_FIELDS: dict[str, PayloadSchemaType]` so each field carries its own schema type. Rename `_ensure_keyword_payload_indexes` to `_ensure_payload_indexes` since the function now creates more than just KEYWORD indexes. The per-field log line now includes the schema type ("Created KEYWORD payload index on 'doc_id'", "Created BOOL payload index on 'is_placeholder'") so operators can tell which type was created without checking the source. - Correct the misleading `wait=True` docstring in `_apply_backfill_writes`. The previous wording said `_ensure_payload_indexes` runs "immediately after this function", but `_apply_backfill_writes` is called in a loop inside `_backfill_doc_id_to_string` — the index creation runs after the backfill function *returns*, not after each write. Rewrote the docstring to capture both load-bearing reasons: (1) per-batch commit ordering for crash-recovery safety, and (2) ensuring the keyword index built later covers committed payloads only. Adds `test_ensure_payload_indexes_includes_is_placeholder_as_bool` asserting the schema type is BOOL specifically. Existing tests updated to use the new dict-based registry (side_effect lists now extend to all four entries; field-set assertions derive from the registry instead of hardcoding 3 KEYWORD names). Co-Authored-By: Claude Opus 4.7 (1M context) --- nextcloud_mcp_server/vector/qdrant_client.py | 50 ++++++--- tests/unit/vector/test_qdrant_client.py | 104 ++++++++++++------- 2 files changed, 104 insertions(+), 50 deletions(-) diff --git a/nextcloud_mcp_server/vector/qdrant_client.py b/nextcloud_mcp_server/vector/qdrant_client.py index 3937b07a..57fe1697 100644 --- a/nextcloud_mcp_server/vector/qdrant_client.py +++ b/nextcloud_mcp_server/vector/qdrant_client.py @@ -20,10 +20,18 @@ logger = logging.getLogger(__name__) # Payload fields filtered by exact-match in scanner/processor/placeholder/eviction. # Qdrant requires a payload index for any field used in a FieldCondition; without -# one, queries fail with HTTP 400 ("Index required but not found"). All three -# carry string values after producer normalization, so a KEYWORD index is the -# correct schema (see ADR notes in commit message). -_KEYWORD_PAYLOAD_FIELDS: tuple[str, ...] = ("doc_id", "user_id", "doc_type") +# one, queries fail with HTTP 400 ("Index required but not found") on instances +# that enforce strict-mode index-required filtering (Qdrant Cloud, network mode +# with strict settings). The three string fields (doc_id, user_id, doc_type) +# carry str values after producer normalization, so KEYWORD is the correct +# schema; is_placeholder is the bool used by ``get_placeholder_filter`` and +# ``delete_placeholder_point`` (see vector/placeholder.py), so it gets BOOL. +_PAYLOAD_INDEX_FIELDS: dict[str, PayloadSchemaType] = { + "doc_id": PayloadSchemaType.KEYWORD, + "user_id": PayloadSchemaType.KEYWORD, + "doc_type": PayloadSchemaType.KEYWORD, + "is_placeholder": PayloadSchemaType.BOOL, +} # Sentinel point that records "this collection has been backfilled to str # doc_id". Written after a successful pass of _backfill_doc_id_to_string so @@ -39,11 +47,13 @@ _DOC_ID_BACKFILL_SENTINEL_PAYLOAD: dict[str, str] = {"_migration_marker": "doc_i _qdrant_client: AsyncQdrantClient | None = None -async def _ensure_keyword_payload_indexes( +async def _ensure_payload_indexes( client: AsyncQdrantClient, collection_name: str ) -> None: - """Create KEYWORD payload indexes for fields used in exact-match filters. + """Create payload indexes for fields used in exact-match filters. + Each entry in ``_PAYLOAD_INDEX_FIELDS`` is created with its declared + schema type (KEYWORD for string fields, BOOL for ``is_placeholder``). Pre-fetches the existing payload schema and skips fields that are already indexed, so routine restarts make no Qdrant write round-trips and emit no INFO log lines. Schema conflicts (a pre-existing index @@ -70,7 +80,7 @@ async def _ensure_keyword_payload_indexes( existing_schema = collection_info.payload_schema or {} failed_fields: list[str] = [] - for field in _KEYWORD_PAYLOAD_FIELDS: + for field, schema_type in _PAYLOAD_INDEX_FIELDS.items(): if field in existing_schema: # Index already present — silent skip. Logging here on every # restart would be noise that hides the genuinely interesting @@ -80,10 +90,10 @@ async def _ensure_keyword_payload_indexes( await client.create_payload_index( collection_name=collection_name, field_name=field, - field_schema=PayloadSchemaType.KEYWORD, + field_schema=schema_type, wait=True, ) - logger.info("Created KEYWORD payload index on '%s'", field) + logger.info("Created %s payload index on '%s'", schema_type.name, field) except UnexpectedResponse as e: body = getattr(e, "content", b"") or b"" body_text = body.decode("utf-8", errors="replace") @@ -151,10 +161,20 @@ async def _apply_backfill_writes( ) -> int: """Apply one ``set_payload`` per stringified doc_id; return rewritten count. - ``wait=True`` is required because ``_ensure_keyword_payload_indexes`` runs - immediately after this function (see ``get_qdrant_client`` near the call - site) and only indexes committed data — fire-and-forget writes would - leave int payloads invisible to KEYWORD filters. + ``wait=True`` is load-bearing for two reasons: + + 1. It ensures each batch commits before the scroll loop advances to + the next page (and before the sentinel is written by the caller + after ``_backfill_doc_id_to_string`` returns). A crash mid-scroll + leaves no sentinel, so the next restart re-scrolls — and that + re-scroll only sees a deterministic, committed partial state when + each batch was committed synchronously. Fire-and-forget writes + would race the next scroll page against still-in-flight rewrites. + 2. ``_ensure_payload_indexes`` runs after this backfill returns and + can only index already-committed payload values. Without + ``wait=True``, the keyword index could be built over points whose + payloads are still int values in flight to disk, leaving them + silently invisible to ``FieldCondition`` filters. """ rewritten = 0 for str_val, point_ids in by_value.items(): @@ -427,7 +447,7 @@ async def get_qdrant_client() -> AsyncQdrantClient: await _backfill_doc_id_to_string( _qdrant_client, collection_name, expected_dimension ) - await _ensure_keyword_payload_indexes(_qdrant_client, collection_name) + await _ensure_payload_indexes(_qdrant_client, collection_name) else: # Collection doesn't exist - create it @@ -460,6 +480,6 @@ async def get_qdrant_client() -> AsyncQdrantClient: f" Distance: COSINE\n" f"Background sync will index all documents with dense + sparse vectors." ) - await _ensure_keyword_payload_indexes(_qdrant_client, collection_name) + await _ensure_payload_indexes(_qdrant_client, collection_name) return _qdrant_client diff --git a/tests/unit/vector/test_qdrant_client.py b/tests/unit/vector/test_qdrant_client.py index b345d24c..126f3d92 100644 --- a/tests/unit/vector/test_qdrant_client.py +++ b/tests/unit/vector/test_qdrant_client.py @@ -25,16 +25,16 @@ from qdrant_client.models import PayloadSchemaType from nextcloud_mcp_server.vector.qdrant_client import ( _DOC_ID_BACKFILL_SENTINEL_ID, - _KEYWORD_PAYLOAD_FIELDS, + _PAYLOAD_INDEX_FIELDS, _backfill_doc_id_to_string, - _ensure_keyword_payload_indexes, + _ensure_payload_indexes, ) def _empty_collection_info() -> SimpleNamespace: """Stand-in for a CollectionInfo with no payload indexes yet. - Tests for _ensure_keyword_payload_indexes only read ``payload_schema`` + Tests for _ensure_payload_indexes only read ``payload_schema`` off the result. None / empty dict both signal "no indexes" — use {} here to match the production-code default. """ @@ -71,38 +71,66 @@ def _record(point_id: int | str, doc_id: int | str | None) -> SimpleNamespace: # --------------------------------------------------------------------------- -# _ensure_keyword_payload_indexes +# _ensure_payload_indexes # --------------------------------------------------------------------------- @pytest.mark.unit -async def test_ensure_keyword_payload_indexes_creates_each_field(mocker): - """Happy path: every field in _KEYWORD_PAYLOAD_FIELDS gets a KEYWORD index.""" +async def test_ensure_payload_indexes_creates_each_field(mocker): + """Happy path: every field in _PAYLOAD_INDEX_FIELDS gets its declared schema. + + The dict-of-(field, schema_type) registry pairs string fields with + KEYWORD and the boolean ``is_placeholder`` with BOOL — both are + required because Qdrant's strict-mode index-required filtering + enforces a payload index for any ``FieldCondition`` regardless of + value type. + """ client = mocker.AsyncMock() client.get_collection.return_value = _empty_collection_info() - await _ensure_keyword_payload_indexes(client, "test-collection") + await _ensure_payload_indexes(client, "test-collection") - assert client.create_payload_index.await_count == len(_KEYWORD_PAYLOAD_FIELDS) + assert client.create_payload_index.await_count == len(_PAYLOAD_INDEX_FIELDS) expected_calls = [ call( collection_name="test-collection", field_name=field, - field_schema=PayloadSchemaType.KEYWORD, + field_schema=schema_type, wait=True, ) - for field in _KEYWORD_PAYLOAD_FIELDS + for field, schema_type in _PAYLOAD_INDEX_FIELDS.items() ] client.create_payload_index.assert_has_awaits(expected_calls, any_order=False) @pytest.mark.unit -async def test_ensure_keyword_payload_indexes_skips_fields_already_indexed( - mocker, caplog -): +async def test_ensure_payload_indexes_includes_is_placeholder_as_bool(mocker): + """is_placeholder must be created with BOOL schema, not KEYWORD. + + ``get_placeholder_filter`` and ``delete_placeholder_point`` filter on + ``is_placeholder`` (a bool); creating it with KEYWORD would still + fail strict-mode index-required filtering on Qdrant Cloud because + the index type wouldn't match the value type. + """ + client = mocker.AsyncMock() + client.get_collection.return_value = _empty_collection_info() + + await _ensure_payload_indexes(client, "test-collection") + + bool_calls = [ + c + for c in client.create_payload_index.await_args_list + if c.kwargs.get("field_name") == "is_placeholder" + ] + assert len(bool_calls) == 1, "is_placeholder must be created exactly once" + assert bool_calls[0].kwargs["field_schema"] is PayloadSchemaType.BOOL + + +@pytest.mark.unit +async def test_ensure_payload_indexes_skips_fields_already_indexed(mocker, caplog): """Routine restart path: existing payload indexes are silently skipped. - Without the pre-fetch, every restart logs `Created KEYWORD payload + Without the pre-fetch, every restart logs `Created payload index on ''` for every field — noise that hides genuinely interesting first-time-creation lines. With the pre-fetch, no log fires and no Qdrant write round-trip happens for already-indexed @@ -114,21 +142,23 @@ async def test_ensure_keyword_payload_indexes_skips_fields_already_indexed( ) with caplog.at_level("INFO", logger="nextcloud_mcp_server.vector.qdrant_client"): - await _ensure_keyword_payload_indexes(client, "test-collection") + await _ensure_payload_indexes(client, "test-collection") - # Only the two missing fields are created. - assert client.create_payload_index.await_count == 2 + # Only the missing fields are created — every entry in the registry + # other than the one already in the schema. + expected_missing = set(_PAYLOAD_INDEX_FIELDS) - {"doc_id"} + assert client.create_payload_index.await_count == len(expected_missing) created_fields = { c.kwargs["field_name"] for c in client.create_payload_index.await_args_list } - assert created_fields == {"user_id", "doc_type"} + assert created_fields == expected_missing # No INFO log fires for the already-indexed field. info_messages = [r.getMessage() for r in caplog.records if r.levelname == "INFO"] assert not any("doc_id" in m for m in info_messages), info_messages @pytest.mark.unit -async def test_ensure_keyword_payload_indexes_logs_400_as_warning(mocker, caplog): +async def test_ensure_payload_indexes_logs_400_as_warning(mocker, caplog): """Any 400 from create_payload_index is logged at WARNING and skipped. Real Qdrant returns 200 when the index already exists with a matching @@ -138,20 +168,21 @@ async def test_ensure_keyword_payload_indexes_logs_400_as_warning(mocker, caplog """ client = mocker.AsyncMock() client.get_collection.return_value = _empty_collection_info() + # First field fails with 400; remaining fields succeed. One side_effect + # entry per item in _PAYLOAD_INDEX_FIELDS so the iteration is exhaustive. client.create_payload_index.side_effect = [ _make_unexpected( 400, b'{"status":{"error":"field \\"doc_id\\" indexed with different schema"}}', ), - None, - None, + *([None] * (len(_PAYLOAD_INDEX_FIELDS) - 1)), ] with caplog.at_level("WARNING", logger="nextcloud_mcp_server.vector.qdrant_client"): - await _ensure_keyword_payload_indexes(client, "test-collection") + await _ensure_payload_indexes(client, "test-collection") - # Loop continued past the failing field; all three were attempted. - assert client.create_payload_index.await_count == len(_KEYWORD_PAYLOAD_FIELDS) + # Loop continued past the failing field; every field was attempted. + assert client.create_payload_index.await_count == len(_PAYLOAD_INDEX_FIELDS) warnings = [r for r in caplog.records if r.levelname == "WARNING"] # 400s do not contribute to the partial-failure summary (which fires # only for non-400 errors), so this is the per-field warning, not the @@ -163,7 +194,7 @@ async def test_ensure_keyword_payload_indexes_logs_400_as_warning(mocker, caplog @pytest.mark.unit -async def test_ensure_keyword_payload_indexes_logs_non_400_as_error(mocker, caplog): +async def test_ensure_payload_indexes_logs_non_400_as_error(mocker, caplog): """A non-400 status from create_payload_index escalates to ERROR. A 5xx response (e.g., Qdrant temporarily unavailable) should not be @@ -172,16 +203,16 @@ async def test_ensure_keyword_payload_indexes_logs_non_400_as_error(mocker, capl """ client = mocker.AsyncMock() client.get_collection.return_value = _empty_collection_info() + # First field fails with 500; remaining fields succeed. client.create_payload_index.side_effect = [ _make_unexpected(500, b'{"status":{"error":"internal server error"}}'), - None, - None, + *([None] * (len(_PAYLOAD_INDEX_FIELDS) - 1)), ] with caplog.at_level("ERROR", logger="nextcloud_mcp_server.vector.qdrant_client"): - await _ensure_keyword_payload_indexes(client, "test-collection") + await _ensure_payload_indexes(client, "test-collection") - assert client.create_payload_index.await_count == len(_KEYWORD_PAYLOAD_FIELDS) + assert client.create_payload_index.await_count == len(_PAYLOAD_INDEX_FIELDS) errors = [r for r in caplog.records if r.levelname == "ERROR"] assert len(errors) == 1 msg = errors[0].getMessage() @@ -190,7 +221,7 @@ async def test_ensure_keyword_payload_indexes_logs_non_400_as_error(mocker, capl @pytest.mark.unit -async def test_ensure_keyword_payload_indexes_logs_and_returns_when_get_collection_raises( +async def test_ensure_payload_indexes_logs_and_returns_when_get_collection_raises( mocker, caplog ): """A get_collection failure is logged and swallowed; no indexes are attempted. @@ -212,7 +243,7 @@ async def test_ensure_keyword_payload_indexes_logs_and_returns_when_get_collecti client.get_collection.side_effect = _get_collection_raises with caplog.at_level("ERROR", logger="nextcloud_mcp_server.vector.qdrant_client"): - await _ensure_keyword_payload_indexes(client, "test-collection") + await _ensure_payload_indexes(client, "test-collection") # No index creation was attempted — the function returned early. client.create_payload_index.assert_not_awaited() @@ -554,7 +585,7 @@ async def test_backfill_emits_progress_log_every_20_batches(mocker, caplog): @pytest.mark.unit -async def test_ensure_keyword_payload_indexes_summarises_failed_fields(mocker, caplog): +async def test_ensure_payload_indexes_summarises_failed_fields(mocker, caplog): """A non-400 failure surfaces both as ERROR and a WARNING summary. Per-field ERROR lines are easy to miss in startup noise; the @@ -564,7 +595,9 @@ async def test_ensure_keyword_payload_indexes_summarises_failed_fields(mocker, c """ client = mocker.AsyncMock() client.get_collection.return_value = SimpleNamespace(payload_schema={}) - # Two of the three fields fail with 5xx; one succeeds. + # All but the second field fail with 5xx. _PAYLOAD_INDEX_FIELDS has + # insertion-ordered keys (doc_id, user_id, doc_type, is_placeholder), + # so call #2 (user_id) is the success case. call_count = {"n": 0} async def _create_index(*args, **kwargs): @@ -578,7 +611,7 @@ async def test_ensure_keyword_payload_indexes_summarises_failed_fields(mocker, c client.create_payload_index.side_effect = _create_index with caplog.at_level("WARNING", logger="nextcloud_mcp_server.vector.qdrant_client"): - await _ensure_keyword_payload_indexes(client, "test-collection") + await _ensure_payload_indexes(client, "test-collection") summary = [ r.getMessage() @@ -586,8 +619,9 @@ async def test_ensure_keyword_payload_indexes_summarises_failed_fields(mocker, c if "Payload index creation incomplete" in r.getMessage() ] assert len(summary) == 1 - # Field order matches _KEYWORD_PAYLOAD_FIELDS = ("doc_id", "user_id", "doc_type") + # All fields except user_id should appear in the summary. assert "doc_id" in summary[0] assert "doc_type" in summary[0] + assert "is_placeholder" in summary[0] assert "user_id" not in summary[0] # The one that succeeded. assert "test-collection" in summary[0] From d390b3a4b8121001d2705827209da8c0c1d29fc1 Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Sat, 9 May 2026 13:48:54 +0200 Subject: [PATCH 09/27] =?UTF-8?q?fix(vector):=20address=20PR=20review=20ro?= =?UTF-8?q?und=208=20=E2=80=94=20anyio=20convention=20+=20cosine-safe=20se?= =?UTF-8?q?ntinel=20+=20dedup=20get=5Fcollection?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewer findings (1 blocking + 2 important): - 🔴 Replace `import asyncio` / `await asyncio.sleep(0)` with `import anyio` / `await anyio.sleep(0)` in the four async side-effect helpers (_scroll_raises, _upsert_raises, _get_collection_raises, _create_index). CLAUDE.md mandates anyio for all async operations; conftest pins the backend to asyncio so the asyncio.sleep call worked today, but the inconsistency would surface the moment that pin moves. - 🟡 Replace the sentinel's zero dense vector with a single non-zero element (`[1e-9] + [0.0] * (dimension - 1)`). Cosine distance is mathematically undefined for the zero vector and Qdrant Cloud strict mode rejects zero-vector upserts. The exact value doesn't matter (sentinel never participates in a search — no user_id/doc_id/doc_type payload) but the upsert itself must be valid. - 🟡 Avoid the duplicate `get_collection` round-trip on every restart. `_ensure_payload_indexes` now accepts an optional `existing_schema: dict | None` parameter; when None it fetches collection_info itself (and the get_collection-failure swallow still applies), but `get_qdrant_client` already fetches collection_info for dimension validation in the existing-collection branch — pass `collection_info.payload_schema or {}` through to skip the second call. The new-collection branch passes `existing_schema={}` explicitly since a freshly created collection has no payload schema. The 🟡 deck_card iteration-fallback finding doesn't apply: the `isdigit()` guard at context.py:612 returns early before either the fast-path or the iteration fallback runs, so non-numeric doc_ids cannot reach the inner `c.id == int(doc_id)` comparison. Co-Authored-By: Claude Opus 4.7 (1M context) --- nextcloud_mcp_server/vector/qdrant_client.py | 81 +++++++++++++------- tests/unit/vector/test_qdrant_client.py | 12 +-- 2 files changed, 59 insertions(+), 34 deletions(-) diff --git a/nextcloud_mcp_server/vector/qdrant_client.py b/nextcloud_mcp_server/vector/qdrant_client.py index 57fe1697..6bcb9a85 100644 --- a/nextcloud_mcp_server/vector/qdrant_client.py +++ b/nextcloud_mcp_server/vector/qdrant_client.py @@ -48,18 +48,28 @@ _qdrant_client: AsyncQdrantClient | None = None async def _ensure_payload_indexes( - client: AsyncQdrantClient, collection_name: str + client: AsyncQdrantClient, + collection_name: str, + existing_schema: dict[str, Any] | None = None, ) -> None: """Create payload indexes for fields used in exact-match filters. Each entry in ``_PAYLOAD_INDEX_FIELDS`` is created with its declared schema type (KEYWORD for string fields, BOOL for ``is_placeholder``). - Pre-fetches the existing payload schema and skips fields that are - already indexed, so routine restarts make no Qdrant write round-trips - and emit no INFO log lines. Schema conflicts (a pre-existing index - with a different type) still surface as a 400 — log loudly so - operators can intervene, but keep going so the remaining fields still - get indexed. + Skips fields that are already in ``existing_schema`` so routine + restarts make no Qdrant write round-trips and emit no INFO log lines. + Schema conflicts (a pre-existing index with a different type) still + surface as a 400 — log loudly so operators can intervene, but keep + going so the remaining fields still get indexed. + + Args: + client: Qdrant client instance. + collection_name: Target collection. + existing_schema: The collection's current ``payload_schema``. If + ``None``, this function fetches it via ``get_collection``; + callers that have already fetched the collection info (e.g. + ``get_qdrant_client``'s dimension-validation step) should pass + it through to avoid a duplicate round-trip. """ # Mirror the broad swallow in `_backfill_doc_id_to_string`: the singleton # in `get_qdrant_client` is already assigned by the time this function @@ -67,17 +77,18 @@ async def _ensure_payload_indexes( # propagating out would leave the process holding a usable client with # the migration silently skipped on every subsequent call. Log ERROR # with exc_info and return; the next process restart retries from scratch. - try: - collection_info = await client.get_collection(collection_name) - except Exception: - logger.error( - "Failed to fetch collection info for '%s'; payload indexes not " - "created. Will retry on next restart.", - collection_name, - exc_info=True, - ) - return - existing_schema = collection_info.payload_schema or {} + if existing_schema is None: + try: + collection_info = await client.get_collection(collection_name) + except Exception: + logger.error( + "Failed to fetch collection info for '%s'; payload indexes not " + "created. Will retry on next restart.", + collection_name, + exc_info=True, + ) + return + existing_schema = collection_info.payload_schema or {} failed_fields: list[str] = [] for field, schema_type in _PAYLOAD_INDEX_FIELDS.items(): @@ -301,16 +312,20 @@ async def _backfill_doc_id_to_string( # Data backfill succeeded — write the sentinel so a future restart can # short-circuit. Empty sparse vector mirrors the placeholder.py - # convention (vector/placeholder.py); zero dense vector is fine - # because the sentinel never participates in a search (no user_id / - # doc_id / doc_type payload to match). A failure here is non-fatal: - # the data is correct; only the short-circuit marker is missing, so - # the next restart will re-scroll an already-clean collection (idempotent - # zero-write) before retrying the upsert. + # convention (vector/placeholder.py). The dense vector uses a single + # non-zero element instead of all zeros: cosine distance is undefined + # for the zero vector and Qdrant Cloud's strict mode rejects zero-vector + # upserts. The sentinel still never participates in a search (no + # user_id / doc_id / doc_type payload to match), so the exact value + # doesn't matter — it just has to be normalisable. + # A failure here is non-fatal: the data is correct; only the short-circuit + # marker is missing, so the next restart will re-scroll an already-clean + # collection (idempotent zero-write) before retrying the upsert. + sentinel_dense = [1e-9] + [0.0] * (dimension - 1) sentinel_point = PointStruct( id=_DOC_ID_BACKFILL_SENTINEL_ID, vector={ - "dense": [0.0] * dimension, + "dense": sentinel_dense, "sparse": models.SparseVector(indices=[], values=[]), }, payload=dict(_DOC_ID_BACKFILL_SENTINEL_PAYLOAD), @@ -443,11 +458,17 @@ async def get_qdrant_client() -> AsyncQdrantClient: # Existing collections may pre-date the doc_id normalization / # payload-index work. Backfill before creating the index so the - # index covers every point. + # index covers every point. Pass the already-fetched + # collection_info.payload_schema through to avoid a redundant + # get_collection round-trip on every restart. await _backfill_doc_id_to_string( _qdrant_client, collection_name, expected_dimension ) - await _ensure_payload_indexes(_qdrant_client, collection_name) + await _ensure_payload_indexes( + _qdrant_client, + collection_name, + existing_schema=collection_info.payload_schema or {}, + ) else: # Collection doesn't exist - create it @@ -480,6 +501,10 @@ async def get_qdrant_client() -> AsyncQdrantClient: f" Distance: COSINE\n" f"Background sync will index all documents with dense + sparse vectors." ) - await _ensure_payload_indexes(_qdrant_client, collection_name) + # Freshly created collection has no payload schema yet; pass {} + # explicitly to skip the otherwise-redundant get_collection call. + await _ensure_payload_indexes( + _qdrant_client, collection_name, existing_schema={} + ) return _qdrant_client diff --git a/tests/unit/vector/test_qdrant_client.py b/tests/unit/vector/test_qdrant_client.py index 126f3d92..32c8bd3a 100644 --- a/tests/unit/vector/test_qdrant_client.py +++ b/tests/unit/vector/test_qdrant_client.py @@ -14,10 +14,10 @@ run at startup. Producer-side normalization is exercised by the existing scanner tests. """ -import asyncio from types import SimpleNamespace from unittest.mock import call +import anyio import httpx import pytest from qdrant_client.http.exceptions import UnexpectedResponse @@ -237,7 +237,7 @@ async def test_ensure_payload_indexes_logs_and_returns_when_get_collection_raise async def _get_collection_raises(*args, **kwargs): # See _scroll_raises in the backfill section for why this is async. - await asyncio.sleep(0) + await anyio.sleep(0) raise RuntimeError("connection refused") client.get_collection.side_effect = _get_collection_raises @@ -481,11 +481,11 @@ async def test_backfill_logs_and_returns_when_scroll_raises(mocker, caplog): # An async-callable side_effect lets AsyncMock await the coroutine # before the exception propagates; assigning a bare exception class # leaks an un-awaited coroutine and trips RuntimeWarning at gc time. - # The `await asyncio.sleep(0)` is a no-op event-loop yield that + # The `await anyio.sleep(0)` is a no-op event-loop yield that # satisfies static analysis ("async function uses no async features") # without changing observable behavior. async def _scroll_raises(*args, **kwargs): - await asyncio.sleep(0) + await anyio.sleep(0) raise RuntimeError("boom") client.scroll.side_effect = _scroll_raises @@ -525,7 +525,7 @@ async def test_backfill_logs_warning_when_sentinel_upsert_fails(mocker, caplog): async def _upsert_raises(*args, **kwargs): # See _scroll_raises above for why this is async + sleep(0). - await asyncio.sleep(0) + await anyio.sleep(0) raise RuntimeError("sentinel write blip") client.upsert.side_effect = _upsert_raises @@ -602,7 +602,7 @@ async def test_ensure_payload_indexes_summarises_failed_fields(mocker, caplog): async def _create_index(*args, **kwargs): # See _scroll_raises above for why this is async + sleep(0). - await asyncio.sleep(0) + await anyio.sleep(0) call_count["n"] += 1 if call_count["n"] != 2: raise _make_unexpected(500, b'{"status":{"error":"boom"}}') From fd8c037eea869133ffc41f7c323fdc1836b96c60 Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Sat, 9 May 2026 14:03:42 +0200 Subject: [PATCH 10/27] fix(login-flow): allow Astrolabe's OAuth client on the management API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `mcp-login-flow` profile's `ALLOWED_MGMT_CLIENT` was set to the test-fixture id `nextcloudMcpServerUIPublicClient` only, but the actual Astrolabe app provisions its OIDC client as `astrolabeMcpClientOAuth00000000000` (see `app-hooks/before-starting/26-configure-astrolabe-oauth.sh:39`). All tokens issued through the "Enable Semantic Search" flow were rejected with HTTP 401 by `unified_verifier.py:222-227`'s allowlist check, and the Astrolabe UI's retry loop subsequently exhausted the `api/passwords.py` 5/hr rate limit (HTTP 429). Switch the `mcp-login-flow` allowlist to Astrolabe's client id so production-shaped traffic actually validates. The `mcp-multi-user-basic` profile keeps `nextcloudMcpServerUIPublicClient` for the `configure_astrolabe_for_mcp_server` test fixture. Also bump `third_party/astrolabe` 0.13.12 → 0.14.0 to pull in the chunk-context indexed-lookup fix (#75) and the PDF bbox highlight overlay (#76) that match the master-side changes already merged on this branch. Co-Authored-By: Claude Opus 4.7 (1M context) --- docker-compose.yml | 2 +- third_party/astrolabe | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 208ce05e..46984db5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -293,7 +293,7 @@ services: # the same configure_astrolabe_for_mcp_server fixture (which creates the # static `nextcloudMcpServerUIPublicClient` OIDC client) works for tests # against this profile too. - - ALLOWED_MGMT_CLIENT=nextcloudMcpServerUIPublicClient + - ALLOWED_MGMT_CLIENT=astrolabeMcpClientOAuth00000000000 volumes: - login-flow-data:/app/data - login-flow-oauth-storage:/app/.oauth diff --git a/third_party/astrolabe b/third_party/astrolabe index d9e641b9..7e08b498 160000 --- a/third_party/astrolabe +++ b/third_party/astrolabe @@ -1 +1 @@ -Subproject commit d9e641b93e6511fba0c27acc06ee0733fcdd5add +Subproject commit 7e08b4983ae03675a81039d85c7056eb7a3bd70e From d83c32a9ddbb99515ee366864400d91d3dd42db4 Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Sat, 9 May 2026 14:06:02 +0200 Subject: [PATCH 11/27] test(login-flow): use Astrolabe's client id for management API tests The previous commit moved `mcp-login-flow`'s `ALLOWED_MGMT_CLIENT` to `astrolabeMcpClientOAuth00000000000` so production-shaped Astrolabe traffic actually validates. Update the management API test fixture to match: the static OIDC client created in `tests/server/login_flow/conftest.py:login_flow_static_client_credentials` now uses the same id `app-hooks/before-starting/26-configure-astrolabe-oauth.sh` provisions in real deployments, so the test path exercises the same code as production rather than a substituted fixture-only id. `mcp-multi-user-basic`'s allowlist is unchanged (`nextcloudMcpServerUIPublicClient`) and the shared `configure_astrolabe_for_mcp_server` fixture in `tests/conftest.py` keeps that as its default, so multi-user-basic tests are unaffected. Co-Authored-By: Claude Opus 4.7 (1M context) --- tests/server/login_flow/conftest.py | 23 +++++++++++-------- .../server/login_flow/test_management_api.py | 8 ++++--- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/tests/server/login_flow/conftest.py b/tests/server/login_flow/conftest.py index 900f02fc..a7714a87 100644 --- a/tests/server/login_flow/conftest.py +++ b/tests/server/login_flow/conftest.py @@ -936,21 +936,26 @@ async def diana_login_flow_mcp_client( # Static OIDC client used by the management API integration tests. -# Matches the value `mcp-login-flow` and `mcp-multi-user-basic` allowlist -# (`ALLOWED_MGMT_CLIENT=nextcloudMcpServerUIPublicClient`) so tokens issued -# to it pass the management API allowlist check. -STATIC_MGMT_CLIENT_ID = "nextcloudMcpServerUIPublicClient" +# Matches the `mcp-login-flow` allowlist +# (`ALLOWED_MGMT_CLIENT=astrolabeMcpClientOAuth00000000000`) — i.e. the same +# id `app-hooks/before-starting/26-configure-astrolabe-oauth.sh` provisions +# in real deployments — so tokens issued to it pass the management API +# allowlist check on `mcp-login-flow` and exercise the production-shaped +# code path. +STATIC_MGMT_CLIENT_ID = "astrolabeMcpClientOAuth00000000000" @pytest.fixture(scope="session") async def login_flow_static_client_credentials(anyio_backend, oauth_callback_server): - """Pre-create the static OIDC client `nextcloudMcpServerUIPublicClient` + """Pre-create the static OIDC client `astrolabeMcpClientOAuth00000000000` via `occ oidc:create` with the test's OAuth callback URL. - The static client_id is allowlisted on `mcp-login-flow` (and - `mcp-multi-user-basic`) via `ALLOWED_MGMT_CLIENT`, so tokens it issues - pass the management API allowlist check. Uses a confidential JWT-token - client to match production Astrolabe configuration. + The static client_id matches the id provisioned by + `app-hooks/before-starting/26-configure-astrolabe-oauth.sh` in real + deployments, and is allowlisted on `mcp-login-flow` via + `ALLOWED_MGMT_CLIENT`, so tokens it issues pass the management API + allowlist check. Uses a confidential JWT-token client to match + production Astrolabe configuration. Yields: (client_id, client_secret, callback_url, token_endpoint, authorization_endpoint) """ diff --git a/tests/server/login_flow/test_management_api.py b/tests/server/login_flow/test_management_api.py index 397066dd..9830c145 100644 --- a/tests/server/login_flow/test_management_api.py +++ b/tests/server/login_flow/test_management_api.py @@ -1,9 +1,11 @@ """Integration tests for the management API on the login-flow MCP server. These tests drive a real OAuth flow against Nextcloud's `oidc` app using the -static `nextcloudMcpServerUIPublicClient` client (which is allowlisted on the -`mcp-login-flow` container via `ALLOWED_MGMT_CLIENT`), then hit the -management API endpoints with the resulting bearer token. +static `astrolabeMcpClientOAuth00000000000` client (which is allowlisted on +the `mcp-login-flow` container via `ALLOWED_MGMT_CLIENT` and matches the id +provisioned by `app-hooks/before-starting/26-configure-astrolabe-oauth.sh` +in real deployments), then hit the management API endpoints with the +resulting bearer token. Regression coverage for the bug where /api/v1/apps proxied to OCS v1 /cloud/apps and always 401'd. The handler now uses /ocs/v2.php/cloud/capabilities, From a27f738dbff5e49de55fd50907635ac99081f0c2 Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Sat, 9 May 2026 14:32:33 +0200 Subject: [PATCH 12/27] fix(vector): tighten get_chunk_bbox_and_page_from_qdrant doc_id to str MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🔴 Blocking finding from PR #773 latest review: `get_chunk_bbox_and_page_from_qdrant` (`search/context.py:199`) still declared `doc_id: int | str` and passed the raw value into `MatchValue(value=doc_id)` at lines 239 and 256 without `str()` coercion. After this branch's startup backfill normalises every Qdrant `doc_id` payload to a string, an `int` filter would silently match zero points — the function would return `(None, None)` instead of the chunk bbox / page, and PDF highlight overlays would fail in production. Take option 2 from the reviewer's two suggestions (annotation tightening over inline coercion): the producer side of this PR has already narrowed every other `doc_id` annotation to `str`, so this function is the last hold-out. Pushing the contract into the type system means `ty` will catch any future regression at the call site. Production callers in `api/visualization.py` and `auth/viz_routes.py` already pass `doc_id` (str) verbatim after the recent merge with master's chunk_index-first refactor, so no caller-side changes needed. Update the 9 calls in `tests/unit/test_chunk_bbox_helper.py` to use string literals (`"42"` / `"99"` / `"1"`) instead of integers. The mock doesn't validate `MatchValue` value types, so the tests passed with stale int doc_ids today — but they were exercising a path production no longer takes. Co-Authored-By: Claude Opus 4.7 (1M context) --- nextcloud_mcp_server/search/context.py | 8 ++++++-- tests/unit/test_chunk_bbox_helper.py | 18 +++++++++--------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/nextcloud_mcp_server/search/context.py b/nextcloud_mcp_server/search/context.py index a3449b5c..f818e278 100644 --- a/nextcloud_mcp_server/search/context.py +++ b/nextcloud_mcp_server/search/context.py @@ -198,7 +198,7 @@ async def _get_deck_metadata_from_qdrant( async def get_chunk_bbox_and_page_from_qdrant( user_id: str, - doc_id: int | str, + doc_id: str, chunk_index: int | None, chunk_start: int, chunk_end: int, @@ -214,7 +214,11 @@ async def get_chunk_bbox_and_page_from_qdrant( Args: user_id: User ID who owns the document - doc_id: Document ID (int for file/note, str for some doc types) + doc_id: Document ID — always a string. Producers stringify their + native ID before writing to Qdrant so the keyword payload + index on ``doc_id`` matches every point regardless of source + doc_type. An ``int`` filter against the str-indexed payload + would silently match zero points. chunk_index: Zero-based chunk index, or None to use offset fallback chunk_start: Character offset where chunk starts (used when chunk_index is None) diff --git a/tests/unit/test_chunk_bbox_helper.py b/tests/unit/test_chunk_bbox_helper.py index af542353..f7abed91 100644 --- a/tests/unit/test_chunk_bbox_helper.py +++ b/tests/unit/test_chunk_bbox_helper.py @@ -58,7 +58,7 @@ class TestIndexedPath: with ctx: result = await get_chunk_bbox_and_page_from_qdrant( user_id="alice", - doc_id=42, + doc_id="42", chunk_index=3, chunk_start=0, chunk_end=100, @@ -84,7 +84,7 @@ class TestOffsetFallbackPath: with ctx: result = await get_chunk_bbox_and_page_from_qdrant( user_id="bob", - doc_id=99, + doc_id="99", chunk_index=None, chunk_start=500, chunk_end=600, @@ -105,7 +105,7 @@ class TestOffsetFallbackPath: with ctx, caplog.at_level("WARNING"): result = await get_chunk_bbox_and_page_from_qdrant( user_id="bob", - doc_id=99, + doc_id="99", chunk_index=None, chunk_start=0, chunk_end=100, @@ -124,7 +124,7 @@ class TestPayloadShape: with ctx: result = await get_chunk_bbox_and_page_from_qdrant( user_id="alice", - doc_id=1, + doc_id="1", chunk_index=0, chunk_start=0, chunk_end=10, @@ -143,7 +143,7 @@ class TestPayloadShape: with ctx: result = await get_chunk_bbox_and_page_from_qdrant( user_id="alice", - doc_id=42, + doc_id="42", chunk_index=3, chunk_start=0, chunk_end=100, @@ -157,7 +157,7 @@ class TestPayloadShape: with ctx: result = await get_chunk_bbox_and_page_from_qdrant( user_id="alice", - doc_id=42, + doc_id="42", chunk_index=3, chunk_start=0, chunk_end=100, @@ -171,7 +171,7 @@ class TestPayloadShape: with ctx: result = await get_chunk_bbox_and_page_from_qdrant( user_id="alice", - doc_id=42, + doc_id="42", chunk_index=3, chunk_start=0, chunk_end=100, @@ -188,7 +188,7 @@ class TestPayloadShape: with ctx: result = await get_chunk_bbox_and_page_from_qdrant( user_id="alice", - doc_id=42, + doc_id="42", chunk_index=3, chunk_start=0, chunk_end=100, @@ -205,7 +205,7 @@ class TestExceptionHandling: with ctx, caplog.at_level("WARNING"): result = await get_chunk_bbox_and_page_from_qdrant( user_id="alice", - doc_id=42, + doc_id="42", chunk_index=3, chunk_start=0, chunk_end=100, From 22a2a24941088881e0145f4ad38d08e00e67781b Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Sat, 9 May 2026 14:32:54 +0200 Subject: [PATCH 13/27] test(integration): add login-flow Astrolabe provisioning regression test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the missing end-to-end coverage for the seam that PR #773's `ALLOWED_MGMT_CLIENT` ↔ `astrolabeMcpClientOAuth00000000000` drift bug exposed: browser → Astrolabe (NC PHP app) → MCP server's management API on the `mcp-login-flow` profile. Every existing `tests/integration/test_astrolabe_*.py` is marked `multi_user_basic` and exercises the BasicAuth flow, not Login Flow v2 / OAuth. The new test mirrors the production-shaped flow exactly: 1. Log in as admin via Playwright. 2. Navigate to `/settings/user/astrolabe`. 3. Click the "Enable Semantic Search" OAuth link rendered by `oauth-required.php`. (Same selector Astrolabe's own e2e helper uses — `third_party/astrolabe/tests/e2e/helpers/authorize.ts`.) 4. Click "Allow" on the Nextcloud OIDC consent screen. 5. Wait for the redirect back to the Astrolabe settings page. 6. Assert the "Enable Semantic Search" link is no longer visible. Step 6 is the canary for the drift class: if Astrolabe's management API call to `/api/v1/users/{id}/session` is rejected (HTTP 401, the original bug), the session lookup falls back to "no token" and the same `oauth-required.php` template re-renders with the link still present — so the test fails loudly with a message naming the likely cause. Reuses `login_to_nextcloud` and `navigate_to_astrolabe_settings` helpers from `tests/integration/test_astrolabe_multi_user_background_sync.py` (already pattern-imported by the Plotly viz test). No fixture-level OIDC client creation: `app-hooks/before-starting/26-configure-astrolabe-oauth.sh` already provisions `astrolabeMcpClientOAuth00000000000` with the correct redirect URI and scopes when `MCP_SERVER_URL` is set in the shell that runs `docker compose --profile login-flow up`. The test skips cleanly when admin is already authorized (typical state on a re-run against a long-lived dev stack), so it's safe to run repeatedly without manual reset. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../test_astrolabe_login_flow_provisioning.py | 137 ++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 tests/integration/test_astrolabe_login_flow_provisioning.py diff --git a/tests/integration/test_astrolabe_login_flow_provisioning.py b/tests/integration/test_astrolabe_login_flow_provisioning.py new file mode 100644 index 00000000..bb1c81cb --- /dev/null +++ b/tests/integration/test_astrolabe_login_flow_provisioning.py @@ -0,0 +1,137 @@ +"""Integration test for Astrolabe's "Enable Semantic Search" OAuth flow on +the `mcp-login-flow` profile. + +Cross-system interface test. Brings together Astrolabe (Nextcloud PHP app +installed at container start by ``app-hooks/post-installation``) with the +``mcp-login-flow`` MCP server over OAuth + the management API. Mirrors +the production-shaped flow that PR #773's recent +`ALLOWED_MGMT_CLIENT` ↔ `astrolabeMcpClientOAuth00000000000` drift was +masking — every management API call from Astrolabe (e.g. +``/api/v1/users/admin/session``) was returning 401 because the +real-deployment client id was not in the test-fixture allowlist, so the +Astrolabe settings page never updated to reflect a successful +authorization. + +This test is **regression coverage** for that class of drift. If the +Astrolabe client id ever falls out of `mcp-login-flow`'s +``ALLOWED_MGMT_CLIENT`` again, the post-redirect assertions here will +fail because the page state stays on ``oauth-required.php``. + +Requires the login-flow stack to be running: + + MCP_SERVER_URL=http://mcp-login-flow:8004 \\ + docker compose --profile login-flow up -d app db mcp-login-flow + +The ``app-hooks/before-starting/26-configure-astrolabe-oauth.sh`` hook +creates the OAuth client with the production-shaped id +``astrolabeMcpClientOAuth00000000000`` automatically when +``MCP_SERVER_URL`` is set, so no fixture-level OIDC client creation is +needed here. +""" + +import logging +import os +import re + +import pytest +from playwright.async_api import Page + +# Reuse helpers from the multi-user-basic Astrolabe test for login + nav. +from tests.integration.test_astrolabe_multi_user_background_sync import ( + login_to_nextcloud, + navigate_to_astrolabe_settings, +) + +logger = logging.getLogger(__name__) + +pytestmark = [pytest.mark.integration, pytest.mark.login_flow] + +NEXTCLOUD_URL = "http://localhost:8080" +ASTROLABE_SETTINGS_URL = f"{NEXTCLOUD_URL}/settings/user/astrolabe" + + +async def _click_enable_semantic_search(page: Page) -> None: + """Click the "Enable Semantic Search" OAuth link on the + ``oauth-required.php`` template that login-flow mode renders to a + not-yet-authorized user. + + Astrolabe's own e2e helper (``third_party/astrolabe/tests/e2e/helpers/ + authorize.ts``) targets the same link by accessible name. + """ + enable_link = page.get_by_role("link", name="Enable Semantic Search") + await enable_link.wait_for(state="visible", timeout=10_000) + logger.info("Clicking 'Enable Semantic Search' OAuth link") + await enable_link.click() + + +async def _grant_oidc_consent(page: Page) -> None: + """Click "Allow" on the Nextcloud OIDC consent screen, if shown. + + Nextcloud may auto-redirect for already-trusted clients, in which + case the consent button never appears — that's not an error. + """ + allow_button = page.get_by_role("button", name=re.compile(r"^allow$", re.I)) + try: + await allow_button.wait_for(state="visible", timeout=10_000) + logger.info("Clicking 'Allow' on OIDC consent") + await allow_button.click(force=True) + except Exception: + logger.info( + "OIDC consent screen not visible — assuming auto-grant for " + "already-trusted client" + ) + + +@pytest.mark.timeout(180) +async def test_enable_semantic_search_completes_oauth_for_login_flow(browser): + """Click the "Enable Semantic Search" link, grant consent, and assert + the post-redirect page reflects a completed authorization. + + The success criterion is intentionally negative: after the OAuth + flow, the original "Enable Semantic Search" link must be gone. If + Astrolabe's management API call is rejected by the MCP server (HTTP + 401, the original bug), the page falls back to the same + ``oauth-required.php`` template and the link reappears — making this + test the canary for the drift class. + """ + admin_password = os.environ.get("NEXTCLOUD_PASSWORD", "admin") + page = await browser.new_page() + try: + await login_to_nextcloud(page, "admin", admin_password) + await navigate_to_astrolabe_settings(page) + + # Sanity-check we're on the not-yet-authorized template. + enable_link = page.get_by_role("link", name="Enable Semantic Search") + if await enable_link.count() == 0: + pytest.skip( + "Astrolabe is already authorized for admin (oauth-required.php " + "not rendered). Reset by clearing the user's OAuth tokens " + "before re-running this test." + ) + + await _click_enable_semantic_search(page) + await _grant_oidc_consent(page) + + # OAuth callback returns to /apps/astrolabe/oauth/callback then the + # controller redirects to /settings/user/astrolabe. + await page.wait_for_url(re.compile(r"/settings/user/astrolabe"), timeout=30_000) + await page.wait_for_load_state("networkidle", timeout=15_000) + + # Regression assertion for the ALLOWED_MGMT_CLIENT drift bug: + # the page must have moved past oauth-required.php. If + # Astrolabe's management API call to /api/v1/users/{id}/session + # is rejected (401), the session lookup falls back to "no token", + # and the same oauth-required.php template re-renders with the + # link still present. + post_auth_count = await page.get_by_role( + "link", name="Enable Semantic Search" + ).count() + assert post_auth_count == 0, ( + "'Enable Semantic Search' link still visible after completing " + "OAuth flow — Astrolabe could not read the user's session from " + "the MCP server. Most likely cause: " + "`astrolabeMcpClientOAuth00000000000` missing from " + "`ALLOWED_MGMT_CLIENT` on `mcp-login-flow`." + ) + finally: + await page.close() From fc9786c3a9e372db972afb18fe1e6e7f4c5fe213 Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Sat, 9 May 2026 14:39:55 +0200 Subject: [PATCH 14/27] test: Raise on missing NEXTCLOUD_PASSWORD --- tests/integration/test_astrolabe_login_flow_provisioning.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_astrolabe_login_flow_provisioning.py b/tests/integration/test_astrolabe_login_flow_provisioning.py index bb1c81cb..15d2abbb 100644 --- a/tests/integration/test_astrolabe_login_flow_provisioning.py +++ b/tests/integration/test_astrolabe_login_flow_provisioning.py @@ -94,7 +94,9 @@ async def test_enable_semantic_search_completes_oauth_for_login_flow(browser): ``oauth-required.php`` template and the link reappears — making this test the canary for the drift class. """ - admin_password = os.environ.get("NEXTCLOUD_PASSWORD", "admin") + admin_password = os.getenv("NEXTCLOUD_PASSWORD") + if admin_password is None: + raise RuntimeError("NEXTCLOUD_PASSWORD must be set") page = await browser.new_page() try: await login_to_nextcloud(page, "admin", admin_password) From 64f08429778ddc426d715cf75e91f07407ec1332 Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Sat, 9 May 2026 14:53:12 +0200 Subject: [PATCH 15/27] fix(vector): guard _group_int_doc_ids against non-int doc_id values Skip and warn instead of stringifying floats / unexpected types in the backfill helper. A stray doc_id=3.0 would otherwise be rewritten to "3.0", which producers (str(int)) and the keyword index would never match, and which int() on the verification side would reject. Also add a doc_id=0 case to the backfill test to guard against a future falsy-skip regression. Co-Authored-By: Claude Opus 4.7 (1M context) --- nextcloud_mcp_server/vector/qdrant_client.py | 12 +++++++++ tests/unit/vector/test_qdrant_client.py | 27 +++++++++++++++----- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/nextcloud_mcp_server/vector/qdrant_client.py b/nextcloud_mcp_server/vector/qdrant_client.py index 6bcb9a85..bd3a3e54 100644 --- a/nextcloud_mcp_server/vector/qdrant_client.py +++ b/nextcloud_mcp_server/vector/qdrant_client.py @@ -161,6 +161,18 @@ def _group_int_doc_ids(points: list[Any]) -> tuple[dict[str, list[Any]], int]: value = payload.get("doc_id") if value is None or isinstance(value, str): continue + if not isinstance(value, int): + # Producers only ever write int or str; anything else is a + # producer bug. Stringifying e.g. a float would write "3.0", + # which producers (str(int)) and the keyword index would + # never match, and which int() on the verification side + # would later reject. Skip and log loudly instead. + logger.warning( + "Unexpected doc_id type %s on point %s; skipping rewrite", + type(value).__name__, + point.id, + ) + continue by_value.setdefault(str(value), []).append(point.id) return by_value, scanned diff --git a/tests/unit/vector/test_qdrant_client.py b/tests/unit/vector/test_qdrant_client.py index 32c8bd3a..146b1f89 100644 --- a/tests/unit/vector/test_qdrant_client.py +++ b/tests/unit/vector/test_qdrant_client.py @@ -335,27 +335,40 @@ async def test_backfill_writes_sentinel_after_successful_scroll(mocker): @pytest.mark.unit async def test_backfill_rewrites_int_doc_ids_to_str(mocker): - """Mixed int/str payload across two scroll pages: only ints get rewritten.""" + """Mixed int/str payload across two scroll pages: only ints get rewritten. + + Includes ``doc_id=0`` to guard against a future "early-exit on + falsy" refactor — the helper must rewrite zero alongside other + ints, not skip it. + """ client = mocker.AsyncMock() client.retrieve.return_value = [] - # Two scroll calls: batch 1 is mixed and reports a next_offset; batch 2 - # is mixed with next_offset=None to terminate. + # Two scroll calls: batch 1 mixes int (incl. 0) + str and reports a + # next_offset; batch 2 mixes int + str with next_offset=None to terminate. client.scroll.side_effect = [ - ([_record(1, 100), _record(2, "abc")], "next-offset-123"), + ( + [_record(1, 100), _record(2, "abc"), _record(5, 0)], + "next-offset-123", + ), ([_record(3, 200), _record(4, "def")], None), ] await _backfill_doc_id_to_string(client, "test-collection", _backfill_dimension()) - # One set_payload per *unique* int value — point 1 (100) and point 3 - # (200) are in different batches with different values, so two calls. - assert client.set_payload.await_count == 2 + # One set_payload per *unique* int value across all batches: 100, 0, 200. + assert client.set_payload.await_count == 3 client.set_payload.assert_any_await( collection_name="test-collection", payload={"doc_id": "100"}, points=[1], wait=True, ) + client.set_payload.assert_any_await( + collection_name="test-collection", + payload={"doc_id": "0"}, + points=[5], + wait=True, + ) client.set_payload.assert_any_await( collection_name="test-collection", payload={"doc_id": "200"}, From 0c14501a2be4a37e37f0e17b2e7a58b5625915cb Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Sat, 9 May 2026 15:19:46 +0200 Subject: [PATCH 16/27] test: align CI assertions with documented contracts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two unrelated CI failures on this branch, one fix each: - tests/integration/test_deck_vector_search.py: pass str(card.id) to get_chunk_with_context. The function's contract is doc_id: str (keyword-indexed in Qdrant), and real callers (viz_routes.py URL path, server/semantic.py via str(result.id)) all stringify. The test was the only int caller, hitting the .isdigit() guard added earlier on this branch. - tests/server/login_flow/test_login_flow_integration.py: test_check_status_provisioned now accepts scopes=None as valid. Per ProvisionStatusResponse in models/auth.py, None is the documented sentinel for "all scopes granted" — and the web provisioning path (provision_routes.py, used by Astrolabe's "Enable Semantic Search" flow exercised by the new regression test added on this branch) stores exactly that. The previous is-not-None assertion hid behind test order until that flow ran. - Replace anyio.sleep(0) with anyio.lowlevel.checkpoint() Co-Authored-By: Claude Opus 4.7 (1M context) --- tests/integration/test_deck_vector_search.py | 5 ++++- tests/server/login_flow/test_login_flow_integration.py | 8 +++++++- tests/unit/vector/test_qdrant_client.py | 10 +++++----- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_deck_vector_search.py b/tests/integration/test_deck_vector_search.py index 3f6433bc..7524816f 100644 --- a/tests/integration/test_deck_vector_search.py +++ b/tests/integration/test_deck_vector_search.py @@ -207,10 +207,13 @@ async def test_deck_card_chunk_context(nc_client): # Fetch chunk context (simulates viz UI request) # The chunk spans the title, so start=0 and end=len(card_title) + # doc_id is str — keyword-indexed in Qdrant payload; the real + # callers (viz_routes.py from URL path; server/semantic.py from + # str(result.id)) all stringify before reaching this entry point. context = await get_chunk_with_context( nc_client=nc_client, user_id=nc_client.username, - doc_id=card.id, + doc_id=str(card.id), doc_type="deck_card", chunk_start=0, chunk_end=len(card_title), diff --git a/tests/server/login_flow/test_login_flow_integration.py b/tests/server/login_flow/test_login_flow_integration.py index f2db50ba..262be92d 100644 --- a/tests/server/login_flow/test_login_flow_integration.py +++ b/tests/server/login_flow/test_login_flow_integration.py @@ -43,7 +43,13 @@ class TestLoginFlowAuthTools: data = json.loads(result.content[0].text) assert data["status"] == "provisioned" assert data["username"] is not None - assert data["scopes"] is not None + # ``scopes`` may legitimately be ``None`` — per ProvisionStatusResponse + # in models/auth.py, ``None`` is the documented sentinel for "all + # scopes granted" and is what the web provisioning path + # (``provision_routes.py``, used by Astrolabe's "Enable Semantic + # Search" flow) stores. So accept either a non-empty list or None; + # the field's *presence* in the payload is what we care about here. + assert data["scopes"] is None or len(data["scopes"]) > 0 logger.info(f"Provisioned as: {data['username']}, scopes: {data['scopes']}") async def test_provision_access_already_provisioned( diff --git a/tests/unit/vector/test_qdrant_client.py b/tests/unit/vector/test_qdrant_client.py index 146b1f89..9479acdc 100644 --- a/tests/unit/vector/test_qdrant_client.py +++ b/tests/unit/vector/test_qdrant_client.py @@ -237,7 +237,7 @@ async def test_ensure_payload_indexes_logs_and_returns_when_get_collection_raise async def _get_collection_raises(*args, **kwargs): # See _scroll_raises in the backfill section for why this is async. - await anyio.sleep(0) + await anyio.lowlevel.checkpoint() raise RuntimeError("connection refused") client.get_collection.side_effect = _get_collection_raises @@ -494,11 +494,11 @@ async def test_backfill_logs_and_returns_when_scroll_raises(mocker, caplog): # An async-callable side_effect lets AsyncMock await the coroutine # before the exception propagates; assigning a bare exception class # leaks an un-awaited coroutine and trips RuntimeWarning at gc time. - # The `await anyio.sleep(0)` is a no-op event-loop yield that + # The `await anyio.lowlevel.checkpoint()` is a no-op event-loop yield that # satisfies static analysis ("async function uses no async features") # without changing observable behavior. async def _scroll_raises(*args, **kwargs): - await anyio.sleep(0) + await anyio.lowlevel.checkpoint() raise RuntimeError("boom") client.scroll.side_effect = _scroll_raises @@ -538,7 +538,7 @@ async def test_backfill_logs_warning_when_sentinel_upsert_fails(mocker, caplog): async def _upsert_raises(*args, **kwargs): # See _scroll_raises above for why this is async + sleep(0). - await anyio.sleep(0) + await anyio.lowlevel.checkpoint() raise RuntimeError("sentinel write blip") client.upsert.side_effect = _upsert_raises @@ -615,7 +615,7 @@ async def test_ensure_payload_indexes_summarises_failed_fields(mocker, caplog): async def _create_index(*args, **kwargs): # See _scroll_raises above for why this is async + sleep(0). - await anyio.sleep(0) + await anyio.lowlevel.checkpoint() call_count["n"] += 1 if call_count["n"] != 2: raise _make_unexpected(500, b'{"status":{"error":"boom"}}') From fec1596784a4ace18b0fa0649bea1fcf4ceaa6ee Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Sat, 9 May 2026 15:49:43 +0200 Subject: [PATCH 17/27] =?UTF-8?q?fix(vector):=20address=20PR=20review=20ro?= =?UTF-8?q?und=209=20=E2=80=94=20drop=20redundant=20guard,=20add=20init=20?= =?UTF-8?q?lock,=20test=20float=20doc=5Fid=20path?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the four 🟡 important findings from claude-bot review on PR #773: str (non-Optional) and the guard would silently skip the Qdrant lookup for an empty string. Removing the guard matches the type signature. (`all([…, doc_id, …])` rejects None and empty string, plus `assert doc_id is not None`). No code change needed. `get_qdrant_client()` with a module-level `anyio.Lock`. Double-checked locking keeps the steady-state hot path lock-free. Without this, parallel cold-start callers could all enter the init block and run `_backfill_doc_id_to_string` + `_ensure_payload_indexes` redundantly (idempotent, but noisy). Pattern matches `auth/storage.py:2071`. behavior with three tests covering the float-warning path (the gap called out in the review), the str/None silent-skip paths, and the int-grouping happy path. Verification: - ruff check / format: clean - ty check -- nextcloud_mcp_server: clean - uv run pytest tests/unit/: 969 passed Co-Authored-By: Claude Opus 4.7 (1M context) --- .claude/skills/pre-push-review/SKILL.md | 1 - nextcloud_mcp_server/search/context.py | 29 +-- nextcloud_mcp_server/vector/qdrant_client.py | 256 ++++++++++--------- tests/unit/vector/test_qdrant_client.py | 78 ++++++ 4 files changed, 233 insertions(+), 131 deletions(-) diff --git a/.claude/skills/pre-push-review/SKILL.md b/.claude/skills/pre-push-review/SKILL.md index efc906b0..26883d49 100644 --- a/.claude/skills/pre-push-review/SKILL.md +++ b/.claude/skills/pre-push-review/SKILL.md @@ -7,7 +7,6 @@ description: | in this repo's automated PR reviews. Use when the user is about to push, says "ready to push", "review my work", "check before PR", or invokes /pre-push-review. Report-only — does not modify code. -model: sonnet allowed-tools: - Bash - Read diff --git a/nextcloud_mcp_server/search/context.py b/nextcloud_mcp_server/search/context.py index f818e278..c9baf084 100644 --- a/nextcloud_mcp_server/search/context.py +++ b/nextcloud_mcp_server/search/context.py @@ -369,21 +369,20 @@ async def get_chunk_with_context( # Prefer chunk_index lookup (always-indexed field) when caller supplied it; # fall back to (chunk_start, chunk_end) lookup otherwise. chunk_text: str | None = None - if doc_id: - if chunk_index is not None: - chunk_text = await _get_chunk_by_index_from_qdrant( - user_id, doc_id, doc_type, chunk_index - ) - # Skip the offset fallback for files when the indexed chunk_index - # lookup already ran: chunk_start/end_offset aren't indexed in Qdrant - # Cloud strict mode, so the call returns 400 and surfaces a misleading - # logger.error. The file fast-fail below correctly handles the miss - # without it. - skip_offset_lookup = chunk_index is not None and doc_type == "file" - if chunk_text is None and not skip_offset_lookup: - chunk_text = await _get_chunk_from_qdrant( - user_id, doc_id, doc_type, chunk_start, chunk_end - ) + if chunk_index is not None: + chunk_text = await _get_chunk_by_index_from_qdrant( + user_id, doc_id, doc_type, chunk_index + ) + # Skip the offset fallback for files when the indexed chunk_index + # lookup already ran: chunk_start/end_offset aren't indexed in Qdrant + # Cloud strict mode, so the call returns 400 and surfaces a misleading + # logger.error. The file fast-fail below correctly handles the miss + # without it. + skip_offset_lookup = chunk_index is not None and doc_type == "file" + if chunk_text is None and not skip_offset_lookup: + chunk_text = await _get_chunk_from_qdrant( + user_id, doc_id, doc_type, chunk_start, chunk_end + ) if chunk_text: logger.info( diff --git a/nextcloud_mcp_server/vector/qdrant_client.py b/nextcloud_mcp_server/vector/qdrant_client.py index bd3a3e54..4cdb8434 100644 --- a/nextcloud_mcp_server/vector/qdrant_client.py +++ b/nextcloud_mcp_server/vector/qdrant_client.py @@ -3,6 +3,7 @@ import logging from typing import Any +import anyio from qdrant_client import AsyncQdrantClient, models from qdrant_client.http.exceptions import UnexpectedResponse from qdrant_client.models import ( @@ -43,8 +44,13 @@ _PAYLOAD_INDEX_FIELDS: dict[str, PayloadSchemaType] = { _DOC_ID_BACKFILL_SENTINEL_ID: str = "00000000-0000-0000-0000-d0c1d0d1d0c1" _DOC_ID_BACKFILL_SENTINEL_PAYLOAD: dict[str, str] = {"_migration_marker": "doc_id_v1"} -# Singleton instance +# Singleton instance + init lock. The lock serialises concurrent first +# callers so the idempotent-but-expensive startup migration +# (``_backfill_doc_id_to_string`` + ``_ensure_payload_indexes``) only runs +# once per process. Steady-state callers hit the fast path above the lock +# and never acquire it. _qdrant_client: AsyncQdrantClient | None = None +_qdrant_init_lock: anyio.Lock = anyio.Lock() async def _ensure_payload_indexes( @@ -392,131 +398,151 @@ async def get_qdrant_client() -> AsyncQdrantClient: """ global _qdrant_client - if _qdrant_client is None: - settings = get_settings() + # Fast path: already initialized — skip lock acquisition for the + # steady-state hot path (every MCP tool call after first start). + if _qdrant_client is not None: + return _qdrant_client - # Detect mode and initialize client accordingly - if settings.qdrant_url: - # Network mode - logger.info(f"Using Qdrant network mode: {settings.qdrant_url}") - _qdrant_client = AsyncQdrantClient( - url=settings.qdrant_url, - api_key=settings.qdrant_api_key, - timeout=30, - ) - elif settings.qdrant_location: - # Local mode (either :memory: or persistent path) - if settings.qdrant_location == ":memory:": - logger.info("Using Qdrant in-memory mode: :memory:") + # Slow path: serialise concurrent first-callers so the idempotent-but- + # expensive startup migration (``_backfill_doc_id_to_string`` + + # ``_ensure_payload_indexes``) runs exactly once. Without this lock, + # parallel cold-start callers would all enter the init block, run the + # migration N times, and emit duplicate "skip-because-exists" warnings + # from the index helper — annoying log noise but not data corruption. + async with _qdrant_init_lock: + # Double-checked: another waiter may have initialized while we + # blocked on the lock. + if _qdrant_client is None: + settings = get_settings() + + # Detect mode and initialize client accordingly + if settings.qdrant_url: + # Network mode + logger.info(f"Using Qdrant network mode: {settings.qdrant_url}") + _qdrant_client = AsyncQdrantClient( + url=settings.qdrant_url, + api_key=settings.qdrant_api_key, + timeout=30, + ) + elif settings.qdrant_location: + # Local mode (either :memory: or persistent path) + if settings.qdrant_location == ":memory:": + logger.info("Using Qdrant in-memory mode: :memory:") + _qdrant_client = AsyncQdrantClient(":memory:") + else: + # Persistent local mode - use path parameter + logger.info( + f"Using Qdrant persistent mode: {settings.qdrant_location}" + ) + _qdrant_client = AsyncQdrantClient(path=settings.qdrant_location) + else: + # Should not happen due to __post_init__ validation, but handle gracefully + logger.warning("No Qdrant mode configured, defaulting to :memory:") _qdrant_client = AsyncQdrantClient(":memory:") - else: - # Persistent local mode - use path parameter - logger.info(f"Using Qdrant persistent mode: {settings.qdrant_location}") - _qdrant_client = AsyncQdrantClient(path=settings.qdrant_location) - else: - # Should not happen due to __post_init__ validation, but handle gracefully - logger.warning("No Qdrant mode configured, defaulting to :memory:") - _qdrant_client = AsyncQdrantClient(":memory:") - # Get collection name (auto-generated from deployment ID + model) - collection_name = settings.get_collection_name() + # Get collection name (auto-generated from deployment ID + model) + collection_name = settings.get_collection_name() - embedding_service = get_embedding_service() + embedding_service = get_embedding_service() - # Detect dimension dynamically (for OllamaEmbeddingProvider) - if hasattr(embedding_service.provider, "_detect_dimension"): - await embedding_service.provider._detect_dimension() # type: ignore[call-non-callable] + # Detect dimension dynamically (for OllamaEmbeddingProvider) + if hasattr(embedding_service.provider, "_detect_dimension"): + await embedding_service.provider._detect_dimension() # type: ignore[call-non-callable] - expected_dimension = embedding_service.get_dimension() + expected_dimension = embedding_service.get_dimension() - # Explicitly check if collection exists - logger.debug(f"Checking if collection '{collection_name}' exists...") - collections = await _qdrant_client.get_collections() - collection_names = [c.name for c in collections.collections] + # Explicitly check if collection exists + logger.debug(f"Checking if collection '{collection_name}' exists...") + collections = await _qdrant_client.get_collections() + collection_names = [c.name for c in collections.collections] - if collection_name in collection_names: - # Collection exists - validate dimensions - logger.debug( - f"Collection '{collection_name}' found, validating dimensions..." - ) - collection_info = await _qdrant_client.get_collection(collection_name) - # Handle both named vectors (dict) and legacy single vector - vectors = collection_info.config.params.vectors - if isinstance(vectors, dict): - actual_dimension = vectors["dense"].size - else: - # Type narrowing: vectors must be VectorParams if not dict - assert isinstance(vectors, VectorParams) - actual_dimension = vectors.size + if collection_name in collection_names: + # Collection exists - validate dimensions + logger.debug( + f"Collection '{collection_name}' found, validating dimensions..." + ) + collection_info = await _qdrant_client.get_collection(collection_name) + # Handle both named vectors (dict) and legacy single vector + vectors = collection_info.config.params.vectors + if isinstance(vectors, dict): + actual_dimension = vectors["dense"].size + else: + # Type narrowing: vectors must be VectorParams if not dict + assert isinstance(vectors, VectorParams) + actual_dimension = vectors.size - # Validate dimension matches - if actual_dimension != expected_dimension: - embedding_model = settings.get_embedding_model_name() - raise ValueError( - f"Dimension mismatch for collection '{collection_name}':\n" - f" Expected: {expected_dimension} (from embedding model '{embedding_model}')\n" - f" Found: {actual_dimension}\n" - f"This usually means you changed the embedding model.\n" - f"Solutions:\n" - f" 1. Delete the old collection: Collection will be recreated with new dimensions\n" - f" 2. Set QDRANT_COLLECTION to use a different collection name\n" - f" 3. Revert to the original embedding model" + # Validate dimension matches + if actual_dimension != expected_dimension: + embedding_model = settings.get_embedding_model_name() + raise ValueError( + f"Dimension mismatch for collection '{collection_name}':\n" + f" Expected: {expected_dimension} (from embedding model '{embedding_model}')\n" + f" Found: {actual_dimension}\n" + f"This usually means you changed the embedding model.\n" + f"Solutions:\n" + f" 1. Delete the old collection: Collection will be recreated with new dimensions\n" + f" 2. Set QDRANT_COLLECTION to use a different collection name\n" + f" 3. Revert to the original embedding model" + ) + + logger.info( + f"Using existing Qdrant collection: {collection_name} " + f"(dimension={actual_dimension}, model={settings.get_embedding_model_name()})" ) - logger.info( - f"Using existing Qdrant collection: {collection_name} " - f"(dimension={actual_dimension}, model={settings.get_embedding_model_name()})" - ) + # Existing collections may pre-date the doc_id normalization / + # payload-index work. Backfill before creating the index so the + # index covers every point. Pass the already-fetched + # collection_info.payload_schema through to avoid a redundant + # get_collection round-trip on every restart. + await _backfill_doc_id_to_string( + _qdrant_client, collection_name, expected_dimension + ) + await _ensure_payload_indexes( + _qdrant_client, + collection_name, + existing_schema=collection_info.payload_schema or {}, + ) - # Existing collections may pre-date the doc_id normalization / - # payload-index work. Backfill before creating the index so the - # index covers every point. Pass the already-fetched - # collection_info.payload_schema through to avoid a redundant - # get_collection round-trip on every restart. - await _backfill_doc_id_to_string( - _qdrant_client, collection_name, expected_dimension - ) - await _ensure_payload_indexes( - _qdrant_client, - collection_name, - existing_schema=collection_info.payload_schema or {}, - ) - - else: - # Collection doesn't exist - create it - embedding_model = settings.get_embedding_model_name() - logger.info( - f"Collection '{collection_name}' not found, creating with " - f"dimension={expected_dimension}, model={embedding_model}..." - ) - await _qdrant_client.create_collection( - collection_name=collection_name, - vectors_config={ - "dense": VectorParams( - size=expected_dimension, - distance=Distance.COSINE, - ), - }, - sparse_vectors_config={ - "sparse": models.SparseVectorParams( - index=models.SparseIndexParams( - on_disk=False, - ) - ), - }, - ) - logger.info( - f"Created Qdrant collection: {collection_name}\n" - f" Dense vector dimension: {expected_dimension}\n" - f" Dense embedding model: {embedding_model}\n" - f" Sparse vectors: BM25 (for hybrid search)\n" - f" Distance: COSINE\n" - f"Background sync will index all documents with dense + sparse vectors." - ) - # Freshly created collection has no payload schema yet; pass {} - # explicitly to skip the otherwise-redundant get_collection call. - await _ensure_payload_indexes( - _qdrant_client, collection_name, existing_schema={} - ) + else: + # Collection doesn't exist - create it + embedding_model = settings.get_embedding_model_name() + logger.info( + f"Collection '{collection_name}' not found, creating with " + f"dimension={expected_dimension}, model={embedding_model}..." + ) + await _qdrant_client.create_collection( + collection_name=collection_name, + vectors_config={ + "dense": VectorParams( + size=expected_dimension, + distance=Distance.COSINE, + ), + }, + sparse_vectors_config={ + "sparse": models.SparseVectorParams( + index=models.SparseIndexParams( + on_disk=False, + ) + ), + }, + ) + logger.info( + f"Created Qdrant collection: {collection_name}\n" + f" Dense vector dimension: {expected_dimension}\n" + f" Dense embedding model: {embedding_model}\n" + f" Sparse vectors: BM25 (for hybrid search)\n" + f" Distance: COSINE\n" + f"Background sync will index all documents with dense + sparse vectors." + ) + # Freshly created collection has no payload schema yet; pass {} + # explicitly to skip the otherwise-redundant get_collection call. + await _ensure_payload_indexes( + _qdrant_client, collection_name, existing_schema={} + ) + # Lock released. ``_qdrant_client`` is guaranteed non-None here: + # either the fast path returned earlier, the lock-protected branch + # set it, or a sibling waiter set it before we got the lock. + assert _qdrant_client is not None return _qdrant_client diff --git a/tests/unit/vector/test_qdrant_client.py b/tests/unit/vector/test_qdrant_client.py index 9479acdc..4743648a 100644 --- a/tests/unit/vector/test_qdrant_client.py +++ b/tests/unit/vector/test_qdrant_client.py @@ -28,6 +28,7 @@ from nextcloud_mcp_server.vector.qdrant_client import ( _PAYLOAD_INDEX_FIELDS, _backfill_doc_id_to_string, _ensure_payload_indexes, + _group_int_doc_ids, ) @@ -597,6 +598,83 @@ async def test_backfill_emits_progress_log_every_20_batches(mocker, caplog): assert "test-collection" in progress_messages[0] +# --------------------------------------------------------------------------- +# _group_int_doc_ids +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_group_int_doc_ids_skips_float_and_warns(caplog): + """A float doc_id is not stringified; it logs WARNING and is skipped. + + Producers always write int or str. A float would round-trip to e.g. + ``"3.0"``, which the keyword index and verification path + (``int(doc_id)``) would never match. Skipping with a loud warning is + the only safe choice. + """ + float_point = SimpleNamespace(id=99, payload={"doc_id": 3.0}) + int_point = SimpleNamespace(id=42, payload={"doc_id": 7}) + + with caplog.at_level("WARNING", logger="nextcloud_mcp_server.vector.qdrant_client"): + by_value, scanned = _group_int_doc_ids([float_point, int_point]) + + # Only the int point made it into by_value; float was dropped. + assert by_value == {"7": [42]} + # Both points still count toward the scanned total — the warning + # should not hide them from progress logs. + assert scanned == 2 + + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert len(warnings) == 1 + msg = warnings[0].getMessage() + assert "float" in msg + assert "99" in msg + + +@pytest.mark.unit +def test_group_int_doc_ids_handles_str_and_missing_silently(caplog): + """str / missing doc_id payloads are skipped without warning. + + These are the steady-state paths — already-migrated str values and + sentinel-style points without a doc_id key. Neither should noise up + the log on every restart. + """ + str_point = SimpleNamespace(id=1, payload={"doc_id": "abc"}) + none_payload_point = SimpleNamespace(id=2, payload=None) + missing_key_point = SimpleNamespace(id=3, payload={"other": "value"}) + explicit_none_point = SimpleNamespace(id=4, payload={"doc_id": None}) + + with caplog.at_level("WARNING", logger="nextcloud_mcp_server.vector.qdrant_client"): + by_value, scanned = _group_int_doc_ids( + [str_point, none_payload_point, missing_key_point, explicit_none_point] + ) + + assert by_value == {} + assert scanned == 4 + # No warnings — these paths are expected and silent. + assert not [r for r in caplog.records if r.levelname == "WARNING"] + + +@pytest.mark.unit +def test_group_int_doc_ids_groups_ints_by_str_value(): + """Multiple int-doc_id points sharing a value collapse into one entry. + + Pins the chunk-batching contract: all chunks of one document share its + doc_id, so the helper hands ``_apply_backfill_writes`` a single key + with all chunk point-ids attached. + """ + by_value, scanned = _group_int_doc_ids( + [ + SimpleNamespace(id=10, payload={"doc_id": 42}), + SimpleNamespace(id=11, payload={"doc_id": 42}), + SimpleNamespace(id=12, payload={"doc_id": 7}), + ] + ) + + assert by_value == {"42": [10, 11], "7": [12]} + assert scanned == 3 + + @pytest.mark.unit async def test_ensure_payload_indexes_summarises_failed_fields(mocker, caplog): """A non-400 failure surfaces both as ERROR and a WARNING summary. From d60348e77b8a4468808795ded1a41ce31b106e1f Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Sat, 9 May 2026 16:54:06 +0200 Subject: [PATCH 18/27] fix(api): validate doc_id at chunk-context handler boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an .isdigit() guard at the top of both chunk-context handlers so a non-numeric doc_id fails fast with a clear 400 ("doc_id must be numeric, got 'abc'") rather than silently bottoming out as a 404 from deep inside get_chunk_with_context. The earlier int(doc_id) coercion was removed when doc_id became a pure pass-through to Qdrant's keyword payload index, which also dropped this boundary validation. Also align test_backfill_emits_progress_log_every_20_batches' scroll stub with real Qdrant: next_offset is now "next-1" (str) instead of 1 (int), matching the sibling test_backfill_rewrites_int_doc_ids_to_str. Pure stub-fidelity fix; production code already treats next_offset as opaque. Addresses both 🟡 Important items from PR #773 review round 10. Co-Authored-By: Claude Opus 4.7 (1M context) --- nextcloud_mcp_server/api/visualization.py | 14 ++++++++++++++ nextcloud_mcp_server/auth/viz_routes.py | 14 ++++++++++++++ tests/unit/vector/test_qdrant_client.py | 7 +++++-- 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/nextcloud_mcp_server/api/visualization.py b/nextcloud_mcp_server/api/visualization.py index cec34e92..62d8e3f3 100644 --- a/nextcloud_mcp_server/api/visualization.py +++ b/nextcloud_mcp_server/api/visualization.py @@ -498,6 +498,20 @@ async def get_chunk_context(request: Request) -> JSONResponse: assert doc_id is not None assert doc_type is not None + # Validate doc_id at the handler boundary: a malformed doc_id would + # otherwise pass through to get_chunk_with_context and bottom out as a + # 404 from deep inside, not a clear 400. Nextcloud IDs are unsigned + # ints from MySQL auto_increment; doc_id stays a str downstream + # (Qdrant payload index is keyword-typed). + if not doc_id.isdigit(): + return JSONResponse( + { + "success": False, + "error": f"doc_id must be numeric, got {doc_id!r}", + }, + status_code=400, + ) + # Parse and validate integer parameters with bounds checking try: context_chars = _parse_int_param( diff --git a/nextcloud_mcp_server/auth/viz_routes.py b/nextcloud_mcp_server/auth/viz_routes.py index c189ac0a..5c2eee3d 100644 --- a/nextcloud_mcp_server/auth/viz_routes.py +++ b/nextcloud_mcp_server/auth/viz_routes.py @@ -560,6 +560,20 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse: assert start_str is not None assert end_str is not None + # Validate doc_id at the handler boundary: a malformed doc_id would + # otherwise pass through to get_chunk_with_context and bottom out as a + # 404 from deep inside, not a clear 400. Nextcloud IDs are unsigned + # ints from MySQL auto_increment; doc_id stays a str downstream + # (Qdrant payload index is keyword-typed). + if not doc_id.isdigit(): + return JSONResponse( + { + "success": False, + "error": f"doc_id must be numeric, got {doc_id!r}", + }, + status_code=400, + ) + context_chars = _parse_int_param( request.query_params.get("context"), 500, diff --git a/tests/unit/vector/test_qdrant_client.py b/tests/unit/vector/test_qdrant_client.py index 4743648a..32a498a2 100644 --- a/tests/unit/vector/test_qdrant_client.py +++ b/tests/unit/vector/test_qdrant_client.py @@ -577,8 +577,11 @@ async def test_backfill_emits_progress_log_every_20_batches(mocker, caplog): # set_payload calls happen — the test focuses on the progress log # cadence, not the rewrite path. str_point = SimpleNamespace(id=1, payload={"doc_id": "abc"}) - batches: list[tuple[list[SimpleNamespace], int | None]] = [ - ([str_point], 1) for _ in range(21) + # Real Qdrant returns next_offset as a UUID string (or None to terminate). + # Match that shape so the stub remains accurate if scroll's return type is + # ever tightened — and aligns with test_backfill_rewrites_int_doc_ids_to_str. + batches: list[tuple[list[SimpleNamespace], str | None]] = [ + ([str_point], "next-1") for _ in range(21) ] + [([], None)] client.scroll.side_effect = batches From 47c531969f9169b1b8b5b0041520e32ce2ccc086 Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Sat, 9 May 2026 17:07:15 +0200 Subject: [PATCH 19/27] =?UTF-8?q?fix(vector):=20address=20PR=20review=20ro?= =?UTF-8?q?und=2010=20=E2=80=94=20index=20chunk=5Findex,=20harden=20index?= =?UTF-8?q?=20loop,=20lazy-init=20lock?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three coordinated fixes flagged as Important in the round-10 review of PR #773: 1. Index chunk_index. The chunk-context fast path in _get_chunk_by_index_from_qdrant and get_chunk_bbox_and_page_from_qdrant filters on chunk_index, but the field was absent from _PAYLOAD_INDEX_FIELDS. On Qdrant Cloud strict mode every chunk-context lookup via chunk_index would 400 and silently fall back to the document re-fetch path — the exact failure mode the chunk_index shortcut exists to avoid. Added as INTEGER schema. 2. Catch raw network errors in _ensure_payload_indexes. The create_payload_index loop only caught UnexpectedResponse, so an httpx.ConnectError or asyncio.TimeoutError mid-loop would propagate uncaught — leaving _qdrant_client assigned and silently skipping all remaining fields. Added a broad Exception catch with the same per-field containment as the 5xx path: log at ERROR with exc_info, append to failed_fields, continue. New test covers the path. 3. Lazy-initialise _qdrant_init_lock. Constructing anyio.Lock() at module import time works for the asyncio backend but anyio's docs advise instantiating synchronization primitives within an async context, and pyproject.toml's anyio_mode = "auto" means tests can run under trio. Moved the construction into get_qdrant_client; safe under cooperative multitasking because there is no await between the None-check and the assignment. Co-Authored-By: Claude Opus 4.7 (1M context) --- nextcloud_mcp_server/vector/qdrant_client.py | 80 +++++++++++++++----- tests/unit/vector/test_qdrant_client.py | 44 +++++++++++ 2 files changed, 104 insertions(+), 20 deletions(-) diff --git a/nextcloud_mcp_server/vector/qdrant_client.py b/nextcloud_mcp_server/vector/qdrant_client.py index 4cdb8434..5162cbb4 100644 --- a/nextcloud_mcp_server/vector/qdrant_client.py +++ b/nextcloud_mcp_server/vector/qdrant_client.py @@ -19,19 +19,25 @@ from nextcloud_mcp_server.embedding import get_embedding_service logger = logging.getLogger(__name__) -# Payload fields filtered by exact-match in scanner/processor/placeholder/eviction. -# Qdrant requires a payload index for any field used in a FieldCondition; without -# one, queries fail with HTTP 400 ("Index required but not found") on instances -# that enforce strict-mode index-required filtering (Qdrant Cloud, network mode -# with strict settings). The three string fields (doc_id, user_id, doc_type) -# carry str values after producer normalization, so KEYWORD is the correct -# schema; is_placeholder is the bool used by ``get_placeholder_filter`` and -# ``delete_placeholder_point`` (see vector/placeholder.py), so it gets BOOL. +# Payload fields filtered by exact-match in scanner/processor/placeholder/eviction +# and the chunk-context lookup path. Qdrant requires a payload index for any +# field used in a FieldCondition; without one, queries fail with HTTP 400 +# ("Index required but not found") on instances that enforce strict-mode +# index-required filtering (Qdrant Cloud, network mode with strict settings). +# The three string fields (doc_id, user_id, doc_type) carry str values after +# producer normalization, so KEYWORD is the correct schema. is_placeholder is +# the bool used by ``get_placeholder_filter`` and ``delete_placeholder_point`` +# (see vector/placeholder.py), so it gets BOOL. chunk_index is the int used by +# ``_get_chunk_by_index_from_qdrant`` and ``get_chunk_bbox_and_page_from_qdrant`` +# (see search/context.py) — the always-indexed fast path that the offset-based +# fallback exists to avoid; it has to actually be indexed for that promise to +# hold on Qdrant Cloud strict mode. _PAYLOAD_INDEX_FIELDS: dict[str, PayloadSchemaType] = { "doc_id": PayloadSchemaType.KEYWORD, "user_id": PayloadSchemaType.KEYWORD, "doc_type": PayloadSchemaType.KEYWORD, "is_placeholder": PayloadSchemaType.BOOL, + "chunk_index": PayloadSchemaType.INTEGER, } # Sentinel point that records "this collection has been backfilled to str @@ -48,9 +54,16 @@ _DOC_ID_BACKFILL_SENTINEL_PAYLOAD: dict[str, str] = {"_migration_marker": "doc_i # callers so the idempotent-but-expensive startup migration # (``_backfill_doc_id_to_string`` + ``_ensure_payload_indexes``) only runs # once per process. Steady-state callers hit the fast path above the lock -# and never acquire it. +# and never acquire it. The lock is lazy-initialised inside +# ``get_qdrant_client`` rather than constructed at module import time: +# anyio's docs are explicit that synchronization primitives should be +# instantiated within an async context, and ``anyio_mode = "auto"`` in +# pyproject.toml means tests can run under trio where eager construction +# would fail. Construction is safe under cooperative multitasking — there +# is no ``await`` between the None-check and the assignment, so two +# coroutines cannot both create a lock. _qdrant_client: AsyncQdrantClient | None = None -_qdrant_init_lock: anyio.Lock = anyio.Lock() +_qdrant_init_lock: anyio.Lock | None = None async def _ensure_payload_indexes( @@ -61,12 +74,16 @@ async def _ensure_payload_indexes( """Create payload indexes for fields used in exact-match filters. Each entry in ``_PAYLOAD_INDEX_FIELDS`` is created with its declared - schema type (KEYWORD for string fields, BOOL for ``is_placeholder``). - Skips fields that are already in ``existing_schema`` so routine - restarts make no Qdrant write round-trips and emit no INFO log lines. - Schema conflicts (a pre-existing index with a different type) still - surface as a 400 — log loudly so operators can intervene, but keep - going so the remaining fields still get indexed. + schema type (KEYWORD for string fields, BOOL for ``is_placeholder``, + INTEGER for ``chunk_index``). Skips fields that are already in + ``existing_schema`` so routine restarts make no Qdrant write round-trips + and emit no INFO log lines. Schema conflicts (a pre-existing index with + a different type) still surface as a 400 — log loudly so operators can + intervene, but keep going so the remaining fields still get indexed. + The same per-field error containment applies to raw network errors + (e.g. ``httpx.ConnectError`` from a transient Qdrant unavailability): + log at ERROR with ``exc_info`` and continue, so a single transient + failure on one field does not skip the rest. Args: client: Qdrant client instance. @@ -115,9 +132,9 @@ async def _ensure_payload_indexes( body = getattr(e, "content", b"") or b"" body_text = body.decode("utf-8", errors="replace") # 400 is the expected schema-conflict path (index already exists - # with a different type). 5xx / network-shaped errors should not - # be silently downgraded — keep the loop going so the remaining - # fields still get attempted, but log at error so operators see it. + # with a different type). 5xx is unexpected — keep the loop going + # so the remaining fields still get attempted, but log at error + # so operators see it. if e.status_code == 400: logger.warning( "Schema conflict on payload index '%s': %s", field, body_text @@ -130,6 +147,21 @@ async def _ensure_payload_indexes( body_text, ) failed_fields.append(field) + except Exception: + # Raw network / timeout failures (httpx.ConnectError, + # asyncio.TimeoutError, etc.) reach here — outside the HTTP-status + # taxonomy that UnexpectedResponse covers. Same containment + # rationale as above: one transient failure on one field must not + # skip the rest, and the singleton in get_qdrant_client is already + # assigned by this point so re-raising would leave the process + # holding a usable client with the migration silently incomplete. + logger.error( + "Network error creating payload index on '%s'; " + "field will remain unindexed until next successful restart", + field, + exc_info=True, + ) + failed_fields.append(field) # A single per-field ERROR line is easy to miss in startup noise. Surface # the partial-failure summary at WARNING so operators auditing the log @@ -396,13 +428,21 @@ async def get_qdrant_client() -> AsyncQdrantClient: Raises: Exception: If Qdrant connection fails or collection creation fails """ - global _qdrant_client + global _qdrant_client, _qdrant_init_lock # Fast path: already initialized — skip lock acquisition for the # steady-state hot path (every MCP tool call after first start). if _qdrant_client is not None: return _qdrant_client + # Lazy-create the init lock on first cold-start. Safe under cooperative + # multitasking: there is no ``await`` between the None-check and the + # assignment, so two coroutines cannot both reach the construction. + # See the rationale on _qdrant_init_lock for why eager construction + # would break under the trio backend. + if _qdrant_init_lock is None: + _qdrant_init_lock = anyio.Lock() + # Slow path: serialise concurrent first-callers so the idempotent-but- # expensive startup migration (``_backfill_doc_id_to_string`` + # ``_ensure_payload_indexes``) runs exactly once. Without this lock, diff --git a/tests/unit/vector/test_qdrant_client.py b/tests/unit/vector/test_qdrant_client.py index 32a498a2..4c2ebadd 100644 --- a/tests/unit/vector/test_qdrant_client.py +++ b/tests/unit/vector/test_qdrant_client.py @@ -221,6 +221,50 @@ async def test_ensure_payload_indexes_logs_non_400_as_error(mocker, caplog): assert "internal server error" in msg +@pytest.mark.unit +async def test_ensure_payload_indexes_continues_past_raw_network_error(mocker, caplog): + """A raw network error (e.g. ConnectError, TimeoutError) must not skip the rest. + + UnexpectedResponse covers HTTP-shaped failures, but transport-level + failures (httpx.ConnectError, asyncio.TimeoutError) reach the loop as + bare Exceptions. Without a broad catch, the first network blip + propagates, leaves _qdrant_client assigned, and silently skips every + remaining field. The fix is per-field containment matching the 5xx + behaviour: log at ERROR with exc_info, append to failed_fields, and + continue. + """ + client = mocker.AsyncMock() + client.get_collection.return_value = _empty_collection_info() + # First field hits a connection failure; remaining fields succeed. + client.create_payload_index.side_effect = [ + ConnectionError("Connection refused"), + *([None] * (len(_PAYLOAD_INDEX_FIELDS) - 1)), + ] + + with caplog.at_level("WARNING", logger="nextcloud_mcp_server.vector.qdrant_client"): + await _ensure_payload_indexes(client, "test-collection") + + # Loop continued past the failing field; every field was attempted. + assert client.create_payload_index.await_count == len(_PAYLOAD_INDEX_FIELDS) + errors = [r for r in caplog.records if r.levelname == "ERROR"] + assert len(errors) == 1 + assert "Network error creating payload index" in errors[0].getMessage() + # exc_info is preserved so operators can see the underlying cause. + assert errors[0].exc_info is not None + assert errors[0].exc_info[0] is ConnectionError + # The partial-failure summary surfaces the field as missing. + summary_warnings = [ + r + for r in caplog.records + if r.levelname == "WARNING" + and "Payload index creation incomplete" in r.getMessage() + ] + assert len(summary_warnings) == 1 + # The first field in _PAYLOAD_INDEX_FIELDS is the one that raised. + failing_field = next(iter(_PAYLOAD_INDEX_FIELDS)) + assert failing_field in summary_warnings[0].getMessage() + + @pytest.mark.unit async def test_ensure_payload_indexes_logs_and_returns_when_get_collection_raises( mocker, caplog From f3ce46da0ffc55222a1a27701091bbfbcae13d40 Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Sat, 9 May 2026 17:59:41 +0200 Subject: [PATCH 20/27] =?UTF-8?q?fix(vector):=20address=20PR=20review=20ro?= =?UTF-8?q?und=2011=20=E2=80=94=20broaden=20offset-skip=20gate,=20clarify?= =?UTF-8?q?=20ordering?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - search/context.py: drop the doc_type=='file' guard on skip_offset_lookup so notes / deck cards / news items also bypass the unindexed offset fallback when chunk_index is available. Legacy chunk_index=None data still uses the offset path. - vector/qdrant_client.py: clarify the backfill/_ensure_payload_indexes ordering invariant (backfill rewrites payload values only, never schema or indexes). Acknowledge OSS-vs-Cloud uncertainty in the 400-branch comment and the new-collection call-site comment. - vector/scanner.py: hoist qdrant_client to function scope so the file-scroll block doesn't depend on a name bound inside the notes-scroll block. - tests/unit/test_chunk_context_offset_gate.py: flip the note-with- chunk_index test to assert the offset fallback is skipped. Co-Authored-By: Claude Opus 4.7 (1M context) --- nextcloud_mcp_server/search/context.py | 15 ++++++----- nextcloud_mcp_server/vector/qdrant_client.py | 28 +++++++++++++++----- nextcloud_mcp_server/vector/scanner.py | 11 ++++++-- tests/unit/test_chunk_context_offset_gate.py | 22 ++++++++++----- 4 files changed, 55 insertions(+), 21 deletions(-) diff --git a/nextcloud_mcp_server/search/context.py b/nextcloud_mcp_server/search/context.py index c9baf084..6cd583a7 100644 --- a/nextcloud_mcp_server/search/context.py +++ b/nextcloud_mcp_server/search/context.py @@ -373,12 +373,15 @@ async def get_chunk_with_context( chunk_text = await _get_chunk_by_index_from_qdrant( user_id, doc_id, doc_type, chunk_index ) - # Skip the offset fallback for files when the indexed chunk_index - # lookup already ran: chunk_start/end_offset aren't indexed in Qdrant - # Cloud strict mode, so the call returns 400 and surfaces a misleading - # logger.error. The file fast-fail below correctly handles the miss - # without it. - skip_offset_lookup = chunk_index is not None and doc_type == "file" + # When chunk_index is available, treat the indexed lookup as canonical + # for every doc_type. A miss means the chunk is genuinely absent, not + # "fall back to the unindexed slow path". chunk_start/end_offset aren't + # in _PAYLOAD_INDEX_FIELDS, so the offset filter 400s in Qdrant Cloud + # strict mode and surfaces a misleading logger.error. Legacy data + # without chunk_index (pre-cbcoutinho/astrolabe#75) still hits the + # offset path and degrades to a None chunk with a WARNING; that's the + # same behavior get_chunk_bbox_and_page_from_qdrant already documents. + skip_offset_lookup = chunk_index is not None if chunk_text is None and not skip_offset_lookup: chunk_text = await _get_chunk_from_qdrant( user_id, doc_id, doc_type, chunk_start, chunk_end diff --git a/nextcloud_mcp_server/vector/qdrant_client.py b/nextcloud_mcp_server/vector/qdrant_client.py index 5162cbb4..c4876bc4 100644 --- a/nextcloud_mcp_server/vector/qdrant_client.py +++ b/nextcloud_mcp_server/vector/qdrant_client.py @@ -132,9 +132,14 @@ async def _ensure_payload_indexes( body = getattr(e, "content", b"") or b"" body_text = body.decode("utf-8", errors="replace") # 400 is the expected schema-conflict path (index already exists - # with a different type). 5xx is unexpected — keep the loop going - # so the remaining fields still get attempted, but log at error - # so operators see it. + # with a different type). Verified for Qdrant OSS, where an + # idempotent re-create against a matching schema returns 200; if + # Qdrant Cloud diverges and returns 400 for benign re-creates, + # the WARNING below will fire on every restart against an + # already-indexed collection — read the response body before + # treating that as a real schema conflict. 5xx is unexpected — + # keep the loop going so the remaining fields still get + # attempted, but log at error so operators see it. if e.status_code == 400: logger.warning( "Schema conflict on payload index '%s': %s", field, body_text @@ -534,7 +539,10 @@ async def get_qdrant_client() -> AsyncQdrantClient: # payload-index work. Backfill before creating the index so the # index covers every point. Pass the already-fetched # collection_info.payload_schema through to avoid a redundant - # get_collection round-trip on every restart. + # get_collection round-trip on every restart — safe because + # _backfill_doc_id_to_string only rewrites payload *values*, + # never schema or indexes, so the snapshot remains accurate + # across the backfill call. await _backfill_doc_id_to_string( _qdrant_client, collection_name, expected_dimension ) @@ -575,8 +583,16 @@ async def get_qdrant_client() -> AsyncQdrantClient: f" Distance: COSINE\n" f"Background sync will index all documents with dense + sparse vectors." ) - # Freshly created collection has no payload schema yet; pass {} - # explicitly to skip the otherwise-redundant get_collection call. + # Freshly created collection has no payload schema yet; pass + # {} explicitly to skip the otherwise-redundant + # get_collection call. Every field in _PAYLOAD_INDEX_FIELDS + # then goes through create_payload_index; on a brand-new + # collection none of them exist yet, so the WARNING in the + # 400-handler should *never* fire on this path. If it does + # on Qdrant Cloud first-start, that points at a + # deployment-level issue (race with a concurrent creator, + # implicit auto-indexes, etc.) worth investigating before + # suppressing. await _ensure_payload_indexes( _qdrant_client, collection_name, existing_schema={} ) diff --git a/nextcloud_mcp_server/vector/scanner.py b/nextcloud_mcp_server/vector/scanner.py index 6322411e..2dccdb3f 100644 --- a/nextcloud_mcp_server/vector/scanner.py +++ b/nextcloud_mcp_server/vector/scanner.py @@ -210,10 +210,16 @@ async def scan_user_documents( ) # For deletion tracking, get all doc_ids in Qdrant (for incremental sync) - # Note: We no longer bulk-query indexed_at, instead check per-document + # Note: We no longer bulk-query indexed_at, instead check per-document. + # Hoisted to function scope so the file-scroll block below doesn't + # depend on a name bound inside the notes-scroll block; future + # refactors that add an early return between the two blocks would + # otherwise hit an UnboundLocalError. get_qdrant_client is a + # singleton call, so the cost is identical. + qdrant_client = await get_qdrant_client() if not initial_sync else None indexed_doc_ids = set() if not initial_sync: - qdrant_client = await get_qdrant_client() + assert qdrant_client is not None # narrow for the type checker scroll_result = await qdrant_client.scroll( collection_name=get_settings().get_collection_name(), scroll_filter=Filter( @@ -387,6 +393,7 @@ async def scan_user_documents( # Get indexed file IDs from Qdrant (for deletion tracking) indexed_file_ids = set() if not initial_sync: + assert qdrant_client is not None # narrow for the type checker file_scroll_result = await qdrant_client.scroll( collection_name=settings.get_collection_name(), scroll_filter=Filter( diff --git a/tests/unit/test_chunk_context_offset_gate.py b/tests/unit/test_chunk_context_offset_gate.py index d46dc69d..8c7f85cc 100644 --- a/tests/unit/test_chunk_context_offset_gate.py +++ b/tests/unit/test_chunk_context_offset_gate.py @@ -28,8 +28,12 @@ def mock_nc_client() -> MagicMock: class TestOffsetFallbackGate: - """When chunk_index is provided AND doc_type=='file', the offset fallback - must be skipped — see PR #767 review (🟡 spurious Qdrant error log). + """When chunk_index is provided, the offset fallback must be skipped for + every doc_type. The original gate was file-only (PR #767 review, 🟡 + spurious Qdrant error log); PR #773 round 11 broadened it to all + doc_types because chunk_start/end_offset aren't in + ``_PAYLOAD_INDEX_FIELDS`` and 400 in Qdrant Cloud strict mode for + notes / deck cards / news items the same way they do for files. """ async def test_file_with_chunk_index_skips_offset_fallback_on_miss( @@ -64,11 +68,15 @@ class TestOffsetFallbackGate: mock_indexed.assert_awaited_once() mock_offset.assert_not_awaited() - async def test_note_with_chunk_index_still_uses_offset_fallback( + async def test_note_with_chunk_index_skips_offset_fallback_on_miss( self, mock_nc_client ): - """Notes/deck cards keep the offset fallback (cheap, useful for legacy - data): the gate is file-specific. + """Notes (and other non-file doc_types) trust the indexed + chunk_index lookup as canonical too. A miss means absent — don't + hit the unindexed offset path that 400s in Qdrant Cloud strict + mode. Legacy data without chunk_index still uses the offset path + via the chunk_index=None branch (see + test_file_without_chunk_index_uses_offset_fallback). """ with ( patch.object( @@ -81,7 +89,7 @@ class TestOffsetFallbackGate: context_module, "_get_chunk_from_qdrant", new_callable=AsyncMock, - return_value=None, + return_value="should-not-be-returned", ) as mock_offset, patch.object( context_module, @@ -102,7 +110,7 @@ class TestOffsetFallbackGate: ) mock_indexed.assert_awaited_once() - mock_offset.assert_awaited_once() + mock_offset.assert_not_awaited() async def test_file_without_chunk_index_uses_offset_fallback(self, mock_nc_client): """Files with no chunk_index supplied still use the offset path — From f9ad7dc52ee4f91bbf9dfecfd424226570c871de Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Sat, 9 May 2026 20:28:47 +0200 Subject: [PATCH 21/27] =?UTF-8?q?fix(vector):=20address=20PR=20review=20ro?= =?UTF-8?q?und=2012=20=E2=80=94=20bool=20guard=20+=20strict=20doc=5Fid=20v?= =?UTF-8?q?alidation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - _group_int_doc_ids: use type(value) is not int instead of isinstance, since bool is an int subclass and would otherwise stringify to "True"/"False" and corrupt legacy payloads on backfill. - Replace doc_id.isdigit() guards in 5 boundary sites (api/visualization, auth/viz_routes, search/context note/news_item/ deck_card branches) with a shared is_valid_nextcloud_doc_id helper that rejects "0", leading zeros, and Unicode digit classes (superscripts, Arabic-Indic, Devanagari) which pass isdigit() but cannot be valid MySQL AUTO_INCREMENT IDs. Co-Authored-By: Claude Opus 4.7 (1M context) --- nextcloud_mcp_server/api/visualization.py | 6 ++- nextcloud_mcp_server/auth/viz_routes.py | 6 ++- nextcloud_mcp_server/search/context.py | 32 +++++++----- nextcloud_mcp_server/utils/validation.py | 15 ++++++ nextcloud_mcp_server/vector/qdrant_client.py | 14 ++--- tests/unit/utils/__init__.py | 0 tests/unit/utils/test_validation.py | 54 ++++++++++++++++++++ tests/unit/vector/test_qdrant_client.py | 30 +++++++++++ 8 files changed, 133 insertions(+), 24 deletions(-) create mode 100644 nextcloud_mcp_server/utils/validation.py create mode 100644 tests/unit/utils/__init__.py create mode 100644 tests/unit/utils/test_validation.py diff --git a/nextcloud_mcp_server/api/visualization.py b/nextcloud_mcp_server/api/visualization.py index 62d8e3f3..8203ef9a 100644 --- a/nextcloud_mcp_server/api/visualization.py +++ b/nextcloud_mcp_server/api/visualization.py @@ -34,6 +34,7 @@ from nextcloud_mcp_server.search.context import ( get_chunk_bbox_and_page_from_qdrant, get_chunk_with_context, ) +from nextcloud_mcp_server.utils.validation import is_valid_nextcloud_doc_id from nextcloud_mcp_server.vector.oauth_sync import ( NotProvisionedError, get_user_client_basic_auth, @@ -502,8 +503,9 @@ async def get_chunk_context(request: Request) -> JSONResponse: # otherwise pass through to get_chunk_with_context and bottom out as a # 404 from deep inside, not a clear 400. Nextcloud IDs are unsigned # ints from MySQL auto_increment; doc_id stays a str downstream - # (Qdrant payload index is keyword-typed). - if not doc_id.isdigit(): + # (Qdrant payload index is keyword-typed). is_valid_nextcloud_doc_id + # rejects "0", leading zeros, and Unicode digits that pass isdigit(). + if not is_valid_nextcloud_doc_id(doc_id): return JSONResponse( { "success": False, diff --git a/nextcloud_mcp_server/auth/viz_routes.py b/nextcloud_mcp_server/auth/viz_routes.py index 5c2eee3d..688d03e9 100644 --- a/nextcloud_mcp_server/auth/viz_routes.py +++ b/nextcloud_mcp_server/auth/viz_routes.py @@ -37,6 +37,7 @@ from nextcloud_mcp_server.search.context import ( get_chunk_bbox_and_page_from_qdrant, get_chunk_with_context, ) +from nextcloud_mcp_server.utils.validation import is_valid_nextcloud_doc_id from nextcloud_mcp_server.vector.oauth_sync import ( NotProvisionedError, get_user_client_basic_auth, @@ -564,8 +565,9 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse: # otherwise pass through to get_chunk_with_context and bottom out as a # 404 from deep inside, not a clear 400. Nextcloud IDs are unsigned # ints from MySQL auto_increment; doc_id stays a str downstream - # (Qdrant payload index is keyword-typed). - if not doc_id.isdigit(): + # (Qdrant payload index is keyword-typed). is_valid_nextcloud_doc_id + # rejects "0", leading zeros, and Unicode digits that pass isdigit(). + if not is_valid_nextcloud_doc_id(doc_id): return JSONResponse( { "success": False, diff --git a/nextcloud_mcp_server/search/context.py b/nextcloud_mcp_server/search/context.py index 6cd583a7..c233007f 100644 --- a/nextcloud_mcp_server/search/context.py +++ b/nextcloud_mcp_server/search/context.py @@ -11,6 +11,7 @@ from qdrant_client.models import FieldCondition, Filter, MatchValue from nextcloud_mcp_server.client import NextcloudClient from nextcloud_mcp_server.config import get_settings +from nextcloud_mcp_server.utils.validation import is_valid_nextcloud_doc_id from nextcloud_mcp_server.vector.html_processor import html_to_markdown from nextcloud_mcp_server.vector.placeholder import get_placeholder_filter from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client @@ -577,10 +578,11 @@ async def _fetch_document_text( """ try: if doc_type == "note": - # Note IDs are integers in the Nextcloud API; reject non-numeric - # doc_ids explicitly so a malformed payload surfaces in logs - # rather than getting silently swallowed by `except Exception`. - if not doc_id.isdigit(): + # Note IDs are positive ASCII integers (MySQL AUTO_INCREMENT). + # is_valid_nextcloud_doc_id rejects "0", leading zeros, and Unicode + # digits that pass str.isdigit(); a malformed payload surfaces in + # logs rather than getting silently swallowed by `except Exception`. + if not is_valid_nextcloud_doc_id(doc_id): logger.warning( "Expected numeric note doc_id, got %r — skipping document fetch", doc_id, @@ -594,10 +596,11 @@ async def _fetch_document_text( content = note.get("content", "") return f"{title}\n\n{content}" elif doc_type == "news_item": - # News item IDs are integers in the Nextcloud News API; reject - # non-numeric doc_ids explicitly so malformed payloads surface - # rather than getting swallowed by the broad except below. - if not doc_id.isdigit(): + # News item IDs are positive ASCII integers (MySQL AUTO_INCREMENT). + # is_valid_nextcloud_doc_id rejects "0", leading zeros, and Unicode + # digits that pass str.isdigit(); malformed payloads surface in + # logs rather than getting swallowed by the broad except below. + if not is_valid_nextcloud_doc_id(doc_id): logger.warning( "Expected numeric news_item doc_id, got %r — skipping document fetch", doc_id, @@ -621,12 +624,13 @@ async def _fetch_document_text( content_parts.append(body_markdown) return "\n".join(content_parts) elif doc_type == "deck_card": - # Deck card IDs are integers in the Nextcloud Deck API; reject - # non-numeric doc_ids explicitly so malformed payloads surface - # rather than getting swallowed by the broad except below. The - # numeric check covers both the metadata-fast-path (line ~600) - # and the iteration fallback (line ~635). - if not doc_id.isdigit(): + # Deck card IDs are positive ASCII integers (MySQL AUTO_INCREMENT). + # is_valid_nextcloud_doc_id rejects "0", leading zeros, and Unicode + # digits that pass str.isdigit(); malformed payloads surface in + # logs rather than getting swallowed by the broad except below. + # The numeric check covers both the metadata-fast-path and the + # iteration fallback below. + if not is_valid_nextcloud_doc_id(doc_id): logger.warning( "Expected numeric deck_card doc_id, got %r — skipping document fetch", doc_id, diff --git a/nextcloud_mcp_server/utils/validation.py b/nextcloud_mcp_server/utils/validation.py new file mode 100644 index 00000000..2f9d0287 --- /dev/null +++ b/nextcloud_mcp_server/utils/validation.py @@ -0,0 +1,15 @@ +"""Shared validators for primitive types crossing system boundaries.""" + +import re + +# Nextcloud object IDs are unsigned ints from MySQL AUTO_INCREMENT, which +# starts at 1. Restrict to ASCII positive integers to exclude Unicode digit +# classes (e.g. superscripts, Arabic-Indic numerals) that pass str.isdigit() +# / str.isdecimal() but would never be valid Nextcloud IDs, and to reject "0" +# and leading zeros. +_NEXTCLOUD_DOC_ID_RE = re.compile(r"^[1-9][0-9]*$") + + +def is_valid_nextcloud_doc_id(value: str) -> bool: + """True iff `value` is the str form of a positive ASCII integer (>= 1).""" + return bool(_NEXTCLOUD_DOC_ID_RE.fullmatch(value)) diff --git a/nextcloud_mcp_server/vector/qdrant_client.py b/nextcloud_mcp_server/vector/qdrant_client.py index c4876bc4..b51b74bf 100644 --- a/nextcloud_mcp_server/vector/qdrant_client.py +++ b/nextcloud_mcp_server/vector/qdrant_client.py @@ -204,12 +204,14 @@ def _group_int_doc_ids(points: list[Any]) -> tuple[dict[str, list[Any]], int]: value = payload.get("doc_id") if value is None or isinstance(value, str): continue - if not isinstance(value, int): - # Producers only ever write int or str; anything else is a - # producer bug. Stringifying e.g. a float would write "3.0", - # which producers (str(int)) and the keyword index would - # never match, and which int() on the verification side - # would later reject. Skip and log loudly instead. + # Strict type check: bool is a subclass of int in Python, so an + # `isinstance(value, int)` guard would let `True`/`False` slip + # through and be stringified to `"True"`/`"False"` — which the + # keyword index would never match and the verification side + # would later reject. Producers only ever write int or str; + # anything else (bool, float, etc.) is a producer bug. Skip + # and log loudly instead. + if type(value) is not int: logger.warning( "Unexpected doc_id type %s on point %s; skipping rewrite", type(value).__name__, diff --git a/tests/unit/utils/__init__.py b/tests/unit/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/utils/test_validation.py b/tests/unit/utils/test_validation.py new file mode 100644 index 00000000..8e1f38ed --- /dev/null +++ b/tests/unit/utils/test_validation.py @@ -0,0 +1,54 @@ +"""Unit tests for shared boundary validators.""" + +import pytest + +from nextcloud_mcp_server.utils.validation import is_valid_nextcloud_doc_id + + +@pytest.mark.unit +@pytest.mark.parametrize( + "value", + [ + "1", + "42", + "1234567890", + "9999999999999999999", + ], +) +def test_accepts_positive_ascii_integers(value): + """Any positive ASCII integer (no leading zero) is a valid doc_id.""" + assert is_valid_nextcloud_doc_id(value) is True + + +@pytest.mark.unit +@pytest.mark.parametrize( + "value,reason", + [ + ("", "empty string"), + ("0", "MySQL AUTO_INCREMENT starts at 1"), + ("01", "leading zero"), + ("00", "leading zeros"), + ("-1", "negative"), + ("+1", "explicit sign"), + ("1.0", "float-like"), + (" 1", "leading whitespace"), + ("1 ", "trailing whitespace"), + ("1\n", "trailing newline"), + ("abc", "alphabetic"), + ("1a", "trailing letter"), + ("a1", "leading letter"), + # Unicode digit classes that pass str.isdigit() but are not ASCII. + # `²` (U+00B2) is a superscript and would slip past the old guard. + ("²", "Unicode superscript-2"), + # `٢` (U+0662) Arabic-Indic digit two — passes both isdigit() and + # isdecimal(), so only an explicit ASCII regex catches it. + ("٢", "Arabic-Indic digit two"), + # `१` (U+0967) Devanagari digit one — same story. + ("१", "Devanagari digit one"), + # Mixed ASCII + Unicode digits. + ("1٢", "mixed ASCII + Arabic-Indic"), + ], +) +def test_rejects_invalid_doc_ids(value, reason): + """Reject empty/zero/leading-zero/non-ASCII/non-digit inputs.""" + assert is_valid_nextcloud_doc_id(value) is False, f"should reject: {reason}" diff --git a/tests/unit/vector/test_qdrant_client.py b/tests/unit/vector/test_qdrant_client.py index 4c2ebadd..6611aedc 100644 --- a/tests/unit/vector/test_qdrant_client.py +++ b/tests/unit/vector/test_qdrant_client.py @@ -678,6 +678,36 @@ def test_group_int_doc_ids_skips_float_and_warns(caplog): assert "99" in msg +@pytest.mark.unit +def test_group_int_doc_ids_skips_bool_and_warns(caplog): + """A bool doc_id is not stringified to "True"/"False"; it logs and skips. + + ``isinstance(True, int)`` is ``True`` because ``bool`` is a subclass of + ``int`` in Python, so a naive ``isinstance(value, int)`` guard would let + a boolean payload through and write ``str(True)`` → ``"True"`` into + Qdrant. Producers never write bools, but the strict ``type(value) is + int`` guard ensures any future producer bug surfaces as a WARNING and is + not silently stringified. + """ + bool_point = SimpleNamespace(id=33, payload={"doc_id": True}) + int_point = SimpleNamespace(id=42, payload={"doc_id": 7}) + + with caplog.at_level("WARNING", logger="nextcloud_mcp_server.vector.qdrant_client"): + by_value, scanned = _group_int_doc_ids([bool_point, int_point]) + + # Only the int point made it into by_value — "True" is *not* a key. + assert by_value == {"7": [42]} + assert "True" not in by_value + assert "False" not in by_value + assert scanned == 2 + + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert len(warnings) == 1 + msg = warnings[0].getMessage() + assert "bool" in msg + assert "33" in msg + + @pytest.mark.unit def test_group_int_doc_ids_handles_str_and_missing_silently(caplog): """str / missing doc_id payloads are skipped without warning. From ae23bbe8b8558e7abdddfcdb3f457d3284e2926d Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Sun, 10 May 2026 10:27:57 +0200 Subject: [PATCH 22/27] =?UTF-8?q?fix(vector):=20address=20PR=20review=20ro?= =?UTF-8?q?und=2013=20=E2=80=94=20index=20offset=20fields=20+=20tighten=20?= =?UTF-8?q?test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add chunk_start_offset / chunk_end_offset to _PAYLOAD_INDEX_FIELDS so the legacy offset-based fallback in search/context.py works on Qdrant Cloud strict mode (pre-#75 clients have no chunk_index payload). - Cover chunk_index / chunk_start_offset / chunk_end_offset in the payload-index summary test; refresh the stale field-list comment. - Flag the is_valid_nextcloud_doc_id gate at both chunk-context handler sites with a TODO for future non-numeric doc_types. Co-Authored-By: Claude Opus 4.7 (1M context) --- nextcloud_mcp_server/api/visualization.py | 3 +++ nextcloud_mcp_server/auth/viz_routes.py | 3 +++ nextcloud_mcp_server/vector/qdrant_client.py | 7 ++++++- tests/unit/vector/test_qdrant_client.py | 14 ++++++++++---- 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/nextcloud_mcp_server/api/visualization.py b/nextcloud_mcp_server/api/visualization.py index 8203ef9a..370d4ef4 100644 --- a/nextcloud_mcp_server/api/visualization.py +++ b/nextcloud_mcp_server/api/visualization.py @@ -505,6 +505,9 @@ async def get_chunk_context(request: Request) -> JSONResponse: # ints from MySQL auto_increment; doc_id stays a str downstream # (Qdrant payload index is keyword-typed). is_valid_nextcloud_doc_id # rejects "0", leading zeros, and Unicode digits that pass isdigit(). + # TODO: when chunk-context support extends to non-numeric doc_types + # (calendar VEVENT UIDs, CardDAV hrefs, …), relax this gate or make + # it doc_type-aware. Today every indexed doc_type is numeric. if not is_valid_nextcloud_doc_id(doc_id): return JSONResponse( { diff --git a/nextcloud_mcp_server/auth/viz_routes.py b/nextcloud_mcp_server/auth/viz_routes.py index 688d03e9..f73717b9 100644 --- a/nextcloud_mcp_server/auth/viz_routes.py +++ b/nextcloud_mcp_server/auth/viz_routes.py @@ -567,6 +567,9 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse: # ints from MySQL auto_increment; doc_id stays a str downstream # (Qdrant payload index is keyword-typed). is_valid_nextcloud_doc_id # rejects "0", leading zeros, and Unicode digits that pass isdigit(). + # TODO: when chunk-context support extends to non-numeric doc_types + # (calendar VEVENT UIDs, CardDAV hrefs, …), relax this gate or make + # it doc_type-aware. Today every indexed doc_type is numeric. if not is_valid_nextcloud_doc_id(doc_id): return JSONResponse( { diff --git a/nextcloud_mcp_server/vector/qdrant_client.py b/nextcloud_mcp_server/vector/qdrant_client.py index b51b74bf..7217966a 100644 --- a/nextcloud_mcp_server/vector/qdrant_client.py +++ b/nextcloud_mcp_server/vector/qdrant_client.py @@ -31,13 +31,18 @@ logger = logging.getLogger(__name__) # ``_get_chunk_by_index_from_qdrant`` and ``get_chunk_bbox_and_page_from_qdrant`` # (see search/context.py) — the always-indexed fast path that the offset-based # fallback exists to avoid; it has to actually be indexed for that promise to -# hold on Qdrant Cloud strict mode. +# hold on Qdrant Cloud strict mode. chunk_start_offset / chunk_end_offset are +# the ints used by the legacy offset fallback in the same module — pre-#75 +# clients have no chunk_index payload, so the offset path still has to work +# (or 400 silently and return None on Qdrant Cloud strict mode). _PAYLOAD_INDEX_FIELDS: dict[str, PayloadSchemaType] = { "doc_id": PayloadSchemaType.KEYWORD, "user_id": PayloadSchemaType.KEYWORD, "doc_type": PayloadSchemaType.KEYWORD, "is_placeholder": PayloadSchemaType.BOOL, "chunk_index": PayloadSchemaType.INTEGER, + "chunk_start_offset": PayloadSchemaType.INTEGER, + "chunk_end_offset": PayloadSchemaType.INTEGER, } # Sentinel point that records "this collection has been backfilled to str diff --git a/tests/unit/vector/test_qdrant_client.py b/tests/unit/vector/test_qdrant_client.py index 6611aedc..30b71338 100644 --- a/tests/unit/vector/test_qdrant_client.py +++ b/tests/unit/vector/test_qdrant_client.py @@ -763,9 +763,10 @@ async def test_ensure_payload_indexes_summarises_failed_fields(mocker, caplog): """ client = mocker.AsyncMock() client.get_collection.return_value = SimpleNamespace(payload_schema={}) - # All but the second field fail with 5xx. _PAYLOAD_INDEX_FIELDS has - # insertion-ordered keys (doc_id, user_id, doc_type, is_placeholder), - # so call #2 (user_id) is the success case. + # _PAYLOAD_INDEX_FIELDS preserves insertion order; user_id is the second + # entry, so call #2 is the success case and every other field fails. Don't + # hard-code the full field list here — it grows as new fields move into + # the index dict, and the assertions below are what enforce coverage. call_count = {"n": 0} async def _create_index(*args, **kwargs): @@ -787,9 +788,14 @@ async def test_ensure_payload_indexes_summarises_failed_fields(mocker, caplog): if "Payload index creation incomplete" in r.getMessage() ] assert len(summary) == 1 - # All fields except user_id should appear in the summary. + # Every field that failed must appear in the summary — operators rely on + # this single log line to spot the degraded state, so any missing entry + # is a silent gap. assert "doc_id" in summary[0] assert "doc_type" in summary[0] assert "is_placeholder" in summary[0] + assert "chunk_index" in summary[0] + assert "chunk_start_offset" in summary[0] + assert "chunk_end_offset" in summary[0] assert "user_id" not in summary[0] # The one that succeeded. assert "test-collection" in summary[0] From c5020d9629d96a844efc166cf505964380af09a5 Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Sun, 10 May 2026 11:50:46 +0200 Subject: [PATCH 23/27] =?UTF-8?q?fix(vector):=20address=20PR=20review=20ro?= =?UTF-8?q?und=2014=20=E2=80=94=20accurate=20offset-skip=20comment=20+=20n?= =?UTF-8?q?ews=5Fitem=20doc=5Fid=20guard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-14 review surfaced one blocking and one important issue. context.py: the comment justifying `skip_offset_lookup` claimed chunk_start/end_offset weren't in _PAYLOAD_INDEX_FIELDS — round 13 indexed both as INTEGER, so the comment now actively misleads. Replace with the real reason: an indexed chunk_index miss is canonical (both paths hit the same Qdrant collection), and skipping the offset filter avoids a redundant round-trip. verification.py: hoist an is_valid_nextcloud_doc_id guard before the `int(d)` cast in _verify_news_items, mirroring the boundary-validation pattern already in _fetch_document_text. Coerce via `str(d)` because SearchResult.id is `int | str` (D1 forward-compat widening). Malformed ids now surface as a logger.warning rather than a generic debug line; fail-open semantics are preserved. Co-Authored-By: Claude Opus 4.7 (1M context) --- nextcloud_mcp_server/search/context.py | 15 +++++++-------- nextcloud_mcp_server/search/verification.py | 10 ++++++++++ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/nextcloud_mcp_server/search/context.py b/nextcloud_mcp_server/search/context.py index c233007f..11ef544c 100644 --- a/nextcloud_mcp_server/search/context.py +++ b/nextcloud_mcp_server/search/context.py @@ -374,14 +374,13 @@ async def get_chunk_with_context( chunk_text = await _get_chunk_by_index_from_qdrant( user_id, doc_id, doc_type, chunk_index ) - # When chunk_index is available, treat the indexed lookup as canonical - # for every doc_type. A miss means the chunk is genuinely absent, not - # "fall back to the unindexed slow path". chunk_start/end_offset aren't - # in _PAYLOAD_INDEX_FIELDS, so the offset filter 400s in Qdrant Cloud - # strict mode and surfaces a misleading logger.error. Legacy data - # without chunk_index (pre-cbcoutinho/astrolabe#75) still hits the - # offset path and degrades to a None chunk with a WARNING; that's the - # same behavior get_chunk_bbox_and_page_from_qdrant already documents. + # When chunk_index is supplied, the indexed lookup is canonical: both the + # index path and the offset path query the same Qdrant collection, so an + # indexed miss means the chunk is genuinely absent. Skipping the offset + # filter avoids a redundant Qdrant round-trip. Legacy data without + # chunk_index (pre-cbcoutinho/astrolabe#75) still hits the offset path + # and degrades to a None chunk with a WARNING; that's the same behavior + # get_chunk_bbox_and_page_from_qdrant already documents. skip_offset_lookup = chunk_index is not None if chunk_text is None and not skip_offset_lookup: chunk_text = await _get_chunk_from_qdrant( diff --git a/nextcloud_mcp_server/search/verification.py b/nextcloud_mcp_server/search/verification.py index a4b0a0d6..291b2cae 100644 --- a/nextcloud_mcp_server/search/verification.py +++ b/nextcloud_mcp_server/search/verification.py @@ -42,6 +42,7 @@ from nextcloud_mcp_server.search.algorithms import ( NextcloudClientProtocol, SearchResult, ) +from nextcloud_mcp_server.utils.validation import is_valid_nextcloud_doc_id from nextcloud_mcp_server.vector.eviction import delete_document_points logger = logging.getLogger(__name__) @@ -388,6 +389,15 @@ async def _verify_news_items( # above for why this is narrower than the API-response failure path. accessible: set[str] = set() for d in doc_ids: + # SearchResult.id is `int | str` (D1: forward-compat widening). Coerce + # to str so the validator's regex applies consistently to both shapes. + if not is_valid_nextcloud_doc_id(str(d)): + logger.warning( + "Malformed news_item doc_id %r in verifier; keeping (cannot verify)", + d, + ) + accessible.add(d) + continue try: if int(d) in present_ids: accessible.add(d) From 68506f96c522a06ce1c3124126c74a98bfc17a82 Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Sun, 10 May 2026 13:49:17 +0200 Subject: [PATCH 24/27] =?UTF-8?q?fix(vector):=20address=20PR=20review=20ro?= =?UTF-8?q?und=2015=20=E2=80=94=20concurrency,=20pagination,=20stale=20coe?= =?UTF-8?q?rcion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Defer publication of `_qdrant_client` until after the in-lock backfill + payload-index migration awaits complete. The fast-path check at the top of `get_qdrant_client` reads the singleton without holding the init lock, so publishing the constructed-but-unmigrated client let concurrent fast-path callers fire filtered searches before `_ensure_payload_indexes` ran — producing HTTP 400 ("Index required but not found") on Qdrant Cloud strict mode. Local `provisional` is now used for every await inside the lock; the global is assigned exactly once, last. Replace the five hand-rolled `scroll(..., limit=10000)` calls in `vector/scanner.py` (notes / files / news / deck-cards deletion tracking, plus the timestamp scroll) with a single paginated `_scroll_all_points` helper. The previous single-page cap silently dropped deletion-tracking points beyond the first 10 k for any user past that threshold. Pagination follows Qdrant's documented contract (loop until `next_page_offset is None`) with a fixed per-page `_DELETION_TRACKING_PAGE_SIZE = 1024`. Extract `_create_one_payload_index` from `_ensure_payload_indexes` to drop its cognitive complexity below the SonarQube limit (17 → ≤ 15) without losing the per-field error-containment rationale; every comment is preserved verbatim on the helper. Drop the stale `SearchResult.id` `int | str` comment and the redundant `str(d)` coercion in `_verify_news_items` — the contract has been str-only since the producer-side stringification landed earlier in this PR. Fix eight `doc_id=` test calls in `test_chunk_context_offset_gate.py` that violated the `doc_id: str` signature of `get_chunk_with_context`, plus align `_make_result` in `test_verification.py` to coerce `id=str(...)` matching the production contract — and update 30+ assertions from int sets (`{1, 2, 3}`) to str sets (`{"1", "2", "3"}`) so the tests now model the post-PR `SearchResult.id: str` reality end-to-end. Previously these were masked by the `str(d)` coercion now removed from production. Co-Authored-By: Claude Opus 4.7 (1M context) --- nextcloud_mcp_server/search/verification.py | 6 +- nextcloud_mcp_server/vector/qdrant_client.py | 164 +++++++++++-------- nextcloud_mcp_server/vector/scanner.py | 100 +++++++---- tests/unit/search/test_verification.py | 77 ++++----- tests/unit/test_chunk_context_offset_gate.py | 16 +- 5 files changed, 220 insertions(+), 143 deletions(-) diff --git a/nextcloud_mcp_server/search/verification.py b/nextcloud_mcp_server/search/verification.py index 291b2cae..ca8f5c0e 100644 --- a/nextcloud_mcp_server/search/verification.py +++ b/nextcloud_mcp_server/search/verification.py @@ -389,9 +389,9 @@ async def _verify_news_items( # above for why this is narrower than the API-response failure path. accessible: set[str] = set() for d in doc_ids: - # SearchResult.id is `int | str` (D1: forward-compat widening). Coerce - # to str so the validator's regex applies consistently to both shapes. - if not is_valid_nextcloud_doc_id(str(d)): + # SearchResult.id is always str (Qdrant payload doc_id is keyword- + # indexed; producers stringify on write). Pass through verbatim. + if not is_valid_nextcloud_doc_id(d): logger.warning( "Malformed news_item doc_id %r in verifier; keeping (cannot verify)", d, diff --git a/nextcloud_mcp_server/vector/qdrant_client.py b/nextcloud_mcp_server/vector/qdrant_client.py index 7217966a..0263cb92 100644 --- a/nextcloud_mcp_server/vector/qdrant_client.py +++ b/nextcloud_mcp_server/vector/qdrant_client.py @@ -71,6 +71,71 @@ _qdrant_client: AsyncQdrantClient | None = None _qdrant_init_lock: anyio.Lock | None = None +async def _create_one_payload_index( + client: AsyncQdrantClient, + collection_name: str, + field: str, + schema_type: PayloadSchemaType, +) -> bool: + """Create one payload index with per-field error containment. + + Returns True on success or benign 400 schema-conflict (caller treats as + indexed). Returns False if the field should be added to the caller's + failed-fields list. Never re-raises: the singleton in + ``get_qdrant_client`` is already assigned by the time this runs, so + propagating a network blip would leave the process holding a usable + client with the migration silently incomplete. + """ + try: + await client.create_payload_index( + collection_name=collection_name, + field_name=field, + field_schema=schema_type, + wait=True, + ) + logger.info("Created %s payload index on '%s'", schema_type.name, field) + return True + except UnexpectedResponse as e: + body = getattr(e, "content", b"") or b"" + body_text = body.decode("utf-8", errors="replace") + # 400 is the expected schema-conflict path (index already exists + # with a different type). Verified for Qdrant OSS, where an + # idempotent re-create against a matching schema returns 200; if + # Qdrant Cloud diverges and returns 400 for benign re-creates, + # the WARNING below will fire on every restart against an + # already-indexed collection — read the response body before + # treating that as a real schema conflict. 5xx is unexpected — + # keep the loop going so the remaining fields still get + # attempted, but log at error so operators see it. + if e.status_code == 400: + logger.warning( + "Schema conflict on payload index '%s': %s", field, body_text + ) + return True + logger.error( + "Unexpected error creating payload index on '%s' (status %s): %s", + field, + e.status_code, + body_text, + ) + return False + except Exception: + # Raw network / timeout failures (httpx.ConnectError, + # asyncio.TimeoutError, etc.) reach here — outside the HTTP-status + # taxonomy that UnexpectedResponse covers. Same containment + # rationale as above: one transient failure on one field must not + # skip the rest, and the singleton in get_qdrant_client is already + # assigned by this point so re-raising would leave the process + # holding a usable client with the migration silently incomplete. + logger.error( + "Network error creating payload index on '%s'; " + "field will remain unindexed until next successful restart", + field, + exc_info=True, + ) + return False + + async def _ensure_payload_indexes( client: AsyncQdrantClient, collection_name: str, @@ -82,13 +147,9 @@ async def _ensure_payload_indexes( schema type (KEYWORD for string fields, BOOL for ``is_placeholder``, INTEGER for ``chunk_index``). Skips fields that are already in ``existing_schema`` so routine restarts make no Qdrant write round-trips - and emit no INFO log lines. Schema conflicts (a pre-existing index with - a different type) still surface as a 400 — log loudly so operators can - intervene, but keep going so the remaining fields still get indexed. - The same per-field error containment applies to raw network errors - (e.g. ``httpx.ConnectError`` from a transient Qdrant unavailability): - log at ERROR with ``exc_info`` and continue, so a single transient - failure on one field does not skip the rest. + and emit no INFO log lines. Per-field error handling (schema conflicts, + network errors) lives in ``_create_one_payload_index``; this loop is + flat so a single transient failure on one field does not skip the rest. Args: client: Qdrant client instance. @@ -117,60 +178,17 @@ async def _ensure_payload_indexes( ) return existing_schema = collection_info.payload_schema or {} - failed_fields: list[str] = [] + failed_fields: list[str] = [] for field, schema_type in _PAYLOAD_INDEX_FIELDS.items(): if field in existing_schema: # Index already present — silent skip. Logging here on every # restart would be noise that hides the genuinely interesting - # "first-time creation" line below. + # "first-time creation" line in _create_one_payload_index. continue - try: - await client.create_payload_index( - collection_name=collection_name, - field_name=field, - field_schema=schema_type, - wait=True, - ) - logger.info("Created %s payload index on '%s'", schema_type.name, field) - except UnexpectedResponse as e: - body = getattr(e, "content", b"") or b"" - body_text = body.decode("utf-8", errors="replace") - # 400 is the expected schema-conflict path (index already exists - # with a different type). Verified for Qdrant OSS, where an - # idempotent re-create against a matching schema returns 200; if - # Qdrant Cloud diverges and returns 400 for benign re-creates, - # the WARNING below will fire on every restart against an - # already-indexed collection — read the response body before - # treating that as a real schema conflict. 5xx is unexpected — - # keep the loop going so the remaining fields still get - # attempted, but log at error so operators see it. - if e.status_code == 400: - logger.warning( - "Schema conflict on payload index '%s': %s", field, body_text - ) - else: - logger.error( - "Unexpected error creating payload index on '%s' (status %s): %s", - field, - e.status_code, - body_text, - ) - failed_fields.append(field) - except Exception: - # Raw network / timeout failures (httpx.ConnectError, - # asyncio.TimeoutError, etc.) reach here — outside the HTTP-status - # taxonomy that UnexpectedResponse covers. Same containment - # rationale as above: one transient failure on one field must not - # skip the rest, and the singleton in get_qdrant_client is already - # assigned by this point so re-raising would leave the process - # holding a usable client with the migration silently incomplete. - logger.error( - "Network error creating payload index on '%s'; " - "field will remain unindexed until next successful restart", - field, - exc_info=True, - ) + if not await _create_one_payload_index( + client, collection_name, field, schema_type + ): failed_fields.append(field) # A single per-field ERROR line is easy to miss in startup noise. Surface @@ -467,11 +485,21 @@ async def get_qdrant_client() -> AsyncQdrantClient: if _qdrant_client is None: settings = get_settings() + # Build the client into a local ``provisional`` and only publish + # it to the global ``_qdrant_client`` after the migration awaits + # below have all completed. The fast-path check at the top of + # this function reads ``_qdrant_client`` without the lock, so + # publishing the constructed-but-unmigrated client would let a + # concurrent caller short-circuit the lock and fire a filtered + # search before ``_ensure_payload_indexes`` runs — that search + # would 400 with "Index required but not found". + provisional: AsyncQdrantClient + # Detect mode and initialize client accordingly if settings.qdrant_url: # Network mode logger.info(f"Using Qdrant network mode: {settings.qdrant_url}") - _qdrant_client = AsyncQdrantClient( + provisional = AsyncQdrantClient( url=settings.qdrant_url, api_key=settings.qdrant_api_key, timeout=30, @@ -480,17 +508,17 @@ async def get_qdrant_client() -> AsyncQdrantClient: # Local mode (either :memory: or persistent path) if settings.qdrant_location == ":memory:": logger.info("Using Qdrant in-memory mode: :memory:") - _qdrant_client = AsyncQdrantClient(":memory:") + provisional = AsyncQdrantClient(":memory:") else: # Persistent local mode - use path parameter logger.info( f"Using Qdrant persistent mode: {settings.qdrant_location}" ) - _qdrant_client = AsyncQdrantClient(path=settings.qdrant_location) + provisional = AsyncQdrantClient(path=settings.qdrant_location) else: # Should not happen due to __post_init__ validation, but handle gracefully logger.warning("No Qdrant mode configured, defaulting to :memory:") - _qdrant_client = AsyncQdrantClient(":memory:") + provisional = AsyncQdrantClient(":memory:") # Get collection name (auto-generated from deployment ID + model) collection_name = settings.get_collection_name() @@ -505,7 +533,7 @@ async def get_qdrant_client() -> AsyncQdrantClient: # Explicitly check if collection exists logger.debug(f"Checking if collection '{collection_name}' exists...") - collections = await _qdrant_client.get_collections() + collections = await provisional.get_collections() collection_names = [c.name for c in collections.collections] if collection_name in collection_names: @@ -513,7 +541,7 @@ async def get_qdrant_client() -> AsyncQdrantClient: logger.debug( f"Collection '{collection_name}' found, validating dimensions..." ) - collection_info = await _qdrant_client.get_collection(collection_name) + collection_info = await provisional.get_collection(collection_name) # Handle both named vectors (dict) and legacy single vector vectors = collection_info.config.params.vectors if isinstance(vectors, dict): @@ -551,10 +579,10 @@ async def get_qdrant_client() -> AsyncQdrantClient: # never schema or indexes, so the snapshot remains accurate # across the backfill call. await _backfill_doc_id_to_string( - _qdrant_client, collection_name, expected_dimension + provisional, collection_name, expected_dimension ) await _ensure_payload_indexes( - _qdrant_client, + provisional, collection_name, existing_schema=collection_info.payload_schema or {}, ) @@ -566,7 +594,7 @@ async def get_qdrant_client() -> AsyncQdrantClient: f"Collection '{collection_name}' not found, creating with " f"dimension={expected_dimension}, model={embedding_model}..." ) - await _qdrant_client.create_collection( + await provisional.create_collection( collection_name=collection_name, vectors_config={ "dense": VectorParams( @@ -601,9 +629,15 @@ async def get_qdrant_client() -> AsyncQdrantClient: # implicit auto-indexes, etc.) worth investigating before # suppressing. await _ensure_payload_indexes( - _qdrant_client, collection_name, existing_schema={} + provisional, collection_name, existing_schema={} ) + # Publish only after the migration awaits completed. From this + # point on, fast-path callers may short-circuit the lock and + # use the client; every payload index they could filter on now + # exists. + _qdrant_client = provisional + # Lock released. ``_qdrant_client`` is guaranteed non-None here: # either the fast path returned earlier, the lock-protected branch # set it, or a sibling waiter set it before we got the lock. diff --git a/nextcloud_mcp_server/vector/scanner.py b/nextcloud_mcp_server/vector/scanner.py index 2dccdb3f..0f2204e4 100644 --- a/nextcloud_mcp_server/vector/scanner.py +++ b/nextcloud_mcp_server/vector/scanner.py @@ -13,7 +13,8 @@ from email.utils import parsedate_to_datetime import anyio from anyio.abc import TaskStatus from anyio.streams.memory import MemoryObjectSendStream -from qdrant_client.models import FieldCondition, Filter, MatchValue +from qdrant_client import AsyncQdrantClient +from qdrant_client.models import FieldCondition, Filter, MatchValue, Record from nextcloud_mcp_server.client import NextcloudClient from nextcloud_mcp_server.client.news import NewsItemType @@ -43,6 +44,49 @@ INDEXED_DOC_TYPES: frozenset[str] = frozenset( ) +# Page size for paginated deletion-tracking scrolls. Chosen to keep per-page +# memory bounded while making the round-trip count manageable in the typical +# < 100 k point per (user_id, doc_type) case. The previous single-page +# ``limit=10_000`` silently truncated deletion sets for any user past the +# cap, so anything indexed beyond the first 10 k was never reconciled. +_DELETION_TRACKING_PAGE_SIZE: int = 1024 + + +async def _scroll_all_points( + qdrant_client: AsyncQdrantClient, + *, + collection_name: str, + scroll_filter: Filter, + payload_fields: list[str], + page_size: int = _DELETION_TRACKING_PAGE_SIZE, +) -> list[Record]: + """Scroll every point matching the filter, paginating until exhausted. + + Replaces the prior single-page ``limit=10_000`` calls that silently + dropped points beyond the first page. Pagination follows Qdrant's + documented contract: ``scroll`` returns ``(points, next_page_offset)`` + and ``next_page_offset`` is ``None`` once the cursor reaches the end. + Errors propagate to the caller — the scanner's outer ``try`` already + handles them by skipping the deletion-tracking pass for this scan + (worse: extra-scan latency; never: bad data). + """ + all_points: list[Record] = [] + offset = None + while True: + points, offset = await qdrant_client.scroll( + collection_name=collection_name, + scroll_filter=scroll_filter, + with_payload=payload_fields, + with_vectors=False, + limit=page_size, + offset=offset, + ) + all_points.extend(points) + if offset is None: + break + return all_points + + @dataclass class DocumentTask: """Document task for processing queue.""" @@ -79,8 +123,11 @@ async def get_last_indexed_timestamp(user_id: str) -> int | None: try: qdrant_client = await get_qdrant_client() - # Query for user's notes, ordered by indexed_at descending, limit 1 - scroll_result = await qdrant_client.scroll( + # Scroll across every indexed note for this user — paginated so users + # with > 10 k indexed notes still produce a correct max (the prior + # single-page ``limit=10_000`` would have silently undercounted). + points = await _scroll_all_points( + qdrant_client, collection_name=get_settings().get_collection_name(), scroll_filter=Filter( must=[ @@ -88,19 +135,16 @@ async def get_last_indexed_timestamp(user_id: str) -> int | None: FieldCondition(key="doc_type", match=MatchValue(value="note")), ] ), - with_payload=["indexed_at"], - with_vectors=False, - limit=10000, # Get all to find max + payload_fields=["indexed_at"], ) - # Find max indexed_at across all results - num_points = len(scroll_result[0]) if scroll_result[0] else 0 + num_points = len(points) logger.info(f"Found {num_points} indexed notes in Qdrant for user {user_id}") - if scroll_result[0]: + if points: timestamps = [ point.payload.get("indexed_at", 0) - for point in scroll_result[0] + for point in points if point.payload is not None ] max_timestamp = max(timestamps) if timestamps else 0 @@ -220,7 +264,8 @@ async def scan_user_documents( indexed_doc_ids = set() if not initial_sync: assert qdrant_client is not None # narrow for the type checker - scroll_result = await qdrant_client.scroll( + points = await _scroll_all_points( + qdrant_client, collection_name=get_settings().get_collection_name(), scroll_filter=Filter( must=[ @@ -228,14 +273,12 @@ async def scan_user_documents( FieldCondition(key="doc_type", match=MatchValue(value="note")), ] ), - with_payload=["doc_id"], - with_vectors=False, - limit=10000, + payload_fields=["doc_id"], ) indexed_doc_ids = { str(point.payload["doc_id"]) - for point in (scroll_result[0] or []) + for point in points if point.payload is not None and "doc_id" in point.payload } @@ -394,7 +437,8 @@ async def scan_user_documents( indexed_file_ids = set() if not initial_sync: assert qdrant_client is not None # narrow for the type checker - file_scroll_result = await qdrant_client.scroll( + points = await _scroll_all_points( + qdrant_client, collection_name=settings.get_collection_name(), scroll_filter=Filter( must=[ @@ -402,14 +446,12 @@ async def scan_user_documents( FieldCondition(key="doc_type", match=MatchValue(value="file")), ] ), - limit=10000, # Reasonable limit for file count - with_payload=["doc_id"], - with_vectors=False, + payload_fields=["doc_id"], ) indexed_file_ids = { str(point.payload["doc_id"]) - for point in (file_scroll_result[0] or []) + for point in points if point.payload is not None and "doc_id" in point.payload } @@ -675,7 +717,8 @@ async def scan_news_items( indexed_item_ids: set[str] = set() if not initial_sync: qdrant_client = await get_qdrant_client() - scroll_result = await qdrant_client.scroll( + points = await _scroll_all_points( + qdrant_client, collection_name=settings.get_collection_name(), scroll_filter=Filter( must=[ @@ -683,13 +726,11 @@ async def scan_news_items( FieldCondition(key="doc_type", match=MatchValue(value="news_item")), ] ), - with_payload=["doc_id"], - with_vectors=False, - limit=10000, + payload_fields=["doc_id"], ) indexed_item_ids = { str(point.payload["doc_id"]) - for point in (scroll_result[0] or []) + for point in points if point.payload is not None and "doc_id" in point.payload } logger.debug(f"Found {len(indexed_item_ids)} indexed news items in Qdrant") @@ -854,7 +895,8 @@ async def scan_deck_cards( indexed_card_ids: set[str] = set() if not initial_sync: qdrant_client = await get_qdrant_client() - scroll_result = await qdrant_client.scroll( + points = await _scroll_all_points( + qdrant_client, collection_name=settings.get_collection_name(), scroll_filter=Filter( must=[ @@ -862,13 +904,11 @@ async def scan_deck_cards( FieldCondition(key="doc_type", match=MatchValue(value="deck_card")), ] ), - with_payload=["doc_id"], - with_vectors=False, - limit=10000, + payload_fields=["doc_id"], ) indexed_card_ids = { str(point.payload["doc_id"]) - for point in (scroll_result[0] or []) + for point in points if point.payload is not None and "doc_id" in point.payload } logger.debug(f"Found {len(indexed_card_ids)} indexed deck cards in Qdrant") diff --git a/tests/unit/search/test_verification.py b/tests/unit/search/test_verification.py index da5b8065..3cee6981 100644 --- a/tests/unit/search/test_verification.py +++ b/tests/unit/search/test_verification.py @@ -35,8 +35,11 @@ def _make_result( score: float = 0.9, metadata: dict | None = None, ) -> SearchResult: + # Mirror the producer-side stringification (scanner writes str(note["id"]) + # etc. into Qdrant payloads). Tests pass int literals for readability; + # the SearchResult contract is ``id: str``. return SearchResult( - id=doc_id, + id=str(doc_id), doc_type=doc_type, title=f"{doc_type}_{doc_id}", excerpt="...", @@ -84,7 +87,7 @@ async def test_verify_notes_200_keeps_all(mocker): client, [_make_result(1), _make_result(2), _make_result(3)], _sem() ) - assert result == {1, 2, 3} + assert result == {"1", "2", "3"} assert notes_client.get_note.await_count == 3 @@ -122,7 +125,7 @@ async def test_verify_notes_transient_5xx_keeps(mocker): result = await _verify_notes(client, [_make_result(42)], _sem()) - assert result == {42} + assert result == {"42"} @pytest.mark.unit @@ -140,7 +143,7 @@ async def test_verify_notes_429_keeps_as_transient(mocker): result = await _verify_notes(client, [_make_result(42)], _sem()) - assert result == {42} + assert result == {"42"} @pytest.mark.unit @@ -152,7 +155,7 @@ async def test_verify_notes_unexpected_exception_keeps(mocker): result = await _verify_notes(client, [_make_result(7)], _sem()) - assert result == {7} + assert result == {"7"} @pytest.mark.unit @@ -193,7 +196,7 @@ async def test_verify_notes_mixed_outcomes(mocker): client, [_make_result(1), _make_result(2), _make_result(3)], _sem() ) - assert result == {1, 3} + assert result == {"1", "3"} @pytest.mark.unit @@ -242,7 +245,7 @@ async def test_verify_news_items_intersects_with_fetched_set(mocker): _sem(), ) - assert result == {10, 20} + assert result == {"10", "20"} assert news_client.get_items.await_count == 1 @@ -304,7 +307,7 @@ async def test_verify_news_items_transient_keeps_all(mocker): _sem(), ) - assert result == {1, 2, 3} + assert result == {"1", "2", "3"} @pytest.mark.unit @@ -325,7 +328,7 @@ async def test_verify_news_items_429_keeps_as_transient(mocker): _sem(), ) - assert result == {1, 2, 3} + assert result == {"1", "2", "3"} @pytest.mark.unit @@ -350,7 +353,7 @@ async def test_verify_news_items_unexpected_exception_keeps_all(mocker): _sem(), ) - assert result == {1, 2} + assert result == {"1", "2"} @pytest.mark.unit @@ -377,7 +380,7 @@ async def test_verify_news_items_non_numeric_id_keeps_only_bad_item(mocker): ) # 10 and 20 are verified present; "abc" is unverifiable so kept fail-open. - assert result == {10, 20, "abc"} + assert result == {"10", "20", "abc"} @pytest.mark.unit @@ -402,7 +405,7 @@ async def test_verify_news_items_drops_missing_when_other_id_is_non_numeric( ) # 10 verified present, 20 verified missing (dropped), "abc" unverifiable. - assert result == {10, "abc"} + assert result == {"10", "abc"} @pytest.mark.unit @@ -426,7 +429,7 @@ async def test_verify_news_items_malformed_api_response_keeps_all(mocker): ) # Batch fail-open: API broken, every requested id preserved. - assert result == {10, 20} + assert result == {"10", "20"} # --------------------------------------------------------------------------- @@ -448,7 +451,7 @@ async def test_verify_files_uses_path_from_metadata(mocker): _sem(), ) - assert result == {100} + assert result == {"100"} webdav_client.get_file_info.assert_awaited_once_with("Documents/foo.txt") @@ -487,7 +490,7 @@ async def test_verify_files_malformed_propfind_keeps_result(mocker): _sem(), ) - assert result == {123}, "ambiguous None must keep result, not evict" + assert result == {"123"}, "ambiguous None must keep result, not evict" @pytest.mark.unit @@ -517,14 +520,14 @@ async def test_verify_files_missing_path_metadata_keeps_unverified(mocker): # No metadata at all result = await _verify_files(client, [_make_result(555, doc_type="file")], _sem()) - assert result == {555} + assert result == {"555"} webdav_client.get_file_info.assert_not_awaited() # Metadata present but no "path" key result = await _verify_files( client, [_make_result(556, doc_type="file", metadata={})], _sem() ) - assert result == {556} + assert result == {"556"} webdav_client.get_file_info.assert_not_awaited() @@ -541,7 +544,7 @@ async def test_verify_files_transient_5xx_keeps(mocker): _sem(), ) - assert result == {7} + assert result == {"7"} @pytest.mark.unit @@ -558,7 +561,7 @@ async def test_verify_files_429_keeps_as_transient(mocker): _sem(), ) - assert result == {7} + assert result == {"7"} @pytest.mark.unit @@ -580,7 +583,7 @@ async def test_verify_files_unexpected_exception_keeps(mocker): _sem(), ) - assert result == {8} + assert result == {"8"} # --------------------------------------------------------------------------- @@ -606,7 +609,7 @@ async def test_verify_deck_cards_uses_metadata_fast_path(mocker): _sem(), ) - assert result == {42} + assert result == {"42"} deck_client.get_card.assert_awaited_once_with(board_id=1, stack_id=2, card_id=42) @@ -676,7 +679,7 @@ async def test_verify_deck_cards_transient_5xx_keeps(mocker): _sem(), ) - assert result == {42} + assert result == {"42"} @pytest.mark.unit @@ -699,7 +702,7 @@ async def test_verify_deck_cards_429_keeps_as_transient(mocker): _sem(), ) - assert result == {42} + assert result == {"42"} @pytest.mark.unit @@ -722,7 +725,7 @@ async def test_verify_deck_cards_unexpected_exception_keeps(mocker): _sem(), ) - assert result == {42} + assert result == {"42"} @pytest.mark.unit @@ -749,7 +752,7 @@ async def test_verify_deck_cards_non_numeric_metadata_keeps(mocker): ], _sem(), ) - assert result == {42} + assert result == {"42"} # Non-numeric stack_id result = await _verify_deck_cards( @@ -763,7 +766,7 @@ async def test_verify_deck_cards_non_numeric_metadata_keeps(mocker): ], _sem(), ) - assert result == {43} + assert result == {"43"} # Non-numeric card_id (doc_id itself) result = await _verify_deck_cards( @@ -794,7 +797,7 @@ async def test_verify_deck_cards_missing_metadata_keeps_unverified(mocker): result = await _verify_deck_cards( client, [_make_result(42, doc_type="deck_card")], _sem() ) - assert result == {42} + assert result == {"42"} # Only board_id (stack_id missing) result = await _verify_deck_cards( @@ -802,7 +805,7 @@ async def test_verify_deck_cards_missing_metadata_keeps_unverified(mocker): [_make_result(43, doc_type="deck_card", metadata={"board_id": 1})], _sem(), ) - assert result == {43} + assert result == {"43"} # Only stack_id (board_id missing) result = await _verify_deck_cards( @@ -810,7 +813,7 @@ async def test_verify_deck_cards_missing_metadata_keeps_unverified(mocker): [_make_result(44, doc_type="deck_card", metadata={"stack_id": 2})], _sem(), ) - assert result == {44} + assert result == {"44"} deck_client.get_card.assert_not_awaited() @@ -829,7 +832,7 @@ async def test_verify_search_results_empty_input_passthrough(): @pytest.mark.unit async def test_verify_search_results_dedupes_chunks_per_document(mocker): """Two chunks of the same note → ONE call to the underlying verifier.""" - spy = mocker.AsyncMock(return_value={1}) + spy = mocker.AsyncMock(return_value={"1"}) mocker.patch.dict(verification._VERIFIERS, {"note": spy}, clear=False) mocker.patch.object(verification, "delete_document_points", mocker.AsyncMock()) @@ -848,7 +851,7 @@ async def test_verify_search_results_dedupes_chunks_per_document(mocker): # Verifier received exactly one SearchResult (the deduplicated representative) args, _kwargs = spy.call_args assert len(args[1]) == 1 - assert args[1][0].id == 1 + assert args[1][0].id == "1" # And a semaphore as the third arg assert isinstance(args[2], anyio.Semaphore) @@ -1035,7 +1038,7 @@ async def test_verify_search_results_verifier_blowup_keeps_all(mocker): kept, dropped_count = await verify_search_results(client, results) - assert [r.id for r in kept] == [1, 2] + assert [r.id for r in kept] == ["1", "2"] assert dropped_count == 0 # fail-open: nothing dropped spy_evict.assert_not_awaited() @@ -1043,7 +1046,7 @@ async def test_verify_search_results_verifier_blowup_keeps_all(mocker): @pytest.mark.unit async def test_verify_search_results_preserves_order(mocker): """Order of original results must be preserved after filtering.""" - note_verifier = mocker.AsyncMock(return_value={1, 3}) + note_verifier = mocker.AsyncMock(return_value={"1", "3"}) mocker.patch.dict(verification._VERIFIERS, {"note": note_verifier}, clear=False) mocker.patch.object(verification, "delete_document_points", mocker.AsyncMock()) @@ -1056,7 +1059,7 @@ async def test_verify_search_results_preserves_order(mocker): kept, dropped_count = await verify_search_results(client, results) - assert [r.id for r in kept] == [1, 3] + assert [r.id for r in kept] == ["1", "3"] assert dropped_count == 1 @@ -1083,8 +1086,8 @@ async def test_verify_search_results_eviction_failure_does_not_propagate(mocker) @pytest.mark.unit async def test_verify_search_results_dispatches_per_doc_type_concurrently(mocker): """Mixed doc_types must be routed to their respective verifiers.""" - note_verifier = mocker.AsyncMock(return_value={1}) - file_verifier = mocker.AsyncMock(return_value={500}) + note_verifier = mocker.AsyncMock(return_value={"1"}) + file_verifier = mocker.AsyncMock(return_value={"500"}) mocker.patch.dict( verification._VERIFIERS, {"note": note_verifier, "file": file_verifier}, @@ -1101,7 +1104,7 @@ async def test_verify_search_results_dispatches_per_doc_type_concurrently(mocker kept, dropped_count = await verify_search_results(client, results) - assert {(r.id, r.doc_type) for r in kept} == {(1, "note"), (500, "file")} + assert {(r.id, r.doc_type) for r in kept} == {("1", "note"), ("500", "file")} assert dropped_count == 1 note_verifier.assert_awaited_once() file_verifier.assert_awaited_once() diff --git a/tests/unit/test_chunk_context_offset_gate.py b/tests/unit/test_chunk_context_offset_gate.py index 8c7f85cc..c8e6dedd 100644 --- a/tests/unit/test_chunk_context_offset_gate.py +++ b/tests/unit/test_chunk_context_offset_gate.py @@ -56,7 +56,7 @@ class TestOffsetFallbackGate: result = await get_chunk_with_context( nc_client=mock_nc_client, user_id="alice", - doc_id=12345, + doc_id="12345", doc_type="file", chunk_start=0, chunk_end=100, @@ -101,7 +101,7 @@ class TestOffsetFallbackGate: await get_chunk_with_context( nc_client=mock_nc_client, user_id="alice", - doc_id=42, + doc_id="42", doc_type="note", chunk_start=0, chunk_end=10, @@ -133,7 +133,7 @@ class TestOffsetFallbackGate: await get_chunk_with_context( nc_client=mock_nc_client, user_id="alice", - doc_id=12345, + doc_id="12345", doc_type="file", chunk_start=0, chunk_end=100, @@ -175,7 +175,7 @@ class TestNullableChunkIndexPropagation: result = await get_chunk_with_context( nc_client=mock_nc_client, user_id="alice", - doc_id=42, + doc_id="42", doc_type="note", chunk_start=100, chunk_end=200, @@ -222,7 +222,7 @@ class TestNullableChunkIndexPropagation: result = await get_chunk_with_context( nc_client=mock_nc_client, user_id="alice", - doc_id=42, + doc_id="42", doc_type="note", chunk_start=0, chunk_end=10, @@ -264,7 +264,7 @@ class TestNullableChunkIndexPropagation: result = await get_chunk_with_context( nc_client=mock_nc_client, user_id="alice", - doc_id=42, + doc_id="42", doc_type="note", chunk_start=100, chunk_end=200, @@ -307,7 +307,7 @@ class TestAdjacentChunkBoundary: result = await get_chunk_with_context( nc_client=mock_nc_client, user_id="alice", - doc_id=42, + doc_id="42", doc_type="note", chunk_start=0, chunk_end=10, @@ -348,7 +348,7 @@ class TestAdjacentChunkBoundary: result = await get_chunk_with_context( nc_client=mock_nc_client, user_id="alice", - doc_id=42, + doc_id="42", doc_type="note", chunk_start=0, chunk_end=10, From 8f4f5c007956e8fbea2bb4e16aaee3e54be925b9 Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Sun, 10 May 2026 17:50:23 +0200 Subject: [PATCH 25/27] =?UTF-8?q?fix(vector):=20address=20PR=20review=20ro?= =?UTF-8?q?und=2016=20=E2=80=94=20type-aware=20index=20check,=20comments?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Detect pre-existing payload indexes with the wrong schema type in `_ensure_payload_indexes`. The previous "field already in existing_schema → skip" branch silently survived a collection migrated from the int-doc_id era where `doc_id` is indexed as INTEGER, letting `MatchValue(value="123")` searches keep failing with HTTP 400 on Qdrant Cloud strict mode — exactly the production failure this PR was meant to fix. New behaviour: compare `existing_schema[field].data_type` against the declared type; on mismatch log a WARNING and append to `failed_fields` so the consolidated end-of-function summary picks it up. No auto-repair (operator intervention only — see docs/configuration.md recovery procedure). New test exercises the doc_id-INTEGER scenario end-to-end and asserts both the per-field WARNING and the summary line. Clarify the `_verify_news_items` malformed-doc_id rationale: the news API has no per-item endpoint, so a malformed doc_id genuinely cannot be verified against the source of truth. We err toward false-positive (keep) over false-negative (drop) — same conservative posture as `_verify_notes` and `_verify_deck_cards`. The producer-side validation is the real security boundary; the verifier is defence-in-depth. Both the inline comment and the WARNING message now spell this out. Add a TODO in `get_last_indexed_timestamp` flagging the O(N) cost on every incremental sync tick. The previous single-page `limit=10_000` silently bounded the scroll; paginating fixed correctness but made the unbounded cost visible. The follow-up tracker (canonical TODO at `api/visualization.py`) covers migrating the max-`indexed_at` to a sentinel point or collection metadata for O(1) lookup. Consolidate the duplicate non-numeric-doc_type TODOs at `api/visualization.py:508` and `auth/viz_routes.py:570` into a single canonical comment in `visualization.py`; `viz_routes.py` is reduced to a back-reference. Removes the rot risk of "fixed in one place, forgotten in the other." The canonical comment also references the O(1) timestamp follow-up in `scanner.py`. Document the `batch_size = 256` (qdrant_client.py) vs `_DELETION_TRACKING_PAGE_SIZE = 1024` (scanner.py) split with cross-referencing comments at each site: the smaller batch is for the read-write backfill upsert path (Qdrant accepts ~256-point chunks comfortably); the larger page is for read-only deletion-tracking scrolls where no per-page write round-trip applies. Replace `assert qdrant_client is not None` in `scan_user_documents` with `cast(AsyncQdrantClient, qdrant_client)` plus an explanatory comment. `assert` is silently elided under `-O`; `cast` is the conventional zero-cost narrower for branches the type checker can't infer from the surrounding `if not initial_sync` ternary. Co-Authored-By: Claude Opus 4.7 (1M context) --- nextcloud_mcp_server/api/visualization.py | 12 ++++-- nextcloud_mcp_server/auth/viz_routes.py | 14 +++---- nextcloud_mcp_server/search/verification.py | 12 +++++- nextcloud_mcp_server/vector/qdrant_client.py | 34 ++++++++++++++-- nextcloud_mcp_server/vector/scanner.py | 23 ++++++++++- tests/unit/vector/test_qdrant_client.py | 43 ++++++++++++++++++++ 6 files changed, 121 insertions(+), 17 deletions(-) diff --git a/nextcloud_mcp_server/api/visualization.py b/nextcloud_mcp_server/api/visualization.py index 370d4ef4..c3a9b790 100644 --- a/nextcloud_mcp_server/api/visualization.py +++ b/nextcloud_mcp_server/api/visualization.py @@ -505,9 +505,15 @@ async def get_chunk_context(request: Request) -> JSONResponse: # ints from MySQL auto_increment; doc_id stays a str downstream # (Qdrant payload index is keyword-typed). is_valid_nextcloud_doc_id # rejects "0", leading zeros, and Unicode digits that pass isdigit(). - # TODO: when chunk-context support extends to non-numeric doc_types - # (calendar VEVENT UIDs, CardDAV hrefs, …), relax this gate or make - # it doc_type-aware. Today every indexed doc_type is numeric. + # + # Canonical TODO (referenced by ``auth/viz_routes.py`` and + # ``vector/scanner.py:get_last_indexed_timestamp``): when chunk- + # context support extends to non-numeric doc_types (calendar VEVENT + # UIDs, CardDAV hrefs, …), relax this gate or make it doc_type- + # aware. Today every indexed doc_type is numeric. The follow-up + # tracker also covers the O(N) → O(1) migration of + # ``get_last_indexed_timestamp`` (currently re-scans every + # ``indexed_at`` on each tick). if not is_valid_nextcloud_doc_id(doc_id): return JSONResponse( { diff --git a/nextcloud_mcp_server/auth/viz_routes.py b/nextcloud_mcp_server/auth/viz_routes.py index f73717b9..1c6ad627 100644 --- a/nextcloud_mcp_server/auth/viz_routes.py +++ b/nextcloud_mcp_server/auth/viz_routes.py @@ -561,15 +561,11 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse: assert start_str is not None assert end_str is not None - # Validate doc_id at the handler boundary: a malformed doc_id would - # otherwise pass through to get_chunk_with_context and bottom out as a - # 404 from deep inside, not a clear 400. Nextcloud IDs are unsigned - # ints from MySQL auto_increment; doc_id stays a str downstream - # (Qdrant payload index is keyword-typed). is_valid_nextcloud_doc_id - # rejects "0", leading zeros, and Unicode digits that pass isdigit(). - # TODO: when chunk-context support extends to non-numeric doc_types - # (calendar VEVENT UIDs, CardDAV hrefs, …), relax this gate or make - # it doc_type-aware. Today every indexed doc_type is numeric. + # Same numeric-doc_id gate as ``api/visualization.py`` — see the + # canonical TODO and rationale there. Kept in sync so both + # OAuth-protected and direct-access handlers reject malformed + # IDs at the boundary instead of bottoming out as a 404 from + # deep inside ``get_chunk_with_context``. if not is_valid_nextcloud_doc_id(doc_id): return JSONResponse( { diff --git a/nextcloud_mcp_server/search/verification.py b/nextcloud_mcp_server/search/verification.py index ca8f5c0e..480f3515 100644 --- a/nextcloud_mcp_server/search/verification.py +++ b/nextcloud_mcp_server/search/verification.py @@ -392,8 +392,18 @@ async def _verify_news_items( # SearchResult.id is always str (Qdrant payload doc_id is keyword- # indexed; producers stringify on write). Pass through verbatim. if not is_valid_nextcloud_doc_id(d): + # The news API has no per-item endpoint, so a malformed doc_id + # cannot be verified against the source of truth. Err toward + # false-positive (keep in results) over false-negative (drop a + # potentially legitimate result) — matches the same conservative + # posture _verify_notes and _verify_deck_cards take for + # non-numeric IDs. The producer-side validation is the real + # security boundary; the verifier is defence-in-depth. logger.warning( - "Malformed news_item doc_id %r in verifier; keeping (cannot verify)", + "Malformed news_item doc_id %r in verifier; keeping to " + "avoid dropping a potentially legitimate result (news API " + "has no per-item endpoint, so cannot verify against source " + "of truth — false-positive preferred over false-negative)", d, ) accessible.add(d) diff --git a/nextcloud_mcp_server/vector/qdrant_client.py b/nextcloud_mcp_server/vector/qdrant_client.py index 0263cb92..b205cf46 100644 --- a/nextcloud_mcp_server/vector/qdrant_client.py +++ b/nextcloud_mcp_server/vector/qdrant_client.py @@ -182,9 +182,31 @@ async def _ensure_payload_indexes( failed_fields: list[str] = [] for field, schema_type in _PAYLOAD_INDEX_FIELDS.items(): if field in existing_schema: - # Index already present — silent skip. Logging here on every - # restart would be noise that hides the genuinely interesting - # "first-time creation" line in _create_one_payload_index. + # Index already present. Confirm the existing schema type matches + # what we'd create — a pre-existing collection with `doc_id` + # indexed as INTEGER (the bug this PR fixes) would otherwise + # silently survive here, and searches using + # MatchValue(value="123") would keep failing with HTTP 400 on + # Qdrant Cloud strict mode. Compare via PayloadSchemaType + # equality; PayloadIndexInfo.data_type is the same enum + # we wrote with. + existing_info = existing_schema[field] + existing_type = getattr(existing_info, "data_type", None) + if existing_type is not None and existing_type != schema_type: + logger.warning( + "Payload index on '%s' has wrong schema type " + "(got %s, expected %s); searches filtering on this " + "field will fail with HTTP 400 until the index is " + "dropped and recreated. See docs/configuration.md " + "for the recovery procedure.", + field, + getattr(existing_type, "name", existing_type), + schema_type.name, + ) + failed_fields.append(field) + # Either way, skip the create call: a matching index needs no + # work, and a mismatch must not be auto-repaired (operator + # intervention only — see docs/configuration.md). continue if not await _create_one_payload_index( client, collection_name, field, schema_type @@ -338,6 +360,12 @@ async def _backfill_doc_id_to_string( # Qdrant scroll returns next_offset as PointId | None — keep it untyped here # so the qdrant client's full union (UUID/int/str/PointId) flows through. next_offset = None + # Smaller than ``_DELETION_TRACKING_PAGE_SIZE = 1024`` in + # ``vector/scanner.py`` because this is a read-write path: every batch + # is followed by a ``set_payload`` upsert, and 256-point upserts are + # the working size where Qdrant comfortably accepts writes without + # timing out under load. The scanner-side scroll has no per-page write + # round-trip, so it can use a larger page. batch_size = 256 # Log progress every N batches so a long-running migration on a large # collection (≥ 50k points) doesn't look like a startup hang. At batch diff --git a/nextcloud_mcp_server/vector/scanner.py b/nextcloud_mcp_server/vector/scanner.py index 0f2204e4..c4a2e6a2 100644 --- a/nextcloud_mcp_server/vector/scanner.py +++ b/nextcloud_mcp_server/vector/scanner.py @@ -9,6 +9,7 @@ import random import time from dataclasses import dataclass from email.utils import parsedate_to_datetime +from typing import cast import anyio from anyio.abc import TaskStatus @@ -49,6 +50,13 @@ INDEXED_DOC_TYPES: frozenset[str] = frozenset( # < 100 k point per (user_id, doc_type) case. The previous single-page # ``limit=10_000`` silently truncated deletion sets for any user past the # cap, so anything indexed beyond the first 10 k was never reconciled. +# +# Intentionally larger than the ``batch_size = 256`` used by +# ``_backfill_doc_id_to_string`` in ``vector/qdrant_client.py``: this is a +# read-only scroll that just collects payloads (no write round-trip per +# point), so the per-page memory budget is the only relevant constraint. +# The 256 there is sized for read-write upsert batches where Qdrant +# accepts ~256-point chunks comfortably without timing out under load. _DELETION_TRACKING_PAGE_SIZE: int = 1024 @@ -120,6 +128,14 @@ async def get_last_indexed_timestamp(user_id: str) -> int | None: Returns: Unix timestamp of most recently indexed note, or None if no notes indexed yet """ + # TODO: This is O(N) over a user's indexed notes on every incremental + # sync tick. Was accidentally bounded at 10 k before this PR (single- + # page scroll silently truncated); paginating fixed correctness but + # made the unbounded cost visible. Track the max ``indexed_at`` as + # collection metadata or a dedicated sentinel point so this becomes + # O(1). Out of scope for the current PR — see the chunk-context / + # vector-sync follow-up tracker (referenced by the canonical TODO at + # ``api/visualization.py``). try: qdrant_client = await get_qdrant_client() @@ -263,7 +279,12 @@ async def scan_user_documents( qdrant_client = await get_qdrant_client() if not initial_sync else None indexed_doc_ids = set() if not initial_sync: - assert qdrant_client is not None # narrow for the type checker + # ``assert ... is not None`` would also narrow but raises an + # opaque AssertionError under ``-O`` and at runtime — ``cast`` + # is the conventional zero-cost narrower for branches the type + # checker can't infer from the surrounding ``if not + # initial_sync`` (the ternary above ties the two together). + qdrant_client = cast(AsyncQdrantClient, qdrant_client) points = await _scroll_all_points( qdrant_client, collection_name=get_settings().get_collection_name(), diff --git a/tests/unit/vector/test_qdrant_client.py b/tests/unit/vector/test_qdrant_client.py index 30b71338..15bdc18c 100644 --- a/tests/unit/vector/test_qdrant_client.py +++ b/tests/unit/vector/test_qdrant_client.py @@ -158,6 +158,49 @@ async def test_ensure_payload_indexes_skips_fields_already_indexed(mocker, caplo assert not any("doc_id" in m for m in info_messages), info_messages +@pytest.mark.unit +async def test_ensure_payload_indexes_warns_on_wrong_schema_type(mocker, caplog): + """Pre-existing index with wrong schema type surfaces as a WARNING. + + The bug this PR fixes: a collection migrated from the int-doc_id era + can have ``doc_id`` indexed as INTEGER, which silently survives the + "field already in schema → skip" branch and lets ``MatchValue(value="123")`` + keep failing with HTTP 400 on Qdrant Cloud strict mode. Confirm the + type-aware check fires a WARNING, marks the field as failed (so the + consolidated end-of-function summary picks it up), and does NOT + attempt to recreate the index — operator intervention is the only + safe path. + """ + client = mocker.AsyncMock() + # PayloadIndexInfo-like stand-in: only ``data_type`` is read. + wrong = SimpleNamespace(data_type=PayloadSchemaType.INTEGER) + client.get_collection.return_value = SimpleNamespace( + payload_schema={"doc_id": wrong} + ) + + with caplog.at_level("WARNING", logger="nextcloud_mcp_server.vector.qdrant_client"): + await _ensure_payload_indexes(client, "test-collection") + + # No create attempt for the mismatched field. + created_fields = { + c.kwargs["field_name"] for c in client.create_payload_index.await_args_list + } + assert "doc_id" not in created_fields + + warning_messages = [ + r.getMessage() for r in caplog.records if r.levelname == "WARNING" + ] + # Per-field warning describes both observed and expected types. + assert any( + "doc_id" in m and "INTEGER" in m and "KEYWORD" in m for m in warning_messages + ), warning_messages + # Consolidated summary at end of function includes the field too. + assert any( + "Payload index creation incomplete" in m and "doc_id" in m + for m in warning_messages + ), warning_messages + + @pytest.mark.unit async def test_ensure_payload_indexes_logs_400_as_warning(mocker, caplog): """Any 400 from create_payload_index is logged at WARNING and skipped. From 8246d9a08867004f7967773b6861780d9717ecd4 Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Sun, 10 May 2026 18:33:51 +0200 Subject: [PATCH 26/27] fix(vector): address PR review round 17 + local-mode collection-creation regression MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 17 reviewer (🟡 Important): 1. docs/configuration.md degraded-migration runbook said `doc_id backfill failed on …` but the actual log line in qdrant_client.py:415 is `doc_id backfill scroll failed on …`. Operators grepping the runbook string would have missed it. Insert the `scroll` qualifier. 2. _create_one_payload_index returned True on the 400 schema-conflict path, so a wrong-type index discovered at create time skipped the consolidated `Payload index creation incomplete` summary — but a wrong-type index discovered via the existing-schema check at line 195-206 did fire it. Tenants whose payload_schema is hidden from their JWT (Qdrant Cloud collection-scoped tokens) only ever observe the create-time path, so they never saw the operator-level summary. Return False so the summary fires in both cases. 3. docs/configuration.md said the upgrade-time delay was `proportional to point count while writes are issued` — overstating the cost. Writes are proportional to int-typed points only; the scroll itself is proportional to total point count. Reword. Local-mode collection-creation regression (root-cause of failing single-user / login-flow / multi-user-basic CI jobs): PR #779 changed the existence probe in get_qdrant_client from collection_exists() (returned bool in both modes) to get_collection() + except UnexpectedResponse(status_code=404). The HTTP-mode client raises UnexpectedResponse with a 404 body, but the local/in-memory client raises ValueError(f"Collection {name} not found") — see qdrant_client/local/async_qdrant_local.py. The narrow except clause let the ValueError propagate, app.py's lifespan re-raised as RuntimeError, and the mcp container crashed on first start. Catch ValueError too, with a `not found` substring guard so genuine programming bugs (bad collection_name, etc.) still surface. Tests: extend the existing 400-path test to assert the new failed_fields contract; add two get_qdrant_client unit tests pinning the local-mode VE catch (positive case + propagation case). Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/configuration.md | 7 +- nextcloud_mcp_server/vector/qdrant_client.py | 25 ++- tests/unit/vector/test_qdrant_client.py | 166 +++++++++++++++++-- 3 files changed, 184 insertions(+), 14 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 8ca27842..a1879596 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -341,8 +341,9 @@ server runs two idempotent migrations: legacy integer `doc_id` payloads to strings so they match the keyword index. Idempotent: on a clean collection (all `doc_id` values already `str`), the scroll runs but emits zero writes. On the first start after - the upgrade, expect a delay proportional to point count while writes - are issued. + the upgrade, expect a delay proportional to total point count for the + scroll itself, plus an additional delay proportional to any `int`-typed + `doc_id` points found while their payloads are rewritten. Both steps emit INFO-level log lines so operators can track progress. @@ -363,7 +364,7 @@ Both steps emit INFO-level log lines so operators can track progress. > the index was not created. Searches filtering on that field will keep > returning HTTP 400 (`Index required but not found`) until a subsequent > restart succeeds in creating it. -> - `doc_id backfill failed on ''; will retry on next restart` — +> - `doc_id backfill scroll failed on ''; will retry on next restart` — > the migration sentinel was not written. Legacy integer `doc_id` > payloads remain invisible to the keyword index in the meantime; the > scroll re-runs from scratch on the next process start. diff --git a/nextcloud_mcp_server/vector/qdrant_client.py b/nextcloud_mcp_server/vector/qdrant_client.py index 2c82bafb..e1e745af 100644 --- a/nextcloud_mcp_server/vector/qdrant_client.py +++ b/nextcloud_mcp_server/vector/qdrant_client.py @@ -111,7 +111,15 @@ async def _create_one_payload_index( logger.warning( "Schema conflict on payload index '%s': %s", field, body_text ) - return True + # Treat schema conflict the same as a wrong-type index + # discovered via `existing_schema` in `_ensure_payload_indexes` + # (lines 195-206) — both are "the index present is not the + # one we'd build", so the consolidated `Payload index + # creation incomplete` summary should fire in both cases. + # Without this, tenants whose payload_schema is hidden from + # their JWT (Qdrant Cloud collection-scoped tokens) would + # only see the per-field WARNING and miss the summary. + return False logger.error( "Unexpected error creating payload index on '%s' (status %s): %s", field, @@ -580,6 +588,21 @@ async def get_qdrant_client() -> AsyncQdrantClient: if exc.status_code != 404: raise logger.debug(f"Collection '{collection_name}' not found (404).") + except ValueError as exc: + # Local/in-memory qdrant_client raises ValueError(f"Collection + # {name} not found") instead of UnexpectedResponse — see + # qdrant_client/local/async_qdrant_local.py. Match on the + # message rather than catching every ValueError so genuine + # programming bugs (bad collection_name validation, etc.) + # still propagate. PR #779 introduced this regression by + # switching the existence probe from `collection_exists()` + # (which returned a bool in both modes) to `get_collection` + # without accounting for the local-mode signalling + # convention; the failing single-user / login-flow / + # multi-user-basic CI jobs all exercise this path. + if "not found" not in str(exc): + raise + logger.debug(f"Collection '{collection_name}' not found (local mode).") if collection_info is not None: # Collection exists - validate dimensions diff --git a/tests/unit/vector/test_qdrant_client.py b/tests/unit/vector/test_qdrant_client.py index 15bdc18c..cb0f5236 100644 --- a/tests/unit/vector/test_qdrant_client.py +++ b/tests/unit/vector/test_qdrant_client.py @@ -23,12 +23,14 @@ import pytest from qdrant_client.http.exceptions import UnexpectedResponse from qdrant_client.models import PayloadSchemaType +from nextcloud_mcp_server.vector import qdrant_client as qdrant_module from nextcloud_mcp_server.vector.qdrant_client import ( _DOC_ID_BACKFILL_SENTINEL_ID, _PAYLOAD_INDEX_FIELDS, _backfill_doc_id_to_string, _ensure_payload_indexes, _group_int_doc_ids, + get_qdrant_client, ) @@ -203,12 +205,17 @@ async def test_ensure_payload_indexes_warns_on_wrong_schema_type(mocker, caplog) @pytest.mark.unit async def test_ensure_payload_indexes_logs_400_as_warning(mocker, caplog): - """Any 400 from create_payload_index is logged at WARNING and skipped. + """A 400 from create_payload_index logs WARNING *and* fires the summary. Real Qdrant returns 200 when the index already exists with a matching schema, so 400s indicate a genuine problem (e.g., schema conflict on a pre-existing index). The loop continues past the failure so the - remaining fields still get indexed. + remaining fields still get indexed, *and* the field accumulates into + ``failed_fields`` so the consolidated `Payload index creation + incomplete` summary fires — without this, tenants whose + ``payload_schema`` is hidden from their JWT (Qdrant Cloud + collection-scoped tokens) would only see the per-field warning and + miss the operator-level summary that `wrong_schema_type` paths emit. """ client = mocker.AsyncMock() client.get_collection.return_value = _empty_collection_info() @@ -227,14 +234,21 @@ async def test_ensure_payload_indexes_logs_400_as_warning(mocker, caplog): # Loop continued past the failing field; every field was attempted. assert client.create_payload_index.await_count == len(_PAYLOAD_INDEX_FIELDS) - warnings = [r for r in caplog.records if r.levelname == "WARNING"] - # 400s do not contribute to the partial-failure summary (which fires - # only for non-400 errors), so this is the per-field warning, not the - # summary. Match the message prefix exactly so a future change adding - # 400s to the summary would surface here as a count mismatch. - assert len(warnings) == 1 - assert warnings[0].getMessage().startswith("Schema conflict on payload index") - assert "different schema" in warnings[0].getMessage() + warning_messages = [ + r.getMessage() for r in caplog.records if r.levelname == "WARNING" + ] + # Per-field warning describes the schema conflict. + assert any( + m.startswith("Schema conflict on payload index") and "different schema" in m + for m in warning_messages + ), warning_messages + # Consolidated summary names the failed field too — see the docstring + # for why this matters in tenant-scoped Qdrant Cloud setups. + first_field = next(iter(_PAYLOAD_INDEX_FIELDS)) + assert any( + "Payload index creation incomplete" in m and first_field in m + for m in warning_messages + ), warning_messages @pytest.mark.unit @@ -842,3 +856,135 @@ async def test_ensure_payload_indexes_summarises_failed_fields(mocker, caplog): assert "chunk_end_offset" in summary[0] assert "user_id" not in summary[0] # The one that succeeded. assert "test-collection" in summary[0] + + +# --------------------------------------------------------------------------- +# get_qdrant_client — collection-existence probe across modes +# --------------------------------------------------------------------------- + + +@pytest.fixture +def reset_qdrant_singleton(): + """Reset the module-level singleton + init lock around each test. + + ``get_qdrant_client`` short-circuits on a non-None ``_qdrant_client`` + via the unsynchronized fast path, so any prior test that initialised + the singleton would mask the cold-start logic these tests exercise. + Restore the original after the test so a leak doesn't bleed into + later tests in the same process. + """ + original_client = qdrant_module._qdrant_client + original_lock = qdrant_module._qdrant_init_lock + qdrant_module._qdrant_client = None + qdrant_module._qdrant_init_lock = None + yield + qdrant_module._qdrant_client = original_client + qdrant_module._qdrant_init_lock = original_lock + + +def _stub_provisional(mocker, get_collection_side_effect): + """Build a fake AsyncQdrantClient suitable for cold-start get_qdrant_client. + + ``get_collection`` is wired up to ``get_collection_side_effect``; + every other awaited method returns an AsyncMock so the migration + helpers (``_ensure_payload_indexes`` etc.) don't blow up on the + create-collection path. Returns the mock so tests can assert against + the awaited methods. + """ + provisional = mocker.AsyncMock() + provisional.get_collection.side_effect = get_collection_side_effect + # _ensure_payload_indexes pulls payload_schema off the freshly-created + # collection's get_collection result; on the create path it's passed + # an explicit {} so this branch isn't exercised, but make it safe + # anyway in case the order of init shifts. + provisional.create_payload_index.return_value = None + return provisional + + +def _stub_settings_and_embedding(mocker, monkeypatch): + """Replace get_settings and the embedding service with deterministic stubs.""" + from nextcloud_mcp_server.config import Settings + + settings = Settings( + qdrant_location=":memory:", + ollama_embedding_model="nomic-embed-text", + vector_sync_enabled=False, + ) + monkeypatch.setattr( + "nextcloud_mcp_server.vector.qdrant_client.get_settings", lambda: settings + ) + + embedding_service = mocker.Mock() + # No _detect_dimension attribute → the dynamic-detection branch is + # skipped. Real Ollama provider has it, but tests don't need to. + embedding_service.provider = mocker.Mock(spec_set=[]) + embedding_service.get_dimension = lambda: 4 + monkeypatch.setattr( + "nextcloud_mcp_server.embedding.get_embedding_service", + lambda: embedding_service, + ) + return settings + + +@pytest.mark.unit +async def test_get_qdrant_client_creates_collection_on_local_mode_value_error( + mocker, monkeypatch, reset_qdrant_singleton +): + """Local-mode `ValueError("Collection X not found")` must trigger create. + + The local/in-memory ``AsyncQdrantClient`` raises ``ValueError`` (see + ``qdrant_client/local/async_qdrant_local.py``) where the HTTP-mode + client would raise ``UnexpectedResponse(status_code=404)``. Both must + be treated as "the collection doesn't exist yet — create it." + Without this dual-path catch, the ``mcp`` container fails on first + start with `Failed to initialize Qdrant collection: Collection X not + found` and the ``app.py`` lifespan re-raises as ``RuntimeError``, + crashing every single-user / login-flow / multi-user-basic CI job. + """ + settings = _stub_settings_and_embedding(mocker, monkeypatch) + collection_name = settings.get_collection_name() + + provisional = _stub_provisional( + mocker, ValueError(f"Collection {collection_name} not found") + ) + monkeypatch.setattr( + "nextcloud_mcp_server.vector.qdrant_client.AsyncQdrantClient", + lambda *a, **kw: provisional, + ) + + client = await get_qdrant_client() + + assert client is provisional + provisional.create_collection.assert_awaited_once() + # The created collection should be the auto-generated name from + # settings — guards against accidental collection-name drift. + assert ( + provisional.create_collection.await_args.kwargs["collection_name"] + == collection_name + ) + + +@pytest.mark.unit +async def test_get_qdrant_client_propagates_unrelated_value_error( + mocker, monkeypatch, reset_qdrant_singleton +): + """A ValueError that is *not* a missing-collection signal must propagate. + + The ``except ValueError`` clause in ``get_qdrant_client`` matches on + the ``"not found"`` substring rather than catching every + ``ValueError`` so genuine programming bugs (bad ``collection_name`` + validation, dimension assertions, etc.) still surface to the caller. + Loosening the guard to a bare ``except ValueError`` would silently + treat any of those as "create the collection" and mask the bug. + """ + _stub_settings_and_embedding(mocker, monkeypatch) + provisional = _stub_provisional(mocker, ValueError("Bad collection_name")) + monkeypatch.setattr( + "nextcloud_mcp_server.vector.qdrant_client.AsyncQdrantClient", + lambda *a, **kw: provisional, + ) + + with pytest.raises(ValueError, match="Bad collection_name"): + await get_qdrant_client() + + provisional.create_collection.assert_not_awaited() From f3a66cf6bb1247d1f13b54a281766d4783dc5c24 Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Sun, 10 May 2026 18:37:27 +0200 Subject: [PATCH 27/27] chore: ruff format --- nextcloud_mcp_server/server/contacts.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nextcloud_mcp_server/server/contacts.py b/nextcloud_mcp_server/server/contacts.py index f495f6fd..ffa35bcf 100644 --- a/nextcloud_mcp_server/server/contacts.py +++ b/nextcloud_mcp_server/server/contacts.py @@ -197,7 +197,11 @@ def configure_contacts_tools(mcp: FastMCP): hay_parts: list[str] = [] if contact.fn: hay_parts.append(contact.fn.lower()) - nickname = contact.custom_fields.get("nickname") if contact.custom_fields else None + nickname = ( + contact.custom_fields.get("nickname") + if contact.custom_fields + else None + ) if nickname: hay_parts.append(str(nickname).lower()) for e in contact.emails: