fix(vector): address PR review — wait=True backfill, batched writes, search helper
Addresses reviewer feedback on PR #773: - Backfill set_payload now uses wait=True to avoid a race where _ensure_keyword_payload_indexes builds the KEYWORD index before fire-and-forget writes have committed, leaving int payloads invisible to filters. - Batch points sharing the same int doc_id into a single set_payload call (one document → many chunks → one round-trip instead of N). - Drop _has_int_doc_id_sample short-circuit. The sample's false-negative window (clean first 256 results, ints further in) is gone; full scroll is the dominant cost on first run anyway. - Simplify _ensure_keyword_payload_indexes: the "already exists" 400 branch was dead code (Qdrant returns 200 on identical re-create); any 400 now logs a warning and continues. - search/context.py: comment the broadened file-type guard. Add explicit not doc_id.isdigit() checks at the top of note/news_item/deck_card branches in _fetch_document_text so malformed payloads surface as warnings instead of being swallowed by the broad except. Also extracts build_search_result_from_point into search/algorithms.py to deduplicate the 71-line payload-extraction loop shared by SemanticSearchAlgorithm and BM25HybridSearchAlgorithm. This fixes SonarQube's quality-gate failure (4.0% new-code duplication, max 3%). Test coverage: - 7 new unit tests for build_search_result_from_point covering missing payload, note/file/deck_card metadata, int doc_id coercion, and metadata_extras merging. - Replace _has_int_doc_id_sample tests with clean-collection no-op and per-batch grouping tests. - Update set_payload assertions from wait=False to wait=True. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
719b3b5034
commit
6aba589a6e
@@ -5,7 +5,7 @@ from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Protocol, runtime_checkable
|
||||
|
||||
from qdrant_client.models import FieldCondition, Filter, MatchValue
|
||||
from qdrant_client.models import FieldCondition, Filter, MatchValue, ScoredPoint
|
||||
|
||||
from nextcloud_mcp_server.config import get_settings
|
||||
from nextcloud_mcp_server.vector.placeholder import get_placeholder_filter
|
||||
@@ -181,6 +181,70 @@ class SearchResult:
|
||||
raise ValueError(f"Score must be non-negative, got {self.score}")
|
||||
|
||||
|
||||
def build_search_result_from_point(
|
||||
point: ScoredPoint,
|
||||
*,
|
||||
metadata_extras: dict[str, Any] | None = None,
|
||||
) -> SearchResult | None:
|
||||
"""Construct a SearchResult from a Qdrant ScoredPoint payload.
|
||||
|
||||
Returns ``None`` when the payload is missing — callers should skip the
|
||||
point. The defensive ``str()`` coercion on ``doc_id`` covers legacy int
|
||||
payloads until the startup backfill has run everywhere (see
|
||||
``vector/qdrant_client.py:_backfill_doc_id_to_string``).
|
||||
|
||||
Args:
|
||||
point: A Qdrant ``ScoredPoint`` from a search response.
|
||||
metadata_extras: Algorithm-specific metadata merged into the result's
|
||||
``metadata`` dict (e.g., ``{"search_method": "bm25_hybrid_rrf"}``).
|
||||
|
||||
Returns:
|
||||
A populated ``SearchResult``, or ``None`` if ``point.payload`` is
|
||||
missing.
|
||||
"""
|
||||
if point.payload is None:
|
||||
return None
|
||||
|
||||
doc_id = str(point.payload["doc_id"])
|
||||
doc_type = point.payload.get("doc_type", "note")
|
||||
|
||||
metadata: dict[str, Any] = {
|
||||
"chunk_index": point.payload.get("chunk_index"),
|
||||
"total_chunks": point.payload.get("total_chunks"),
|
||||
}
|
||||
if metadata_extras:
|
||||
metadata.update(metadata_extras)
|
||||
|
||||
# File-specific metadata for PDF viewer
|
||||
if doc_type == "file" and (path := point.payload.get("file_path")):
|
||||
metadata["path"] = path
|
||||
|
||||
# Deck-card metadata for frontend URL construction and verify-on-read
|
||||
# (ADR-019) — both board_id and stack_id are required to call
|
||||
# deck.get_card without an O(boards × stacks) iteration fallback.
|
||||
if doc_type == "deck_card":
|
||||
if board_id := point.payload.get("board_id"):
|
||||
metadata["board_id"] = board_id
|
||||
if stack_id := point.payload.get("stack_id"):
|
||||
metadata["stack_id"] = stack_id
|
||||
|
||||
return SearchResult(
|
||||
id=doc_id,
|
||||
doc_type=doc_type,
|
||||
title=point.payload.get("title", "Untitled"),
|
||||
excerpt=point.payload.get("excerpt", ""),
|
||||
score=point.score,
|
||||
metadata=metadata,
|
||||
chunk_start_offset=point.payload.get("chunk_start_offset"),
|
||||
chunk_end_offset=point.payload.get("chunk_end_offset"),
|
||||
page_number=point.payload.get("page_number"),
|
||||
page_count=point.payload.get("page_count"),
|
||||
chunk_index=point.payload.get("chunk_index", 0),
|
||||
total_chunks=point.payload.get("total_chunks", 1),
|
||||
point_id=str(point.id),
|
||||
)
|
||||
|
||||
|
||||
class SearchAlgorithm(ABC):
|
||||
"""Abstract base class for search algorithms.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user