fix(vector): address PR review — wait=True backfill, batched writes, search helper
Addresses reviewer feedback on PR #773: - Backfill set_payload now uses wait=True to avoid a race where _ensure_keyword_payload_indexes builds the KEYWORD index before fire-and-forget writes have committed, leaving int payloads invisible to filters. - Batch points sharing the same int doc_id into a single set_payload call (one document → many chunks → one round-trip instead of N). - Drop _has_int_doc_id_sample short-circuit. The sample's false-negative window (clean first 256 results, ints further in) is gone; full scroll is the dominant cost on first run anyway. - Simplify _ensure_keyword_payload_indexes: the "already exists" 400 branch was dead code (Qdrant returns 200 on identical re-create); any 400 now logs a warning and continues. - search/context.py: comment the broadened file-type guard. Add explicit not doc_id.isdigit() checks at the top of note/news_item/deck_card branches in _fetch_document_text so malformed payloads surface as warnings instead of being swallowed by the broad except. Also extracts build_search_result_from_point into search/algorithms.py to deduplicate the 71-line payload-extraction loop shared by SemanticSearchAlgorithm and BM25HybridSearchAlgorithm. This fixes SonarQube's quality-gate failure (4.0% new-code duplication, max 3%). Test coverage: - 7 new unit tests for build_search_result_from_point covering missing payload, note/file/deck_card metadata, int doc_id coercion, and metadata_extras merging. - Replace _has_int_doc_id_sample tests with clean-collection no-op and per-batch grouping tests. - Update set_payload assertions from wait=False to wait=True. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
719b3b5034
commit
6aba589a6e
@@ -1,8 +1,22 @@
|
||||
"""Unit tests for SearchResult validation."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from nextcloud_mcp_server.search.algorithms import SearchResult
|
||||
from nextcloud_mcp_server.search.algorithms import (
|
||||
SearchResult,
|
||||
build_search_result_from_point,
|
||||
)
|
||||
|
||||
|
||||
def _make_point(point_id, payload, score=0.5):
|
||||
"""Stand-in for qdrant_client.models.ScoredPoint.
|
||||
|
||||
The helper only reads ``id``, ``payload``, and ``score`` — full Pydantic
|
||||
validation isn't required for unit tests.
|
||||
"""
|
||||
return SimpleNamespace(id=point_id, payload=payload, score=score)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@@ -133,3 +147,145 @@ def test_search_result_with_chunk_offsets():
|
||||
|
||||
assert result.chunk_start_offset == 100
|
||||
assert result.chunk_end_offset == 500
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# build_search_result_from_point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_build_search_result_from_point_returns_none_when_payload_missing():
|
||||
"""Helper signals the caller to skip the point by returning None."""
|
||||
point = _make_point(point_id="p1", payload=None)
|
||||
|
||||
assert build_search_result_from_point(point) is None
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_build_search_result_from_point_note_payload():
|
||||
"""Note-type payload populates the SearchResult fields without metadata extras."""
|
||||
point = _make_point(
|
||||
point_id="p-1",
|
||||
payload={
|
||||
"doc_id": "42",
|
||||
"doc_type": "note",
|
||||
"title": "Hello",
|
||||
"excerpt": "world",
|
||||
"chunk_start_offset": 0,
|
||||
"chunk_end_offset": 100,
|
||||
"chunk_index": 0,
|
||||
"total_chunks": 2,
|
||||
},
|
||||
score=0.91,
|
||||
)
|
||||
|
||||
sr = build_search_result_from_point(point)
|
||||
|
||||
assert sr is not None
|
||||
assert sr.id == "42"
|
||||
assert sr.doc_type == "note"
|
||||
assert sr.title == "Hello"
|
||||
assert sr.excerpt == "world"
|
||||
assert sr.score == 0.91
|
||||
assert sr.chunk_start_offset == 0
|
||||
assert sr.chunk_end_offset == 100
|
||||
assert sr.chunk_index == 0
|
||||
assert sr.total_chunks == 2
|
||||
assert sr.point_id == "p-1"
|
||||
assert sr.metadata == {"chunk_index": 0, "total_chunks": 2}
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_build_search_result_from_point_coerces_int_doc_id_to_str():
|
||||
"""Legacy int doc_id payloads are stringified defensively."""
|
||||
point = _make_point(
|
||||
point_id=1,
|
||||
payload={"doc_id": 7, "doc_type": "note"},
|
||||
score=0.5,
|
||||
)
|
||||
|
||||
sr = build_search_result_from_point(point)
|
||||
|
||||
assert sr is not None
|
||||
assert sr.id == "7"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_build_search_result_from_point_file_metadata_includes_path():
|
||||
"""File-type payloads with a file_path attach it under metadata['path']."""
|
||||
point = _make_point(
|
||||
point_id="p-2",
|
||||
payload={
|
||||
"doc_id": "100",
|
||||
"doc_type": "file",
|
||||
"file_path": "/Documents/report.pdf",
|
||||
"page_number": 3,
|
||||
"page_count": 12,
|
||||
},
|
||||
)
|
||||
|
||||
sr = build_search_result_from_point(point)
|
||||
|
||||
assert sr is not None
|
||||
assert sr.doc_type == "file"
|
||||
assert sr.metadata["path"] == "/Documents/report.pdf"
|
||||
assert sr.page_number == 3
|
||||
assert sr.page_count == 12
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_build_search_result_from_point_deck_card_metadata():
|
||||
"""Deck-card payloads carry board_id/stack_id forward for verify-on-read."""
|
||||
point = _make_point(
|
||||
point_id="p-3",
|
||||
payload={
|
||||
"doc_id": "55",
|
||||
"doc_type": "deck_card",
|
||||
"board_id": 7,
|
||||
"stack_id": 12,
|
||||
"title": "Card",
|
||||
},
|
||||
)
|
||||
|
||||
sr = build_search_result_from_point(point)
|
||||
|
||||
assert sr is not None
|
||||
assert sr.metadata["board_id"] == 7
|
||||
assert sr.metadata["stack_id"] == 12
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_build_search_result_from_point_merges_metadata_extras():
|
||||
"""metadata_extras override/augment the helper's computed metadata dict."""
|
||||
point = _make_point(
|
||||
point_id="p-4",
|
||||
payload={"doc_id": "1", "doc_type": "note"},
|
||||
)
|
||||
|
||||
sr = build_search_result_from_point(
|
||||
point, metadata_extras={"search_method": "bm25_hybrid_rrf"}
|
||||
)
|
||||
|
||||
assert sr is not None
|
||||
assert sr.metadata["search_method"] == "bm25_hybrid_rrf"
|
||||
# Common fields still present
|
||||
assert "chunk_index" in sr.metadata
|
||||
assert "total_chunks" in sr.metadata
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_build_search_result_from_point_defaults_when_optional_fields_missing():
|
||||
"""Missing optional payload keys fall back to documented defaults."""
|
||||
point = _make_point(point_id="p-5", payload={"doc_id": "1"})
|
||||
|
||||
sr = build_search_result_from_point(point)
|
||||
|
||||
assert sr is not None
|
||||
assert sr.doc_type == "note" # default doc_type
|
||||
assert sr.title == "Untitled"
|
||||
assert sr.excerpt == ""
|
||||
assert sr.chunk_index == 0
|
||||
assert sr.total_chunks == 1
|
||||
assert sr.chunk_start_offset is None
|
||||
assert sr.chunk_end_offset is None
|
||||
|
||||
Reference in New Issue
Block a user