fix(vector): address PR review round 3 — sentinel guard, skip indexed fields, narrow types
- Add a fixed-UUID sentinel point written after a successful doc_id backfill so subsequent restarts retrieve it and short-circuit the O(N) scroll. Sentinel has no user_id/doc_id/doc_type payload so production search filters never see it. - Pre-fetch payload_schema in _ensure_keyword_payload_indexes and silently skip fields that are already indexed; the "Created KEYWORD payload index" INFO log fires only on actual creation. - Narrow stale `int | str` doc_id annotations to `str` across search/verification.py (BatchVerifier return type, per-verifier accessible sets, by_type / accessible_by_type / inaccessible collections); drop the now-redundant `type(d).__name__` prefix in the dropped-docs log. - Align the backfill log message with the PR description's "Running doc_id backfill" promise; add a caller cross-reference to the wait=True comment. - Fix _get_file_path_from_qdrant docstring (file_id is str, not numeric). - Convert legacy `id=1` to `id="1"` in test_search_result.py to match the SearchResult.id: str annotation. Three new unit tests cover sentinel-found, sentinel-written, and skip-existing-index branches; existing backfill tests pass dimension and explicit retrieve.return_value=[] for the no-sentinel path. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
b5b4025bb4
commit
92b2d50cd7
@@ -151,7 +151,8 @@ async def _get_file_path_from_qdrant(
|
||||
|
||||
Args:
|
||||
user_id: User ID who owns the file
|
||||
file_id: Numeric file ID
|
||||
file_id: Stringified file ID (Qdrant payload value, post-doc_id
|
||||
normalization — see vector/qdrant_client.py)
|
||||
chunk_start: Character offset where chunk starts
|
||||
chunk_end: Character offset where chunk ends
|
||||
|
||||
|
||||
@@ -49,7 +49,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
BatchVerifier = Callable[
|
||||
[NextcloudClientProtocol, list[SearchResult], anyio.Semaphore],
|
||||
Awaitable[set[int | str]],
|
||||
Awaitable[set[str]],
|
||||
]
|
||||
"""(client, results, semaphore) -> set of doc_ids accessible to the user."""
|
||||
|
||||
@@ -75,9 +75,9 @@ async def _verify_notes(
|
||||
client: NextcloudClientProtocol,
|
||||
results: list[SearchResult],
|
||||
semaphore: anyio.Semaphore,
|
||||
) -> set[int | str]:
|
||||
) -> set[str]:
|
||||
# safe: cooperative concurrency, no lock needed (see verify_search_results)
|
||||
accessible: set[int | str] = set()
|
||||
accessible: set[str] = set()
|
||||
|
||||
async def check(result: SearchResult) -> None:
|
||||
doc_id = result.id
|
||||
@@ -130,9 +130,9 @@ async def _verify_files(
|
||||
client: NextcloudClientProtocol,
|
||||
results: list[SearchResult],
|
||||
semaphore: anyio.Semaphore,
|
||||
) -> set[int | str]:
|
||||
) -> set[str]:
|
||||
# safe: cooperative concurrency, no lock needed (see verify_search_results)
|
||||
accessible: set[int | str] = set()
|
||||
accessible: set[str] = set()
|
||||
|
||||
async def check(result: SearchResult) -> None:
|
||||
doc_id = result.id
|
||||
@@ -201,9 +201,9 @@ async def _verify_deck_cards(
|
||||
client: NextcloudClientProtocol,
|
||||
results: list[SearchResult],
|
||||
semaphore: anyio.Semaphore,
|
||||
) -> set[int | str]:
|
||||
) -> set[str]:
|
||||
# safe: cooperative concurrency, no lock needed (see verify_search_results)
|
||||
accessible: set[int | str] = set()
|
||||
accessible: set[str] = set()
|
||||
|
||||
async def check(result: SearchResult) -> None:
|
||||
doc_id = result.id
|
||||
@@ -283,7 +283,7 @@ async def _verify_news_items(
|
||||
client: NextcloudClientProtocol,
|
||||
results: list[SearchResult],
|
||||
semaphore: anyio.Semaphore,
|
||||
) -> set[int | str]:
|
||||
) -> set[str]:
|
||||
"""Batch-verify news items with a single fetch.
|
||||
|
||||
The Nextcloud News API has no per-item endpoint, so ``news.get_item`` is
|
||||
@@ -386,7 +386,7 @@ async def _verify_news_items(
|
||||
# for THAT item only — not the whole batch. Mirrors the per-item
|
||||
# shape of the notes/files/deck verifiers. See the granularity note
|
||||
# above for why this is narrower than the API-response failure path.
|
||||
accessible: set[int | str] = set()
|
||||
accessible: set[str] = set()
|
||||
for d in doc_ids:
|
||||
try:
|
||||
if int(d) in present_ids:
|
||||
@@ -474,7 +474,7 @@ async def verify_search_results(
|
||||
# deduplicated batch. We pick one SearchResult per (id, doc_type) to carry
|
||||
# metadata (path, board_id/stack_id) into the verifier — chunks of the
|
||||
# same document share these fields, so any chunk works.
|
||||
by_type: dict[str, dict[int | str, SearchResult]] = {}
|
||||
by_type: dict[str, dict[str, SearchResult]] = {}
|
||||
for r in results:
|
||||
by_type.setdefault(r.doc_type, {}).setdefault(r.id, r)
|
||||
|
||||
@@ -494,7 +494,7 @@ async def verify_search_results(
|
||||
# same write. Adding a lock would be dead weight; using ``anyio.Lock``
|
||||
# here would force serialization on a path that is intentionally
|
||||
# parallel.
|
||||
accessible_by_type: dict[str, set[int | str]] = {}
|
||||
accessible_by_type: dict[str, set[str]] = {}
|
||||
|
||||
async def run_verifier(doc_type: str, unique_results: list[SearchResult]) -> None:
|
||||
verifier = _VERIFIERS.get(doc_type)
|
||||
@@ -526,7 +526,7 @@ async def verify_search_results(
|
||||
tg.start_soon(run_verifier, doc_type, list(id_to_result.values()))
|
||||
|
||||
# Compute (doc_id, doc_type) pairs that failed verification
|
||||
inaccessible: set[tuple[int | str, str]] = set()
|
||||
inaccessible: set[tuple[str, str]] = set()
|
||||
for doc_type, id_to_result in by_type.items():
|
||||
# The .get() default is defensive only — run_verifier always populates
|
||||
# accessible_by_type[doc_type], either with the verifier's result or
|
||||
@@ -537,12 +537,10 @@ async def verify_search_results(
|
||||
inaccessible.add((doc_id, doc_type))
|
||||
|
||||
if inaccessible:
|
||||
# Tag ids with their type (int vs str) so ghost-record logs are
|
||||
# unambiguous: int 42 and str "42" both render as "42" otherwise.
|
||||
logger.info(
|
||||
"Verification dropped %d inaccessible document(s): %s",
|
||||
len(inaccessible),
|
||||
sorted((f"{type(d).__name__}:{d}", t) for d, t in inaccessible),
|
||||
sorted(inaccessible),
|
||||
)
|
||||
|
||||
# Filter results, preserving order. All chunks of an inaccessible document
|
||||
|
||||
Reference in New Issue
Block a user