fix(vector): address PR review round 11 — broaden offset-skip gate, clarify ordering

- search/context.py: drop the doc_type=='file' guard on skip_offset_lookup
  so notes / deck cards / news items also bypass the unindexed offset
  fallback when chunk_index is available. Legacy chunk_index=None data
  still uses the offset path.
- vector/qdrant_client.py: clarify the backfill/_ensure_payload_indexes
  ordering invariant (backfill rewrites payload values only, never schema
  or indexes). Acknowledge OSS-vs-Cloud uncertainty in the 400-branch
  comment and the new-collection call-site comment.
- vector/scanner.py: hoist qdrant_client to function scope so the
  file-scroll block doesn't depend on a name bound inside the
  notes-scroll block.
- tests/unit/test_chunk_context_offset_gate.py: flip the note-with-
  chunk_index test to assert the offset fallback is skipped.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
2026-05-09 17:59:41 +02:00
co-authored by Claude Opus 4.7
parent 47c531969f
commit f3ce46da0f
4 changed files with 55 additions and 21 deletions
+22 -6
View File
@@ -132,9 +132,14 @@ async def _ensure_payload_indexes(
body = getattr(e, "content", b"") or b""
body_text = body.decode("utf-8", errors="replace")
# 400 is the expected schema-conflict path (index already exists
# with a different type). 5xx is unexpected — keep the loop going
# so the remaining fields still get attempted, but log at error
# so operators see it.
# with a different type). Verified for Qdrant OSS, where an
# idempotent re-create against a matching schema returns 200; if
# Qdrant Cloud diverges and returns 400 for benign re-creates,
# the WARNING below will fire on every restart against an
# already-indexed collection — read the response body before
# treating that as a real schema conflict. 5xx is unexpected —
# keep the loop going so the remaining fields still get
# attempted, but log at error so operators see it.
if e.status_code == 400:
logger.warning(
"Schema conflict on payload index '%s': %s", field, body_text
@@ -534,7 +539,10 @@ async def get_qdrant_client() -> AsyncQdrantClient:
# payload-index work. Backfill before creating the index so the
# index covers every point. Pass the already-fetched
# collection_info.payload_schema through to avoid a redundant
# get_collection round-trip on every restart.
# get_collection round-trip on every restart — safe because
# _backfill_doc_id_to_string only rewrites payload *values*,
# never schema or indexes, so the snapshot remains accurate
# across the backfill call.
await _backfill_doc_id_to_string(
_qdrant_client, collection_name, expected_dimension
)
@@ -575,8 +583,16 @@ async def get_qdrant_client() -> AsyncQdrantClient:
f" Distance: COSINE\n"
f"Background sync will index all documents with dense + sparse vectors."
)
# Freshly created collection has no payload schema yet; pass {}
# explicitly to skip the otherwise-redundant get_collection call.
# Freshly created collection has no payload schema yet; pass
# {} explicitly to skip the otherwise-redundant
# get_collection call. Every field in _PAYLOAD_INDEX_FIELDS
# then goes through create_payload_index; on a brand-new
# collection none of them exist yet, so the WARNING in the
# 400-handler should *never* fire on this path. If it does
# on Qdrant Cloud first-start, that points at a
# deployment-level issue (race with a concurrent creator,
# implicit auto-indexes, etc.) worth investigating before
# suppressing.
await _ensure_payload_indexes(
_qdrant_client, collection_name, existing_schema={}
)
+9 -2
View File
@@ -210,10 +210,16 @@ async def scan_user_documents(
)
# For deletion tracking, get all doc_ids in Qdrant (for incremental sync)
# Note: We no longer bulk-query indexed_at, instead check per-document
# Note: We no longer bulk-query indexed_at, instead check per-document.
# Hoisted to function scope so the file-scroll block below doesn't
# depend on a name bound inside the notes-scroll block; future
# refactors that add an early return between the two blocks would
# otherwise hit an UnboundLocalError. get_qdrant_client is a
# singleton call, so the cost is identical.
qdrant_client = await get_qdrant_client() if not initial_sync else None
indexed_doc_ids = set()
if not initial_sync:
qdrant_client = await get_qdrant_client()
assert qdrant_client is not None # narrow for the type checker
scroll_result = await qdrant_client.scroll(
collection_name=get_settings().get_collection_name(),
scroll_filter=Filter(
@@ -387,6 +393,7 @@ async def scan_user_documents(
# Get indexed file IDs from Qdrant (for deletion tracking)
indexed_file_ids = set()
if not initial_sync:
assert qdrant_client is not None # narrow for the type checker
file_scroll_result = await qdrant_client.scroll(
collection_name=settings.get_collection_name(),
scroll_filter=Filter(