fix(vector-sync): address PR review — dict guard, symmetric backstop, metrics

- purge route: 400 (not 500) on a valid-JSON non-object body
- scanner: backstop-purge admin-disabled note/news_item/deck_card points
  (their deletion-tracking lives inside the skipped scan_* fns), mirroring the
  files path; gated on a concrete allow-set so fail-open never deletes
- processor: record_ingest_dropped("admin_disabled") so consent-skipped index
  tasks are observable/alertable
- app.py: list /api/v1/vector-sync/purge in the endpoints log line
- capabilities: drop empty-string doc types; return frozenset throughout
- purge: document the count-before-delete approximation
- tests: non-object body -> 400, ProvisioningRequiredError -> 428, cache TTL
  expiry refetch, and the scanner consent backstop

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
2026-06-16 00:54:50 +02:00
co-authored by Claude Opus 4.8
parent ef5b3f3873
commit 477fb02b0a
10 changed files with 258 additions and 43 deletions
+3
View File
@@ -590,6 +590,9 @@ async def process_document(
doc_task.doc_id,
doc_task.user_id,
)
# Alertable counter so a flood of webhook events for a
# disabled source is observable (not silently swallowed).
record_ingest_dropped("admin_disabled")
record_vector_sync_processing(time.time() - start_time, "skipped")
return
+7 -3
View File
@@ -35,6 +35,11 @@ async def purge_doc_types(doc_types: list[str]) -> dict[str, int]:
deletion). Each doc type is purged independently so a failure on one does
not abort the rest; failures re-raise after the loop only if every doc type
failed, otherwise partial progress is returned.
The count is taken just before the delete (two separate Qdrant calls), so
it is approximate — a point indexed in the gap is deleted but not counted.
This is acceptable: indexing of a disabled source is already gated upstream,
so the window is effectively empty in practice.
"""
qdrant_client = await get_qdrant_client()
collection = get_settings().get_collection_name()
@@ -61,10 +66,9 @@ async def purge_doc_types(doc_types: list[str]) -> dict[str, int]:
)
except Exception as exc: # noqa: BLE001 — record and continue
last_error = exc
logger.error(
"Failed to purge indexed points for doc_type=%s: %s",
logger.exception(
"Failed to purge indexed points for doc_type=%s",
doc_type,
exc,
)
if not purged and last_error is not None:
+79
View File
@@ -280,6 +280,75 @@ def _app_enabled(app_id: str, enabled_apps: set[str] | None) -> bool:
return enabled_apps is None or app_id in enabled_apps
# Text doc types whose deletion-tracking lives *inside* their scan_* function,
# so skipping that function (when admin-disabled) leaves indexed points with no
# grace-period backstop. ``file`` is intentionally excluded: its scan path
# empties discovery and lets the existing reconcile loop purge on disable.
_TEXT_BACKSTOP_DOC_TYPES: tuple[str, ...] = ("note", "news_item", "deck_card")
async def _enqueue_deletes_for_disabled_types(
user_id: str,
send_stream: TaskProducer,
allowed: frozenset[str] | None,
scan_id: int,
) -> int:
"""Enqueue delete tasks for indexed text-source points the admin disabled.
Backstop for a failed eager purge: scrolls this user's indexed points for
each admin-disallowed text doc_type and queues a delete. No-op when
``allowed`` is ``None`` (fail-open — never delete on a transient capability
read failure). Returns the number of delete tasks enqueued.
"""
if allowed is None:
return 0
disabled = [dt for dt in _TEXT_BACKSTOP_DOC_TYPES if dt not in allowed]
if not disabled:
return 0
qdrant_client = await get_qdrant_client()
collection = get_settings().get_collection_name()
queued = 0
for doc_type in disabled:
points = await _scroll_all_points(
qdrant_client,
collection_name=collection,
scroll_filter=Filter(
must=[
FieldCondition(key="user_id", match=MatchValue(value=user_id)),
FieldCondition(key="doc_type", match=MatchValue(value=doc_type)),
]
),
payload_fields=["doc_id"],
)
doc_ids = {
str(p.payload["doc_id"])
for p in points
if p.payload is not None and "doc_id" in p.payload
}
if doc_ids:
logger.info(
"[SCAN-%s] %s disabled by admin for %s; enqueueing %d delete(s) (backstop)",
scan_id,
doc_type,
user_id,
len(doc_ids),
)
for doc_id in doc_ids:
await send_stream.send(
DocumentTask(
user_id=user_id,
doc_id=doc_id,
doc_type=doc_type,
operation="delete",
modified_at=0,
)
)
queued += 1
return queued
async def scan_user_documents(
user_id: str,
send_stream: TaskProducer,
@@ -381,6 +450,16 @@ async def scan_user_documents(
current_time = time.time()
queued = 0
# Backstop purge for admin-disabled text sources. Their deletion-
# tracking lives inside the scan_* function we skip below, so (unlike
# files, whose discovery-empties-then-reconcile path purges on disable)
# they'd linger if Astrolabe's eager purge failed. Enqueue deletes for
# any indexed points of a now-disallowed type. Gated on a concrete
# allow-set, so a fail-open None never triggers deletion.
queued += await _enqueue_deletes_for_disabled_types(
user_id, send_stream, allowed, scan_id
)
if _app_enabled("notes", enabled_apps) and is_doc_type_allowed("note", allowed):
try:
queued += await scan_notes(