fix(vector-sync): address PR review — dict guard, symmetric backstop, metrics

- purge route: 400 (not 500) on a valid-JSON non-object body
- scanner: backstop-purge admin-disabled note/news_item/deck_card points
  (their deletion-tracking lives inside the skipped scan_* fns), mirroring the
  files path; gated on a concrete allow-set so fail-open never deletes
- processor: record_ingest_dropped("admin_disabled") so consent-skipped index
  tasks are observable/alertable
- app.py: list /api/v1/vector-sync/purge in the endpoints log line
- capabilities: drop empty-string doc types; return frozenset throughout
- purge: document the count-before-delete approximation
- tests: non-object body -> 400, ProvisioningRequiredError -> 428, cache TTL
  expiry refetch, and the scanner consent backstop

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
2026-06-16 00:54:50 +02:00
co-authored by Claude Opus 4.8
parent ef5b3f3873
commit 477fb02b0a
10 changed files with 258 additions and 43 deletions
+7 -1
View File
@@ -65,6 +65,12 @@ async def purge_doc_types_route(request: Request) -> JSONResponse:
status_code=400,
)
if not isinstance(body, dict):
return JSONResponse(
{"error": "Bad request", "message": "body must be a JSON object"},
status_code=400,
)
raw = body.get("doc_types")
if not isinstance(raw, list) or not all(isinstance(d, str) for d in raw):
return JSONResponse(
@@ -115,7 +121,7 @@ async def purge_doc_types_route(request: Request) -> JSONResponse:
status_code=428,
)
except Exception as e:
logger.error("Error purging doc types for user %s: %s", user_id, e)
logger.exception("Error purging doc types for user %s", user_id)
return JSONResponse(
{
"error": "Internal error",
+1 -1
View File
@@ -2454,7 +2454,7 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
"/api/v1/users/{user_id}/app-password, /api/v1/users/{user_id}/access, "
"/api/v1/users/{user_id}/scopes, /api/v1/scopes, "
"/api/v1/vector-viz/search, /api/v1/search, /api/v1/apps, "
"/api/v1/webhooks, /api/v1/pdf-preview"
"/api/v1/webhooks, /api/v1/vector-sync/purge, /api/v1/pdf-preview"
)
# Note: Metrics endpoint is NOT exposed on main HTTP port for security reasons.
+6 -7
View File
@@ -38,13 +38,13 @@ class _CapabilitiesClientProtocol(Protocol):
async def capabilities(self) -> Any: ...
def _parse_enabled_doc_types(payload: Any) -> set[str] | None:
def _parse_enabled_doc_types(payload: Any) -> frozenset[str] | None:
"""Extract ``enabled_doc_types`` from an OCS capabilities payload.
Returns ``None`` when the ``astrolabe.semantic_search`` block is absent or
malformed (treated as "no restriction"). Returns a set (possibly empty) when
the block is present and well-formed; an empty set means the admin disabled
every source.
malformed (treated as "no restriction"). Returns a frozenset (possibly
empty) when the block is present and well-formed; an empty set means the
admin disabled every source.
"""
if not isinstance(payload, dict):
return None
@@ -63,7 +63,7 @@ def _parse_enabled_doc_types(payload: Any) -> set[str] | None:
raw = semantic.get("enabled_doc_types")
if not isinstance(raw, list):
return None
return {dt for dt in raw if isinstance(dt, str)}
return frozenset(dt for dt in raw if isinstance(dt, str) and dt)
async def allowed_doc_types(
@@ -92,8 +92,7 @@ async def allowed_doc_types(
)
return None # don't cache failures — retry next call
parsed = _parse_enabled_doc_types(payload)
result = frozenset(parsed) if parsed is not None else None
result = _parse_enabled_doc_types(payload)
_cache[user_id] = (now, result)
_cache.move_to_end(user_id)
while len(_cache) > _CACHE_MAXSIZE:
+3
View File
@@ -590,6 +590,9 @@ async def process_document(
doc_task.doc_id,
doc_task.user_id,
)
# Alertable counter so a flood of webhook events for a
# disabled source is observable (not silently swallowed).
record_ingest_dropped("admin_disabled")
record_vector_sync_processing(time.time() - start_time, "skipped")
return
+7 -3
View File
@@ -35,6 +35,11 @@ async def purge_doc_types(doc_types: list[str]) -> dict[str, int]:
deletion). Each doc type is purged independently so a failure on one does
not abort the rest; failures re-raise after the loop only if every doc type
failed, otherwise partial progress is returned.
The count is taken just before the delete (two separate Qdrant calls), so
it is approximate — a point indexed in the gap is deleted but not counted.
This is acceptable: indexing of a disabled source is already gated upstream,
so the window is effectively empty in practice.
"""
qdrant_client = await get_qdrant_client()
collection = get_settings().get_collection_name()
@@ -61,10 +66,9 @@ async def purge_doc_types(doc_types: list[str]) -> dict[str, int]:
)
except Exception as exc: # noqa: BLE001 — record and continue
last_error = exc
logger.error(
"Failed to purge indexed points for doc_type=%s: %s",
logger.exception(
"Failed to purge indexed points for doc_type=%s",
doc_type,
exc,
)
if not purged and last_error is not None:
+79
View File
@@ -280,6 +280,75 @@ def _app_enabled(app_id: str, enabled_apps: set[str] | None) -> bool:
return enabled_apps is None or app_id in enabled_apps
# Text doc types whose deletion-tracking lives *inside* their scan_* function,
# so skipping that function (when admin-disabled) leaves indexed points with no
# grace-period backstop. ``file`` is intentionally excluded: its scan path
# empties discovery and lets the existing reconcile loop purge on disable.
_TEXT_BACKSTOP_DOC_TYPES: tuple[str, ...] = ("note", "news_item", "deck_card")
async def _enqueue_deletes_for_disabled_types(
user_id: str,
send_stream: TaskProducer,
allowed: frozenset[str] | None,
scan_id: int,
) -> int:
"""Enqueue delete tasks for indexed text-source points the admin disabled.
Backstop for a failed eager purge: scrolls this user's indexed points for
each admin-disallowed text doc_type and queues a delete. No-op when
``allowed`` is ``None`` (fail-open — never delete on a transient capability
read failure). Returns the number of delete tasks enqueued.
"""
if allowed is None:
return 0
disabled = [dt for dt in _TEXT_BACKSTOP_DOC_TYPES if dt not in allowed]
if not disabled:
return 0
qdrant_client = await get_qdrant_client()
collection = get_settings().get_collection_name()
queued = 0
for doc_type in disabled:
points = await _scroll_all_points(
qdrant_client,
collection_name=collection,
scroll_filter=Filter(
must=[
FieldCondition(key="user_id", match=MatchValue(value=user_id)),
FieldCondition(key="doc_type", match=MatchValue(value=doc_type)),
]
),
payload_fields=["doc_id"],
)
doc_ids = {
str(p.payload["doc_id"])
for p in points
if p.payload is not None and "doc_id" in p.payload
}
if doc_ids:
logger.info(
"[SCAN-%s] %s disabled by admin for %s; enqueueing %d delete(s) (backstop)",
scan_id,
doc_type,
user_id,
len(doc_ids),
)
for doc_id in doc_ids:
await send_stream.send(
DocumentTask(
user_id=user_id,
doc_id=doc_id,
doc_type=doc_type,
operation="delete",
modified_at=0,
)
)
queued += 1
return queued
async def scan_user_documents(
user_id: str,
send_stream: TaskProducer,
@@ -381,6 +450,16 @@ async def scan_user_documents(
current_time = time.time()
queued = 0
# Backstop purge for admin-disabled text sources. Their deletion-
# tracking lives inside the scan_* function we skip below, so (unlike
# files, whose discovery-empties-then-reconcile path purges on disable)
# they'd linger if Astrolabe's eager purge failed. Enqueue deletes for
# any indexed points of a now-disallowed type. Gated on a concrete
# allow-set, so a fail-open None never triggers deletion.
queued += await _enqueue_deletes_for_disabled_types(
user_id, send_stream, allowed, scan_id
)
if _app_enabled("notes", enabled_apps) and is_doc_type_allowed("note", allowed):
try:
queued += await scan_notes(