fix(vector-sync): address PR review — dict guard, symmetric backstop, metrics
- purge route: 400 (not 500) on a valid-JSON non-object body
- scanner: backstop-purge admin-disabled note/news_item/deck_card points
(their deletion-tracking lives inside the skipped scan_* fns), mirroring the
files path; gated on a concrete allow-set so fail-open never deletes
- processor: record_ingest_dropped("admin_disabled") so consent-skipped index
tasks are observable/alertable
- app.py: list /api/v1/vector-sync/purge in the endpoints log line
- capabilities: drop empty-string doc types; return frozenset throughout
- purge: document the count-before-delete approximation
- tests: non-object body -> 400, ProvisioningRequiredError -> 428, cache TTL
expiry refetch, and the scanner consent backstop
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
ef5b3f3873
commit
477fb02b0a
@@ -65,6 +65,12 @@ async def purge_doc_types_route(request: Request) -> JSONResponse:
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
if not isinstance(body, dict):
|
||||
return JSONResponse(
|
||||
{"error": "Bad request", "message": "body must be a JSON object"},
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
raw = body.get("doc_types")
|
||||
if not isinstance(raw, list) or not all(isinstance(d, str) for d in raw):
|
||||
return JSONResponse(
|
||||
@@ -115,7 +121,7 @@ async def purge_doc_types_route(request: Request) -> JSONResponse:
|
||||
status_code=428,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Error purging doc types for user %s: %s", user_id, e)
|
||||
logger.exception("Error purging doc types for user %s", user_id)
|
||||
return JSONResponse(
|
||||
{
|
||||
"error": "Internal error",
|
||||
|
||||
@@ -2454,7 +2454,7 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
|
||||
"/api/v1/users/{user_id}/app-password, /api/v1/users/{user_id}/access, "
|
||||
"/api/v1/users/{user_id}/scopes, /api/v1/scopes, "
|
||||
"/api/v1/vector-viz/search, /api/v1/search, /api/v1/apps, "
|
||||
"/api/v1/webhooks, /api/v1/pdf-preview"
|
||||
"/api/v1/webhooks, /api/v1/vector-sync/purge, /api/v1/pdf-preview"
|
||||
)
|
||||
|
||||
# Note: Metrics endpoint is NOT exposed on main HTTP port for security reasons.
|
||||
|
||||
@@ -38,13 +38,13 @@ class _CapabilitiesClientProtocol(Protocol):
|
||||
async def capabilities(self) -> Any: ...
|
||||
|
||||
|
||||
def _parse_enabled_doc_types(payload: Any) -> set[str] | None:
|
||||
def _parse_enabled_doc_types(payload: Any) -> frozenset[str] | None:
|
||||
"""Extract ``enabled_doc_types`` from an OCS capabilities payload.
|
||||
|
||||
Returns ``None`` when the ``astrolabe.semantic_search`` block is absent or
|
||||
malformed (treated as "no restriction"). Returns a set (possibly empty) when
|
||||
the block is present and well-formed; an empty set means the admin disabled
|
||||
every source.
|
||||
malformed (treated as "no restriction"). Returns a frozenset (possibly
|
||||
empty) when the block is present and well-formed; an empty set means the
|
||||
admin disabled every source.
|
||||
"""
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
@@ -63,7 +63,7 @@ def _parse_enabled_doc_types(payload: Any) -> set[str] | None:
|
||||
raw = semantic.get("enabled_doc_types")
|
||||
if not isinstance(raw, list):
|
||||
return None
|
||||
return {dt for dt in raw if isinstance(dt, str)}
|
||||
return frozenset(dt for dt in raw if isinstance(dt, str) and dt)
|
||||
|
||||
|
||||
async def allowed_doc_types(
|
||||
@@ -92,8 +92,7 @@ async def allowed_doc_types(
|
||||
)
|
||||
return None # don't cache failures — retry next call
|
||||
|
||||
parsed = _parse_enabled_doc_types(payload)
|
||||
result = frozenset(parsed) if parsed is not None else None
|
||||
result = _parse_enabled_doc_types(payload)
|
||||
_cache[user_id] = (now, result)
|
||||
_cache.move_to_end(user_id)
|
||||
while len(_cache) > _CACHE_MAXSIZE:
|
||||
|
||||
@@ -590,6 +590,9 @@ async def process_document(
|
||||
doc_task.doc_id,
|
||||
doc_task.user_id,
|
||||
)
|
||||
# Alertable counter so a flood of webhook events for a
|
||||
# disabled source is observable (not silently swallowed).
|
||||
record_ingest_dropped("admin_disabled")
|
||||
record_vector_sync_processing(time.time() - start_time, "skipped")
|
||||
return
|
||||
|
||||
|
||||
@@ -35,6 +35,11 @@ async def purge_doc_types(doc_types: list[str]) -> dict[str, int]:
|
||||
deletion). Each doc type is purged independently so a failure on one does
|
||||
not abort the rest; failures re-raise after the loop only if every doc type
|
||||
failed, otherwise partial progress is returned.
|
||||
|
||||
The count is taken just before the delete (two separate Qdrant calls), so
|
||||
it is approximate — a point indexed in the gap is deleted but not counted.
|
||||
This is acceptable: indexing of a disabled source is already gated upstream,
|
||||
so the window is effectively empty in practice.
|
||||
"""
|
||||
qdrant_client = await get_qdrant_client()
|
||||
collection = get_settings().get_collection_name()
|
||||
@@ -61,10 +66,9 @@ async def purge_doc_types(doc_types: list[str]) -> dict[str, int]:
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — record and continue
|
||||
last_error = exc
|
||||
logger.error(
|
||||
"Failed to purge indexed points for doc_type=%s: %s",
|
||||
logger.exception(
|
||||
"Failed to purge indexed points for doc_type=%s",
|
||||
doc_type,
|
||||
exc,
|
||||
)
|
||||
|
||||
if not purged and last_error is not None:
|
||||
|
||||
@@ -280,6 +280,75 @@ def _app_enabled(app_id: str, enabled_apps: set[str] | None) -> bool:
|
||||
return enabled_apps is None or app_id in enabled_apps
|
||||
|
||||
|
||||
# Text doc types whose deletion-tracking lives *inside* their scan_* function,
|
||||
# so skipping that function (when admin-disabled) leaves indexed points with no
|
||||
# grace-period backstop. ``file`` is intentionally excluded: its scan path
|
||||
# empties discovery and lets the existing reconcile loop purge on disable.
|
||||
_TEXT_BACKSTOP_DOC_TYPES: tuple[str, ...] = ("note", "news_item", "deck_card")
|
||||
|
||||
|
||||
async def _enqueue_deletes_for_disabled_types(
|
||||
user_id: str,
|
||||
send_stream: TaskProducer,
|
||||
allowed: frozenset[str] | None,
|
||||
scan_id: int,
|
||||
) -> int:
|
||||
"""Enqueue delete tasks for indexed text-source points the admin disabled.
|
||||
|
||||
Backstop for a failed eager purge: scrolls this user's indexed points for
|
||||
each admin-disallowed text doc_type and queues a delete. No-op when
|
||||
``allowed`` is ``None`` (fail-open — never delete on a transient capability
|
||||
read failure). Returns the number of delete tasks enqueued.
|
||||
"""
|
||||
if allowed is None:
|
||||
return 0
|
||||
|
||||
disabled = [dt for dt in _TEXT_BACKSTOP_DOC_TYPES if dt not in allowed]
|
||||
if not disabled:
|
||||
return 0
|
||||
|
||||
qdrant_client = await get_qdrant_client()
|
||||
collection = get_settings().get_collection_name()
|
||||
queued = 0
|
||||
for doc_type in disabled:
|
||||
points = await _scroll_all_points(
|
||||
qdrant_client,
|
||||
collection_name=collection,
|
||||
scroll_filter=Filter(
|
||||
must=[
|
||||
FieldCondition(key="user_id", match=MatchValue(value=user_id)),
|
||||
FieldCondition(key="doc_type", match=MatchValue(value=doc_type)),
|
||||
]
|
||||
),
|
||||
payload_fields=["doc_id"],
|
||||
)
|
||||
doc_ids = {
|
||||
str(p.payload["doc_id"])
|
||||
for p in points
|
||||
if p.payload is not None and "doc_id" in p.payload
|
||||
}
|
||||
if doc_ids:
|
||||
logger.info(
|
||||
"[SCAN-%s] %s disabled by admin for %s; enqueueing %d delete(s) (backstop)",
|
||||
scan_id,
|
||||
doc_type,
|
||||
user_id,
|
||||
len(doc_ids),
|
||||
)
|
||||
for doc_id in doc_ids:
|
||||
await send_stream.send(
|
||||
DocumentTask(
|
||||
user_id=user_id,
|
||||
doc_id=doc_id,
|
||||
doc_type=doc_type,
|
||||
operation="delete",
|
||||
modified_at=0,
|
||||
)
|
||||
)
|
||||
queued += 1
|
||||
return queued
|
||||
|
||||
|
||||
async def scan_user_documents(
|
||||
user_id: str,
|
||||
send_stream: TaskProducer,
|
||||
@@ -381,6 +450,16 @@ async def scan_user_documents(
|
||||
current_time = time.time()
|
||||
queued = 0
|
||||
|
||||
# Backstop purge for admin-disabled text sources. Their deletion-
|
||||
# tracking lives inside the scan_* function we skip below, so (unlike
|
||||
# files, whose discovery-empties-then-reconcile path purges on disable)
|
||||
# they'd linger if Astrolabe's eager purge failed. Enqueue deletes for
|
||||
# any indexed points of a now-disallowed type. Gated on a concrete
|
||||
# allow-set, so a fail-open None never triggers deletion.
|
||||
queued += await _enqueue_deletes_for_disabled_types(
|
||||
user_id, send_stream, allowed, scan_id
|
||||
)
|
||||
|
||||
if _app_enabled("notes", enabled_apps) and is_doc_type_allowed("note", allowed):
|
||||
try:
|
||||
queued += await scan_notes(
|
||||
|
||||
Reference in New Issue
Block a user