fix(vector-sync): address round-4 review — processor test, partial eviction

- tests: cover the process_document consent gate (drops an admin-disabled
  index task with record_ingest_dropped("admin_disabled"); allows approved)
- scanner: _consent_backstop_done is now an insertion-ordered dict and evicts
  the oldest entries to half capacity on overflow, so a bound hit re-fires the
  backstop for only the oldest markers instead of the whole fleet at once
- semantic: reword the short-circuit log (consent, not installation)
- capabilities: comment why move_to_end is needed after an expired-key update
- test: assert the global purge delete-filter is owner-agnostic (doc_type only);
  fix a pre-existing ty error on UnexpectedResponse(headers=None) in the file

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
2026-06-16 01:33:18 +02:00
co-authored by Claude Opus 4.8
parent cef477b877
commit 6b9f910a14
5 changed files with 98 additions and 10 deletions
+15 -8
View File
@@ -291,13 +291,14 @@ _TEXT_BACKSTOP_DOC_TYPES: tuple[str, ...] = tuple(sorted(INDEXED_DOC_TYPES - {"f
# already been enqueued, so a *standing* admin-disable doesn't re-flood the
# processor with idempotent deletes on every scan tick. An entry is cleared once
# the type is allowed again, so a later re-disable re-triggers the backstop.
_consent_backstop_done: set[tuple[str, str]] = set()
# A dict (not a set) so it stays insertion-ordered for oldest-first eviction.
_consent_backstop_done: dict[tuple[str, str], None] = {}
# Safety bound on the tracking set so a long-running multi-tenant process with
# Safety bound on the tracking dict so a long-running multi-tenant process with
# heavy user churn (deprovisioned users leave stale entries) can't grow it
# without limit. At <= len(INDEXED_DOC_TYPES) entries per user this is generous;
# on overflow we clear the whole set, which at worst re-fires the (idempotent)
# backstop once for currently-disabled types.
# on overflow we evict the *oldest* entries down to half capacity (not a full
# clear) so the backstop re-fires for only those, avoiding a fleet-wide burst.
_CONSENT_BACKSTOP_MAX = 50_000
@@ -321,7 +322,7 @@ async def _enqueue_deletes_for_disabled_types(
# re-triggers the backstop.
for doc_type in _TEXT_BACKSTOP_DOC_TYPES:
if doc_type in allowed:
_consent_backstop_done.discard((user_id, doc_type))
_consent_backstop_done.pop((user_id, doc_type), None)
# Disabled types not yet backstopped this episode.
disabled = [
@@ -374,12 +375,18 @@ async def _enqueue_deletes_for_disabled_types(
# Mark this (user, doc_type) backstopped for the current disable episode
# so subsequent scans don't re-enqueue the same idempotent deletes.
if len(_consent_backstop_done) >= _CONSENT_BACKSTOP_MAX:
# Evict oldest-first down to half capacity (insertion-ordered dict),
# so overflow re-fires the backstop for only the oldest markers
# rather than the whole fleet at once.
overage = len(_consent_backstop_done) - _CONSENT_BACKSTOP_MAX // 2
logger.info(
"consent backstop tracking set hit %d entries; clearing",
"consent backstop tracking hit %d entries; evicting %d oldest",
_CONSENT_BACKSTOP_MAX,
overage,
)
_consent_backstop_done.clear()
_consent_backstop_done.add((user_id, doc_type))
for stale_key in list(_consent_backstop_done)[:overage]:
del _consent_backstop_done[stale_key]
_consent_backstop_done[(user_id, doc_type)] = None
return queued