fix(vector-sync): address round-4 review — processor test, partial eviction
- tests: cover the process_document consent gate (drops an admin-disabled
index task with record_ingest_dropped("admin_disabled"); allows approved)
- scanner: _consent_backstop_done is now an insertion-ordered dict and evicts
the oldest entries to half capacity on overflow, so a bound hit re-fires the
backstop for only the oldest markers instead of the whole fleet at once
- semantic: reword the short-circuit log (consent, not installation)
- capabilities: comment why move_to_end is needed after an expired-key update
- test: assert the global purge delete-filter is owner-agnostic (doc_type only);
fix a pre-existing ty error on UnexpectedResponse(headers=None) in the file
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
cef477b877
commit
6b9f910a14
@@ -291,13 +291,14 @@ _TEXT_BACKSTOP_DOC_TYPES: tuple[str, ...] = tuple(sorted(INDEXED_DOC_TYPES - {"f
|
||||
# already been enqueued, so a *standing* admin-disable doesn't re-flood the
|
||||
# processor with idempotent deletes on every scan tick. An entry is cleared once
|
||||
# the type is allowed again, so a later re-disable re-triggers the backstop.
|
||||
_consent_backstop_done: set[tuple[str, str]] = set()
|
||||
# A dict (not a set) so it stays insertion-ordered for oldest-first eviction.
|
||||
_consent_backstop_done: dict[tuple[str, str], None] = {}
|
||||
|
||||
# Safety bound on the tracking set so a long-running multi-tenant process with
|
||||
# Safety bound on the tracking dict so a long-running multi-tenant process with
|
||||
# heavy user churn (deprovisioned users leave stale entries) can't grow it
|
||||
# without limit. At <= len(INDEXED_DOC_TYPES) entries per user this is generous;
|
||||
# on overflow we clear the whole set, which at worst re-fires the (idempotent)
|
||||
# backstop once for currently-disabled types.
|
||||
# on overflow we evict the *oldest* entries down to half capacity (not a full
|
||||
# clear) so the backstop re-fires for only those, avoiding a fleet-wide burst.
|
||||
_CONSENT_BACKSTOP_MAX = 50_000
|
||||
|
||||
|
||||
@@ -321,7 +322,7 @@ async def _enqueue_deletes_for_disabled_types(
|
||||
# re-triggers the backstop.
|
||||
for doc_type in _TEXT_BACKSTOP_DOC_TYPES:
|
||||
if doc_type in allowed:
|
||||
_consent_backstop_done.discard((user_id, doc_type))
|
||||
_consent_backstop_done.pop((user_id, doc_type), None)
|
||||
|
||||
# Disabled types not yet backstopped this episode.
|
||||
disabled = [
|
||||
@@ -374,12 +375,18 @@ async def _enqueue_deletes_for_disabled_types(
|
||||
# Mark this (user, doc_type) backstopped for the current disable episode
|
||||
# so subsequent scans don't re-enqueue the same idempotent deletes.
|
||||
if len(_consent_backstop_done) >= _CONSENT_BACKSTOP_MAX:
|
||||
# Evict oldest-first down to half capacity (insertion-ordered dict),
|
||||
# so overflow re-fires the backstop for only the oldest markers
|
||||
# rather than the whole fleet at once.
|
||||
overage = len(_consent_backstop_done) - _CONSENT_BACKSTOP_MAX // 2
|
||||
logger.info(
|
||||
"consent backstop tracking set hit %d entries; clearing",
|
||||
"consent backstop tracking hit %d entries; evicting %d oldest",
|
||||
_CONSENT_BACKSTOP_MAX,
|
||||
overage,
|
||||
)
|
||||
_consent_backstop_done.clear()
|
||||
_consent_backstop_done.add((user_id, doc_type))
|
||||
for stale_key in list(_consent_backstop_done)[:overage]:
|
||||
del _consent_backstop_done[stale_key]
|
||||
_consent_backstop_done[(user_id, doc_type)] = None
|
||||
return queued
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user