fix(vector): self-heal stale app passwords on auth failure
Deleted/disabled Nextcloud users left their app_passwords row in storage, so user_manager_task re-spawned their scanner every poll interval only to 401 again — an endless re-spawn/auth-failure loop (observed on tenant-blackbox-demo: ~534 respawns/3h, matching the 60s poll interval). - Delete the stored app password on a hard 401/403 in user_scanner_task (both the pre-validation and in-scan-loop paths), breaking the re-spawn loop at the source so the user-manager stops recreating the scanner. - Add a periodic credential_cleanup_task backstop (hourly) that sweeps cleanup_invalid_app_passwords for anything the per-scanner path misses. - Run the startup cleanup for all deployment modes: drop the stale `not oauth_enabled` guard so login_flow tenants (the cloud default) are covered. NOTE: login_flow startup now makes one concurrent OCS validation call per stored user before readiness. Refs Deck #198. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
60df141442
commit
3790cf6d60
@@ -125,6 +125,34 @@ class UserSyncState:
|
||||
started_at: float = field(default_factory=time.time)
|
||||
|
||||
|
||||
async def _remove_stale_credential(user_id: str, status_code: int) -> None:
|
||||
"""Delete a user's stored app password after a hard auth failure.
|
||||
|
||||
A 401/403 from Nextcloud means the stored app password is no longer usable —
|
||||
the user was deleted, disabled, or revoked the password. Without removing it
|
||||
the credential lingers in storage and ``user_manager_task`` re-spawns this
|
||||
scanner on its next poll (the user is still "provisioned"), only to fail auth
|
||||
again — an endless re-spawn/401 loop that keeps hammering Nextcloud (Deck
|
||||
#198). Deleting the row drops the user out of
|
||||
``get_all_app_password_user_ids()`` so the scanner is not recreated.
|
||||
|
||||
Best-effort: a storage failure here is logged, not raised — the periodic
|
||||
``credential_cleanup_task`` sweep (and the next startup sweep) are backstops.
|
||||
"""
|
||||
try:
|
||||
storage = await _get_initialized_basic_auth_storage()
|
||||
if await storage.delete_app_password(user_id):
|
||||
logger.info(
|
||||
"[BasicAuth] Removed stale app password for %s after HTTP %s",
|
||||
user_id,
|
||||
status_code,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"[BasicAuth] Failed to remove stale app password for %s: %s", user_id, e
|
||||
)
|
||||
|
||||
|
||||
async def get_user_client_basic_auth(
|
||||
user_id: str,
|
||||
nextcloud_host: str,
|
||||
@@ -215,10 +243,12 @@ async def user_scanner_task(
|
||||
except HTTPStatusError as e:
|
||||
if e.response.status_code in (401, 403):
|
||||
logger.warning(
|
||||
"[BasicAuth] Credential validation failed for %s (HTTP %s), not starting scan loop",
|
||||
"[BasicAuth] Credential validation failed for %s (HTTP %s), "
|
||||
"removing stale credential and not starting scan loop",
|
||||
user_id,
|
||||
e.response.status_code,
|
||||
)
|
||||
await _remove_stale_credential(user_id, e.response.status_code)
|
||||
return
|
||||
raise
|
||||
finally:
|
||||
@@ -262,10 +292,12 @@ async def user_scanner_task(
|
||||
status_code = e.response.status_code
|
||||
if status_code in (401, 403):
|
||||
logger.warning(
|
||||
"[BasicAuth] Scanner auth failed for %s (HTTP %s), stopping scanner. User may need to re-provision credentials.",
|
||||
"[BasicAuth] Scanner auth failed for %s (HTTP %s), removing stale "
|
||||
"credential and stopping scanner. User must re-provision to resume sync.",
|
||||
user_id,
|
||||
status_code,
|
||||
)
|
||||
await _remove_stale_credential(user_id, status_code)
|
||||
break
|
||||
elif status_code == 429:
|
||||
retry_after = min(int(e.response.headers.get("Retry-After", "60")), 300)
|
||||
@@ -410,6 +442,63 @@ async def multi_user_processor_task(
|
||||
oauth_processor_task = multi_user_processor_task
|
||||
|
||||
|
||||
# Backstop sweep cadence for cleanup_invalid_app_passwords (seconds). The
|
||||
# per-scanner 401 deletion (_remove_stale_credential) is the primary self-heal;
|
||||
# this periodic sweep is defense-in-depth for credentials whose scanner never
|
||||
# ran or whose in-scanner deletion failed. One hour keeps the per-user OCS
|
||||
# validation load negligible.
|
||||
CREDENTIAL_CLEANUP_INTERVAL = 3600
|
||||
|
||||
|
||||
async def credential_cleanup_task(
|
||||
storage: "RefreshTokenStorage",
|
||||
shutdown_event: anyio.Event,
|
||||
nextcloud_host: str,
|
||||
*,
|
||||
task_status: TaskStatus = anyio.TASK_STATUS_IGNORED,
|
||||
) -> None:
|
||||
"""Periodically remove app passwords that no longer authenticate.
|
||||
|
||||
Backstop for the per-scanner self-heal (``_remove_stale_credential``):
|
||||
validates every stored app password against Nextcloud and deletes the ones
|
||||
that return 401/403, so credentials for deleted/disabled users cannot
|
||||
accumulate even if their scanner never ran (Deck #198). Runs on a fixed
|
||||
cadence until ``shutdown_event`` is set. A fresh sweep already runs once at
|
||||
startup (app lifespan), so this sleeps first.
|
||||
"""
|
||||
logger.info(
|
||||
"[BasicAuth] Credential cleanup task started (interval: %ss)",
|
||||
CREDENTIAL_CLEANUP_INTERVAL,
|
||||
)
|
||||
task_status.started()
|
||||
|
||||
while not shutdown_event.is_set():
|
||||
# Sleep first — startup already swept; wake early on shutdown.
|
||||
try:
|
||||
with anyio.move_on_after(CREDENTIAL_CLEANUP_INTERVAL):
|
||||
await shutdown_event.wait()
|
||||
except anyio.get_cancelled_exc_class():
|
||||
break
|
||||
if shutdown_event.is_set():
|
||||
break
|
||||
|
||||
try:
|
||||
removed = await storage.cleanup_invalid_app_passwords(nextcloud_host)
|
||||
if removed:
|
||||
logger.info(
|
||||
"[BasicAuth] Periodic cleanup removed %s stale app password(s): %s",
|
||||
len(removed),
|
||||
removed,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"[BasicAuth] Periodic credential cleanup failed (non-fatal): %s",
|
||||
format_exception_group(e),
|
||||
)
|
||||
|
||||
logger.info("[BasicAuth] Credential cleanup task stopped")
|
||||
|
||||
|
||||
async def _run_user_scanner_with_scope(
|
||||
user_id: str,
|
||||
cancel_scope: anyio.CancelScope,
|
||||
|
||||
Reference in New Issue
Block a user