Merge remote-tracking branch 'origin/master' into feat/decomp-hook-points

# Conflicts:
#	nextcloud_mcp_server/vector/scanner.py
This commit is contained in:
Chris Coutinho
2026-05-29 18:31:05 +02:00
40 changed files with 2070 additions and 1804 deletions
+36 -3
View File
@@ -43,6 +43,38 @@ class NotProvisionedError(Exception):
pass
# Process-wide app-password storage for the BasicAuth client path.
#
# get_user_client_basic_auth is on the search hot path (Unified Search and the
# /api/v1 viz endpoints call it per request). Creating a fresh
# RefreshTokenStorage and running ``initialize()`` — a full Alembic upgrade in
# a worker thread — on every call is both wasteful and unsafe: concurrent
# upgrades race on Alembic's non-thread-safe module-global EnvironmentContext
# proxy, surfacing as ``KeyError: 'script'``. Cache one initialized instance,
# guarded by a lock so the one-time migration runs exactly once. The lock is
# created lazily inside an async context (anyio primitives must not be built at
# import time — trio compatibility), mirroring vector/qdrant_client.py.
_basic_auth_storage: "RefreshTokenStorage | None" = None
_basic_auth_storage_lock: anyio.Lock | None = None
async def _get_initialized_basic_auth_storage() -> "RefreshTokenStorage":
"""Return the process-wide, already-initialized app-password storage."""
global _basic_auth_storage, _basic_auth_storage_lock
if _basic_auth_storage is not None:
return _basic_auth_storage
# Safe under cooperative scheduling: no await between the None-check and the
# assignment, so two coroutines cannot both create a lock.
if _basic_auth_storage_lock is None:
_basic_auth_storage_lock = anyio.Lock()
async with _basic_auth_storage_lock:
if _basic_auth_storage is None:
storage = RefreshTokenStorage.from_env()
await storage.initialize()
_basic_auth_storage = storage
return _basic_auth_storage
@dataclass
class UserSyncState:
"""State for a single user's scanner task."""
@@ -74,10 +106,11 @@ async def get_user_client_basic_auth(
Raises:
NotProvisionedError: If user has not provisioned an app password
"""
# Get or create storage instance
# Get or create storage instance. Reuse a process-wide initialized instance
# rather than building one (and running an Alembic upgrade) per call — see
# _get_initialized_basic_auth_storage for why (hot path + Alembic race).
if storage is None:
storage = RefreshTokenStorage.from_env()
await storage.initialize()
storage = await _get_initialized_basic_auth_storage()
# Retrieve app password from local storage
app_password = await storage.get_app_password(user_id)
+10
View File
@@ -717,6 +717,16 @@ async def _index_document(
},
payload={
"user_id": doc_task.user_id,
# owner_id is the UID of the file's owner — what
# search-time ACL expansion filters on. Today the scanner
# always runs as the file's owner (per-user crawl, only
# surfaces files the user owns or that fall under their
# WebDAV root), so owner_id == user_id is correct for
# every doc type indexed here. The fields are kept
# separate so a future indexer change that lets a user
# crawl shared-with-them content can set owner_id to the
# true owner without losing the "who indexed this" trail.
"owner_id": doc_task.owner_id or doc_task.user_id,
"doc_id": doc_task.doc_id,
"doc_type": doc_task.doc_type,
"is_placeholder": False, # Real indexed document (not placeholder)
@@ -38,6 +38,15 @@ logger = logging.getLogger(__name__)
_PAYLOAD_INDEX_FIELDS: dict[str, PayloadSchemaType] = {
"doc_id": PayloadSchemaType.KEYWORD,
"user_id": PayloadSchemaType.KEYWORD,
# owner_id is the ACL-aware filter field: every search applies
# MatchAny(key="owner_id", any=accessible_owners) (see
# search/access_filter.py). Without a keyword index Qdrant full-scans the
# collection to evaluate it — invisible at small scale, but a latency
# regression at tens of thousands of points and an HTTP 400 on Qdrant
# Cloud strict payload-validation mode. Mirrors the user_id treatment;
# _ensure_payload_indexes is idempotent so existing collections migrate
# at startup without operator intervention.
"owner_id": PayloadSchemaType.KEYWORD,
"doc_type": PayloadSchemaType.KEYWORD,
"is_placeholder": PayloadSchemaType.BOOL,
"chunk_index": PayloadSchemaType.INTEGER,
+6
View File
@@ -113,6 +113,12 @@ class DocumentTask:
# when it is None (deletes, or sources whose etag isn't threaded). Harmless
# in local mode — the in-process processor reads its own etag.
etag: str | None = None
# UID of the true owner of the indexed object, used by the search-time
# ACL filter. None today (scanner always runs as the owner, so the
# processor falls back to user_id), but settable so a future
# shared-with-me crawl can pass through the actual owner without
# reshaping the payload contract.
owner_id: str | None = None
# Track documents potentially deleted (grace period before actual deletion)