Merge remote-tracking branch 'origin/master' into feat/decomp-hook-points
# Conflicts: # nextcloud_mcp_server/vector/scanner.py
This commit is contained in:
@@ -43,6 +43,38 @@ class NotProvisionedError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
# Process-wide app-password storage for the BasicAuth client path.
|
||||
#
|
||||
# get_user_client_basic_auth is on the search hot path (Unified Search and the
|
||||
# /api/v1 viz endpoints call it per request). Creating a fresh
|
||||
# RefreshTokenStorage and running ``initialize()`` — a full Alembic upgrade in
|
||||
# a worker thread — on every call is both wasteful and unsafe: concurrent
|
||||
# upgrades race on Alembic's non-thread-safe module-global EnvironmentContext
|
||||
# proxy, surfacing as ``KeyError: 'script'``. Cache one initialized instance,
|
||||
# guarded by a lock so the one-time migration runs exactly once. The lock is
|
||||
# created lazily inside an async context (anyio primitives must not be built at
|
||||
# import time — trio compatibility), mirroring vector/qdrant_client.py.
|
||||
_basic_auth_storage: "RefreshTokenStorage | None" = None
|
||||
_basic_auth_storage_lock: anyio.Lock | None = None
|
||||
|
||||
|
||||
async def _get_initialized_basic_auth_storage() -> "RefreshTokenStorage":
|
||||
"""Return the process-wide, already-initialized app-password storage."""
|
||||
global _basic_auth_storage, _basic_auth_storage_lock
|
||||
if _basic_auth_storage is not None:
|
||||
return _basic_auth_storage
|
||||
# Safe under cooperative scheduling: no await between the None-check and the
|
||||
# assignment, so two coroutines cannot both create a lock.
|
||||
if _basic_auth_storage_lock is None:
|
||||
_basic_auth_storage_lock = anyio.Lock()
|
||||
async with _basic_auth_storage_lock:
|
||||
if _basic_auth_storage is None:
|
||||
storage = RefreshTokenStorage.from_env()
|
||||
await storage.initialize()
|
||||
_basic_auth_storage = storage
|
||||
return _basic_auth_storage
|
||||
|
||||
|
||||
@dataclass
|
||||
class UserSyncState:
|
||||
"""State for a single user's scanner task."""
|
||||
@@ -74,10 +106,11 @@ async def get_user_client_basic_auth(
|
||||
Raises:
|
||||
NotProvisionedError: If user has not provisioned an app password
|
||||
"""
|
||||
# Get or create storage instance
|
||||
# Get or create storage instance. Reuse a process-wide initialized instance
|
||||
# rather than building one (and running an Alembic upgrade) per call — see
|
||||
# _get_initialized_basic_auth_storage for why (hot path + Alembic race).
|
||||
if storage is None:
|
||||
storage = RefreshTokenStorage.from_env()
|
||||
await storage.initialize()
|
||||
storage = await _get_initialized_basic_auth_storage()
|
||||
|
||||
# Retrieve app password from local storage
|
||||
app_password = await storage.get_app_password(user_id)
|
||||
|
||||
@@ -717,6 +717,16 @@ async def _index_document(
|
||||
},
|
||||
payload={
|
||||
"user_id": doc_task.user_id,
|
||||
# owner_id is the UID of the file's owner — what
|
||||
# search-time ACL expansion filters on. Today the scanner
|
||||
# always runs as the file's owner (per-user crawl, only
|
||||
# surfaces files the user owns or that fall under their
|
||||
# WebDAV root), so owner_id == user_id is correct for
|
||||
# every doc type indexed here. The fields are kept
|
||||
# separate so a future indexer change that lets a user
|
||||
# crawl shared-with-them content can set owner_id to the
|
||||
# true owner without losing the "who indexed this" trail.
|
||||
"owner_id": doc_task.owner_id or doc_task.user_id,
|
||||
"doc_id": doc_task.doc_id,
|
||||
"doc_type": doc_task.doc_type,
|
||||
"is_placeholder": False, # Real indexed document (not placeholder)
|
||||
|
||||
@@ -38,6 +38,15 @@ logger = logging.getLogger(__name__)
|
||||
_PAYLOAD_INDEX_FIELDS: dict[str, PayloadSchemaType] = {
|
||||
"doc_id": PayloadSchemaType.KEYWORD,
|
||||
"user_id": PayloadSchemaType.KEYWORD,
|
||||
# owner_id is the ACL-aware filter field: every search applies
|
||||
# MatchAny(key="owner_id", any=accessible_owners) (see
|
||||
# search/access_filter.py). Without a keyword index Qdrant full-scans the
|
||||
# collection to evaluate it — invisible at small scale, but a latency
|
||||
# regression at tens of thousands of points and an HTTP 400 on Qdrant
|
||||
# Cloud strict payload-validation mode. Mirrors the user_id treatment;
|
||||
# _ensure_payload_indexes is idempotent so existing collections migrate
|
||||
# at startup without operator intervention.
|
||||
"owner_id": PayloadSchemaType.KEYWORD,
|
||||
"doc_type": PayloadSchemaType.KEYWORD,
|
||||
"is_placeholder": PayloadSchemaType.BOOL,
|
||||
"chunk_index": PayloadSchemaType.INTEGER,
|
||||
|
||||
@@ -113,6 +113,12 @@ class DocumentTask:
|
||||
# when it is None (deletes, or sources whose etag isn't threaded). Harmless
|
||||
# in local mode — the in-process processor reads its own etag.
|
||||
etag: str | None = None
|
||||
# UID of the true owner of the indexed object, used by the search-time
|
||||
# ACL filter. None today (scanner always runs as the owner, so the
|
||||
# processor falls back to user_id), but settable so a future
|
||||
# shared-with-me crawl can pass through the actual owner without
|
||||
# reshaping the payload contract.
|
||||
owner_id: str | None = None
|
||||
|
||||
|
||||
# Track documents potentially deleted (grace period before actual deletion)
|
||||
|
||||
Reference in New Issue
Block a user