Merge pull request #911 from cbcoutinho/feat/admin-searchable-sources
feat(vector-sync): honor Astrolabe admin consent for searchable sources
This commit is contained in:
Chris Coutinho
@@ -36,6 +36,9 @@ from nextcloud_mcp_server.api.passwords import (
|
||||
get_app_password_status,
|
||||
provision_app_password,
|
||||
)
|
||||
from nextcloud_mcp_server.api.vector_sync import (
|
||||
purge_doc_types_route,
|
||||
)
|
||||
from nextcloud_mcp_server.api.visualization import (
|
||||
get_chunk_context,
|
||||
get_pdf_preview,
|
||||
@@ -78,6 +81,8 @@ __all__ = [
|
||||
"list_webhooks",
|
||||
"create_webhook",
|
||||
"delete_webhook",
|
||||
# Vector-sync admin endpoints (from vector_sync.py)
|
||||
"purge_doc_types_route",
|
||||
# Visualization endpoints (from visualization.py)
|
||||
"unified_search",
|
||||
"vector_search",
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Vector-sync admin API endpoints.
|
||||
|
||||
Provides the purge endpoint Astrolabe calls when an admin disables a content
|
||||
source for semantic search. Consent is binding on data-at-rest, so the
|
||||
already-indexed content for the disabled source's doc type(s) is deleted
|
||||
globally (every owner) — see :mod:`nextcloud_mcp_server.vector.purge`.
|
||||
|
||||
Auth: the OAuth bearer identifies the caller (``validate_token_and_get_user``);
|
||||
because the purge deletes every owner's content for a doc type, it is further
|
||||
restricted to Nextcloud administrators (verified via the ``admin`` group using
|
||||
the caller's app password). This is stricter than the per-user webhook routes
|
||||
in :mod:`nextcloud_mcp_server.api.webhooks` precisely because the blast radius
|
||||
is global.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import JSONResponse
|
||||
|
||||
from nextcloud_mcp_server.api._auth import get_basic_auth_for_user
|
||||
from nextcloud_mcp_server.api.management import (
|
||||
_sanitize_error_for_client,
|
||||
validate_token_and_get_user,
|
||||
)
|
||||
from nextcloud_mcp_server.auth.scope_authorization import ProvisioningRequiredError
|
||||
from nextcloud_mcp_server.client.users import UsersClient
|
||||
from nextcloud_mcp_server.vector.purge import purge_doc_types
|
||||
|
||||
from ..http import nextcloud_httpx_client
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Upper bound on doc_types per purge request. There are only a handful of real
|
||||
# indexed types; this caps a hostile/buggy caller's fan-out of count+delete
|
||||
# calls without constraining legitimate use.
|
||||
_MAX_PURGE_DOC_TYPES = 64
|
||||
|
||||
|
||||
def _bad_request(message: str) -> JSONResponse:
|
||||
return JSONResponse({"error": "Bad request", "message": message}, status_code=400)
|
||||
|
||||
|
||||
async def purge_doc_types_route(request: Request) -> JSONResponse:
|
||||
"""POST /api/v1/vector-sync/purge — delete indexed vectors by doc type.
|
||||
|
||||
Request body::
|
||||
|
||||
{"doc_types": ["file", "note"]}
|
||||
|
||||
Returns ``{"purged": {doc_type: deleted_count}}``. Admin-only.
|
||||
|
||||
Requires OAuth bearer token for authentication.
|
||||
"""
|
||||
try:
|
||||
user_id, _ = await validate_token_and_get_user(request)
|
||||
except Exception as e:
|
||||
logger.warning("Unauthorized access to /api/v1/vector-sync/purge: %s", e)
|
||||
return JSONResponse(
|
||||
{
|
||||
"error": "Unauthorized",
|
||||
"message": _sanitize_error_for_client(e, "purge_doc_types"),
|
||||
},
|
||||
status_code=401,
|
||||
)
|
||||
|
||||
try:
|
||||
body = await request.json()
|
||||
except Exception as e:
|
||||
logger.warning("Purge payload was not valid JSON: %s", e)
|
||||
return _bad_request("invalid JSON")
|
||||
|
||||
if not isinstance(body, dict):
|
||||
return _bad_request("body must be a JSON object")
|
||||
|
||||
raw = body.get("doc_types")
|
||||
if raw is None:
|
||||
return _bad_request("doc_types is required")
|
||||
if not isinstance(raw, list) or not all(isinstance(d, str) for d in raw):
|
||||
return _bad_request("doc_types must be a list of strings")
|
||||
doc_types = [d for d in raw if d]
|
||||
# No whitelist against INDEXED_DOC_TYPES on purpose: an unknown type yields a
|
||||
# zero-match Qdrant filter (harmless no-op), and the canonical set lives with
|
||||
# the indexer — the route shouldn't need a server update to purge a new type.
|
||||
# Bound the batch: there are only a handful of real indexed types, so a huge
|
||||
# list is abuse — cap it rather than fan out unbounded count+delete calls.
|
||||
if len(doc_types) > _MAX_PURGE_DOC_TYPES:
|
||||
return _bad_request(f"doc_types exceeds the maximum of {_MAX_PURGE_DOC_TYPES}")
|
||||
|
||||
try:
|
||||
username, app_password = await get_basic_auth_for_user(user_id)
|
||||
|
||||
oauth_ctx = request.app.state.oauth_context
|
||||
nextcloud_host = oauth_ctx.get("config", {}).get("nextcloud_host", "")
|
||||
if not nextcloud_host:
|
||||
raise ValueError("Nextcloud host not configured")
|
||||
|
||||
# Verify admin via the caller's own app password before any deletion —
|
||||
# enforced even for an empty (no-op) request, since this is a
|
||||
# destructive admin route.
|
||||
async with nextcloud_httpx_client(
|
||||
base_url=nextcloud_host,
|
||||
auth=httpx.BasicAuth(username, app_password),
|
||||
timeout=30.0,
|
||||
) as client:
|
||||
users_client = UsersClient(client, username)
|
||||
user_groups = await users_client.get_user_groups(username)
|
||||
if "admin" not in user_groups:
|
||||
logger.warning("Non-admin user %s attempted vector-sync purge", user_id)
|
||||
return JSONResponse(
|
||||
{
|
||||
"error": "Forbidden",
|
||||
"message": "Administrator privileges required",
|
||||
},
|
||||
status_code=403,
|
||||
)
|
||||
|
||||
if not doc_types:
|
||||
return JSONResponse({"purged": {}})
|
||||
|
||||
purged = await purge_doc_types(doc_types)
|
||||
# Surface a partial-failure signal so Astrolabe knows which types were
|
||||
# NOT purged (consent not yet enforced for them) — the scanner backstop
|
||||
# still catches these, but the caller shouldn't assume full success.
|
||||
failed = [dt for dt in dict.fromkeys(doc_types) if dt not in purged]
|
||||
resp: dict = {"purged": purged}
|
||||
if failed:
|
||||
resp["failed"] = failed
|
||||
logger.info(
|
||||
"Vector-sync purge by admin %s: purged=%s failed=%s",
|
||||
user_id,
|
||||
purged,
|
||||
failed,
|
||||
)
|
||||
return JSONResponse(resp)
|
||||
|
||||
except ProvisioningRequiredError as e:
|
||||
logger.info("Provisioning required for user %s: %s", user_id, e)
|
||||
return JSONResponse(
|
||||
{"error": "Provisioning required", "message": str(e)},
|
||||
status_code=428,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("Error purging doc types for user %s", user_id)
|
||||
return JSONResponse(
|
||||
{
|
||||
"error": "Internal error",
|
||||
"message": _sanitize_error_for_client(e, "purge_doc_types"),
|
||||
},
|
||||
status_code=500,
|
||||
)
|
||||
@@ -47,6 +47,7 @@ from nextcloud_mcp_server.api import (
|
||||
list_supported_scopes,
|
||||
list_webhooks,
|
||||
provision_app_password,
|
||||
purge_doc_types_route,
|
||||
revoke_user_access,
|
||||
unified_search,
|
||||
update_user_scopes,
|
||||
@@ -2422,6 +2423,19 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
|
||||
routes.append(
|
||||
Route("/api/v1/webhooks/{webhook_id}", delete_webhook, methods=["DELETE"])
|
||||
)
|
||||
# Vector-sync admin: purge indexed vectors by doc type (admin consent —
|
||||
# called by Astrolabe when a source is disabled for semantic search).
|
||||
# Gated on vector_sync_enabled: without it there is no Qdrant client, so
|
||||
# the purge would 500 rather than no-op.
|
||||
if settings.vector_sync_enabled:
|
||||
routes.append(
|
||||
Route(
|
||||
"/api/v1/vector-sync/purge",
|
||||
purge_doc_types_route,
|
||||
methods=["POST"],
|
||||
)
|
||||
)
|
||||
logger.info("Vector-sync admin endpoint enabled: /api/v1/vector-sync/purge")
|
||||
# Access and scope management endpoints (ADR-022)
|
||||
routes.append(
|
||||
Route(
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
"""Reads admin-approved searchable sources from the Astrolabe capability.
|
||||
|
||||
The Astrolabe Nextcloud app advertises, per user, which content sources an
|
||||
admin has approved for semantic search, under
|
||||
``capabilities.astrolabe.semantic_search.enabled_doc_types`` on the OCS
|
||||
capabilities endpoint (``/ocs/v2.php/cloud/capabilities``). This is the single
|
||||
source of truth for admin consent: the search layer filters results to these
|
||||
doc types, and the indexing layer (scanner + webhook ingest) skips everything
|
||||
else (the hard data-at-rest guarantee is the eager purge Astrolabe triggers on
|
||||
disable; see ``vector.purge``).
|
||||
|
||||
Fail-open for *availability*: if the capability block is absent (an older
|
||||
Astrolabe that predates this feature) or the OCS call fails, ``allowed_doc_types``
|
||||
returns ``None`` meaning "no restriction", so search keeps working. ``None`` is
|
||||
distinct from an empty set, which means "the admin disabled every source".
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from typing import Any, Protocol
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Short-lived per-user cache for the OCS capabilities lookup. Admin consent
|
||||
# changes rarely, but search/scan paths consult it frequently, so trade a little
|
||||
# staleness for keeping the OCS round-trip off the hot path. Mirrors the
|
||||
# list_accessible_owners cache in search/access_filter.py.
|
||||
#
|
||||
# Keyed by user_id even though enabled_doc_types is an admin-wide value: the OCS
|
||||
# call is authenticated per-user (and ``installed`` resolves per-user on the
|
||||
# Astrolabe side), so we cache per-user for correctness. The redundancy is
|
||||
# bounded by _CACHE_MAXSIZE; on an admin change all entries reconverge within
|
||||
# one TTL window.
|
||||
_CACHE_TTL_SECONDS = 30.0
|
||||
_CACHE_MAXSIZE = 1024
|
||||
# user_id -> (monotonic_ts, frozenset[doc_type] | None). None = no restriction.
|
||||
_cache: OrderedDict[str, tuple[float, frozenset[str] | None]] = OrderedDict()
|
||||
|
||||
|
||||
class _CapabilitiesClientProtocol(Protocol):
|
||||
async def capabilities(self) -> Any: ...
|
||||
|
||||
|
||||
def _parse_enabled_doc_types(payload: Any) -> frozenset[str] | None:
|
||||
"""Extract ``enabled_doc_types`` from an OCS capabilities payload.
|
||||
|
||||
Returns ``None`` when the ``astrolabe.semantic_search`` block is absent or
|
||||
malformed (treated as "no restriction"). Returns a frozenset (possibly
|
||||
empty) when the block is present and well-formed; an empty set means the
|
||||
admin disabled every source.
|
||||
"""
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
try:
|
||||
caps = payload["ocs"]["data"]["capabilities"]
|
||||
except (KeyError, TypeError):
|
||||
return None
|
||||
if not isinstance(caps, dict):
|
||||
return None
|
||||
block = caps.get("astrolabe")
|
||||
if not isinstance(block, dict):
|
||||
return None
|
||||
semantic = block.get("semantic_search")
|
||||
if not isinstance(semantic, dict):
|
||||
return None
|
||||
raw = semantic.get("enabled_doc_types")
|
||||
if not isinstance(raw, list):
|
||||
return None
|
||||
return frozenset(dt for dt in raw if isinstance(dt, str) and dt)
|
||||
|
||||
|
||||
async def allowed_doc_types(
|
||||
client: _CapabilitiesClientProtocol, user_id: str
|
||||
) -> frozenset[str] | None:
|
||||
"""Admin-approved doc types for ``user_id``, or ``None`` for "no restriction".
|
||||
|
||||
Cached per user with a short TTL (+ LRU eviction). Failures are not cached so
|
||||
a transient OCS hiccup retries on the next call. Fail-open: a missing
|
||||
capability block or an error yields ``None`` so search remains available.
|
||||
"""
|
||||
now = time.monotonic()
|
||||
cached = _cache.get(user_id)
|
||||
if cached is not None and now - cached[0] < _CACHE_TTL_SECONDS:
|
||||
_cache.move_to_end(user_id) # mark recently used (LRU)
|
||||
return cached[1]
|
||||
|
||||
try:
|
||||
payload = await client.capabilities()
|
||||
except Exception as exc: # noqa: BLE001 — degrade gracefully (fail-open)
|
||||
logger.warning(
|
||||
"Astrolabe capabilities unavailable for user %s (%s); "
|
||||
"not restricting doc types this cycle",
|
||||
user_id,
|
||||
exc,
|
||||
)
|
||||
return None # don't cache failures — retry next call
|
||||
|
||||
result = _parse_enabled_doc_types(payload)
|
||||
_cache[user_id] = (now, result)
|
||||
# Needed only for an existing (expired) key: __setitem__ updates it in place,
|
||||
# keeping its old position, so move it to the end to preserve LRU order. For
|
||||
# a brand-new key __setitem__ already appends, so this is a harmless no-op.
|
||||
_cache.move_to_end(user_id)
|
||||
while len(_cache) > _CACHE_MAXSIZE:
|
||||
_cache.popitem(last=False) # evict least-recently-used
|
||||
return result
|
||||
|
||||
|
||||
def is_doc_type_allowed(doc_type: str, allowed: frozenset[str] | None) -> bool:
|
||||
"""Whether ``doc_type`` may be indexed/searched given an allow-set.
|
||||
|
||||
``allowed=None`` means "no restriction" (fail-open / older Astrolabe), so
|
||||
everything is permitted.
|
||||
"""
|
||||
return allowed is None or doc_type in allowed
|
||||
|
||||
|
||||
def clear_cache() -> None:
|
||||
"""Test hook: drop all cached entries."""
|
||||
_cache.clear()
|
||||
@@ -20,6 +20,7 @@ from mcp.types import (
|
||||
from pydantic import Field
|
||||
|
||||
from nextcloud_mcp_server.auth import require_scopes
|
||||
from nextcloud_mcp_server.capabilities import allowed_doc_types
|
||||
from nextcloud_mcp_server.config import get_settings
|
||||
from nextcloud_mcp_server.context import get_client
|
||||
from nextcloud_mcp_server.models.semantic import (
|
||||
@@ -56,6 +57,25 @@ logger = logging.getLogger(__name__)
|
||||
_USAGE_METADATA_MAX_DOC_TYPES = 16
|
||||
|
||||
|
||||
def _consent_narrowed_doc_types(
|
||||
doc_types: list[str] | None, allowed: frozenset[str]
|
||||
) -> list[str]:
|
||||
"""Apply the admin allow-set to a requested ``doc_types`` filter.
|
||||
|
||||
Caller has already established ``allowed is not None`` (a concrete allow-set;
|
||||
``None`` means "no restriction" and is handled by skipping this call). When
|
||||
no explicit ``doc_types`` are requested, restrict to the full allow-set
|
||||
(returned ``sorted`` for determinism only — order is a filter, not a ranking
|
||||
hint); otherwise intersect (preserving the caller's order). An empty result
|
||||
means nothing the caller asked for is admin-approved — the caller
|
||||
short-circuits to an empty response rather than falling through to an
|
||||
all-types search.
|
||||
"""
|
||||
if doc_types is None:
|
||||
return sorted(allowed)
|
||||
return [dt for dt in doc_types if dt in allowed]
|
||||
|
||||
|
||||
async def record_search_usage(
|
||||
*,
|
||||
enabled: bool,
|
||||
@@ -300,6 +320,37 @@ def configure_semantic_tools(mcp: FastMCP):
|
||||
# files under their own user_id.
|
||||
accessible_owners = await list_accessible_owners(client.sharing, username)
|
||||
|
||||
# Admin consent gate: restrict to source types the Astrolabe admin has
|
||||
# approved (and that are installed for this user). This mirrors
|
||||
# Astrolabe's own server-side enforcement but is independent because
|
||||
# this tool queries Qdrant directly. ``None`` = no restriction
|
||||
# (fail-open / Astrolabe predating this feature). An empty allow-set
|
||||
# means the admin disabled every source.
|
||||
#
|
||||
# Perf trade-off (accepted): when Astrolabe is present and the caller
|
||||
# passed no doc_types, narrowing turns ``None`` into a concrete list, so
|
||||
# the search takes the per-type query branch (N queries) instead of the
|
||||
# single cross-type query. N is the count of admin-approved types
|
||||
# (typically 1-4), so the overhead is small; left as-is rather than
|
||||
# adding a "search all approved in one query" fast path.
|
||||
allowed = await allowed_doc_types(client, username)
|
||||
if allowed is not None:
|
||||
doc_types = _consent_narrowed_doc_types(doc_types, allowed)
|
||||
if not doc_types:
|
||||
logger.info(
|
||||
"Semantic search short-circuited for user %s: no requested "
|
||||
"doc_type is admin-approved for semantic search",
|
||||
username,
|
||||
)
|
||||
return SemanticSearchResponse(
|
||||
results=[],
|
||||
query=query,
|
||||
total_found=0,
|
||||
search_method=f"bm25_hybrid_{fusion}",
|
||||
verified_chunk_count=0,
|
||||
dropped_document_count=0,
|
||||
)
|
||||
|
||||
try:
|
||||
# The nc_semantic_search tool deliberately uses BM25-hybrid (dense +
|
||||
# sparse with RRF/DBSF fusion) as the single tool-layer algorithm.
|
||||
|
||||
@@ -21,6 +21,7 @@ if TYPE_CHECKING:
|
||||
from nextcloud_mcp_server.document_processors.registry import ProcessorRegistry
|
||||
|
||||
from nextcloud_mcp_server.acl_hash import compute_acl_hash
|
||||
from nextcloud_mcp_server.capabilities import allowed_doc_types, is_doc_type_allowed
|
||||
from nextcloud_mcp_server.client import NextcloudClient
|
||||
from nextcloud_mcp_server.config import get_settings
|
||||
from nextcloud_mcp_server.embedding import get_bm25_service, get_embedding_service
|
||||
@@ -590,6 +591,27 @@ async def process_document(
|
||||
):
|
||||
await _reconcile_tag_event(doc_task, nc_client)
|
||||
|
||||
# Admin consent gate (Astrolabe): never index a source the admin has
|
||||
# disabled for semantic search — this catches near-real-time webhook
|
||||
# events that bypass the scanner's discovery gate. Deletes always
|
||||
# proceed (removing data honours consent). ``None`` from the reader
|
||||
# means no restriction (fail-open / older Astrolabe), so a transient
|
||||
# capabilities failure never silently drops indexing.
|
||||
if doc_task.operation == "index":
|
||||
allowed = await allowed_doc_types(nc_client, doc_task.user_id)
|
||||
if not is_doc_type_allowed(doc_task.doc_type, allowed):
|
||||
logger.info(
|
||||
"Skipping index of %s_%s for %s: doc_type disabled by admin",
|
||||
doc_task.doc_type,
|
||||
doc_task.doc_id,
|
||||
doc_task.user_id,
|
||||
)
|
||||
# Alertable counter so a flood of webhook events for a
|
||||
# disabled source is observable (not silently swallowed).
|
||||
record_ingest_dropped("admin_disabled")
|
||||
record_vector_sync_processing(time.time() - start_time, "skipped")
|
||||
return
|
||||
|
||||
# Handle deletion
|
||||
if doc_task.operation == "delete":
|
||||
# Release this user rather than blind-delete: a file shared across
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Global purge of indexed vectors by doc type (admin consent enforcement).
|
||||
|
||||
When an admin disables a content source for semantic search in Astrolabe,
|
||||
consent is binding on data-at-rest: the already-indexed content for that
|
||||
source's doc type(s) must be deleted, not merely hidden. Astrolabe calls the
|
||||
``/api/v1/vector-sync/purge`` route on disable, which delegates here.
|
||||
|
||||
The purge is global (every owner) because the admin disable is a global
|
||||
decision. It is safe to call for a doc type with no indexed points — Qdrant
|
||||
deletes zero points and reports a count of 0.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from qdrant_client.models import FieldCondition, Filter, MatchValue
|
||||
|
||||
from nextcloud_mcp_server.config import get_settings
|
||||
from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _doc_type_filter(doc_type: str) -> Filter:
|
||||
return Filter(
|
||||
must=[FieldCondition(key="doc_type", match=MatchValue(value=doc_type))]
|
||||
)
|
||||
|
||||
|
||||
async def purge_doc_types(doc_types: list[str]) -> dict[str, int]:
|
||||
"""Delete every indexed point whose ``doc_type`` is in ``doc_types``.
|
||||
|
||||
Returns a mapping of doc_type -> number of points deleted (counted before
|
||||
deletion). Each doc type is purged independently so a failure on one does
|
||||
not abort the rest; failures re-raise after the loop only if every doc type
|
||||
failed, otherwise partial progress is returned.
|
||||
|
||||
The count is taken just before the delete (two separate Qdrant calls), so
|
||||
it is approximate — a point indexed in the gap is deleted but not counted.
|
||||
This is acceptable: indexing of a disabled source is already gated upstream,
|
||||
so the window is effectively empty in practice.
|
||||
"""
|
||||
qdrant_client = await get_qdrant_client()
|
||||
collection = get_settings().get_collection_name()
|
||||
|
||||
purged: dict[str, int] = {}
|
||||
last_error: Exception | None = None
|
||||
for doc_type in dict.fromkeys(doc_types): # de-dupe, preserve order
|
||||
flt = _doc_type_filter(doc_type)
|
||||
try:
|
||||
count_result = await qdrant_client.count(
|
||||
collection_name=collection,
|
||||
count_filter=flt,
|
||||
exact=True,
|
||||
)
|
||||
await qdrant_client.delete(
|
||||
collection_name=collection,
|
||||
points_selector=flt,
|
||||
)
|
||||
purged[doc_type] = int(count_result.count)
|
||||
logger.info(
|
||||
"Purged %d indexed point(s) for disabled doc_type=%s",
|
||||
purged[doc_type],
|
||||
doc_type,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — record and continue
|
||||
last_error = exc
|
||||
logger.exception(
|
||||
"Failed to purge indexed points for doc_type=%s",
|
||||
doc_type,
|
||||
)
|
||||
|
||||
if not purged and last_error is not None:
|
||||
# Nothing succeeded — surface the failure to the caller (HTTP 500).
|
||||
raise last_error
|
||||
return purged
|
||||
@@ -16,6 +16,7 @@ from httpx import HTTPStatusError
|
||||
from qdrant_client import AsyncQdrantClient
|
||||
from qdrant_client.models import FieldCondition, Filter, MatchValue, Record
|
||||
|
||||
from nextcloud_mcp_server.capabilities import allowed_doc_types, is_doc_type_allowed
|
||||
from nextcloud_mcp_server.client import NextcloudClient
|
||||
from nextcloud_mcp_server.client.news import NewsItemType
|
||||
from nextcloud_mcp_server.config import get_settings
|
||||
@@ -279,6 +280,150 @@ def _app_enabled(app_id: str, enabled_apps: set[str] | None) -> bool:
|
||||
return enabled_apps is None or app_id in enabled_apps
|
||||
|
||||
|
||||
def _should_scan(
|
||||
app_id: str,
|
||||
doc_type: str,
|
||||
enabled_apps: set[str] | None,
|
||||
allowed: frozenset[str] | None,
|
||||
) -> bool:
|
||||
"""Whether to scan ``app_id``: installed for the user AND admin-approved."""
|
||||
return _app_enabled(app_id, enabled_apps) and is_doc_type_allowed(doc_type, allowed)
|
||||
|
||||
|
||||
# Text doc types whose deletion-tracking lives *inside* their scan_* function,
|
||||
# so skipping that function (when admin-disabled) leaves indexed points with no
|
||||
# grace-period backstop. Derived from INDEXED_DOC_TYPES so a newly-indexed type
|
||||
# automatically gets the backstop. ``file`` is excluded: its scan path empties
|
||||
# discovery and lets the existing reconcile loop purge on disable.
|
||||
_TEXT_BACKSTOP_DOC_TYPES: tuple[str, ...] = tuple(sorted(INDEXED_DOC_TYPES - {"file"}))
|
||||
|
||||
# Per-process record of (user_id, doc_type) whose consent backstop deletes have
|
||||
# already been enqueued, so a *standing* admin-disable doesn't re-flood the
|
||||
# processor with idempotent deletes on every scan tick. An entry is cleared once
|
||||
# the type is allowed again, so a later re-disable re-triggers the backstop.
|
||||
# A dict (not a set) so it stays insertion-ordered for oldest-first eviction.
|
||||
_consent_backstop_done: dict[tuple[str, str], None] = {}
|
||||
|
||||
# Safety bound on the tracking dict so a long-running multi-tenant process with
|
||||
# heavy user churn (deprovisioned users leave stale entries) can't grow it
|
||||
# without limit. At <= len(INDEXED_DOC_TYPES) entries per user this is generous;
|
||||
# on overflow we evict the *oldest* entries down to half capacity (not a full
|
||||
# clear) so the backstop re-fires for only those, avoiding a fleet-wide burst.
|
||||
_CONSENT_BACKSTOP_MAX = 50_000
|
||||
|
||||
|
||||
def _mark_backstop_done(key: tuple[str, str]) -> None:
|
||||
"""Record a one-shot backstop marker, evicting oldest entries on overflow.
|
||||
|
||||
Evicts oldest-first down to half capacity (insertion-ordered dict) rather
|
||||
than clearing wholesale, so a bound hit re-fires the backstop only for the
|
||||
oldest markers, not the whole fleet. A re-fire is idempotent regardless.
|
||||
"""
|
||||
if len(_consent_backstop_done) >= _CONSENT_BACKSTOP_MAX:
|
||||
overage = len(_consent_backstop_done) - _CONSENT_BACKSTOP_MAX // 2
|
||||
logger.info(
|
||||
"consent backstop tracking hit %d entries; evicting %d oldest",
|
||||
_CONSENT_BACKSTOP_MAX,
|
||||
overage,
|
||||
)
|
||||
for stale_key in list(_consent_backstop_done)[:overage]:
|
||||
del _consent_backstop_done[stale_key]
|
||||
_consent_backstop_done[key] = None
|
||||
|
||||
|
||||
async def _backstop_delete_doc_type(
|
||||
user_id: str,
|
||||
send_stream: TaskProducer,
|
||||
doc_type: str,
|
||||
qdrant_client: AsyncQdrantClient,
|
||||
collection: str,
|
||||
scan_id: int,
|
||||
) -> int:
|
||||
"""Enqueue delete tasks for every indexed point of one disabled doc_type.
|
||||
|
||||
Returns the number of delete tasks enqueued.
|
||||
"""
|
||||
points = await _scroll_all_points(
|
||||
qdrant_client,
|
||||
collection_name=collection,
|
||||
scroll_filter=Filter(
|
||||
must=[
|
||||
FieldCondition(key="user_id", match=MatchValue(value=user_id)),
|
||||
FieldCondition(key="doc_type", match=MatchValue(value=doc_type)),
|
||||
]
|
||||
),
|
||||
payload_fields=["doc_id"],
|
||||
)
|
||||
doc_ids = {
|
||||
str(p.payload["doc_id"])
|
||||
for p in points
|
||||
if p.payload is not None and "doc_id" in p.payload
|
||||
}
|
||||
if doc_ids:
|
||||
logger.info(
|
||||
"[SCAN-%s] %s disabled by admin for %s; enqueueing %d delete(s) (backstop)",
|
||||
scan_id,
|
||||
doc_type,
|
||||
user_id,
|
||||
len(doc_ids),
|
||||
)
|
||||
for doc_id in doc_ids:
|
||||
await send_stream.send(
|
||||
DocumentTask(
|
||||
user_id=user_id,
|
||||
doc_id=doc_id,
|
||||
doc_type=doc_type,
|
||||
operation="delete",
|
||||
modified_at=0,
|
||||
)
|
||||
)
|
||||
return len(doc_ids)
|
||||
|
||||
|
||||
async def _enqueue_deletes_for_disabled_types(
|
||||
user_id: str,
|
||||
send_stream: TaskProducer,
|
||||
allowed: frozenset[str] | None,
|
||||
scan_id: int,
|
||||
) -> int:
|
||||
"""Enqueue delete tasks for indexed text-source points the admin disabled.
|
||||
|
||||
Backstop for a failed eager purge: scrolls this user's indexed points for
|
||||
each admin-disallowed text doc_type and queues a delete. No-op when
|
||||
``allowed`` is ``None`` (fail-open — never delete on a transient capability
|
||||
read failure). Returns the number of delete tasks enqueued.
|
||||
"""
|
||||
if allowed is None:
|
||||
return 0
|
||||
|
||||
# Re-enabled types: clear their one-shot marker so a later re-disable
|
||||
# re-triggers the backstop.
|
||||
for doc_type in _TEXT_BACKSTOP_DOC_TYPES:
|
||||
if doc_type in allowed:
|
||||
_consent_backstop_done.pop((user_id, doc_type), None)
|
||||
|
||||
# Disabled types not yet backstopped this episode.
|
||||
disabled = [
|
||||
dt
|
||||
for dt in _TEXT_BACKSTOP_DOC_TYPES
|
||||
if dt not in allowed and (user_id, dt) not in _consent_backstop_done
|
||||
]
|
||||
if not disabled:
|
||||
return 0
|
||||
|
||||
qdrant_client = await get_qdrant_client()
|
||||
collection = get_settings().get_collection_name()
|
||||
queued = 0
|
||||
for doc_type in disabled:
|
||||
queued += await _backstop_delete_doc_type(
|
||||
user_id, send_stream, doc_type, qdrant_client, collection, scan_id
|
||||
)
|
||||
# Mark backstopped (even when nothing was found) so subsequent scans
|
||||
# don't re-scroll/re-enqueue for this disable episode.
|
||||
_mark_backstop_done((user_id, doc_type))
|
||||
return queued
|
||||
|
||||
|
||||
async def scan_user_documents(
|
||||
user_id: str,
|
||||
send_stream: TaskProducer,
|
||||
@@ -365,6 +510,13 @@ async def scan_user_documents(
|
||||
# detection failed: fall back to scanning every app (prior behaviour).
|
||||
enabled_apps = await _get_enabled_apps_or_none(nc_client, user_id, scan_id)
|
||||
|
||||
# Admin consent gate (Astrolabe): only index sources the admin has
|
||||
# approved for semantic search. ``None`` = no restriction (fail-open /
|
||||
# older Astrolabe), so a transient capabilities failure never silently
|
||||
# halts (or worse, mass-deletes) indexing. This is independent of
|
||||
# ``enabled_apps``, which reflects only what the user has installed.
|
||||
allowed = await allowed_doc_types(nc_client, user_id)
|
||||
|
||||
# Notes (isolated so an uninstalled or disabled Notes app — whose API
|
||||
# returns 404 — cannot abort scanning of the other apps; this mirrors the
|
||||
# per-app try/except guards already wrapping files/news/deck below).
|
||||
@@ -373,7 +525,17 @@ async def scan_user_documents(
|
||||
current_time = time.time()
|
||||
queued = 0
|
||||
|
||||
if _app_enabled("notes", enabled_apps):
|
||||
# Backstop purge for admin-disabled text sources. Their deletion-
|
||||
# tracking lives inside the scan_* function we skip below, so (unlike
|
||||
# files, whose discovery-empties-then-reconcile path purges on disable)
|
||||
# they'd linger if Astrolabe's eager purge failed. Enqueue deletes for
|
||||
# any indexed points of a now-disallowed type. Gated on a concrete
|
||||
# allow-set, so a fail-open None never triggers deletion.
|
||||
queued += await _enqueue_deletes_for_disabled_types(
|
||||
user_id, send_stream, allowed, scan_id
|
||||
)
|
||||
|
||||
if _should_scan("notes", "note", enabled_apps, allowed):
|
||||
try:
|
||||
queued += await scan_notes(
|
||||
user_id=user_id,
|
||||
@@ -454,9 +616,24 @@ async def scan_user_documents(
|
||||
# folder applies to every PDF beneath it.
|
||||
settings = get_settings()
|
||||
tag_name = settings.vector_sync_pdf_tag
|
||||
tagged_files = await nc_client.find_files_by_tag(
|
||||
tag_name, mime_type_filter="application/pdf"
|
||||
)
|
||||
if is_doc_type_allowed("file", allowed):
|
||||
tagged_files = await nc_client.find_files_by_tag(
|
||||
tag_name, mime_type_filter="application/pdf"
|
||||
)
|
||||
else:
|
||||
# Files disabled by admin: discover nothing so no new file is
|
||||
# indexed. The deletion-reconcile below then sees every indexed
|
||||
# file as "missing" and purges it after the grace period — the
|
||||
# backstop for the eager purge Astrolabe runs on disable.
|
||||
# Asymmetry (intentional): files purge up to 1.5x scan_interval
|
||||
# later than text types, which get immediate one-shot backstop
|
||||
# deletes via _enqueue_deletes_for_disabled_types.
|
||||
logger.debug(
|
||||
"[SCAN-%s] Files disabled by admin for %s; skipping tagged-file discovery",
|
||||
scan_id,
|
||||
user_id,
|
||||
)
|
||||
tagged_files = []
|
||||
|
||||
# Apply EXCLUDED_TAGS as defense-in-depth: a folder marked
|
||||
# off-limits via the exclusion tag must not be indexed even if
|
||||
@@ -710,7 +887,7 @@ async def scan_user_documents(
|
||||
|
||||
# Scan News items (starred + unread)
|
||||
news_queued = 0
|
||||
if _app_enabled("news", enabled_apps):
|
||||
if _should_scan("news", "news_item", enabled_apps, allowed):
|
||||
try:
|
||||
news_queued = await scan_news_items(
|
||||
user_id=user_id,
|
||||
@@ -731,7 +908,7 @@ async def scan_user_documents(
|
||||
|
||||
# Scan Deck cards
|
||||
deck_queued = 0
|
||||
if _app_enabled("deck", enabled_apps):
|
||||
if _should_scan("deck", "deck_card", enabled_apps, allowed):
|
||||
try:
|
||||
deck_queued = await scan_deck_cards(
|
||||
user_id=user_id,
|
||||
|
||||
Reference in New Issue
Block a user