Merge pull request #911 from cbcoutinho/feat/admin-searchable-sources
feat(vector-sync): honor Astrolabe admin consent for searchable sources
This commit is contained in:
Chris Coutinho
@@ -36,6 +36,9 @@ from nextcloud_mcp_server.api.passwords import (
|
||||
get_app_password_status,
|
||||
provision_app_password,
|
||||
)
|
||||
from nextcloud_mcp_server.api.vector_sync import (
|
||||
purge_doc_types_route,
|
||||
)
|
||||
from nextcloud_mcp_server.api.visualization import (
|
||||
get_chunk_context,
|
||||
get_pdf_preview,
|
||||
@@ -78,6 +81,8 @@ __all__ = [
|
||||
"list_webhooks",
|
||||
"create_webhook",
|
||||
"delete_webhook",
|
||||
# Vector-sync admin endpoints (from vector_sync.py)
|
||||
"purge_doc_types_route",
|
||||
# Visualization endpoints (from visualization.py)
|
||||
"unified_search",
|
||||
"vector_search",
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Vector-sync admin API endpoints.
|
||||
|
||||
Provides the purge endpoint Astrolabe calls when an admin disables a content
|
||||
source for semantic search. Consent is binding on data-at-rest, so the
|
||||
already-indexed content for the disabled source's doc type(s) is deleted
|
||||
globally (every owner) — see :mod:`nextcloud_mcp_server.vector.purge`.
|
||||
|
||||
Auth: the OAuth bearer identifies the caller (``validate_token_and_get_user``);
|
||||
because the purge deletes every owner's content for a doc type, it is further
|
||||
restricted to Nextcloud administrators (verified via the ``admin`` group using
|
||||
the caller's app password). This is stricter than the per-user webhook routes
|
||||
in :mod:`nextcloud_mcp_server.api.webhooks` precisely because the blast radius
|
||||
is global.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import JSONResponse
|
||||
|
||||
from nextcloud_mcp_server.api._auth import get_basic_auth_for_user
|
||||
from nextcloud_mcp_server.api.management import (
|
||||
_sanitize_error_for_client,
|
||||
validate_token_and_get_user,
|
||||
)
|
||||
from nextcloud_mcp_server.auth.scope_authorization import ProvisioningRequiredError
|
||||
from nextcloud_mcp_server.client.users import UsersClient
|
||||
from nextcloud_mcp_server.vector.purge import purge_doc_types
|
||||
|
||||
from ..http import nextcloud_httpx_client
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Upper bound on doc_types per purge request. There are only a handful of real
|
||||
# indexed types; this caps a hostile/buggy caller's fan-out of count+delete
|
||||
# calls without constraining legitimate use.
|
||||
_MAX_PURGE_DOC_TYPES = 64
|
||||
|
||||
|
||||
def _bad_request(message: str) -> JSONResponse:
|
||||
return JSONResponse({"error": "Bad request", "message": message}, status_code=400)
|
||||
|
||||
|
||||
async def purge_doc_types_route(request: Request) -> JSONResponse:
|
||||
"""POST /api/v1/vector-sync/purge — delete indexed vectors by doc type.
|
||||
|
||||
Request body::
|
||||
|
||||
{"doc_types": ["file", "note"]}
|
||||
|
||||
Returns ``{"purged": {doc_type: deleted_count}}``. Admin-only.
|
||||
|
||||
Requires OAuth bearer token for authentication.
|
||||
"""
|
||||
try:
|
||||
user_id, _ = await validate_token_and_get_user(request)
|
||||
except Exception as e:
|
||||
logger.warning("Unauthorized access to /api/v1/vector-sync/purge: %s", e)
|
||||
return JSONResponse(
|
||||
{
|
||||
"error": "Unauthorized",
|
||||
"message": _sanitize_error_for_client(e, "purge_doc_types"),
|
||||
},
|
||||
status_code=401,
|
||||
)
|
||||
|
||||
try:
|
||||
body = await request.json()
|
||||
except Exception as e:
|
||||
logger.warning("Purge payload was not valid JSON: %s", e)
|
||||
return _bad_request("invalid JSON")
|
||||
|
||||
if not isinstance(body, dict):
|
||||
return _bad_request("body must be a JSON object")
|
||||
|
||||
raw = body.get("doc_types")
|
||||
if raw is None:
|
||||
return _bad_request("doc_types is required")
|
||||
if not isinstance(raw, list) or not all(isinstance(d, str) for d in raw):
|
||||
return _bad_request("doc_types must be a list of strings")
|
||||
doc_types = [d for d in raw if d]
|
||||
# No whitelist against INDEXED_DOC_TYPES on purpose: an unknown type yields a
|
||||
# zero-match Qdrant filter (harmless no-op), and the canonical set lives with
|
||||
# the indexer — the route shouldn't need a server update to purge a new type.
|
||||
# Bound the batch: there are only a handful of real indexed types, so a huge
|
||||
# list is abuse — cap it rather than fan out unbounded count+delete calls.
|
||||
if len(doc_types) > _MAX_PURGE_DOC_TYPES:
|
||||
return _bad_request(f"doc_types exceeds the maximum of {_MAX_PURGE_DOC_TYPES}")
|
||||
|
||||
try:
|
||||
username, app_password = await get_basic_auth_for_user(user_id)
|
||||
|
||||
oauth_ctx = request.app.state.oauth_context
|
||||
nextcloud_host = oauth_ctx.get("config", {}).get("nextcloud_host", "")
|
||||
if not nextcloud_host:
|
||||
raise ValueError("Nextcloud host not configured")
|
||||
|
||||
# Verify admin via the caller's own app password before any deletion —
|
||||
# enforced even for an empty (no-op) request, since this is a
|
||||
# destructive admin route.
|
||||
async with nextcloud_httpx_client(
|
||||
base_url=nextcloud_host,
|
||||
auth=httpx.BasicAuth(username, app_password),
|
||||
timeout=30.0,
|
||||
) as client:
|
||||
users_client = UsersClient(client, username)
|
||||
user_groups = await users_client.get_user_groups(username)
|
||||
if "admin" not in user_groups:
|
||||
logger.warning("Non-admin user %s attempted vector-sync purge", user_id)
|
||||
return JSONResponse(
|
||||
{
|
||||
"error": "Forbidden",
|
||||
"message": "Administrator privileges required",
|
||||
},
|
||||
status_code=403,
|
||||
)
|
||||
|
||||
if not doc_types:
|
||||
return JSONResponse({"purged": {}})
|
||||
|
||||
purged = await purge_doc_types(doc_types)
|
||||
# Surface a partial-failure signal so Astrolabe knows which types were
|
||||
# NOT purged (consent not yet enforced for them) — the scanner backstop
|
||||
# still catches these, but the caller shouldn't assume full success.
|
||||
failed = [dt for dt in dict.fromkeys(doc_types) if dt not in purged]
|
||||
resp: dict = {"purged": purged}
|
||||
if failed:
|
||||
resp["failed"] = failed
|
||||
logger.info(
|
||||
"Vector-sync purge by admin %s: purged=%s failed=%s",
|
||||
user_id,
|
||||
purged,
|
||||
failed,
|
||||
)
|
||||
return JSONResponse(resp)
|
||||
|
||||
except ProvisioningRequiredError as e:
|
||||
logger.info("Provisioning required for user %s: %s", user_id, e)
|
||||
return JSONResponse(
|
||||
{"error": "Provisioning required", "message": str(e)},
|
||||
status_code=428,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("Error purging doc types for user %s", user_id)
|
||||
return JSONResponse(
|
||||
{
|
||||
"error": "Internal error",
|
||||
"message": _sanitize_error_for_client(e, "purge_doc_types"),
|
||||
},
|
||||
status_code=500,
|
||||
)
|
||||
@@ -47,6 +47,7 @@ from nextcloud_mcp_server.api import (
|
||||
list_supported_scopes,
|
||||
list_webhooks,
|
||||
provision_app_password,
|
||||
purge_doc_types_route,
|
||||
revoke_user_access,
|
||||
unified_search,
|
||||
update_user_scopes,
|
||||
@@ -2422,6 +2423,19 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
|
||||
routes.append(
|
||||
Route("/api/v1/webhooks/{webhook_id}", delete_webhook, methods=["DELETE"])
|
||||
)
|
||||
# Vector-sync admin: purge indexed vectors by doc type (admin consent —
|
||||
# called by Astrolabe when a source is disabled for semantic search).
|
||||
# Gated on vector_sync_enabled: without it there is no Qdrant client, so
|
||||
# the purge would 500 rather than no-op.
|
||||
if settings.vector_sync_enabled:
|
||||
routes.append(
|
||||
Route(
|
||||
"/api/v1/vector-sync/purge",
|
||||
purge_doc_types_route,
|
||||
methods=["POST"],
|
||||
)
|
||||
)
|
||||
logger.info("Vector-sync admin endpoint enabled: /api/v1/vector-sync/purge")
|
||||
# Access and scope management endpoints (ADR-022)
|
||||
routes.append(
|
||||
Route(
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
"""Reads admin-approved searchable sources from the Astrolabe capability.
|
||||
|
||||
The Astrolabe Nextcloud app advertises, per user, which content sources an
|
||||
admin has approved for semantic search, under
|
||||
``capabilities.astrolabe.semantic_search.enabled_doc_types`` on the OCS
|
||||
capabilities endpoint (``/ocs/v2.php/cloud/capabilities``). This is the single
|
||||
source of truth for admin consent: the search layer filters results to these
|
||||
doc types, and the indexing layer (scanner + webhook ingest) skips everything
|
||||
else (the hard data-at-rest guarantee is the eager purge Astrolabe triggers on
|
||||
disable; see ``vector.purge``).
|
||||
|
||||
Fail-open for *availability*: if the capability block is absent (an older
|
||||
Astrolabe that predates this feature) or the OCS call fails, ``allowed_doc_types``
|
||||
returns ``None`` meaning "no restriction", so search keeps working. ``None`` is
|
||||
distinct from an empty set, which means "the admin disabled every source".
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from typing import Any, Protocol
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Short-lived per-user cache for the OCS capabilities lookup. Admin consent
|
||||
# changes rarely, but search/scan paths consult it frequently, so trade a little
|
||||
# staleness for keeping the OCS round-trip off the hot path. Mirrors the
|
||||
# list_accessible_owners cache in search/access_filter.py.
|
||||
#
|
||||
# Keyed by user_id even though enabled_doc_types is an admin-wide value: the OCS
|
||||
# call is authenticated per-user (and ``installed`` resolves per-user on the
|
||||
# Astrolabe side), so we cache per-user for correctness. The redundancy is
|
||||
# bounded by _CACHE_MAXSIZE; on an admin change all entries reconverge within
|
||||
# one TTL window.
|
||||
_CACHE_TTL_SECONDS = 30.0
|
||||
_CACHE_MAXSIZE = 1024
|
||||
# user_id -> (monotonic_ts, frozenset[doc_type] | None). None = no restriction.
|
||||
_cache: OrderedDict[str, tuple[float, frozenset[str] | None]] = OrderedDict()
|
||||
|
||||
|
||||
class _CapabilitiesClientProtocol(Protocol):
|
||||
async def capabilities(self) -> Any: ...
|
||||
|
||||
|
||||
def _parse_enabled_doc_types(payload: Any) -> frozenset[str] | None:
|
||||
"""Extract ``enabled_doc_types`` from an OCS capabilities payload.
|
||||
|
||||
Returns ``None`` when the ``astrolabe.semantic_search`` block is absent or
|
||||
malformed (treated as "no restriction"). Returns a frozenset (possibly
|
||||
empty) when the block is present and well-formed; an empty set means the
|
||||
admin disabled every source.
|
||||
"""
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
try:
|
||||
caps = payload["ocs"]["data"]["capabilities"]
|
||||
except (KeyError, TypeError):
|
||||
return None
|
||||
if not isinstance(caps, dict):
|
||||
return None
|
||||
block = caps.get("astrolabe")
|
||||
if not isinstance(block, dict):
|
||||
return None
|
||||
semantic = block.get("semantic_search")
|
||||
if not isinstance(semantic, dict):
|
||||
return None
|
||||
raw = semantic.get("enabled_doc_types")
|
||||
if not isinstance(raw, list):
|
||||
return None
|
||||
return frozenset(dt for dt in raw if isinstance(dt, str) and dt)
|
||||
|
||||
|
||||
async def allowed_doc_types(
|
||||
client: _CapabilitiesClientProtocol, user_id: str
|
||||
) -> frozenset[str] | None:
|
||||
"""Admin-approved doc types for ``user_id``, or ``None`` for "no restriction".
|
||||
|
||||
Cached per user with a short TTL (+ LRU eviction). Failures are not cached so
|
||||
a transient OCS hiccup retries on the next call. Fail-open: a missing
|
||||
capability block or an error yields ``None`` so search remains available.
|
||||
"""
|
||||
now = time.monotonic()
|
||||
cached = _cache.get(user_id)
|
||||
if cached is not None and now - cached[0] < _CACHE_TTL_SECONDS:
|
||||
_cache.move_to_end(user_id) # mark recently used (LRU)
|
||||
return cached[1]
|
||||
|
||||
try:
|
||||
payload = await client.capabilities()
|
||||
except Exception as exc: # noqa: BLE001 — degrade gracefully (fail-open)
|
||||
logger.warning(
|
||||
"Astrolabe capabilities unavailable for user %s (%s); "
|
||||
"not restricting doc types this cycle",
|
||||
user_id,
|
||||
exc,
|
||||
)
|
||||
return None # don't cache failures — retry next call
|
||||
|
||||
result = _parse_enabled_doc_types(payload)
|
||||
_cache[user_id] = (now, result)
|
||||
# Needed only for an existing (expired) key: __setitem__ updates it in place,
|
||||
# keeping its old position, so move it to the end to preserve LRU order. For
|
||||
# a brand-new key __setitem__ already appends, so this is a harmless no-op.
|
||||
_cache.move_to_end(user_id)
|
||||
while len(_cache) > _CACHE_MAXSIZE:
|
||||
_cache.popitem(last=False) # evict least-recently-used
|
||||
return result
|
||||
|
||||
|
||||
def is_doc_type_allowed(doc_type: str, allowed: frozenset[str] | None) -> bool:
|
||||
"""Whether ``doc_type`` may be indexed/searched given an allow-set.
|
||||
|
||||
``allowed=None`` means "no restriction" (fail-open / older Astrolabe), so
|
||||
everything is permitted.
|
||||
"""
|
||||
return allowed is None or doc_type in allowed
|
||||
|
||||
|
||||
def clear_cache() -> None:
|
||||
"""Test hook: drop all cached entries."""
|
||||
_cache.clear()
|
||||
@@ -20,6 +20,7 @@ from mcp.types import (
|
||||
from pydantic import Field
|
||||
|
||||
from nextcloud_mcp_server.auth import require_scopes
|
||||
from nextcloud_mcp_server.capabilities import allowed_doc_types
|
||||
from nextcloud_mcp_server.config import get_settings
|
||||
from nextcloud_mcp_server.context import get_client
|
||||
from nextcloud_mcp_server.models.semantic import (
|
||||
@@ -56,6 +57,25 @@ logger = logging.getLogger(__name__)
|
||||
_USAGE_METADATA_MAX_DOC_TYPES = 16
|
||||
|
||||
|
||||
def _consent_narrowed_doc_types(
|
||||
doc_types: list[str] | None, allowed: frozenset[str]
|
||||
) -> list[str]:
|
||||
"""Apply the admin allow-set to a requested ``doc_types`` filter.
|
||||
|
||||
Caller has already established ``allowed is not None`` (a concrete allow-set;
|
||||
``None`` means "no restriction" and is handled by skipping this call). When
|
||||
no explicit ``doc_types`` are requested, restrict to the full allow-set
|
||||
(returned ``sorted`` for determinism only — order is a filter, not a ranking
|
||||
hint); otherwise intersect (preserving the caller's order). An empty result
|
||||
means nothing the caller asked for is admin-approved — the caller
|
||||
short-circuits to an empty response rather than falling through to an
|
||||
all-types search.
|
||||
"""
|
||||
if doc_types is None:
|
||||
return sorted(allowed)
|
||||
return [dt for dt in doc_types if dt in allowed]
|
||||
|
||||
|
||||
async def record_search_usage(
|
||||
*,
|
||||
enabled: bool,
|
||||
@@ -300,6 +320,37 @@ def configure_semantic_tools(mcp: FastMCP):
|
||||
# files under their own user_id.
|
||||
accessible_owners = await list_accessible_owners(client.sharing, username)
|
||||
|
||||
# Admin consent gate: restrict to source types the Astrolabe admin has
|
||||
# approved (and that are installed for this user). This mirrors
|
||||
# Astrolabe's own server-side enforcement but is independent because
|
||||
# this tool queries Qdrant directly. ``None`` = no restriction
|
||||
# (fail-open / Astrolabe predating this feature). An empty allow-set
|
||||
# means the admin disabled every source.
|
||||
#
|
||||
# Perf trade-off (accepted): when Astrolabe is present and the caller
|
||||
# passed no doc_types, narrowing turns ``None`` into a concrete list, so
|
||||
# the search takes the per-type query branch (N queries) instead of the
|
||||
# single cross-type query. N is the count of admin-approved types
|
||||
# (typically 1-4), so the overhead is small; left as-is rather than
|
||||
# adding a "search all approved in one query" fast path.
|
||||
allowed = await allowed_doc_types(client, username)
|
||||
if allowed is not None:
|
||||
doc_types = _consent_narrowed_doc_types(doc_types, allowed)
|
||||
if not doc_types:
|
||||
logger.info(
|
||||
"Semantic search short-circuited for user %s: no requested "
|
||||
"doc_type is admin-approved for semantic search",
|
||||
username,
|
||||
)
|
||||
return SemanticSearchResponse(
|
||||
results=[],
|
||||
query=query,
|
||||
total_found=0,
|
||||
search_method=f"bm25_hybrid_{fusion}",
|
||||
verified_chunk_count=0,
|
||||
dropped_document_count=0,
|
||||
)
|
||||
|
||||
try:
|
||||
# The nc_semantic_search tool deliberately uses BM25-hybrid (dense +
|
||||
# sparse with RRF/DBSF fusion) as the single tool-layer algorithm.
|
||||
|
||||
@@ -21,6 +21,7 @@ if TYPE_CHECKING:
|
||||
from nextcloud_mcp_server.document_processors.registry import ProcessorRegistry
|
||||
|
||||
from nextcloud_mcp_server.acl_hash import compute_acl_hash
|
||||
from nextcloud_mcp_server.capabilities import allowed_doc_types, is_doc_type_allowed
|
||||
from nextcloud_mcp_server.client import NextcloudClient
|
||||
from nextcloud_mcp_server.config import get_settings
|
||||
from nextcloud_mcp_server.embedding import get_bm25_service, get_embedding_service
|
||||
@@ -590,6 +591,27 @@ async def process_document(
|
||||
):
|
||||
await _reconcile_tag_event(doc_task, nc_client)
|
||||
|
||||
# Admin consent gate (Astrolabe): never index a source the admin has
|
||||
# disabled for semantic search — this catches near-real-time webhook
|
||||
# events that bypass the scanner's discovery gate. Deletes always
|
||||
# proceed (removing data honours consent). ``None`` from the reader
|
||||
# means no restriction (fail-open / older Astrolabe), so a transient
|
||||
# capabilities failure never silently drops indexing.
|
||||
if doc_task.operation == "index":
|
||||
allowed = await allowed_doc_types(nc_client, doc_task.user_id)
|
||||
if not is_doc_type_allowed(doc_task.doc_type, allowed):
|
||||
logger.info(
|
||||
"Skipping index of %s_%s for %s: doc_type disabled by admin",
|
||||
doc_task.doc_type,
|
||||
doc_task.doc_id,
|
||||
doc_task.user_id,
|
||||
)
|
||||
# Alertable counter so a flood of webhook events for a
|
||||
# disabled source is observable (not silently swallowed).
|
||||
record_ingest_dropped("admin_disabled")
|
||||
record_vector_sync_processing(time.time() - start_time, "skipped")
|
||||
return
|
||||
|
||||
# Handle deletion
|
||||
if doc_task.operation == "delete":
|
||||
# Release this user rather than blind-delete: a file shared across
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Global purge of indexed vectors by doc type (admin consent enforcement).
|
||||
|
||||
When an admin disables a content source for semantic search in Astrolabe,
|
||||
consent is binding on data-at-rest: the already-indexed content for that
|
||||
source's doc type(s) must be deleted, not merely hidden. Astrolabe calls the
|
||||
``/api/v1/vector-sync/purge`` route on disable, which delegates here.
|
||||
|
||||
The purge is global (every owner) because the admin disable is a global
|
||||
decision. It is safe to call for a doc type with no indexed points — Qdrant
|
||||
deletes zero points and reports a count of 0.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from qdrant_client.models import FieldCondition, Filter, MatchValue
|
||||
|
||||
from nextcloud_mcp_server.config import get_settings
|
||||
from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _doc_type_filter(doc_type: str) -> Filter:
|
||||
return Filter(
|
||||
must=[FieldCondition(key="doc_type", match=MatchValue(value=doc_type))]
|
||||
)
|
||||
|
||||
|
||||
async def purge_doc_types(doc_types: list[str]) -> dict[str, int]:
|
||||
"""Delete every indexed point whose ``doc_type`` is in ``doc_types``.
|
||||
|
||||
Returns a mapping of doc_type -> number of points deleted (counted before
|
||||
deletion). Each doc type is purged independently so a failure on one does
|
||||
not abort the rest; failures re-raise after the loop only if every doc type
|
||||
failed, otherwise partial progress is returned.
|
||||
|
||||
The count is taken just before the delete (two separate Qdrant calls), so
|
||||
it is approximate — a point indexed in the gap is deleted but not counted.
|
||||
This is acceptable: indexing of a disabled source is already gated upstream,
|
||||
so the window is effectively empty in practice.
|
||||
"""
|
||||
qdrant_client = await get_qdrant_client()
|
||||
collection = get_settings().get_collection_name()
|
||||
|
||||
purged: dict[str, int] = {}
|
||||
last_error: Exception | None = None
|
||||
for doc_type in dict.fromkeys(doc_types): # de-dupe, preserve order
|
||||
flt = _doc_type_filter(doc_type)
|
||||
try:
|
||||
count_result = await qdrant_client.count(
|
||||
collection_name=collection,
|
||||
count_filter=flt,
|
||||
exact=True,
|
||||
)
|
||||
await qdrant_client.delete(
|
||||
collection_name=collection,
|
||||
points_selector=flt,
|
||||
)
|
||||
purged[doc_type] = int(count_result.count)
|
||||
logger.info(
|
||||
"Purged %d indexed point(s) for disabled doc_type=%s",
|
||||
purged[doc_type],
|
||||
doc_type,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — record and continue
|
||||
last_error = exc
|
||||
logger.exception(
|
||||
"Failed to purge indexed points for doc_type=%s",
|
||||
doc_type,
|
||||
)
|
||||
|
||||
if not purged and last_error is not None:
|
||||
# Nothing succeeded — surface the failure to the caller (HTTP 500).
|
||||
raise last_error
|
||||
return purged
|
||||
@@ -16,6 +16,7 @@ from httpx import HTTPStatusError
|
||||
from qdrant_client import AsyncQdrantClient
|
||||
from qdrant_client.models import FieldCondition, Filter, MatchValue, Record
|
||||
|
||||
from nextcloud_mcp_server.capabilities import allowed_doc_types, is_doc_type_allowed
|
||||
from nextcloud_mcp_server.client import NextcloudClient
|
||||
from nextcloud_mcp_server.client.news import NewsItemType
|
||||
from nextcloud_mcp_server.config import get_settings
|
||||
@@ -279,6 +280,150 @@ def _app_enabled(app_id: str, enabled_apps: set[str] | None) -> bool:
|
||||
return enabled_apps is None or app_id in enabled_apps
|
||||
|
||||
|
||||
def _should_scan(
|
||||
app_id: str,
|
||||
doc_type: str,
|
||||
enabled_apps: set[str] | None,
|
||||
allowed: frozenset[str] | None,
|
||||
) -> bool:
|
||||
"""Whether to scan ``app_id``: installed for the user AND admin-approved."""
|
||||
return _app_enabled(app_id, enabled_apps) and is_doc_type_allowed(doc_type, allowed)
|
||||
|
||||
|
||||
# Text doc types whose deletion-tracking lives *inside* their scan_* function,
|
||||
# so skipping that function (when admin-disabled) leaves indexed points with no
|
||||
# grace-period backstop. Derived from INDEXED_DOC_TYPES so a newly-indexed type
|
||||
# automatically gets the backstop. ``file`` is excluded: its scan path empties
|
||||
# discovery and lets the existing reconcile loop purge on disable.
|
||||
_TEXT_BACKSTOP_DOC_TYPES: tuple[str, ...] = tuple(sorted(INDEXED_DOC_TYPES - {"file"}))
|
||||
|
||||
# Per-process record of (user_id, doc_type) whose consent backstop deletes have
|
||||
# already been enqueued, so a *standing* admin-disable doesn't re-flood the
|
||||
# processor with idempotent deletes on every scan tick. An entry is cleared once
|
||||
# the type is allowed again, so a later re-disable re-triggers the backstop.
|
||||
# A dict (not a set) so it stays insertion-ordered for oldest-first eviction.
|
||||
_consent_backstop_done: dict[tuple[str, str], None] = {}
|
||||
|
||||
# Safety bound on the tracking dict so a long-running multi-tenant process with
|
||||
# heavy user churn (deprovisioned users leave stale entries) can't grow it
|
||||
# without limit. At <= len(INDEXED_DOC_TYPES) entries per user this is generous;
|
||||
# on overflow we evict the *oldest* entries down to half capacity (not a full
|
||||
# clear) so the backstop re-fires for only those, avoiding a fleet-wide burst.
|
||||
_CONSENT_BACKSTOP_MAX = 50_000
|
||||
|
||||
|
||||
def _mark_backstop_done(key: tuple[str, str]) -> None:
|
||||
"""Record a one-shot backstop marker, evicting oldest entries on overflow.
|
||||
|
||||
Evicts oldest-first down to half capacity (insertion-ordered dict) rather
|
||||
than clearing wholesale, so a bound hit re-fires the backstop only for the
|
||||
oldest markers, not the whole fleet. A re-fire is idempotent regardless.
|
||||
"""
|
||||
if len(_consent_backstop_done) >= _CONSENT_BACKSTOP_MAX:
|
||||
overage = len(_consent_backstop_done) - _CONSENT_BACKSTOP_MAX // 2
|
||||
logger.info(
|
||||
"consent backstop tracking hit %d entries; evicting %d oldest",
|
||||
_CONSENT_BACKSTOP_MAX,
|
||||
overage,
|
||||
)
|
||||
for stale_key in list(_consent_backstop_done)[:overage]:
|
||||
del _consent_backstop_done[stale_key]
|
||||
_consent_backstop_done[key] = None
|
||||
|
||||
|
||||
async def _backstop_delete_doc_type(
|
||||
user_id: str,
|
||||
send_stream: TaskProducer,
|
||||
doc_type: str,
|
||||
qdrant_client: AsyncQdrantClient,
|
||||
collection: str,
|
||||
scan_id: int,
|
||||
) -> int:
|
||||
"""Enqueue delete tasks for every indexed point of one disabled doc_type.
|
||||
|
||||
Returns the number of delete tasks enqueued.
|
||||
"""
|
||||
points = await _scroll_all_points(
|
||||
qdrant_client,
|
||||
collection_name=collection,
|
||||
scroll_filter=Filter(
|
||||
must=[
|
||||
FieldCondition(key="user_id", match=MatchValue(value=user_id)),
|
||||
FieldCondition(key="doc_type", match=MatchValue(value=doc_type)),
|
||||
]
|
||||
),
|
||||
payload_fields=["doc_id"],
|
||||
)
|
||||
doc_ids = {
|
||||
str(p.payload["doc_id"])
|
||||
for p in points
|
||||
if p.payload is not None and "doc_id" in p.payload
|
||||
}
|
||||
if doc_ids:
|
||||
logger.info(
|
||||
"[SCAN-%s] %s disabled by admin for %s; enqueueing %d delete(s) (backstop)",
|
||||
scan_id,
|
||||
doc_type,
|
||||
user_id,
|
||||
len(doc_ids),
|
||||
)
|
||||
for doc_id in doc_ids:
|
||||
await send_stream.send(
|
||||
DocumentTask(
|
||||
user_id=user_id,
|
||||
doc_id=doc_id,
|
||||
doc_type=doc_type,
|
||||
operation="delete",
|
||||
modified_at=0,
|
||||
)
|
||||
)
|
||||
return len(doc_ids)
|
||||
|
||||
|
||||
async def _enqueue_deletes_for_disabled_types(
|
||||
user_id: str,
|
||||
send_stream: TaskProducer,
|
||||
allowed: frozenset[str] | None,
|
||||
scan_id: int,
|
||||
) -> int:
|
||||
"""Enqueue delete tasks for indexed text-source points the admin disabled.
|
||||
|
||||
Backstop for a failed eager purge: scrolls this user's indexed points for
|
||||
each admin-disallowed text doc_type and queues a delete. No-op when
|
||||
``allowed`` is ``None`` (fail-open — never delete on a transient capability
|
||||
read failure). Returns the number of delete tasks enqueued.
|
||||
"""
|
||||
if allowed is None:
|
||||
return 0
|
||||
|
||||
# Re-enabled types: clear their one-shot marker so a later re-disable
|
||||
# re-triggers the backstop.
|
||||
for doc_type in _TEXT_BACKSTOP_DOC_TYPES:
|
||||
if doc_type in allowed:
|
||||
_consent_backstop_done.pop((user_id, doc_type), None)
|
||||
|
||||
# Disabled types not yet backstopped this episode.
|
||||
disabled = [
|
||||
dt
|
||||
for dt in _TEXT_BACKSTOP_DOC_TYPES
|
||||
if dt not in allowed and (user_id, dt) not in _consent_backstop_done
|
||||
]
|
||||
if not disabled:
|
||||
return 0
|
||||
|
||||
qdrant_client = await get_qdrant_client()
|
||||
collection = get_settings().get_collection_name()
|
||||
queued = 0
|
||||
for doc_type in disabled:
|
||||
queued += await _backstop_delete_doc_type(
|
||||
user_id, send_stream, doc_type, qdrant_client, collection, scan_id
|
||||
)
|
||||
# Mark backstopped (even when nothing was found) so subsequent scans
|
||||
# don't re-scroll/re-enqueue for this disable episode.
|
||||
_mark_backstop_done((user_id, doc_type))
|
||||
return queued
|
||||
|
||||
|
||||
async def scan_user_documents(
|
||||
user_id: str,
|
||||
send_stream: TaskProducer,
|
||||
@@ -365,6 +510,13 @@ async def scan_user_documents(
|
||||
# detection failed: fall back to scanning every app (prior behaviour).
|
||||
enabled_apps = await _get_enabled_apps_or_none(nc_client, user_id, scan_id)
|
||||
|
||||
# Admin consent gate (Astrolabe): only index sources the admin has
|
||||
# approved for semantic search. ``None`` = no restriction (fail-open /
|
||||
# older Astrolabe), so a transient capabilities failure never silently
|
||||
# halts (or worse, mass-deletes) indexing. This is independent of
|
||||
# ``enabled_apps``, which reflects only what the user has installed.
|
||||
allowed = await allowed_doc_types(nc_client, user_id)
|
||||
|
||||
# Notes (isolated so an uninstalled or disabled Notes app — whose API
|
||||
# returns 404 — cannot abort scanning of the other apps; this mirrors the
|
||||
# per-app try/except guards already wrapping files/news/deck below).
|
||||
@@ -373,7 +525,17 @@ async def scan_user_documents(
|
||||
current_time = time.time()
|
||||
queued = 0
|
||||
|
||||
if _app_enabled("notes", enabled_apps):
|
||||
# Backstop purge for admin-disabled text sources. Their deletion-
|
||||
# tracking lives inside the scan_* function we skip below, so (unlike
|
||||
# files, whose discovery-empties-then-reconcile path purges on disable)
|
||||
# they'd linger if Astrolabe's eager purge failed. Enqueue deletes for
|
||||
# any indexed points of a now-disallowed type. Gated on a concrete
|
||||
# allow-set, so a fail-open None never triggers deletion.
|
||||
queued += await _enqueue_deletes_for_disabled_types(
|
||||
user_id, send_stream, allowed, scan_id
|
||||
)
|
||||
|
||||
if _should_scan("notes", "note", enabled_apps, allowed):
|
||||
try:
|
||||
queued += await scan_notes(
|
||||
user_id=user_id,
|
||||
@@ -454,9 +616,24 @@ async def scan_user_documents(
|
||||
# folder applies to every PDF beneath it.
|
||||
settings = get_settings()
|
||||
tag_name = settings.vector_sync_pdf_tag
|
||||
if is_doc_type_allowed("file", allowed):
|
||||
tagged_files = await nc_client.find_files_by_tag(
|
||||
tag_name, mime_type_filter="application/pdf"
|
||||
)
|
||||
else:
|
||||
# Files disabled by admin: discover nothing so no new file is
|
||||
# indexed. The deletion-reconcile below then sees every indexed
|
||||
# file as "missing" and purges it after the grace period — the
|
||||
# backstop for the eager purge Astrolabe runs on disable.
|
||||
# Asymmetry (intentional): files purge up to 1.5x scan_interval
|
||||
# later than text types, which get immediate one-shot backstop
|
||||
# deletes via _enqueue_deletes_for_disabled_types.
|
||||
logger.debug(
|
||||
"[SCAN-%s] Files disabled by admin for %s; skipping tagged-file discovery",
|
||||
scan_id,
|
||||
user_id,
|
||||
)
|
||||
tagged_files = []
|
||||
|
||||
# Apply EXCLUDED_TAGS as defense-in-depth: a folder marked
|
||||
# off-limits via the exclusion tag must not be indexed even if
|
||||
@@ -710,7 +887,7 @@ async def scan_user_documents(
|
||||
|
||||
# Scan News items (starred + unread)
|
||||
news_queued = 0
|
||||
if _app_enabled("news", enabled_apps):
|
||||
if _should_scan("news", "news_item", enabled_apps, allowed):
|
||||
try:
|
||||
news_queued = await scan_news_items(
|
||||
user_id=user_id,
|
||||
@@ -731,7 +908,7 @@ async def scan_user_documents(
|
||||
|
||||
# Scan Deck cards
|
||||
deck_queued = 0
|
||||
if _app_enabled("deck", enabled_apps):
|
||||
if _should_scan("deck", "deck_card", enabled_apps, allowed):
|
||||
try:
|
||||
deck_queued = await scan_deck_cards(
|
||||
user_id=user_id,
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Unit tests for the search-side consent narrowing in nc_semantic_search.
|
||||
|
||||
The narrowing logic (intersect requested doc_types with the admin allow-set,
|
||||
or restrict to the allow-set when none requested) is extracted into
|
||||
``_consent_narrowed_doc_types`` so it can be tested without exercising the full
|
||||
search path. ``allowed is None`` (no restriction / fail-open) is handled by the
|
||||
caller skipping this helper entirely.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from nextcloud_mcp_server.server.semantic import _consent_narrowed_doc_types
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def test_none_request_restricts_to_allow_set():
|
||||
# No explicit doc_types -> search exactly the allowed set (sorted).
|
||||
assert _consent_narrowed_doc_types(None, frozenset({"file", "note"})) == [
|
||||
"file",
|
||||
"note",
|
||||
]
|
||||
|
||||
|
||||
def test_request_intersected_with_allow_set_preserves_order():
|
||||
result = _consent_narrowed_doc_types(
|
||||
["deck_card", "note", "file"], frozenset({"note", "file"})
|
||||
)
|
||||
assert result == ["note", "file"]
|
||||
|
||||
|
||||
def test_disjoint_request_yields_empty():
|
||||
# Caller short-circuits to an empty response on [].
|
||||
assert _consent_narrowed_doc_types(["deck_card"], frozenset({"note"})) == []
|
||||
|
||||
|
||||
def test_empty_allow_set_blocks_all():
|
||||
assert _consent_narrowed_doc_types(None, frozenset()) == []
|
||||
assert _consent_narrowed_doc_types(["note"], frozenset()) == []
|
||||
@@ -0,0 +1,162 @@
|
||||
"""Unit tests for the Astrolabe searchable-sources capability reader."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
import nextcloud_mcp_server.capabilities as cap
|
||||
from nextcloud_mcp_server.capabilities import (
|
||||
_parse_enabled_doc_types,
|
||||
allowed_doc_types,
|
||||
clear_cache,
|
||||
is_doc_type_allowed,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def _payload(enabled_doc_types) -> dict:
|
||||
"""Build an OCS capabilities envelope carrying the astrolabe block.
|
||||
|
||||
``enabled_doc_types=...`` (Ellipsis) omits the key entirely.
|
||||
"""
|
||||
semantic: dict = {}
|
||||
if enabled_doc_types is not ...:
|
||||
semantic["enabled_doc_types"] = enabled_doc_types
|
||||
return {
|
||||
"ocs": {
|
||||
"meta": {"status": "ok"},
|
||||
"data": {"capabilities": {"astrolabe": {"semantic_search": semantic}}},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _parse_enabled_doc_types
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_parse_present_list_returns_set():
|
||||
assert _parse_enabled_doc_types(_payload(["note", "file"])) == {"note", "file"}
|
||||
|
||||
|
||||
def test_parse_empty_list_returns_empty_set():
|
||||
# Admin disabled every source — distinct from "no restriction".
|
||||
assert _parse_enabled_doc_types(_payload([])) == set()
|
||||
|
||||
|
||||
def test_parse_missing_astrolabe_block_returns_none():
|
||||
payload = {"ocs": {"data": {"capabilities": {}}}}
|
||||
assert _parse_enabled_doc_types(payload) is None
|
||||
|
||||
|
||||
def test_parse_missing_enabled_key_returns_none():
|
||||
assert _parse_enabled_doc_types(_payload(...)) is None
|
||||
|
||||
|
||||
def test_parse_malformed_payload_returns_none():
|
||||
assert _parse_enabled_doc_types(None) is None
|
||||
assert _parse_enabled_doc_types({"ocs": "nope"}) is None
|
||||
assert _parse_enabled_doc_types(_payload("not-a-list")) is None
|
||||
|
||||
|
||||
def test_parse_drops_non_string_entries():
|
||||
assert _parse_enabled_doc_types(_payload(["note", 5, None])) == {"note"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# is_doc_type_allowed
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_is_doc_type_allowed_none_means_no_restriction():
|
||||
assert is_doc_type_allowed("anything", None) is True
|
||||
|
||||
|
||||
def test_is_doc_type_allowed_respects_set():
|
||||
allowed = frozenset({"note"})
|
||||
assert is_doc_type_allowed("note", allowed) is True
|
||||
assert is_doc_type_allowed("file", allowed) is False
|
||||
|
||||
|
||||
def test_is_doc_type_allowed_empty_set_blocks_all():
|
||||
assert is_doc_type_allowed("note", frozenset()) is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# allowed_doc_types (cache + fail-open)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _client(payload=None, raises: Exception | None = None) -> AsyncMock:
|
||||
"""An object with an async ``capabilities()`` method (AsyncMock-backed)."""
|
||||
m = AsyncMock()
|
||||
if raises is not None:
|
||||
m.capabilities.side_effect = raises
|
||||
else:
|
||||
m.capabilities.return_value = payload
|
||||
return m
|
||||
|
||||
|
||||
async def test_allowed_doc_types_parses_and_caches():
|
||||
clear_cache()
|
||||
client = _client(_payload(["note", "file"]))
|
||||
|
||||
first = await allowed_doc_types(client, "alice")
|
||||
second = await allowed_doc_types(client, "alice")
|
||||
|
||||
assert first == frozenset({"note", "file"})
|
||||
assert second == frozenset({"note", "file"})
|
||||
# Second call served from the cache — only one OCS round-trip.
|
||||
assert client.capabilities.await_count == 1
|
||||
|
||||
|
||||
async def test_allowed_doc_types_missing_block_returns_none():
|
||||
clear_cache()
|
||||
client = _client({"ocs": {"data": {"capabilities": {}}}})
|
||||
assert await allowed_doc_types(client, "bob") is None
|
||||
|
||||
|
||||
async def test_allowed_doc_types_fail_open_not_cached():
|
||||
clear_cache()
|
||||
client = _client(raises=RuntimeError("ocs down"))
|
||||
|
||||
assert await allowed_doc_types(client, "carol") is None
|
||||
# Failures are not cached — the next call retries the OCS lookup.
|
||||
assert await allowed_doc_types(client, "carol") is None
|
||||
assert client.capabilities.await_count == 2
|
||||
|
||||
|
||||
async def test_allowed_doc_types_cache_is_per_user():
|
||||
clear_cache()
|
||||
alice = _client(_payload(["note"]))
|
||||
bob = _client(_payload(["file"]))
|
||||
|
||||
assert await allowed_doc_types(alice, "alice") == frozenset({"note"})
|
||||
assert await allowed_doc_types(bob, "bob") == frozenset({"file"})
|
||||
|
||||
|
||||
async def test_allowed_doc_types_refetches_after_ttl(monkeypatch):
|
||||
clear_cache()
|
||||
client = _client(_payload(["note"]))
|
||||
|
||||
# Drive the module clock so the second call lands past the TTL window.
|
||||
clock = {"now": 1000.0}
|
||||
monkeypatch.setattr(cap.time, "monotonic", lambda: clock["now"])
|
||||
|
||||
await allowed_doc_types(client, "erin")
|
||||
clock["now"] += cap._CACHE_TTL_SECONDS + 1
|
||||
await allowed_doc_types(client, "erin")
|
||||
|
||||
assert client.capabilities.await_count == 2
|
||||
|
||||
|
||||
async def test_clear_cache_forces_refetch():
|
||||
clear_cache()
|
||||
client = _client(_payload(["note"]))
|
||||
await allowed_doc_types(client, "dave")
|
||||
cap.clear_cache()
|
||||
await allowed_doc_types(client, "dave")
|
||||
assert client.capabilities.await_count == 2
|
||||
@@ -72,7 +72,7 @@ def test_nested_exception_group_descends_to_leaf():
|
||||
def test_qdrant_namespace_classified():
|
||||
from qdrant_client.http.exceptions import UnexpectedResponse
|
||||
|
||||
exc = UnexpectedResponse(500, "err", b"", headers=None)
|
||||
exc = UnexpectedResponse(500, "err", b"", headers=httpx.Headers())
|
||||
assert processor._drop_reason(exc) == "qdrant"
|
||||
|
||||
|
||||
@@ -109,3 +109,71 @@ async def test_process_document_records_drop_on_exhausted_retries(mocker):
|
||||
await processor.process_document(doc_task, mocker.MagicMock(), max_retries=1)
|
||||
|
||||
rec.assert_called_once_with("connection")
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_process_document_drops_admin_disabled_index_task(mocker):
|
||||
"""A near-real-time index task for an admin-disabled doc_type is dropped
|
||||
before indexing, and recorded under the ``admin_disabled`` reason."""
|
||||
from nextcloud_mcp_server.vector.scanner import DocumentTask
|
||||
|
||||
doc_task = DocumentTask(
|
||||
user_id="alice",
|
||||
doc_id="42",
|
||||
doc_type="note",
|
||||
operation="index",
|
||||
modified_at=0,
|
||||
file_path="/x.md", # set so the tag-reconcile branch is skipped
|
||||
)
|
||||
|
||||
mocker.patch.object(
|
||||
processor,
|
||||
"get_qdrant_client",
|
||||
mocker.AsyncMock(return_value=mocker.MagicMock()),
|
||||
)
|
||||
# Admin disabled everything → note is not allowed.
|
||||
mocker.patch.object(
|
||||
processor, "allowed_doc_types", mocker.AsyncMock(return_value=frozenset())
|
||||
)
|
||||
index = mocker.patch.object(processor, "_index_document")
|
||||
rec = mocker.patch.object(processor, "record_ingest_dropped")
|
||||
|
||||
await processor.process_document(doc_task, mocker.MagicMock(), max_retries=1)
|
||||
|
||||
index.assert_not_called()
|
||||
rec.assert_called_once_with("admin_disabled")
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_process_document_allows_when_doc_type_approved(mocker):
|
||||
"""The consent gate does not drop an index task for an allowed doc_type."""
|
||||
from nextcloud_mcp_server.vector.scanner import DocumentTask
|
||||
|
||||
doc_task = DocumentTask(
|
||||
user_id="alice",
|
||||
doc_id="42",
|
||||
doc_type="note",
|
||||
operation="index",
|
||||
modified_at=0,
|
||||
file_path="/x.md",
|
||||
)
|
||||
|
||||
mocker.patch.object(
|
||||
processor,
|
||||
"get_qdrant_client",
|
||||
mocker.AsyncMock(return_value=mocker.MagicMock()),
|
||||
)
|
||||
mocker.patch.object(
|
||||
processor,
|
||||
"allowed_doc_types",
|
||||
mocker.AsyncMock(return_value=frozenset({"note"})),
|
||||
)
|
||||
index = mocker.patch.object(
|
||||
processor, "_index_document", mocker.AsyncMock(return_value=1)
|
||||
)
|
||||
rec = mocker.patch.object(processor, "record_ingest_dropped")
|
||||
|
||||
await processor.process_document(doc_task, mocker.MagicMock(), max_retries=1)
|
||||
|
||||
index.assert_awaited() # indexing proceeded
|
||||
rec.assert_not_called()
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
"""Unit tests for the /api/v1/vector-sync/purge admin route.
|
||||
|
||||
The purge is global and destructive (deletes every owner's content for a doc
|
||||
type), so the route must: authenticate the bearer, restrict to Nextcloud
|
||||
admins, validate the body, and only then delegate to the global purge.
|
||||
"""
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from starlette.applications import Starlette
|
||||
from starlette.routing import Route
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
from nextcloud_mcp_server.api.vector_sync import purge_doc_types_route
|
||||
from nextcloud_mcp_server.auth.scope_authorization import ProvisioningRequiredError
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def _build_app() -> Starlette:
|
||||
app = Starlette(
|
||||
routes=[
|
||||
Route(
|
||||
"/api/v1/vector-sync/purge",
|
||||
purge_doc_types_route,
|
||||
methods=["POST"],
|
||||
)
|
||||
]
|
||||
)
|
||||
app.state.oauth_context = {"config": {"nextcloud_host": "http://nc.test"}}
|
||||
return app
|
||||
|
||||
|
||||
def _patch_token(mocker, user_id="admin"):
|
||||
mocker.patch(
|
||||
"nextcloud_mcp_server.api.vector_sync.validate_token_and_get_user",
|
||||
new=AsyncMock(return_value=(user_id, {"sub": user_id})),
|
||||
)
|
||||
|
||||
|
||||
def _patch_basic_auth(mocker, username="admin"):
|
||||
mocker.patch(
|
||||
"nextcloud_mcp_server.api.vector_sync.get_basic_auth_for_user",
|
||||
new=AsyncMock(return_value=(username, "app-pwd")),
|
||||
)
|
||||
|
||||
|
||||
def _patch_outbound_client(mocker):
|
||||
client = AsyncMock()
|
||||
client.__aenter__ = AsyncMock(return_value=client)
|
||||
client.__aexit__ = AsyncMock(return_value=False)
|
||||
mocker.patch(
|
||||
"nextcloud_mcp_server.api.vector_sync.nextcloud_httpx_client",
|
||||
MagicMock(return_value=client),
|
||||
)
|
||||
return client
|
||||
|
||||
|
||||
def _patch_groups(mocker, groups):
|
||||
instance = MagicMock()
|
||||
instance.get_user_groups = AsyncMock(return_value=groups)
|
||||
mocker.patch(
|
||||
"nextcloud_mcp_server.api.vector_sync.UsersClient",
|
||||
MagicMock(return_value=instance),
|
||||
)
|
||||
|
||||
|
||||
def _patch_purge(mocker, result=None):
|
||||
return mocker.patch(
|
||||
"nextcloud_mcp_server.api.vector_sync.purge_doc_types",
|
||||
new=AsyncMock(return_value=result or {}),
|
||||
)
|
||||
|
||||
|
||||
def test_unauthorized_when_token_invalid(mocker):
|
||||
mocker.patch(
|
||||
"nextcloud_mcp_server.api.vector_sync.validate_token_and_get_user",
|
||||
new=AsyncMock(side_effect=ValueError("bad token")),
|
||||
)
|
||||
purge = _patch_purge(mocker)
|
||||
|
||||
client = TestClient(_build_app())
|
||||
resp = client.post("/api/v1/vector-sync/purge", json={"doc_types": ["file"]})
|
||||
|
||||
assert resp.status_code == 401
|
||||
purge.assert_not_called()
|
||||
|
||||
|
||||
def test_bad_request_when_doc_types_not_list(mocker):
|
||||
_patch_token(mocker)
|
||||
purge = _patch_purge(mocker)
|
||||
|
||||
client = TestClient(_build_app())
|
||||
resp = client.post("/api/v1/vector-sync/purge", json={"doc_types": "file"})
|
||||
|
||||
assert resp.status_code == 400
|
||||
purge.assert_not_called()
|
||||
|
||||
|
||||
def test_bad_request_when_doc_types_has_non_string(mocker):
|
||||
# Covers the all(isinstance(d, str)) branch (a list with non-string items).
|
||||
_patch_token(mocker)
|
||||
purge = _patch_purge(mocker)
|
||||
|
||||
client = TestClient(_build_app())
|
||||
resp = client.post("/api/v1/vector-sync/purge", json={"doc_types": [1, 2]})
|
||||
|
||||
assert resp.status_code == 400
|
||||
purge.assert_not_called()
|
||||
|
||||
|
||||
def test_total_failure_returns_500(mocker):
|
||||
# purge_doc_types raising (total failure) hits the route's except -> 500.
|
||||
_patch_token(mocker, "admin")
|
||||
_patch_basic_auth(mocker, "admin")
|
||||
_patch_outbound_client(mocker)
|
||||
_patch_groups(mocker, ["admin"])
|
||||
mocker.patch(
|
||||
"nextcloud_mcp_server.api.vector_sync.purge_doc_types",
|
||||
new=AsyncMock(side_effect=RuntimeError("qdrant down")),
|
||||
)
|
||||
|
||||
client = TestClient(_build_app())
|
||||
resp = client.post("/api/v1/vector-sync/purge", json={"doc_types": ["file"]})
|
||||
|
||||
assert resp.status_code == 500
|
||||
|
||||
|
||||
def test_forbidden_when_not_admin(mocker):
|
||||
_patch_token(mocker, "bob")
|
||||
_patch_basic_auth(mocker, "bob")
|
||||
_patch_outbound_client(mocker)
|
||||
_patch_groups(mocker, ["users"]) # not an admin
|
||||
purge = _patch_purge(mocker)
|
||||
|
||||
client = TestClient(_build_app())
|
||||
resp = client.post("/api/v1/vector-sync/purge", json={"doc_types": ["file"]})
|
||||
|
||||
assert resp.status_code == 403
|
||||
purge.assert_not_called()
|
||||
|
||||
|
||||
def test_missing_doc_types_key_returns_400(mocker):
|
||||
_patch_token(mocker)
|
||||
purge = _patch_purge(mocker)
|
||||
|
||||
client = TestClient(_build_app())
|
||||
resp = client.post("/api/v1/vector-sync/purge", json={})
|
||||
|
||||
assert resp.status_code == 400
|
||||
purge.assert_not_called()
|
||||
|
||||
|
||||
def test_empty_doc_types_is_admin_gated_noop(mocker):
|
||||
# An empty (no-op) request still requires admin — this is a destructive route.
|
||||
_patch_token(mocker, "admin")
|
||||
_patch_basic_auth(mocker, "admin")
|
||||
_patch_outbound_client(mocker)
|
||||
_patch_groups(mocker, ["admin"])
|
||||
purge = _patch_purge(mocker)
|
||||
|
||||
client = TestClient(_build_app())
|
||||
resp = client.post("/api/v1/vector-sync/purge", json={"doc_types": []})
|
||||
|
||||
assert resp.status_code == 200
|
||||
assert resp.json() == {"purged": {}}
|
||||
purge.assert_not_called()
|
||||
|
||||
|
||||
def test_empty_doc_types_forbidden_for_non_admin(mocker):
|
||||
_patch_token(mocker, "bob")
|
||||
_patch_basic_auth(mocker, "bob")
|
||||
_patch_outbound_client(mocker)
|
||||
_patch_groups(mocker, ["users"])
|
||||
purge = _patch_purge(mocker)
|
||||
|
||||
client = TestClient(_build_app())
|
||||
resp = client.post("/api/v1/vector-sync/purge", json={"doc_types": []})
|
||||
|
||||
assert resp.status_code == 403
|
||||
purge.assert_not_called()
|
||||
|
||||
|
||||
def test_admin_purge_happy_path(mocker):
|
||||
_patch_token(mocker, "admin")
|
||||
_patch_basic_auth(mocker, "admin")
|
||||
_patch_outbound_client(mocker)
|
||||
_patch_groups(mocker, ["admin"])
|
||||
purge = _patch_purge(mocker, {"file": 12})
|
||||
|
||||
client = TestClient(_build_app())
|
||||
resp = client.post("/api/v1/vector-sync/purge", json={"doc_types": ["file"]})
|
||||
|
||||
assert resp.status_code == 200
|
||||
assert resp.json() == {"purged": {"file": 12}}
|
||||
purge.assert_awaited_once_with(["file"])
|
||||
|
||||
|
||||
def test_partial_failure_reports_failed_types(mocker):
|
||||
# purge_doc_types returns only the succeeded types; the route must tell the
|
||||
# caller which requested types were NOT purged.
|
||||
_patch_token(mocker, "admin")
|
||||
_patch_basic_auth(mocker, "admin")
|
||||
_patch_outbound_client(mocker)
|
||||
_patch_groups(mocker, ["admin"])
|
||||
_patch_purge(mocker, {"file": 3}) # "note" failed
|
||||
|
||||
client = TestClient(_build_app())
|
||||
resp = client.post(
|
||||
"/api/v1/vector-sync/purge", json={"doc_types": ["file", "note"]}
|
||||
)
|
||||
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["purged"] == {"file": 3}
|
||||
assert body["failed"] == ["note"]
|
||||
|
||||
|
||||
def test_bad_request_when_body_not_object(mocker):
|
||||
# A valid JSON non-object (e.g. a list) must 400, not 500.
|
||||
_patch_token(mocker)
|
||||
purge = _patch_purge(mocker)
|
||||
|
||||
client = TestClient(_build_app())
|
||||
resp = client.post("/api/v1/vector-sync/purge", json=[1, 2, 3])
|
||||
|
||||
assert resp.status_code == 400
|
||||
purge.assert_not_called()
|
||||
|
||||
|
||||
def test_bad_request_when_too_many_doc_types(mocker):
|
||||
_patch_token(mocker)
|
||||
purge = _patch_purge(mocker)
|
||||
|
||||
client = TestClient(_build_app())
|
||||
resp = client.post(
|
||||
"/api/v1/vector-sync/purge",
|
||||
json={"doc_types": [f"t{i}" for i in range(65)]},
|
||||
)
|
||||
|
||||
assert resp.status_code == 400
|
||||
purge.assert_not_called()
|
||||
|
||||
|
||||
def test_provisioning_required_returns_428(mocker):
|
||||
_patch_token(mocker, "admin")
|
||||
mocker.patch(
|
||||
"nextcloud_mcp_server.api.vector_sync.get_basic_auth_for_user",
|
||||
new=AsyncMock(side_effect=ProvisioningRequiredError("not provisioned")),
|
||||
)
|
||||
purge = _patch_purge(mocker)
|
||||
|
||||
client = TestClient(_build_app())
|
||||
resp = client.post("/api/v1/vector-sync/purge", json={"doc_types": ["file"]})
|
||||
|
||||
assert resp.status_code == 428
|
||||
purge.assert_not_called()
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Unit tests for global purge-by-doc-type (admin consent enforcement)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
import nextcloud_mcp_server.vector.purge as purge_module
|
||||
from nextcloud_mcp_server.vector.purge import purge_doc_types
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def _patch_qdrant(monkeypatch, *, counts: dict[str, int], delete_raises=None):
|
||||
"""Wire a fake Qdrant client whose ``count`` reflects ``counts`` per
|
||||
doc_type (read off the filter's MatchValue) and whose ``delete`` optionally
|
||||
raises for given doc_types."""
|
||||
client = AsyncMock()
|
||||
|
||||
def _doc_type_of(flt):
|
||||
return flt.must[0].match.value
|
||||
|
||||
# Sync side_effects: AsyncMock awaits the call and returns the value, so the
|
||||
# helpers don't need to be coroutines themselves.
|
||||
def fake_count(*, collection_name, count_filter, exact):
|
||||
return SimpleNamespace(count=counts.get(_doc_type_of(count_filter), 0))
|
||||
|
||||
def fake_delete(*, collection_name, points_selector):
|
||||
dt = _doc_type_of(points_selector)
|
||||
if delete_raises and dt in delete_raises:
|
||||
raise RuntimeError(f"delete failed for {dt}")
|
||||
|
||||
client.count.side_effect = fake_count
|
||||
client.delete.side_effect = fake_delete
|
||||
|
||||
monkeypatch.setattr(
|
||||
purge_module, "get_qdrant_client", AsyncMock(return_value=client)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
purge_module,
|
||||
"get_settings",
|
||||
lambda: SimpleNamespace(get_collection_name=lambda: "test_collection"),
|
||||
)
|
||||
return client
|
||||
|
||||
|
||||
async def test_purges_each_doc_type_and_reports_counts(monkeypatch):
|
||||
client = _patch_qdrant(monkeypatch, counts={"file": 7, "note": 3})
|
||||
|
||||
result = await purge_doc_types(["file", "note"])
|
||||
|
||||
assert result == {"file": 7, "note": 3}
|
||||
assert client.delete.await_count == 2
|
||||
|
||||
|
||||
async def test_purge_is_owner_agnostic_global(monkeypatch):
|
||||
# The admin disable is global, so the delete filter must match by doc_type
|
||||
# ONLY — no owner_id/user_id condition that would scope it to one user.
|
||||
client = _patch_qdrant(monkeypatch, counts={"file": 1})
|
||||
|
||||
await purge_doc_types(["file"])
|
||||
|
||||
flt = client.delete.await_args.kwargs["points_selector"]
|
||||
keys = [c.key for c in flt.must]
|
||||
assert keys == ["doc_type"]
|
||||
assert flt.must[0].match.value == "file"
|
||||
|
||||
|
||||
async def test_dedupes_doc_types(monkeypatch):
|
||||
client = _patch_qdrant(monkeypatch, counts={"file": 2})
|
||||
|
||||
result = await purge_doc_types(["file", "file"])
|
||||
|
||||
assert result == {"file": 2}
|
||||
assert client.delete.await_count == 1
|
||||
|
||||
|
||||
async def test_zero_points_is_safe(monkeypatch):
|
||||
_patch_qdrant(monkeypatch, counts={})
|
||||
assert await purge_doc_types(["deck_card"]) == {"deck_card": 0}
|
||||
|
||||
|
||||
async def test_partial_failure_returns_partial(monkeypatch):
|
||||
_patch_qdrant(
|
||||
monkeypatch,
|
||||
counts={"file": 5, "note": 4},
|
||||
delete_raises={"note"},
|
||||
)
|
||||
# "note" delete fails, "file" succeeds — partial progress is returned.
|
||||
result = await purge_doc_types(["file", "note"])
|
||||
assert result == {"file": 5}
|
||||
|
||||
|
||||
async def test_total_failure_raises(monkeypatch):
|
||||
_patch_qdrant(monkeypatch, counts={"file": 5}, delete_raises={"file"})
|
||||
with pytest.raises(RuntimeError):
|
||||
await purge_doc_types(["file"])
|
||||
@@ -0,0 +1,151 @@
|
||||
"""Unit tests for the scanner's admin-consent backstop deletion.
|
||||
|
||||
When an admin disables a text source (note/news_item/deck_card), the scanner
|
||||
skips its scan_* function, so the in-function deletion-tracking never runs. The
|
||||
backstop enqueues deletes for any indexed points of the disabled type, mirroring
|
||||
the files path — but only on a concrete allow-set (never on fail-open None).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import cast
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from nextcloud_mcp_server.vector import scanner as scanner_module
|
||||
from nextcloud_mcp_server.vector.queue.ports import TaskProducer
|
||||
from nextcloud_mcp_server.vector.scanner import _enqueue_deletes_for_disabled_types
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_backstop_state():
|
||||
"""The one-shot guard is module-level; reset it between tests."""
|
||||
scanner_module._consent_backstop_done.clear()
|
||||
yield
|
||||
scanner_module._consent_backstop_done.clear()
|
||||
|
||||
|
||||
def _producer(send: AsyncMock) -> TaskProducer:
|
||||
"""A minimal stand-in for the TaskProducer protocol (only ``send`` is used)."""
|
||||
return cast(TaskProducer, SimpleNamespace(send=send))
|
||||
|
||||
|
||||
def _patch_qdrant(monkeypatch, points_by_type: dict[str, list[str]]):
|
||||
client = AsyncMock()
|
||||
|
||||
def fake_scroll(
|
||||
*, collection_name, scroll_filter, with_payload, with_vectors, limit, offset
|
||||
):
|
||||
# must=[user_id, doc_type] — doc_type is the second condition.
|
||||
doc_type = scroll_filter.must[1].match.value
|
||||
points = [
|
||||
SimpleNamespace(payload={"doc_id": doc_id})
|
||||
for doc_id in points_by_type.get(doc_type, [])
|
||||
]
|
||||
return (points, None)
|
||||
|
||||
client.scroll.side_effect = fake_scroll
|
||||
monkeypatch.setattr(
|
||||
scanner_module, "get_qdrant_client", AsyncMock(return_value=client)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
scanner_module,
|
||||
"get_settings",
|
||||
lambda: SimpleNamespace(get_collection_name=lambda: "c"),
|
||||
)
|
||||
|
||||
|
||||
async def test_enqueues_deletes_for_disabled_text_type(monkeypatch):
|
||||
_patch_qdrant(monkeypatch, {"note": ["n1", "n2"], "deck_card": ["d1"]})
|
||||
sent: list = []
|
||||
stream = _producer(AsyncMock(side_effect=lambda t: sent.append(t)))
|
||||
|
||||
# note disabled; news_item + deck_card still allowed.
|
||||
allowed = frozenset({"file", "news_item", "deck_card"})
|
||||
queued = await _enqueue_deletes_for_disabled_types("alice", stream, allowed, 1)
|
||||
|
||||
assert queued == 2
|
||||
assert {t.doc_id for t in sent} == {"n1", "n2"}
|
||||
assert all(t.operation == "delete" and t.doc_type == "note" for t in sent)
|
||||
|
||||
|
||||
async def test_all_text_types_disabled_enqueues_all(monkeypatch):
|
||||
# Admin disabled everything at once (empty allow-set): every text type's
|
||||
# indexed points are enqueued for deletion in a single call.
|
||||
_patch_qdrant(
|
||||
monkeypatch, {"note": ["n1"], "news_item": ["ni1"], "deck_card": ["d1"]}
|
||||
)
|
||||
sent: list = []
|
||||
stream = _producer(AsyncMock(side_effect=lambda t: sent.append(t)))
|
||||
|
||||
queued = await _enqueue_deletes_for_disabled_types("alice", stream, frozenset(), 1)
|
||||
|
||||
assert queued == 3
|
||||
assert {(t.doc_type, t.doc_id) for t in sent} == {
|
||||
("note", "n1"),
|
||||
("news_item", "ni1"),
|
||||
("deck_card", "d1"),
|
||||
}
|
||||
|
||||
|
||||
async def test_noop_when_allowed_is_none(monkeypatch):
|
||||
# Fail-open: a transient capability read must never trigger deletion.
|
||||
send = AsyncMock()
|
||||
queued = await _enqueue_deletes_for_disabled_types(
|
||||
"alice", _producer(send), None, 1
|
||||
)
|
||||
assert queued == 0
|
||||
send.assert_not_called()
|
||||
|
||||
|
||||
async def test_noop_when_all_text_types_allowed(monkeypatch):
|
||||
send = AsyncMock()
|
||||
allowed = frozenset({"note", "news_item", "deck_card", "file"})
|
||||
queued = await _enqueue_deletes_for_disabled_types(
|
||||
"alice", _producer(send), allowed, 1
|
||||
)
|
||||
assert queued == 0
|
||||
send.assert_not_called()
|
||||
|
||||
|
||||
async def test_one_shot_does_not_reflood_on_subsequent_scans(monkeypatch):
|
||||
_patch_qdrant(monkeypatch, {"note": ["n1", "n2"]})
|
||||
send = AsyncMock()
|
||||
allowed = frozenset({"file"}) # note disabled
|
||||
|
||||
first = await _enqueue_deletes_for_disabled_types(
|
||||
"alice", _producer(send), allowed, 1
|
||||
)
|
||||
second = await _enqueue_deletes_for_disabled_types(
|
||||
"alice", _producer(send), allowed, 2
|
||||
)
|
||||
|
||||
assert first == 2
|
||||
# Standing disable: the next scan must not re-enqueue the same deletes.
|
||||
assert second == 0
|
||||
|
||||
|
||||
async def test_re_enable_then_disable_retriggers_backstop(monkeypatch):
|
||||
_patch_qdrant(monkeypatch, {"note": ["n1"]})
|
||||
send = AsyncMock()
|
||||
disabled = frozenset({"file"})
|
||||
enabled = frozenset({"file", "note"})
|
||||
|
||||
assert (
|
||||
await _enqueue_deletes_for_disabled_types("alice", _producer(send), disabled, 1)
|
||||
== 1
|
||||
)
|
||||
# Re-enabled: clears the one-shot marker.
|
||||
assert (
|
||||
await _enqueue_deletes_for_disabled_types("alice", _producer(send), enabled, 2)
|
||||
== 0
|
||||
)
|
||||
# Disabled again: backstop fires once more.
|
||||
assert (
|
||||
await _enqueue_deletes_for_disabled_types("alice", _producer(send), disabled, 3)
|
||||
== 1
|
||||
)
|
||||
Reference in New Issue
Block a user