Merge remote-tracking branch 'origin/master' into feat/decomp-hook-points
# Conflicts: # nextcloud_mcp_server/vector/scanner.py
This commit is contained in:
@@ -0,0 +1,179 @@
|
||||
"""ACL-aware ownership filter for semantic / BM25 search.
|
||||
|
||||
The vector store payload carries an ``owner_id`` field — the UID of the user
|
||||
who owns the underlying Nextcloud document. At query time, a user should
|
||||
be able to find every document whose owner has shared it (directly or via
|
||||
group / link) with them, without re-indexing.
|
||||
|
||||
This module turns "who can user X read?" into a Qdrant filter:
|
||||
``owner_id IN accessible_owners`` where ``accessible_owners`` is
|
||||
``{X} ∪ {owners of files / objects shared with X}``.
|
||||
|
||||
A second OR-branch matches the legacy ``user_id`` field so points indexed
|
||||
before this change (which carry only ``user_id``) continue to be findable
|
||||
by their original indexer. New points carry both fields.
|
||||
|
||||
Operator note (existing data): a Qdrant ``owner_id`` field condition matches
|
||||
nothing on points that lack the field, so documents indexed *before* this
|
||||
change never surface to share recipients — only to their original indexer via
|
||||
the legacy ``user_id`` branch. ACL-aware search is therefore effectively a
|
||||
no-op for pre-existing data until each owner's scanner re-indexes it. Trigger a
|
||||
re-index after deploying this feature if it should apply to already-indexed
|
||||
content immediately.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from typing import Any, Protocol
|
||||
|
||||
from qdrant_client.models import Condition, FieldCondition, Filter, MatchAny, MatchValue
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Short-lived per-user cache for the OCS shares lookup, which otherwise runs on
|
||||
# every search/viz request. Trades up to this many seconds of share-visibility
|
||||
# staleness (a freshly-granted share is searchable a little late) for avoiding
|
||||
# an OCS round-trip per query. Safe: verify-on-read still gates each result
|
||||
# against Nextcloud, so a revoked share is caught there regardless of this cache.
|
||||
_OWNERS_CACHE_TTL_SECONDS = 30.0
|
||||
# Cap the number of cached users so the process-global cache can't grow
|
||||
# unboundedly in a long-running multi-user deployment (one entry per active
|
||||
# user, never evicted otherwise). LRU eviction by insertion/access order via
|
||||
# OrderedDict; the cap is generous relative to any realistic concurrent-user
|
||||
# count, so steady state is effectively all-hit.
|
||||
_OWNERS_CACHE_MAXSIZE = 1024
|
||||
_owners_cache: OrderedDict[str, tuple[float, list[str]]] = OrderedDict()
|
||||
|
||||
|
||||
def clear_accessible_owners_cache() -> None:
|
||||
"""Drop all cached accessible-owners entries (used by tests)."""
|
||||
_owners_cache.clear()
|
||||
|
||||
|
||||
class _SharingClientProtocol(Protocol):
|
||||
"""Subset of SharingClient that this module actually uses."""
|
||||
|
||||
async def list_shares(
|
||||
self, path: str | None = None, shared_with_me: bool = False
|
||||
) -> list[dict[str, Any]]: ...
|
||||
|
||||
|
||||
async def list_accessible_owners(
|
||||
sharing_client: _SharingClientProtocol,
|
||||
user_id: str,
|
||||
) -> list[str]:
|
||||
"""Return every owner UID whose content `user_id` should be able to search.
|
||||
|
||||
The set is ``{user_id} ∪ {uid_owner of each share with shared_with_me=True}``.
|
||||
Duplicates are removed; ordering is not significant (Qdrant ``MatchAny``
|
||||
treats the list as a set).
|
||||
|
||||
Results are cached per user for ``_OWNERS_CACHE_TTL_SECONDS`` to keep the
|
||||
OCS round-trip off the search hot path. Failures are not cached.
|
||||
|
||||
Note: ``list_shares(shared_with_me=True)`` returns whatever the OCS endpoint
|
||||
yields in a single page (SharingClient does not paginate today). A user with
|
||||
more incoming shares than the OCS page size could have some owners omitted;
|
||||
if that becomes real, add pagination to SharingClient.
|
||||
|
||||
Granularity / over-fetch limitation (TODO, finer-grained filtering): this
|
||||
expansion is *owner-level*, not *file-level*. If a prolific content creator
|
||||
shares a single item with the querying user, that owner's whole indexed
|
||||
corpus becomes a Qdrant candidate set for the querier even though only the
|
||||
shared item is accessible. Verify-on-read correctly drops the inaccessible
|
||||
"ghost" candidates, but because there is no second Qdrant pass to replenish,
|
||||
a ``limit=N`` search can return fewer than N results when the over-fetch
|
||||
buffer (2× in nc_semantic_search / viz_routes) is dominated by ghosts. A
|
||||
per-file ownership index would remove this tension and is the natural
|
||||
starting point for future work (intentionally out of scope here).
|
||||
|
||||
Sharing API failures are non-fatal — we degrade to ``[user_id]`` and log
|
||||
so a hiccup in OCS doesn't black-hole the user's own search.
|
||||
"""
|
||||
now = time.monotonic()
|
||||
cached = _owners_cache.get(user_id)
|
||||
if cached is not None and now - cached[0] < _OWNERS_CACHE_TTL_SECONDS:
|
||||
_owners_cache.move_to_end(user_id) # mark as recently used (LRU)
|
||||
return list(cached[1]) # copy so callers can't mutate the cached value
|
||||
|
||||
owners: set[str] = {user_id}
|
||||
try:
|
||||
shares = await sharing_client.list_shares(shared_with_me=True)
|
||||
except Exception as exc: # noqa: BLE001 — degrade gracefully
|
||||
logger.warning(
|
||||
"Sharing API unavailable; falling back to self-only owner filter "
|
||||
"for user %s (%s)",
|
||||
user_id,
|
||||
exc,
|
||||
)
|
||||
return [user_id] # don't cache failures — retry on the next search
|
||||
|
||||
for share in shares:
|
||||
# OCS returns the share owner under `uid_owner` (the file owner,
|
||||
# not the share recipient). Some Nextcloud versions also surface
|
||||
# `owner` as a fallback display field — we tolerate both. The intent is
|
||||
# "absent, not empty": a missing/blank `uid_owner` falls through to
|
||||
# `owner`, and a non-string or empty result skips the (malformed) share.
|
||||
owner = share.get("uid_owner") or share.get("owner") or None
|
||||
if not isinstance(owner, str) or not owner:
|
||||
continue
|
||||
owners.add(owner)
|
||||
|
||||
result = list(owners)
|
||||
_owners_cache[user_id] = (now, result)
|
||||
# Promote to the most-recently-used end. This is a no-op for a brand-new
|
||||
# key (dict insertion already appends) but is needed when re-inserting an
|
||||
# existing key after its TTL expired.
|
||||
_owners_cache.move_to_end(user_id)
|
||||
while len(_owners_cache) > _OWNERS_CACHE_MAXSIZE:
|
||||
_owners_cache.popitem(last=False) # evict the least-recently-used entry
|
||||
logger.debug(
|
||||
"Accessible owners for user %s: %d entries (%d other owner(s))",
|
||||
user_id,
|
||||
len(result),
|
||||
len(result) - 1,
|
||||
)
|
||||
return list(result)
|
||||
|
||||
|
||||
def build_ownership_filter(
|
||||
user_id: str, accessible_owners: list[str] | None = None
|
||||
) -> Filter:
|
||||
"""Build the Qdrant ``Filter`` constraining a search to readable points.
|
||||
|
||||
Matches points whose ``owner_id`` is in ``accessible_owners`` (excluding
|
||||
self) OR whose ``user_id`` equals ``user_id``. The ``user_id`` branch covers
|
||||
*all* of the caller's own content — both new points (where
|
||||
``owner_id == user_id``) and legacy points indexed before ``owner_id``
|
||||
existed — so self is intentionally NOT repeated in the ``owner_id`` branch.
|
||||
|
||||
Args:
|
||||
user_id: Querying user (matched by the ``user_id`` branch, which is the
|
||||
self-only default when ``accessible_owners`` is None).
|
||||
accessible_owners: Pre-computed list of owner UIDs the user has
|
||||
access to. When None, defaults to ``[user_id]`` (no shares
|
||||
expansion — used by callers that genuinely want self-only
|
||||
scope such as eviction sweeps).
|
||||
|
||||
Returns:
|
||||
A Qdrant ``Filter`` ready to be nested under a parent ``must`` clause.
|
||||
"""
|
||||
owners = accessible_owners if accessible_owners is not None else [user_id]
|
||||
# The ``user_id`` branch is always present and already covers self-owned
|
||||
# content (new + legacy). The ``owner_id`` branch is added only for OTHER
|
||||
# owners (share senders) — listing self there too would overlap the
|
||||
# ``user_id`` branch for no benefit. When there are no other owners the
|
||||
# ``owner_id`` branch is omitted entirely, so we never depend on
|
||||
# ``MatchAny(any=[])`` matching nothing (not a documented Qdrant guarantee).
|
||||
other_owners = [owner for owner in owners if owner != user_id]
|
||||
conditions: list[Condition] = [
|
||||
FieldCondition(key="user_id", match=MatchValue(value=user_id)),
|
||||
]
|
||||
if other_owners:
|
||||
conditions.insert(
|
||||
0, FieldCondition(key="owner_id", match=MatchAny(any=other_owners))
|
||||
)
|
||||
return Filter(should=conditions)
|
||||
@@ -5,9 +5,10 @@ from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Protocol, runtime_checkable
|
||||
|
||||
from qdrant_client.models import FieldCondition, Filter, MatchValue, ScoredPoint
|
||||
from qdrant_client.models import Filter, ScoredPoint
|
||||
|
||||
from nextcloud_mcp_server.config import get_settings
|
||||
from nextcloud_mcp_server.search.access_filter import build_ownership_filter
|
||||
from nextcloud_mcp_server.vector.placeholder import get_placeholder_filter
|
||||
from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client
|
||||
|
||||
@@ -75,14 +76,24 @@ class NextcloudClientProtocol(Protocol):
|
||||
...
|
||||
|
||||
|
||||
async def get_indexed_doc_types(user_id: str) -> set[str]:
|
||||
async def get_indexed_doc_types(
|
||||
user_id: str, accessible_owners: list[str] | None = None
|
||||
) -> set[str]:
|
||||
"""Query Qdrant to get actually-indexed document types for a user.
|
||||
|
||||
This enables search algorithms to check which document types are available
|
||||
before attempting to search/verify them, allowing graceful cross-app search.
|
||||
|
||||
Args:
|
||||
user_id: User ID to filter by
|
||||
user_id: User ID to filter by.
|
||||
accessible_owners: Owner UIDs the user may read (self + share senders),
|
||||
as computed by ``access_filter.list_accessible_owners``. When
|
||||
provided, doc-type discovery is ACL-aware and matches the same
|
||||
ownership scope as the actual search (so a share recipient discovers
|
||||
cross-user doc_types). When ``None`` (the default), discovery is
|
||||
**self-only** — a recipient won't see doc_types that exist only in
|
||||
another owner's shared content. Pass the expanded set for cross-user
|
||||
discovery.
|
||||
|
||||
Returns:
|
||||
Set of document type strings (e.g., {"note", "file", "calendar"})
|
||||
@@ -106,7 +117,9 @@ async def get_indexed_doc_types(user_id: str) -> set[str]:
|
||||
scroll_filter=Filter(
|
||||
must=[
|
||||
get_placeholder_filter(), # Exclude placeholders from doc_type discovery
|
||||
FieldCondition(key="user_id", match=MatchValue(value=user_id)),
|
||||
# ACL-aware ownership scope (owner_id IN owners OR legacy
|
||||
# user_id == user_id), matching the real search filter.
|
||||
build_ownership_filter(user_id, accessible_owners),
|
||||
]
|
||||
),
|
||||
limit=1000, # Sample size to discover types
|
||||
@@ -168,6 +181,9 @@ class SearchResult:
|
||||
chunk_index: int = 0
|
||||
total_chunks: int = 1
|
||||
point_id: str | None = None
|
||||
# Pre-normalization score, set by the visualization route before it rescales
|
||||
# ``score`` to [0, 1] for visual encoding (see auth/viz_routes.py).
|
||||
original_score: float | None = None
|
||||
|
||||
def __post_init__(self):
|
||||
"""Validate score is non-negative.
|
||||
@@ -271,6 +287,8 @@ class SearchAlgorithm(ABC):
|
||||
user_id: str,
|
||||
limit: int = 10,
|
||||
doc_type: str | None = None,
|
||||
*,
|
||||
accessible_owners: list[str] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> list[SearchResult]:
|
||||
"""Execute search with the given parameters.
|
||||
@@ -280,6 +298,12 @@ class SearchAlgorithm(ABC):
|
||||
user_id: User ID for multi-tenant filtering
|
||||
limit: Maximum number of results to return
|
||||
doc_type: Optional document type filter (note, file, calendar, etc.)
|
||||
accessible_owners: Owner UIDs the user is allowed to read (self plus
|
||||
the owners of content shared with them), pre-computed from the
|
||||
OCS Sharing API by the caller. Declared explicitly — rather than
|
||||
buried in ``**kwargs`` — so a misspelled keyword is a type error
|
||||
instead of a silent fall back to self-only scope. ``None`` means
|
||||
self-only (``[user_id]``).
|
||||
**kwargs: Algorithm-specific parameters
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -10,6 +10,7 @@ from nextcloud_mcp_server.config import get_settings
|
||||
from nextcloud_mcp_server.embedding import get_bm25_service, get_embedding_service
|
||||
from nextcloud_mcp_server.observability.metrics import record_qdrant_operation
|
||||
from nextcloud_mcp_server.observability.tracing import trace_operation
|
||||
from nextcloud_mcp_server.search.access_filter import build_ownership_filter
|
||||
from nextcloud_mcp_server.search.algorithms import (
|
||||
SearchAlgorithm,
|
||||
SearchResult,
|
||||
@@ -70,6 +71,8 @@ class BM25HybridSearchAlgorithm(SearchAlgorithm):
|
||||
user_id: str,
|
||||
limit: int = 10,
|
||||
doc_type: str | None = None,
|
||||
*,
|
||||
accessible_owners: list[str] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> list[SearchResult]:
|
||||
"""
|
||||
@@ -88,6 +91,9 @@ class BM25HybridSearchAlgorithm(SearchAlgorithm):
|
||||
user_id: User ID for filtering
|
||||
limit: Maximum results to return
|
||||
doc_type: Optional document type filter
|
||||
accessible_owners: Owner UIDs the user can read (self + share
|
||||
senders), pre-computed by the caller from the OCS Sharing API.
|
||||
Defaults to ``[user_id]`` (self-only) when ``None``.
|
||||
**kwargs: Additional parameters (score_threshold override)
|
||||
|
||||
Returns:
|
||||
@@ -131,10 +137,7 @@ class BM25HybridSearchAlgorithm(SearchAlgorithm):
|
||||
# Build Qdrant filter
|
||||
filter_conditions = [
|
||||
get_placeholder_filter(), # Always exclude placeholders from user-facing queries
|
||||
FieldCondition(
|
||||
key="user_id",
|
||||
match=MatchValue(value=user_id),
|
||||
),
|
||||
build_ownership_filter(user_id, accessible_owners),
|
||||
]
|
||||
|
||||
# Add doc_type filter if specified
|
||||
@@ -238,12 +241,10 @@ class BM25HybridSearchAlgorithm(SearchAlgorithm):
|
||||
if len(results) >= limit:
|
||||
break
|
||||
|
||||
# Log the count only — NOT titles. These results are unverified: with
|
||||
# owner-level share expansion the candidate set can include other users'
|
||||
# documents that verify-on-read will drop, so titles must not be logged
|
||||
# until after verification (the verifying callers log verified titles).
|
||||
logger.info("Returning %s unverified results after deduplication", len(results))
|
||||
if results:
|
||||
result_details = [
|
||||
f"{r.doc_type}_{r.id} (score={r.score:.3f}, title='{r.title}')"
|
||||
for r in results[:5] # Show top 5
|
||||
]
|
||||
logger.debug("Top results: %s", ", ".join(result_details))
|
||||
|
||||
return results
|
||||
|
||||
@@ -7,10 +7,12 @@ position markers for better visualization and understanding of search results.
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
|
||||
from httpx import HTTPStatusError
|
||||
from qdrant_client.models import FieldCondition, Filter, MatchValue
|
||||
|
||||
from nextcloud_mcp_server.client import NextcloudClient
|
||||
from nextcloud_mcp_server.config import get_settings
|
||||
from nextcloud_mcp_server.search.access_filter import build_ownership_filter
|
||||
from nextcloud_mcp_server.utils.validation import is_valid_nextcloud_doc_id
|
||||
from nextcloud_mcp_server.vector.html_processor import html_to_markdown
|
||||
from nextcloud_mcp_server.vector.placeholder import get_placeholder_filter
|
||||
@@ -20,7 +22,12 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def _get_chunk_from_qdrant(
|
||||
user_id: str, doc_id: str, doc_type: str, chunk_start: int, chunk_end: int
|
||||
user_id: str,
|
||||
doc_id: str,
|
||||
doc_type: str,
|
||||
chunk_start: int,
|
||||
chunk_end: int,
|
||||
accessible_owners: list[str] | None = None,
|
||||
) -> str | None:
|
||||
"""Retrieve full chunk text from Qdrant payload.
|
||||
|
||||
@@ -28,11 +35,15 @@ async def _get_chunk_from_qdrant(
|
||||
chunk content already stored in Qdrant.
|
||||
|
||||
Args:
|
||||
user_id: User ID who owns the document
|
||||
user_id: Querying user.
|
||||
doc_id: Document ID
|
||||
doc_type: Document type (e.g., "note", "file")
|
||||
chunk_start: Character offset where chunk starts
|
||||
chunk_end: Character offset where chunk ends
|
||||
accessible_owners: Owner UIDs the caller may read (self + share senders).
|
||||
When None, the lookup is self-only. Callers must only pass an
|
||||
expanded set after confirming the caller can access the document
|
||||
(see ``get_chunk_with_context``) — the filter is owner-level.
|
||||
|
||||
Returns:
|
||||
Full chunk text from Qdrant excerpt field, or None if not found
|
||||
@@ -46,7 +57,7 @@ async def _get_chunk_from_qdrant(
|
||||
collection_name=settings.get_collection_name(),
|
||||
scroll_filter=Filter(
|
||||
must=[
|
||||
FieldCondition(key="user_id", match=MatchValue(value=user_id)),
|
||||
build_ownership_filter(user_id, accessible_owners),
|
||||
FieldCondition(key="doc_id", match=MatchValue(value=doc_id)),
|
||||
FieldCondition(key="doc_type", match=MatchValue(value=doc_type)),
|
||||
FieldCondition(
|
||||
@@ -93,17 +104,24 @@ async def _get_chunk_from_qdrant(
|
||||
|
||||
|
||||
async def _get_chunk_by_index_from_qdrant(
|
||||
user_id: str, doc_id: str, doc_type: str, chunk_index: int
|
||||
user_id: str,
|
||||
doc_id: str,
|
||||
doc_type: str,
|
||||
chunk_index: int,
|
||||
accessible_owners: list[str] | None = None,
|
||||
) -> str | None:
|
||||
"""Retrieve chunk text by chunk_index from Qdrant payload.
|
||||
|
||||
Used to fetch adjacent chunks for context expansion.
|
||||
|
||||
Args:
|
||||
user_id: User ID who owns the document
|
||||
user_id: Querying user.
|
||||
doc_id: Document ID
|
||||
doc_type: Document type (e.g., "note", "file")
|
||||
chunk_index: Zero-based chunk index in document
|
||||
accessible_owners: Owner UIDs the caller may read; None ⇒ self-only.
|
||||
Only pass an expanded set after a per-document access check (see
|
||||
``get_chunk_with_context``).
|
||||
|
||||
Returns:
|
||||
Full chunk text from Qdrant excerpt field, or None if not found
|
||||
@@ -117,7 +135,7 @@ async def _get_chunk_by_index_from_qdrant(
|
||||
collection_name=settings.get_collection_name(),
|
||||
scroll_filter=Filter(
|
||||
must=[
|
||||
FieldCondition(key="user_id", match=MatchValue(value=user_id)),
|
||||
build_ownership_filter(user_id, accessible_owners),
|
||||
FieldCondition(key="doc_id", match=MatchValue(value=doc_id)),
|
||||
FieldCondition(key="doc_type", match=MatchValue(value=doc_type)),
|
||||
FieldCondition(
|
||||
@@ -172,7 +190,13 @@ async def _get_deck_metadata_from_qdrant(
|
||||
qdrant_client = await get_qdrant_client()
|
||||
settings = get_settings()
|
||||
|
||||
# Query for any chunk of this card (we just need metadata)
|
||||
# Query for any chunk of this card (we just need metadata).
|
||||
# Intentionally self-only (raw user_id, not build_ownership_filter):
|
||||
# deck cards are a documented cross-user gap — the Deck API is per-user,
|
||||
# so cross-user deck context can't be fetched with the caller's
|
||||
# credentials anyway (see the doc_type=="file"-only gate in
|
||||
# get_chunk_with_context). Every other internal Qdrant lookup here is
|
||||
# ACL-aware; this one is the deliberate exception.
|
||||
scroll_result = await qdrant_client.scroll(
|
||||
collection_name=settings.get_collection_name(),
|
||||
scroll_filter=Filter(
|
||||
@@ -217,6 +241,7 @@ async def get_chunk_bbox_and_page_from_qdrant(
|
||||
chunk_index: int | None,
|
||||
chunk_start: int,
|
||||
chunk_end: int,
|
||||
accessible_owners: list[str] | None = None,
|
||||
) -> tuple[list | None, int | None]:
|
||||
"""Fetch chunk_bbox and page_number for a chunk from Qdrant payload.
|
||||
|
||||
@@ -256,7 +281,7 @@ async def get_chunk_bbox_and_page_from_qdrant(
|
||||
must=[
|
||||
get_placeholder_filter(),
|
||||
FieldCondition(key="doc_id", match=MatchValue(value=doc_id)),
|
||||
FieldCondition(key="user_id", match=MatchValue(value=user_id)),
|
||||
build_ownership_filter(user_id, accessible_owners),
|
||||
FieldCondition(
|
||||
key="chunk_index", match=MatchValue(value=chunk_index)
|
||||
),
|
||||
@@ -273,7 +298,7 @@ async def get_chunk_bbox_and_page_from_qdrant(
|
||||
must=[
|
||||
get_placeholder_filter(),
|
||||
FieldCondition(key="doc_id", match=MatchValue(value=doc_id)),
|
||||
FieldCondition(key="user_id", match=MatchValue(value=user_id)),
|
||||
build_ownership_filter(user_id, accessible_owners),
|
||||
FieldCondition(
|
||||
key="chunk_start_offset",
|
||||
match=MatchValue(value=chunk_start),
|
||||
@@ -352,6 +377,7 @@ async def get_chunk_with_context(
|
||||
chunk_index: int | None = None,
|
||||
total_chunks: int = 1,
|
||||
context_chars: int = 300,
|
||||
accessible_owners: list[str] | None = None,
|
||||
) -> ChunkContext | None:
|
||||
"""Fetch chunk with surrounding context.
|
||||
|
||||
@@ -361,7 +387,7 @@ async def get_chunk_with_context(
|
||||
|
||||
Args:
|
||||
nc_client: Authenticated Nextcloud client
|
||||
user_id: User ID who owns the document
|
||||
user_id: Querying user.
|
||||
doc_id: Document ID (str — keyword-indexed in Qdrant payload)
|
||||
doc_type: Type of document ("note", "file", etc.)
|
||||
chunk_start: Character offset where chunk starts
|
||||
@@ -372,6 +398,10 @@ async def get_chunk_with_context(
|
||||
field). When None, falls back to the (chunk_start, chunk_end) lookup.
|
||||
total_chunks: Total number of chunks in document
|
||||
context_chars: Number of characters to include before/after chunk
|
||||
accessible_owners: Owner UIDs the caller may read (self + share senders).
|
||||
Used to support cross-user context for SHARED FILES only, and only
|
||||
after a per-file access check (see ``lookup_owners`` below). For
|
||||
non-file types the lookup stays self-only.
|
||||
|
||||
Returns:
|
||||
ChunkContext with expanded context and markers, or None if document
|
||||
@@ -380,13 +410,53 @@ async def get_chunk_with_context(
|
||||
# doc_id is keyword-indexed in Qdrant as str — pass through verbatim
|
||||
# (no int coercion; producers always stringify on write).
|
||||
|
||||
# Determine the ownership scope for the Qdrant cached-chunk lookups.
|
||||
#
|
||||
# ``accessible_owners`` is OWNER-level (every owner who shared anything with
|
||||
# the caller), so widening the lookup to it unconditionally would let a
|
||||
# recipient of a single shared file read ANY of that owner's cached chunks
|
||||
# by guessing doc_ids. We therefore honour it only for FILES, and only after
|
||||
# confirming the caller can access THIS file by id (``file_accessible_by_id``
|
||||
# is cross-user-safe: a WebDAV SEARCH over the caller's whole tree incl.
|
||||
# mounted shares). For per-user types (note/deck/news) there is no
|
||||
# share-mounted by-id access via the caller's credentials, so the lookup
|
||||
# stays self-only — cross-user context for those types is a known gap.
|
||||
lookup_owners: list[str] | None = None # None ⇒ self-only
|
||||
if doc_type == "file" and accessible_owners:
|
||||
try:
|
||||
if await nc_client.webdav.file_accessible_by_id(int(doc_id)):
|
||||
lookup_owners = accessible_owners
|
||||
else:
|
||||
# Not owned and not shared with the caller → no access. Return
|
||||
# early rather than falling back to a self-only lookup that
|
||||
# would also miss (and so the result is the same None, but this
|
||||
# is explicit and skips a pointless Qdrant round-trip).
|
||||
logger.debug(
|
||||
"File %s not accessible to %s; no cross-user chunk context",
|
||||
doc_id,
|
||||
user_id,
|
||||
)
|
||||
return None
|
||||
except (ValueError, TypeError):
|
||||
# Non-numeric doc_id: shouldn't happen (endpoints validate), but
|
||||
# degrade to self-only rather than raising.
|
||||
logger.warning("Non-numeric file doc_id %r; using self-only scope", doc_id)
|
||||
except HTTPStatusError as exc:
|
||||
# Transient transport/server error — treat as inconclusive and fall
|
||||
# back to self-only so the caller's own files still resolve.
|
||||
logger.warning(
|
||||
"file_accessible_by_id(%s) failed (%s); using self-only scope",
|
||||
doc_id,
|
||||
exc,
|
||||
)
|
||||
|
||||
# Try to get chunk from Qdrant (fast path).
|
||||
# Prefer chunk_index lookup (always-indexed field) when caller supplied it;
|
||||
# fall back to (chunk_start, chunk_end) lookup otherwise.
|
||||
chunk_text: str | None = None
|
||||
if chunk_index is not None:
|
||||
chunk_text = await _get_chunk_by_index_from_qdrant(
|
||||
user_id, doc_id, doc_type, chunk_index
|
||||
user_id, doc_id, doc_type, chunk_index, accessible_owners=lookup_owners
|
||||
)
|
||||
# When chunk_index is supplied, the indexed lookup is canonical: both the
|
||||
# index path and the offset path query the same Qdrant collection, so an
|
||||
@@ -398,7 +468,12 @@ async def get_chunk_with_context(
|
||||
skip_offset_lookup = chunk_index is not None
|
||||
if chunk_text is None and not skip_offset_lookup:
|
||||
chunk_text = await _get_chunk_from_qdrant(
|
||||
user_id, doc_id, doc_type, chunk_start, chunk_end
|
||||
user_id,
|
||||
doc_id,
|
||||
doc_type,
|
||||
chunk_start,
|
||||
chunk_end,
|
||||
accessible_owners=lookup_owners,
|
||||
)
|
||||
|
||||
if chunk_text:
|
||||
@@ -422,7 +497,11 @@ async def get_chunk_with_context(
|
||||
# Fetch previous chunk if not first chunk
|
||||
if chunk_index > 0:
|
||||
before_chunk = await _get_chunk_by_index_from_qdrant(
|
||||
user_id, doc_id, doc_type, chunk_index - 1
|
||||
user_id,
|
||||
doc_id,
|
||||
doc_type,
|
||||
chunk_index - 1,
|
||||
accessible_owners=lookup_owners,
|
||||
)
|
||||
if before_chunk:
|
||||
# Remove overlap: the last chunk_overlap chars of previous chunk
|
||||
@@ -443,7 +522,11 @@ async def get_chunk_with_context(
|
||||
# Fetch next chunk if not last chunk
|
||||
if chunk_index < total_chunks - 1:
|
||||
after_chunk = await _get_chunk_by_index_from_qdrant(
|
||||
user_id, doc_id, doc_type, chunk_index + 1
|
||||
user_id,
|
||||
doc_id,
|
||||
doc_type,
|
||||
chunk_index + 1,
|
||||
accessible_owners=lookup_owners,
|
||||
)
|
||||
if after_chunk:
|
||||
# Remove overlap: the first chunk_overlap chars of next chunk
|
||||
|
||||
@@ -9,6 +9,7 @@ from nextcloud_mcp_server.acl_hash import accessible_hash_set
|
||||
from nextcloud_mcp_server.config import get_settings
|
||||
from nextcloud_mcp_server.embedding import get_embedding_service
|
||||
from nextcloud_mcp_server.observability.metrics import record_qdrant_operation
|
||||
from nextcloud_mcp_server.search.access_filter import build_ownership_filter
|
||||
from nextcloud_mcp_server.search.algorithms import (
|
||||
SearchAlgorithm,
|
||||
SearchResult,
|
||||
@@ -50,6 +51,8 @@ class SemanticSearchAlgorithm(SearchAlgorithm):
|
||||
user_id: str,
|
||||
limit: int = 10,
|
||||
doc_type: str | None = None,
|
||||
*,
|
||||
accessible_owners: list[str] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> list[SearchResult]:
|
||||
"""Execute semantic search using vector similarity.
|
||||
@@ -67,7 +70,11 @@ class SemanticSearchAlgorithm(SearchAlgorithm):
|
||||
user_id: User ID for filtering
|
||||
limit: Maximum results to return
|
||||
doc_type: Optional document type filter
|
||||
**kwargs: Additional parameters (score_threshold override)
|
||||
accessible_owners: Owner UIDs the user can read (self + share
|
||||
senders), pre-computed by the caller from the OCS Sharing API.
|
||||
Defaults to ``[user_id]`` (self-only) when ``None``.
|
||||
**kwargs:
|
||||
- score_threshold (float): override the instance default
|
||||
|
||||
Returns:
|
||||
List of unverified SearchResult objects ranked by similarity score
|
||||
@@ -99,10 +106,7 @@ class SemanticSearchAlgorithm(SearchAlgorithm):
|
||||
# Build Qdrant filter
|
||||
filter_conditions = [
|
||||
get_placeholder_filter(), # Always exclude placeholders from user-facing queries
|
||||
FieldCondition(
|
||||
key="user_id",
|
||||
match=MatchValue(value=user_id),
|
||||
),
|
||||
build_ownership_filter(user_id, accessible_owners),
|
||||
]
|
||||
|
||||
# Add doc_type filter if specified
|
||||
@@ -175,12 +179,10 @@ class SemanticSearchAlgorithm(SearchAlgorithm):
|
||||
if len(results) >= limit:
|
||||
break
|
||||
|
||||
# Log the count only — NOT titles. These results are unverified: with
|
||||
# owner-level share expansion the candidate set can include other users'
|
||||
# documents that verify-on-read will drop, so titles must not be logged
|
||||
# until after verification (the verifying callers log verified titles).
|
||||
logger.info("Returning %s unverified results after deduplication", len(results))
|
||||
if results:
|
||||
result_details = [
|
||||
f"{r.doc_type}_{r.id} (score={r.score:.3f}, title='{r.title}')"
|
||||
for r in results[:5] # Show top 5
|
||||
]
|
||||
logger.debug("Top results: %s", ", ".join(result_details))
|
||||
|
||||
return results
|
||||
|
||||
@@ -132,45 +132,55 @@ async def _verify_files(
|
||||
results: list[SearchResult],
|
||||
semaphore: anyio.Semaphore,
|
||||
) -> set[str]:
|
||||
"""Return the doc_ids of file results this user may actually access.
|
||||
|
||||
Verifies each file by its *global* Nextcloud file id via an ACL-aware
|
||||
WebDAV SEARCH (``webdav.file_accessible_by_id``), NOT by path. This is the
|
||||
ACL-aware-search fix: a file an owner shared with the querying user mounts
|
||||
at a different path under each tree, so the previous path-based check
|
||||
(``get_file_info``) produced false 404s and dropped legitimate shared-file
|
||||
hits. Definitive 403/404 → inaccessible (dropped + scheduled for eviction
|
||||
by the caller); transient/ambiguous errors → kept (fail-open).
|
||||
"""
|
||||
# safe: cooperative concurrency, no lock needed (see verify_search_results)
|
||||
accessible: set[str] = set()
|
||||
|
||||
async def check(result: SearchResult) -> None:
|
||||
doc_id = result.id
|
||||
# file_path is propagated from the Qdrant payload by the algorithm
|
||||
# layer (bm25_hybrid.py / semantic.py). No extra Qdrant round-trip.
|
||||
# layer (bm25_hybrid.py / semantic.py); kept here only for log context.
|
||||
file_path = (result.metadata or {}).get("path")
|
||||
if not file_path:
|
||||
# Cannot verify without a path; treat as accessible to avoid
|
||||
# silently dropping legitimate results when payload is missing
|
||||
# (legacy data, or a future doc_type that doesn't propagate path).
|
||||
|
||||
# Verify by *global* file ID via an ACL-aware WebDAV SEARCH, NOT by
|
||||
# path. For files the vector ``doc_id`` IS the Nextcloud file ID, and
|
||||
# file_accessible_by_id searches the user's whole tree (incl. mounted
|
||||
# shares), so a file an owner shared with this user verifies as
|
||||
# accessible even though it lives at a different path under the owner's
|
||||
# root. A path-based check (the old behaviour) would 404 on shared
|
||||
# files mounted at the recipient's root by basename and silently drop
|
||||
# legitimate ACL-aware-search results.
|
||||
#
|
||||
# Hoisted cast mirrors _verify_notes: a malformed id keeps the result
|
||||
# (fail open) with a specific log line rather than a generic
|
||||
# "unexpected error" from the catch-all below.
|
||||
try:
|
||||
file_id_int = int(doc_id)
|
||||
except (TypeError, ValueError) as e:
|
||||
logger.warning(
|
||||
"No file path in metadata for file_id %s; keeping result "
|
||||
"(verification skipped)",
|
||||
"Non-numeric file id %r (%s): %s; keeping result",
|
||||
doc_id,
|
||||
file_path,
|
||||
e,
|
||||
)
|
||||
accessible.add(doc_id)
|
||||
return
|
||||
|
||||
async with semaphore:
|
||||
try:
|
||||
info = await client.webdav.get_file_info(file_path)
|
||||
if info is None:
|
||||
# Contract (see WebDAVClient.get_file_info docstring):
|
||||
# `None` means a malformed PROPFIND response — an
|
||||
# ambiguous state, not a definitive 404. Treat as
|
||||
# transient and KEEP the result rather than evicting.
|
||||
# Real 404s raise HTTPStatusError and land in the
|
||||
# _is_definitive_404_or_403 branch below.
|
||||
logger.warning(
|
||||
"Malformed PROPFIND response verifying file %s (%s); "
|
||||
"keeping result (ambiguous state, not a definitive 404)",
|
||||
doc_id,
|
||||
file_path,
|
||||
)
|
||||
if await client.webdav.file_accessible_by_id(file_id_int):
|
||||
accessible.add(doc_id)
|
||||
return
|
||||
accessible.add(doc_id)
|
||||
# else: definitively inaccessible (not owned, not shared) —
|
||||
# drop and let the caller schedule eviction.
|
||||
except HTTPStatusError as e:
|
||||
if _is_definitive_404_or_403(e):
|
||||
return
|
||||
@@ -183,6 +193,8 @@ async def _verify_files(
|
||||
)
|
||||
accessible.add(doc_id)
|
||||
except Exception as e:
|
||||
# Network blip / unexpected WebDAV error — ambiguous, not a
|
||||
# definitive denial. Keep the result; the next query re-verifies.
|
||||
logger.warning(
|
||||
"Unexpected error verifying file %s (%s): %s; keeping result",
|
||||
doc_id,
|
||||
@@ -584,6 +596,15 @@ async def verify_search_results(
|
||||
if evict_on_missing and inaccessible:
|
||||
|
||||
async def evict(doc_id: str, doc_type: str) -> None:
|
||||
# Eviction is scoped to the QUERYING user's own points
|
||||
# (user_id == the searcher). For a cross-user shared document
|
||||
# (owner_id=alice surfaced to bob via accessible_owners), bob
|
||||
# failing verification evicts with user_id=bob — a deliberate
|
||||
# no-op, because alice's points carry user_id=alice and must NOT
|
||||
# be deleted just because bob's share was revoked. Bob's view
|
||||
# self-heals via list_accessible_owners (alice drops out of his
|
||||
# accessible owners once OCS no longer reports the share). See the
|
||||
# legacy-user_id semantics note in build_ownership_filter.
|
||||
try:
|
||||
await delete_document_points(doc_id, doc_type, user_id)
|
||||
except Exception as e:
|
||||
|
||||
Reference in New Issue
Block a user