fix(search): address PR #813 review (viz verify-on-read, owners cache, docs)
- viz_routes: run verify_search_results before returning results. After the accessible_owners expansion the viz can surface OTHER users' shared docs, so it must drop ones the caller can no longer access (revoked share) — same as the nc_semantic_search tool path. (Blocking review item.) - access_filter: cache list_accessible_owners per user for 30s to keep the OCS shares round-trip off the search hot path (failures aren't cached); document the single-page OCS limitation; add a clear_accessible_owners_cache() test helper. Comment the empty-accessible_owners MatchAny([]) edge case. - verification: comment why cross-user eviction is a deliberate no-op (eviction is scoped to the querying user's id, so a recipient's revoked access never deletes the owner's points; the recipient self-heals via accessible_owners). - algorithms: declare SearchResult.original_score (set by the viz route) so the now-precisely-typed result list type-checks. - tests: cross-user eviction-no-op safety test; autouse owners-cache reset in the access_filter + shared-search tests; replace async-no-await qdrant fakes with AsyncMock (clears SonarCloud S7503). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
6206f4a634
commit
b1fac2d7a8
@@ -38,6 +38,7 @@ from nextcloud_mcp_server.search.context import (
|
|||||||
get_chunk_bbox_and_page_from_qdrant,
|
get_chunk_bbox_and_page_from_qdrant,
|
||||||
get_chunk_with_context,
|
get_chunk_with_context,
|
||||||
)
|
)
|
||||||
|
from nextcloud_mcp_server.search.verification import verify_search_results
|
||||||
from nextcloud_mcp_server.utils.validation import is_valid_nextcloud_doc_id
|
from nextcloud_mcp_server.utils.validation import is_valid_nextcloud_doc_id
|
||||||
from nextcloud_mcp_server.vector.oauth_sync import (
|
from nextcloud_mcp_server.vector.oauth_sync import (
|
||||||
NotProvisionedError,
|
NotProvisionedError,
|
||||||
@@ -226,10 +227,18 @@ async def vector_visualization_search(request: Request) -> JSONResponse:
|
|||||||
# Sort by score before verification
|
# Sort by score before verification
|
||||||
all_results.sort(key=lambda r: r.score, reverse=True)
|
all_results.sort(key=lambda r: r.score, reverse=True)
|
||||||
|
|
||||||
# No verification needed for visualization - we only need Qdrant metadata
|
# Verify-on-read (ADR-019). Now that accessible_owners is expanded
|
||||||
# (title, excerpt, doc_type) which is already in search results.
|
# via OCS shares, the result set can include OTHER users' shared
|
||||||
# Verification is only needed for sampling (LLM needs full content).
|
# documents — so we must drop any the caller can no longer access
|
||||||
search_results = all_results[:limit]
|
# (e.g. a revoked share whose index entry hasn't reconciled yet),
|
||||||
|
# exactly as the nc_semantic_search tool path does. Skipping this
|
||||||
|
# would let the viz surface stale titles/excerpts from another
|
||||||
|
# user's index after a share is revoked.
|
||||||
|
with trace_operation("vector_viz.verify_on_read"):
|
||||||
|
verified_results, _dropped = await verify_search_results(
|
||||||
|
nc_client, all_results
|
||||||
|
)
|
||||||
|
search_results = verified_results[:limit]
|
||||||
search_duration = time.perf_counter() - search_start
|
search_duration = time.perf_counter() - search_start
|
||||||
|
|
||||||
# Store original scores and normalize for visualization
|
# Store original scores and normalize for visualization
|
||||||
|
|||||||
@@ -17,12 +17,26 @@ by their original indexer. New points carry both fields.
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import time
|
||||||
from typing import Any, Protocol
|
from typing import Any, Protocol
|
||||||
|
|
||||||
from qdrant_client.models import FieldCondition, Filter, MatchAny, MatchValue
|
from qdrant_client.models import FieldCondition, Filter, MatchAny, MatchValue
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Short-lived per-user cache for the OCS shares lookup, which otherwise runs on
|
||||||
|
# every search/viz request. Trades up to this many seconds of share-visibility
|
||||||
|
# staleness (a freshly-granted share is searchable a little late) for avoiding
|
||||||
|
# an OCS round-trip per query. Safe: verify-on-read still gates each result
|
||||||
|
# against Nextcloud, so a revoked share is caught there regardless of this cache.
|
||||||
|
_OWNERS_CACHE_TTL_SECONDS = 30.0
|
||||||
|
_owners_cache: dict[str, tuple[float, list[str]]] = {}
|
||||||
|
|
||||||
|
|
||||||
|
def clear_accessible_owners_cache() -> None:
|
||||||
|
"""Drop all cached accessible-owners entries (used by tests)."""
|
||||||
|
_owners_cache.clear()
|
||||||
|
|
||||||
|
|
||||||
class _SharingClientProtocol(Protocol):
|
class _SharingClientProtocol(Protocol):
|
||||||
"""Subset of SharingClient that this module actually uses."""
|
"""Subset of SharingClient that this module actually uses."""
|
||||||
@@ -42,9 +56,22 @@ async def list_accessible_owners(
|
|||||||
Duplicates are removed; ordering is not significant (Qdrant ``MatchAny``
|
Duplicates are removed; ordering is not significant (Qdrant ``MatchAny``
|
||||||
treats the list as a set).
|
treats the list as a set).
|
||||||
|
|
||||||
|
Results are cached per user for ``_OWNERS_CACHE_TTL_SECONDS`` to keep the
|
||||||
|
OCS round-trip off the search hot path. Failures are not cached.
|
||||||
|
|
||||||
|
Note: ``list_shares(shared_with_me=True)`` returns whatever the OCS endpoint
|
||||||
|
yields in a single page (SharingClient does not paginate today). A user with
|
||||||
|
more incoming shares than the OCS page size could have some owners omitted;
|
||||||
|
if that becomes real, add pagination to SharingClient.
|
||||||
|
|
||||||
Sharing API failures are non-fatal — we degrade to ``[user_id]`` and log
|
Sharing API failures are non-fatal — we degrade to ``[user_id]`` and log
|
||||||
so a hiccup in OCS doesn't black-hole the user's own search.
|
so a hiccup in OCS doesn't black-hole the user's own search.
|
||||||
"""
|
"""
|
||||||
|
now = time.monotonic()
|
||||||
|
cached = _owners_cache.get(user_id)
|
||||||
|
if cached is not None and now - cached[0] < _OWNERS_CACHE_TTL_SECONDS:
|
||||||
|
return list(cached[1]) # copy so callers can't mutate the cached value
|
||||||
|
|
||||||
owners: set[str] = {user_id}
|
owners: set[str] = {user_id}
|
||||||
try:
|
try:
|
||||||
shares = await sharing_client.list_shares(shared_with_me=True)
|
shares = await sharing_client.list_shares(shared_with_me=True)
|
||||||
@@ -55,7 +82,7 @@ async def list_accessible_owners(
|
|||||||
user_id,
|
user_id,
|
||||||
exc,
|
exc,
|
||||||
)
|
)
|
||||||
return [user_id]
|
return [user_id] # don't cache failures — retry on the next search
|
||||||
|
|
||||||
for share in shares:
|
for share in shares:
|
||||||
# OCS returns the share owner under `uid_owner` (the file owner,
|
# OCS returns the share owner under `uid_owner` (the file owner,
|
||||||
@@ -65,8 +92,10 @@ async def list_accessible_owners(
|
|||||||
if isinstance(owner, str) and owner:
|
if isinstance(owner, str) and owner:
|
||||||
owners.add(owner)
|
owners.add(owner)
|
||||||
|
|
||||||
logger.debug("Accessible owners for user %s: %d entries", user_id, len(owners))
|
result = list(owners)
|
||||||
return list(owners)
|
_owners_cache[user_id] = (now, result)
|
||||||
|
logger.debug("Accessible owners for user %s: %d entries", user_id, len(result))
|
||||||
|
return list(result)
|
||||||
|
|
||||||
|
|
||||||
def build_ownership_filter(
|
def build_ownership_filter(
|
||||||
@@ -90,6 +119,11 @@ def build_ownership_filter(
|
|||||||
A Qdrant ``Filter`` ready to be nested under a parent ``must`` clause.
|
A Qdrant ``Filter`` ready to be nested under a parent ``must`` clause.
|
||||||
"""
|
"""
|
||||||
owners = accessible_owners if accessible_owners is not None else [user_id]
|
owners = accessible_owners if accessible_owners is not None else [user_id]
|
||||||
|
# Edge case: an explicit empty ``accessible_owners`` yields
|
||||||
|
# ``MatchAny(any=[])``, which Qdrant treats as matching nothing — so the
|
||||||
|
# owner_id branch contributes no results and access comes solely from the
|
||||||
|
# legacy ``user_id`` branch below (self-owned content). Callers that want
|
||||||
|
# share expansion must pass a non-empty list.
|
||||||
return Filter(
|
return Filter(
|
||||||
should=[
|
should=[
|
||||||
FieldCondition(key="owner_id", match=MatchAny(any=owners)),
|
FieldCondition(key="owner_id", match=MatchAny(any=owners)),
|
||||||
|
|||||||
@@ -168,6 +168,9 @@ class SearchResult:
|
|||||||
chunk_index: int = 0
|
chunk_index: int = 0
|
||||||
total_chunks: int = 1
|
total_chunks: int = 1
|
||||||
point_id: str | None = None
|
point_id: str | None = None
|
||||||
|
# Pre-normalization score, set by the visualization route before it rescales
|
||||||
|
# ``score`` to [0, 1] for visual encoding (see auth/viz_routes.py).
|
||||||
|
original_score: float | None = None
|
||||||
|
|
||||||
def __post_init__(self):
|
def __post_init__(self):
|
||||||
"""Validate score is non-negative.
|
"""Validate score is non-negative.
|
||||||
|
|||||||
@@ -586,6 +586,15 @@ async def verify_search_results(
|
|||||||
if evict_on_missing and inaccessible:
|
if evict_on_missing and inaccessible:
|
||||||
|
|
||||||
async def evict(doc_id: str, doc_type: str) -> None:
|
async def evict(doc_id: str, doc_type: str) -> None:
|
||||||
|
# Eviction is scoped to the QUERYING user's own points
|
||||||
|
# (user_id == the searcher). For a cross-user shared document
|
||||||
|
# (owner_id=alice surfaced to bob via accessible_owners), bob
|
||||||
|
# failing verification evicts with user_id=bob — a deliberate
|
||||||
|
# no-op, because alice's points carry user_id=alice and must NOT
|
||||||
|
# be deleted just because bob's share was revoked. Bob's view
|
||||||
|
# self-heals via list_accessible_owners (alice drops out of his
|
||||||
|
# accessible owners once OCS no longer reports the share). See the
|
||||||
|
# legacy-user_id semantics note in build_ownership_filter.
|
||||||
try:
|
try:
|
||||||
await delete_document_points(doc_id, doc_type, user_id)
|
await delete_document_points(doc_id, doc_type, user_id)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|||||||
@@ -15,6 +15,8 @@ real-Nextcloud flow (share + verify-on-read) lives in
|
|||||||
``test_acl_shared_search.py``.
|
``test_acl_shared_search.py``.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
from unittest.mock import AsyncMock
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from qdrant_client import AsyncQdrantClient
|
from qdrant_client import AsyncQdrantClient
|
||||||
from qdrant_client.models import Distance, PointStruct, VectorParams
|
from qdrant_client.models import Distance, PointStruct, VectorParams
|
||||||
@@ -82,12 +84,9 @@ async def seeded_collection(monkeypatch):
|
|||||||
await client.upsert(collection_name=collection, points=points, wait=True)
|
await client.upsert(collection_name=collection, points=points, wait=True)
|
||||||
|
|
||||||
# Point the algorithm at the in-memory client + deterministic embeddings.
|
# Point the algorithm at the in-memory client + deterministic embeddings.
|
||||||
async def _fake_get_qdrant_client():
|
|
||||||
return client
|
|
||||||
|
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"nextcloud_mcp_server.search.semantic.get_qdrant_client",
|
"nextcloud_mcp_server.search.semantic.get_qdrant_client",
|
||||||
_fake_get_qdrant_client,
|
AsyncMock(return_value=client),
|
||||||
)
|
)
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"nextcloud_mcp_server.search.semantic.get_embedding_service",
|
"nextcloud_mcp_server.search.semantic.get_embedding_service",
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ proves the real share → accessible_owners → filter → verify chain.
|
|||||||
|
|
||||||
import os
|
import os
|
||||||
import uuid
|
import uuid
|
||||||
|
from unittest.mock import AsyncMock
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from httpx import BasicAuth
|
from httpx import BasicAuth
|
||||||
@@ -32,12 +33,25 @@ from qdrant_client.models import Distance, PointStruct, VectorParams
|
|||||||
from nextcloud_mcp_server.client import NextcloudClient
|
from nextcloud_mcp_server.client import NextcloudClient
|
||||||
from nextcloud_mcp_server.config import get_settings
|
from nextcloud_mcp_server.config import get_settings
|
||||||
from nextcloud_mcp_server.embedding import SimpleEmbeddingProvider
|
from nextcloud_mcp_server.embedding import SimpleEmbeddingProvider
|
||||||
from nextcloud_mcp_server.search.access_filter import list_accessible_owners
|
from nextcloud_mcp_server.search.access_filter import (
|
||||||
|
clear_accessible_owners_cache,
|
||||||
|
list_accessible_owners,
|
||||||
|
)
|
||||||
from nextcloud_mcp_server.search.semantic import SemanticSearchAlgorithm
|
from nextcloud_mcp_server.search.semantic import SemanticSearchAlgorithm
|
||||||
from nextcloud_mcp_server.search.verification import verify_search_results
|
from nextcloud_mcp_server.search.verification import verify_search_results
|
||||||
|
|
||||||
pytestmark = pytest.mark.integration
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _reset_owners_cache():
|
||||||
|
"""Reset the process-global accessible-owners cache around each test so a
|
||||||
|
real OCS share created in a fixture isn't masked by a stale cached entry."""
|
||||||
|
clear_accessible_owners_cache()
|
||||||
|
yield
|
||||||
|
clear_accessible_owners_cache()
|
||||||
|
|
||||||
|
|
||||||
_DOC_TEXT = "Confidential quarterly infrastructure budget and capacity plan"
|
_DOC_TEXT = "Confidential quarterly infrastructure budget and capacity plan"
|
||||||
|
|
||||||
|
|
||||||
@@ -127,12 +141,9 @@ async def seeded_semantic(monkeypatch, shared_file):
|
|||||||
wait=True,
|
wait=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
async def _fake_get_qdrant_client():
|
|
||||||
return client
|
|
||||||
|
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"nextcloud_mcp_server.search.semantic.get_qdrant_client",
|
"nextcloud_mcp_server.search.semantic.get_qdrant_client",
|
||||||
_fake_get_qdrant_client,
|
AsyncMock(return_value=client),
|
||||||
)
|
)
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"nextcloud_mcp_server.search.semantic.get_embedding_service",
|
"nextcloud_mcp_server.search.semantic.get_embedding_service",
|
||||||
|
|||||||
@@ -8,10 +8,20 @@ import pytest
|
|||||||
|
|
||||||
from nextcloud_mcp_server.search.access_filter import (
|
from nextcloud_mcp_server.search.access_filter import (
|
||||||
build_ownership_filter,
|
build_ownership_filter,
|
||||||
|
clear_accessible_owners_cache,
|
||||||
list_accessible_owners,
|
list_accessible_owners,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _reset_owners_cache():
|
||||||
|
"""The accessible-owners cache is process-global; reset it around each test
|
||||||
|
so the shared "alice" user_id can't leak cached results between tests."""
|
||||||
|
clear_accessible_owners_cache()
|
||||||
|
yield
|
||||||
|
clear_accessible_owners_cache()
|
||||||
|
|
||||||
|
|
||||||
class TestListAccessibleOwners:
|
class TestListAccessibleOwners:
|
||||||
@pytest.mark.unit
|
@pytest.mark.unit
|
||||||
async def test_includes_self_even_with_no_shares(self) -> None:
|
async def test_includes_self_even_with_no_shares(self) -> None:
|
||||||
|
|||||||
@@ -874,6 +874,37 @@ async def test_verify_search_results_drops_inaccessible_and_evicts(mocker):
|
|||||||
spy_evict.assert_awaited_once_with("99", "note", "alice")
|
spy_evict.assert_awaited_once_with("99", "note", "alice")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
async def test_verify_evicts_cross_user_file_under_querying_user_id(mocker):
|
||||||
|
"""A shared file the recipient can no longer access is evicted under the
|
||||||
|
QUERYING user's id, never the owner's.
|
||||||
|
|
||||||
|
This guards the cross-user eviction no-op: a point owned by alice
|
||||||
|
(user_id=alice) surfaced to bob via accessible_owners and then found
|
||||||
|
inaccessible must be evicted with user_id=bob — which deletes nothing of
|
||||||
|
alice's (her points carry user_id=alice). So a recipient's revoked access
|
||||||
|
can never delete the owner's index entries; bob's view self-heals via
|
||||||
|
list_accessible_owners instead. A future change that evicted under the
|
||||||
|
owner's id would corrupt the owner's index, and this test would catch it.
|
||||||
|
"""
|
||||||
|
spy_evict = mocker.AsyncMock()
|
||||||
|
mocker.patch.object(verification, "delete_document_points", spy_evict)
|
||||||
|
|
||||||
|
webdav_client = SimpleNamespace(
|
||||||
|
file_accessible_by_id=mocker.AsyncMock(return_value=False)
|
||||||
|
)
|
||||||
|
client = SimpleNamespace(webdav=webdav_client, username="bob")
|
||||||
|
|
||||||
|
kept, dropped_count = await verify_search_results(
|
||||||
|
client,
|
||||||
|
[_make_result(777, doc_type="file", metadata={"path": "shared.txt"})],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert kept == []
|
||||||
|
assert dropped_count == 1
|
||||||
|
spy_evict.assert_awaited_once_with("777", "file", "bob")
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.unit
|
@pytest.mark.unit
|
||||||
async def test_verify_search_results_fire_and_forget_eviction(mocker):
|
async def test_verify_search_results_fire_and_forget_eviction(mocker):
|
||||||
"""When eviction_task_group is provided, eviction does not block the response.
|
"""When eviction_task_group is provided, eviction does not block the response.
|
||||||
|
|||||||
Reference in New Issue
Block a user