refactor: Optimize Nextcloud access verification with centralized filtering
Move access verification from individual search algorithms to final output stage, eliminating redundant API calls and improving performance. ## Changes **New:** - `search/verification.py`: Centralized verification using anyio task groups - Deduplicates results by (doc_id, doc_type) before verification - Verifies all unique documents in parallel using structured concurrency - Filters out inaccessible documents in single pass **Modified Search Algorithms:** - `search/semantic.py`: Removed _deduplicate_and_verify() and _verify_document_access() - `search/keyword.py`: Removed _verify_access() and parallel verification - `search/fuzzy.py`: Removed _verify_access() and parallel verification - `search/hybrid.py`: Removed nextcloud_client parameter passing All algorithms now return unverified results from Qdrant payload. **Modified Output Stages:** - `server/semantic.py`: Added verify_search_results() call after search - `auth/viz_routes.py`: Added verify_search_results() call after search Both endpoints now verify access once at final stage with deduplication. ## Performance Impact **Before:** - Hybrid mode (limit=10): 30 API calls (10 per algorithm × 3 algorithms) - Single algorithm: 10-20 API calls (with verification buffer) **After:** - Hybrid mode (limit=10): 10 API calls (deduplicated verification) - Single algorithm: 10 API calls (deduplicated verification) **Performance Gain:** 3x reduction in API calls for hybrid search ## Architecture Benefits - **Separation of concerns**: Algorithms handle scoring, output stage handles security - **Deduplication**: Each document verified exactly once - **Parallel execution**: All verifications run concurrently via anyio task groups - **Consistency**: Same verification logic across MCP tools and viz endpoints 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
@@ -3,15 +3,10 @@
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from httpx import HTTPStatusError
|
||||
from qdrant_client.models import FieldCondition, Filter, MatchValue
|
||||
|
||||
from nextcloud_mcp_server.config import get_settings
|
||||
from nextcloud_mcp_server.search.algorithms import (
|
||||
NextcloudClientProtocol,
|
||||
SearchAlgorithm,
|
||||
SearchResult,
|
||||
)
|
||||
from nextcloud_mcp_server.search.algorithms import SearchAlgorithm, SearchResult
|
||||
from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -46,25 +41,23 @@ class FuzzySearchAlgorithm(SearchAlgorithm):
|
||||
user_id: str,
|
||||
limit: int = 10,
|
||||
doc_type: str | None = None,
|
||||
nextcloud_client: NextcloudClientProtocol | None = None,
|
||||
**kwargs: Any,
|
||||
) -> list[SearchResult]:
|
||||
"""Execute fuzzy search using character overlap on Qdrant payload.
|
||||
|
||||
Queries Qdrant for all indexed documents, then scores based on character
|
||||
overlap in title and excerpt fields. Only verifies access with Nextcloud
|
||||
at the end for security.
|
||||
overlap in title and excerpt fields. Returns unverified results - access
|
||||
verification should be performed separately at the final output stage.
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
user_id: User ID for filtering
|
||||
limit: Maximum results to return
|
||||
doc_type: Optional document type filter (None = all types)
|
||||
nextcloud_client: NextcloudClient for access verification (optional)
|
||||
**kwargs: Additional parameters (threshold override)
|
||||
|
||||
Returns:
|
||||
List of SearchResult objects ranked by character overlap score
|
||||
List of unverified SearchResult objects ranked by character overlap score
|
||||
"""
|
||||
settings = get_settings()
|
||||
threshold = kwargs.get("threshold", self.threshold)
|
||||
@@ -157,66 +150,23 @@ class FuzzySearchAlgorithm(SearchAlgorithm):
|
||||
|
||||
# Sort by score (descending) and limit
|
||||
scored_results.sort(key=lambda x: x["score"], reverse=True)
|
||||
top_results = scored_results[: limit * 2] # Get extra for access verification
|
||||
top_results = scored_results[:limit]
|
||||
|
||||
# Verify access with Nextcloud (optional, for security)
|
||||
# Parallelize verification to avoid sequential HTTP calls
|
||||
# Return unverified results (verification happens at output stage)
|
||||
final_results = []
|
||||
if nextcloud_client:
|
||||
from asyncio import gather
|
||||
|
||||
# Create verification coroutines for all top results
|
||||
verification_coros = [
|
||||
self._verify_access(
|
||||
nextcloud_client, result["doc_id"], result["doc_type"]
|
||||
for result in top_results:
|
||||
final_results.append(
|
||||
SearchResult(
|
||||
id=result["doc_id"],
|
||||
doc_type=result["doc_type"],
|
||||
title=result["title"],
|
||||
excerpt=result["excerpt"],
|
||||
score=result["score"],
|
||||
metadata={"match_location": result["match_location"]},
|
||||
)
|
||||
for result in top_results
|
||||
]
|
||||
|
||||
# Execute all verifications in parallel
|
||||
# return_exceptions=True prevents one failure from canceling others
|
||||
verification_results = await gather(
|
||||
*verification_coros, return_exceptions=True
|
||||
)
|
||||
|
||||
# Build final results from verified documents
|
||||
for result, verified in zip(top_results, verification_results):
|
||||
# Skip if verification failed or raised exception
|
||||
if isinstance(verified, Exception) or verified is None:
|
||||
continue
|
||||
|
||||
final_results.append(
|
||||
SearchResult(
|
||||
id=result["doc_id"],
|
||||
doc_type=result["doc_type"],
|
||||
title=result["title"],
|
||||
excerpt=result["excerpt"],
|
||||
score=result["score"],
|
||||
metadata={
|
||||
**verified.get("metadata", {}),
|
||||
"match_location": result["match_location"],
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
# Stop once we have enough results
|
||||
if len(final_results) >= limit:
|
||||
break
|
||||
else:
|
||||
# No verification, return results directly
|
||||
for result in top_results[:limit]:
|
||||
final_results.append(
|
||||
SearchResult(
|
||||
id=result["doc_id"],
|
||||
doc_type=result["doc_type"],
|
||||
title=result["title"],
|
||||
excerpt=result["excerpt"],
|
||||
score=result["score"],
|
||||
metadata={"match_location": result["match_location"]},
|
||||
)
|
||||
)
|
||||
|
||||
logger.info(f"Fuzzy search returned {len(final_results)} matching documents")
|
||||
logger.info(f"Fuzzy search returned {len(final_results)} unverified results")
|
||||
if final_results:
|
||||
result_details = [
|
||||
f"{r.doc_type}_{r.id} (score={r.score:.3f}, title='{r.title}')"
|
||||
@@ -226,45 +176,6 @@ class FuzzySearchAlgorithm(SearchAlgorithm):
|
||||
|
||||
return final_results
|
||||
|
||||
async def _verify_access(
|
||||
self, nextcloud_client: NextcloudClientProtocol, doc_id: int, doc_type: str
|
||||
) -> dict[str, Any] | None:
|
||||
"""Verify user has access to a document via Nextcloud API.
|
||||
|
||||
Args:
|
||||
nextcloud_client: Client for API access
|
||||
doc_id: Document ID
|
||||
doc_type: Document type
|
||||
|
||||
Returns:
|
||||
Dict with metadata if access verified, None otherwise
|
||||
"""
|
||||
try:
|
||||
if doc_type == "note":
|
||||
note = await nextcloud_client.notes.get_note(doc_id)
|
||||
return {
|
||||
"metadata": {
|
||||
"category": note.get("category", ""),
|
||||
"modified": note.get("modified"),
|
||||
}
|
||||
}
|
||||
else:
|
||||
logger.debug(
|
||||
f"Skipping verification for {doc_type} {doc_id} (not implemented)"
|
||||
)
|
||||
return {"metadata": {}}
|
||||
except HTTPStatusError as e:
|
||||
if e.response.status_code in (403, 404):
|
||||
logger.debug(
|
||||
f"Access denied for {doc_type} {doc_id}: {e.response.status_code}"
|
||||
)
|
||||
return None
|
||||
else:
|
||||
logger.warning(
|
||||
f"Error verifying {doc_type} {doc_id}: {e.response.status_code}"
|
||||
)
|
||||
return None
|
||||
|
||||
def _calculate_char_overlap(self, query: str, text: str) -> float:
|
||||
"""Calculate character overlap ratio between query and text.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user