refactor!: Make all search algorithms query Qdrant payload, not Nextcloud

BREAKING CHANGE: Search algorithms now require Qdrant to be populated.
Vector sync must be enabled and documents indexed for search to work.

- Keyword and fuzzy search now query Qdrant scroll API for title/excerpt
- Remove inefficient Nextcloud API fetching pattern
- Add optional Nextcloud verification for security
- Deduplicate by (doc_id, doc_type) tuple, keeping chunk_index=0
- Align with document processor pattern that already stores text in Qdrant

nextcloud_mcp_server/search/fuzzy.py

@@ -1,14 +1,18 @@
-"""Fuzzy search algorithm using character overlap matching."""
+"""Fuzzy search algorithm using character overlap matching on Qdrant payload."""
 
 import logging
 from typing import Any
 
+from httpx import HTTPStatusError
+from qdrant_client.models import FieldCondition, Filter, MatchValue
+
+from nextcloud_mcp_server.config import get_settings
 from nextcloud_mcp_server.search.algorithms import (
     NextcloudClientProtocol,
     SearchAlgorithm,
     SearchResult,
-    get_indexed_doc_types,
 )
+from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client
 
 logger = logging.getLogger(__name__)
 
@@ -45,25 +49,24 @@ class FuzzySearchAlgorithm(SearchAlgorithm):
         nextcloud_client: NextcloudClientProtocol | None = None,
         **kwargs: Any,
     ) -> list[SearchResult]:
-        """Execute fuzzy search using character overlap.
+        """Execute fuzzy search using character overlap on Qdrant payload.
+
+        Queries Qdrant for all indexed documents, then scores based on character
+        overlap in title and excerpt fields. Only verifies access with Nextcloud
+        at the end for security.
 
         Args:
             query: Search query
             user_id: User ID for filtering
             limit: Maximum results to return
-            doc_type: Optional document type filter (currently only "note" supported)
+            doc_type: Optional document type filter (None = all types)
-            nextcloud_client: NextcloudClient for fetching documents
+            nextcloud_client: NextcloudClient for access verification (optional)
             **kwargs: Additional parameters (threshold override)
 
         Returns:
             List of SearchResult objects ranked by character overlap score
-
-        Raises:
-            ValueError: If nextcloud_client not provided
         """
-        if not nextcloud_client:
+        settings = get_settings()
-            raise ValueError("FuzzySearch requires nextcloud_client parameter")
-
         threshold = kwargs.get("threshold", self.threshold)
 
         logger.info(
@@ -71,112 +74,176 @@ class FuzzySearchAlgorithm(SearchAlgorithm):
             f"limit={limit}, threshold={threshold}, doc_type={doc_type}"
         )
 
-        # Get available document types from Qdrant
+        # Build Qdrant filter
-        indexed_types = await get_indexed_doc_types(user_id)
+        filter_conditions = [
-        logger.debug(f"Indexed document types for user: {indexed_types}")
+            FieldCondition(key="user_id", match=MatchValue(value=user_id))
-
+        ]
-        # Determine which types to search
         if doc_type:
-            # Search specific type if requested
+            filter_conditions.append(
-            search_types = [doc_type] if doc_type in indexed_types else []
+                FieldCondition(key="doc_type", match=MatchValue(value=doc_type))
-            if not search_types:
+            )
-                logger.info(f"Doc type '{doc_type}' not indexed for user {user_id}")
-                return []
-        else:
-            # Search all indexed types
-            search_types = list(indexed_types)
 
-        # Fetch documents for each type and score them
+        # Scroll through Qdrant to get all matching documents
-        all_documents = []
+        qdrant_client = await get_qdrant_client()
-        for dtype in search_types:
+        collection = settings.qdrant_collection
-            documents = await self._fetch_documents(nextcloud_client, dtype)
-            for doc in documents:
-                doc["_doc_type"] = dtype  # Tag with type
-            all_documents.extend(documents)
 
-        logger.debug(f"Fetched {len(all_documents)} total documents for fuzzy search")
+        all_points = []
+        offset = None
 
-        # Score and filter documents
+        # Scroll through all points matching filter
+        while True:
+            scroll_result, next_offset = await qdrant_client.scroll(
+                collection_name=collection,
+                scroll_filter=Filter(must=filter_conditions),
+                limit=100,  # Batch size
+                offset=offset,
+                with_payload=["doc_id", "doc_type", "title", "excerpt", "chunk_index"],
+                with_vectors=False,  # Don't need vectors
+            )
+
+            all_points.extend(scroll_result)
+
+            if next_offset is None:
+                break
+            offset = next_offset
+
+        logger.debug(f"Retrieved {len(all_points)} points from Qdrant for fuzzy search")
+
+        # Deduplicate by (doc_id, doc_type) - keep first chunk
+        seen_docs = {}
+        for point in all_points:
+            doc_id = int(point.payload["doc_id"])
+            dtype = point.payload.get("doc_type", "note")
+            doc_key = (doc_id, dtype)
+
+            chunk_idx = point.payload.get("chunk_index", 0)
+            if doc_key not in seen_docs or chunk_idx == 0:
+                seen_docs[doc_key] = point
+
+        logger.debug(f"Deduplicated to {len(seen_docs)} unique documents")
+
+        # Score each document based on fuzzy matches
         scored_results = []
         query_lower = query.lower()
 
-        for doc in all_documents:
+        for doc_key, point in seen_docs.items():
-            dtype = doc.get("_doc_type", "note")
+            doc_id, dtype = doc_key
-            title = doc.get("title", "")
+            title = point.payload.get("title", "")
-            content = doc.get("content", "")
+            excerpt = point.payload.get("excerpt", "")
 
             # Check title match
             title_score = self._calculate_char_overlap(query_lower, title.lower())
 
-            # Check content match
+            # Check excerpt match
-            content_score = self._calculate_char_overlap(query_lower, content.lower())
+            excerpt_score = self._calculate_char_overlap(query_lower, excerpt.lower())
 
             # Use best score
-            best_score = max(title_score, content_score)
+            best_score = max(title_score, excerpt_score)
 
             if best_score >= threshold:
-                # Extract excerpt based on which matched better
+                match_location = "title" if title_score >= excerpt_score else "excerpt"
-                if title_score >= content_score:
-                    excerpt = f"Title match: {title}"
-                else:
-                    excerpt = self._extract_excerpt(content, max_length=200)
-
                 scored_results.append(
-                    SearchResult(
+                    {
-                        id=doc["id"],
+                        "doc_id": doc_id,
-                        doc_type=dtype,
+                        "doc_type": dtype,
-                        title=title or "Untitled",
+                        "title": title,
-                        excerpt=excerpt,
+                        "excerpt": excerpt
-                        score=best_score,
+                        if excerpt_score >= title_score
-                        metadata={
+                        else f"Title match: {title}",
-                            "category": doc.get("category", ""),
+                        "score": best_score,
-                            "modified": doc.get("modified"),
+                        "match_location": match_location,
-                            "match_location": "title"
+                    }
-                            if title_score >= content_score
-                            else "content",
-                        },
-                    )
                 )
 
         # Sort by score (descending) and limit
-        scored_results.sort(key=lambda x: x.score, reverse=True)
+        scored_results.sort(key=lambda x: x["score"], reverse=True)
-        results = scored_results[:limit]
+        top_results = scored_results[: limit * 2]  # Get extra for access verification
 
-        logger.info(f"Fuzzy search returned {len(results)} matching notes")
+        # Verify access with Nextcloud (optional, for security)
-        if results:
+        final_results = []
+        if nextcloud_client:
+            for result in top_results:
+                verified = await self._verify_access(
+                    nextcloud_client, result["doc_id"], result["doc_type"]
+                )
+                if verified:
+                    final_results.append(
+                        SearchResult(
+                            id=result["doc_id"],
+                            doc_type=result["doc_type"],
+                            title=result["title"],
+                            excerpt=result["excerpt"],
+                            score=result["score"],
+                            metadata={
+                                **verified.get("metadata", {}),
+                                "match_location": result["match_location"],
+                            },
+                        )
+                    )
+                if len(final_results) >= limit:
+                    break
+        else:
+            # No verification, return results directly
+            for result in top_results[:limit]:
+                final_results.append(
+                    SearchResult(
+                        id=result["doc_id"],
+                        doc_type=result["doc_type"],
+                        title=result["title"],
+                        excerpt=result["excerpt"],
+                        score=result["score"],
+                        metadata={"match_location": result["match_location"]},
+                    )
+                )
+
+        logger.info(f"Fuzzy search returned {len(final_results)} matching documents")
+        if final_results:
             result_details = [
-                f"note_{r.id} (score={r.score:.3f}, title='{r.title}')"
+                f"{r.doc_type}_{r.id} (score={r.score:.3f}, title='{r.title}')"
-                for r in results[:5]
+                for r in final_results[:5]
             ]
             logger.debug(f"Top fuzzy results: {', '.join(result_details)}")
 
-        return results
+        return final_results
 
-    async def _fetch_documents(
+    async def _verify_access(
-        self, nextcloud_client: NextcloudClientProtocol, doc_type: str
+        self, nextcloud_client: NextcloudClientProtocol, doc_id: int, doc_type: str
-    ) -> list[dict[str, Any]]:
+    ) -> dict[str, Any] | None:
-        """Fetch documents of a specific type from Nextcloud.
+        """Verify user has access to a document via Nextcloud API.
 
         Args:
             nextcloud_client: Client for API access
-            doc_type: Document type to fetch ("note", "file", "calendar", etc.)
+            doc_id: Document ID
+            doc_type: Document type
 
         Returns:
-            List of document dictionaries with at minimum: id, title, content
+            Dict with metadata if access verified, None otherwise
         """
-        if doc_type == "note":
+        try:
-            return await nextcloud_client.notes.get_notes()
+            if doc_type == "note":
-        elif doc_type == "file":
+                note = await nextcloud_client.notes.get_note(doc_id)
-            # Future: fetch files when indexed
+                return {
-            logger.info("File documents not yet supported for fuzzy search")
+                    "metadata": {
-            return []
+                        "category": note.get("category", ""),
-        elif doc_type == "calendar":
+                        "modified": note.get("modified"),
-            # Future: fetch calendar events when indexed
+                    }
-            logger.info("Calendar documents not yet supported for fuzzy search")
+                }
-            return []
+            else:
-        else:
+                logger.debug(
-            logger.warning(f"Unknown document type '{doc_type}' for fuzzy search")
+                    f"Skipping verification for {doc_type} {doc_id} (not implemented)"
-            return []
+                )
+                return {"metadata": {}}
+        except HTTPStatusError as e:
+            if e.response.status_code in (403, 404):
+                logger.debug(
+                    f"Access denied for {doc_type} {doc_id}: {e.response.status_code}"
+                )
+                return None
+            else:
+                logger.warning(
+                    f"Error verifying {doc_type} {doc_id}: {e.response.status_code}"
+                )
+                return None
 
     def _calculate_char_overlap(self, query: str, text: str) -> float:
         """Calculate character overlap ratio between query and text.

nextcloud_mcp_server/search/keyword.py

@@ -1,14 +1,18 @@
-"""Keyword search algorithm using token-based matching (ADR-001)."""
+"""Keyword search algorithm using token-based matching on Qdrant payload (ADR-001)."""
 
 import logging
 from typing import Any
 
+from httpx import HTTPStatusError
+from qdrant_client.models import FieldCondition, Filter, MatchValue
+
+from nextcloud_mcp_server.config import get_settings
 from nextcloud_mcp_server.search.algorithms import (
     NextcloudClientProtocol,
     SearchAlgorithm,
     SearchResult,
-    get_indexed_doc_types,
 )
+from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client
 
 logger = logging.getLogger(__name__)
 
@@ -39,24 +43,24 @@ class KeywordSearchAlgorithm(SearchAlgorithm):
         nextcloud_client: NextcloudClientProtocol | None = None,
         **kwargs: Any,
     ) -> list[SearchResult]:
-        """Execute keyword search using token matching.
+        """Execute keyword search using token matching on Qdrant payload.
+
+        Queries Qdrant for all indexed documents, then scores based on token
+        matches in title and excerpt fields. Only verifies access with Nextcloud
+        at the end for security.
 
         Args:
             query: Search query to tokenize and match
             user_id: User ID for filtering
             limit: Maximum results to return
-            doc_type: Optional document type filter (currently only "note" supported)
+            doc_type: Optional document type filter (None = all types)
-            nextcloud_client: NextcloudClient for fetching documents
+            nextcloud_client: NextcloudClient for access verification (optional)
             **kwargs: Additional parameters (unused)
 
         Returns:
             List of SearchResult objects ranked by keyword match score
-
-        Raises:
-            ValueError: If nextcloud_client not provided
         """
-        if not nextcloud_client:
+        settings = get_settings()
-            raise ValueError("KeywordSearch requires nextcloud_client parameter")
 
         logger.info(
             f"Keyword search: query='{query}', user={user_id}, "
@@ -67,102 +71,173 @@ class KeywordSearchAlgorithm(SearchAlgorithm):
         query_tokens = self._process_query(query)
         logger.debug(f"Query tokens: {query_tokens}")
 
-        # Get available document types from Qdrant
+        # Build Qdrant filter
-        indexed_types = await get_indexed_doc_types(user_id)
+        filter_conditions = [
-        logger.debug(f"Indexed document types for user: {indexed_types}")
+            FieldCondition(key="user_id", match=MatchValue(value=user_id))
-
+        ]
-        # Determine which types to search
         if doc_type:
-            # Search specific type if requested
+            filter_conditions.append(
-            search_types = [doc_type] if doc_type in indexed_types else []
+                FieldCondition(key="doc_type", match=MatchValue(value=doc_type))
-            if not search_types:
-                logger.info(f"Doc type '{doc_type}' not indexed for user {user_id}")
-                return []
-        else:
-            # Search all indexed types
-            search_types = list(indexed_types)
-
-        # Fetch documents for each type and score them
-        all_documents = []
-        for dtype in search_types:
-            documents = await self._fetch_documents(nextcloud_client, dtype)
-            for doc in documents:
-                doc["_doc_type"] = dtype  # Tag with type
-            all_documents.extend(documents)
-
-        logger.debug(f"Fetched {len(all_documents)} total documents for keyword search")
-
-        # Score and filter documents
-        scored_results = []
-        for doc in all_documents:
-            dtype = doc.get("_doc_type", "note")
-            score = self._calculate_score(
-                query_tokens,
-                doc.get("title", ""),
-                doc.get("content", ""),
             )
 
-            if score > 0:  # Only include matches
+        # Scroll through Qdrant to get all matching documents
-                # Extract excerpt with context
+        # We need title and excerpt from payload for token matching
-                excerpt = self._extract_excerpt(
+        qdrant_client = await get_qdrant_client()
-                    doc.get("content", ""),
+        collection = settings.qdrant_collection
-                    query_tokens,
-                    max_length=200,
-                )
 
+        all_points = []
+        offset = None
+
+        # Scroll through all points matching filter
+        while True:
+            scroll_result, next_offset = await qdrant_client.scroll(
+                collection_name=collection,
+                scroll_filter=Filter(must=filter_conditions),
+                limit=100,  # Batch size
+                offset=offset,
+                with_payload=[
+                    "doc_id",
+                    "doc_type",
+                    "title",
+                    "excerpt",
+                    "chunk_index",
+                    "total_chunks",
+                ],
+                with_vectors=False,  # Don't need vectors for keyword search
+            )
+
+            all_points.extend(scroll_result)
+
+            if next_offset is None:
+                break
+            offset = next_offset
+
+        logger.debug(
+            f"Retrieved {len(all_points)} points from Qdrant for keyword search"
+        )
+
+        # Deduplicate by (doc_id, doc_type) - keep best chunk per document
+        seen_docs = {}
+        for point in all_points:
+            doc_id = int(point.payload["doc_id"])
+            dtype = point.payload.get("doc_type", "note")
+            doc_key = (doc_id, dtype)
+
+            # Keep first chunk (chunk_index=0) as it has the most relevant content
+            chunk_idx = point.payload.get("chunk_index", 0)
+            if doc_key not in seen_docs or chunk_idx == 0:
+                seen_docs[doc_key] = point
+
+        logger.debug(f"Deduplicated to {len(seen_docs)} unique documents")
+
+        # Score each document based on keyword matches
+        scored_results = []
+        for doc_key, point in seen_docs.items():
+            doc_id, dtype = doc_key
+            title = point.payload.get("title", "")
+            excerpt = point.payload.get("excerpt", "")
+
+            # Calculate keyword match score
+            score = self._calculate_score(query_tokens, title, excerpt)
+
+            if score > 0:  # Only include matches
                 scored_results.append(
-                    SearchResult(
+                    {
-                        id=doc["id"],
+                        "doc_id": doc_id,
-                        doc_type=dtype,
+                        "doc_type": dtype,
-                        title=doc.get("title", "Untitled"),
+                        "title": title,
-                        excerpt=excerpt,
+                        "excerpt": excerpt,
-                        score=score,
+                        "score": score,
-                        metadata={
+                    }
-                            "category": doc.get("category", ""),
-                            "modified": doc.get("modified"),
-                        },
-                    )
                 )
 
         # Sort by score (descending) and limit
-        scored_results.sort(key=lambda x: x.score, reverse=True)
+        scored_results.sort(key=lambda x: x["score"], reverse=True)
-        results = scored_results[:limit]
+        top_results = scored_results[: limit * 2]  # Get extra for access verification
 
-        logger.info(f"Keyword search returned {len(results)} matching notes")
+        # Verify access with Nextcloud (optional, for security)
-        if results:
+        final_results = []
+        if nextcloud_client:
+            for result in top_results:
+                verified = await self._verify_access(
+                    nextcloud_client, result["doc_id"], result["doc_type"]
+                )
+                if verified:
+                    final_results.append(
+                        SearchResult(
+                            id=result["doc_id"],
+                            doc_type=result["doc_type"],
+                            title=result["title"],
+                            excerpt=result["excerpt"],
+                            score=result["score"],
+                            metadata=verified.get("metadata", {}),
+                        )
+                    )
+                if len(final_results) >= limit:
+                    break
+        else:
+            # No verification, return results directly
+            for result in top_results[:limit]:
+                final_results.append(
+                    SearchResult(
+                        id=result["doc_id"],
+                        doc_type=result["doc_type"],
+                        title=result["title"],
+                        excerpt=result["excerpt"],
+                        score=result["score"],
+                        metadata={},
+                    )
+                )
+
+        logger.info(f"Keyword search returned {len(final_results)} matching documents")
+        if final_results:
             result_details = [
-                f"note_{r.id} (score={r.score:.3f}, title='{r.title}')"
+                f"{r.doc_type}_{r.id} (score={r.score:.3f}, title='{r.title}')"
-                for r in results[:5]
+                for r in final_results[:5]
             ]
             logger.debug(f"Top keyword results: {', '.join(result_details)}")
 
-        return results
+        return final_results
 
-    async def _fetch_documents(
+    async def _verify_access(
-        self, nextcloud_client: NextcloudClientProtocol, doc_type: str
+        self, nextcloud_client: NextcloudClientProtocol, doc_id: int, doc_type: str
-    ) -> list[dict[str, Any]]:
+    ) -> dict[str, Any] | None:
-        """Fetch documents of a specific type from Nextcloud.
+        """Verify user has access to a document via Nextcloud API.
 
         Args:
             nextcloud_client: Client for API access
-            doc_type: Document type to fetch ("note", "file", "calendar", etc.)
+            doc_id: Document ID
+            doc_type: Document type
 
         Returns:
-            List of document dictionaries with at minimum: id, title, content
+            Dict with metadata if access verified, None otherwise
         """
-        if doc_type == "note":
+        try:
-            return await nextcloud_client.notes.get_notes()
+            if doc_type == "note":
-        elif doc_type == "file":
+                note = await nextcloud_client.notes.get_note(doc_id)
-            # Future: fetch files when indexed
+                return {
-            logger.info("File documents not yet supported for keyword search")
+                    "metadata": {
-            return []
+                        "category": note.get("category", ""),
-        elif doc_type == "calendar":
+                        "modified": note.get("modified"),
-            # Future: fetch calendar events when indexed
+                    }
-            logger.info("Calendar documents not yet supported for keyword search")
+                }
-            return []
+            # Future: Add verification for other document types
-        else:
+            else:
-            logger.warning(f"Unknown document type '{doc_type}' for keyword search")
+                logger.debug(
-            return []
+                    f"Skipping verification for {doc_type} {doc_id} (not implemented)"
+                )
+                return {"metadata": {}}
+        except HTTPStatusError as e:
+            if e.response.status_code in (403, 404):
+                logger.debug(
+                    f"Access denied for {doc_type} {doc_id}: {e.response.status_code}"
+                )
+                return None
+            else:
+                logger.warning(
+                    f"Error verifying {doc_type} {doc_id}: {e.response.status_code}"
+                )
+                return None
 
     def _process_query(self, query: str) -> list[str]:
         """Tokenize and normalize query.

nextcloud_mcp_server/server/semantic.py

@@ -45,6 +45,7 @@ def configure_semantic_tools(mcp: FastMCP):
         query: str,
         ctx: Context,
         limit: int = 10,
+        doc_types: list[str] | None = None,
         score_threshold: float = 0.7,
         algorithm: Literal["semantic", "keyword", "fuzzy", "hybrid"] = "hybrid",
         semantic_weight: float = 0.5,
@@ -52,7 +53,7 @@ def configure_semantic_tools(mcp: FastMCP):
         fuzzy_weight: float = 0.2,
     ) -> SemanticSearchResponse:
         """
-        Search Nextcloud content using configurable algorithms.
+        Search Nextcloud content using configurable algorithms with cross-app support.
 
         Supports multiple search algorithms with client-configurable weighting:
         - semantic: Vector similarity search (requires VECTOR_SYNC_ENABLED=true)
@@ -60,9 +61,13 @@ def configure_semantic_tools(mcp: FastMCP):
         - fuzzy: Character overlap matching (typo-tolerant)
         - hybrid: Combines all algorithms using Reciprocal Rank Fusion (default)
 
+        Document types are queried from the vector database to determine what's
+        actually indexed. Currently only "note" documents are fully supported.
+
         Args:
             query: Natural language search query
             limit: Maximum number of results to return (default: 10)
+            doc_types: Document types to search (e.g., ["note", "file"]). None = search all indexed types (default)
             score_threshold: Minimum similarity score for semantic/hybrid (0-1, default: 0.7)
             algorithm: Search algorithm to use (default: "hybrid")
             semantic_weight: Weight for semantic results in hybrid mode (default: 0.5)
@@ -116,15 +121,42 @@ def configure_semantic_tools(mcp: FastMCP):
                     ErrorData(code=-1, message=f"Unknown algorithm: {algorithm}")
                 )
 
-            # Execute search (currently limited to notes doc_type)
+            # Execute search across requested document types
-            search_results = await search_algo.search(
+            # If doc_types is None, search all indexed types (cross-app search)
-                query=query,
+            # If doc_types is a list, search only those types
-                user_id=username,
+            all_results = []
-                limit=limit,
+
-                doc_type="note",
+            if doc_types is None:
-                nextcloud_client=client,
+                # Cross-app search: search all indexed types
-                score_threshold=score_threshold,
+                # Pass None to search algorithm to let it query Qdrant for available types
-            )
+                search_results = await search_algo.search(
+                    query=query,
+                    user_id=username,
+                    limit=limit,
+                    doc_type=None,  # Signal to search all types
+                    nextcloud_client=client,
+                    score_threshold=score_threshold,
+                )
+                all_results.extend(search_results)
+            else:
+                # Search specific document types
+                # For each requested type, execute search and combine results
+                for dtype in doc_types:
+                    search_results = await search_algo.search(
+                        query=query,
+                        user_id=username,
+                        limit=limit * 2,  # Get extra for combining
+                        doc_type=dtype,
+                        nextcloud_client=client,
+                        score_threshold=score_threshold,
+                    )
+                    all_results.extend(search_results)
+
+                # Sort combined results by score and limit
+                all_results.sort(key=lambda r: r.score, reverse=True)
+                all_results = all_results[:limit]
+
+            search_results = all_results
 
             # Convert SearchResult objects to SemanticSearchResult for response
             results = []