Merge branch 'feature/bm25'
This commit is contained in:
Chris Coutinho
@@ -1,13 +1,11 @@
|
||||
"""Search algorithms module for unified multi-algorithm search.
|
||||
"""Search algorithms module for BM25 hybrid search.
|
||||
|
||||
This module provides a unified interface for different search algorithms:
|
||||
- Semantic search (vector similarity)
|
||||
- Keyword search (token-based matching)
|
||||
- Fuzzy search (character overlap)
|
||||
- Hybrid search (RRF fusion of multiple algorithms)
|
||||
This module provides BM25 hybrid search combining:
|
||||
- Dense semantic vectors (vector similarity via embeddings)
|
||||
- Sparse BM25 vectors (keyword-based retrieval)
|
||||
|
||||
All algorithms share the same interface and can be used interchangeably by both
|
||||
MCP tools and the visualization pane.
|
||||
Results are fused using Qdrant's native Reciprocal Rank Fusion (RRF) for
|
||||
optimal relevance across both semantic and keyword queries.
|
||||
"""
|
||||
|
||||
from nextcloud_mcp_server.search.algorithms import (
|
||||
@@ -16,9 +14,7 @@ from nextcloud_mcp_server.search.algorithms import (
|
||||
SearchResult,
|
||||
get_indexed_doc_types,
|
||||
)
|
||||
from nextcloud_mcp_server.search.fuzzy import FuzzySearchAlgorithm
|
||||
from nextcloud_mcp_server.search.hybrid import HybridSearchAlgorithm
|
||||
from nextcloud_mcp_server.search.keyword import KeywordSearchAlgorithm
|
||||
from nextcloud_mcp_server.search.bm25_hybrid import BM25HybridSearchAlgorithm
|
||||
from nextcloud_mcp_server.search.semantic import SemanticSearchAlgorithm
|
||||
|
||||
__all__ = [
|
||||
@@ -27,7 +23,5 @@ __all__ = [
|
||||
"SearchResult",
|
||||
"get_indexed_doc_types",
|
||||
"SemanticSearchAlgorithm",
|
||||
"KeywordSearchAlgorithm",
|
||||
"FuzzySearchAlgorithm",
|
||||
"HybridSearchAlgorithm",
|
||||
"BM25HybridSearchAlgorithm",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
"""BM25 hybrid search algorithm using Qdrant native RRF fusion."""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from qdrant_client import models
|
||||
from qdrant_client.models import FieldCondition, Filter, MatchValue
|
||||
|
||||
from nextcloud_mcp_server.config import get_settings
|
||||
from nextcloud_mcp_server.embedding import get_bm25_service, get_embedding_service
|
||||
from nextcloud_mcp_server.observability.metrics import record_qdrant_operation
|
||||
from nextcloud_mcp_server.search.algorithms import SearchAlgorithm, SearchResult
|
||||
from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BM25HybridSearchAlgorithm(SearchAlgorithm):
|
||||
"""
|
||||
Hybrid search combining dense semantic vectors with BM25 sparse vectors.
|
||||
|
||||
Uses Qdrant's native Reciprocal Rank Fusion (RRF) to automatically merge
|
||||
results from both dense (semantic) and sparse (BM25 keyword) searches.
|
||||
This provides the best of both worlds: semantic understanding for conceptual
|
||||
queries and precise keyword matching for specific terms, acronyms, and codes.
|
||||
|
||||
The fusion happens efficiently in the database using the prefetch mechanism,
|
||||
eliminating the need for application-layer result merging.
|
||||
"""
|
||||
|
||||
def __init__(self, score_threshold: float = 0.0):
|
||||
"""
|
||||
Initialize BM25 hybrid search algorithm.
|
||||
|
||||
Args:
|
||||
score_threshold: Minimum RRF score (0-1, default: 0.0 to allow RRF scoring)
|
||||
Note: RRF produces normalized scores, so threshold is typically lower
|
||||
"""
|
||||
self.score_threshold = score_threshold
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "bm25_hybrid"
|
||||
|
||||
@property
|
||||
def requires_vector_db(self) -> bool:
|
||||
return True
|
||||
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
user_id: str,
|
||||
limit: int = 10,
|
||||
doc_type: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> list[SearchResult]:
|
||||
"""
|
||||
Execute hybrid search using dense + sparse vectors with native RRF fusion.
|
||||
|
||||
Returns unverified results from Qdrant. Access verification should be
|
||||
performed separately at the final output stage using verify_search_results().
|
||||
|
||||
Args:
|
||||
query: Natural language or keyword search query
|
||||
user_id: User ID for filtering
|
||||
limit: Maximum results to return
|
||||
doc_type: Optional document type filter
|
||||
**kwargs: Additional parameters (score_threshold override)
|
||||
|
||||
Returns:
|
||||
List of unverified SearchResult objects ranked by RRF fusion score
|
||||
|
||||
Raises:
|
||||
McpError: If vector sync is not enabled or search fails
|
||||
"""
|
||||
settings = get_settings()
|
||||
score_threshold = kwargs.get("score_threshold", self.score_threshold)
|
||||
|
||||
logger.info(
|
||||
f"BM25 hybrid search: query='{query}', user={user_id}, "
|
||||
f"limit={limit}, score_threshold={score_threshold}, doc_type={doc_type}"
|
||||
)
|
||||
|
||||
# Generate dense embedding for semantic search
|
||||
embedding_service = get_embedding_service()
|
||||
dense_embedding = await embedding_service.embed(query)
|
||||
logger.debug(f"Generated dense embedding (dimension={len(dense_embedding)})")
|
||||
|
||||
# Generate sparse embedding for BM25 keyword search
|
||||
bm25_service = get_bm25_service()
|
||||
sparse_embedding = bm25_service.encode(query)
|
||||
logger.debug(
|
||||
f"Generated sparse embedding "
|
||||
f"({len(sparse_embedding['indices'])} non-zero terms)"
|
||||
)
|
||||
|
||||
# Build Qdrant filter
|
||||
filter_conditions = [
|
||||
FieldCondition(
|
||||
key="user_id",
|
||||
match=MatchValue(value=user_id),
|
||||
)
|
||||
]
|
||||
|
||||
# Add doc_type filter if specified
|
||||
if doc_type:
|
||||
filter_conditions.append(
|
||||
FieldCondition(
|
||||
key="doc_type",
|
||||
match=MatchValue(value=doc_type),
|
||||
)
|
||||
)
|
||||
|
||||
query_filter = Filter(must=filter_conditions)
|
||||
|
||||
# Execute hybrid search with Qdrant native RRF fusion
|
||||
qdrant_client = await get_qdrant_client()
|
||||
try:
|
||||
# Use prefetch to run both dense and sparse searches
|
||||
# Qdrant will automatically merge results using RRF
|
||||
search_response = await qdrant_client.query_points(
|
||||
collection_name=settings.get_collection_name(),
|
||||
prefetch=[
|
||||
# Dense semantic search
|
||||
models.Prefetch(
|
||||
query=dense_embedding,
|
||||
using="dense",
|
||||
limit=limit * 2, # Get extra for deduplication
|
||||
filter=query_filter,
|
||||
),
|
||||
# Sparse BM25 search
|
||||
models.Prefetch(
|
||||
query=models.SparseVector(
|
||||
indices=sparse_embedding["indices"],
|
||||
values=sparse_embedding["values"],
|
||||
),
|
||||
using="sparse",
|
||||
limit=limit * 2, # Get extra for deduplication
|
||||
filter=query_filter,
|
||||
),
|
||||
],
|
||||
# RRF fusion query (no additional query needed, just fusion)
|
||||
query=models.FusionQuery(fusion=models.Fusion.RRF),
|
||||
limit=limit * 2, # Get extra for deduplication
|
||||
score_threshold=score_threshold,
|
||||
with_payload=True,
|
||||
with_vectors=False, # Don't return vectors to save bandwidth
|
||||
)
|
||||
record_qdrant_operation("search", "success")
|
||||
except Exception:
|
||||
record_qdrant_operation("search", "error")
|
||||
raise
|
||||
|
||||
logger.info(
|
||||
f"Qdrant RRF fusion returned {len(search_response.points)} results "
|
||||
f"(before deduplication)"
|
||||
)
|
||||
|
||||
if search_response.points:
|
||||
# Log top 3 RRF scores to help with threshold tuning
|
||||
top_scores = [p.score for p in search_response.points[:3]]
|
||||
logger.debug(f"Top 3 RRF fusion scores: {top_scores}")
|
||||
|
||||
# Deduplicate by (doc_id, doc_type) - multiple chunks per document
|
||||
seen_docs = set()
|
||||
results = []
|
||||
|
||||
for result in search_response.points:
|
||||
doc_id = int(result.payload["doc_id"])
|
||||
doc_type = result.payload.get("doc_type", "note")
|
||||
doc_key = (doc_id, doc_type)
|
||||
|
||||
# Skip if we've already seen this document
|
||||
if doc_key in seen_docs:
|
||||
continue
|
||||
|
||||
seen_docs.add(doc_key)
|
||||
|
||||
# Return unverified results (verification happens at output stage)
|
||||
results.append(
|
||||
SearchResult(
|
||||
id=doc_id,
|
||||
doc_type=doc_type,
|
||||
title=result.payload.get("title", "Untitled"),
|
||||
excerpt=result.payload.get("excerpt", ""),
|
||||
score=result.score, # RRF fusion score
|
||||
metadata={
|
||||
"chunk_index": result.payload.get("chunk_index"),
|
||||
"total_chunks": result.payload.get("total_chunks"),
|
||||
"search_method": "bm25_hybrid_rrf",
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
if len(results) >= limit:
|
||||
break
|
||||
|
||||
logger.info(f"Returning {len(results)} unverified results after deduplication")
|
||||
if results:
|
||||
result_details = [
|
||||
f"{r.doc_type}_{r.id} (score={r.score:.3f}, title='{r.title}')"
|
||||
for r in results[:5] # Show top 5
|
||||
]
|
||||
logger.debug(f"Top results: {', '.join(result_details)}")
|
||||
|
||||
return results
|
||||
@@ -1,219 +0,0 @@
|
||||
"""Fuzzy search algorithm using character overlap matching on Qdrant payload."""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from qdrant_client.models import FieldCondition, Filter, MatchValue
|
||||
|
||||
from nextcloud_mcp_server.config import get_settings
|
||||
from nextcloud_mcp_server.search.algorithms import SearchAlgorithm, SearchResult
|
||||
from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FuzzySearchAlgorithm(SearchAlgorithm):
|
||||
"""Fuzzy search using simple character-based similarity.
|
||||
|
||||
Implements character overlap matching with configurable threshold:
|
||||
- Compares character sets between query and text
|
||||
- Requires configurable % character overlap to match (default: 70%)
|
||||
- Tolerant to typos and minor variations
|
||||
"""
|
||||
|
||||
def __init__(self, threshold: float = 0.7):
|
||||
"""Initialize fuzzy search algorithm.
|
||||
|
||||
Args:
|
||||
threshold: Minimum character overlap ratio (0-1, default: 0.7)
|
||||
"""
|
||||
if not 0.0 <= threshold <= 1.0:
|
||||
raise ValueError(f"Threshold must be between 0.0 and 1.0, got {threshold}")
|
||||
self.threshold = threshold
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "fuzzy"
|
||||
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
user_id: str,
|
||||
limit: int = 10,
|
||||
doc_type: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> list[SearchResult]:
|
||||
"""Execute fuzzy search using character overlap on Qdrant payload.
|
||||
|
||||
Queries Qdrant for all indexed documents, then scores based on character
|
||||
overlap in title and excerpt fields. Returns unverified results - access
|
||||
verification should be performed separately at the final output stage.
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
user_id: User ID for filtering
|
||||
limit: Maximum results to return
|
||||
doc_type: Optional document type filter (None = all types)
|
||||
**kwargs: Additional parameters (threshold override)
|
||||
|
||||
Returns:
|
||||
List of unverified SearchResult objects ranked by character overlap score
|
||||
"""
|
||||
settings = get_settings()
|
||||
threshold = kwargs.get("threshold", self.threshold)
|
||||
|
||||
logger.info(
|
||||
f"Fuzzy search: query='{query}', user={user_id}, "
|
||||
f"limit={limit}, threshold={threshold}, doc_type={doc_type}"
|
||||
)
|
||||
|
||||
# Build Qdrant filter
|
||||
filter_conditions = [
|
||||
FieldCondition(key="user_id", match=MatchValue(value=user_id))
|
||||
]
|
||||
if doc_type:
|
||||
filter_conditions.append(
|
||||
FieldCondition(key="doc_type", match=MatchValue(value=doc_type))
|
||||
)
|
||||
|
||||
# Scroll through Qdrant to get all matching documents
|
||||
qdrant_client = await get_qdrant_client()
|
||||
collection = settings.get_collection_name()
|
||||
|
||||
all_points = []
|
||||
offset = None
|
||||
|
||||
# Scroll through all points matching filter
|
||||
while True:
|
||||
scroll_result, next_offset = await qdrant_client.scroll(
|
||||
collection_name=collection,
|
||||
scroll_filter=Filter(must=filter_conditions),
|
||||
limit=100, # Batch size
|
||||
offset=offset,
|
||||
with_payload=["doc_id", "doc_type", "title", "excerpt", "chunk_index"],
|
||||
with_vectors=False, # Don't need vectors
|
||||
)
|
||||
|
||||
all_points.extend(scroll_result)
|
||||
|
||||
if next_offset is None:
|
||||
break
|
||||
offset = next_offset
|
||||
|
||||
logger.debug(f"Retrieved {len(all_points)} points from Qdrant for fuzzy search")
|
||||
|
||||
# Deduplicate by (doc_id, doc_type) - keep first chunk
|
||||
seen_docs = {}
|
||||
for point in all_points:
|
||||
doc_id = int(point.payload["doc_id"])
|
||||
dtype = point.payload.get("doc_type", "note")
|
||||
doc_key = (doc_id, dtype)
|
||||
|
||||
chunk_idx = point.payload.get("chunk_index", 0)
|
||||
if doc_key not in seen_docs or chunk_idx == 0:
|
||||
seen_docs[doc_key] = point
|
||||
|
||||
logger.debug(f"Deduplicated to {len(seen_docs)} unique documents")
|
||||
|
||||
# Score each document based on fuzzy matches
|
||||
scored_results = []
|
||||
query_lower = query.lower()
|
||||
|
||||
for doc_key, point in seen_docs.items():
|
||||
doc_id, dtype = doc_key
|
||||
title = point.payload.get("title", "")
|
||||
excerpt = point.payload.get("excerpt", "")
|
||||
|
||||
# Check title match
|
||||
title_score = self._calculate_char_overlap(query_lower, title.lower())
|
||||
|
||||
# Check excerpt match
|
||||
excerpt_score = self._calculate_char_overlap(query_lower, excerpt.lower())
|
||||
|
||||
# Use best score
|
||||
best_score = max(title_score, excerpt_score)
|
||||
|
||||
if best_score >= threshold:
|
||||
match_location = "title" if title_score >= excerpt_score else "excerpt"
|
||||
scored_results.append(
|
||||
{
|
||||
"doc_id": doc_id,
|
||||
"doc_type": dtype,
|
||||
"title": title,
|
||||
"excerpt": excerpt
|
||||
if excerpt_score >= title_score
|
||||
else f"Title match: {title}",
|
||||
"score": best_score,
|
||||
"match_location": match_location,
|
||||
}
|
||||
)
|
||||
|
||||
# Sort by score (descending) and limit
|
||||
scored_results.sort(key=lambda x: x["score"], reverse=True)
|
||||
top_results = scored_results[:limit]
|
||||
|
||||
# Return unverified results (verification happens at output stage)
|
||||
final_results = []
|
||||
for result in top_results:
|
||||
final_results.append(
|
||||
SearchResult(
|
||||
id=result["doc_id"],
|
||||
doc_type=result["doc_type"],
|
||||
title=result["title"],
|
||||
excerpt=result["excerpt"],
|
||||
score=result["score"],
|
||||
metadata={"match_location": result["match_location"]},
|
||||
)
|
||||
)
|
||||
|
||||
logger.info(f"Fuzzy search returned {len(final_results)} unverified results")
|
||||
if final_results:
|
||||
result_details = [
|
||||
f"{r.doc_type}_{r.id} (score={r.score:.3f}, title='{r.title}')"
|
||||
for r in final_results[:5]
|
||||
]
|
||||
logger.debug(f"Top fuzzy results: {', '.join(result_details)}")
|
||||
|
||||
return final_results
|
||||
|
||||
def _calculate_char_overlap(self, query: str, text: str) -> float:
|
||||
"""Calculate character overlap ratio between query and text.
|
||||
|
||||
Args:
|
||||
query: Query string (normalized)
|
||||
text: Text to compare (normalized)
|
||||
|
||||
Returns:
|
||||
Overlap ratio (0.0-1.0)
|
||||
"""
|
||||
if not query or not text:
|
||||
return 0.0
|
||||
|
||||
# Convert to character sets
|
||||
query_chars = set(query)
|
||||
text_chars = set(text)
|
||||
|
||||
# Calculate overlap
|
||||
overlap = query_chars & text_chars
|
||||
overlap_ratio = len(overlap) / len(query_chars)
|
||||
|
||||
return overlap_ratio
|
||||
|
||||
def _extract_excerpt(self, content: str, max_length: int = 200) -> str:
|
||||
"""Extract excerpt from content.
|
||||
|
||||
Args:
|
||||
content: Full document content
|
||||
max_length: Maximum excerpt length
|
||||
|
||||
Returns:
|
||||
Excerpt string
|
||||
"""
|
||||
if not content:
|
||||
return ""
|
||||
|
||||
excerpt = content[:max_length].strip()
|
||||
if len(content) > max_length:
|
||||
excerpt += "..."
|
||||
|
||||
return excerpt
|
||||
@@ -1,278 +0,0 @@
|
||||
"""Hybrid search algorithm using Reciprocal Rank Fusion (RRF)."""
|
||||
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from typing import Any
|
||||
|
||||
import anyio
|
||||
|
||||
from nextcloud_mcp_server.search.algorithms import SearchAlgorithm, SearchResult
|
||||
from nextcloud_mcp_server.search.fuzzy import FuzzySearchAlgorithm
|
||||
from nextcloud_mcp_server.search.keyword import KeywordSearchAlgorithm
|
||||
from nextcloud_mcp_server.search.semantic import SemanticSearchAlgorithm
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HybridSearchAlgorithm(SearchAlgorithm):
|
||||
"""Hybrid search combining multiple algorithms using Reciprocal Rank Fusion.
|
||||
|
||||
Implements RRF from ADR-003 to combine results from:
|
||||
- Semantic search (vector similarity)
|
||||
- Keyword search (token matching)
|
||||
- Fuzzy search (character overlap)
|
||||
|
||||
RRF formula: score = weight / (k + rank)
|
||||
where k=60 (standard value) and rank is 1-indexed position.
|
||||
"""
|
||||
|
||||
DEFAULT_RRF_K = 60 # Standard RRF constant
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
semantic_weight: float = 0.5,
|
||||
keyword_weight: float = 0.3,
|
||||
fuzzy_weight: float = 0.2,
|
||||
rrf_k: int = DEFAULT_RRF_K,
|
||||
):
|
||||
"""Initialize hybrid search with algorithm weights.
|
||||
|
||||
Args:
|
||||
semantic_weight: Weight for semantic results (default: 0.5)
|
||||
keyword_weight: Weight for keyword results (default: 0.3)
|
||||
fuzzy_weight: Weight for fuzzy results (default: 0.2)
|
||||
rrf_k: RRF constant for rank decay (default: 60)
|
||||
|
||||
Raises:
|
||||
ValueError: If weights are invalid
|
||||
"""
|
||||
# Validate weights
|
||||
if semantic_weight < 0 or keyword_weight < 0 or fuzzy_weight < 0:
|
||||
raise ValueError("Weights must be non-negative")
|
||||
|
||||
total_weight = semantic_weight + keyword_weight + fuzzy_weight
|
||||
if total_weight > 1.0:
|
||||
raise ValueError(f"Weights sum to {total_weight:.2f}, must be ≤1.0")
|
||||
|
||||
if total_weight == 0.0:
|
||||
raise ValueError("At least one weight must be > 0")
|
||||
|
||||
self.semantic_weight = semantic_weight
|
||||
self.keyword_weight = keyword_weight
|
||||
self.fuzzy_weight = fuzzy_weight
|
||||
self.rrf_k = rrf_k
|
||||
self.total_weight = total_weight
|
||||
|
||||
# Initialize sub-algorithms
|
||||
self.semantic = SemanticSearchAlgorithm()
|
||||
self.keyword = KeywordSearchAlgorithm()
|
||||
self.fuzzy = FuzzySearchAlgorithm()
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "hybrid"
|
||||
|
||||
@property
|
||||
def requires_vector_db(self) -> bool:
|
||||
# Requires vector DB if semantic search has non-zero weight
|
||||
return self.semantic_weight > 0
|
||||
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
user_id: str,
|
||||
limit: int = 10,
|
||||
doc_type: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> list[SearchResult]:
|
||||
"""Execute hybrid search using RRF to combine algorithms.
|
||||
|
||||
Returns unverified results from combined algorithms. Access verification
|
||||
should be performed separately at the final output stage.
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
user_id: User ID for filtering
|
||||
limit: Maximum results to return
|
||||
doc_type: Optional document type filter
|
||||
**kwargs: Additional parameters passed to sub-algorithms
|
||||
|
||||
Returns:
|
||||
List of unverified SearchResult objects ranked by RRF combined score
|
||||
"""
|
||||
logger.info(
|
||||
f"Hybrid search: query='{query}', user={user_id}, limit={limit}, "
|
||||
f"weights=(semantic={self.semantic_weight}, keyword={self.keyword_weight}, "
|
||||
f"fuzzy={self.fuzzy_weight})"
|
||||
)
|
||||
|
||||
# Prepare algorithm configurations for parallel execution
|
||||
algo_configs = []
|
||||
if self.semantic_weight > 0:
|
||||
algo_configs.append(
|
||||
(
|
||||
"semantic",
|
||||
self.semantic.search,
|
||||
query,
|
||||
user_id,
|
||||
limit * 2,
|
||||
doc_type,
|
||||
kwargs,
|
||||
)
|
||||
)
|
||||
if self.keyword_weight > 0:
|
||||
algo_configs.append(
|
||||
(
|
||||
"keyword",
|
||||
self.keyword.search,
|
||||
query,
|
||||
user_id,
|
||||
limit * 2,
|
||||
doc_type,
|
||||
kwargs,
|
||||
)
|
||||
)
|
||||
if self.fuzzy_weight > 0:
|
||||
algo_configs.append(
|
||||
(
|
||||
"fuzzy",
|
||||
self.fuzzy.search,
|
||||
query,
|
||||
user_id,
|
||||
limit * 2,
|
||||
doc_type,
|
||||
kwargs,
|
||||
)
|
||||
)
|
||||
|
||||
# Pre-allocate results list and extract algorithm names
|
||||
results_list = [None] * len(algo_configs)
|
||||
algo_names = [name for name, *_ in algo_configs]
|
||||
|
||||
async def search_one(
|
||||
index: int,
|
||||
search_func,
|
||||
query_arg: str,
|
||||
user_id_arg: str,
|
||||
limit_arg: int,
|
||||
doc_type_arg: str | None,
|
||||
kwargs_arg: dict,
|
||||
):
|
||||
"""Execute one search algorithm and store result at index."""
|
||||
result = await search_func(
|
||||
query_arg, user_id_arg, limit_arg, doc_type_arg, **kwargs_arg
|
||||
)
|
||||
results_list[index] = result
|
||||
|
||||
# Execute searches in parallel using anyio task group
|
||||
async with anyio.create_task_group() as tg:
|
||||
for idx, (name, search_func, q, uid, lim, dt, kw) in enumerate(
|
||||
algo_configs
|
||||
):
|
||||
tg.start_soon(search_one, idx, search_func, q, uid, lim, dt, kw)
|
||||
|
||||
# Build results dict
|
||||
algo_results = {}
|
||||
for algo_name, results in zip(algo_names, results_list):
|
||||
algo_results[algo_name] = results
|
||||
logger.debug(f"{algo_name} returned {len(results)} results")
|
||||
|
||||
# Combine using RRF
|
||||
combined_results = self._reciprocal_rank_fusion(
|
||||
algo_results,
|
||||
{
|
||||
"semantic": self.semantic_weight,
|
||||
"keyword": self.keyword_weight,
|
||||
"fuzzy": self.fuzzy_weight,
|
||||
},
|
||||
limit,
|
||||
)
|
||||
|
||||
logger.info(f"Hybrid search returned {len(combined_results)} combined results")
|
||||
if combined_results:
|
||||
result_details = [
|
||||
f"{r.doc_type}_{r.id} (score={r.score:.3f}, title='{r.title}')"
|
||||
for r in combined_results[:5]
|
||||
]
|
||||
logger.debug(f"Top hybrid results: {', '.join(result_details)}")
|
||||
|
||||
return combined_results
|
||||
|
||||
def _reciprocal_rank_fusion(
|
||||
self,
|
||||
algo_results: dict[str, list[SearchResult]],
|
||||
weights: dict[str, float],
|
||||
limit: int,
|
||||
) -> list[SearchResult]:
|
||||
"""Combine multiple ranked result lists using RRF.
|
||||
|
||||
Args:
|
||||
algo_results: Dict of algorithm_name -> ranked results
|
||||
weights: Dict of algorithm_name -> weight (0-1)
|
||||
limit: Maximum results to return
|
||||
|
||||
Returns:
|
||||
Combined and re-ranked results
|
||||
"""
|
||||
# Track RRF scores per document
|
||||
rrf_scores: dict[tuple[int, str], float] = defaultdict(float)
|
||||
# Track best result object for each document
|
||||
best_results: dict[tuple[int, str], SearchResult] = {}
|
||||
|
||||
for algo_name, results in algo_results.items():
|
||||
weight = weights.get(algo_name, 0.0)
|
||||
if weight == 0:
|
||||
continue
|
||||
|
||||
for rank, result in enumerate(results, start=1):
|
||||
doc_key = (result.id, result.doc_type)
|
||||
|
||||
# RRF formula: weight / (k + rank)
|
||||
rrf_score = weight / (self.rrf_k + rank)
|
||||
rrf_scores[doc_key] += rrf_score
|
||||
|
||||
# Track best result object (prefer higher original scores)
|
||||
if doc_key not in best_results:
|
||||
best_results[doc_key] = result
|
||||
elif result.score > best_results[doc_key].score:
|
||||
best_results[doc_key] = result
|
||||
|
||||
# Sort by combined RRF score
|
||||
sorted_docs = sorted(
|
||||
rrf_scores.items(),
|
||||
key=lambda x: x[1],
|
||||
reverse=True,
|
||||
)[:limit]
|
||||
|
||||
# Calculate normalization factor to scale RRF scores to 0-1 range
|
||||
# Theoretical max RRF score = total_weight / (rrf_k + 1)
|
||||
# Normalization factor = (rrf_k + 1) / total_weight
|
||||
normalization_factor = (self.rrf_k + 1) / self.total_weight
|
||||
|
||||
# Build final results with normalized RRF scores
|
||||
final_results = []
|
||||
for doc_key, rrf_score in sorted_docs:
|
||||
result = best_results[doc_key]
|
||||
|
||||
# Normalize RRF score to 0-1 range for better user comprehension
|
||||
normalized_score = rrf_score * normalization_factor
|
||||
|
||||
# Create new result with normalized score
|
||||
# Keep original metadata but add RRF details
|
||||
metadata = result.metadata or {}
|
||||
metadata["rrf_score_raw"] = rrf_score # Original RRF score
|
||||
metadata["original_score"] = result.score # Original algorithm score
|
||||
metadata["normalization_factor"] = normalization_factor
|
||||
|
||||
final_results.append(
|
||||
SearchResult(
|
||||
id=result.id,
|
||||
doc_type=result.doc_type,
|
||||
title=result.title,
|
||||
excerpt=result.excerpt,
|
||||
score=normalized_score, # Use normalized score (0-1 range)
|
||||
metadata=metadata,
|
||||
)
|
||||
)
|
||||
|
||||
return final_results
|
||||
@@ -1,277 +0,0 @@
|
||||
"""Keyword search algorithm using token-based matching on Qdrant payload (ADR-001)."""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from qdrant_client.models import FieldCondition, Filter, MatchValue
|
||||
|
||||
from nextcloud_mcp_server.config import get_settings
|
||||
from nextcloud_mcp_server.search.algorithms import SearchAlgorithm, SearchResult
|
||||
from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class KeywordSearchAlgorithm(SearchAlgorithm):
|
||||
"""Keyword search using token-based matching with weighted scoring.
|
||||
|
||||
Implements token-based search from ADR-001:
|
||||
- Title matches weighted 3x higher than content matches
|
||||
- Case-insensitive token matching
|
||||
- Relevance scoring based on match frequency and location
|
||||
"""
|
||||
|
||||
# Weighting constants from ADR-001
|
||||
TITLE_WEIGHT = 3.0
|
||||
CONTENT_WEIGHT = 1.0
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "keyword"
|
||||
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
user_id: str,
|
||||
limit: int = 10,
|
||||
doc_type: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> list[SearchResult]:
|
||||
"""Execute keyword search using token matching on Qdrant payload.
|
||||
|
||||
Queries Qdrant for all indexed documents, then scores based on token
|
||||
matches in title and excerpt fields. Returns unverified results - access
|
||||
verification should be performed separately at the final output stage.
|
||||
|
||||
Args:
|
||||
query: Search query to tokenize and match
|
||||
user_id: User ID for filtering
|
||||
limit: Maximum results to return
|
||||
doc_type: Optional document type filter (None = all types)
|
||||
**kwargs: Additional parameters (unused)
|
||||
|
||||
Returns:
|
||||
List of unverified SearchResult objects ranked by keyword match score
|
||||
"""
|
||||
settings = get_settings()
|
||||
|
||||
logger.info(
|
||||
f"Keyword search: query='{query}', user={user_id}, "
|
||||
f"limit={limit}, doc_type={doc_type}"
|
||||
)
|
||||
|
||||
# Tokenize query
|
||||
query_tokens = self._process_query(query)
|
||||
logger.debug(f"Query tokens: {query_tokens}")
|
||||
|
||||
# Build Qdrant filter
|
||||
filter_conditions = [
|
||||
FieldCondition(key="user_id", match=MatchValue(value=user_id))
|
||||
]
|
||||
if doc_type:
|
||||
filter_conditions.append(
|
||||
FieldCondition(key="doc_type", match=MatchValue(value=doc_type))
|
||||
)
|
||||
|
||||
# Scroll through Qdrant to get all matching documents
|
||||
# We need title and excerpt from payload for token matching
|
||||
qdrant_client = await get_qdrant_client()
|
||||
collection = settings.get_collection_name()
|
||||
|
||||
all_points = []
|
||||
offset = None
|
||||
|
||||
# Scroll through all points matching filter
|
||||
while True:
|
||||
scroll_result, next_offset = await qdrant_client.scroll(
|
||||
collection_name=collection,
|
||||
scroll_filter=Filter(must=filter_conditions),
|
||||
limit=100, # Batch size
|
||||
offset=offset,
|
||||
with_payload=[
|
||||
"doc_id",
|
||||
"doc_type",
|
||||
"title",
|
||||
"excerpt",
|
||||
"chunk_index",
|
||||
"total_chunks",
|
||||
],
|
||||
with_vectors=False, # Don't need vectors for keyword search
|
||||
)
|
||||
|
||||
all_points.extend(scroll_result)
|
||||
|
||||
if next_offset is None:
|
||||
break
|
||||
offset = next_offset
|
||||
|
||||
logger.debug(
|
||||
f"Retrieved {len(all_points)} points from Qdrant for keyword search"
|
||||
)
|
||||
|
||||
# Deduplicate by (doc_id, doc_type) - keep best chunk per document
|
||||
seen_docs = {}
|
||||
for point in all_points:
|
||||
doc_id = int(point.payload["doc_id"])
|
||||
dtype = point.payload.get("doc_type", "note")
|
||||
doc_key = (doc_id, dtype)
|
||||
|
||||
# Keep first chunk (chunk_index=0) as it has the most relevant content
|
||||
chunk_idx = point.payload.get("chunk_index", 0)
|
||||
if doc_key not in seen_docs or chunk_idx == 0:
|
||||
seen_docs[doc_key] = point
|
||||
|
||||
logger.debug(f"Deduplicated to {len(seen_docs)} unique documents")
|
||||
|
||||
# Score each document based on keyword matches
|
||||
scored_results = []
|
||||
for doc_key, point in seen_docs.items():
|
||||
doc_id, dtype = doc_key
|
||||
title = point.payload.get("title", "")
|
||||
excerpt = point.payload.get("excerpt", "")
|
||||
|
||||
# Calculate keyword match score
|
||||
score = self._calculate_score(query_tokens, title, excerpt)
|
||||
|
||||
if score > 0: # Only include matches
|
||||
scored_results.append(
|
||||
{
|
||||
"doc_id": doc_id,
|
||||
"doc_type": dtype,
|
||||
"title": title,
|
||||
"excerpt": excerpt,
|
||||
"score": score,
|
||||
}
|
||||
)
|
||||
|
||||
# Sort by score (descending) and limit
|
||||
scored_results.sort(key=lambda x: x["score"], reverse=True)
|
||||
top_results = scored_results[:limit]
|
||||
|
||||
# Return unverified results (verification happens at output stage)
|
||||
final_results = []
|
||||
for result in top_results:
|
||||
final_results.append(
|
||||
SearchResult(
|
||||
id=result["doc_id"],
|
||||
doc_type=result["doc_type"],
|
||||
title=result["title"],
|
||||
excerpt=result["excerpt"],
|
||||
score=result["score"],
|
||||
metadata={},
|
||||
)
|
||||
)
|
||||
|
||||
logger.info(f"Keyword search returned {len(final_results)} unverified results")
|
||||
if final_results:
|
||||
result_details = [
|
||||
f"{r.doc_type}_{r.id} (score={r.score:.3f}, title='{r.title}')"
|
||||
for r in final_results[:5]
|
||||
]
|
||||
logger.debug(f"Top keyword results: {', '.join(result_details)}")
|
||||
|
||||
return final_results
|
||||
|
||||
def _process_query(self, query: str) -> list[str]:
|
||||
"""Tokenize and normalize query.
|
||||
|
||||
Args:
|
||||
query: Raw query string
|
||||
|
||||
Returns:
|
||||
List of normalized tokens
|
||||
"""
|
||||
# Convert to lowercase and split into tokens
|
||||
tokens = query.lower().split()
|
||||
|
||||
# Filter out very short tokens (optional)
|
||||
tokens = [token for token in tokens if len(token) > 1]
|
||||
|
||||
return tokens
|
||||
|
||||
def _calculate_score(
|
||||
self, query_tokens: list[str], title: str, content: str
|
||||
) -> float:
|
||||
"""Calculate relevance score based on token matches.
|
||||
|
||||
Args:
|
||||
query_tokens: List of query tokens
|
||||
title: Document title
|
||||
content: Document content
|
||||
|
||||
Returns:
|
||||
Relevance score (0.0-1.0)
|
||||
"""
|
||||
if not query_tokens:
|
||||
return 0.0
|
||||
|
||||
# Process title and content
|
||||
title_tokens = title.lower().split()
|
||||
content_tokens = content.lower().split()
|
||||
|
||||
score = 0.0
|
||||
|
||||
# Count matches in title
|
||||
title_matches = sum(1 for qt in query_tokens if qt in title_tokens)
|
||||
if query_tokens: # Avoid division by zero
|
||||
title_match_ratio = title_matches / len(query_tokens)
|
||||
score += self.TITLE_WEIGHT * title_match_ratio
|
||||
|
||||
# Count matches in content
|
||||
content_matches = sum(1 for qt in query_tokens if qt in content_tokens)
|
||||
if query_tokens:
|
||||
content_match_ratio = content_matches / len(query_tokens)
|
||||
score += self.CONTENT_WEIGHT * content_match_ratio
|
||||
|
||||
# Normalize score to 0-1 range
|
||||
# Max score would be TITLE_WEIGHT + CONTENT_WEIGHT if all tokens match everywhere
|
||||
max_score = self.TITLE_WEIGHT + self.CONTENT_WEIGHT
|
||||
normalized_score = min(score / max_score, 1.0)
|
||||
|
||||
return normalized_score
|
||||
|
||||
def _extract_excerpt(
|
||||
self, content: str, query_tokens: list[str], max_length: int = 200
|
||||
) -> str:
|
||||
"""Extract excerpt showing match context.
|
||||
|
||||
Args:
|
||||
content: Full document content
|
||||
query_tokens: Query tokens to find
|
||||
max_length: Maximum excerpt length in characters
|
||||
|
||||
Returns:
|
||||
Excerpt string with context around matches
|
||||
"""
|
||||
if not content:
|
||||
return ""
|
||||
|
||||
content_lower = content.lower()
|
||||
|
||||
# Find first occurrence of any query token
|
||||
first_match_pos = -1
|
||||
for token in query_tokens:
|
||||
pos = content_lower.find(token)
|
||||
if pos != -1:
|
||||
if first_match_pos == -1 or pos < first_match_pos:
|
||||
first_match_pos = pos
|
||||
|
||||
if first_match_pos == -1:
|
||||
# No matches found, return beginning
|
||||
return content[:max_length].strip() + (
|
||||
"..." if len(content) > max_length else ""
|
||||
)
|
||||
|
||||
# Extract context around match
|
||||
start = max(0, first_match_pos - max_length // 2)
|
||||
end = min(len(content), first_match_pos + max_length // 2)
|
||||
|
||||
excerpt = content[start:end].strip()
|
||||
|
||||
# Add ellipsis if truncated
|
||||
if start > 0:
|
||||
excerpt = "..." + excerpt
|
||||
if end < len(content):
|
||||
excerpt = excerpt + "..."
|
||||
|
||||
return excerpt
|
||||
Reference in New Issue
Block a user