Merge branch 'feature/bm25'

nextcloud_mcp_server/vector/processor.py

@@ -15,7 +15,7 @@ from qdrant_client.models import FieldCondition, Filter, MatchValue, PointStruct
 
 from nextcloud_mcp_server.client import NextcloudClient
 from nextcloud_mcp_server.config import get_settings
-from nextcloud_mcp_server.embedding import get_embedding_service
+from nextcloud_mcp_server.embedding import get_bm25_service, get_embedding_service
 from nextcloud_mcp_server.observability.metrics import (
     record_qdrant_operation,
     record_vector_sync_processing,
@@ -233,15 +233,21 @@ async def _index_document(
     )
     chunks = chunker.chunk_text(content)
 
-    # Generate embeddings (I/O bound - external API call)
+    # Generate dense embeddings (I/O bound - external API call)
     embedding_service = get_embedding_service()
-    embeddings = await embedding_service.embed_batch(chunks)
+    dense_embeddings = await embedding_service.embed_batch(chunks)
+
+    # Generate sparse embeddings (BM25 for keyword matching)
+    bm25_service = get_bm25_service()
+    sparse_embeddings = bm25_service.encode_batch(chunks)
 
     # Prepare Qdrant points
     indexed_at = int(time.time())
     points = []
 
-    for i, (chunk, embedding) in enumerate(zip(chunks, embeddings)):
+    for i, (chunk, dense_emb, sparse_emb) in enumerate(
+        zip(chunks, dense_embeddings, sparse_embeddings)
+    ):
         # Generate deterministic UUID for point ID
         # Using uuid5 with DNS namespace and combining doc info
         point_name = f"{doc_task.doc_type}:{doc_task.doc_id}:chunk:{i}"
@@ -250,7 +256,10 @@ async def _index_document(
         points.append(
             PointStruct(
                 id=point_id,
-                vector=embedding,
+                vector={
+                    "dense": dense_emb,
+                    "sparse": sparse_emb,
+                },
                 payload={
                     "user_id": doc_task.user_id,
                     "doc_id": doc_task.doc_id,

nextcloud_mcp_server/vector/qdrant_client.py

@@ -2,7 +2,7 @@
 
 import logging
 
-from qdrant_client import AsyncQdrantClient
+from qdrant_client import AsyncQdrantClient, models
 from qdrant_client.models import Distance, VectorParams
 
 from nextcloud_mcp_server.config import get_settings
@@ -84,7 +84,12 @@ async def get_qdrant_client() -> AsyncQdrantClient:
                 f"Collection '{collection_name}' found, validating dimensions..."
             )
             collection_info = await _qdrant_client.get_collection(collection_name)
-            actual_dimension = collection_info.config.params.vectors.size
+            # Handle both named vectors (dict) and legacy single vector
+            vectors = collection_info.config.params.vectors
+            if isinstance(vectors, dict):
+                actual_dimension = vectors["dense"].size
+            else:
+                actual_dimension = vectors.size
 
             # Validate dimension matches
             if actual_dimension != expected_dimension:
@@ -112,17 +117,27 @@ async def get_qdrant_client() -> AsyncQdrantClient:
             )
             await _qdrant_client.create_collection(
                 collection_name=collection_name,
-                vectors_config=VectorParams(
-                    size=expected_dimension,
-                    distance=Distance.COSINE,
-                ),
+                vectors_config={
+                    "dense": VectorParams(
+                        size=expected_dimension,
+                        distance=Distance.COSINE,
+                    ),
+                },
+                sparse_vectors_config={
+                    "sparse": models.SparseVectorParams(
+                        index=models.SparseIndexParams(
+                            on_disk=False,
+                        )
+                    ),
+                },
             )
             logger.info(
                 f"Created Qdrant collection: {collection_name}\n"
-                f"  Dimension: {expected_dimension}\n"
-                f"  Model: {settings.ollama_embedding_model}\n"
+                f"  Dense vector dimension: {expected_dimension}\n"
+                f"  Dense embedding model: {settings.ollama_embedding_model}\n"
+                f"  Sparse vectors: BM25 (for hybrid search)\n"
                 f"  Distance: COSINE\n"
-                f"Background sync will index all documents with this embedding model."
+                f"Background sync will index all documents with dense + sparse vectors."
             )
 
     return _qdrant_client