feat: implement vector sync scanner and processor (ADR-007 Phase 2)
Implements background vector database synchronization using anyio TaskGroups for BasicAuth mode with single-user credentials. Scanner Implementation: - Periodic document discovery (hourly, configurable) - Timestamp-based change detection (Nextcloud vs Qdrant) - Wake event for immediate scanning on-demand - Supports both initial sync (all docs) and incremental sync (changes only) - Detects deleted documents and queues for removal Processor Implementation: - Concurrent document processing pool (3 workers default) - I/O-bound embedding generation via Ollama API - Retry logic with exponential backoff (3 retries) - Document chunking (512 words, 50-word overlap) - Handles both index and delete operations - Upserts vectors to Qdrant with rich metadata App Lifespan Integration: - Extended AppContext with background task state - Modified app_lifespan_basic() to start tasks via anyio TaskGroups - Graceful shutdown with coordinated task cancellation - Only activates when VECTOR_SYNC_ENABLED=true Embedding Service: - OllamaEmbeddingProvider with TLS support - Singleton pattern for shared client instances - Batch embedding support for efficiency - Auto-detects embedding dimension (768 for nomic-embed-text) Qdrant Client: - Async client wrapper with singleton pattern - Auto-creates collection on first use - COSINE distance metric for semantic similarity - Integrates with embedding service for dimension detection Health Check Enhancement: - Added Qdrant status check to /health/ready endpoint - Only checks when VECTOR_SYNC_ENABLED=true - 2-second timeout for health probe - Reports connection errors with details Configuration: - VECTOR_SYNC_ENABLED: Enable background sync - VECTOR_SYNC_SCAN_INTERVAL: Scanner frequency (3600s default) - VECTOR_SYNC_PROCESSOR_WORKERS: Concurrent processors (3 default) - QDRANT_URL, QDRANT_API_KEY, QDRANT_COLLECTION: Vector DB config - OLLAMA_BASE_URL, OLLAMA_EMBEDDING_MODEL: Embedding service config Dependencies Added: - qdrant-client>=1.7.0: Vector database client Docker Compose: - Added Qdrant service with health check - Exposed ports 6333 (REST) and 6334 (gRPC) - Configured MCP service with vector sync environment - Added qdrant-data volume for persistence Known Issue: - FastMCP lifespan not triggering for streamable-http transport - Background tasks will start once lifespan integration is complete - Lifespan triggers on MCP session establishment, not server startup Related: ADR-007 Background Vector Database Synchronization 🤖 Generated with Claude Code (https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
@@ -1,3 +1,4 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from collections.abc import AsyncIterator
|
||||
@@ -8,6 +9,7 @@ from typing import TYPE_CHECKING, Optional
|
||||
if TYPE_CHECKING:
|
||||
from nextcloud_mcp_server.auth.refresh_token_storage import RefreshTokenStorage
|
||||
|
||||
import anyio
|
||||
import click
|
||||
import httpx
|
||||
import uvicorn
|
||||
@@ -32,6 +34,7 @@ from nextcloud_mcp_server.client import NextcloudClient
|
||||
from nextcloud_mcp_server.config import (
|
||||
LOGGING_CONFIG,
|
||||
get_document_processor_config,
|
||||
get_settings,
|
||||
setup_logging,
|
||||
)
|
||||
from nextcloud_mcp_server.context import get_client as get_nextcloud_client
|
||||
@@ -47,6 +50,7 @@ from nextcloud_mcp_server.server import (
|
||||
configure_webdav_tools,
|
||||
)
|
||||
from nextcloud_mcp_server.server.oauth_tools import register_oauth_tools
|
||||
from nextcloud_mcp_server.vector import processor_task, scanner_task
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -206,6 +210,9 @@ class AppContext:
|
||||
"""Application context for BasicAuth mode."""
|
||||
|
||||
client: NextcloudClient
|
||||
document_queue: Optional[asyncio.Queue] = None
|
||||
shutdown_event: Optional[anyio.Event] = None
|
||||
scanner_wake_event: Optional[anyio.Event] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -369,6 +376,9 @@ async def app_lifespan_basic(server: FastMCP) -> AsyncIterator[AppContext]:
|
||||
|
||||
Creates a single Nextcloud client with basic authentication
|
||||
that is shared across all requests.
|
||||
|
||||
If vector sync is enabled (VECTOR_SYNC_ENABLED=true), also starts
|
||||
background tasks for automatic document indexing (ADR-007).
|
||||
"""
|
||||
logger.info("Starting MCP server in BasicAuth mode")
|
||||
logger.info("Creating Nextcloud client with BasicAuth")
|
||||
@@ -379,11 +389,74 @@ async def app_lifespan_basic(server: FastMCP) -> AsyncIterator[AppContext]:
|
||||
# Initialize document processors
|
||||
initialize_document_processors()
|
||||
|
||||
try:
|
||||
yield AppContext(client=client)
|
||||
finally:
|
||||
logger.info("Shutting down BasicAuth mode")
|
||||
await client.close()
|
||||
settings = get_settings()
|
||||
|
||||
# Check if vector sync is enabled
|
||||
if settings.vector_sync_enabled:
|
||||
logger.info("Vector sync enabled - starting background tasks")
|
||||
|
||||
# Get username from environment for BasicAuth mode
|
||||
username = os.getenv("NEXTCLOUD_USERNAME")
|
||||
if not username:
|
||||
raise ValueError(
|
||||
"NEXTCLOUD_USERNAME is required for vector sync in BasicAuth mode"
|
||||
)
|
||||
|
||||
# Initialize shared state
|
||||
document_queue = asyncio.Queue(maxsize=settings.vector_sync_queue_max_size)
|
||||
shutdown_event = anyio.Event()
|
||||
scanner_wake_event = anyio.Event()
|
||||
|
||||
# Start background tasks using anyio TaskGroup
|
||||
async with anyio.create_task_group() as tg:
|
||||
# Start scanner task
|
||||
tg.start_soon(
|
||||
scanner_task,
|
||||
document_queue,
|
||||
shutdown_event,
|
||||
scanner_wake_event,
|
||||
client,
|
||||
username,
|
||||
)
|
||||
|
||||
# Start processor pool
|
||||
for i in range(settings.vector_sync_processor_workers):
|
||||
tg.start_soon(
|
||||
processor_task,
|
||||
i,
|
||||
document_queue,
|
||||
shutdown_event,
|
||||
client,
|
||||
username,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"Background sync tasks started: 1 scanner + {settings.vector_sync_processor_workers} processors"
|
||||
)
|
||||
|
||||
# Yield with background tasks running
|
||||
try:
|
||||
yield AppContext(
|
||||
client=client,
|
||||
document_queue=document_queue,
|
||||
shutdown_event=shutdown_event,
|
||||
scanner_wake_event=scanner_wake_event,
|
||||
)
|
||||
finally:
|
||||
# Shutdown signal
|
||||
logger.info("Shutting down background sync tasks")
|
||||
shutdown_event.set()
|
||||
|
||||
# TaskGroup automatically cancels all tasks on exit
|
||||
logger.info("Background sync tasks stopped")
|
||||
await client.close()
|
||||
else:
|
||||
# No vector sync - simple lifecycle
|
||||
try:
|
||||
yield AppContext(client=client)
|
||||
finally:
|
||||
logger.info("Shutting down BasicAuth mode")
|
||||
await client.close()
|
||||
|
||||
|
||||
async def setup_oauth_config():
|
||||
@@ -946,7 +1019,7 @@ def get_app(transport: str = "sse", enabled_apps: list[str] | None = None):
|
||||
"""Readiness probe endpoint.
|
||||
|
||||
Returns 200 OK if the application is ready to serve traffic.
|
||||
Checks that required configuration is present.
|
||||
Checks that required configuration is present and Qdrant if vector sync enabled.
|
||||
"""
|
||||
checks = {}
|
||||
is_ready = True
|
||||
@@ -976,6 +1049,24 @@ def get_app(transport: str = "sse", enabled_apps: list[str] | None = None):
|
||||
checks["auth_configured"] = "error: credentials not set"
|
||||
is_ready = False
|
||||
|
||||
# Check Qdrant status if vector sync is enabled
|
||||
vector_sync_enabled = (
|
||||
os.getenv("VECTOR_SYNC_ENABLED", "false").lower() == "true"
|
||||
)
|
||||
if vector_sync_enabled:
|
||||
try:
|
||||
qdrant_url = os.getenv("QDRANT_URL", "http://qdrant:6333")
|
||||
async with httpx.AsyncClient(timeout=2.0) as client:
|
||||
response = await client.get(f"{qdrant_url}/readyz")
|
||||
if response.status_code == 200:
|
||||
checks["qdrant"] = "ok"
|
||||
else:
|
||||
checks["qdrant"] = f"error: status {response.status_code}"
|
||||
is_ready = False
|
||||
except Exception as e:
|
||||
checks["qdrant"] = f"error: {str(e)}"
|
||||
is_ready = False
|
||||
|
||||
status_code = 200 if is_ready else 503
|
||||
return JSONResponse(
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user