refactor: simplify OpenTelemetry tracing configuration
Simplifies the OpenTelemetry tracing setup by removing the redundant OTEL_ENABLED flag and using the presence of OTEL_EXPORTER_OTLP_ENDPOINT to determine if tracing should be enabled. This follows the standard OpenTelemetry environment variable conventions more closely. Changes: - Remove OTEL_ENABLED/tracing_enabled flag in favor of checking if OTEL_EXPORTER_OTLP_ENDPOINT is set - Add OTEL_EXPORTER_VERIFY_SSL configuration option for OTLP endpoints with self-signed certificates (defaults to false for development) - Move HTTPXClientInstrumentor initialization to module level to ensure httpx calls are traced across all Nextcloud API requests - Add tracing spans to vector sync operations (scan_user_documents) - Fix authorization header logging to only warn about missing headers in OAuth mode (BasicAuth mode doesn't use Authorization headers) - Update observability documentation to reflect simplified configuration - Refactor Dockerfile to use --no-editable flag for uv sync Breaking changes: - OTEL_ENABLED environment variable is removed - Tracing is now automatically enabled when OTEL_EXPORTER_OTLP_ENDPOINT is set Migration guide: - Remove OTEL_ENABLED=true from environment configuration - Tracing will be enabled automatically if OTEL_EXPORTER_OTLP_ENDPOINT is configured 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -5,5 +5,7 @@ __pycache__/
|
|||||||
.env.local
|
.env.local
|
||||||
.env.*.local
|
.env.*.local
|
||||||
|
|
||||||
|
docker-compose.override.yml
|
||||||
|
|
||||||
# Generated by pytest used to login users
|
# Generated by pytest used to login users
|
||||||
.nextcloud_oauth_*.json
|
.nextcloud_oauth_*.json
|
||||||
|
|||||||
+1
-1
@@ -9,7 +9,7 @@ WORKDIR /app
|
|||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
RUN uv sync --locked --no-dev
|
RUN uv sync --locked --no-dev --no-editable
|
||||||
|
|
||||||
ENV PYTHONUNBUFFERED=1
|
ENV PYTHONUNBUFFERED=1
|
||||||
ENV VIRTUAL_ENV=/app/.venv
|
ENV VIRTUAL_ENV=/app/.venv
|
||||||
|
|||||||
@@ -218,8 +218,6 @@ spec:
|
|||||||
- name: METRICS_PORT
|
- name: METRICS_PORT
|
||||||
value: {{ .Values.observability.metrics.port | quote }}
|
value: {{ .Values.observability.metrics.port | quote }}
|
||||||
{{- if .Values.observability.tracing.enabled }}
|
{{- if .Values.observability.tracing.enabled }}
|
||||||
- name: OTEL_ENABLED
|
|
||||||
value: "true"
|
|
||||||
- name: OTEL_EXPORTER_OTLP_ENDPOINT
|
- name: OTEL_EXPORTER_OTLP_ENDPOINT
|
||||||
value: {{ .Values.observability.tracing.endpoint | quote }}
|
value: {{ .Values.observability.tracing.endpoint | quote }}
|
||||||
- name: OTEL_SERVICE_NAME
|
- name: OTEL_SERVICE_NAME
|
||||||
|
|||||||
+6
-2
@@ -98,16 +98,20 @@ services:
|
|||||||
#- QDRANT_URL=http://qdrant:6333 # Uncomment for network mode
|
#- QDRANT_URL=http://qdrant:6333 # Uncomment for network mode
|
||||||
#- QDRANT_API_KEY=${QDRANT_API_KEY:-my_secret_api_key} # Only for network mode
|
#- QDRANT_API_KEY=${QDRANT_API_KEY:-my_secret_api_key} # Only for network mode
|
||||||
|
|
||||||
|
# Observability
|
||||||
|
#- OTEL_SERVICE_NAME=nextcloud-mcp-docker-compose
|
||||||
|
#- OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
|
||||||
|
|
||||||
# Collection naming: Auto-generated as {deployment-id}-{model-name}
|
# Collection naming: Auto-generated as {deployment-id}-{model-name}
|
||||||
# - Deployment ID: OTEL_SERVICE_NAME (if set) or hostname (fallback)
|
# - Deployment ID: OTEL_SERVICE_NAME (if set) or hostname (fallback)
|
||||||
# - Model name: OLLAMA_EMBEDDING_MODEL
|
# - Model name: OLLAMA_EMBEDDING_MODEL
|
||||||
# - Example: "nextcloud-mcp-server-nomic-embed-text"
|
# - Example: "nextcloud-mcp-server-nomic-embed-text"
|
||||||
# - Changing models creates new collection (requires re-embedding)
|
# - Changing models creates new collection (requires re-embedding)
|
||||||
# - Set QDRANT_COLLECTION to override auto-generation:
|
# - Set QDRANT_COLLECTION to override auto-generation:
|
||||||
- QDRANT_COLLECTION=nextcloud_content
|
#- QDRANT_COLLECTION=nextcloud_content
|
||||||
|
|
||||||
# Ollama configuration (optional - uses SimpleEmbeddingProvider if not set)
|
# Ollama configuration (optional - uses SimpleEmbeddingProvider if not set)
|
||||||
# - OLLAMA_BASE_URL=https://ollama.internal.coutinho.io:443
|
# - OLLAMA_BASE_URL=http://ollama:11434
|
||||||
# - OLLAMA_EMBEDDING_MODEL=nomic-embed-text # Changing this creates new collection
|
# - OLLAMA_EMBEDDING_MODEL=nomic-embed-text # Changing this creates new collection
|
||||||
# - OLLAMA_VERIFY_SSL=false
|
# - OLLAMA_VERIFY_SSL=false
|
||||||
|
|
||||||
|
|||||||
@@ -16,8 +16,7 @@ The Nextcloud MCP Server includes comprehensive observability features for produ
|
|||||||
export METRICS_ENABLED=true
|
export METRICS_ENABLED=true
|
||||||
export METRICS_PORT=9090
|
export METRICS_PORT=9090
|
||||||
|
|
||||||
# Enable tracing (optional)
|
# Enable tracing (optional - tracing is enabled when OTEL_EXPORTER_OTLP_ENDPOINT is set)
|
||||||
export OTEL_ENABLED=true
|
|
||||||
export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
|
export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
|
||||||
|
|
||||||
# Start the server
|
# Start the server
|
||||||
@@ -46,8 +45,7 @@ helm install nextcloud-mcp charts/nextcloud-mcp-server \
|
|||||||
|----------|---------|-------------|
|
|----------|---------|-------------|
|
||||||
| `METRICS_ENABLED` | `true` | Enable Prometheus metrics |
|
| `METRICS_ENABLED` | `true` | Enable Prometheus metrics |
|
||||||
| `METRICS_PORT` | `9090` | Port for metrics endpoint |
|
| `METRICS_PORT` | `9090` | Port for metrics endpoint |
|
||||||
| `OTEL_ENABLED` | `false` | Enable OpenTelemetry tracing |
|
| `OTEL_EXPORTER_OTLP_ENDPOINT` | - | OTLP gRPC endpoint (e.g., `http://otel-collector:4317`). Tracing is enabled when this is set. |
|
||||||
| `OTEL_EXPORTER_OTLP_ENDPOINT` | - | OTLP gRPC endpoint (e.g., `http://otel-collector:4317`) |
|
|
||||||
| `OTEL_SERVICE_NAME` | `nextcloud-mcp-server` | Service name in traces |
|
| `OTEL_SERVICE_NAME` | `nextcloud-mcp-server` | Service name in traces |
|
||||||
| `OTEL_TRACES_SAMPLER` | `always_on` | Trace sampling strategy |
|
| `OTEL_TRACES_SAMPLER` | `always_on` | Trace sampling strategy |
|
||||||
| `OTEL_TRACES_SAMPLER_ARG` | `1.0` | Sampling rate (0.0-1.0) |
|
| `OTEL_TRACES_SAMPLER_ARG` | `1.0` | Sampling rate (0.0-1.0) |
|
||||||
|
|||||||
@@ -5,9 +5,12 @@ from contextlib import AsyncExitStack, asynccontextmanager
|
|||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import TYPE_CHECKING, Optional
|
from typing import TYPE_CHECKING, Optional
|
||||||
|
|
||||||
|
from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from nextcloud_mcp_server.auth.refresh_token_storage import RefreshTokenStorage
|
from nextcloud_mcp_server.auth.refresh_token_storage import RefreshTokenStorage
|
||||||
|
|
||||||
|
|
||||||
import anyio
|
import anyio
|
||||||
import click
|
import click
|
||||||
import httpx
|
import httpx
|
||||||
@@ -58,6 +61,7 @@ from nextcloud_mcp_server.server.oauth_tools import register_oauth_tools
|
|||||||
from nextcloud_mcp_server.vector import processor_task, scanner_task
|
from nextcloud_mcp_server.vector import processor_task, scanner_task
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
HTTPXClientInstrumentor().instrument()
|
||||||
|
|
||||||
|
|
||||||
def initialize_document_processors():
|
def initialize_document_processors():
|
||||||
@@ -791,17 +795,20 @@ def get_app(transport: str = "sse", enabled_apps: list[str] | None = None):
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Setup OpenTelemetry tracing (optional)
|
# Setup OpenTelemetry tracing (optional)
|
||||||
if settings.tracing_enabled:
|
if settings.otel_exporter_otlp_endpoint:
|
||||||
setup_tracing(
|
setup_tracing(
|
||||||
service_name=settings.otel_service_name,
|
service_name=settings.otel_service_name,
|
||||||
otlp_endpoint=settings.otel_exporter_otlp_endpoint,
|
otlp_endpoint=settings.otel_exporter_otlp_endpoint,
|
||||||
|
otlp_verify_ssl=settings.otel_exporter_verify_ssl,
|
||||||
sampling_rate=settings.otel_traces_sampler_arg,
|
sampling_rate=settings.otel_traces_sampler_arg,
|
||||||
)
|
)
|
||||||
logger.info(
|
logger.info(
|
||||||
f"OpenTelemetry tracing enabled (endpoint: {settings.otel_exporter_otlp_endpoint})"
|
f"OpenTelemetry tracing enabled (endpoint: {settings.otel_exporter_otlp_endpoint})"
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
logger.info("OpenTelemetry tracing disabled (set OTEL_ENABLED=true to enable)")
|
logger.info(
|
||||||
|
"OpenTelemetry tracing disabled (set OTEL_EXPORTER_OTLP_ENDPOINT to enable)"
|
||||||
|
)
|
||||||
|
|
||||||
# Determine authentication mode
|
# Determine authentication mode
|
||||||
oauth_enabled = is_oauth_mode()
|
oauth_enabled = is_oauth_mode()
|
||||||
@@ -1391,6 +1398,9 @@ def get_app(transport: str = "sse", enabled_apps: list[str] | None = None):
|
|||||||
)
|
)
|
||||||
logger.info(f"🔑 /mcp request with Authorization: {token_preview}")
|
logger.info(f"🔑 /mcp request with Authorization: {token_preview}")
|
||||||
else:
|
else:
|
||||||
|
# Only warn about missing Authorization in OAuth mode
|
||||||
|
# In BasicAuth mode, /mcp requests without Authorization are expected
|
||||||
|
if oauth_enabled:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
f"⚠️ /mcp request WITHOUT Authorization header from {request.client}"
|
f"⚠️ /mcp request WITHOUT Authorization header from {request.client}"
|
||||||
)
|
)
|
||||||
@@ -1454,7 +1464,7 @@ def get_app(transport: str = "sse", enabled_apps: list[str] | None = None):
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Add observability middleware (metrics + tracing)
|
# Add observability middleware (metrics + tracing)
|
||||||
if settings.metrics_enabled or settings.tracing_enabled:
|
if settings.metrics_enabled or settings.otel_exporter_otlp_endpoint:
|
||||||
app.add_middleware(ObservabilityMiddleware)
|
app.add_middleware(ObservabilityMiddleware)
|
||||||
logger.info("Observability middleware enabled (metrics and/or tracing)")
|
logger.info("Observability middleware enabled (metrics and/or tracing)")
|
||||||
|
|
||||||
|
|||||||
@@ -231,17 +231,21 @@ class UnifiedTokenVerifier(TokenVerifier):
|
|||||||
token,
|
token,
|
||||||
signing_key.key,
|
signing_key.key,
|
||||||
algorithms=["RS256"],
|
algorithms=["RS256"],
|
||||||
issuer=self.settings.oidc_issuer
|
issuer=(
|
||||||
|
self.settings.oidc_issuer
|
||||||
if hasattr(self.settings, "oidc_issuer")
|
if hasattr(self.settings, "oidc_issuer")
|
||||||
else None,
|
else None
|
||||||
|
),
|
||||||
options={
|
options={
|
||||||
"verify_signature": True,
|
"verify_signature": True,
|
||||||
"verify_exp": True,
|
"verify_exp": True,
|
||||||
"verify_iat": True,
|
"verify_iat": True,
|
||||||
"verify_iss": True
|
"verify_iss": (
|
||||||
|
True
|
||||||
if hasattr(self.settings, "oidc_issuer")
|
if hasattr(self.settings, "oidc_issuer")
|
||||||
and self.settings.oidc_issuer
|
and self.settings.oidc_issuer
|
||||||
else False,
|
else False
|
||||||
|
),
|
||||||
"verify_aud": False, # We handle audience validation separately
|
"verify_aud": False, # We handle audience validation separately
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ from httpx import (
|
|||||||
BasicAuth,
|
BasicAuth,
|
||||||
Request,
|
Request,
|
||||||
Response,
|
Response,
|
||||||
|
Timeout,
|
||||||
)
|
)
|
||||||
|
|
||||||
from ..controllers.notes_search import NotesSearchController
|
from ..controllers.notes_search import NotesSearchController
|
||||||
@@ -66,6 +67,7 @@ class NextcloudClient:
|
|||||||
auth=auth,
|
auth=auth,
|
||||||
transport=AsyncDisableCookieTransport(AsyncHTTPTransport()),
|
transport=AsyncDisableCookieTransport(AsyncHTTPTransport()),
|
||||||
event_hooks={"request": [log_request], "response": [log_response]},
|
event_hooks={"request": [log_request], "response": [log_response]},
|
||||||
|
timeout=Timeout(timeout=30, connect=5),
|
||||||
)
|
)
|
||||||
|
|
||||||
# Initialize app clients
|
# Initialize app clients
|
||||||
|
|||||||
@@ -181,8 +181,8 @@ class Settings:
|
|||||||
# Observability settings
|
# Observability settings
|
||||||
metrics_enabled: bool = True
|
metrics_enabled: bool = True
|
||||||
metrics_port: int = 9090
|
metrics_port: int = 9090
|
||||||
tracing_enabled: bool = False
|
|
||||||
otel_exporter_otlp_endpoint: Optional[str] = None
|
otel_exporter_otlp_endpoint: Optional[str] = None
|
||||||
|
otel_exporter_verify_ssl: bool = False
|
||||||
otel_service_name: str = "nextcloud-mcp-server"
|
otel_service_name: str = "nextcloud-mcp-server"
|
||||||
otel_traces_sampler: str = "always_on"
|
otel_traces_sampler: str = "always_on"
|
||||||
otel_traces_sampler_arg: float = 1.0
|
otel_traces_sampler_arg: float = 1.0
|
||||||
@@ -334,8 +334,9 @@ def get_settings() -> Settings:
|
|||||||
# Observability settings
|
# Observability settings
|
||||||
metrics_enabled=os.getenv("METRICS_ENABLED", "true").lower() == "true",
|
metrics_enabled=os.getenv("METRICS_ENABLED", "true").lower() == "true",
|
||||||
metrics_port=int(os.getenv("METRICS_PORT", "9090")),
|
metrics_port=int(os.getenv("METRICS_PORT", "9090")),
|
||||||
tracing_enabled=os.getenv("OTEL_ENABLED", "false").lower() == "true",
|
|
||||||
otel_exporter_otlp_endpoint=os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT"),
|
otel_exporter_otlp_endpoint=os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT"),
|
||||||
|
otel_exporter_verify_ssl=os.getenv("OTEL_EXPORTER_VERIFY_SSL", "false").lower()
|
||||||
|
== "true",
|
||||||
otel_service_name=os.getenv("OTEL_SERVICE_NAME", "nextcloud-mcp-server"),
|
otel_service_name=os.getenv("OTEL_SERVICE_NAME", "nextcloud-mcp-server"),
|
||||||
otel_traces_sampler=os.getenv("OTEL_TRACES_SAMPLER", "always_on"),
|
otel_traces_sampler=os.getenv("OTEL_TRACES_SAMPLER", "always_on"),
|
||||||
otel_traces_sampler_arg=float(os.getenv("OTEL_TRACES_SAMPLER_ARG", "1.0")),
|
otel_traces_sampler_arg=float(os.getenv("OTEL_TRACES_SAMPLER_ARG", "1.0")),
|
||||||
|
|||||||
@@ -13,9 +13,9 @@ import logging
|
|||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
from importlib_metadata import version
|
||||||
from opentelemetry import trace
|
from opentelemetry import trace
|
||||||
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
|
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
|
||||||
from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
|
|
||||||
from opentelemetry.instrumentation.logging import LoggingInstrumentor
|
from opentelemetry.instrumentation.logging import LoggingInstrumentor
|
||||||
from opentelemetry.sdk.resources import Resource
|
from opentelemetry.sdk.resources import Resource
|
||||||
from opentelemetry.sdk.trace import TracerProvider
|
from opentelemetry.sdk.trace import TracerProvider
|
||||||
@@ -27,10 +27,13 @@ logger = logging.getLogger(__name__)
|
|||||||
# Global tracer instance (initialized in setup_tracing)
|
# Global tracer instance (initialized in setup_tracing)
|
||||||
_tracer: Tracer | None = None
|
_tracer: Tracer | None = None
|
||||||
|
|
||||||
|
# Auto-instrument httpx for Nextcloud API calls
|
||||||
|
|
||||||
|
|
||||||
def setup_tracing(
|
def setup_tracing(
|
||||||
service_name: str = "nextcloud-mcp-server",
|
service_name: str = "nextcloud-mcp-server",
|
||||||
otlp_endpoint: str | None = None,
|
otlp_endpoint: str | None = None,
|
||||||
|
otlp_verify_ssl: bool = False,
|
||||||
sampling_rate: float = 1.0,
|
sampling_rate: float = 1.0,
|
||||||
) -> Tracer:
|
) -> Tracer:
|
||||||
"""
|
"""
|
||||||
@@ -40,6 +43,8 @@ def setup_tracing(
|
|||||||
service_name: Service name for traces (default: "nextcloud-mcp-server")
|
service_name: Service name for traces (default: "nextcloud-mcp-server")
|
||||||
otlp_endpoint: OTLP gRPC endpoint (e.g., "http://otel-collector:4317")
|
otlp_endpoint: OTLP gRPC endpoint (e.g., "http://otel-collector:4317")
|
||||||
If None, tracing is initialized but no exporter is configured
|
If None, tracing is initialized but no exporter is configured
|
||||||
|
otlp_verify_ssl: Enable TLS verification for otlp_endpoint. If True,
|
||||||
|
`insecure` will eval to False
|
||||||
sampling_rate: Sampling rate (0.0-1.0). Default 1.0 (100% sampling)
|
sampling_rate: Sampling rate (0.0-1.0). Default 1.0 (100% sampling)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
@@ -51,7 +56,7 @@ def setup_tracing(
|
|||||||
resource = Resource.create(
|
resource = Resource.create(
|
||||||
{
|
{
|
||||||
"service.name": service_name,
|
"service.name": service_name,
|
||||||
"service.version": "0.27.2", # TODO: Extract from pyproject.toml
|
"service.version": version(__package__.split(".")[0]),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -61,7 +66,9 @@ def setup_tracing(
|
|||||||
# Configure OTLP exporter if endpoint is provided
|
# Configure OTLP exporter if endpoint is provided
|
||||||
if otlp_endpoint:
|
if otlp_endpoint:
|
||||||
try:
|
try:
|
||||||
otlp_exporter = OTLPSpanExporter(endpoint=otlp_endpoint, insecure=True)
|
otlp_exporter = OTLPSpanExporter(
|
||||||
|
endpoint=otlp_endpoint, insecure=not otlp_verify_ssl
|
||||||
|
)
|
||||||
span_processor = BatchSpanProcessor(otlp_exporter)
|
span_processor = BatchSpanProcessor(otlp_exporter)
|
||||||
provider.add_span_processor(span_processor)
|
provider.add_span_processor(span_processor)
|
||||||
logger.info(
|
logger.info(
|
||||||
@@ -79,9 +86,6 @@ def setup_tracing(
|
|||||||
# Set global tracer provider
|
# Set global tracer provider
|
||||||
trace.set_tracer_provider(provider)
|
trace.set_tracer_provider(provider)
|
||||||
|
|
||||||
# Auto-instrument httpx for Nextcloud API calls
|
|
||||||
HTTPXClientInstrumentor().instrument()
|
|
||||||
|
|
||||||
# Auto-instrument logging to inject trace context
|
# Auto-instrument logging to inject trace context
|
||||||
LoggingInstrumentor().instrument(set_logging_format=True)
|
LoggingInstrumentor().instrument(set_logging_format=True)
|
||||||
|
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ from qdrant_client.models import FieldCondition, Filter, MatchValue, PointStruct
|
|||||||
from nextcloud_mcp_server.client import NextcloudClient
|
from nextcloud_mcp_server.client import NextcloudClient
|
||||||
from nextcloud_mcp_server.config import get_settings
|
from nextcloud_mcp_server.config import get_settings
|
||||||
from nextcloud_mcp_server.embedding import get_embedding_service
|
from nextcloud_mcp_server.embedding import get_embedding_service
|
||||||
|
from nextcloud_mcp_server.observability.tracing import trace_operation
|
||||||
from nextcloud_mcp_server.vector.document_chunker import DocumentChunker
|
from nextcloud_mcp_server.vector.document_chunker import DocumentChunker
|
||||||
from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client
|
from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client
|
||||||
from nextcloud_mcp_server.vector.scanner import DocumentTask
|
from nextcloud_mcp_server.vector.scanner import DocumentTask
|
||||||
@@ -94,6 +95,16 @@ async def process_document(doc_task: DocumentTask, nc_client: NextcloudClient):
|
|||||||
f"for {doc_task.user_id} ({doc_task.operation})"
|
f"for {doc_task.user_id} ({doc_task.operation})"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
with trace_operation(
|
||||||
|
"vector_sync.process_document",
|
||||||
|
attributes={
|
||||||
|
"vector_sync.operation": "process",
|
||||||
|
"vector_sync.user_id": doc_task.user_id,
|
||||||
|
"vector_sync.doc_id": doc_task.doc_id,
|
||||||
|
"vector_sync.doc_type": doc_task.doc_type,
|
||||||
|
"vector_sync.doc_operation": doc_task.operation,
|
||||||
|
},
|
||||||
|
):
|
||||||
qdrant_client = await get_qdrant_client()
|
qdrant_client = await get_qdrant_client()
|
||||||
settings = get_settings()
|
settings = get_settings()
|
||||||
|
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ from qdrant_client.models import FieldCondition, Filter, MatchValue
|
|||||||
|
|
||||||
from nextcloud_mcp_server.client import NextcloudClient
|
from nextcloud_mcp_server.client import NextcloudClient
|
||||||
from nextcloud_mcp_server.config import get_settings
|
from nextcloud_mcp_server.config import get_settings
|
||||||
|
from nextcloud_mcp_server.observability.tracing import trace_operation
|
||||||
from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client
|
from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -154,9 +155,20 @@ async def scan_user_documents(
|
|||||||
f"[SCAN-{scan_id}] Starting scan for user: {user_id}, initial_sync={initial_sync}"
|
f"[SCAN-{scan_id}] Starting scan for user: {user_id}, initial_sync={initial_sync}"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
with trace_operation(
|
||||||
|
"vector_sync.scan_user_documents",
|
||||||
|
attributes={
|
||||||
|
"vector_sync.operation": "scan",
|
||||||
|
"vector_sync.user_id": user_id,
|
||||||
|
"vector_sync.initial_sync": initial_sync,
|
||||||
|
"vector_sync.scan_id": scan_id,
|
||||||
|
},
|
||||||
|
):
|
||||||
# Calculate prune timestamp for optimized data transfer
|
# Calculate prune timestamp for optimized data transfer
|
||||||
# Only notes modified after this will be sent with full data
|
# Only notes modified after this will be sent with full data
|
||||||
prune_before = None if initial_sync else await get_last_indexed_timestamp(user_id)
|
prune_before = (
|
||||||
|
None if initial_sync else await get_last_indexed_timestamp(user_id)
|
||||||
|
)
|
||||||
if prune_before:
|
if prune_before:
|
||||||
logger.info(
|
logger.info(
|
||||||
f"[SCAN-{scan_id}] Using pruneBefore={prune_before} to optimize data transfer"
|
f"[SCAN-{scan_id}] Using pruneBefore={prune_before} to optimize data transfer"
|
||||||
@@ -164,7 +176,8 @@ async def scan_user_documents(
|
|||||||
|
|
||||||
# Fetch all notes from Nextcloud
|
# Fetch all notes from Nextcloud
|
||||||
notes = [
|
notes = [
|
||||||
note async for note in nc_client.notes.get_all_notes(prune_before=prune_before)
|
note
|
||||||
|
async for note in nc_client.notes.get_all_notes(prune_before=prune_before)
|
||||||
]
|
]
|
||||||
logger.info(f"[SCAN-{scan_id}] Found {len(notes)} notes for {user_id}")
|
logger.info(f"[SCAN-{scan_id}] Found {len(notes)} notes for {user_id}")
|
||||||
|
|
||||||
@@ -239,7 +252,9 @@ async def scan_user_documents(
|
|||||||
# Check for deleted documents (in Qdrant but not in Nextcloud)
|
# Check for deleted documents (in Qdrant but not in Nextcloud)
|
||||||
# Use grace period: only delete after 2 consecutive scans confirm absence
|
# Use grace period: only delete after 2 consecutive scans confirm absence
|
||||||
settings = get_settings()
|
settings = get_settings()
|
||||||
grace_period = settings.vector_sync_scan_interval * 1.5 # Allow 1.5 scan intervals
|
grace_period = (
|
||||||
|
settings.vector_sync_scan_interval * 1.5
|
||||||
|
) # Allow 1.5 scan intervals
|
||||||
current_time = time.time()
|
current_time = time.time()
|
||||||
|
|
||||||
for doc_id in indexed_docs:
|
for doc_id in indexed_docs:
|
||||||
|
|||||||
Reference in New Issue
Block a user