diff --git a/.claude/hooks/sonar-secrets/build-scripts/pretool-secrets.sh b/.claude/hooks/sonar-secrets/build-scripts/pretool-secrets.sh new file mode 100755 index 00000000..24f537c2 --- /dev/null +++ b/.claude/hooks/sonar-secrets/build-scripts/pretool-secrets.sh @@ -0,0 +1,5 @@ +#!/bin/bash +if ! command -v sonar &> /dev/null; then + exit 0 +fi +sonar hook claude-pre-tool-use diff --git a/.claude/hooks/sonar-secrets/build-scripts/prompt-secrets.sh b/.claude/hooks/sonar-secrets/build-scripts/prompt-secrets.sh new file mode 100755 index 00000000..2bb89ce2 --- /dev/null +++ b/.claude/hooks/sonar-secrets/build-scripts/prompt-secrets.sh @@ -0,0 +1,5 @@ +#!/bin/bash +if ! command -v sonar &> /dev/null; then + exit 0 +fi +sonar hook claude-prompt-submit diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 00000000..81d3bbac --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,28 @@ +{ + "hooks": { + "PreToolUse": [ + { + "matcher": "Read", + "hooks": [ + { + "type": "command", + "command": ".claude/hooks/sonar-secrets/build-scripts/pretool-secrets.sh", + "timeout": 60 + } + ] + } + ], + "UserPromptSubmit": [ + { + "matcher": "*", + "hooks": [ + { + "type": "command", + "command": ".claude/hooks/sonar-secrets/build-scripts/prompt-secrets.sh", + "timeout": 60 + } + ] + } + ] + } +} \ No newline at end of file diff --git a/.claude/skills/pre-push-review/SKILL.md b/.claude/skills/pre-push-review/SKILL.md index 26883d49..efc906b0 100644 --- a/.claude/skills/pre-push-review/SKILL.md +++ b/.claude/skills/pre-push-review/SKILL.md @@ -7,6 +7,7 @@ description: | in this repo's automated PR reviews. Use when the user is about to push, says "ready to push", "review my work", "check before PR", or invokes /pre-push-review. Report-only — does not modify code. +model: sonnet allowed-tools: - Bash - Read diff --git a/CHANGELOG.md b/CHANGELOG.md index cc5079ad..9380c671 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,30 @@ All notable changes to the Nextcloud MCP Server will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://peps.python.org/pep-0440/). +## v0.83.0 (2026-05-08) + +### Feat + +- **vector**: replace inline page-image payloads with chunk_bbox (Deck #76) + +### Refactor + +- **vector**: address PR #775 review round 3 — fix unused var, harden boundary lookup, rename trace span +- **vector**: address PR #775 review round 2 — drop dead page field, add omission tests +- **vector**: address PR #775 review — drop unused payload key, fix resource leaks + +## v0.82.0 (2026-05-08) + +### Feat + +- **providers**: add Mistral embedding provider, route registry through dynaconf + +### Refactor + +- **providers**: address PR #772 review round 3 — hermetic test, lazy logging, defensive-guard tests +- **providers**: address PR #772 review round 2 — guard, naming, docs, tests +- **providers**: address PR #772 review — shared retry, cleaner imports, no-op close + ## v0.81.0 (2026-05-07) ### Feat diff --git a/docker-compose.yml b/docker-compose.yml index 6e8a1f8c..208ce05e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -334,7 +334,7 @@ services: - claude-funnel qdrant: - image: docker.io/qdrant/qdrant:v1.17.1@sha256:94728574965d17c6485dd361aa3c0818b325b9016dac5ea6afec7b4b2700865f + image: docker.io/qdrant/qdrant:v1.18.0@sha256:1cf3e07c9f269030c7cfed0d5c0beea7bb081848a88e2ae13b35a613d4dd5019 restart: always ports: - 127.0.0.1:6333:6333 # REST API diff --git a/docs/ADR-015-unified-provider-architecture.md b/docs/ADR-015-unified-provider-architecture.md index 8922c307..b81f73b4 100644 --- a/docs/ADR-015-unified-provider-architecture.md +++ b/docs/ADR-015-unified-provider-architecture.md @@ -118,10 +118,16 @@ class ProviderRegistry: @staticmethod def create_provider() -> Provider: # 1. Bedrock (AWS_REGION or BEDROCK_*_MODEL) - # 2. Ollama (OLLAMA_BASE_URL) - # 3. Simple (fallback) + # 2. OpenAI (OPENAI_API_KEY) + # 3. Mistral (MISTRAL_API_KEY) + # 4. Ollama (OLLAMA_BASE_URL) + # 5. Simple (fallback) ``` +Configuration is sourced via the dynaconf-backed `Settings` dataclass in +`config.py`; the registry reads `get_settings()` rather than `os.getenv` +directly, so settings files and env vars share one resolution path. + **Environment Variables:** **Bedrock:** @@ -131,6 +137,17 @@ class ProviderRegistry: - `BEDROCK_EMBEDDING_MODEL`: Model ID for embeddings (e.g., "amazon.titan-embed-text-v2:0") - `BEDROCK_GENERATION_MODEL`: Model ID for text generation (e.g., "anthropic.claude-3-sonnet-20240229-v1:0") +**OpenAI:** +- `OPENAI_API_KEY`: OpenAI API key (or `GITHUB_TOKEN` for GitHub Models) +- `OPENAI_BASE_URL`: Optional base URL override for OpenAI-compatible APIs +- `OPENAI_EMBEDDING_MODEL`: Embedding model (default: "text-embedding-3-small") +- `OPENAI_GENERATION_MODEL`: Generation model (e.g., "gpt-4o-mini") + +**Mistral (embeddings only):** +- `MISTRAL_API_KEY`: Mistral API key from console.mistral.ai +- `MISTRAL_EMBEDDING_MODEL`: Embedding model (default: "mistral-embed", 1024-dim) +- `MISTRAL_BASE_URL`: Optional server URL override (proxies, on-prem) + **Ollama:** - `OLLAMA_BASE_URL`: Ollama API base URL (e.g., "http://localhost:11434") - `OLLAMA_EMBEDDING_MODEL`: Model for embeddings (default: "nomic-embed-text") diff --git a/docs/configuration.md b/docs/configuration.md index f6f5d8e0..dd71e300 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -410,9 +410,16 @@ DOCUMENT_CHUNK_OVERLAP=50 # Overlapping words between chunks (defaul ### Embedding Service Configuration -The server uses an embedding service to generate vector representations. Two options are available: +The server picks an embedding provider via auto-detection. Priority order +(see `nextcloud_mcp_server/providers/registry.py`): -#### Ollama (Recommended) +1. **Bedrock** — if `AWS_REGION` or `BEDROCK_EMBEDDING_MODEL` is set +2. **OpenAI** — if `OPENAI_API_KEY` is set +3. **Mistral** — if `MISTRAL_API_KEY` is set +4. **Ollama** — if `OLLAMA_BASE_URL` is set +5. **Simple** — fallback when nothing else is configured + +#### Ollama (Recommended for self-hosted) Use a local Ollama instance for embeddings: @@ -422,9 +429,52 @@ OLLAMA_EMBEDDING_MODEL=nomic-embed-text # Default model OLLAMA_VERIFY_SSL=true # Verify SSL certificates ``` +#### OpenAI + +Hosted OpenAI embeddings (or any OpenAI-compatible API via `OPENAI_BASE_URL`): + +```dotenv +OPENAI_API_KEY=sk-... +OPENAI_EMBEDDING_MODEL=text-embedding-3-small # default +# OPENAI_BASE_URL=https://models.github.ai/inference # optional +``` + +#### Mistral + +Hosted Mistral embeddings. Requires a Mistral API key from +[console.mistral.ai](https://console.mistral.ai). Currently embeddings only +(no text generation). + +```dotenv +MISTRAL_API_KEY=... +MISTRAL_EMBEDDING_MODEL=mistral-embed # default; produces 1024-dim vectors +# MISTRAL_BASE_URL=https://api.mistral.ai # optional override (proxies, on-prem) +``` + +Switching to or from Mistral forces a new Qdrant collection because the +collection name encodes the model (see "Qdrant Collection Naming" above). + +#### Amazon Bedrock + +Bedrock provides hosted embedding models (Titan, Cohere) and uses the AWS +credential chain (env vars, profiles, or IAM role): + +```dotenv +AWS_REGION=us-east-1 +BEDROCK_EMBEDDING_MODEL=amazon.titan-embed-text-v2:0 +# AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY are optional — boto3 will use +# the standard credential chain if not set. +``` + #### Simple Embedding Provider (Fallback) -If `OLLAMA_BASE_URL` is not set, the server uses a simple random embedding provider for testing. This is **not suitable for production** as it generates random embeddings with no semantic meaning. +If no provider env var is set, the server falls back to a simple deterministic +embedding provider for testing. This is **not suitable for production** as +its embeddings have no semantic meaning. + +```dotenv +SIMPLE_EMBEDDING_DIMENSION=384 # optional; default 384 +``` ### Document Chunking Configuration @@ -533,7 +583,21 @@ equivalent.** Operators who need a runtime toggle should open an issue. | `VECTOR_SYNC_QUEUE_MAX_SIZE` | ⚠️ Optional | `10000` | Max queued documents | | `OLLAMA_BASE_URL` | ⚠️ Optional | - | Ollama API endpoint for embeddings | | `OLLAMA_EMBEDDING_MODEL` | ⚠️ Optional | `nomic-embed-text` | Embedding model to use | +| `OLLAMA_GENERATION_MODEL` | ⚠️ Optional | - | Ollama model for text generation | | `OLLAMA_VERIFY_SSL` | ⚠️ Optional | `true` | Verify SSL certificates | +| `OPENAI_API_KEY` | ⚠️ Optional | - | OpenAI API key (selects OpenAI provider) | +| `OPENAI_BASE_URL` | ⚠️ Optional | - | OpenAI base URL override (for compatible APIs) | +| `OPENAI_EMBEDDING_MODEL` | ⚠️ Optional | `text-embedding-3-small` | OpenAI embedding model | +| `OPENAI_GENERATION_MODEL` | ⚠️ Optional | - | OpenAI model for text generation | +| `MISTRAL_API_KEY` | ⚠️ Optional | - | Mistral API key (selects Mistral provider) | +| `MISTRAL_EMBEDDING_MODEL` | ⚠️ Optional | `mistral-embed` | Mistral embedding model (1024-dim) | +| `MISTRAL_BASE_URL` | ⚠️ Optional | - | Mistral base URL override (proxies, on-prem) | +| `AWS_REGION` | ⚠️ Optional | - | AWS region (selects Bedrock provider) | +| `AWS_ACCESS_KEY_ID` | ⚠️ Optional | - | AWS access key (boto3 credential chain fallback) | +| `AWS_SECRET_ACCESS_KEY` | ⚠️ Optional | - | AWS secret key (boto3 credential chain fallback) | +| `BEDROCK_EMBEDDING_MODEL` | ⚠️ Optional | - | Bedrock embedding model ID | +| `BEDROCK_GENERATION_MODEL` | ⚠️ Optional | - | Bedrock generation model ID | +| `SIMPLE_EMBEDDING_DIMENSION` | ⚠️ Optional | `384` | Dimension for the fallback Simple provider | | `DOCUMENT_CHUNK_SIZE` | ⚠️ Optional | `512` | Words per chunk for document embedding | | `DOCUMENT_CHUNK_OVERLAP` | ⚠️ Optional | `50` | Overlapping words between chunks (must be < chunk size) | diff --git a/nextcloud_mcp_server/api/visualization.py b/nextcloud_mcp_server/api/visualization.py index 0027af66..95199429 100644 --- a/nextcloud_mcp_server/api/visualization.py +++ b/nextcloud_mcp_server/api/visualization.py @@ -565,9 +565,10 @@ async def get_chunk_context(request: Request) -> JSONResponse: status_code=404, ) - # For PDF files, also fetch the highlighted page image from Qdrant if available - # This is useful for clients that want to show a pre-rendered image - highlighted_page_image = None + # For PDF files, also fetch the chunk's bounding box from Qdrant if + # available so the client can overlay a highlight on top of a + # render-on-demand page image (Deck #76). + chunk_bbox = None page_number = chunk_context.page_number if doc_type == "file": @@ -575,7 +576,7 @@ async def get_chunk_context(request: Request) -> JSONResponse: settings = get_settings() qdrant_client = await get_qdrant_client() - # Prefer chunk_index for the highlighted-image lookup (always indexed); + # Prefer chunk_index for the chunk-bbox lookup (always indexed); # fall back to (chunk_start_offset, chunk_end_offset) when not provided. if chunk_index is not None: points_response = await qdrant_client.scroll( @@ -589,10 +590,6 @@ async def get_chunk_context(request: Request) -> JSONResponse: FieldCondition( key="user_id", match=MatchValue(value=user_id) ), - FieldCondition( - key="doc_type", - match=MatchValue(value=doc_type), - ), FieldCondition( key="chunk_index", match=MatchValue(value=chunk_index), @@ -601,7 +598,7 @@ async def get_chunk_context(request: Request) -> JSONResponse: ), limit=1, with_vectors=False, - with_payload=["highlighted_page_image", "page_number"], + with_payload=["chunk_bbox", "page_number"], ) else: points_response = await qdrant_client.scroll( @@ -615,10 +612,6 @@ async def get_chunk_context(request: Request) -> JSONResponse: FieldCondition( key="user_id", match=MatchValue(value=user_id) ), - FieldCondition( - key="doc_type", - match=MatchValue(value=doc_type), - ), FieldCondition( key="chunk_start_offset", match=MatchValue(value=start), @@ -631,19 +624,19 @@ async def get_chunk_context(request: Request) -> JSONResponse: ), limit=1, with_vectors=False, - with_payload=["highlighted_page_image", "page_number"], + with_payload=["chunk_bbox", "page_number"], ) if points_response[0]: payload = points_response[0][0].payload if payload: - highlighted_page_image = payload.get("highlighted_page_image") + chunk_bbox = payload.get("chunk_bbox") # Trust Qdrant page number if available (might be more accurate than context expansion logic) if payload.get("page_number") is not None: page_number = payload.get("page_number") except Exception as e: - logger.warning(f"Failed to fetch highlighted image: {e}") + logger.warning(f"Failed to fetch chunk bbox: {e}") # Build response response_data = { @@ -658,8 +651,8 @@ async def get_chunk_context(request: Request) -> JSONResponse: "total_chunks": chunk_context.total_chunks, } - if highlighted_page_image: - response_data["highlighted_page_image"] = highlighted_page_image + if chunk_bbox: + response_data["chunk_bbox"] = chunk_bbox return JSONResponse(response_data) diff --git a/nextcloud_mcp_server/app.py b/nextcloud_mcp_server/app.py index 3fb9194f..d2c1d1ab 100644 --- a/nextcloud_mcp_server/app.py +++ b/nextcloud_mcp_server/app.py @@ -125,12 +125,13 @@ from nextcloud_mcp_server.server import ( ) from nextcloud_mcp_server.server.auth_tools import register_auth_tools from nextcloud_mcp_server.server.oauth_tools import register_oauth_tools -from nextcloud_mcp_server.vector import processor_task, scanner_task from nextcloud_mcp_server.vector.oauth_sync import ( oauth_processor_task, user_manager_task, ) +from nextcloud_mcp_server.vector.processor import processor_task from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client +from nextcloud_mcp_server.vector.scanner import scanner_task from nextcloud_mcp_server.vector.webhook_receiver import handle_nextcloud_webhook logger = logging.getLogger(__name__) diff --git a/nextcloud_mcp_server/auth/viz_routes.py b/nextcloud_mcp_server/auth/viz_routes.py index cb91e4b0..b3a33886 100644 --- a/nextcloud_mcp_server/auth/viz_routes.py +++ b/nextcloud_mcp_server/auth/viz_routes.py @@ -623,8 +623,10 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse: f"after_len={len(chunk_context.after_context)}" ) - # For PDF files, also fetch the highlighted page image from Qdrant - highlighted_page_image = None + # For PDF files, also fetch the chunk bbox from Qdrant so the client + # can overlay a highlight on top of a render-on-demand page image + # (Deck #76). + chunk_bbox = None page_number = chunk_context.page_number if doc_type == "file": try: @@ -632,7 +634,7 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse: qdrant_client = await get_qdrant_client() username = request.user.display_name - # Prefer chunk_index for the highlighted-image lookup (always indexed); + # Prefer chunk_index for the chunk-bbox lookup (always indexed); # fall back to (chunk_start_offset, chunk_end_offset) when not provided. if chunk_index is not None: points_response = await qdrant_client.scroll( @@ -646,10 +648,6 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse: FieldCondition( key="user_id", match=MatchValue(value=username) ), - FieldCondition( - key="doc_type", - match=MatchValue(value=doc_type), - ), FieldCondition( key="chunk_index", match=MatchValue(value=chunk_index), @@ -658,7 +656,7 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse: ), limit=1, with_vectors=False, - with_payload=["highlighted_page_image", "page_number"], + with_payload=["chunk_bbox", "page_number"], ) else: points_response = await qdrant_client.scroll( @@ -672,10 +670,6 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse: FieldCondition( key="user_id", match=MatchValue(value=username) ), - FieldCondition( - key="doc_type", - match=MatchValue(value=doc_type), - ), FieldCondition( key="chunk_start_offset", match=MatchValue(value=start), @@ -688,22 +682,20 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse: ), limit=1, with_vectors=False, - with_payload=["highlighted_page_image", "page_number"], + with_payload=["chunk_bbox", "page_number"], ) points = points_response[0] if points and points[0].payload: - highlighted_page_image = points[0].payload.get( - "highlighted_page_image" - ) + chunk_bbox = points[0].payload.get("chunk_bbox") page_number = points[0].payload.get("page_number") - if highlighted_page_image: + if chunk_bbox: logger.info( - f"Found highlighted image for chunk: " - f"page={page_number}, image_size={len(highlighted_page_image)}" + f"Found chunk bbox: page={page_number}, " + f"rects={len(chunk_bbox)}" ) except Exception as e: - logger.warning(f"Failed to fetch highlighted image: {e}") + logger.warning(f"Failed to fetch chunk bbox: {e}") # Return response compatible with frontend expectations response_data: dict = { @@ -718,8 +710,8 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse: "total_chunks": chunk_context.total_chunks, } - if highlighted_page_image: - response_data["highlighted_page_image"] = highlighted_page_image + if chunk_bbox: + response_data["chunk_bbox"] = chunk_bbox return JSONResponse(response_data) diff --git a/nextcloud_mcp_server/config.py b/nextcloud_mcp_server/config.py index 60a0a714..3a205110 100644 --- a/nextcloud_mcp_server/config.py +++ b/nextcloud_mcp_server/config.py @@ -77,11 +77,25 @@ _DEFAULTS: dict[str, Any] = { # Ollama "ollama_base_url": None, "ollama_embedding_model": "nomic-embed-text", + "ollama_generation_model": None, "ollama_verify_ssl": True, # OpenAI "openai_api_key": None, "openai_base_url": None, "openai_embedding_model": "text-embedding-3-small", + "openai_generation_model": None, + # Bedrock (AWS) + "aws_region": None, + "aws_access_key_id": None, + "aws_secret_access_key": None, + "bedrock_embedding_model": None, + "bedrock_generation_model": None, + # Mistral + "mistral_api_key": None, + "mistral_embedding_model": "mistral-embed", + "mistral_base_url": None, + # Simple (fallback) embedding dimension + "simple_embedding_dimension": 384, # Document chunking "document_chunk_size": 2048, "document_chunk_overlap": 200, @@ -486,15 +500,32 @@ class Settings: qdrant_api_key: str | None = None qdrant_collection: str = "nextcloud_content" - # Ollama settings (for embeddings) + # Ollama settings (embeddings + optional generation) ollama_base_url: str | None = None ollama_embedding_model: str = "nomic-embed-text" + ollama_generation_model: str | None = None ollama_verify_ssl: bool = True - # OpenAI settings (for embeddings) + # OpenAI settings (embeddings + optional generation) openai_api_key: str | None = None openai_base_url: str | None = None openai_embedding_model: str = "text-embedding-3-small" + openai_generation_model: str | None = None + + # Bedrock (AWS) settings — boto3 also reads these from its credential chain + aws_region: str | None = None + aws_access_key_id: str | None = None + aws_secret_access_key: str | None = None + bedrock_embedding_model: str | None = None + bedrock_generation_model: str | None = None + + # Mistral settings (embeddings only) + mistral_api_key: str | None = None + mistral_embedding_model: str = "mistral-embed" + mistral_base_url: str | None = None + + # Simple (fallback) provider — dimension when no real provider configured + simple_embedding_dimension: int = 384 # Document chunking settings (for vector embeddings) document_chunk_size: int = 2048 # Characters per chunk @@ -573,23 +604,32 @@ class Settings: Get the active embedding model name based on provider priority. Priority order (same as ProviderRegistry): - 1. OpenAI - if OPENAI_API_KEY is set - 2. Ollama - if OLLAMA_BASE_URL is set - 3. Simple - fallback (returns "simple-384") + 1. Bedrock - if AWS_REGION or BEDROCK_EMBEDDING_MODEL is set + 2. OpenAI - if OPENAI_API_KEY is set + 3. Mistral - if MISTRAL_API_KEY is set + 4. Ollama - if OLLAMA_BASE_URL is set + 5. Simple - fallback (returns "simple-{dimension}") Returns: Active embedding model name """ - # Check OpenAI first (higher priority than Ollama in registry) + if ( + self.aws_region + or self.bedrock_embedding_model + or self.bedrock_generation_model + ): + return self.bedrock_embedding_model or "bedrock-default" + if self.openai_api_key: return self.openai_embedding_model - # Check Ollama + if self.mistral_api_key: + return self.mistral_embedding_model + if self.ollama_base_url: return self.ollama_embedding_model - # Fallback to simple provider indicator - return "simple-384" + return f"simple-{self.simple_embedding_dimension}" def get_collection_name(self) -> str: """ @@ -835,11 +875,25 @@ def get_settings() -> Settings: # Ollama settings "ollama_base_url": "OLLAMA_BASE_URL", "ollama_embedding_model": "OLLAMA_EMBEDDING_MODEL", + "ollama_generation_model": "OLLAMA_GENERATION_MODEL", "ollama_verify_ssl": "OLLAMA_VERIFY_SSL", # OpenAI settings "openai_api_key": "OPENAI_API_KEY", "openai_base_url": "OPENAI_BASE_URL", "openai_embedding_model": "OPENAI_EMBEDDING_MODEL", + "openai_generation_model": "OPENAI_GENERATION_MODEL", + # Bedrock (AWS) settings + "aws_region": "AWS_REGION", + "aws_access_key_id": "AWS_ACCESS_KEY_ID", + "aws_secret_access_key": "AWS_SECRET_ACCESS_KEY", + "bedrock_embedding_model": "BEDROCK_EMBEDDING_MODEL", + "bedrock_generation_model": "BEDROCK_GENERATION_MODEL", + # Mistral settings + "mistral_api_key": "MISTRAL_API_KEY", + "mistral_embedding_model": "MISTRAL_EMBEDDING_MODEL", + "mistral_base_url": "MISTRAL_BASE_URL", + # Simple provider + "simple_embedding_dimension": "SIMPLE_EMBEDDING_DIMENSION", # Document chunking settings "document_chunk_size": "DOCUMENT_CHUNK_SIZE", "document_chunk_overlap": "DOCUMENT_CHUNK_OVERLAP", diff --git a/nextcloud_mcp_server/providers/__init__.py b/nextcloud_mcp_server/providers/__init__.py index c1f2ad9d..c4d3be00 100644 --- a/nextcloud_mcp_server/providers/__init__.py +++ b/nextcloud_mcp_server/providers/__init__.py @@ -3,6 +3,7 @@ from .anthropic import AnthropicProvider from .base import Provider from .bedrock import BedrockProvider +from .mistral import MistralProvider from .ollama import OllamaProvider from .openai import OpenAIProvider from .registry import get_provider, reset_provider @@ -13,6 +14,7 @@ __all__ = [ "OllamaProvider", "OpenAIProvider", "AnthropicProvider", + "MistralProvider", "SimpleProvider", "BedrockProvider", "get_provider", diff --git a/nextcloud_mcp_server/providers/_retry.py b/nextcloud_mcp_server/providers/_retry.py new file mode 100644 index 00000000..8fb8011e --- /dev/null +++ b/nextcloud_mcp_server/providers/_retry.py @@ -0,0 +1,78 @@ +"""Shared rate-limit retry helper for provider modules. + +OpenAI and Mistral both retry on 429 with the same exponential-backoff curve; +extracting the loop here keeps the two provider modules thin and lets future +providers (Bedrock throttling, etc.) reuse the same primitive. +""" + +from __future__ import annotations + +import logging +from collections.abc import Awaitable, Callable +from functools import wraps +from typing import Any, TypeVar + +import anyio + +logger = logging.getLogger(__name__) + +MAX_RETRIES = 5 +INITIAL_RETRY_DELAY = 2.0 +MAX_RETRY_DELAY = 60.0 + +T = TypeVar("T") + + +def retry_on_rate_limit( + exception_type: type[BaseException], + is_rate_limit: Callable[[BaseException], bool] = lambda _exc: True, + *, + provider_name: str = "provider", +) -> Callable[[Callable[..., Awaitable[T]]], Callable[..., Awaitable[T]]]: + """Build a decorator that retries on rate-limit exceptions. + + Args: + exception_type: Catch this exception class (e.g. ``openai.RateLimitError``, + ``mistralai.client.errors.SDKError``). + is_rate_limit: Predicate that decides whether a caught exception is + actually a rate-limit (vs. some other error of the same class). + Defaults to "always True" — appropriate when ``exception_type`` is + already a rate-limit-specific class. + provider_name: Used in log messages so operators can tell which + provider exhausted retries. + """ + + def decorator(func: Callable[..., Awaitable[T]]) -> Callable[..., Awaitable[T]]: + @wraps(func) + async def wrapper(*args: Any, **kwargs: Any) -> T: + retry_delay = INITIAL_RETRY_DELAY + last_error: BaseException | None = None + + for attempt in range(1, MAX_RETRIES + 1): + try: + return await func(*args, **kwargs) + except exception_type as e: + if not is_rate_limit(e): + raise + last_error = e + if attempt < MAX_RETRIES: + logger.warning( + "%s rate limit hit (attempt %d/%d), retrying in %.1fs...", + provider_name, + attempt, + MAX_RETRIES, + retry_delay, + ) + await anyio.sleep(retry_delay) + retry_delay = min(retry_delay * 2, MAX_RETRY_DELAY) + + logger.error( + "%s rate limit exceeded after %d attempts", provider_name, MAX_RETRIES + ) + if last_error is None: # pragma: no cover — loop above always sets this + raise RuntimeError("retry loop exited without capturing an error") + raise last_error + + return wrapper + + return decorator diff --git a/nextcloud_mcp_server/providers/mistral.py b/nextcloud_mcp_server/providers/mistral.py new file mode 100644 index 00000000..b5f61584 --- /dev/null +++ b/nextcloud_mcp_server/providers/mistral.py @@ -0,0 +1,199 @@ +"""Mistral provider for embeddings. + +Currently supports embeddings only (``mistral-embed``, 1024-dim). Generation +can be added later if needed; see ADR-015. +""" + +import logging + +# mistralai 2.x ships no top-level __init__.py, so `from mistralai import …` +# raises ImportError. The canonical public paths are `mistralai.client` (which +# re-exports the SDK class via `client/__init__.py`) and `mistralai.client.errors` +# (which lazy-loads SDKError). There is no `mistralai.models` subpackage either. +from mistralai.client import Mistral +from mistralai.client.errors import SDKError + +from ._retry import retry_on_rate_limit +from .base import Provider + +logger = logging.getLogger(__name__) + +# Well-known Mistral embedding model dimensions +MISTRAL_EMBEDDING_DIMENSIONS: dict[str, int] = { + "mistral-embed": 1024, +} + +# Conservative chunk size for batch embeddings. Mistral allows large batches, +# but we keep this in line with sibling providers (OpenAI=100, Ollama=32). +BATCH_SIZE = 64 + +_NO_EMBEDDING_MODEL_MSG = "Embedding not supported - no embedding_model configured" + + +def _is_rate_limit(exc: BaseException) -> bool: + """True only for HTTP 429 SDKErrors.""" + return getattr(exc, "status_code", None) == 429 + + +_retry_429 = retry_on_rate_limit( + SDKError, is_rate_limit=_is_rate_limit, provider_name="Mistral" +) + + +class MistralProvider(Provider): + """ + Mistral provider — embeddings only. + + Uses the official ``mistralai`` SDK. Lazy dimension detection mirrors the + OpenAI provider: known models populate the cached dimension at construction + time; unknown models get their dimension detected on the first ``embed()`` + call. + """ + + def __init__( + self, + api_key: str, + embedding_model: str | None = "mistral-embed", + base_url: str | None = None, + ): + """ + Initialize the Mistral provider. + + Args: + api_key: Mistral API key. + embedding_model: Embedding model ID (default: ``mistral-embed``). + Pass ``None`` to disable embeddings (the provider will then + support no capabilities, which is mostly useful for tests). + base_url: Optional base URL override (e.g. proxies, on-prem). + """ + self.embedding_model = embedding_model + self._dimension: int | None = None + + self.client = Mistral(api_key=api_key, server_url=base_url) + + if embedding_model and embedding_model in MISTRAL_EMBEDDING_DIMENSIONS: + self._dimension = MISTRAL_EMBEDDING_DIMENSIONS[embedding_model] + + logger.info( + "Initialized Mistral provider: base_url=%s, embedding_model=%s, " + "dimension=%s", + base_url or "default", + embedding_model, + self._dimension, + ) + + @property + def supports_embeddings(self) -> bool: + return self.embedding_model is not None + + @property + def supports_generation(self) -> bool: + return False + + @_retry_429 + async def embed(self, text: str) -> list[float]: + """Generate an embedding for a single text.""" + if not self.supports_embeddings: + raise NotImplementedError(_NO_EMBEDDING_MODEL_MSG) + + assert self.embedding_model is not None + response = await self.client.embeddings.create_async( + model=self.embedding_model, + inputs=[text], + ) + + if not response.data or response.data[0].embedding is None: + raise RuntimeError( + f"Mistral embeddings API returned no embedding for model " + f"{self.embedding_model}" + ) + + embedding = response.data[0].embedding + + if self._dimension is None: + self._dimension = len(embedding) + logger.info( + "Detected embedding dimension: %d for model %s", + self._dimension, + self.embedding_model, + ) + + return embedding + + async def embed_batch(self, texts: list[str]) -> list[list[float]]: + """Generate embeddings for multiple texts, chunking by ``BATCH_SIZE``.""" + if not self.supports_embeddings: + raise NotImplementedError(_NO_EMBEDDING_MODEL_MSG) + + if not texts: + return [] + + all_embeddings: list[list[float]] = [] + for i in range(0, len(texts), BATCH_SIZE): + batch = texts[i : i + BATCH_SIZE] + batch_embeddings = await self._embed_batch_request(batch) + all_embeddings.extend(batch_embeddings) + + if self._dimension is None and batch_embeddings: + self._dimension = len(batch_embeddings[0]) + logger.info( + "Detected embedding dimension: %d for model %s", + self._dimension, + self.embedding_model, + ) + + return all_embeddings + + @_retry_429 + async def _embed_batch_request(self, batch: list[str]) -> list[list[float]]: + """Single batch request with rate-limit retry.""" + assert self.embedding_model is not None + response = await self.client.embeddings.create_async( + model=self.embedding_model, + inputs=batch, + ) + + # Defensive: response.data items have Optional fields. Sort by index + # (default 0 if missing) and reject None embeddings explicitly. + sorted_data = sorted(response.data or [], key=lambda x: x.index or 0) + result: list[list[float]] = [] + for item in sorted_data: + if item.embedding is None: + raise RuntimeError( + f"Mistral embeddings API returned a null embedding for " + f"model {self.embedding_model}" + ) + result.append(item.embedding) + + if len(result) != len(batch): + raise RuntimeError( + f"Mistral embeddings API returned {len(result)} embeddings " + f"for {len(batch)} inputs" + ) + return result + + def get_dimension(self) -> int: + if not self.supports_embeddings: + raise NotImplementedError(_NO_EMBEDDING_MODEL_MSG) + + if self._dimension is None: + raise RuntimeError( + f"Embedding dimension not detected yet for model " + f"{self.embedding_model}. Call embed() first or use a known " + "model." + ) + return self._dimension + + async def generate(self, prompt: str, max_tokens: int = 500) -> str: + raise NotImplementedError( + "MistralProvider does not support generation. " + "Use OpenAI, Anthropic, or Bedrock for text generation." + ) + + async def close(self) -> None: + # The mistralai 2.x client (Speakeasy-generated) does not expose a + # public close()/aclose() — only the async-context-manager protocol + # (__aenter__/__aexit__). Calling __aexit__ directly is internal API + # and brittle across SDK patch versions; the underlying httpx client + # is closed during garbage collection, so we leave this as a no-op. + return None diff --git a/nextcloud_mcp_server/providers/openai.py b/nextcloud_mcp_server/providers/openai.py index 201c1c29..536697bd 100644 --- a/nextcloud_mcp_server/providers/openai.py +++ b/nextcloud_mcp_server/providers/openai.py @@ -7,46 +7,17 @@ Supports: """ import logging -from functools import wraps -import anyio from openai import AsyncOpenAI, RateLimitError +from ._retry import retry_on_rate_limit from .base import Provider logger = logging.getLogger(__name__) -# Rate limit retry configuration -MAX_RETRIES = 5 -INITIAL_RETRY_DELAY = 2.0 # seconds -MAX_RETRY_DELAY = 60.0 # seconds - - -def retry_on_rate_limit(func): - """Decorator to retry on OpenAI rate limit errors with exponential backoff.""" - - @wraps(func) - async def wrapper(*args, **kwargs): - retry_delay = INITIAL_RETRY_DELAY - last_error: Exception | None = None - - for attempt in range(1, MAX_RETRIES + 1): - try: - return await func(*args, **kwargs) - except RateLimitError as e: - last_error = e - if attempt < MAX_RETRIES: - logger.warning( - f"Rate limit hit (attempt {attempt}/{MAX_RETRIES}), " - f"retrying in {retry_delay:.1f}s..." - ) - await anyio.sleep(retry_delay) - retry_delay = min(retry_delay * 2, MAX_RETRY_DELAY) - - logger.error(f"Rate limit exceeded after {MAX_RETRIES} attempts") - raise last_error # type: ignore[misc] - - return wrapper +# OpenAI's RateLimitError is itself a 429-specific class, so the default +# is_rate_limit predicate ("always True") matches the previous behavior. +_retry_429 = retry_on_rate_limit(RateLimitError, provider_name="OpenAI") # Well-known embedding dimensions for OpenAI models @@ -106,9 +77,12 @@ class OpenAIProvider(Provider): self._dimension = OPENAI_EMBEDDING_DIMENSIONS[embedding_model] logger.info( - f"Initialized OpenAI provider: base_url={base_url or 'default'} " - f"(embedding_model={embedding_model}, generation_model={generation_model}, " - f"dimension={self._dimension})" + "Initialized OpenAI provider: base_url=%s " + "(embedding_model=%s, generation_model=%s, dimension=%s)", + base_url or "default", + embedding_model, + generation_model, + self._dimension, ) @property @@ -121,7 +95,7 @@ class OpenAIProvider(Provider): """Whether this provider supports text generation.""" return self.generation_model is not None - @retry_on_rate_limit + @_retry_429 async def embed(self, text: str) -> list[float]: """ Generate embedding vector for text. @@ -152,8 +126,9 @@ class OpenAIProvider(Provider): if self._dimension is None: self._dimension = len(embedding) logger.info( - f"Detected embedding dimension: {self._dimension} " - f"for model {self.embedding_model}" + "Detected embedding dimension: %d for model %s", + self._dimension, + self.embedding_model, ) return embedding @@ -196,13 +171,14 @@ class OpenAIProvider(Provider): if self._dimension is None and batch_embeddings: self._dimension = len(batch_embeddings[0]) logger.info( - f"Detected embedding dimension: {self._dimension} " - f"for model {self.embedding_model}" + "Detected embedding dimension: %d for model %s", + self._dimension, + self.embedding_model, ) return all_embeddings - @retry_on_rate_limit + @_retry_429 async def _embed_batch_request(self, batch: list[str]) -> list[list[float]]: """Make a single batch embedding request with retry logic.""" assert self.embedding_model is not None # Type narrowing @@ -237,7 +213,7 @@ class OpenAIProvider(Provider): ) return self._dimension - @retry_on_rate_limit + @_retry_429 async def generate(self, prompt: str, max_tokens: int = 500) -> str: """ Generate text from a prompt. diff --git a/nextcloud_mcp_server/providers/registry.py b/nextcloud_mcp_server/providers/registry.py index 28eb9ecf..4768f998 100644 --- a/nextcloud_mcp_server/providers/registry.py +++ b/nextcloud_mcp_server/providers/registry.py @@ -1,10 +1,11 @@ """Provider registry and factory for auto-detection and instantiation.""" import logging -import os +from ..config import get_settings from .base import Provider from .bedrock import BedrockProvider +from .mistral import MistralProvider from .ollama import OllamaProvider from .openai import OpenAIProvider from .simple import SimpleProvider @@ -16,117 +17,112 @@ class ProviderRegistry: """ Registry for provider auto-detection and instantiation. - Checks environment variables in priority order and creates appropriate provider: - 1. Bedrock (AWS_REGION + BEDROCK_*_MODEL) - 2. OpenAI (OPENAI_API_KEY) - 3. Ollama (OLLAMA_BASE_URL) - 4. Simple (fallback for testing/development) + Reads configuration via dynaconf-backed Settings (see ``config.py``). + Checks provider settings in priority order and creates the appropriate + provider: + + 1. Bedrock (``AWS_REGION`` or ``BEDROCK_*_MODEL``) + 2. OpenAI (``OPENAI_API_KEY``) + 3. Mistral (``MISTRAL_API_KEY``) + 4. Ollama (``OLLAMA_BASE_URL``) + 5. Simple (fallback for testing/development) """ @staticmethod def create_provider() -> Provider: """ - Auto-detect and create provider based on environment variables. + Auto-detect and create provider based on configured settings. + + Settings are sourced via :func:`nextcloud_mcp_server.config.get_settings`, + which reads from settings files and environment variables (env vars + always win, see ADR-024/025). Priority order: - 1. Bedrock - if AWS_REGION or BEDROCK_EMBEDDING_MODEL is set - 2. OpenAI - if OPENAI_API_KEY is set - 3. Ollama - if OLLAMA_BASE_URL is set - 4. Simple - fallback for testing/development + + 1. Bedrock - if ``aws_region`` or ``bedrock_embedding_model`` is set + 2. OpenAI - if ``openai_api_key`` is set + 3. Mistral - if ``mistral_api_key`` is set + 4. Ollama - if ``ollama_base_url`` is set + 5. Simple - fallback for testing/development Returns: Provider instance - - Environment Variables: - Bedrock: - - AWS_REGION: AWS region (e.g., "us-east-1") - - AWS_ACCESS_KEY_ID: AWS access key (optional, uses credential chain) - - AWS_SECRET_ACCESS_KEY: AWS secret key (optional) - - BEDROCK_EMBEDDING_MODEL: Model ID for embeddings (e.g., "amazon.titan-embed-text-v2:0") - - BEDROCK_GENERATION_MODEL: Model ID for text generation (e.g., "anthropic.claude-3-sonnet-20240229-v1:0") - - OpenAI: - - OPENAI_API_KEY: OpenAI API key (or GITHUB_TOKEN for GitHub Models) - - OPENAI_BASE_URL: Base URL override (e.g., "https://models.github.ai/inference") - - OPENAI_EMBEDDING_MODEL: Model for embeddings (default: "text-embedding-3-small") - - OPENAI_GENERATION_MODEL: Model for text generation (e.g., "gpt-4o-mini") - - Ollama: - - OLLAMA_BASE_URL: Ollama API base URL (e.g., "http://localhost:11434") - - OLLAMA_EMBEDDING_MODEL: Model for embeddings (default: "nomic-embed-text") - - OLLAMA_GENERATION_MODEL: Model for text generation (e.g., "llama3.2:1b") - - OLLAMA_VERIFY_SSL: Verify SSL certificates (default: "true") - - Simple (no configuration needed, fallback): - - SIMPLE_EMBEDDING_DIMENSION: Embedding dimension (default: 384) """ - # 1. Check for Bedrock - aws_region = os.getenv("AWS_REGION") - bedrock_embedding_model = os.getenv("BEDROCK_EMBEDDING_MODEL") - bedrock_generation_model = os.getenv("BEDROCK_GENERATION_MODEL") + settings = get_settings() - if aws_region or bedrock_embedding_model or bedrock_generation_model: + # 1. Bedrock + if ( + settings.aws_region + or settings.bedrock_embedding_model + or settings.bedrock_generation_model + ): logger.info( - f"Using Bedrock provider: region={aws_region}, " - f"embedding_model={bedrock_embedding_model}, " - f"generation_model={bedrock_generation_model}" + "Using Bedrock provider: region=%s, embedding_model=%s, " + "generation_model=%s", + settings.aws_region, + settings.bedrock_embedding_model, + settings.bedrock_generation_model, ) return BedrockProvider( - region_name=aws_region, - embedding_model=bedrock_embedding_model, - generation_model=bedrock_generation_model, - aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), - aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), + region_name=settings.aws_region, + embedding_model=settings.bedrock_embedding_model, + generation_model=settings.bedrock_generation_model, + aws_access_key_id=settings.aws_access_key_id, + aws_secret_access_key=settings.aws_secret_access_key, ) - # 2. Check for OpenAI - openai_api_key = os.getenv("OPENAI_API_KEY") - if openai_api_key: - base_url = os.getenv("OPENAI_BASE_URL") - embedding_model = os.getenv( - "OPENAI_EMBEDDING_MODEL", "text-embedding-3-small" - ) - generation_model = os.getenv("OPENAI_GENERATION_MODEL") - + # 2. OpenAI + if settings.openai_api_key: logger.info( - f"Using OpenAI provider: base_url={base_url or 'default'}, " - f"embedding_model={embedding_model}, " - f"generation_model={generation_model}" + "Using OpenAI provider: base_url=%s, embedding_model=%s, " + "generation_model=%s", + settings.openai_base_url or "default", + settings.openai_embedding_model, + settings.openai_generation_model, ) return OpenAIProvider( - api_key=openai_api_key, - base_url=base_url, - embedding_model=embedding_model, - generation_model=generation_model, + api_key=settings.openai_api_key, + base_url=settings.openai_base_url, + embedding_model=settings.openai_embedding_model, + generation_model=settings.openai_generation_model, ) - # 3. Check for Ollama (local LLM) - ollama_url = os.getenv("OLLAMA_BASE_URL") - if ollama_url: - embedding_model = os.getenv("OLLAMA_EMBEDDING_MODEL", "nomic-embed-text") - generation_model = os.getenv("OLLAMA_GENERATION_MODEL") - verify_ssl = os.getenv("OLLAMA_VERIFY_SSL", "true").lower() == "true" - + # 3. Mistral + if settings.mistral_api_key: logger.info( - f"Using Ollama provider: {ollama_url}, " - f"embedding_model={embedding_model}, " - f"generation_model={generation_model}" + "Using Mistral provider: base_url=%s, embedding_model=%s", + settings.mistral_base_url or "default", + settings.mistral_embedding_model, + ) + return MistralProvider( + api_key=settings.mistral_api_key, + base_url=settings.mistral_base_url, + embedding_model=settings.mistral_embedding_model, + ) + + # 4. Ollama + if settings.ollama_base_url: + logger.info( + "Using Ollama provider: %s, embedding_model=%s, generation_model=%s", + settings.ollama_base_url, + settings.ollama_embedding_model, + settings.ollama_generation_model, ) return OllamaProvider( - base_url=ollama_url, - embedding_model=embedding_model, - generation_model=generation_model, - verify_ssl=verify_ssl, + base_url=settings.ollama_base_url, + embedding_model=settings.ollama_embedding_model, + generation_model=settings.ollama_generation_model, + verify_ssl=settings.ollama_verify_ssl, ) - # 4. Fallback to Simple provider for development/testing - dimension = int(os.getenv("SIMPLE_EMBEDDING_DIMENSION", "384")) + # 5. Simple (fallback) logger.warning( - "No provider configured (AWS_REGION, OPENAI_API_KEY, OLLAMA_BASE_URL not set). " + "No provider configured (AWS_REGION, OPENAI_API_KEY, " + "MISTRAL_API_KEY, OLLAMA_BASE_URL not set). " "Using SimpleProvider for testing/development. " - "For production, configure Bedrock, OpenAI, or Ollama." + "For production, configure Bedrock, OpenAI, Mistral, or Ollama." ) - return SimpleProvider(dimension=dimension) + return SimpleProvider(dimension=settings.simple_embedding_dimension) # Singleton instance diff --git a/nextcloud_mcp_server/search/pdf_highlighter.py b/nextcloud_mcp_server/search/pdf_highlighter.py index 3c8212a3..18ebafcd 100644 --- a/nextcloud_mcp_server/search/pdf_highlighter.py +++ b/nextcloud_mcp_server/search/pdf_highlighter.py @@ -691,6 +691,122 @@ class PDFHighlighter: f"Failed to delete temp directory {temp_pdf_path.parent}: {e}" ) + @staticmethod + def compute_chunk_bboxes_batch( + pdf_bytes: bytes, + chunks: list[tuple[int, int, int, int | None, str]], + page_boundaries: list[dict], + full_text: str, + ) -> dict[int, tuple[list[tuple[float, float, float, float]], int]]: + """Compute normalized bounding boxes for chunks without rendering. + + Lightweight alternative to highlight_chunks_batch — opens the PDF, + locates each chunk on its assigned page using the same text-search + path as the highlighter (`_find_chunk_bbox`), and returns + page-normalized rectangles. Skips the get_pixmap + PIL pipeline + entirely, so no PNG bytes are produced. + + Args: + pdf_bytes: PDF file bytes. + chunks: List of (chunk_index, start_offset, end_offset, + stored_page_number, chunk_text). chunk_index is the dict key. + page_boundaries: Pre-computed page boundaries from the document + processor; each entry is {"page", "start_offset", "end_offset"}. + full_text: Full document text (for cross-page chunk handling). + + Returns: + dict mapping chunk_index to (normalized_bboxes, page_number). + Each bbox is (x0, y0, x1, y1) in [0, 1] relative to page width + and height, top-left origin. Chunks whose bbox cannot be located + are omitted from the result. + """ + results: dict[int, tuple[list[tuple[float, float, float, float]], int]] = {} + + if not chunks: + return results + + temp_pdf_path = None + doc = None + try: + temp_dir = Path(tempfile.mkdtemp(prefix="pdf_bbox_batch_")) + temp_pdf_path = temp_dir / "pdf.pdf" + temp_pdf_path.write_bytes(pdf_bytes) + + doc = pymupdf.open(temp_pdf_path) + + for ( + chunk_index, + start_offset, + end_offset, + _, + _, + ) in chunks: + chunk_page_info = PDFHighlighter.find_chunk_page( + start_offset, end_offset, page_boundaries + ) + if not chunk_page_info: + logger.debug("Chunk %s: not found on any page", chunk_index) + continue + + page_num = chunk_page_info["page_num"] + page_boundary = next( + (b for b in page_boundaries if b["page"] == page_num), None + ) + if page_boundary is None: + logger.debug( + "Chunk %s: page %s not found in boundaries", + chunk_index, + page_num, + ) + continue + page_text_length = ( + page_boundary["end_offset"] - page_boundary["start_offset"] + ) + + # Page-relative slice (handles chunks that span page boundaries) + chunk_start_on_page = max(start_offset, page_boundary["start_offset"]) + chunk_end_on_page = min(end_offset, page_boundary["end_offset"]) + page_relative_text = full_text[chunk_start_on_page:chunk_end_on_page] + + page = doc[page_num - 1] + bbox = PDFHighlighter._find_chunk_bbox( + page, + page_relative_text, + chunk_page_info["page_relative_start"], + chunk_page_info["page_relative_end"], + page_text_length, + ) + + if bbox is None: + continue + + page_rect = page.rect + w = page_rect.width or 1.0 + h = page_rect.height or 1.0 + normalized = ( + bbox[0] / w, + bbox[1] / h, + bbox[2] / w, + bbox[3] / h, + ) + results[chunk_index] = ([normalized], page_num) + + logger.info(f"Computed bboxes for {len(results)}/{len(chunks)} chunks") + return results + + except Exception as e: + logger.error(f"Error computing chunk bboxes: {e}", exc_info=True) + return results + + finally: + if doc is not None: + doc.close() + if temp_pdf_path and temp_pdf_path.parent.exists(): + try: + shutil.rmtree(temp_pdf_path.parent) + except Exception as e: + logger.warning(f"Failed to clean up temp dir: {e}") + @staticmethod def highlight_chunks_batch( pdf_bytes: bytes, diff --git a/nextcloud_mcp_server/vector/__init__.py b/nextcloud_mcp_server/vector/__init__.py index 00c11cbe..8f83ef6e 100644 --- a/nextcloud_mcp_server/vector/__init__.py +++ b/nextcloud_mcp_server/vector/__init__.py @@ -1,16 +1,17 @@ -"""Vector database and background sync package.""" +"""Vector database and background sync package. + +`processor` and `scanner` are intentionally NOT re-exported from this +package init: they transitively import `server.semantic` -> +`search.bm25_hybrid`, which forms an import cycle with +`search.algorithms` -> `vector.placeholder` -> `vector/__init__`. +Consumers that need those symbols import them from their submodules +directly (e.g. `from nextcloud_mcp_server.vector.processor import ...`). +""" from .document_chunker import DocumentChunker -from .processor import process_document, processor_task from .qdrant_client import get_qdrant_client -from .scanner import DocumentTask, scan_user_documents, scanner_task __all__ = [ "get_qdrant_client", "DocumentChunker", - "scanner_task", - "scan_user_documents", - "DocumentTask", - "processor_task", - "process_document", ] diff --git a/nextcloud_mcp_server/vector/processor.py b/nextcloud_mcp_server/vector/processor.py index 848b8098..f3acc8ff 100644 --- a/nextcloud_mcp_server/vector/processor.py +++ b/nextcloud_mcp_server/vector/processor.py @@ -3,7 +3,6 @@ Processes documents from stream: fetches content, generates embeddings, stores in Qdrant. """ -import base64 import logging import time import uuid @@ -538,7 +537,11 @@ async def _index_document( # Initialize results containers dense_embeddings: list = [] sparse_embeddings: list = [] - chunk_images: dict[int, dict] = {} + # chunk_index -> list[(x0, y0, x1, y1)] of normalized rectangles + # in [0, 1] relative to page width/height. The page is taken from + # `chunk.page_number` (offset-based) and stored as `page_number` + # in the Qdrant payload, so we don't carry an `actual_page_num` here. + chunk_bboxes: dict[int, list[tuple[float, float, float, float]]] = {} # Determine if we need PDF highlighting is_pdf = doc_task.doc_type == "file" and content_type == "application/pdf" @@ -570,8 +573,8 @@ async def _index_document( sparse_embeddings = await bm25_service.encode_batch(chunk_texts) async def generate_highlights(): - """Generate highlighted page images for PDF chunks (CPU-bound).""" - nonlocal chunk_images + """Compute chunk bounding boxes for PDF chunks (CPU-bound, no rendering).""" + nonlocal chunk_bboxes if not is_pdf: return @@ -579,64 +582,42 @@ async def _index_document( assert content_bytes is not None with trace_operation( - "vector_sync.generate_highlights", + "vector_sync.compute_chunk_bboxes", attributes={ "vector_sync.chunk_count": len(chunks), "vector_sync.pdf_size": len(content_bytes), }, ): - # Build chunk data for batch processing - # Format: (chunk_index, start_offset, end_offset, page_number, chunk_text) chunk_data: list[tuple[int, int, int, int | None, str]] = [ (i, chunk.start_offset, chunk.end_offset, chunk.page_number, chunk.text) for i, chunk in enumerate(chunks) if chunk.page_number is not None ] - # Get pre-computed page boundaries from document processor page_boundaries = file_metadata.get("page_boundaries") if not page_boundaries: - logger.warning("No page boundaries available, skipping highlighting") + logger.warning( + "No page boundaries available, skipping bbox computation" + ) return - # Type narrowing: page_boundaries is guaranteed to be list[dict] here page_boundaries_list = cast(list[dict[str, Any]], page_boundaries) - logger.info( - f"Batch generating highlighted page images for {len(chunk_data)} PDF chunks" - ) + logger.info(f"Computing chunk bboxes for {len(chunk_data)} PDF chunks") - # Run CPU-bound highlighting in thread pool - # Pass pre-computed page boundaries and full text to avoid re-processing the PDF batch_results = await anyio.to_thread.run_sync( # type: ignore[attr-defined] - lambda: PDFHighlighter.highlight_chunks_batch( + lambda: PDFHighlighter.compute_chunk_bboxes_batch( pdf_bytes=content_bytes, chunks=chunk_data, page_boundaries=page_boundaries_list, full_text=content, - color="yellow", - zoom=2.0, ) ) - # Convert results to storage format - for chunk_index, ( - png_bytes, - actual_page_num, - highlight_count, - ) in batch_results.items(): - image_base64 = base64.b64encode(png_bytes).decode("utf-8") - chunk_images[chunk_index] = { - "image": image_base64, - "page": actual_page_num, - "highlights": highlight_count, - "size": len(png_bytes), - } + for chunk_index, (bboxes, _) in batch_results.items(): + chunk_bboxes[chunk_index] = bboxes - logger.info( - f"Generated {len(chunk_images)}/{len(chunks)} highlighted page images " - f"(avg {sum(img['size'] for img in chunk_images.values()) // max(len(chunk_images), 1):,} bytes)" - ) + logger.info(f"Computed bboxes for {len(chunk_bboxes)}/{len(chunks)} chunks") # Run all embedding/highlighting operations in parallel # - Dense embeddings: I/O bound (API call) @@ -752,16 +733,11 @@ async def _index_document( if doc_task.doc_type == "deck_card" else {} ), - # Highlighted page image (PDF only) - **( - { - "highlighted_page_image": chunk_images[i]["image"], - "highlighted_page_number": chunk_images[i]["page"], - "highlight_count": chunk_images[i]["highlights"], - } - if i in chunk_images - else {} - ), + # Chunk bbox (PDF only) — normalized rectangles in [0,1] + # relative to page width/height. Replaces the legacy + # `highlighted_page_image` (Deck #76). The page number + # comes from `page_number` (set above for PDF chunks). + **({"chunk_bbox": chunk_bboxes[i]} if i in chunk_bboxes else {}), }, ) ) @@ -780,15 +756,16 @@ async def _index_document( f"Failed to delete placeholder for {doc_task.doc_type}_{doc_task.doc_id}: {e}" ) - # Upsert to Qdrant in batches to avoid timeout with large payloads - # Each batch is limited to avoid WriteTimeout when sending large image payloads - BATCH_SIZE = 10 # ~2MB per batch with images + # Upsert to Qdrant in batches. Now that we no longer embed PNG payloads, + # per-point payloads are small (chunk text + small metadata), so we can + # safely use a larger batch size. + BATCH_SIZE = 100 with trace_operation( "vector_sync.qdrant_upsert", attributes={ "vector_sync.point_count": len(points), "vector_sync.collection": settings.get_collection_name(), - "vector_sync.images_count": len(chunk_images), + "vector_sync.bboxes_count": len(chunk_bboxes), "vector_sync.batch_size": BATCH_SIZE, }, ): diff --git a/pyproject.toml b/pyproject.toml index 58de7ace..38b0702b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "nextcloud-mcp-server" -version = "0.81.0" +version = "0.83.0" description = "Model Context Protocol (MCP) server for Nextcloud integration - enables AI assistants to interact with Nextcloud data" authors = [ {name = "Chris Coutinho", email = "chris@coutinho.io"} @@ -43,6 +43,7 @@ dependencies = [ "pymupdf4llm>=0.2.2", "openai>=2.8.1", "dynaconf>=3.2.13,<4.0", + "mistralai>=2.4.5", ] classifiers = [ "Development Status :: 4 - Beta", diff --git a/scripts/purge_page_images.py b/scripts/purge_page_images.py new file mode 100755 index 00000000..350a792f --- /dev/null +++ b/scripts/purge_page_images.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Purge legacy `highlighted_page_image` payloads from Qdrant (Deck #76). + +Iterates all points in the configured Qdrant collection and deletes the +legacy payload keys `highlighted_page_image`, `highlighted_page_number`, +and `highlight_count`. This relieves disk pressure caused by inline +base64 PNGs that the new code path no longer writes. + +Idempotent: deleting non-existent keys is a no-op, so re-runs are safe. + +Usage: + uv run python scripts/purge_page_images.py [--dry-run] [--batch-size 256] + +Connection settings (Qdrant URL/API key, collection name) are read from +the same `Settings` object the server uses. +""" + +from __future__ import annotations + +import argparse +import logging +import sys +from functools import partial + +import anyio +from qdrant_client import AsyncQdrantClient + +from nextcloud_mcp_server.config import get_settings + +logger = logging.getLogger("purge_page_images") + +LEGACY_FIELDS = [ + "highlighted_page_image", + "highlighted_page_number", + "highlight_count", +] + + +async def purge(dry_run: bool, batch_size: int) -> None: + settings = get_settings() + if not settings.qdrant_url: + raise SystemExit( + "qdrant_url is not configured. Set QDRANT_URL (and QDRANT_API_KEY " + "if required) before running this script." + ) + collection = settings.get_collection_name() + + # AsyncQdrantClient doesn't implement __aenter__/__aexit__, so use + # try/finally to guarantee the underlying aiohttp session is closed. + client = AsyncQdrantClient( + url=settings.qdrant_url, + api_key=settings.qdrant_api_key, + timeout=60, + ) + try: + next_offset = None + total_seen = 0 + total_updated = 0 + + logger.info( + "Scanning collection %s; will delete keys %s%s", + collection, + LEGACY_FIELDS, + " (dry run)" if dry_run else "", + ) + + while True: + points, next_offset = await client.scroll( + collection_name=collection, + limit=batch_size, + offset=next_offset, + with_payload=False, + with_vectors=False, + ) + if not points: + break + + ids = [p.id for p in points] + total_seen += len(ids) + + if not dry_run: + await client.delete_payload( + collection_name=collection, + keys=LEGACY_FIELDS, + points=ids, + ) + total_updated += len(ids) + + logger.info( + "Batch: ids=%d total_seen=%d total_updated=%d", + len(ids), + total_seen, + total_updated, + ) + + if next_offset is None: + break + + logger.info( + "Done. total_seen=%d total_updated=%d%s", + total_seen, + total_updated, + " (dry run, no writes)" if dry_run else "", + ) + finally: + await client.close() + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--dry-run", + action="store_true", + help="Scan only; do not write any changes.", + ) + parser.add_argument( + "--batch-size", + type=int, + default=256, + help="Points per scroll/update batch (default: 256).", + ) + parser.add_argument( + "-v", "--verbose", action="store_true", help="Enable debug logging." + ) + args = parser.parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + + anyio.run(partial(purge, dry_run=args.dry_run, batch_size=args.batch_size)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/unit/providers/test_mistral.py b/tests/unit/providers/test_mistral.py new file mode 100644 index 00000000..0e38da15 --- /dev/null +++ b/tests/unit/providers/test_mistral.py @@ -0,0 +1,269 @@ +"""Unit tests for Mistral provider.""" + +from unittest.mock import AsyncMock, MagicMock + +import pytest +from mistralai.client.errors import SDKError + +from nextcloud_mcp_server.providers.mistral import ( + BATCH_SIZE, + MISTRAL_EMBEDDING_DIMENSIONS, + MistralProvider, + _is_rate_limit, +) + + +def _make_data(embedding: list[float], index: int) -> MagicMock: + """Build a mock EmbeddingResponseData entry.""" + item = MagicMock() + item.embedding = embedding + item.index = index + return item + + +def _make_response(embeddings: list[list[float]]) -> MagicMock: + """Build a mock EmbeddingResponse with `embeddings` indexed in order.""" + response = MagicMock() + response.data = [_make_data(emb, i) for i, emb in enumerate(embeddings)] + return response + + +@pytest.fixture +def mock_mistral_client(mocker): + """Mock the Mistral SDK constructor.""" + mock_client = MagicMock() + mock_client.embeddings = MagicMock() + mocker.patch( + "nextcloud_mcp_server.providers.mistral.Mistral", return_value=mock_client + ) + return mock_client + + +@pytest.mark.unit +async def test_mistral_embedding_single(mock_mistral_client): + """Single text embed: round-trip through SDK with correct kwargs.""" + mock_mistral_client.embeddings.create_async = AsyncMock( + return_value=_make_response([[0.1, 0.2, 0.3]]) + ) + + provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed") + embedding = await provider.embed("hello world") + + assert embedding == [0.1, 0.2, 0.3] + mock_mistral_client.embeddings.create_async.assert_awaited_once_with( + model="mistral-embed", + inputs=["hello world"], + ) + + +@pytest.mark.unit +async def test_mistral_embedding_batch_single_call(mock_mistral_client): + """Batch smaller than BATCH_SIZE issues a single API call.""" + mock_mistral_client.embeddings.create_async = AsyncMock( + return_value=_make_response([[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]]) + ) + + provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed") + embeddings = await provider.embed_batch(["a", "b", "c"]) + + assert embeddings == [[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]] + assert mock_mistral_client.embeddings.create_async.await_count == 1 + + +@pytest.mark.unit +async def test_mistral_embedding_batch_chunking(mock_mistral_client): + """Batches exceeding BATCH_SIZE are split into multiple API calls.""" + + # Each call returns one embedding per input it received; capture by side + # effect so we can inspect lengths per chunk. + def _side_effect(*, model, inputs, **_kwargs): + return _make_response([[float(i)] for i in range(len(inputs))]) + + mock_mistral_client.embeddings.create_async = AsyncMock(side_effect=_side_effect) + + provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed") + total = BATCH_SIZE * 2 + 5 # forces three chunks: 64, 64, 5 (with default) + embeddings = await provider.embed_batch([f"text-{i}" for i in range(total)]) + + assert len(embeddings) == total + assert mock_mistral_client.embeddings.create_async.await_count == 3 + # Verify the chunk sizes the SDK was actually called with. + chunk_sizes = [ + len(call.kwargs["inputs"]) + for call in mock_mistral_client.embeddings.create_async.await_args_list + ] + assert chunk_sizes == [BATCH_SIZE, BATCH_SIZE, 5] + + +@pytest.mark.unit +async def test_mistral_embedding_batch_order_preserved(mock_mistral_client): + """Out-of-order index in response data is sorted before returning.""" + response = MagicMock() + response.data = [ + _make_data([0.3, 0.3], 2), + _make_data([0.1, 0.1], 0), + _make_data([0.2, 0.2], 1), + ] + mock_mistral_client.embeddings.create_async = AsyncMock(return_value=response) + + provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed") + embeddings = await provider.embed_batch(["x", "y", "z"]) + + assert embeddings == [[0.1, 0.1], [0.2, 0.2], [0.3, 0.3]] + + +@pytest.mark.unit +async def test_mistral_supports_capabilities(mock_mistral_client): + """Mistral provider advertises embeddings only.""" + provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed") + assert provider.supports_embeddings is True + assert provider.supports_generation is False + + +@pytest.mark.unit +async def test_mistral_generate_not_implemented(mock_mistral_client): + """generate() always raises NotImplementedError.""" + provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed") + with pytest.raises(NotImplementedError, match="does not support generation"): + await provider.generate("test prompt") + + +@pytest.mark.unit +async def test_mistral_get_dimension_known_model(mock_mistral_client): + """Known model: dimension available without an API call.""" + provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed") + assert provider.get_dimension() == MISTRAL_EMBEDDING_DIMENSIONS["mistral-embed"] + mock_mistral_client.embeddings.create_async.assert_not_called() + + +@pytest.mark.unit +async def test_mistral_get_dimension_unknown_model_detected(mock_mistral_client): + """Unknown model: dimension detected on first embed() call.""" + mock_mistral_client.embeddings.create_async = AsyncMock( + return_value=_make_response([[0.1] * 768]) + ) + + provider = MistralProvider(api_key="test-key", embedding_model="custom-mistral") + + with pytest.raises(RuntimeError, match="not detected yet"): + provider.get_dimension() + + await provider.embed("test") + assert provider.get_dimension() == 768 + + +@pytest.mark.unit +async def test_mistral_no_embeddings_disabled(mock_mistral_client): + """Setting embedding_model=None disables the embedding capability.""" + provider = MistralProvider(api_key="test-key", embedding_model=None) + assert provider.supports_embeddings is False + + with pytest.raises(NotImplementedError, match="no embedding_model configured"): + await provider.embed("test") + with pytest.raises(NotImplementedError, match="no embedding_model configured"): + await provider.embed_batch(["test"]) + with pytest.raises(NotImplementedError, match="no embedding_model configured"): + provider.get_dimension() + + +@pytest.mark.unit +async def test_mistral_empty_batch(mock_mistral_client): + """An empty batch returns [] without calling the API.""" + provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed") + assert await provider.embed_batch([]) == [] + mock_mistral_client.embeddings.create_async.assert_not_called() + + +@pytest.mark.unit +async def test_mistral_close_no_error(mock_mistral_client): + """close() is best-effort and does not raise.""" + provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed") + # No __aexit__ on the mock by default → close() should silently no-op. + await provider.close() + + +@pytest.mark.unit +async def test_mistral_base_url_passed_to_sdk(mocker): + """base_url is forwarded as server_url to the Mistral SDK constructor.""" + mock_ctor = mocker.patch( + "nextcloud_mcp_server.providers.mistral.Mistral", return_value=MagicMock() + ) + + MistralProvider( + api_key="test-key", + embedding_model="mistral-embed", + base_url="https://example.com/mistral", + ) + mock_ctor.assert_called_once_with( + api_key="test-key", + server_url="https://example.com/mistral", + ) + + +@pytest.mark.unit +async def test_mistral_embed_raises_on_empty_response_data(mock_mistral_client): + """embed(): empty response.data triggers the defensive RuntimeError guard.""" + empty_response = MagicMock() + empty_response.data = [] + mock_mistral_client.embeddings.create_async = AsyncMock(return_value=empty_response) + + provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed") + with pytest.raises(RuntimeError, match="returned no embedding"): + await provider.embed("test") + + +@pytest.mark.unit +async def test_mistral_embed_raises_on_null_embedding(mock_mistral_client): + """embed(): a single response item with embedding=None is rejected.""" + null_item = MagicMock() + null_item.embedding = None + null_item.index = 0 + null_response = MagicMock() + null_response.data = [null_item] + mock_mistral_client.embeddings.create_async = AsyncMock(return_value=null_response) + + provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed") + with pytest.raises(RuntimeError, match="returned no embedding"): + await provider.embed("test") + + +@pytest.mark.unit +async def test_mistral_batch_raises_on_null_embedding(mock_mistral_client): + """_embed_batch_request: a null embedding inside a batch raises explicitly.""" + good = _make_data([0.1, 0.2], 0) + bad = MagicMock() + bad.embedding = None + bad.index = 1 + response = MagicMock() + response.data = [good, bad] + mock_mistral_client.embeddings.create_async = AsyncMock(return_value=response) + + provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed") + with pytest.raises(RuntimeError, match="null embedding"): + await provider.embed_batch(["a", "b"]) + + +@pytest.mark.unit +async def test_mistral_batch_raises_on_count_mismatch(mock_mistral_client): + """_embed_batch_request: fewer embeddings returned than inputs sent.""" + # Two inputs sent, one embedding returned. + response = _make_response([[0.1, 0.2]]) + mock_mistral_client.embeddings.create_async = AsyncMock(return_value=response) + + provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed") + with pytest.raises(RuntimeError, match="returned 1 embeddings for 2 inputs"): + await provider.embed_batch(["a", "b"]) + + +@pytest.mark.unit +def test_mistral_is_rate_limit_predicate(): + """_is_rate_limit returns True only for SDKErrors with status_code == 429.""" + err_429 = MagicMock(spec=SDKError) + err_429.status_code = 429 + err_500 = MagicMock(spec=SDKError) + err_500.status_code = 500 + + assert _is_rate_limit(err_429) is True + assert _is_rate_limit(err_500) is False + # ValueError has no status_code attr → getattr returns None → False. + assert _is_rate_limit(ValueError()) is False diff --git a/tests/unit/providers/test_registry.py b/tests/unit/providers/test_registry.py new file mode 100644 index 00000000..5d4c3226 --- /dev/null +++ b/tests/unit/providers/test_registry.py @@ -0,0 +1,136 @@ +"""Unit tests for ProviderRegistry — dynaconf-driven auto-detection.""" + +import pytest + +from nextcloud_mcp_server.config import _reload_config +from nextcloud_mcp_server.providers import ( + BedrockProvider, + MistralProvider, + OllamaProvider, + OpenAIProvider, + SimpleProvider, + get_provider, + reset_provider, +) +from nextcloud_mcp_server.providers.bedrock import BOTO3_AVAILABLE + + +def _clear_provider_envs(monkeypatch: pytest.MonkeyPatch) -> None: + """Strip every provider-selection env var so each test starts clean.""" + for name in ( + "AWS_REGION", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "BEDROCK_EMBEDDING_MODEL", + "BEDROCK_GENERATION_MODEL", + "OPENAI_API_KEY", + "OPENAI_BASE_URL", + "OPENAI_EMBEDDING_MODEL", + "OPENAI_GENERATION_MODEL", + "MISTRAL_API_KEY", + "MISTRAL_BASE_URL", + "MISTRAL_EMBEDDING_MODEL", + "OLLAMA_BASE_URL", + "OLLAMA_EMBEDDING_MODEL", + "OLLAMA_GENERATION_MODEL", + "OLLAMA_VERIFY_SSL", + "SIMPLE_EMBEDDING_DIMENSION", + ): + monkeypatch.delenv(name, raising=False) + + +@pytest.fixture +def clean_provider_env(monkeypatch): + """Reset provider singleton + dynaconf cache around each test.""" + _clear_provider_envs(monkeypatch) + reset_provider() + _reload_config() + yield monkeypatch + reset_provider() + + +@pytest.mark.unit +def test_registry_falls_back_to_simple(clean_provider_env): + """No provider env set → SimpleProvider (with default dimension).""" + provider = get_provider() + assert isinstance(provider, SimpleProvider) + assert provider.get_dimension() == 384 + + +@pytest.mark.unit +def test_registry_picks_simple_with_custom_dimension(clean_provider_env): + """SIMPLE_EMBEDDING_DIMENSION flows through dynaconf to SimpleProvider.""" + clean_provider_env.setenv("SIMPLE_EMBEDDING_DIMENSION", "512") + _reload_config() + + provider = get_provider() + assert isinstance(provider, SimpleProvider) + assert provider.get_dimension() == 512 + + +@pytest.mark.unit +def test_registry_picks_mistral_when_api_key_set(clean_provider_env, mocker): + """MISTRAL_API_KEY alone is enough to select MistralProvider.""" + # MistralProvider eagerly constructs the SDK client in __init__; stub it + # so the test doesn't depend on the SDK accepting arbitrary keys. + mocker.patch("nextcloud_mcp_server.providers.mistral.Mistral") + clean_provider_env.setenv("MISTRAL_API_KEY", "test-key") + _reload_config() + + provider = get_provider() + assert isinstance(provider, MistralProvider) + + +@pytest.mark.unit +def test_registry_picks_ollama_when_base_url_set(clean_provider_env, mocker): + """OLLAMA_BASE_URL selects OllamaProvider.""" + # OllamaProvider eagerly probes /api/tags in __init__; stub it out. + mocker.patch( + "nextcloud_mcp_server.providers.ollama.OllamaProvider._check_model_is_loaded" + ) + clean_provider_env.setenv("OLLAMA_BASE_URL", "http://localhost:11434") + _reload_config() + + provider = get_provider() + assert isinstance(provider, OllamaProvider) + + +@pytest.mark.unit +def test_registry_openai_wins_over_mistral_and_ollama(clean_provider_env): + """OpenAI takes priority when multiple provider env vars are set.""" + clean_provider_env.setenv("OPENAI_API_KEY", "openai-key") + clean_provider_env.setenv("MISTRAL_API_KEY", "mistral-key") + clean_provider_env.setenv("OLLAMA_BASE_URL", "http://localhost:11434") + _reload_config() + + provider = get_provider() + assert isinstance(provider, OpenAIProvider) + + +@pytest.mark.unit +def test_registry_mistral_wins_over_ollama(clean_provider_env, mocker): + """Mistral takes priority over Ollama when both are configured.""" + # Stub the Mistral SDK constructor for the same reason as the sibling + # picker test — keeps the registry test independent of SDK key validation. + mocker.patch("nextcloud_mcp_server.providers.mistral.Mistral") + clean_provider_env.setenv("MISTRAL_API_KEY", "mistral-key") + clean_provider_env.setenv("OLLAMA_BASE_URL", "http://localhost:11434") + _reload_config() + + provider = get_provider() + assert isinstance(provider, MistralProvider) + + +@pytest.mark.unit +def test_registry_bedrock_wins_when_aws_region_set(clean_provider_env): + """AWS_REGION alone routes to Bedrock, even with other providers configured.""" + if not BOTO3_AVAILABLE: + pytest.skip("boto3 not installed") + + clean_provider_env.setenv("AWS_REGION", "us-east-1") + clean_provider_env.setenv("OPENAI_API_KEY", "openai-key") + clean_provider_env.setenv("MISTRAL_API_KEY", "mistral-key") + _reload_config() + + provider = get_provider() + assert isinstance(provider, BedrockProvider) diff --git a/tests/unit/providers/test_retry.py b/tests/unit/providers/test_retry.py new file mode 100644 index 00000000..305acdd3 --- /dev/null +++ b/tests/unit/providers/test_retry.py @@ -0,0 +1,103 @@ +"""Unit tests for the shared rate-limit retry decorator.""" + +from unittest.mock import AsyncMock + +import pytest + +from nextcloud_mcp_server.providers import _retry + + +class _FakeError(Exception): + """Stand-in for an SDK exception with an HTTP status code attached.""" + + def __init__(self, status_code: int): + super().__init__(f"status {status_code}") + self.status_code = status_code + + +@pytest.fixture(autouse=True) +def _no_real_sleep(monkeypatch): + """Replace anyio.sleep with an awaitable no-op so retries don't waste time.""" + monkeypatch.setattr(_retry.anyio, "sleep", AsyncMock(return_value=None)) + + +@pytest.mark.unit +async def test_retry_succeeds_after_429(): + """A 429 followed by success returns the success value.""" + calls = {"n": 0} + + @_retry.retry_on_rate_limit( + _FakeError, is_rate_limit=lambda e: e.status_code == 429 + ) + async def flaky(): + calls["n"] += 1 + if calls["n"] < 3: + raise _FakeError(429) + return "ok" + + result = await flaky() + assert result == "ok" + assert calls["n"] == 3 + + +@pytest.mark.unit +async def test_retry_reraises_non_rate_limit_immediately(): + """A non-rate-limit error of the same class is re-raised on first hit.""" + calls = {"n": 0} + + @_retry.retry_on_rate_limit( + _FakeError, is_rate_limit=lambda e: e.status_code == 429 + ) + async def boom(): + calls["n"] += 1 + raise _FakeError(500) + + with pytest.raises(_FakeError, match="status 500"): + await boom() + assert calls["n"] == 1 # No retries on non-429. + + +@pytest.mark.unit +async def test_retry_gives_up_after_max_retries(): + """After MAX_RETRIES failed attempts the last error is re-raised.""" + calls = {"n": 0} + + @_retry.retry_on_rate_limit( + _FakeError, is_rate_limit=lambda e: e.status_code == 429 + ) + async def always_429(): + calls["n"] += 1 + raise _FakeError(429) + + with pytest.raises(_FakeError, match="status 429"): + await always_429() + assert calls["n"] == _retry.MAX_RETRIES + + +@pytest.mark.unit +async def test_retry_default_predicate_treats_all_as_rate_limit(): + """Default predicate (`lambda _: True`) retries every caught exception.""" + calls = {"n": 0} + + @_retry.retry_on_rate_limit(_FakeError) + async def fail_once(): + calls["n"] += 1 + if calls["n"] < 2: + raise _FakeError(503) + return "recovered" + + result = await fail_once() + assert result == "recovered" + assert calls["n"] == 2 + + +@pytest.mark.unit +async def test_retry_does_not_catch_unrelated_exceptions(): + """Exceptions of a different class bypass the decorator entirely.""" + + @_retry.retry_on_rate_limit(_FakeError) + async def value_error(): + raise ValueError("nope") + + with pytest.raises(ValueError, match="nope"): + await value_error() diff --git a/tests/unit/search/test_pdf_highlighter_bbox.py b/tests/unit/search/test_pdf_highlighter_bbox.py new file mode 100644 index 00000000..60985916 --- /dev/null +++ b/tests/unit/search/test_pdf_highlighter_bbox.py @@ -0,0 +1,232 @@ +"""Unit tests for PDFHighlighter.compute_chunk_bboxes_batch (Deck #76). + +Replaces the legacy `highlight_chunks_batch`-+-base64 pipeline that inflated +Qdrant payloads with per-chunk PNG screenshots. The new path returns +normalized bounding boxes only. +""" + +from __future__ import annotations + +import pymupdf +import pytest + +from nextcloud_mcp_server.search.pdf_highlighter import PDFHighlighter + + +def _make_pdf(pages: list[str]) -> bytes: + """Build an in-memory PDF whose pages contain the given text.""" + doc = pymupdf.open() + for body in pages: + page = doc.new_page(width=595, height=842) # A4 + page.insert_text((50, 50), body) + pdf_bytes = doc.tobytes() + doc.close() + return pdf_bytes + + +def _page_boundaries(pages: list[str]) -> tuple[list[dict], str]: + """Build (page_boundaries, full_text) compatible with the highlighter API.""" + boundaries: list[dict] = [] + cursor = 0 + parts: list[str] = [] + for i, body in enumerate(pages, start=1): + end = cursor + len(body) + boundaries.append({"page": i, "start_offset": cursor, "end_offset": end}) + parts.append(body) + cursor = end + return boundaries, "".join(parts) + + +@pytest.mark.unit +def test_compute_chunk_bboxes_returns_normalized_rects(): + """Each returned bbox should be 4 floats in [0, 1] tagged with the page.""" + pages = [ + "Chapter 1: Introduction. Nextcloud is a self-hosted collaboration platform " + "covering installation, configuration and maintenance topics.", + "Chapter 2: Installation. Download the package, extract it to the web " + "server directory, and configure the database connection.", + ] + pdf_bytes = _make_pdf(pages) + boundaries, full_text = _page_boundaries(pages) + + chunks = [ + ( + 0, + 0, + len(pages[0]), + 1, + "Chapter 1: Introduction. Nextcloud is a self-hosted collaboration platform.", + ), + ( + 1, + len(pages[0]), + len(pages[0]) + len(pages[1]), + 2, + "Chapter 2: Installation. Download the package.", + ), + ] + + results = PDFHighlighter.compute_chunk_bboxes_batch( + pdf_bytes=pdf_bytes, + chunks=chunks, + page_boundaries=boundaries, + full_text=full_text, + ) + + assert set(results) == {0, 1} + + bboxes_p1, page_p1 = results[0] + bboxes_p2, page_p2 = results[1] + + assert page_p1 == 1 + assert page_p2 == 2 + + for rects in (bboxes_p1, bboxes_p2): + assert len(rects) >= 1 + for rect in rects: + assert len(rect) == 4 + x0, y0, x1, y1 = rect + assert 0.0 <= x0 < x1 <= 1.0 + assert 0.0 <= y0 < y1 <= 1.0 + + +@pytest.mark.unit +def test_compute_chunk_bboxes_empty_input(): + assert ( + PDFHighlighter.compute_chunk_bboxes_batch( + pdf_bytes=b"", + chunks=[], + page_boundaries=[], + full_text="", + ) + == {} + ) + + +@pytest.mark.unit +def test_compute_chunk_bboxes_omits_when_offsets_out_of_range(): + """Chunks whose offsets fall outside every page boundary are omitted. + + Verifies the docstring contract: *"Chunks whose bbox cannot be located + are omitted from the result."* (path: ``find_chunk_page`` returns None). + """ + pages = ["Page one body text content here for the test."] + pdf_bytes = _make_pdf(pages) + boundaries, full_text = _page_boundaries(pages) + + # Offsets way beyond the document end — no page boundary matches. + out_of_range_start = len(full_text) + 1000 + out_of_range_end = out_of_range_start + 50 + chunks = [(0, out_of_range_start, out_of_range_end, 1, "irrelevant")] + + results = PDFHighlighter.compute_chunk_bboxes_batch( + pdf_bytes=pdf_bytes, + chunks=chunks, + page_boundaries=boundaries, + full_text=full_text, + ) + + assert results == {} + + +@pytest.mark.unit +def test_compute_chunk_bboxes_omits_when_text_not_in_pdf(): + """Chunks whose page-relative text isn't on the page are omitted. + + Verifies the second omission path: ``_find_chunk_bbox`` returns None + when the supplied text cannot be located on the rendered page. + """ + pages = ["Hello world."] + pdf_bytes = _make_pdf(pages) + # Build boundaries from the real text but pass a *different* full_text + # so the page-relative slice is content that does not exist in the PDF. + boundaries, _ = _page_boundaries(pages) + bogus_full_text = "Z" * len(pages[0]) + + chunks = [(0, 0, len(pages[0]), 1, "ignored")] + + results = PDFHighlighter.compute_chunk_bboxes_batch( + pdf_bytes=pdf_bytes, + chunks=chunks, + page_boundaries=boundaries, + full_text=bogus_full_text, + ) + + assert results == {} + + +@pytest.mark.unit +@pytest.mark.parametrize("page_index", [0, 1]) +def test_compute_chunk_bboxes_assigns_correct_page(page_index: int): + """Verify the page number returned matches the page the chunk lives on.""" + pages = [ + "Page one talks about apples and oranges in detail.", + "Page two discusses bananas and grapes thoroughly.", + ] + pdf_bytes = _make_pdf(pages) + boundaries, full_text = _page_boundaries(pages) + + if page_index == 0: + chunk_text = "apples and oranges" + offsets = (0, len(pages[0])) + else: + chunk_text = "bananas and grapes" + offsets = (len(pages[0]), len(pages[0]) + len(pages[1])) + + chunks = [(0, offsets[0], offsets[1], page_index + 1, chunk_text)] + + results = PDFHighlighter.compute_chunk_bboxes_batch( + pdf_bytes=pdf_bytes, + chunks=chunks, + page_boundaries=boundaries, + full_text=full_text, + ) + + assert results, "expected a bbox for the chunk" + _, page_num = results[0] + assert page_num == page_index + 1 + + +@pytest.mark.unit +def test_compute_chunk_bboxes_handles_unordered_page_boundaries(): + """Page lookup must match by ``page`` key, not by list position. + + Regression guard: an earlier implementation indexed + ``page_boundaries[page_num - 1]``, which silently produces a wrong + bbox if boundaries are passed out of order. Reverse the boundaries + and assert the result is identical to the in-order case. + """ + pages = [ + "Page one talks about apples and oranges in detail.", + "Page two discusses bananas and grapes thoroughly.", + ] + pdf_bytes = _make_pdf(pages) + boundaries, full_text = _page_boundaries(pages) + + chunks = [ + (0, 0, len(pages[0]), 1, "apples and oranges"), + ( + 1, + len(pages[0]), + len(pages[0]) + len(pages[1]), + 2, + "bananas and grapes", + ), + ] + + in_order = PDFHighlighter.compute_chunk_bboxes_batch( + pdf_bytes=pdf_bytes, + chunks=chunks, + page_boundaries=boundaries, + full_text=full_text, + ) + reversed_order = PDFHighlighter.compute_chunk_bboxes_batch( + pdf_bytes=pdf_bytes, + chunks=chunks, + page_boundaries=list(reversed(boundaries)), + full_text=full_text, + ) + + assert in_order == reversed_order + assert reversed_order[0][1] == 1 + assert reversed_order[1][1] == 2 diff --git a/uv.lock b/uv.lock index a077072c..80de7994 100644 --- a/uv.lock +++ b/uv.lock @@ -769,6 +769,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/97/43/11d6e5d2c00bf000b5329717c74563bf76a9193f4a41cb0c4ef277dde4fa/dynaconf-3.2.13-py2.py3-none-any.whl", hash = "sha256:4305527aef4834bdba3e39479b23c005186e83fb85f65bcaa4bcea58fa26759b", size = 238041, upload-time = "2026-03-17T19:38:45.337Z" }, ] +[[package]] +name = "eval-type-backport" +version = "0.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fb/a3/cafafb4558fd638aadfe4121dc6cefb8d743368c085acb2f521df0f3d9d7/eval_type_backport-0.3.1.tar.gz", hash = "sha256:57e993f7b5b69d271e37482e62f74e76a0276c82490cf8e4f0dffeb6b332d5ed", size = 9445, upload-time = "2025-12-02T11:51:42.987Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cf/22/fdc2e30d43ff853720042fa15baa3e6122722be1a7950a98233ebb55cd71/eval_type_backport-0.3.1-py3-none-any.whl", hash = "sha256:279ab641905e9f11129f56a8a78f493518515b83402b860f6f06dd7c011fdfa8", size = 6063, upload-time = "2025-12-02T11:51:41.665Z" }, +] + [[package]] name = "executing" version = "2.2.1" @@ -1478,6 +1487,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/73/07/02e16ed01e04a374e644b575638ec7987ae846d25ad97bcc9945a3ee4b0e/jsonpatch-1.33-py2.py3-none-any.whl", hash = "sha256:0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade", size = 12898, upload-time = "2023-06-16T21:01:28.466Z" }, ] +[[package]] +name = "jsonpath-python" +version = "1.1.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/98/18/4ca8742534a5993ff383f7602e325ce2d5d7cc93d72ac5e1cdedbea8a458/jsonpath_python-1.1.6.tar.gz", hash = "sha256:dded9932b4ec41fb8726e09c83afa4e6be618f938c2db287cc2a81723c639671", size = 88178, upload-time = "2026-05-07T01:26:34.482Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/55/8a/1270a6803bd821cbfcdda387eaa13cb41a7b1f7b9bd145979b3bfb9d6cb7/jsonpath_python-1.1.6-py3-none-any.whl", hash = "sha256:a1c50afd8d3fbbaf47a4873bc890dcb3c15da96f5c020327977d844d8731a2d4", size = 14453, upload-time = "2026-05-07T01:26:33.306Z" }, +] + [[package]] name = "jsonpointer" version = "3.0.0" @@ -1842,6 +1860,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "mistralai" +version = "2.4.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "eval-type-backport" }, + { name = "httpx" }, + { name = "jsonpath-python" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "pydantic" }, + { name = "python-dateutil" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8e/3f/5624d57c5897c83c55d3e4c7dd4127de42ad14fd3183e26566cdc7dca1bf/mistralai-2.4.5.tar.gz", hash = "sha256:ef165bb004ec4423cbf19a440bf0983ca0c3fc92ab12a35ebca097bdf418e33a", size = 424611, upload-time = "2026-05-07T11:46:43.888Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/48/2c5c4f853dec32a625c1a3d23809b80cf2e135c3441fe1764f72910dfea9/mistralai-2.4.5-py3-none-any.whl", hash = "sha256:bf3b6550258ab16dec8547b90e9c18bebf9099f55b7fc25a884bf0bbeffced0f", size = 995999, upload-time = "2026-05-07T11:46:41.915Z" }, +] + [[package]] name = "mmh3" version = "5.2.0" @@ -2086,7 +2123,7 @@ wheels = [ [[package]] name = "nextcloud-mcp-server" -version = "0.81.0" +version = "0.83.0" source = { editable = "." } dependencies = [ { name = "aiosqlite" }, @@ -2104,6 +2141,7 @@ dependencies = [ { name = "langchain-text-splitters" }, { name = "markdownify" }, { name = "mcp", extra = ["cli"] }, + { name = "mistralai" }, { name = "openai" }, { name = "opentelemetry-api" }, { name = "opentelemetry-exporter-otlp-proto-grpc" }, @@ -2157,6 +2195,7 @@ requires-dist = [ { name = "langchain-text-splitters", specifier = ">=1.0.0" }, { name = "markdownify", specifier = ">=0.14.1" }, { name = "mcp", extras = ["cli"], specifier = ">=1.27,<1.28" }, + { name = "mistralai", specifier = ">=2.4.5" }, { name = "openai", specifier = ">=2.8.1" }, { name = "opentelemetry-api", specifier = ">=1.28.2" }, { name = "opentelemetry-exporter-otlp-proto-grpc", specifier = ">=1.28.2" }, @@ -2341,45 +2380,45 @@ wheels = [ [[package]] name = "opentelemetry-api" -version = "1.39.0" +version = "1.39.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "importlib-metadata" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c0/0b/e5428c009d4d9af0515b0a8371a8aaae695371af291f45e702f7969dce6b/opentelemetry_api-1.39.0.tar.gz", hash = "sha256:6130644268c5ac6bdffaf660ce878f10906b3e789f7e2daa5e169b047a2933b9", size = 65763, upload-time = "2025-12-03T13:19:56.378Z" } +sdist = { url = "https://files.pythonhosted.org/packages/97/b9/3161be15bb8e3ad01be8be5a968a9237c3027c5be504362ff800fca3e442/opentelemetry_api-1.39.1.tar.gz", hash = "sha256:fbde8c80e1b937a2c61f20347e91c0c18a1940cecf012d62e65a7caf08967c9c", size = 65767, upload-time = "2025-12-11T13:32:39.182Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/05/85/d831a9bc0a9e0e1a304ff3d12c1489a5fbc9bf6690a15dcbdae372bbca45/opentelemetry_api-1.39.0-py3-none-any.whl", hash = "sha256:3c3b3ca5c5687b1b5b37e5c5027ff68eacea8675241b29f13110a8ffbb8f0459", size = 66357, upload-time = "2025-12-03T13:19:33.043Z" }, + { url = "https://files.pythonhosted.org/packages/cf/df/d3f1ddf4bb4cb50ed9b1139cc7b1c54c34a1e7ce8fd1b9a37c0d1551a6bd/opentelemetry_api-1.39.1-py3-none-any.whl", hash = "sha256:2edd8463432a7f8443edce90972169b195e7d6a05500cd29e6d13898187c9950", size = 66356, upload-time = "2025-12-11T13:32:17.304Z" }, ] [[package]] name = "opentelemetry-exporter-otlp" -version = "1.39.0" +version = "1.39.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "opentelemetry-exporter-otlp-proto-grpc" }, { name = "opentelemetry-exporter-otlp-proto-http" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/13/be/0e9d889f47e55cadc4041e5b53d4e0cc688f9a74811134fb0ba7cbee6905/opentelemetry_exporter_otlp-1.39.0.tar.gz", hash = "sha256:b405da0287b895fe4e2450dedb2a5b072debba1dfcfed5bdb3d1d183d8daa296", size = 6146, upload-time = "2025-12-03T13:19:58.381Z" } +sdist = { url = "https://files.pythonhosted.org/packages/30/9c/3ab1db90f32da200dba332658f2bbe602369e3d19f6aba394031a42635be/opentelemetry_exporter_otlp-1.39.1.tar.gz", hash = "sha256:7cf7470e9fd0060c8a38a23e4f695ac686c06a48ad97f8d4867bc9b420180b9c", size = 6147, upload-time = "2025-12-11T13:32:40.309Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/35/212d2cae4fa9a2c02e74438612268b640ab577b8ccb04590371eb4e0f542/opentelemetry_exporter_otlp-1.39.0-py3-none-any.whl", hash = "sha256:fe155d6968d581b325574ad6dc267c8de299397b18d11feeda2206d0a47928a9", size = 7017, upload-time = "2025-12-03T13:19:35.686Z" }, + { url = "https://files.pythonhosted.org/packages/00/6c/bdc82a066e6fb1dcf9e8cc8d4e026358fe0f8690700cc6369a6bf9bd17a7/opentelemetry_exporter_otlp-1.39.1-py3-none-any.whl", hash = "sha256:68ae69775291f04f000eb4b698ff16ff685fdebe5cb52871bc4e87938a7b00fe", size = 7019, upload-time = "2025-12-11T13:32:19.387Z" }, ] [[package]] name = "opentelemetry-exporter-otlp-proto-common" -version = "1.39.0" +version = "1.39.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "opentelemetry-proto" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/11/cb/3a29ce606b10c76d413d6edd42d25a654af03e73e50696611e757d2602f3/opentelemetry_exporter_otlp_proto_common-1.39.0.tar.gz", hash = "sha256:a135fceed1a6d767f75be65bd2845da344dd8b9258eeed6bc48509d02b184409", size = 20407, upload-time = "2025-12-03T13:19:59.003Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e9/9d/22d241b66f7bbde88a3bfa6847a351d2c46b84de23e71222c6aae25c7050/opentelemetry_exporter_otlp_proto_common-1.39.1.tar.gz", hash = "sha256:763370d4737a59741c89a67b50f9e39271639ee4afc999dadfe768541c027464", size = 20409, upload-time = "2025-12-11T13:32:40.885Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/c6/215edba62d13a3948c718b289539f70e40965bc37fc82ecd55bb0b749c1a/opentelemetry_exporter_otlp_proto_common-1.39.0-py3-none-any.whl", hash = "sha256:3d77be7c4bdf90f1a76666c934368b8abed730b5c6f0547a2ec57feb115849ac", size = 18367, upload-time = "2025-12-03T13:19:36.906Z" }, + { url = "https://files.pythonhosted.org/packages/8c/02/ffc3e143d89a27ac21fd557365b98bd0653b98de8a101151d5805b5d4c33/opentelemetry_exporter_otlp_proto_common-1.39.1-py3-none-any.whl", hash = "sha256:08f8a5862d64cc3435105686d0216c1365dc5701f86844a8cd56597d0c764fde", size = 18366, upload-time = "2025-12-11T13:32:20.2Z" }, ] [[package]] name = "opentelemetry-exporter-otlp-proto-grpc" -version = "1.39.0" +version = "1.39.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "googleapis-common-protos" }, @@ -2390,14 +2429,14 @@ dependencies = [ { name = "opentelemetry-sdk" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7e/62/4db083ee9620da3065eeb559e9fc128f41a1d15e7c48d7c83aafbccd354c/opentelemetry_exporter_otlp_proto_grpc-1.39.0.tar.gz", hash = "sha256:7e7bb3f436006836c0e0a42ac619097746ad5553ad7128a5bd4d3e727f37fc06", size = 24650, upload-time = "2025-12-03T13:20:00.06Z" } +sdist = { url = "https://files.pythonhosted.org/packages/53/48/b329fed2c610c2c32c9366d9dc597202c9d1e58e631c137ba15248d8850f/opentelemetry_exporter_otlp_proto_grpc-1.39.1.tar.gz", hash = "sha256:772eb1c9287485d625e4dbe9c879898e5253fea111d9181140f51291b5fec3ad", size = 24650, upload-time = "2025-12-11T13:32:41.429Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/56/e8/d420b94ffddfd8cff85bb4aa5d98da26ce7935dc3cf3eca6b83cd39ab436/opentelemetry_exporter_otlp_proto_grpc-1.39.0-py3-none-any.whl", hash = "sha256:758641278050de9bb895738f35ff8840e4a47685b7e6ef4a201fe83196ba7a05", size = 19765, upload-time = "2025-12-03T13:19:38.143Z" }, + { url = "https://files.pythonhosted.org/packages/81/a3/cc9b66575bd6597b98b886a2067eea2693408d2d5f39dad9ab7fc264f5f3/opentelemetry_exporter_otlp_proto_grpc-1.39.1-py3-none-any.whl", hash = "sha256:fa1c136a05c7e9b4c09f739469cbdb927ea20b34088ab1d959a849b5cc589c18", size = 19766, upload-time = "2025-12-11T13:32:21.027Z" }, ] [[package]] name = "opentelemetry-exporter-otlp-proto-http" -version = "1.39.0" +version = "1.39.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "googleapis-common-protos" }, @@ -2408,14 +2447,14 @@ dependencies = [ { name = "requests" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/81/dc/1e9bf3f6a28e29eba516bc0266e052996d02bc7e92675f3cd38169607609/opentelemetry_exporter_otlp_proto_http-1.39.0.tar.gz", hash = "sha256:28d78fc0eb82d5a71ae552263d5012fa3ebad18dfd189bf8d8095ba0e65ee1ed", size = 17287, upload-time = "2025-12-03T13:20:01.134Z" } +sdist = { url = "https://files.pythonhosted.org/packages/80/04/2a08fa9c0214ae38880df01e8bfae12b067ec0793446578575e5080d6545/opentelemetry_exporter_otlp_proto_http-1.39.1.tar.gz", hash = "sha256:31bdab9745c709ce90a49a0624c2bd445d31a28ba34275951a6a362d16a0b9cb", size = 17288, upload-time = "2025-12-11T13:32:42.029Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bc/46/e4a102e17205bb05a50dbf24ef0e92b66b648cd67db9a68865af06a242fd/opentelemetry_exporter_otlp_proto_http-1.39.0-py3-none-any.whl", hash = "sha256:5789cb1375a8b82653328c0ce13a054d285f774099faf9d068032a49de4c7862", size = 19639, upload-time = "2025-12-03T13:19:39.536Z" }, + { url = "https://files.pythonhosted.org/packages/95/f1/b27d3e2e003cd9a3592c43d099d2ed8d0a947c15281bf8463a256db0b46c/opentelemetry_exporter_otlp_proto_http-1.39.1-py3-none-any.whl", hash = "sha256:d9f5207183dd752a412c4cd564ca8875ececba13be6e9c6c370ffb752fd59985", size = 19641, upload-time = "2025-12-11T13:32:22.248Z" }, ] [[package]] name = "opentelemetry-instrumentation" -version = "0.60b0" +version = "0.60b1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "opentelemetry-api" }, @@ -2423,14 +2462,14 @@ dependencies = [ { name = "packaging" }, { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/55/3c/bd53dbb42eff93d18e3047c7be11224aa9966ce98ac4cc5bfb860a32c95a/opentelemetry_instrumentation-0.60b0.tar.gz", hash = "sha256:4e9fec930f283a2677a2217754b40aaf9ef76edae40499c165bc7f1d15366a74", size = 31707, upload-time = "2025-12-03T13:22:00.352Z" } +sdist = { url = "https://files.pythonhosted.org/packages/41/0f/7e6b713ac117c1f5e4e3300748af699b9902a2e5e34c9cf443dde25a01fa/opentelemetry_instrumentation-0.60b1.tar.gz", hash = "sha256:57ddc7974c6eb35865af0426d1a17132b88b2ed8586897fee187fd5b8944bd6a", size = 31706, upload-time = "2025-12-11T13:36:42.515Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/7b/5b5b9f8cfe727a28553acf9cd287b1d7f706f5c0a00d6e482df55b169483/opentelemetry_instrumentation-0.60b0-py3-none-any.whl", hash = "sha256:aaafa1483543a402819f1bdfb06af721c87d60dd109501f9997332862a35c76a", size = 33096, upload-time = "2025-12-03T13:20:51.785Z" }, + { url = "https://files.pythonhosted.org/packages/77/d2/6788e83c5c86a2690101681aeef27eeb2a6bf22df52d3f263a22cee20915/opentelemetry_instrumentation-0.60b1-py3-none-any.whl", hash = "sha256:04480db952b48fb1ed0073f822f0ee26012b7be7c3eac1a3793122737c78632d", size = 33096, upload-time = "2025-12-11T13:35:33.067Z" }, ] [[package]] name = "opentelemetry-instrumentation-asgi" -version = "0.60b0" +version = "0.60b1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "asgiref" }, @@ -2439,14 +2478,14 @@ dependencies = [ { name = "opentelemetry-semantic-conventions" }, { name = "opentelemetry-util-http" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b0/0a/715ea7044708d3c215385fb2a1c6ffe429aacb3cd23a348060aaeda52834/opentelemetry_instrumentation_asgi-0.60b0.tar.gz", hash = "sha256:928731218050089dca69f0fe980b8bfe109f384be8b89802d7337372ddb67b91", size = 26083, upload-time = "2025-12-03T13:22:05.672Z" } +sdist = { url = "https://files.pythonhosted.org/packages/77/db/851fa88db7441da82d50bd80f2de5ee55213782e25dc858e04d0c9961d60/opentelemetry_instrumentation_asgi-0.60b1.tar.gz", hash = "sha256:16bfbe595cd24cda309a957456d0fc2523f41bc7b076d1f2d7e98a1ad9876d6f", size = 26107, upload-time = "2025-12-11T13:36:47.015Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9b/8c/c6c59127fd996107243ca45669355665a7daff578ddafb86d6d2d3b01428/opentelemetry_instrumentation_asgi-0.60b0-py3-none-any.whl", hash = "sha256:9d76a541269452c718a0384478f3291feb650c5a3f29e578fdc6613ea3729cf3", size = 16907, upload-time = "2025-12-03T13:20:58.962Z" }, + { url = "https://files.pythonhosted.org/packages/76/76/1fb94367cef64420d2171157a6b9509582873bd09a6afe08a78a8d1f59d9/opentelemetry_instrumentation_asgi-0.60b1-py3-none-any.whl", hash = "sha256:d48def2dbed10294c99cfcf41ebbd0c414d390a11773a41f472d20000fcddc25", size = 16933, upload-time = "2025-12-11T13:35:40.462Z" }, ] [[package]] name = "opentelemetry-instrumentation-httpx" -version = "0.60b0" +version = "0.60b1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "opentelemetry-api" }, @@ -2455,70 +2494,70 @@ dependencies = [ { name = "opentelemetry-util-http" }, { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/09/71/9dc0bc5ab14122251f520ffe9fc8dd892ca688d7d591482b5b1843d685d2/opentelemetry_instrumentation_httpx-0.60b0.tar.gz", hash = "sha256:fcf349a92fb0b941a2a18bec65141f4ba62cbf7a457a1aa580794bad44dc477c", size = 20612, upload-time = "2025-12-03T13:22:20.5Z" } +sdist = { url = "https://files.pythonhosted.org/packages/86/08/11208bcfcab4fc2023252c3f322aa397fd9ad948355fea60f5fc98648603/opentelemetry_instrumentation_httpx-0.60b1.tar.gz", hash = "sha256:a506ebaf28c60112cbe70ad4f0338f8603f148938cb7b6794ce1051cd2b270ae", size = 20611, upload-time = "2025-12-11T13:37:01.661Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3a/22/a340c8bfad6f31bfd6a15b0b24d5e68e05e3975e4dbdc3cea6ec4f96e060/opentelemetry_instrumentation_httpx-0.60b0-py3-none-any.whl", hash = "sha256:3f5e6fc4ddf1d9de2aaddb5255110827154dbd4de9187da906c8c2a3cc2219e9", size = 15702, upload-time = "2025-12-03T13:21:20.667Z" }, + { url = "https://files.pythonhosted.org/packages/43/59/b98e84eebf745ffc75397eaad4763795bff8a30cbf2373a50ed4e70646c5/opentelemetry_instrumentation_httpx-0.60b1-py3-none-any.whl", hash = "sha256:f37636dd742ad2af83d896ba69601ed28da51fa4e25d1ab62fde89ce413e275b", size = 15701, upload-time = "2025-12-11T13:36:04.56Z" }, ] [[package]] name = "opentelemetry-instrumentation-logging" -version = "0.60b0" +version = "0.60b1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "opentelemetry-api" }, { name = "opentelemetry-instrumentation" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5b/8b/a1aed0b695a58a1f0bdcf90fae5e468cc0fd7de2b74b9816b17e45c38e43/opentelemetry_instrumentation_logging-0.60b0.tar.gz", hash = "sha256:6d87840666669cbbcd53d2230c7a33476862d0bf7f1adc67a95519c6ccfc8281", size = 9969, upload-time = "2025-12-03T13:22:22.608Z" } +sdist = { url = "https://files.pythonhosted.org/packages/60/a6/4515895b383113677fd2ad21813df5e56108a2df14ebb7916c962c9a0234/opentelemetry_instrumentation_logging-0.60b1.tar.gz", hash = "sha256:98f4b9c7aeb9314a30feee7c002c7ea9abea07c90df5f97fb058b850bc45b89a", size = 9968, upload-time = "2025-12-11T13:37:03.974Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/00/89/6d4f5d8d03376637bff5b8d9e91356104a6f1ce9a34a2099bb2f4bf5e2b5/opentelemetry_instrumentation_logging-0.60b0-py3-none-any.whl", hash = "sha256:af75b3020911b9b6a1b4b19819a165eb131ce9bfdd313062d578fc2dc9a5cd0f", size = 12576, upload-time = "2025-12-03T13:21:24.797Z" }, + { url = "https://files.pythonhosted.org/packages/f1/f9/8a4ce3901bc52277794e4b18c4ac43dc5929806eff01d22812364132f45f/opentelemetry_instrumentation_logging-0.60b1-py3-none-any.whl", hash = "sha256:f2e18cbc7e1dd3628c80e30d243897fdc93c5b7e0c8ae60abd2b9b6a99f82343", size = 12577, upload-time = "2025-12-11T13:36:08.123Z" }, ] [[package]] name = "opentelemetry-proto" -version = "1.39.0" +version = "1.39.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "protobuf" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/48/b5/64d2f8c3393cd13ea2092106118f7b98461ba09333d40179a31444c6f176/opentelemetry_proto-1.39.0.tar.gz", hash = "sha256:c1fa48678ad1a1624258698e59be73f990b7fc1f39e73e16a9d08eef65dd838c", size = 46153, upload-time = "2025-12-03T13:20:08.729Z" } +sdist = { url = "https://files.pythonhosted.org/packages/49/1d/f25d76d8260c156c40c97c9ed4511ec0f9ce353f8108ca6e7561f82a06b2/opentelemetry_proto-1.39.1.tar.gz", hash = "sha256:6c8e05144fc0d3ed4d22c2289c6b126e03bcd0e6a7da0f16cedd2e1c2772e2c8", size = 46152, upload-time = "2025-12-11T13:32:48.681Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e3/4d/d500e1862beed68318705732d1976c390f4a72ca8009c4983ff627acff20/opentelemetry_proto-1.39.0-py3-none-any.whl", hash = "sha256:1e086552ac79acb501485ff0ce75533f70f3382d43d0a30728eeee594f7bf818", size = 72534, upload-time = "2025-12-03T13:19:50.251Z" }, + { url = "https://files.pythonhosted.org/packages/51/95/b40c96a7b5203005a0b03d8ce8cd212ff23f1793d5ba289c87a097571b18/opentelemetry_proto-1.39.1-py3-none-any.whl", hash = "sha256:22cdc78efd3b3765d09e68bfbd010d4fc254c9818afd0b6b423387d9dee46007", size = 72535, upload-time = "2025-12-11T13:32:33.866Z" }, ] [[package]] name = "opentelemetry-sdk" -version = "1.39.0" +version = "1.39.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "opentelemetry-api" }, { name = "opentelemetry-semantic-conventions" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/51/e3/7cd989003e7cde72e0becfe830abff0df55c69d237ee7961a541e0167833/opentelemetry_sdk-1.39.0.tar.gz", hash = "sha256:c22204f12a0529e07aa4d985f1bca9d6b0e7b29fe7f03e923548ae52e0e15dde", size = 171322, upload-time = "2025-12-03T13:20:09.651Z" } +sdist = { url = "https://files.pythonhosted.org/packages/eb/fb/c76080c9ba07e1e8235d24cdcc4d125ef7aa3edf23eb4e497c2e50889adc/opentelemetry_sdk-1.39.1.tar.gz", hash = "sha256:cf4d4563caf7bff906c9f7967e2be22d0d6b349b908be0d90fb21c8e9c995cc6", size = 171460, upload-time = "2025-12-11T13:32:49.369Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a4/b4/2adc8bc83eb1055ecb592708efb6f0c520cc2eb68970b02b0f6ecda149cf/opentelemetry_sdk-1.39.0-py3-none-any.whl", hash = "sha256:90cfb07600dfc0d2de26120cebc0c8f27e69bf77cd80ef96645232372709a514", size = 132413, upload-time = "2025-12-03T13:19:51.364Z" }, + { url = "https://files.pythonhosted.org/packages/7c/98/e91cf858f203d86f4eccdf763dcf01cf03f1dae80c3750f7e635bfa206b6/opentelemetry_sdk-1.39.1-py3-none-any.whl", hash = "sha256:4d5482c478513ecb0a5d938dcc61394e647066e0cc2676bee9f3af3f3f45f01c", size = 132565, upload-time = "2025-12-11T13:32:35.069Z" }, ] [[package]] name = "opentelemetry-semantic-conventions" -version = "0.60b0" +version = "0.60b1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "opentelemetry-api" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/71/0e/176a7844fe4e3cb5de604212094dffaed4e18b32f1c56b5258bcbcba85c2/opentelemetry_semantic_conventions-0.60b0.tar.gz", hash = "sha256:227d7aa73cbb8a2e418029d6b6465553aa01cf7e78ec9d0bc3255c7b3ac5bf8f", size = 137935, upload-time = "2025-12-03T13:20:12.395Z" } +sdist = { url = "https://files.pythonhosted.org/packages/91/df/553f93ed38bf22f4b999d9be9c185adb558982214f33eae539d3b5cd0858/opentelemetry_semantic_conventions-0.60b1.tar.gz", hash = "sha256:87c228b5a0669b748c76d76df6c364c369c28f1c465e50f661e39737e84bc953", size = 137935, upload-time = "2025-12-11T13:32:50.487Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d0/56/af0306666f91bae47db14d620775604688361f0f76a872e0005277311131/opentelemetry_semantic_conventions-0.60b0-py3-none-any.whl", hash = "sha256:069530852691136018087b52688857d97bba61cd641d0f8628d2d92788c4f78a", size = 219981, upload-time = "2025-12-03T13:19:53.585Z" }, + { url = "https://files.pythonhosted.org/packages/7a/5e/5958555e09635d09b75de3c4f8b9cae7335ca545d77392ffe7331534c402/opentelemetry_semantic_conventions-0.60b1-py3-none-any.whl", hash = "sha256:9fa8c8b0c110da289809292b0591220d3a7b53c1526a23021e977d68597893fb", size = 219982, upload-time = "2025-12-11T13:32:36.955Z" }, ] [[package]] name = "opentelemetry-util-http" -version = "0.60b0" +version = "0.60b1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/38/0d/786a713445cf338131fef3a84fab1378e4b2ef3c3ea348eeb0c915eb804a/opentelemetry_util_http-0.60b0.tar.gz", hash = "sha256:e42b7bb49bba43b6f34390327d97e5016eb1c47949ceaf37c4795472a4e3a82d", size = 10576, upload-time = "2025-12-03T13:22:41.224Z" } +sdist = { url = "https://files.pythonhosted.org/packages/50/fc/c47bb04a1d8a941a4061307e1eddfa331ed4d0ab13d8a9781e6db256940a/opentelemetry_util_http-0.60b1.tar.gz", hash = "sha256:0d97152ca8c8a41ced7172d29d3622a219317f74ae6bb3027cfbdcf22c3cc0d6", size = 11053, upload-time = "2025-12-11T13:37:25.115Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/53/5d/a448862f6d10c95685ed0e703596b6bd1784074e7ad90bffdc550abb7b68/opentelemetry_util_http-0.60b0-py3-none-any.whl", hash = "sha256:4f366f1a48adb74ffa6f80aee26f96882e767e01b03cd1cfb948b6e1020341fe", size = 8742, upload-time = "2025-12-03T13:21:54.553Z" }, + { url = "https://files.pythonhosted.org/packages/16/5c/d3f1733665f7cd582ef0842fb1d2ed0bc1fba10875160593342d22bba375/opentelemetry_util_http-0.60b1-py3-none-any.whl", hash = "sha256:66381ba28550c91bee14dcba8979ace443444af1ed609226634596b4b0faf199", size = 8947, upload-time = "2025-12-11T13:36:37.151Z" }, ] [[package]] @@ -3314,7 +3353,7 @@ wheels = [ [[package]] name = "pytest-otel" -version = "2.0.1" +version = "2.0.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "opentelemetry-api" }, @@ -3322,9 +3361,9 @@ dependencies = [ { name = "opentelemetry-sdk" }, { name = "pytest" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ca/5e/771f8dbdf55ae57603d89bb26e046c13e8bee3492c74cb3afb6044166e9a/pytest_otel-2.0.1.tar.gz", hash = "sha256:3d529dc34105862cca39fd1258d00dd3d17f3b7d92ebf0953d55326bb017af3d", size = 17880, upload-time = "2025-12-08T14:56:40.393Z" } +sdist = { url = "https://files.pythonhosted.org/packages/98/14/e4f7cb90c4c93dbdd85034fd65e7a16da328bd71c6ca5270e4f8d37fa4ba/pytest_otel-2.0.3.tar.gz", hash = "sha256:782985fef50acc6922db5e69da4584b384d785c863a96ebc11eaf2a7fef5189c", size = 18587, upload-time = "2026-01-02T16:12:20.394Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/16/fa/0cfe23bac571f68b4f427ad7ae313ddd8de881ec2ff49dcea196a2a96592/pytest_otel-2.0.1-py2.py3-none-any.whl", hash = "sha256:501f36f02f55578ca34c3ccbe55e81cc1e14bb130da13f7cf0b6480d54a9e5db", size = 14530, upload-time = "2025-12-08T14:56:41.731Z" }, + { url = "https://files.pythonhosted.org/packages/ac/f8/d9b93b41b299c7e14b47887eb31fb4cb77d771c2e9f3e0bcac0750f62699/pytest_otel-2.0.3-py2.py3-none-any.whl", hash = "sha256:3c6c331e943609ad7df7c718714090dee91211340f133a8798217e72e3b2cd67", size = 14914, upload-time = "2026-01-02T16:12:21.946Z" }, ] [[package]]