Merge remote-tracking branch 'origin/master' into fix/qdrant-doc-id-keyword-index

This commit is contained in:
Chris Coutinho
2026-05-08 23:08:39 +02:00
23 changed files with 1619 additions and 262 deletions
+1
View File
@@ -7,6 +7,7 @@ description: |
in this repo's automated PR reviews. Use when the user is about to push, says "ready in this repo's automated PR reviews. Use when the user is about to push, says "ready
to push", "review my work", "check before PR", or invokes /pre-push-review. to push", "review my work", "check before PR", or invokes /pre-push-review.
Report-only — does not modify code. Report-only — does not modify code.
model: sonnet
allowed-tools: allowed-tools:
- Bash - Bash
- Read - Read
+24
View File
@@ -5,6 +5,30 @@ All notable changes to the Nextcloud MCP Server will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [PEP 440](https://peps.python.org/pep-0440/). and this project adheres to [PEP 440](https://peps.python.org/pep-0440/).
## v0.83.0 (2026-05-08)
### Feat
- **vector**: replace inline page-image payloads with chunk_bbox (Deck #76)
### Refactor
- **vector**: address PR #775 review round 3 — fix unused var, harden boundary lookup, rename trace span
- **vector**: address PR #775 review round 2 — drop dead page field, add omission tests
- **vector**: address PR #775 review — drop unused payload key, fix resource leaks
## v0.82.0 (2026-05-08)
### Feat
- **providers**: add Mistral embedding provider, route registry through dynaconf
### Refactor
- **providers**: address PR #772 review round 3 — hermetic test, lazy logging, defensive-guard tests
- **providers**: address PR #772 review round 2 — guard, naming, docs, tests
- **providers**: address PR #772 review — shared retry, cleaner imports, no-op close
## v0.81.0 (2026-05-07) ## v0.81.0 (2026-05-07)
### Feat ### Feat
+1 -1
View File
@@ -334,7 +334,7 @@ services:
- claude-funnel - claude-funnel
qdrant: qdrant:
image: docker.io/qdrant/qdrant:v1.17.1@sha256:94728574965d17c6485dd361aa3c0818b325b9016dac5ea6afec7b4b2700865f image: docker.io/qdrant/qdrant:v1.18.0@sha256:1cf3e07c9f269030c7cfed0d5c0beea7bb081848a88e2ae13b35a613d4dd5019
restart: always restart: always
ports: ports:
- 127.0.0.1:6333:6333 # REST API - 127.0.0.1:6333:6333 # REST API
+19 -2
View File
@@ -118,10 +118,16 @@ class ProviderRegistry:
@staticmethod @staticmethod
def create_provider() -> Provider: def create_provider() -> Provider:
# 1. Bedrock (AWS_REGION or BEDROCK_*_MODEL) # 1. Bedrock (AWS_REGION or BEDROCK_*_MODEL)
# 2. Ollama (OLLAMA_BASE_URL) # 2. OpenAI (OPENAI_API_KEY)
# 3. Simple (fallback) # 3. Mistral (MISTRAL_API_KEY)
# 4. Ollama (OLLAMA_BASE_URL)
# 5. Simple (fallback)
``` ```
Configuration is sourced via the dynaconf-backed `Settings` dataclass in
`config.py`; the registry reads `get_settings()` rather than `os.getenv`
directly, so settings files and env vars share one resolution path.
**Environment Variables:** **Environment Variables:**
**Bedrock:** **Bedrock:**
@@ -131,6 +137,17 @@ class ProviderRegistry:
- `BEDROCK_EMBEDDING_MODEL`: Model ID for embeddings (e.g., "amazon.titan-embed-text-v2:0") - `BEDROCK_EMBEDDING_MODEL`: Model ID for embeddings (e.g., "amazon.titan-embed-text-v2:0")
- `BEDROCK_GENERATION_MODEL`: Model ID for text generation (e.g., "anthropic.claude-3-sonnet-20240229-v1:0") - `BEDROCK_GENERATION_MODEL`: Model ID for text generation (e.g., "anthropic.claude-3-sonnet-20240229-v1:0")
**OpenAI:**
- `OPENAI_API_KEY`: OpenAI API key (or `GITHUB_TOKEN` for GitHub Models)
- `OPENAI_BASE_URL`: Optional base URL override for OpenAI-compatible APIs
- `OPENAI_EMBEDDING_MODEL`: Embedding model (default: "text-embedding-3-small")
- `OPENAI_GENERATION_MODEL`: Generation model (e.g., "gpt-4o-mini")
**Mistral (embeddings only):**
- `MISTRAL_API_KEY`: Mistral API key from console.mistral.ai
- `MISTRAL_EMBEDDING_MODEL`: Embedding model (default: "mistral-embed", 1024-dim)
- `MISTRAL_BASE_URL`: Optional server URL override (proxies, on-prem)
**Ollama:** **Ollama:**
- `OLLAMA_BASE_URL`: Ollama API base URL (e.g., "http://localhost:11434") - `OLLAMA_BASE_URL`: Ollama API base URL (e.g., "http://localhost:11434")
- `OLLAMA_EMBEDDING_MODEL`: Model for embeddings (default: "nomic-embed-text") - `OLLAMA_EMBEDDING_MODEL`: Model for embeddings (default: "nomic-embed-text")
+9 -9
View File
@@ -553,9 +553,10 @@ async def get_chunk_context(request: Request) -> JSONResponse:
status_code=404, status_code=404,
) )
# For PDF files, also fetch the highlighted page image from Qdrant if available # For PDF files, also fetch the chunk's bounding box from Qdrant if
# This is useful for clients that want to show a pre-rendered image # available so the client can overlay a highlight on top of a
highlighted_page_image = None # render-on-demand page image (Deck #76).
chunk_bbox = None
page_number = chunk_context.page_number page_number = chunk_context.page_number
if doc_type == "file": if doc_type == "file":
@@ -563,7 +564,6 @@ async def get_chunk_context(request: Request) -> JSONResponse:
settings = get_settings() settings = get_settings()
qdrant_client = await get_qdrant_client() qdrant_client = await get_qdrant_client()
# Query for this specific chunk's highlighted image
points_response = await qdrant_client.scroll( points_response = await qdrant_client.scroll(
collection_name=settings.get_collection_name(), collection_name=settings.get_collection_name(),
scroll_filter=Filter( scroll_filter=Filter(
@@ -585,19 +585,19 @@ async def get_chunk_context(request: Request) -> JSONResponse:
), ),
limit=1, limit=1,
with_vectors=False, with_vectors=False,
with_payload=["highlighted_page_image", "page_number"], with_payload=["chunk_bbox", "page_number"],
) )
if points_response[0]: if points_response[0]:
payload = points_response[0][0].payload payload = points_response[0][0].payload
if payload: if payload:
highlighted_page_image = payload.get("highlighted_page_image") chunk_bbox = payload.get("chunk_bbox")
# Trust Qdrant page number if available (might be more accurate than context expansion logic) # Trust Qdrant page number if available (might be more accurate than context expansion logic)
if payload.get("page_number") is not None: if payload.get("page_number") is not None:
page_number = payload.get("page_number") page_number = payload.get("page_number")
except Exception as e: except Exception as e:
logger.warning(f"Failed to fetch highlighted image: {e}") logger.warning(f"Failed to fetch chunk bbox: {e}")
# Build response # Build response
response_data = { response_data = {
@@ -612,8 +612,8 @@ async def get_chunk_context(request: Request) -> JSONResponse:
"total_chunks": chunk_context.total_chunks, "total_chunks": chunk_context.total_chunks,
} }
if highlighted_page_image: if chunk_bbox:
response_data["highlighted_page_image"] = highlighted_page_image response_data["chunk_bbox"] = chunk_bbox
return JSONResponse(response_data) return JSONResponse(response_data)
+2 -1
View File
@@ -125,12 +125,13 @@ from nextcloud_mcp_server.server import (
) )
from nextcloud_mcp_server.server.auth_tools import register_auth_tools from nextcloud_mcp_server.server.auth_tools import register_auth_tools
from nextcloud_mcp_server.server.oauth_tools import register_oauth_tools from nextcloud_mcp_server.server.oauth_tools import register_oauth_tools
from nextcloud_mcp_server.vector import processor_task, scanner_task
from nextcloud_mcp_server.vector.oauth_sync import ( from nextcloud_mcp_server.vector.oauth_sync import (
oauth_processor_task, oauth_processor_task,
user_manager_task, user_manager_task,
) )
from nextcloud_mcp_server.vector.processor import processor_task
from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client
from nextcloud_mcp_server.vector.scanner import scanner_task
from nextcloud_mcp_server.vector.webhook_receiver import handle_nextcloud_webhook from nextcloud_mcp_server.vector.webhook_receiver import handle_nextcloud_webhook
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
+12 -14
View File
@@ -607,8 +607,10 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse:
f"after_len={len(chunk_context.after_context)}" f"after_len={len(chunk_context.after_context)}"
) )
# For PDF files, also fetch the highlighted page image from Qdrant # For PDF files, also fetch the chunk bbox from Qdrant so the client
highlighted_page_image = None # can overlay a highlight on top of a render-on-demand page image
# (Deck #76).
chunk_bbox = None
page_number = None page_number = None
if doc_type == "file": if doc_type == "file":
try: try:
@@ -616,7 +618,6 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse:
qdrant_client = await get_qdrant_client() qdrant_client = await get_qdrant_client()
username = request.user.display_name username = request.user.display_name
# Query for this specific chunk's highlighted image
points_response = await qdrant_client.scroll( points_response = await qdrant_client.scroll(
collection_name=settings.get_collection_name(), collection_name=settings.get_collection_name(),
scroll_filter=Filter( scroll_filter=Filter(
@@ -638,22 +639,20 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse:
), ),
limit=1, limit=1,
with_vectors=False, with_vectors=False,
with_payload=["highlighted_page_image", "page_number"], with_payload=["chunk_bbox", "page_number"],
) )
points = points_response[0] points = points_response[0]
if points and points[0].payload: if points and points[0].payload:
highlighted_page_image = points[0].payload.get( chunk_bbox = points[0].payload.get("chunk_bbox")
"highlighted_page_image"
)
page_number = points[0].payload.get("page_number") page_number = points[0].payload.get("page_number")
if highlighted_page_image: if chunk_bbox:
logger.info( logger.info(
f"Found highlighted image for chunk: " f"Found chunk bbox: page={page_number}, "
f"page={page_number}, image_size={len(highlighted_page_image)}" f"rects={len(chunk_bbox)}"
) )
except Exception as e: except Exception as e:
logger.warning(f"Failed to fetch highlighted image: {e}") logger.warning(f"Failed to fetch chunk bbox: {e}")
# Return response compatible with frontend expectations # Return response compatible with frontend expectations
response_data: dict = { response_data: dict = {
@@ -665,9 +664,8 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse:
"has_more_after": chunk_context.has_after_truncation, "has_more_after": chunk_context.has_after_truncation,
} }
# Add image data if available if chunk_bbox:
if highlighted_page_image: response_data["chunk_bbox"] = chunk_bbox
response_data["highlighted_page_image"] = highlighted_page_image
response_data["page_number"] = page_number response_data["page_number"] = page_number
return JSONResponse(response_data) return JSONResponse(response_data)
+63 -9
View File
@@ -77,11 +77,25 @@ _DEFAULTS: dict[str, Any] = {
# Ollama # Ollama
"ollama_base_url": None, "ollama_base_url": None,
"ollama_embedding_model": "nomic-embed-text", "ollama_embedding_model": "nomic-embed-text",
"ollama_generation_model": None,
"ollama_verify_ssl": True, "ollama_verify_ssl": True,
# OpenAI # OpenAI
"openai_api_key": None, "openai_api_key": None,
"openai_base_url": None, "openai_base_url": None,
"openai_embedding_model": "text-embedding-3-small", "openai_embedding_model": "text-embedding-3-small",
"openai_generation_model": None,
# Bedrock (AWS)
"aws_region": None,
"aws_access_key_id": None,
"aws_secret_access_key": None,
"bedrock_embedding_model": None,
"bedrock_generation_model": None,
# Mistral
"mistral_api_key": None,
"mistral_embedding_model": "mistral-embed",
"mistral_base_url": None,
# Simple (fallback) embedding dimension
"simple_embedding_dimension": 384,
# Document chunking # Document chunking
"document_chunk_size": 2048, "document_chunk_size": 2048,
"document_chunk_overlap": 200, "document_chunk_overlap": 200,
@@ -486,15 +500,32 @@ class Settings:
qdrant_api_key: str | None = None qdrant_api_key: str | None = None
qdrant_collection: str = "nextcloud_content" qdrant_collection: str = "nextcloud_content"
# Ollama settings (for embeddings) # Ollama settings (embeddings + optional generation)
ollama_base_url: str | None = None ollama_base_url: str | None = None
ollama_embedding_model: str = "nomic-embed-text" ollama_embedding_model: str = "nomic-embed-text"
ollama_generation_model: str | None = None
ollama_verify_ssl: bool = True ollama_verify_ssl: bool = True
# OpenAI settings (for embeddings) # OpenAI settings (embeddings + optional generation)
openai_api_key: str | None = None openai_api_key: str | None = None
openai_base_url: str | None = None openai_base_url: str | None = None
openai_embedding_model: str = "text-embedding-3-small" openai_embedding_model: str = "text-embedding-3-small"
openai_generation_model: str | None = None
# Bedrock (AWS) settings — boto3 also reads these from its credential chain
aws_region: str | None = None
aws_access_key_id: str | None = None
aws_secret_access_key: str | None = None
bedrock_embedding_model: str | None = None
bedrock_generation_model: str | None = None
# Mistral settings (embeddings only)
mistral_api_key: str | None = None
mistral_embedding_model: str = "mistral-embed"
mistral_base_url: str | None = None
# Simple (fallback) provider — dimension when no real provider configured
simple_embedding_dimension: int = 384
# Document chunking settings (for vector embeddings) # Document chunking settings (for vector embeddings)
document_chunk_size: int = 2048 # Characters per chunk document_chunk_size: int = 2048 # Characters per chunk
@@ -573,23 +604,32 @@ class Settings:
Get the active embedding model name based on provider priority. Get the active embedding model name based on provider priority.
Priority order (same as ProviderRegistry): Priority order (same as ProviderRegistry):
1. OpenAI - if OPENAI_API_KEY is set 1. Bedrock - if AWS_REGION or BEDROCK_EMBEDDING_MODEL is set
2. Ollama - if OLLAMA_BASE_URL is set 2. OpenAI - if OPENAI_API_KEY is set
3. Simple - fallback (returns "simple-384") 3. Mistral - if MISTRAL_API_KEY is set
4. Ollama - if OLLAMA_BASE_URL is set
5. Simple - fallback (returns "simple-{dimension}")
Returns: Returns:
Active embedding model name Active embedding model name
""" """
# Check OpenAI first (higher priority than Ollama in registry) if (
self.aws_region
or self.bedrock_embedding_model
or self.bedrock_generation_model
):
return self.bedrock_embedding_model or "bedrock-default"
if self.openai_api_key: if self.openai_api_key:
return self.openai_embedding_model return self.openai_embedding_model
# Check Ollama if self.mistral_api_key:
return self.mistral_embedding_model
if self.ollama_base_url: if self.ollama_base_url:
return self.ollama_embedding_model return self.ollama_embedding_model
# Fallback to simple provider indicator return f"simple-{self.simple_embedding_dimension}"
return "simple-384"
def get_collection_name(self) -> str: def get_collection_name(self) -> str:
""" """
@@ -835,11 +875,25 @@ def get_settings() -> Settings:
# Ollama settings # Ollama settings
"ollama_base_url": "OLLAMA_BASE_URL", "ollama_base_url": "OLLAMA_BASE_URL",
"ollama_embedding_model": "OLLAMA_EMBEDDING_MODEL", "ollama_embedding_model": "OLLAMA_EMBEDDING_MODEL",
"ollama_generation_model": "OLLAMA_GENERATION_MODEL",
"ollama_verify_ssl": "OLLAMA_VERIFY_SSL", "ollama_verify_ssl": "OLLAMA_VERIFY_SSL",
# OpenAI settings # OpenAI settings
"openai_api_key": "OPENAI_API_KEY", "openai_api_key": "OPENAI_API_KEY",
"openai_base_url": "OPENAI_BASE_URL", "openai_base_url": "OPENAI_BASE_URL",
"openai_embedding_model": "OPENAI_EMBEDDING_MODEL", "openai_embedding_model": "OPENAI_EMBEDDING_MODEL",
"openai_generation_model": "OPENAI_GENERATION_MODEL",
# Bedrock (AWS) settings
"aws_region": "AWS_REGION",
"aws_access_key_id": "AWS_ACCESS_KEY_ID",
"aws_secret_access_key": "AWS_SECRET_ACCESS_KEY",
"bedrock_embedding_model": "BEDROCK_EMBEDDING_MODEL",
"bedrock_generation_model": "BEDROCK_GENERATION_MODEL",
# Mistral settings
"mistral_api_key": "MISTRAL_API_KEY",
"mistral_embedding_model": "MISTRAL_EMBEDDING_MODEL",
"mistral_base_url": "MISTRAL_BASE_URL",
# Simple provider
"simple_embedding_dimension": "SIMPLE_EMBEDDING_DIMENSION",
# Document chunking settings # Document chunking settings
"document_chunk_size": "DOCUMENT_CHUNK_SIZE", "document_chunk_size": "DOCUMENT_CHUNK_SIZE",
"document_chunk_overlap": "DOCUMENT_CHUNK_OVERLAP", "document_chunk_overlap": "DOCUMENT_CHUNK_OVERLAP",
@@ -3,6 +3,7 @@
from .anthropic import AnthropicProvider from .anthropic import AnthropicProvider
from .base import Provider from .base import Provider
from .bedrock import BedrockProvider from .bedrock import BedrockProvider
from .mistral import MistralProvider
from .ollama import OllamaProvider from .ollama import OllamaProvider
from .openai import OpenAIProvider from .openai import OpenAIProvider
from .registry import get_provider, reset_provider from .registry import get_provider, reset_provider
@@ -13,6 +14,7 @@ __all__ = [
"OllamaProvider", "OllamaProvider",
"OpenAIProvider", "OpenAIProvider",
"AnthropicProvider", "AnthropicProvider",
"MistralProvider",
"SimpleProvider", "SimpleProvider",
"BedrockProvider", "BedrockProvider",
"get_provider", "get_provider",
+78
View File
@@ -0,0 +1,78 @@
"""Shared rate-limit retry helper for provider modules.
OpenAI and Mistral both retry on 429 with the same exponential-backoff curve;
extracting the loop here keeps the two provider modules thin and lets future
providers (Bedrock throttling, etc.) reuse the same primitive.
"""
from __future__ import annotations
import logging
from collections.abc import Awaitable, Callable
from functools import wraps
from typing import Any, TypeVar
import anyio
logger = logging.getLogger(__name__)
MAX_RETRIES = 5
INITIAL_RETRY_DELAY = 2.0
MAX_RETRY_DELAY = 60.0
T = TypeVar("T")
def retry_on_rate_limit(
exception_type: type[BaseException],
is_rate_limit: Callable[[BaseException], bool] = lambda _exc: True,
*,
provider_name: str = "provider",
) -> Callable[[Callable[..., Awaitable[T]]], Callable[..., Awaitable[T]]]:
"""Build a decorator that retries on rate-limit exceptions.
Args:
exception_type: Catch this exception class (e.g. ``openai.RateLimitError``,
``mistralai.client.errors.SDKError``).
is_rate_limit: Predicate that decides whether a caught exception is
actually a rate-limit (vs. some other error of the same class).
Defaults to "always True" — appropriate when ``exception_type`` is
already a rate-limit-specific class.
provider_name: Used in log messages so operators can tell which
provider exhausted retries.
"""
def decorator(func: Callable[..., Awaitable[T]]) -> Callable[..., Awaitable[T]]:
@wraps(func)
async def wrapper(*args: Any, **kwargs: Any) -> T:
retry_delay = INITIAL_RETRY_DELAY
last_error: BaseException | None = None
for attempt in range(1, MAX_RETRIES + 1):
try:
return await func(*args, **kwargs)
except exception_type as e:
if not is_rate_limit(e):
raise
last_error = e
if attempt < MAX_RETRIES:
logger.warning(
"%s rate limit hit (attempt %d/%d), retrying in %.1fs...",
provider_name,
attempt,
MAX_RETRIES,
retry_delay,
)
await anyio.sleep(retry_delay)
retry_delay = min(retry_delay * 2, MAX_RETRY_DELAY)
logger.error(
"%s rate limit exceeded after %d attempts", provider_name, MAX_RETRIES
)
if last_error is None: # pragma: no cover — loop above always sets this
raise RuntimeError("retry loop exited without capturing an error")
raise last_error
return wrapper
return decorator
+199
View File
@@ -0,0 +1,199 @@
"""Mistral provider for embeddings.
Currently supports embeddings only (``mistral-embed``, 1024-dim). Generation
can be added later if needed; see ADR-015.
"""
import logging
# mistralai 2.x ships no top-level __init__.py, so `from mistralai import …`
# raises ImportError. The canonical public paths are `mistralai.client` (which
# re-exports the SDK class via `client/__init__.py`) and `mistralai.client.errors`
# (which lazy-loads SDKError). There is no `mistralai.models` subpackage either.
from mistralai.client import Mistral
from mistralai.client.errors import SDKError
from ._retry import retry_on_rate_limit
from .base import Provider
logger = logging.getLogger(__name__)
# Well-known Mistral embedding model dimensions
MISTRAL_EMBEDDING_DIMENSIONS: dict[str, int] = {
"mistral-embed": 1024,
}
# Conservative chunk size for batch embeddings. Mistral allows large batches,
# but we keep this in line with sibling providers (OpenAI=100, Ollama=32).
BATCH_SIZE = 64
_NO_EMBEDDING_MODEL_MSG = "Embedding not supported - no embedding_model configured"
def _is_rate_limit(exc: BaseException) -> bool:
"""True only for HTTP 429 SDKErrors."""
return getattr(exc, "status_code", None) == 429
_retry_429 = retry_on_rate_limit(
SDKError, is_rate_limit=_is_rate_limit, provider_name="Mistral"
)
class MistralProvider(Provider):
"""
Mistral provider — embeddings only.
Uses the official ``mistralai`` SDK. Lazy dimension detection mirrors the
OpenAI provider: known models populate the cached dimension at construction
time; unknown models get their dimension detected on the first ``embed()``
call.
"""
def __init__(
self,
api_key: str,
embedding_model: str | None = "mistral-embed",
base_url: str | None = None,
):
"""
Initialize the Mistral provider.
Args:
api_key: Mistral API key.
embedding_model: Embedding model ID (default: ``mistral-embed``).
Pass ``None`` to disable embeddings (the provider will then
support no capabilities, which is mostly useful for tests).
base_url: Optional base URL override (e.g. proxies, on-prem).
"""
self.embedding_model = embedding_model
self._dimension: int | None = None
self.client = Mistral(api_key=api_key, server_url=base_url)
if embedding_model and embedding_model in MISTRAL_EMBEDDING_DIMENSIONS:
self._dimension = MISTRAL_EMBEDDING_DIMENSIONS[embedding_model]
logger.info(
"Initialized Mistral provider: base_url=%s, embedding_model=%s, "
"dimension=%s",
base_url or "default",
embedding_model,
self._dimension,
)
@property
def supports_embeddings(self) -> bool:
return self.embedding_model is not None
@property
def supports_generation(self) -> bool:
return False
@_retry_429
async def embed(self, text: str) -> list[float]:
"""Generate an embedding for a single text."""
if not self.supports_embeddings:
raise NotImplementedError(_NO_EMBEDDING_MODEL_MSG)
assert self.embedding_model is not None
response = await self.client.embeddings.create_async(
model=self.embedding_model,
inputs=[text],
)
if not response.data or response.data[0].embedding is None:
raise RuntimeError(
f"Mistral embeddings API returned no embedding for model "
f"{self.embedding_model}"
)
embedding = response.data[0].embedding
if self._dimension is None:
self._dimension = len(embedding)
logger.info(
"Detected embedding dimension: %d for model %s",
self._dimension,
self.embedding_model,
)
return embedding
async def embed_batch(self, texts: list[str]) -> list[list[float]]:
"""Generate embeddings for multiple texts, chunking by ``BATCH_SIZE``."""
if not self.supports_embeddings:
raise NotImplementedError(_NO_EMBEDDING_MODEL_MSG)
if not texts:
return []
all_embeddings: list[list[float]] = []
for i in range(0, len(texts), BATCH_SIZE):
batch = texts[i : i + BATCH_SIZE]
batch_embeddings = await self._embed_batch_request(batch)
all_embeddings.extend(batch_embeddings)
if self._dimension is None and batch_embeddings:
self._dimension = len(batch_embeddings[0])
logger.info(
"Detected embedding dimension: %d for model %s",
self._dimension,
self.embedding_model,
)
return all_embeddings
@_retry_429
async def _embed_batch_request(self, batch: list[str]) -> list[list[float]]:
"""Single batch request with rate-limit retry."""
assert self.embedding_model is not None
response = await self.client.embeddings.create_async(
model=self.embedding_model,
inputs=batch,
)
# Defensive: response.data items have Optional fields. Sort by index
# (default 0 if missing) and reject None embeddings explicitly.
sorted_data = sorted(response.data or [], key=lambda x: x.index or 0)
result: list[list[float]] = []
for item in sorted_data:
if item.embedding is None:
raise RuntimeError(
f"Mistral embeddings API returned a null embedding for "
f"model {self.embedding_model}"
)
result.append(item.embedding)
if len(result) != len(batch):
raise RuntimeError(
f"Mistral embeddings API returned {len(result)} embeddings "
f"for {len(batch)} inputs"
)
return result
def get_dimension(self) -> int:
if not self.supports_embeddings:
raise NotImplementedError(_NO_EMBEDDING_MODEL_MSG)
if self._dimension is None:
raise RuntimeError(
f"Embedding dimension not detected yet for model "
f"{self.embedding_model}. Call embed() first or use a known "
"model."
)
return self._dimension
async def generate(self, prompt: str, max_tokens: int = 500) -> str:
raise NotImplementedError(
"MistralProvider does not support generation. "
"Use OpenAI, Anthropic, or Bedrock for text generation."
)
async def close(self) -> None:
# The mistralai 2.x client (Speakeasy-generated) does not expose a
# public close()/aclose() — only the async-context-manager protocol
# (__aenter__/__aexit__). Calling __aexit__ directly is internal API
# and brittle across SDK patch versions; the underlying httpx client
# is closed during garbage collection, so we leave this as a no-op.
return None
+19 -43
View File
@@ -7,46 +7,17 @@ Supports:
""" """
import logging import logging
from functools import wraps
import anyio
from openai import AsyncOpenAI, RateLimitError from openai import AsyncOpenAI, RateLimitError
from ._retry import retry_on_rate_limit
from .base import Provider from .base import Provider
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# Rate limit retry configuration # OpenAI's RateLimitError is itself a 429-specific class, so the default
MAX_RETRIES = 5 # is_rate_limit predicate ("always True") matches the previous behavior.
INITIAL_RETRY_DELAY = 2.0 # seconds _retry_429 = retry_on_rate_limit(RateLimitError, provider_name="OpenAI")
MAX_RETRY_DELAY = 60.0 # seconds
def retry_on_rate_limit(func):
"""Decorator to retry on OpenAI rate limit errors with exponential backoff."""
@wraps(func)
async def wrapper(*args, **kwargs):
retry_delay = INITIAL_RETRY_DELAY
last_error: Exception | None = None
for attempt in range(1, MAX_RETRIES + 1):
try:
return await func(*args, **kwargs)
except RateLimitError as e:
last_error = e
if attempt < MAX_RETRIES:
logger.warning(
f"Rate limit hit (attempt {attempt}/{MAX_RETRIES}), "
f"retrying in {retry_delay:.1f}s..."
)
await anyio.sleep(retry_delay)
retry_delay = min(retry_delay * 2, MAX_RETRY_DELAY)
logger.error(f"Rate limit exceeded after {MAX_RETRIES} attempts")
raise last_error # type: ignore[misc]
return wrapper
# Well-known embedding dimensions for OpenAI models # Well-known embedding dimensions for OpenAI models
@@ -106,9 +77,12 @@ class OpenAIProvider(Provider):
self._dimension = OPENAI_EMBEDDING_DIMENSIONS[embedding_model] self._dimension = OPENAI_EMBEDDING_DIMENSIONS[embedding_model]
logger.info( logger.info(
f"Initialized OpenAI provider: base_url={base_url or 'default'} " "Initialized OpenAI provider: base_url=%s "
f"(embedding_model={embedding_model}, generation_model={generation_model}, " "(embedding_model=%s, generation_model=%s, dimension=%s)",
f"dimension={self._dimension})" base_url or "default",
embedding_model,
generation_model,
self._dimension,
) )
@property @property
@@ -121,7 +95,7 @@ class OpenAIProvider(Provider):
"""Whether this provider supports text generation.""" """Whether this provider supports text generation."""
return self.generation_model is not None return self.generation_model is not None
@retry_on_rate_limit @_retry_429
async def embed(self, text: str) -> list[float]: async def embed(self, text: str) -> list[float]:
""" """
Generate embedding vector for text. Generate embedding vector for text.
@@ -152,8 +126,9 @@ class OpenAIProvider(Provider):
if self._dimension is None: if self._dimension is None:
self._dimension = len(embedding) self._dimension = len(embedding)
logger.info( logger.info(
f"Detected embedding dimension: {self._dimension} " "Detected embedding dimension: %d for model %s",
f"for model {self.embedding_model}" self._dimension,
self.embedding_model,
) )
return embedding return embedding
@@ -196,13 +171,14 @@ class OpenAIProvider(Provider):
if self._dimension is None and batch_embeddings: if self._dimension is None and batch_embeddings:
self._dimension = len(batch_embeddings[0]) self._dimension = len(batch_embeddings[0])
logger.info( logger.info(
f"Detected embedding dimension: {self._dimension} " "Detected embedding dimension: %d for model %s",
f"for model {self.embedding_model}" self._dimension,
self.embedding_model,
) )
return all_embeddings return all_embeddings
@retry_on_rate_limit @_retry_429
async def _embed_batch_request(self, batch: list[str]) -> list[list[float]]: async def _embed_batch_request(self, batch: list[str]) -> list[list[float]]:
"""Make a single batch embedding request with retry logic.""" """Make a single batch embedding request with retry logic."""
assert self.embedding_model is not None # Type narrowing assert self.embedding_model is not None # Type narrowing
@@ -237,7 +213,7 @@ class OpenAIProvider(Provider):
) )
return self._dimension return self._dimension
@retry_on_rate_limit @_retry_429
async def generate(self, prompt: str, max_tokens: int = 500) -> str: async def generate(self, prompt: str, max_tokens: int = 500) -> str:
""" """
Generate text from a prompt. Generate text from a prompt.
+78 -82
View File
@@ -1,10 +1,11 @@
"""Provider registry and factory for auto-detection and instantiation.""" """Provider registry and factory for auto-detection and instantiation."""
import logging import logging
import os
from ..config import get_settings
from .base import Provider from .base import Provider
from .bedrock import BedrockProvider from .bedrock import BedrockProvider
from .mistral import MistralProvider
from .ollama import OllamaProvider from .ollama import OllamaProvider
from .openai import OpenAIProvider from .openai import OpenAIProvider
from .simple import SimpleProvider from .simple import SimpleProvider
@@ -16,117 +17,112 @@ class ProviderRegistry:
""" """
Registry for provider auto-detection and instantiation. Registry for provider auto-detection and instantiation.
Checks environment variables in priority order and creates appropriate provider: Reads configuration via dynaconf-backed Settings (see ``config.py``).
1. Bedrock (AWS_REGION + BEDROCK_*_MODEL) Checks provider settings in priority order and creates the appropriate
2. OpenAI (OPENAI_API_KEY) provider:
3. Ollama (OLLAMA_BASE_URL)
4. Simple (fallback for testing/development) 1. Bedrock (``AWS_REGION`` or ``BEDROCK_*_MODEL``)
2. OpenAI (``OPENAI_API_KEY``)
3. Mistral (``MISTRAL_API_KEY``)
4. Ollama (``OLLAMA_BASE_URL``)
5. Simple (fallback for testing/development)
""" """
@staticmethod @staticmethod
def create_provider() -> Provider: def create_provider() -> Provider:
""" """
Auto-detect and create provider based on environment variables. Auto-detect and create provider based on configured settings.
Settings are sourced via :func:`nextcloud_mcp_server.config.get_settings`,
which reads from settings files and environment variables (env vars
always win, see ADR-024/025).
Priority order: Priority order:
1. Bedrock - if AWS_REGION or BEDROCK_EMBEDDING_MODEL is set
2. OpenAI - if OPENAI_API_KEY is set 1. Bedrock - if ``aws_region`` or ``bedrock_embedding_model`` is set
3. Ollama - if OLLAMA_BASE_URL is set 2. OpenAI - if ``openai_api_key`` is set
4. Simple - fallback for testing/development 3. Mistral - if ``mistral_api_key`` is set
4. Ollama - if ``ollama_base_url`` is set
5. Simple - fallback for testing/development
Returns: Returns:
Provider instance Provider instance
Environment Variables:
Bedrock:
- AWS_REGION: AWS region (e.g., "us-east-1")
- AWS_ACCESS_KEY_ID: AWS access key (optional, uses credential chain)
- AWS_SECRET_ACCESS_KEY: AWS secret key (optional)
- BEDROCK_EMBEDDING_MODEL: Model ID for embeddings (e.g., "amazon.titan-embed-text-v2:0")
- BEDROCK_GENERATION_MODEL: Model ID for text generation (e.g., "anthropic.claude-3-sonnet-20240229-v1:0")
OpenAI:
- OPENAI_API_KEY: OpenAI API key (or GITHUB_TOKEN for GitHub Models)
- OPENAI_BASE_URL: Base URL override (e.g., "https://models.github.ai/inference")
- OPENAI_EMBEDDING_MODEL: Model for embeddings (default: "text-embedding-3-small")
- OPENAI_GENERATION_MODEL: Model for text generation (e.g., "gpt-4o-mini")
Ollama:
- OLLAMA_BASE_URL: Ollama API base URL (e.g., "http://localhost:11434")
- OLLAMA_EMBEDDING_MODEL: Model for embeddings (default: "nomic-embed-text")
- OLLAMA_GENERATION_MODEL: Model for text generation (e.g., "llama3.2:1b")
- OLLAMA_VERIFY_SSL: Verify SSL certificates (default: "true")
Simple (no configuration needed, fallback):
- SIMPLE_EMBEDDING_DIMENSION: Embedding dimension (default: 384)
""" """
# 1. Check for Bedrock settings = get_settings()
aws_region = os.getenv("AWS_REGION")
bedrock_embedding_model = os.getenv("BEDROCK_EMBEDDING_MODEL")
bedrock_generation_model = os.getenv("BEDROCK_GENERATION_MODEL")
if aws_region or bedrock_embedding_model or bedrock_generation_model: # 1. Bedrock
if (
settings.aws_region
or settings.bedrock_embedding_model
or settings.bedrock_generation_model
):
logger.info( logger.info(
f"Using Bedrock provider: region={aws_region}, " "Using Bedrock provider: region=%s, embedding_model=%s, "
f"embedding_model={bedrock_embedding_model}, " "generation_model=%s",
f"generation_model={bedrock_generation_model}" settings.aws_region,
settings.bedrock_embedding_model,
settings.bedrock_generation_model,
) )
return BedrockProvider( return BedrockProvider(
region_name=aws_region, region_name=settings.aws_region,
embedding_model=bedrock_embedding_model, embedding_model=settings.bedrock_embedding_model,
generation_model=bedrock_generation_model, generation_model=settings.bedrock_generation_model,
aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), aws_access_key_id=settings.aws_access_key_id,
aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), aws_secret_access_key=settings.aws_secret_access_key,
) )
# 2. Check for OpenAI # 2. OpenAI
openai_api_key = os.getenv("OPENAI_API_KEY") if settings.openai_api_key:
if openai_api_key:
base_url = os.getenv("OPENAI_BASE_URL")
embedding_model = os.getenv(
"OPENAI_EMBEDDING_MODEL", "text-embedding-3-small"
)
generation_model = os.getenv("OPENAI_GENERATION_MODEL")
logger.info( logger.info(
f"Using OpenAI provider: base_url={base_url or 'default'}, " "Using OpenAI provider: base_url=%s, embedding_model=%s, "
f"embedding_model={embedding_model}, " "generation_model=%s",
f"generation_model={generation_model}" settings.openai_base_url or "default",
settings.openai_embedding_model,
settings.openai_generation_model,
) )
return OpenAIProvider( return OpenAIProvider(
api_key=openai_api_key, api_key=settings.openai_api_key,
base_url=base_url, base_url=settings.openai_base_url,
embedding_model=embedding_model, embedding_model=settings.openai_embedding_model,
generation_model=generation_model, generation_model=settings.openai_generation_model,
) )
# 3. Check for Ollama (local LLM) # 3. Mistral
ollama_url = os.getenv("OLLAMA_BASE_URL") if settings.mistral_api_key:
if ollama_url:
embedding_model = os.getenv("OLLAMA_EMBEDDING_MODEL", "nomic-embed-text")
generation_model = os.getenv("OLLAMA_GENERATION_MODEL")
verify_ssl = os.getenv("OLLAMA_VERIFY_SSL", "true").lower() == "true"
logger.info( logger.info(
f"Using Ollama provider: {ollama_url}, " "Using Mistral provider: base_url=%s, embedding_model=%s",
f"embedding_model={embedding_model}, " settings.mistral_base_url or "default",
f"generation_model={generation_model}" settings.mistral_embedding_model,
)
return MistralProvider(
api_key=settings.mistral_api_key,
base_url=settings.mistral_base_url,
embedding_model=settings.mistral_embedding_model,
)
# 4. Ollama
if settings.ollama_base_url:
logger.info(
"Using Ollama provider: %s, embedding_model=%s, generation_model=%s",
settings.ollama_base_url,
settings.ollama_embedding_model,
settings.ollama_generation_model,
) )
return OllamaProvider( return OllamaProvider(
base_url=ollama_url, base_url=settings.ollama_base_url,
embedding_model=embedding_model, embedding_model=settings.ollama_embedding_model,
generation_model=generation_model, generation_model=settings.ollama_generation_model,
verify_ssl=verify_ssl, verify_ssl=settings.ollama_verify_ssl,
) )
# 4. Fallback to Simple provider for development/testing # 5. Simple (fallback)
dimension = int(os.getenv("SIMPLE_EMBEDDING_DIMENSION", "384"))
logger.warning( logger.warning(
"No provider configured (AWS_REGION, OPENAI_API_KEY, OLLAMA_BASE_URL not set). " "No provider configured (AWS_REGION, OPENAI_API_KEY, "
"MISTRAL_API_KEY, OLLAMA_BASE_URL not set). "
"Using SimpleProvider for testing/development. " "Using SimpleProvider for testing/development. "
"For production, configure Bedrock, OpenAI, or Ollama." "For production, configure Bedrock, OpenAI, Mistral, or Ollama."
) )
return SimpleProvider(dimension=dimension) return SimpleProvider(dimension=settings.simple_embedding_dimension)
# Singleton instance # Singleton instance
@@ -691,6 +691,122 @@ class PDFHighlighter:
f"Failed to delete temp directory {temp_pdf_path.parent}: {e}" f"Failed to delete temp directory {temp_pdf_path.parent}: {e}"
) )
@staticmethod
def compute_chunk_bboxes_batch(
pdf_bytes: bytes,
chunks: list[tuple[int, int, int, int | None, str]],
page_boundaries: list[dict],
full_text: str,
) -> dict[int, tuple[list[tuple[float, float, float, float]], int]]:
"""Compute normalized bounding boxes for chunks without rendering.
Lightweight alternative to highlight_chunks_batch — opens the PDF,
locates each chunk on its assigned page using the same text-search
path as the highlighter (`_find_chunk_bbox`), and returns
page-normalized rectangles. Skips the get_pixmap + PIL pipeline
entirely, so no PNG bytes are produced.
Args:
pdf_bytes: PDF file bytes.
chunks: List of (chunk_index, start_offset, end_offset,
stored_page_number, chunk_text). chunk_index is the dict key.
page_boundaries: Pre-computed page boundaries from the document
processor; each entry is {"page", "start_offset", "end_offset"}.
full_text: Full document text (for cross-page chunk handling).
Returns:
dict mapping chunk_index to (normalized_bboxes, page_number).
Each bbox is (x0, y0, x1, y1) in [0, 1] relative to page width
and height, top-left origin. Chunks whose bbox cannot be located
are omitted from the result.
"""
results: dict[int, tuple[list[tuple[float, float, float, float]], int]] = {}
if not chunks:
return results
temp_pdf_path = None
doc = None
try:
temp_dir = Path(tempfile.mkdtemp(prefix="pdf_bbox_batch_"))
temp_pdf_path = temp_dir / "pdf.pdf"
temp_pdf_path.write_bytes(pdf_bytes)
doc = pymupdf.open(temp_pdf_path)
for (
chunk_index,
start_offset,
end_offset,
_,
_,
) in chunks:
chunk_page_info = PDFHighlighter.find_chunk_page(
start_offset, end_offset, page_boundaries
)
if not chunk_page_info:
logger.debug("Chunk %s: not found on any page", chunk_index)
continue
page_num = chunk_page_info["page_num"]
page_boundary = next(
(b for b in page_boundaries if b["page"] == page_num), None
)
if page_boundary is None:
logger.debug(
"Chunk %s: page %s not found in boundaries",
chunk_index,
page_num,
)
continue
page_text_length = (
page_boundary["end_offset"] - page_boundary["start_offset"]
)
# Page-relative slice (handles chunks that span page boundaries)
chunk_start_on_page = max(start_offset, page_boundary["start_offset"])
chunk_end_on_page = min(end_offset, page_boundary["end_offset"])
page_relative_text = full_text[chunk_start_on_page:chunk_end_on_page]
page = doc[page_num - 1]
bbox = PDFHighlighter._find_chunk_bbox(
page,
page_relative_text,
chunk_page_info["page_relative_start"],
chunk_page_info["page_relative_end"],
page_text_length,
)
if bbox is None:
continue
page_rect = page.rect
w = page_rect.width or 1.0
h = page_rect.height or 1.0
normalized = (
bbox[0] / w,
bbox[1] / h,
bbox[2] / w,
bbox[3] / h,
)
results[chunk_index] = ([normalized], page_num)
logger.info(f"Computed bboxes for {len(results)}/{len(chunks)} chunks")
return results
except Exception as e:
logger.error(f"Error computing chunk bboxes: {e}", exc_info=True)
return results
finally:
if doc is not None:
doc.close()
if temp_pdf_path and temp_pdf_path.parent.exists():
try:
shutil.rmtree(temp_pdf_path.parent)
except Exception as e:
logger.warning(f"Failed to clean up temp dir: {e}")
@staticmethod @staticmethod
def highlight_chunks_batch( def highlight_chunks_batch(
pdf_bytes: bytes, pdf_bytes: bytes,
+9 -8
View File
@@ -1,16 +1,17 @@
"""Vector database and background sync package.""" """Vector database and background sync package.
`processor` and `scanner` are intentionally NOT re-exported from this
package init: they transitively import `server.semantic` ->
`search.bm25_hybrid`, which forms an import cycle with
`search.algorithms` -> `vector.placeholder` -> `vector/__init__`.
Consumers that need those symbols import them from their submodules
directly (e.g. `from nextcloud_mcp_server.vector.processor import ...`).
"""
from .document_chunker import DocumentChunker from .document_chunker import DocumentChunker
from .processor import process_document, processor_task
from .qdrant_client import get_qdrant_client from .qdrant_client import get_qdrant_client
from .scanner import DocumentTask, scan_user_documents, scanner_task
__all__ = [ __all__ = [
"get_qdrant_client", "get_qdrant_client",
"DocumentChunker", "DocumentChunker",
"scanner_task",
"scan_user_documents",
"DocumentTask",
"processor_task",
"process_document",
] ]
+26 -49
View File
@@ -3,7 +3,6 @@
Processes documents from stream: fetches content, generates embeddings, stores in Qdrant. Processes documents from stream: fetches content, generates embeddings, stores in Qdrant.
""" """
import base64
import logging import logging
import time import time
import uuid import uuid
@@ -538,7 +537,11 @@ async def _index_document(
# Initialize results containers # Initialize results containers
dense_embeddings: list = [] dense_embeddings: list = []
sparse_embeddings: list = [] sparse_embeddings: list = []
chunk_images: dict[int, dict] = {} # chunk_index -> list[(x0, y0, x1, y1)] of normalized rectangles
# in [0, 1] relative to page width/height. The page is taken from
# `chunk.page_number` (offset-based) and stored as `page_number`
# in the Qdrant payload, so we don't carry an `actual_page_num` here.
chunk_bboxes: dict[int, list[tuple[float, float, float, float]]] = {}
# Determine if we need PDF highlighting # Determine if we need PDF highlighting
is_pdf = doc_task.doc_type == "file" and content_type == "application/pdf" is_pdf = doc_task.doc_type == "file" and content_type == "application/pdf"
@@ -570,8 +573,8 @@ async def _index_document(
sparse_embeddings = await bm25_service.encode_batch(chunk_texts) sparse_embeddings = await bm25_service.encode_batch(chunk_texts)
async def generate_highlights(): async def generate_highlights():
"""Generate highlighted page images for PDF chunks (CPU-bound).""" """Compute chunk bounding boxes for PDF chunks (CPU-bound, no rendering)."""
nonlocal chunk_images nonlocal chunk_bboxes
if not is_pdf: if not is_pdf:
return return
@@ -579,64 +582,42 @@ async def _index_document(
assert content_bytes is not None assert content_bytes is not None
with trace_operation( with trace_operation(
"vector_sync.generate_highlights", "vector_sync.compute_chunk_bboxes",
attributes={ attributes={
"vector_sync.chunk_count": len(chunks), "vector_sync.chunk_count": len(chunks),
"vector_sync.pdf_size": len(content_bytes), "vector_sync.pdf_size": len(content_bytes),
}, },
): ):
# Build chunk data for batch processing
# Format: (chunk_index, start_offset, end_offset, page_number, chunk_text)
chunk_data: list[tuple[int, int, int, int | None, str]] = [ chunk_data: list[tuple[int, int, int, int | None, str]] = [
(i, chunk.start_offset, chunk.end_offset, chunk.page_number, chunk.text) (i, chunk.start_offset, chunk.end_offset, chunk.page_number, chunk.text)
for i, chunk in enumerate(chunks) for i, chunk in enumerate(chunks)
if chunk.page_number is not None if chunk.page_number is not None
] ]
# Get pre-computed page boundaries from document processor
page_boundaries = file_metadata.get("page_boundaries") page_boundaries = file_metadata.get("page_boundaries")
if not page_boundaries: if not page_boundaries:
logger.warning("No page boundaries available, skipping highlighting") logger.warning(
"No page boundaries available, skipping bbox computation"
)
return return
# Type narrowing: page_boundaries is guaranteed to be list[dict] here
page_boundaries_list = cast(list[dict[str, Any]], page_boundaries) page_boundaries_list = cast(list[dict[str, Any]], page_boundaries)
logger.info( logger.info(f"Computing chunk bboxes for {len(chunk_data)} PDF chunks")
f"Batch generating highlighted page images for {len(chunk_data)} PDF chunks"
)
# Run CPU-bound highlighting in thread pool
# Pass pre-computed page boundaries and full text to avoid re-processing the PDF
batch_results = await anyio.to_thread.run_sync( # type: ignore[attr-defined] batch_results = await anyio.to_thread.run_sync( # type: ignore[attr-defined]
lambda: PDFHighlighter.highlight_chunks_batch( lambda: PDFHighlighter.compute_chunk_bboxes_batch(
pdf_bytes=content_bytes, pdf_bytes=content_bytes,
chunks=chunk_data, chunks=chunk_data,
page_boundaries=page_boundaries_list, page_boundaries=page_boundaries_list,
full_text=content, full_text=content,
color="yellow",
zoom=2.0,
) )
) )
# Convert results to storage format for chunk_index, (bboxes, _) in batch_results.items():
for chunk_index, ( chunk_bboxes[chunk_index] = bboxes
png_bytes,
actual_page_num,
highlight_count,
) in batch_results.items():
image_base64 = base64.b64encode(png_bytes).decode("utf-8")
chunk_images[chunk_index] = {
"image": image_base64,
"page": actual_page_num,
"highlights": highlight_count,
"size": len(png_bytes),
}
logger.info( logger.info(f"Computed bboxes for {len(chunk_bboxes)}/{len(chunks)} chunks")
f"Generated {len(chunk_images)}/{len(chunks)} highlighted page images "
f"(avg {sum(img['size'] for img in chunk_images.values()) // max(len(chunk_images), 1):,} bytes)"
)
# Run all embedding/highlighting operations in parallel # Run all embedding/highlighting operations in parallel
# - Dense embeddings: I/O bound (API call) # - Dense embeddings: I/O bound (API call)
@@ -752,16 +733,11 @@ async def _index_document(
if doc_task.doc_type == "deck_card" if doc_task.doc_type == "deck_card"
else {} else {}
), ),
# Highlighted page image (PDF only) # Chunk bbox (PDF only) — normalized rectangles in [0,1]
**( # relative to page width/height. Replaces the legacy
{ # `highlighted_page_image` (Deck #76). The page number
"highlighted_page_image": chunk_images[i]["image"], # comes from `page_number` (set above for PDF chunks).
"highlighted_page_number": chunk_images[i]["page"], **({"chunk_bbox": chunk_bboxes[i]} if i in chunk_bboxes else {}),
"highlight_count": chunk_images[i]["highlights"],
}
if i in chunk_images
else {}
),
}, },
) )
) )
@@ -780,15 +756,16 @@ async def _index_document(
f"Failed to delete placeholder for {doc_task.doc_type}_{doc_task.doc_id}: {e}" f"Failed to delete placeholder for {doc_task.doc_type}_{doc_task.doc_id}: {e}"
) )
# Upsert to Qdrant in batches to avoid timeout with large payloads # Upsert to Qdrant in batches. Now that we no longer embed PNG payloads,
# Each batch is limited to avoid WriteTimeout when sending large image payloads # per-point payloads are small (chunk text + small metadata), so we can
BATCH_SIZE = 10 # ~2MB per batch with images # safely use a larger batch size.
BATCH_SIZE = 100
with trace_operation( with trace_operation(
"vector_sync.qdrant_upsert", "vector_sync.qdrant_upsert",
attributes={ attributes={
"vector_sync.point_count": len(points), "vector_sync.point_count": len(points),
"vector_sync.collection": settings.get_collection_name(), "vector_sync.collection": settings.get_collection_name(),
"vector_sync.images_count": len(chunk_images), "vector_sync.bboxes_count": len(chunk_bboxes),
"vector_sync.batch_size": BATCH_SIZE, "vector_sync.batch_size": BATCH_SIZE,
}, },
): ):
+2 -1
View File
@@ -1,6 +1,6 @@
[project] [project]
name = "nextcloud-mcp-server" name = "nextcloud-mcp-server"
version = "0.81.0" version = "0.83.0"
description = "Model Context Protocol (MCP) server for Nextcloud integration - enables AI assistants to interact with Nextcloud data" description = "Model Context Protocol (MCP) server for Nextcloud integration - enables AI assistants to interact with Nextcloud data"
authors = [ authors = [
{name = "Chris Coutinho", email = "chris@coutinho.io"} {name = "Chris Coutinho", email = "chris@coutinho.io"}
@@ -43,6 +43,7 @@ dependencies = [
"pymupdf4llm>=0.2.2", "pymupdf4llm>=0.2.2",
"openai>=2.8.1", "openai>=2.8.1",
"dynaconf>=3.2.13,<4.0", "dynaconf>=3.2.13,<4.0",
"mistralai>=2.4.5",
] ]
classifiers = [ classifiers = [
"Development Status :: 4 - Beta", "Development Status :: 4 - Beta",
+137
View File
@@ -0,0 +1,137 @@
#!/usr/bin/env python3
"""Purge legacy `highlighted_page_image` payloads from Qdrant (Deck #76).
Iterates all points in the configured Qdrant collection and deletes the
legacy payload keys `highlighted_page_image`, `highlighted_page_number`,
and `highlight_count`. This relieves disk pressure caused by inline
base64 PNGs that the new code path no longer writes.
Idempotent: deleting non-existent keys is a no-op, so re-runs are safe.
Usage:
uv run python scripts/purge_page_images.py [--dry-run] [--batch-size 256]
Connection settings (Qdrant URL/API key, collection name) are read from
the same `Settings` object the server uses.
"""
from __future__ import annotations
import argparse
import logging
import sys
from functools import partial
import anyio
from qdrant_client import AsyncQdrantClient
from nextcloud_mcp_server.config import get_settings
logger = logging.getLogger("purge_page_images")
LEGACY_FIELDS = [
"highlighted_page_image",
"highlighted_page_number",
"highlight_count",
]
async def purge(dry_run: bool, batch_size: int) -> None:
settings = get_settings()
if not settings.qdrant_url:
raise SystemExit(
"qdrant_url is not configured. Set QDRANT_URL (and QDRANT_API_KEY "
"if required) before running this script."
)
collection = settings.get_collection_name()
# AsyncQdrantClient doesn't implement __aenter__/__aexit__, so use
# try/finally to guarantee the underlying aiohttp session is closed.
client = AsyncQdrantClient(
url=settings.qdrant_url,
api_key=settings.qdrant_api_key,
timeout=60,
)
try:
next_offset = None
total_seen = 0
total_updated = 0
logger.info(
"Scanning collection %s; will delete keys %s%s",
collection,
LEGACY_FIELDS,
" (dry run)" if dry_run else "",
)
while True:
points, next_offset = await client.scroll(
collection_name=collection,
limit=batch_size,
offset=next_offset,
with_payload=False,
with_vectors=False,
)
if not points:
break
ids = [p.id for p in points]
total_seen += len(ids)
if not dry_run:
await client.delete_payload(
collection_name=collection,
keys=LEGACY_FIELDS,
points=ids,
)
total_updated += len(ids)
logger.info(
"Batch: ids=%d total_seen=%d total_updated=%d",
len(ids),
total_seen,
total_updated,
)
if next_offset is None:
break
logger.info(
"Done. total_seen=%d total_updated=%d%s",
total_seen,
total_updated,
" (dry run, no writes)" if dry_run else "",
)
finally:
await client.close()
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--dry-run",
action="store_true",
help="Scan only; do not write any changes.",
)
parser.add_argument(
"--batch-size",
type=int,
default=256,
help="Points per scroll/update batch (default: 256).",
)
parser.add_argument(
"-v", "--verbose", action="store_true", help="Enable debug logging."
)
args = parser.parse_args()
logging.basicConfig(
level=logging.DEBUG if args.verbose else logging.INFO,
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
)
anyio.run(partial(purge, dry_run=args.dry_run, batch_size=args.batch_size))
return 0
if __name__ == "__main__":
sys.exit(main())
+269
View File
@@ -0,0 +1,269 @@
"""Unit tests for Mistral provider."""
from unittest.mock import AsyncMock, MagicMock
import pytest
from mistralai.client.errors import SDKError
from nextcloud_mcp_server.providers.mistral import (
BATCH_SIZE,
MISTRAL_EMBEDDING_DIMENSIONS,
MistralProvider,
_is_rate_limit,
)
def _make_data(embedding: list[float], index: int) -> MagicMock:
"""Build a mock EmbeddingResponseData entry."""
item = MagicMock()
item.embedding = embedding
item.index = index
return item
def _make_response(embeddings: list[list[float]]) -> MagicMock:
"""Build a mock EmbeddingResponse with `embeddings` indexed in order."""
response = MagicMock()
response.data = [_make_data(emb, i) for i, emb in enumerate(embeddings)]
return response
@pytest.fixture
def mock_mistral_client(mocker):
"""Mock the Mistral SDK constructor."""
mock_client = MagicMock()
mock_client.embeddings = MagicMock()
mocker.patch(
"nextcloud_mcp_server.providers.mistral.Mistral", return_value=mock_client
)
return mock_client
@pytest.mark.unit
async def test_mistral_embedding_single(mock_mistral_client):
"""Single text embed: round-trip through SDK with correct kwargs."""
mock_mistral_client.embeddings.create_async = AsyncMock(
return_value=_make_response([[0.1, 0.2, 0.3]])
)
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
embedding = await provider.embed("hello world")
assert embedding == [0.1, 0.2, 0.3]
mock_mistral_client.embeddings.create_async.assert_awaited_once_with(
model="mistral-embed",
inputs=["hello world"],
)
@pytest.mark.unit
async def test_mistral_embedding_batch_single_call(mock_mistral_client):
"""Batch smaller than BATCH_SIZE issues a single API call."""
mock_mistral_client.embeddings.create_async = AsyncMock(
return_value=_make_response([[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]])
)
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
embeddings = await provider.embed_batch(["a", "b", "c"])
assert embeddings == [[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]]
assert mock_mistral_client.embeddings.create_async.await_count == 1
@pytest.mark.unit
async def test_mistral_embedding_batch_chunking(mock_mistral_client):
"""Batches exceeding BATCH_SIZE are split into multiple API calls."""
# Each call returns one embedding per input it received; capture by side
# effect so we can inspect lengths per chunk.
def _side_effect(*, model, inputs, **_kwargs):
return _make_response([[float(i)] for i in range(len(inputs))])
mock_mistral_client.embeddings.create_async = AsyncMock(side_effect=_side_effect)
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
total = BATCH_SIZE * 2 + 5 # forces three chunks: 64, 64, 5 (with default)
embeddings = await provider.embed_batch([f"text-{i}" for i in range(total)])
assert len(embeddings) == total
assert mock_mistral_client.embeddings.create_async.await_count == 3
# Verify the chunk sizes the SDK was actually called with.
chunk_sizes = [
len(call.kwargs["inputs"])
for call in mock_mistral_client.embeddings.create_async.await_args_list
]
assert chunk_sizes == [BATCH_SIZE, BATCH_SIZE, 5]
@pytest.mark.unit
async def test_mistral_embedding_batch_order_preserved(mock_mistral_client):
"""Out-of-order index in response data is sorted before returning."""
response = MagicMock()
response.data = [
_make_data([0.3, 0.3], 2),
_make_data([0.1, 0.1], 0),
_make_data([0.2, 0.2], 1),
]
mock_mistral_client.embeddings.create_async = AsyncMock(return_value=response)
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
embeddings = await provider.embed_batch(["x", "y", "z"])
assert embeddings == [[0.1, 0.1], [0.2, 0.2], [0.3, 0.3]]
@pytest.mark.unit
async def test_mistral_supports_capabilities(mock_mistral_client):
"""Mistral provider advertises embeddings only."""
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
assert provider.supports_embeddings is True
assert provider.supports_generation is False
@pytest.mark.unit
async def test_mistral_generate_not_implemented(mock_mistral_client):
"""generate() always raises NotImplementedError."""
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
with pytest.raises(NotImplementedError, match="does not support generation"):
await provider.generate("test prompt")
@pytest.mark.unit
async def test_mistral_get_dimension_known_model(mock_mistral_client):
"""Known model: dimension available without an API call."""
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
assert provider.get_dimension() == MISTRAL_EMBEDDING_DIMENSIONS["mistral-embed"]
mock_mistral_client.embeddings.create_async.assert_not_called()
@pytest.mark.unit
async def test_mistral_get_dimension_unknown_model_detected(mock_mistral_client):
"""Unknown model: dimension detected on first embed() call."""
mock_mistral_client.embeddings.create_async = AsyncMock(
return_value=_make_response([[0.1] * 768])
)
provider = MistralProvider(api_key="test-key", embedding_model="custom-mistral")
with pytest.raises(RuntimeError, match="not detected yet"):
provider.get_dimension()
await provider.embed("test")
assert provider.get_dimension() == 768
@pytest.mark.unit
async def test_mistral_no_embeddings_disabled(mock_mistral_client):
"""Setting embedding_model=None disables the embedding capability."""
provider = MistralProvider(api_key="test-key", embedding_model=None)
assert provider.supports_embeddings is False
with pytest.raises(NotImplementedError, match="no embedding_model configured"):
await provider.embed("test")
with pytest.raises(NotImplementedError, match="no embedding_model configured"):
await provider.embed_batch(["test"])
with pytest.raises(NotImplementedError, match="no embedding_model configured"):
provider.get_dimension()
@pytest.mark.unit
async def test_mistral_empty_batch(mock_mistral_client):
"""An empty batch returns [] without calling the API."""
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
assert await provider.embed_batch([]) == []
mock_mistral_client.embeddings.create_async.assert_not_called()
@pytest.mark.unit
async def test_mistral_close_no_error(mock_mistral_client):
"""close() is best-effort and does not raise."""
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
# No __aexit__ on the mock by default → close() should silently no-op.
await provider.close()
@pytest.mark.unit
async def test_mistral_base_url_passed_to_sdk(mocker):
"""base_url is forwarded as server_url to the Mistral SDK constructor."""
mock_ctor = mocker.patch(
"nextcloud_mcp_server.providers.mistral.Mistral", return_value=MagicMock()
)
MistralProvider(
api_key="test-key",
embedding_model="mistral-embed",
base_url="https://example.com/mistral",
)
mock_ctor.assert_called_once_with(
api_key="test-key",
server_url="https://example.com/mistral",
)
@pytest.mark.unit
async def test_mistral_embed_raises_on_empty_response_data(mock_mistral_client):
"""embed(): empty response.data triggers the defensive RuntimeError guard."""
empty_response = MagicMock()
empty_response.data = []
mock_mistral_client.embeddings.create_async = AsyncMock(return_value=empty_response)
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
with pytest.raises(RuntimeError, match="returned no embedding"):
await provider.embed("test")
@pytest.mark.unit
async def test_mistral_embed_raises_on_null_embedding(mock_mistral_client):
"""embed(): a single response item with embedding=None is rejected."""
null_item = MagicMock()
null_item.embedding = None
null_item.index = 0
null_response = MagicMock()
null_response.data = [null_item]
mock_mistral_client.embeddings.create_async = AsyncMock(return_value=null_response)
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
with pytest.raises(RuntimeError, match="returned no embedding"):
await provider.embed("test")
@pytest.mark.unit
async def test_mistral_batch_raises_on_null_embedding(mock_mistral_client):
"""_embed_batch_request: a null embedding inside a batch raises explicitly."""
good = _make_data([0.1, 0.2], 0)
bad = MagicMock()
bad.embedding = None
bad.index = 1
response = MagicMock()
response.data = [good, bad]
mock_mistral_client.embeddings.create_async = AsyncMock(return_value=response)
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
with pytest.raises(RuntimeError, match="null embedding"):
await provider.embed_batch(["a", "b"])
@pytest.mark.unit
async def test_mistral_batch_raises_on_count_mismatch(mock_mistral_client):
"""_embed_batch_request: fewer embeddings returned than inputs sent."""
# Two inputs sent, one embedding returned.
response = _make_response([[0.1, 0.2]])
mock_mistral_client.embeddings.create_async = AsyncMock(return_value=response)
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
with pytest.raises(RuntimeError, match="returned 1 embeddings for 2 inputs"):
await provider.embed_batch(["a", "b"])
@pytest.mark.unit
def test_mistral_is_rate_limit_predicate():
"""_is_rate_limit returns True only for SDKErrors with status_code == 429."""
err_429 = MagicMock(spec=SDKError)
err_429.status_code = 429
err_500 = MagicMock(spec=SDKError)
err_500.status_code = 500
assert _is_rate_limit(err_429) is True
assert _is_rate_limit(err_500) is False
# ValueError has no status_code attr → getattr returns None → False.
assert _is_rate_limit(ValueError()) is False
+136
View File
@@ -0,0 +1,136 @@
"""Unit tests for ProviderRegistry — dynaconf-driven auto-detection."""
import pytest
from nextcloud_mcp_server.config import _reload_config
from nextcloud_mcp_server.providers import (
BedrockProvider,
MistralProvider,
OllamaProvider,
OpenAIProvider,
SimpleProvider,
get_provider,
reset_provider,
)
from nextcloud_mcp_server.providers.bedrock import BOTO3_AVAILABLE
def _clear_provider_envs(monkeypatch: pytest.MonkeyPatch) -> None:
"""Strip every provider-selection env var so each test starts clean."""
for name in (
"AWS_REGION",
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
"BEDROCK_EMBEDDING_MODEL",
"BEDROCK_GENERATION_MODEL",
"OPENAI_API_KEY",
"OPENAI_BASE_URL",
"OPENAI_EMBEDDING_MODEL",
"OPENAI_GENERATION_MODEL",
"MISTRAL_API_KEY",
"MISTRAL_BASE_URL",
"MISTRAL_EMBEDDING_MODEL",
"OLLAMA_BASE_URL",
"OLLAMA_EMBEDDING_MODEL",
"OLLAMA_GENERATION_MODEL",
"OLLAMA_VERIFY_SSL",
"SIMPLE_EMBEDDING_DIMENSION",
):
monkeypatch.delenv(name, raising=False)
@pytest.fixture
def clean_provider_env(monkeypatch):
"""Reset provider singleton + dynaconf cache around each test."""
_clear_provider_envs(monkeypatch)
reset_provider()
_reload_config()
yield monkeypatch
reset_provider()
@pytest.mark.unit
def test_registry_falls_back_to_simple(clean_provider_env):
"""No provider env set → SimpleProvider (with default dimension)."""
provider = get_provider()
assert isinstance(provider, SimpleProvider)
assert provider.get_dimension() == 384
@pytest.mark.unit
def test_registry_picks_simple_with_custom_dimension(clean_provider_env):
"""SIMPLE_EMBEDDING_DIMENSION flows through dynaconf to SimpleProvider."""
clean_provider_env.setenv("SIMPLE_EMBEDDING_DIMENSION", "512")
_reload_config()
provider = get_provider()
assert isinstance(provider, SimpleProvider)
assert provider.get_dimension() == 512
@pytest.mark.unit
def test_registry_picks_mistral_when_api_key_set(clean_provider_env, mocker):
"""MISTRAL_API_KEY alone is enough to select MistralProvider."""
# MistralProvider eagerly constructs the SDK client in __init__; stub it
# so the test doesn't depend on the SDK accepting arbitrary keys.
mocker.patch("nextcloud_mcp_server.providers.mistral.Mistral")
clean_provider_env.setenv("MISTRAL_API_KEY", "test-key")
_reload_config()
provider = get_provider()
assert isinstance(provider, MistralProvider)
@pytest.mark.unit
def test_registry_picks_ollama_when_base_url_set(clean_provider_env, mocker):
"""OLLAMA_BASE_URL selects OllamaProvider."""
# OllamaProvider eagerly probes /api/tags in __init__; stub it out.
mocker.patch(
"nextcloud_mcp_server.providers.ollama.OllamaProvider._check_model_is_loaded"
)
clean_provider_env.setenv("OLLAMA_BASE_URL", "http://localhost:11434")
_reload_config()
provider = get_provider()
assert isinstance(provider, OllamaProvider)
@pytest.mark.unit
def test_registry_openai_wins_over_mistral_and_ollama(clean_provider_env):
"""OpenAI takes priority when multiple provider env vars are set."""
clean_provider_env.setenv("OPENAI_API_KEY", "openai-key")
clean_provider_env.setenv("MISTRAL_API_KEY", "mistral-key")
clean_provider_env.setenv("OLLAMA_BASE_URL", "http://localhost:11434")
_reload_config()
provider = get_provider()
assert isinstance(provider, OpenAIProvider)
@pytest.mark.unit
def test_registry_mistral_wins_over_ollama(clean_provider_env, mocker):
"""Mistral takes priority over Ollama when both are configured."""
# Stub the Mistral SDK constructor for the same reason as the sibling
# picker test — keeps the registry test independent of SDK key validation.
mocker.patch("nextcloud_mcp_server.providers.mistral.Mistral")
clean_provider_env.setenv("MISTRAL_API_KEY", "mistral-key")
clean_provider_env.setenv("OLLAMA_BASE_URL", "http://localhost:11434")
_reload_config()
provider = get_provider()
assert isinstance(provider, MistralProvider)
@pytest.mark.unit
def test_registry_bedrock_wins_when_aws_region_set(clean_provider_env):
"""AWS_REGION alone routes to Bedrock, even with other providers configured."""
if not BOTO3_AVAILABLE:
pytest.skip("boto3 not installed")
clean_provider_env.setenv("AWS_REGION", "us-east-1")
clean_provider_env.setenv("OPENAI_API_KEY", "openai-key")
clean_provider_env.setenv("MISTRAL_API_KEY", "mistral-key")
_reload_config()
provider = get_provider()
assert isinstance(provider, BedrockProvider)
+103
View File
@@ -0,0 +1,103 @@
"""Unit tests for the shared rate-limit retry decorator."""
from unittest.mock import AsyncMock
import pytest
from nextcloud_mcp_server.providers import _retry
class _FakeError(Exception):
"""Stand-in for an SDK exception with an HTTP status code attached."""
def __init__(self, status_code: int):
super().__init__(f"status {status_code}")
self.status_code = status_code
@pytest.fixture(autouse=True)
def _no_real_sleep(monkeypatch):
"""Replace anyio.sleep with an awaitable no-op so retries don't waste time."""
monkeypatch.setattr(_retry.anyio, "sleep", AsyncMock(return_value=None))
@pytest.mark.unit
async def test_retry_succeeds_after_429():
"""A 429 followed by success returns the success value."""
calls = {"n": 0}
@_retry.retry_on_rate_limit(
_FakeError, is_rate_limit=lambda e: e.status_code == 429
)
async def flaky():
calls["n"] += 1
if calls["n"] < 3:
raise _FakeError(429)
return "ok"
result = await flaky()
assert result == "ok"
assert calls["n"] == 3
@pytest.mark.unit
async def test_retry_reraises_non_rate_limit_immediately():
"""A non-rate-limit error of the same class is re-raised on first hit."""
calls = {"n": 0}
@_retry.retry_on_rate_limit(
_FakeError, is_rate_limit=lambda e: e.status_code == 429
)
async def boom():
calls["n"] += 1
raise _FakeError(500)
with pytest.raises(_FakeError, match="status 500"):
await boom()
assert calls["n"] == 1 # No retries on non-429.
@pytest.mark.unit
async def test_retry_gives_up_after_max_retries():
"""After MAX_RETRIES failed attempts the last error is re-raised."""
calls = {"n": 0}
@_retry.retry_on_rate_limit(
_FakeError, is_rate_limit=lambda e: e.status_code == 429
)
async def always_429():
calls["n"] += 1
raise _FakeError(429)
with pytest.raises(_FakeError, match="status 429"):
await always_429()
assert calls["n"] == _retry.MAX_RETRIES
@pytest.mark.unit
async def test_retry_default_predicate_treats_all_as_rate_limit():
"""Default predicate (`lambda _: True`) retries every caught exception."""
calls = {"n": 0}
@_retry.retry_on_rate_limit(_FakeError)
async def fail_once():
calls["n"] += 1
if calls["n"] < 2:
raise _FakeError(503)
return "recovered"
result = await fail_once()
assert result == "recovered"
assert calls["n"] == 2
@pytest.mark.unit
async def test_retry_does_not_catch_unrelated_exceptions():
"""Exceptions of a different class bypass the decorator entirely."""
@_retry.retry_on_rate_limit(_FakeError)
async def value_error():
raise ValueError("nope")
with pytest.raises(ValueError, match="nope"):
await value_error()
@@ -0,0 +1,232 @@
"""Unit tests for PDFHighlighter.compute_chunk_bboxes_batch (Deck #76).
Replaces the legacy `highlight_chunks_batch`-+-base64 pipeline that inflated
Qdrant payloads with per-chunk PNG screenshots. The new path returns
normalized bounding boxes only.
"""
from __future__ import annotations
import pymupdf
import pytest
from nextcloud_mcp_server.search.pdf_highlighter import PDFHighlighter
def _make_pdf(pages: list[str]) -> bytes:
"""Build an in-memory PDF whose pages contain the given text."""
doc = pymupdf.open()
for body in pages:
page = doc.new_page(width=595, height=842) # A4
page.insert_text((50, 50), body)
pdf_bytes = doc.tobytes()
doc.close()
return pdf_bytes
def _page_boundaries(pages: list[str]) -> tuple[list[dict], str]:
"""Build (page_boundaries, full_text) compatible with the highlighter API."""
boundaries: list[dict] = []
cursor = 0
parts: list[str] = []
for i, body in enumerate(pages, start=1):
end = cursor + len(body)
boundaries.append({"page": i, "start_offset": cursor, "end_offset": end})
parts.append(body)
cursor = end
return boundaries, "".join(parts)
@pytest.mark.unit
def test_compute_chunk_bboxes_returns_normalized_rects():
"""Each returned bbox should be 4 floats in [0, 1] tagged with the page."""
pages = [
"Chapter 1: Introduction. Nextcloud is a self-hosted collaboration platform "
"covering installation, configuration and maintenance topics.",
"Chapter 2: Installation. Download the package, extract it to the web "
"server directory, and configure the database connection.",
]
pdf_bytes = _make_pdf(pages)
boundaries, full_text = _page_boundaries(pages)
chunks = [
(
0,
0,
len(pages[0]),
1,
"Chapter 1: Introduction. Nextcloud is a self-hosted collaboration platform.",
),
(
1,
len(pages[0]),
len(pages[0]) + len(pages[1]),
2,
"Chapter 2: Installation. Download the package.",
),
]
results = PDFHighlighter.compute_chunk_bboxes_batch(
pdf_bytes=pdf_bytes,
chunks=chunks,
page_boundaries=boundaries,
full_text=full_text,
)
assert set(results) == {0, 1}
bboxes_p1, page_p1 = results[0]
bboxes_p2, page_p2 = results[1]
assert page_p1 == 1
assert page_p2 == 2
for rects in (bboxes_p1, bboxes_p2):
assert len(rects) >= 1
for rect in rects:
assert len(rect) == 4
x0, y0, x1, y1 = rect
assert 0.0 <= x0 < x1 <= 1.0
assert 0.0 <= y0 < y1 <= 1.0
@pytest.mark.unit
def test_compute_chunk_bboxes_empty_input():
assert (
PDFHighlighter.compute_chunk_bboxes_batch(
pdf_bytes=b"",
chunks=[],
page_boundaries=[],
full_text="",
)
== {}
)
@pytest.mark.unit
def test_compute_chunk_bboxes_omits_when_offsets_out_of_range():
"""Chunks whose offsets fall outside every page boundary are omitted.
Verifies the docstring contract: *"Chunks whose bbox cannot be located
are omitted from the result."* (path: ``find_chunk_page`` returns None).
"""
pages = ["Page one body text content here for the test."]
pdf_bytes = _make_pdf(pages)
boundaries, full_text = _page_boundaries(pages)
# Offsets way beyond the document end — no page boundary matches.
out_of_range_start = len(full_text) + 1000
out_of_range_end = out_of_range_start + 50
chunks = [(0, out_of_range_start, out_of_range_end, 1, "irrelevant")]
results = PDFHighlighter.compute_chunk_bboxes_batch(
pdf_bytes=pdf_bytes,
chunks=chunks,
page_boundaries=boundaries,
full_text=full_text,
)
assert results == {}
@pytest.mark.unit
def test_compute_chunk_bboxes_omits_when_text_not_in_pdf():
"""Chunks whose page-relative text isn't on the page are omitted.
Verifies the second omission path: ``_find_chunk_bbox`` returns None
when the supplied text cannot be located on the rendered page.
"""
pages = ["Hello world."]
pdf_bytes = _make_pdf(pages)
# Build boundaries from the real text but pass a *different* full_text
# so the page-relative slice is content that does not exist in the PDF.
boundaries, _ = _page_boundaries(pages)
bogus_full_text = "Z" * len(pages[0])
chunks = [(0, 0, len(pages[0]), 1, "ignored")]
results = PDFHighlighter.compute_chunk_bboxes_batch(
pdf_bytes=pdf_bytes,
chunks=chunks,
page_boundaries=boundaries,
full_text=bogus_full_text,
)
assert results == {}
@pytest.mark.unit
@pytest.mark.parametrize("page_index", [0, 1])
def test_compute_chunk_bboxes_assigns_correct_page(page_index: int):
"""Verify the page number returned matches the page the chunk lives on."""
pages = [
"Page one talks about apples and oranges in detail.",
"Page two discusses bananas and grapes thoroughly.",
]
pdf_bytes = _make_pdf(pages)
boundaries, full_text = _page_boundaries(pages)
if page_index == 0:
chunk_text = "apples and oranges"
offsets = (0, len(pages[0]))
else:
chunk_text = "bananas and grapes"
offsets = (len(pages[0]), len(pages[0]) + len(pages[1]))
chunks = [(0, offsets[0], offsets[1], page_index + 1, chunk_text)]
results = PDFHighlighter.compute_chunk_bboxes_batch(
pdf_bytes=pdf_bytes,
chunks=chunks,
page_boundaries=boundaries,
full_text=full_text,
)
assert results, "expected a bbox for the chunk"
_, page_num = results[0]
assert page_num == page_index + 1
@pytest.mark.unit
def test_compute_chunk_bboxes_handles_unordered_page_boundaries():
"""Page lookup must match by ``page`` key, not by list position.
Regression guard: an earlier implementation indexed
``page_boundaries[page_num - 1]``, which silently produces a wrong
bbox if boundaries are passed out of order. Reverse the boundaries
and assert the result is identical to the in-order case.
"""
pages = [
"Page one talks about apples and oranges in detail.",
"Page two discusses bananas and grapes thoroughly.",
]
pdf_bytes = _make_pdf(pages)
boundaries, full_text = _page_boundaries(pages)
chunks = [
(0, 0, len(pages[0]), 1, "apples and oranges"),
(
1,
len(pages[0]),
len(pages[0]) + len(pages[1]),
2,
"bananas and grapes",
),
]
in_order = PDFHighlighter.compute_chunk_bboxes_batch(
pdf_bytes=pdf_bytes,
chunks=chunks,
page_boundaries=boundaries,
full_text=full_text,
)
reversed_order = PDFHighlighter.compute_chunk_bboxes_batch(
pdf_bytes=pdf_bytes,
chunks=chunks,
page_boundaries=list(reversed(boundaries)),
full_text=full_text,
)
assert in_order == reversed_order
assert reversed_order[0][1] == 1
assert reversed_order[1][1] == 2
Generated
+82 -43
View File
@@ -769,6 +769,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/97/43/11d6e5d2c00bf000b5329717c74563bf76a9193f4a41cb0c4ef277dde4fa/dynaconf-3.2.13-py2.py3-none-any.whl", hash = "sha256:4305527aef4834bdba3e39479b23c005186e83fb85f65bcaa4bcea58fa26759b", size = 238041, upload-time = "2026-03-17T19:38:45.337Z" }, { url = "https://files.pythonhosted.org/packages/97/43/11d6e5d2c00bf000b5329717c74563bf76a9193f4a41cb0c4ef277dde4fa/dynaconf-3.2.13-py2.py3-none-any.whl", hash = "sha256:4305527aef4834bdba3e39479b23c005186e83fb85f65bcaa4bcea58fa26759b", size = 238041, upload-time = "2026-03-17T19:38:45.337Z" },
] ]
[[package]]
name = "eval-type-backport"
version = "0.3.1"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/fb/a3/cafafb4558fd638aadfe4121dc6cefb8d743368c085acb2f521df0f3d9d7/eval_type_backport-0.3.1.tar.gz", hash = "sha256:57e993f7b5b69d271e37482e62f74e76a0276c82490cf8e4f0dffeb6b332d5ed", size = 9445, upload-time = "2025-12-02T11:51:42.987Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/cf/22/fdc2e30d43ff853720042fa15baa3e6122722be1a7950a98233ebb55cd71/eval_type_backport-0.3.1-py3-none-any.whl", hash = "sha256:279ab641905e9f11129f56a8a78f493518515b83402b860f6f06dd7c011fdfa8", size = 6063, upload-time = "2025-12-02T11:51:41.665Z" },
]
[[package]] [[package]]
name = "executing" name = "executing"
version = "2.2.1" version = "2.2.1"
@@ -1478,6 +1487,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/73/07/02e16ed01e04a374e644b575638ec7987ae846d25ad97bcc9945a3ee4b0e/jsonpatch-1.33-py2.py3-none-any.whl", hash = "sha256:0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade", size = 12898, upload-time = "2023-06-16T21:01:28.466Z" }, { url = "https://files.pythonhosted.org/packages/73/07/02e16ed01e04a374e644b575638ec7987ae846d25ad97bcc9945a3ee4b0e/jsonpatch-1.33-py2.py3-none-any.whl", hash = "sha256:0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade", size = 12898, upload-time = "2023-06-16T21:01:28.466Z" },
] ]
[[package]]
name = "jsonpath-python"
version = "1.1.6"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/98/18/4ca8742534a5993ff383f7602e325ce2d5d7cc93d72ac5e1cdedbea8a458/jsonpath_python-1.1.6.tar.gz", hash = "sha256:dded9932b4ec41fb8726e09c83afa4e6be618f938c2db287cc2a81723c639671", size = 88178, upload-time = "2026-05-07T01:26:34.482Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/55/8a/1270a6803bd821cbfcdda387eaa13cb41a7b1f7b9bd145979b3bfb9d6cb7/jsonpath_python-1.1.6-py3-none-any.whl", hash = "sha256:a1c50afd8d3fbbaf47a4873bc890dcb3c15da96f5c020327977d844d8731a2d4", size = 14453, upload-time = "2026-05-07T01:26:33.306Z" },
]
[[package]] [[package]]
name = "jsonpointer" name = "jsonpointer"
version = "3.0.0" version = "3.0.0"
@@ -1842,6 +1860,25 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" },
] ]
[[package]]
name = "mistralai"
version = "2.4.5"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "eval-type-backport" },
{ name = "httpx" },
{ name = "jsonpath-python" },
{ name = "opentelemetry-api" },
{ name = "opentelemetry-semantic-conventions" },
{ name = "pydantic" },
{ name = "python-dateutil" },
{ name = "typing-inspection" },
]
sdist = { url = "https://files.pythonhosted.org/packages/8e/3f/5624d57c5897c83c55d3e4c7dd4127de42ad14fd3183e26566cdc7dca1bf/mistralai-2.4.5.tar.gz", hash = "sha256:ef165bb004ec4423cbf19a440bf0983ca0c3fc92ab12a35ebca097bdf418e33a", size = 424611, upload-time = "2026-05-07T11:46:43.888Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/1b/48/2c5c4f853dec32a625c1a3d23809b80cf2e135c3441fe1764f72910dfea9/mistralai-2.4.5-py3-none-any.whl", hash = "sha256:bf3b6550258ab16dec8547b90e9c18bebf9099f55b7fc25a884bf0bbeffced0f", size = 995999, upload-time = "2026-05-07T11:46:41.915Z" },
]
[[package]] [[package]]
name = "mmh3" name = "mmh3"
version = "5.2.0" version = "5.2.0"
@@ -2086,7 +2123,7 @@ wheels = [
[[package]] [[package]]
name = "nextcloud-mcp-server" name = "nextcloud-mcp-server"
version = "0.81.0" version = "0.83.0"
source = { editable = "." } source = { editable = "." }
dependencies = [ dependencies = [
{ name = "aiosqlite" }, { name = "aiosqlite" },
@@ -2104,6 +2141,7 @@ dependencies = [
{ name = "langchain-text-splitters" }, { name = "langchain-text-splitters" },
{ name = "markdownify" }, { name = "markdownify" },
{ name = "mcp", extra = ["cli"] }, { name = "mcp", extra = ["cli"] },
{ name = "mistralai" },
{ name = "openai" }, { name = "openai" },
{ name = "opentelemetry-api" }, { name = "opentelemetry-api" },
{ name = "opentelemetry-exporter-otlp-proto-grpc" }, { name = "opentelemetry-exporter-otlp-proto-grpc" },
@@ -2157,6 +2195,7 @@ requires-dist = [
{ name = "langchain-text-splitters", specifier = ">=1.0.0" }, { name = "langchain-text-splitters", specifier = ">=1.0.0" },
{ name = "markdownify", specifier = ">=0.14.1" }, { name = "markdownify", specifier = ">=0.14.1" },
{ name = "mcp", extras = ["cli"], specifier = ">=1.27,<1.28" }, { name = "mcp", extras = ["cli"], specifier = ">=1.27,<1.28" },
{ name = "mistralai", specifier = ">=2.4.5" },
{ name = "openai", specifier = ">=2.8.1" }, { name = "openai", specifier = ">=2.8.1" },
{ name = "opentelemetry-api", specifier = ">=1.28.2" }, { name = "opentelemetry-api", specifier = ">=1.28.2" },
{ name = "opentelemetry-exporter-otlp-proto-grpc", specifier = ">=1.28.2" }, { name = "opentelemetry-exporter-otlp-proto-grpc", specifier = ">=1.28.2" },
@@ -2341,45 +2380,45 @@ wheels = [
[[package]] [[package]]
name = "opentelemetry-api" name = "opentelemetry-api"
version = "1.39.0" version = "1.39.1"
source = { registry = "https://pypi.org/simple" } source = { registry = "https://pypi.org/simple" }
dependencies = [ dependencies = [
{ name = "importlib-metadata" }, { name = "importlib-metadata" },
{ name = "typing-extensions" }, { name = "typing-extensions" },
] ]
sdist = { url = "https://files.pythonhosted.org/packages/c0/0b/e5428c009d4d9af0515b0a8371a8aaae695371af291f45e702f7969dce6b/opentelemetry_api-1.39.0.tar.gz", hash = "sha256:6130644268c5ac6bdffaf660ce878f10906b3e789f7e2daa5e169b047a2933b9", size = 65763, upload-time = "2025-12-03T13:19:56.378Z" } sdist = { url = "https://files.pythonhosted.org/packages/97/b9/3161be15bb8e3ad01be8be5a968a9237c3027c5be504362ff800fca3e442/opentelemetry_api-1.39.1.tar.gz", hash = "sha256:fbde8c80e1b937a2c61f20347e91c0c18a1940cecf012d62e65a7caf08967c9c", size = 65767, upload-time = "2025-12-11T13:32:39.182Z" }
wheels = [ wheels = [
{ url = "https://files.pythonhosted.org/packages/05/85/d831a9bc0a9e0e1a304ff3d12c1489a5fbc9bf6690a15dcbdae372bbca45/opentelemetry_api-1.39.0-py3-none-any.whl", hash = "sha256:3c3b3ca5c5687b1b5b37e5c5027ff68eacea8675241b29f13110a8ffbb8f0459", size = 66357, upload-time = "2025-12-03T13:19:33.043Z" }, { url = "https://files.pythonhosted.org/packages/cf/df/d3f1ddf4bb4cb50ed9b1139cc7b1c54c34a1e7ce8fd1b9a37c0d1551a6bd/opentelemetry_api-1.39.1-py3-none-any.whl", hash = "sha256:2edd8463432a7f8443edce90972169b195e7d6a05500cd29e6d13898187c9950", size = 66356, upload-time = "2025-12-11T13:32:17.304Z" },
] ]
[[package]] [[package]]
name = "opentelemetry-exporter-otlp" name = "opentelemetry-exporter-otlp"
version = "1.39.0" version = "1.39.1"
source = { registry = "https://pypi.org/simple" } source = { registry = "https://pypi.org/simple" }
dependencies = [ dependencies = [
{ name = "opentelemetry-exporter-otlp-proto-grpc" }, { name = "opentelemetry-exporter-otlp-proto-grpc" },
{ name = "opentelemetry-exporter-otlp-proto-http" }, { name = "opentelemetry-exporter-otlp-proto-http" },
] ]
sdist = { url = "https://files.pythonhosted.org/packages/13/be/0e9d889f47e55cadc4041e5b53d4e0cc688f9a74811134fb0ba7cbee6905/opentelemetry_exporter_otlp-1.39.0.tar.gz", hash = "sha256:b405da0287b895fe4e2450dedb2a5b072debba1dfcfed5bdb3d1d183d8daa296", size = 6146, upload-time = "2025-12-03T13:19:58.381Z" } sdist = { url = "https://files.pythonhosted.org/packages/30/9c/3ab1db90f32da200dba332658f2bbe602369e3d19f6aba394031a42635be/opentelemetry_exporter_otlp-1.39.1.tar.gz", hash = "sha256:7cf7470e9fd0060c8a38a23e4f695ac686c06a48ad97f8d4867bc9b420180b9c", size = 6147, upload-time = "2025-12-11T13:32:40.309Z" }
wheels = [ wheels = [
{ url = "https://files.pythonhosted.org/packages/fc/35/212d2cae4fa9a2c02e74438612268b640ab577b8ccb04590371eb4e0f542/opentelemetry_exporter_otlp-1.39.0-py3-none-any.whl", hash = "sha256:fe155d6968d581b325574ad6dc267c8de299397b18d11feeda2206d0a47928a9", size = 7017, upload-time = "2025-12-03T13:19:35.686Z" }, { url = "https://files.pythonhosted.org/packages/00/6c/bdc82a066e6fb1dcf9e8cc8d4e026358fe0f8690700cc6369a6bf9bd17a7/opentelemetry_exporter_otlp-1.39.1-py3-none-any.whl", hash = "sha256:68ae69775291f04f000eb4b698ff16ff685fdebe5cb52871bc4e87938a7b00fe", size = 7019, upload-time = "2025-12-11T13:32:19.387Z" },
] ]
[[package]] [[package]]
name = "opentelemetry-exporter-otlp-proto-common" name = "opentelemetry-exporter-otlp-proto-common"
version = "1.39.0" version = "1.39.1"
source = { registry = "https://pypi.org/simple" } source = { registry = "https://pypi.org/simple" }
dependencies = [ dependencies = [
{ name = "opentelemetry-proto" }, { name = "opentelemetry-proto" },
] ]
sdist = { url = "https://files.pythonhosted.org/packages/11/cb/3a29ce606b10c76d413d6edd42d25a654af03e73e50696611e757d2602f3/opentelemetry_exporter_otlp_proto_common-1.39.0.tar.gz", hash = "sha256:a135fceed1a6d767f75be65bd2845da344dd8b9258eeed6bc48509d02b184409", size = 20407, upload-time = "2025-12-03T13:19:59.003Z" } sdist = { url = "https://files.pythonhosted.org/packages/e9/9d/22d241b66f7bbde88a3bfa6847a351d2c46b84de23e71222c6aae25c7050/opentelemetry_exporter_otlp_proto_common-1.39.1.tar.gz", hash = "sha256:763370d4737a59741c89a67b50f9e39271639ee4afc999dadfe768541c027464", size = 20409, upload-time = "2025-12-11T13:32:40.885Z" }
wheels = [ wheels = [
{ url = "https://files.pythonhosted.org/packages/ef/c6/215edba62d13a3948c718b289539f70e40965bc37fc82ecd55bb0b749c1a/opentelemetry_exporter_otlp_proto_common-1.39.0-py3-none-any.whl", hash = "sha256:3d77be7c4bdf90f1a76666c934368b8abed730b5c6f0547a2ec57feb115849ac", size = 18367, upload-time = "2025-12-03T13:19:36.906Z" }, { url = "https://files.pythonhosted.org/packages/8c/02/ffc3e143d89a27ac21fd557365b98bd0653b98de8a101151d5805b5d4c33/opentelemetry_exporter_otlp_proto_common-1.39.1-py3-none-any.whl", hash = "sha256:08f8a5862d64cc3435105686d0216c1365dc5701f86844a8cd56597d0c764fde", size = 18366, upload-time = "2025-12-11T13:32:20.2Z" },
] ]
[[package]] [[package]]
name = "opentelemetry-exporter-otlp-proto-grpc" name = "opentelemetry-exporter-otlp-proto-grpc"
version = "1.39.0" version = "1.39.1"
source = { registry = "https://pypi.org/simple" } source = { registry = "https://pypi.org/simple" }
dependencies = [ dependencies = [
{ name = "googleapis-common-protos" }, { name = "googleapis-common-protos" },
@@ -2390,14 +2429,14 @@ dependencies = [
{ name = "opentelemetry-sdk" }, { name = "opentelemetry-sdk" },
{ name = "typing-extensions" }, { name = "typing-extensions" },
] ]
sdist = { url = "https://files.pythonhosted.org/packages/7e/62/4db083ee9620da3065eeb559e9fc128f41a1d15e7c48d7c83aafbccd354c/opentelemetry_exporter_otlp_proto_grpc-1.39.0.tar.gz", hash = "sha256:7e7bb3f436006836c0e0a42ac619097746ad5553ad7128a5bd4d3e727f37fc06", size = 24650, upload-time = "2025-12-03T13:20:00.06Z" } sdist = { url = "https://files.pythonhosted.org/packages/53/48/b329fed2c610c2c32c9366d9dc597202c9d1e58e631c137ba15248d8850f/opentelemetry_exporter_otlp_proto_grpc-1.39.1.tar.gz", hash = "sha256:772eb1c9287485d625e4dbe9c879898e5253fea111d9181140f51291b5fec3ad", size = 24650, upload-time = "2025-12-11T13:32:41.429Z" }
wheels = [ wheels = [
{ url = "https://files.pythonhosted.org/packages/56/e8/d420b94ffddfd8cff85bb4aa5d98da26ce7935dc3cf3eca6b83cd39ab436/opentelemetry_exporter_otlp_proto_grpc-1.39.0-py3-none-any.whl", hash = "sha256:758641278050de9bb895738f35ff8840e4a47685b7e6ef4a201fe83196ba7a05", size = 19765, upload-time = "2025-12-03T13:19:38.143Z" }, { url = "https://files.pythonhosted.org/packages/81/a3/cc9b66575bd6597b98b886a2067eea2693408d2d5f39dad9ab7fc264f5f3/opentelemetry_exporter_otlp_proto_grpc-1.39.1-py3-none-any.whl", hash = "sha256:fa1c136a05c7e9b4c09f739469cbdb927ea20b34088ab1d959a849b5cc589c18", size = 19766, upload-time = "2025-12-11T13:32:21.027Z" },
] ]
[[package]] [[package]]
name = "opentelemetry-exporter-otlp-proto-http" name = "opentelemetry-exporter-otlp-proto-http"
version = "1.39.0" version = "1.39.1"
source = { registry = "https://pypi.org/simple" } source = { registry = "https://pypi.org/simple" }
dependencies = [ dependencies = [
{ name = "googleapis-common-protos" }, { name = "googleapis-common-protos" },
@@ -2408,14 +2447,14 @@ dependencies = [
{ name = "requests" }, { name = "requests" },
{ name = "typing-extensions" }, { name = "typing-extensions" },
] ]
sdist = { url = "https://files.pythonhosted.org/packages/81/dc/1e9bf3f6a28e29eba516bc0266e052996d02bc7e92675f3cd38169607609/opentelemetry_exporter_otlp_proto_http-1.39.0.tar.gz", hash = "sha256:28d78fc0eb82d5a71ae552263d5012fa3ebad18dfd189bf8d8095ba0e65ee1ed", size = 17287, upload-time = "2025-12-03T13:20:01.134Z" } sdist = { url = "https://files.pythonhosted.org/packages/80/04/2a08fa9c0214ae38880df01e8bfae12b067ec0793446578575e5080d6545/opentelemetry_exporter_otlp_proto_http-1.39.1.tar.gz", hash = "sha256:31bdab9745c709ce90a49a0624c2bd445d31a28ba34275951a6a362d16a0b9cb", size = 17288, upload-time = "2025-12-11T13:32:42.029Z" }
wheels = [ wheels = [
{ url = "https://files.pythonhosted.org/packages/bc/46/e4a102e17205bb05a50dbf24ef0e92b66b648cd67db9a68865af06a242fd/opentelemetry_exporter_otlp_proto_http-1.39.0-py3-none-any.whl", hash = "sha256:5789cb1375a8b82653328c0ce13a054d285f774099faf9d068032a49de4c7862", size = 19639, upload-time = "2025-12-03T13:19:39.536Z" }, { url = "https://files.pythonhosted.org/packages/95/f1/b27d3e2e003cd9a3592c43d099d2ed8d0a947c15281bf8463a256db0b46c/opentelemetry_exporter_otlp_proto_http-1.39.1-py3-none-any.whl", hash = "sha256:d9f5207183dd752a412c4cd564ca8875ececba13be6e9c6c370ffb752fd59985", size = 19641, upload-time = "2025-12-11T13:32:22.248Z" },
] ]
[[package]] [[package]]
name = "opentelemetry-instrumentation" name = "opentelemetry-instrumentation"
version = "0.60b0" version = "0.60b1"
source = { registry = "https://pypi.org/simple" } source = { registry = "https://pypi.org/simple" }
dependencies = [ dependencies = [
{ name = "opentelemetry-api" }, { name = "opentelemetry-api" },
@@ -2423,14 +2462,14 @@ dependencies = [
{ name = "packaging" }, { name = "packaging" },
{ name = "wrapt" }, { name = "wrapt" },
] ]
sdist = { url = "https://files.pythonhosted.org/packages/55/3c/bd53dbb42eff93d18e3047c7be11224aa9966ce98ac4cc5bfb860a32c95a/opentelemetry_instrumentation-0.60b0.tar.gz", hash = "sha256:4e9fec930f283a2677a2217754b40aaf9ef76edae40499c165bc7f1d15366a74", size = 31707, upload-time = "2025-12-03T13:22:00.352Z" } sdist = { url = "https://files.pythonhosted.org/packages/41/0f/7e6b713ac117c1f5e4e3300748af699b9902a2e5e34c9cf443dde25a01fa/opentelemetry_instrumentation-0.60b1.tar.gz", hash = "sha256:57ddc7974c6eb35865af0426d1a17132b88b2ed8586897fee187fd5b8944bd6a", size = 31706, upload-time = "2025-12-11T13:36:42.515Z" }
wheels = [ wheels = [
{ url = "https://files.pythonhosted.org/packages/5c/7b/5b5b9f8cfe727a28553acf9cd287b1d7f706f5c0a00d6e482df55b169483/opentelemetry_instrumentation-0.60b0-py3-none-any.whl", hash = "sha256:aaafa1483543a402819f1bdfb06af721c87d60dd109501f9997332862a35c76a", size = 33096, upload-time = "2025-12-03T13:20:51.785Z" }, { url = "https://files.pythonhosted.org/packages/77/d2/6788e83c5c86a2690101681aeef27eeb2a6bf22df52d3f263a22cee20915/opentelemetry_instrumentation-0.60b1-py3-none-any.whl", hash = "sha256:04480db952b48fb1ed0073f822f0ee26012b7be7c3eac1a3793122737c78632d", size = 33096, upload-time = "2025-12-11T13:35:33.067Z" },
] ]
[[package]] [[package]]
name = "opentelemetry-instrumentation-asgi" name = "opentelemetry-instrumentation-asgi"
version = "0.60b0" version = "0.60b1"
source = { registry = "https://pypi.org/simple" } source = { registry = "https://pypi.org/simple" }
dependencies = [ dependencies = [
{ name = "asgiref" }, { name = "asgiref" },
@@ -2439,14 +2478,14 @@ dependencies = [
{ name = "opentelemetry-semantic-conventions" }, { name = "opentelemetry-semantic-conventions" },
{ name = "opentelemetry-util-http" }, { name = "opentelemetry-util-http" },
] ]
sdist = { url = "https://files.pythonhosted.org/packages/b0/0a/715ea7044708d3c215385fb2a1c6ffe429aacb3cd23a348060aaeda52834/opentelemetry_instrumentation_asgi-0.60b0.tar.gz", hash = "sha256:928731218050089dca69f0fe980b8bfe109f384be8b89802d7337372ddb67b91", size = 26083, upload-time = "2025-12-03T13:22:05.672Z" } sdist = { url = "https://files.pythonhosted.org/packages/77/db/851fa88db7441da82d50bd80f2de5ee55213782e25dc858e04d0c9961d60/opentelemetry_instrumentation_asgi-0.60b1.tar.gz", hash = "sha256:16bfbe595cd24cda309a957456d0fc2523f41bc7b076d1f2d7e98a1ad9876d6f", size = 26107, upload-time = "2025-12-11T13:36:47.015Z" }
wheels = [ wheels = [
{ url = "https://files.pythonhosted.org/packages/9b/8c/c6c59127fd996107243ca45669355665a7daff578ddafb86d6d2d3b01428/opentelemetry_instrumentation_asgi-0.60b0-py3-none-any.whl", hash = "sha256:9d76a541269452c718a0384478f3291feb650c5a3f29e578fdc6613ea3729cf3", size = 16907, upload-time = "2025-12-03T13:20:58.962Z" }, { url = "https://files.pythonhosted.org/packages/76/76/1fb94367cef64420d2171157a6b9509582873bd09a6afe08a78a8d1f59d9/opentelemetry_instrumentation_asgi-0.60b1-py3-none-any.whl", hash = "sha256:d48def2dbed10294c99cfcf41ebbd0c414d390a11773a41f472d20000fcddc25", size = 16933, upload-time = "2025-12-11T13:35:40.462Z" },
] ]
[[package]] [[package]]
name = "opentelemetry-instrumentation-httpx" name = "opentelemetry-instrumentation-httpx"
version = "0.60b0" version = "0.60b1"
source = { registry = "https://pypi.org/simple" } source = { registry = "https://pypi.org/simple" }
dependencies = [ dependencies = [
{ name = "opentelemetry-api" }, { name = "opentelemetry-api" },
@@ -2455,70 +2494,70 @@ dependencies = [
{ name = "opentelemetry-util-http" }, { name = "opentelemetry-util-http" },
{ name = "wrapt" }, { name = "wrapt" },
] ]
sdist = { url = "https://files.pythonhosted.org/packages/09/71/9dc0bc5ab14122251f520ffe9fc8dd892ca688d7d591482b5b1843d685d2/opentelemetry_instrumentation_httpx-0.60b0.tar.gz", hash = "sha256:fcf349a92fb0b941a2a18bec65141f4ba62cbf7a457a1aa580794bad44dc477c", size = 20612, upload-time = "2025-12-03T13:22:20.5Z" } sdist = { url = "https://files.pythonhosted.org/packages/86/08/11208bcfcab4fc2023252c3f322aa397fd9ad948355fea60f5fc98648603/opentelemetry_instrumentation_httpx-0.60b1.tar.gz", hash = "sha256:a506ebaf28c60112cbe70ad4f0338f8603f148938cb7b6794ce1051cd2b270ae", size = 20611, upload-time = "2025-12-11T13:37:01.661Z" }
wheels = [ wheels = [
{ url = "https://files.pythonhosted.org/packages/3a/22/a340c8bfad6f31bfd6a15b0b24d5e68e05e3975e4dbdc3cea6ec4f96e060/opentelemetry_instrumentation_httpx-0.60b0-py3-none-any.whl", hash = "sha256:3f5e6fc4ddf1d9de2aaddb5255110827154dbd4de9187da906c8c2a3cc2219e9", size = 15702, upload-time = "2025-12-03T13:21:20.667Z" }, { url = "https://files.pythonhosted.org/packages/43/59/b98e84eebf745ffc75397eaad4763795bff8a30cbf2373a50ed4e70646c5/opentelemetry_instrumentation_httpx-0.60b1-py3-none-any.whl", hash = "sha256:f37636dd742ad2af83d896ba69601ed28da51fa4e25d1ab62fde89ce413e275b", size = 15701, upload-time = "2025-12-11T13:36:04.56Z" },
] ]
[[package]] [[package]]
name = "opentelemetry-instrumentation-logging" name = "opentelemetry-instrumentation-logging"
version = "0.60b0" version = "0.60b1"
source = { registry = "https://pypi.org/simple" } source = { registry = "https://pypi.org/simple" }
dependencies = [ dependencies = [
{ name = "opentelemetry-api" }, { name = "opentelemetry-api" },
{ name = "opentelemetry-instrumentation" }, { name = "opentelemetry-instrumentation" },
] ]
sdist = { url = "https://files.pythonhosted.org/packages/5b/8b/a1aed0b695a58a1f0bdcf90fae5e468cc0fd7de2b74b9816b17e45c38e43/opentelemetry_instrumentation_logging-0.60b0.tar.gz", hash = "sha256:6d87840666669cbbcd53d2230c7a33476862d0bf7f1adc67a95519c6ccfc8281", size = 9969, upload-time = "2025-12-03T13:22:22.608Z" } sdist = { url = "https://files.pythonhosted.org/packages/60/a6/4515895b383113677fd2ad21813df5e56108a2df14ebb7916c962c9a0234/opentelemetry_instrumentation_logging-0.60b1.tar.gz", hash = "sha256:98f4b9c7aeb9314a30feee7c002c7ea9abea07c90df5f97fb058b850bc45b89a", size = 9968, upload-time = "2025-12-11T13:37:03.974Z" }
wheels = [ wheels = [
{ url = "https://files.pythonhosted.org/packages/00/89/6d4f5d8d03376637bff5b8d9e91356104a6f1ce9a34a2099bb2f4bf5e2b5/opentelemetry_instrumentation_logging-0.60b0-py3-none-any.whl", hash = "sha256:af75b3020911b9b6a1b4b19819a165eb131ce9bfdd313062d578fc2dc9a5cd0f", size = 12576, upload-time = "2025-12-03T13:21:24.797Z" }, { url = "https://files.pythonhosted.org/packages/f1/f9/8a4ce3901bc52277794e4b18c4ac43dc5929806eff01d22812364132f45f/opentelemetry_instrumentation_logging-0.60b1-py3-none-any.whl", hash = "sha256:f2e18cbc7e1dd3628c80e30d243897fdc93c5b7e0c8ae60abd2b9b6a99f82343", size = 12577, upload-time = "2025-12-11T13:36:08.123Z" },
] ]
[[package]] [[package]]
name = "opentelemetry-proto" name = "opentelemetry-proto"
version = "1.39.0" version = "1.39.1"
source = { registry = "https://pypi.org/simple" } source = { registry = "https://pypi.org/simple" }
dependencies = [ dependencies = [
{ name = "protobuf" }, { name = "protobuf" },
] ]
sdist = { url = "https://files.pythonhosted.org/packages/48/b5/64d2f8c3393cd13ea2092106118f7b98461ba09333d40179a31444c6f176/opentelemetry_proto-1.39.0.tar.gz", hash = "sha256:c1fa48678ad1a1624258698e59be73f990b7fc1f39e73e16a9d08eef65dd838c", size = 46153, upload-time = "2025-12-03T13:20:08.729Z" } sdist = { url = "https://files.pythonhosted.org/packages/49/1d/f25d76d8260c156c40c97c9ed4511ec0f9ce353f8108ca6e7561f82a06b2/opentelemetry_proto-1.39.1.tar.gz", hash = "sha256:6c8e05144fc0d3ed4d22c2289c6b126e03bcd0e6a7da0f16cedd2e1c2772e2c8", size = 46152, upload-time = "2025-12-11T13:32:48.681Z" }
wheels = [ wheels = [
{ url = "https://files.pythonhosted.org/packages/e3/4d/d500e1862beed68318705732d1976c390f4a72ca8009c4983ff627acff20/opentelemetry_proto-1.39.0-py3-none-any.whl", hash = "sha256:1e086552ac79acb501485ff0ce75533f70f3382d43d0a30728eeee594f7bf818", size = 72534, upload-time = "2025-12-03T13:19:50.251Z" }, { url = "https://files.pythonhosted.org/packages/51/95/b40c96a7b5203005a0b03d8ce8cd212ff23f1793d5ba289c87a097571b18/opentelemetry_proto-1.39.1-py3-none-any.whl", hash = "sha256:22cdc78efd3b3765d09e68bfbd010d4fc254c9818afd0b6b423387d9dee46007", size = 72535, upload-time = "2025-12-11T13:32:33.866Z" },
] ]
[[package]] [[package]]
name = "opentelemetry-sdk" name = "opentelemetry-sdk"
version = "1.39.0" version = "1.39.1"
source = { registry = "https://pypi.org/simple" } source = { registry = "https://pypi.org/simple" }
dependencies = [ dependencies = [
{ name = "opentelemetry-api" }, { name = "opentelemetry-api" },
{ name = "opentelemetry-semantic-conventions" }, { name = "opentelemetry-semantic-conventions" },
{ name = "typing-extensions" }, { name = "typing-extensions" },
] ]
sdist = { url = "https://files.pythonhosted.org/packages/51/e3/7cd989003e7cde72e0becfe830abff0df55c69d237ee7961a541e0167833/opentelemetry_sdk-1.39.0.tar.gz", hash = "sha256:c22204f12a0529e07aa4d985f1bca9d6b0e7b29fe7f03e923548ae52e0e15dde", size = 171322, upload-time = "2025-12-03T13:20:09.651Z" } sdist = { url = "https://files.pythonhosted.org/packages/eb/fb/c76080c9ba07e1e8235d24cdcc4d125ef7aa3edf23eb4e497c2e50889adc/opentelemetry_sdk-1.39.1.tar.gz", hash = "sha256:cf4d4563caf7bff906c9f7967e2be22d0d6b349b908be0d90fb21c8e9c995cc6", size = 171460, upload-time = "2025-12-11T13:32:49.369Z" }
wheels = [ wheels = [
{ url = "https://files.pythonhosted.org/packages/a4/b4/2adc8bc83eb1055ecb592708efb6f0c520cc2eb68970b02b0f6ecda149cf/opentelemetry_sdk-1.39.0-py3-none-any.whl", hash = "sha256:90cfb07600dfc0d2de26120cebc0c8f27e69bf77cd80ef96645232372709a514", size = 132413, upload-time = "2025-12-03T13:19:51.364Z" }, { url = "https://files.pythonhosted.org/packages/7c/98/e91cf858f203d86f4eccdf763dcf01cf03f1dae80c3750f7e635bfa206b6/opentelemetry_sdk-1.39.1-py3-none-any.whl", hash = "sha256:4d5482c478513ecb0a5d938dcc61394e647066e0cc2676bee9f3af3f3f45f01c", size = 132565, upload-time = "2025-12-11T13:32:35.069Z" },
] ]
[[package]] [[package]]
name = "opentelemetry-semantic-conventions" name = "opentelemetry-semantic-conventions"
version = "0.60b0" version = "0.60b1"
source = { registry = "https://pypi.org/simple" } source = { registry = "https://pypi.org/simple" }
dependencies = [ dependencies = [
{ name = "opentelemetry-api" }, { name = "opentelemetry-api" },
{ name = "typing-extensions" }, { name = "typing-extensions" },
] ]
sdist = { url = "https://files.pythonhosted.org/packages/71/0e/176a7844fe4e3cb5de604212094dffaed4e18b32f1c56b5258bcbcba85c2/opentelemetry_semantic_conventions-0.60b0.tar.gz", hash = "sha256:227d7aa73cbb8a2e418029d6b6465553aa01cf7e78ec9d0bc3255c7b3ac5bf8f", size = 137935, upload-time = "2025-12-03T13:20:12.395Z" } sdist = { url = "https://files.pythonhosted.org/packages/91/df/553f93ed38bf22f4b999d9be9c185adb558982214f33eae539d3b5cd0858/opentelemetry_semantic_conventions-0.60b1.tar.gz", hash = "sha256:87c228b5a0669b748c76d76df6c364c369c28f1c465e50f661e39737e84bc953", size = 137935, upload-time = "2025-12-11T13:32:50.487Z" }
wheels = [ wheels = [
{ url = "https://files.pythonhosted.org/packages/d0/56/af0306666f91bae47db14d620775604688361f0f76a872e0005277311131/opentelemetry_semantic_conventions-0.60b0-py3-none-any.whl", hash = "sha256:069530852691136018087b52688857d97bba61cd641d0f8628d2d92788c4f78a", size = 219981, upload-time = "2025-12-03T13:19:53.585Z" }, { url = "https://files.pythonhosted.org/packages/7a/5e/5958555e09635d09b75de3c4f8b9cae7335ca545d77392ffe7331534c402/opentelemetry_semantic_conventions-0.60b1-py3-none-any.whl", hash = "sha256:9fa8c8b0c110da289809292b0591220d3a7b53c1526a23021e977d68597893fb", size = 219982, upload-time = "2025-12-11T13:32:36.955Z" },
] ]
[[package]] [[package]]
name = "opentelemetry-util-http" name = "opentelemetry-util-http"
version = "0.60b0" version = "0.60b1"
source = { registry = "https://pypi.org/simple" } source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/38/0d/786a713445cf338131fef3a84fab1378e4b2ef3c3ea348eeb0c915eb804a/opentelemetry_util_http-0.60b0.tar.gz", hash = "sha256:e42b7bb49bba43b6f34390327d97e5016eb1c47949ceaf37c4795472a4e3a82d", size = 10576, upload-time = "2025-12-03T13:22:41.224Z" } sdist = { url = "https://files.pythonhosted.org/packages/50/fc/c47bb04a1d8a941a4061307e1eddfa331ed4d0ab13d8a9781e6db256940a/opentelemetry_util_http-0.60b1.tar.gz", hash = "sha256:0d97152ca8c8a41ced7172d29d3622a219317f74ae6bb3027cfbdcf22c3cc0d6", size = 11053, upload-time = "2025-12-11T13:37:25.115Z" }
wheels = [ wheels = [
{ url = "https://files.pythonhosted.org/packages/53/5d/a448862f6d10c95685ed0e703596b6bd1784074e7ad90bffdc550abb7b68/opentelemetry_util_http-0.60b0-py3-none-any.whl", hash = "sha256:4f366f1a48adb74ffa6f80aee26f96882e767e01b03cd1cfb948b6e1020341fe", size = 8742, upload-time = "2025-12-03T13:21:54.553Z" }, { url = "https://files.pythonhosted.org/packages/16/5c/d3f1733665f7cd582ef0842fb1d2ed0bc1fba10875160593342d22bba375/opentelemetry_util_http-0.60b1-py3-none-any.whl", hash = "sha256:66381ba28550c91bee14dcba8979ace443444af1ed609226634596b4b0faf199", size = 8947, upload-time = "2025-12-11T13:36:37.151Z" },
] ]
[[package]] [[package]]
@@ -3314,7 +3353,7 @@ wheels = [
[[package]] [[package]]
name = "pytest-otel" name = "pytest-otel"
version = "2.0.1" version = "2.0.3"
source = { registry = "https://pypi.org/simple" } source = { registry = "https://pypi.org/simple" }
dependencies = [ dependencies = [
{ name = "opentelemetry-api" }, { name = "opentelemetry-api" },
@@ -3322,9 +3361,9 @@ dependencies = [
{ name = "opentelemetry-sdk" }, { name = "opentelemetry-sdk" },
{ name = "pytest" }, { name = "pytest" },
] ]
sdist = { url = "https://files.pythonhosted.org/packages/ca/5e/771f8dbdf55ae57603d89bb26e046c13e8bee3492c74cb3afb6044166e9a/pytest_otel-2.0.1.tar.gz", hash = "sha256:3d529dc34105862cca39fd1258d00dd3d17f3b7d92ebf0953d55326bb017af3d", size = 17880, upload-time = "2025-12-08T14:56:40.393Z" } sdist = { url = "https://files.pythonhosted.org/packages/98/14/e4f7cb90c4c93dbdd85034fd65e7a16da328bd71c6ca5270e4f8d37fa4ba/pytest_otel-2.0.3.tar.gz", hash = "sha256:782985fef50acc6922db5e69da4584b384d785c863a96ebc11eaf2a7fef5189c", size = 18587, upload-time = "2026-01-02T16:12:20.394Z" }
wheels = [ wheels = [
{ url = "https://files.pythonhosted.org/packages/16/fa/0cfe23bac571f68b4f427ad7ae313ddd8de881ec2ff49dcea196a2a96592/pytest_otel-2.0.1-py2.py3-none-any.whl", hash = "sha256:501f36f02f55578ca34c3ccbe55e81cc1e14bb130da13f7cf0b6480d54a9e5db", size = 14530, upload-time = "2025-12-08T14:56:41.731Z" }, { url = "https://files.pythonhosted.org/packages/ac/f8/d9b93b41b299c7e14b47887eb31fb4cb77d771c2e9f3e0bcac0750f62699/pytest_otel-2.0.3-py2.py3-none-any.whl", hash = "sha256:3c6c331e943609ad7df7c718714090dee91211340f133a8798217e72e3b2cd67", size = 14914, upload-time = "2026-01-02T16:12:21.946Z" },
] ]
[[package]] [[package]]