Merge remote-tracking branch 'origin/master' into fix/chunk-context-indexed-lookup

# Conflicts:
#	nextcloud_mcp_server/api/visualization.py
#	nextcloud_mcp_server/auth/viz_routes.py
This commit is contained in:
Chris Coutinho
2026-05-08 23:10:09 +02:00
27 changed files with 1728 additions and 282 deletions
@@ -0,0 +1,5 @@
#!/bin/bash
if ! command -v sonar &> /dev/null; then
exit 0
fi
sonar hook claude-pre-tool-use
@@ -0,0 +1,5 @@
#!/bin/bash
if ! command -v sonar &> /dev/null; then
exit 0
fi
sonar hook claude-prompt-submit
+28
View File
@@ -0,0 +1,28 @@
{
"hooks": {
"PreToolUse": [
{
"matcher": "Read",
"hooks": [
{
"type": "command",
"command": ".claude/hooks/sonar-secrets/build-scripts/pretool-secrets.sh",
"timeout": 60
}
]
}
],
"UserPromptSubmit": [
{
"matcher": "*",
"hooks": [
{
"type": "command",
"command": ".claude/hooks/sonar-secrets/build-scripts/prompt-secrets.sh",
"timeout": 60
}
]
}
]
}
}
+1
View File
@@ -7,6 +7,7 @@ description: |
in this repo's automated PR reviews. Use when the user is about to push, says "ready
to push", "review my work", "check before PR", or invokes /pre-push-review.
Report-only — does not modify code.
model: sonnet
allowed-tools:
- Bash
- Read
+24
View File
@@ -5,6 +5,30 @@ All notable changes to the Nextcloud MCP Server will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [PEP 440](https://peps.python.org/pep-0440/).
## v0.83.0 (2026-05-08)
### Feat
- **vector**: replace inline page-image payloads with chunk_bbox (Deck #76)
### Refactor
- **vector**: address PR #775 review round 3 — fix unused var, harden boundary lookup, rename trace span
- **vector**: address PR #775 review round 2 — drop dead page field, add omission tests
- **vector**: address PR #775 review — drop unused payload key, fix resource leaks
## v0.82.0 (2026-05-08)
### Feat
- **providers**: add Mistral embedding provider, route registry through dynaconf
### Refactor
- **providers**: address PR #772 review round 3 — hermetic test, lazy logging, defensive-guard tests
- **providers**: address PR #772 review round 2 — guard, naming, docs, tests
- **providers**: address PR #772 review — shared retry, cleaner imports, no-op close
## v0.81.0 (2026-05-07)
### Feat
+1 -1
View File
@@ -334,7 +334,7 @@ services:
- claude-funnel
qdrant:
image: docker.io/qdrant/qdrant:v1.17.1@sha256:94728574965d17c6485dd361aa3c0818b325b9016dac5ea6afec7b4b2700865f
image: docker.io/qdrant/qdrant:v1.18.0@sha256:1cf3e07c9f269030c7cfed0d5c0beea7bb081848a88e2ae13b35a613d4dd5019
restart: always
ports:
- 127.0.0.1:6333:6333 # REST API
+19 -2
View File
@@ -118,10 +118,16 @@ class ProviderRegistry:
@staticmethod
def create_provider() -> Provider:
# 1. Bedrock (AWS_REGION or BEDROCK_*_MODEL)
# 2. Ollama (OLLAMA_BASE_URL)
# 3. Simple (fallback)
# 2. OpenAI (OPENAI_API_KEY)
# 3. Mistral (MISTRAL_API_KEY)
# 4. Ollama (OLLAMA_BASE_URL)
# 5. Simple (fallback)
```
Configuration is sourced via the dynaconf-backed `Settings` dataclass in
`config.py`; the registry reads `get_settings()` rather than `os.getenv`
directly, so settings files and env vars share one resolution path.
**Environment Variables:**
**Bedrock:**
@@ -131,6 +137,17 @@ class ProviderRegistry:
- `BEDROCK_EMBEDDING_MODEL`: Model ID for embeddings (e.g., "amazon.titan-embed-text-v2:0")
- `BEDROCK_GENERATION_MODEL`: Model ID for text generation (e.g., "anthropic.claude-3-sonnet-20240229-v1:0")
**OpenAI:**
- `OPENAI_API_KEY`: OpenAI API key (or `GITHUB_TOKEN` for GitHub Models)
- `OPENAI_BASE_URL`: Optional base URL override for OpenAI-compatible APIs
- `OPENAI_EMBEDDING_MODEL`: Embedding model (default: "text-embedding-3-small")
- `OPENAI_GENERATION_MODEL`: Generation model (e.g., "gpt-4o-mini")
**Mistral (embeddings only):**
- `MISTRAL_API_KEY`: Mistral API key from console.mistral.ai
- `MISTRAL_EMBEDDING_MODEL`: Embedding model (default: "mistral-embed", 1024-dim)
- `MISTRAL_BASE_URL`: Optional server URL override (proxies, on-prem)
**Ollama:**
- `OLLAMA_BASE_URL`: Ollama API base URL (e.g., "http://localhost:11434")
- `OLLAMA_EMBEDDING_MODEL`: Model for embeddings (default: "nomic-embed-text")
+67 -3
View File
@@ -410,9 +410,16 @@ DOCUMENT_CHUNK_OVERLAP=50 # Overlapping words between chunks (defaul
### Embedding Service Configuration
The server uses an embedding service to generate vector representations. Two options are available:
The server picks an embedding provider via auto-detection. Priority order
(see `nextcloud_mcp_server/providers/registry.py`):
#### Ollama (Recommended)
1. **Bedrock** — if `AWS_REGION` or `BEDROCK_EMBEDDING_MODEL` is set
2. **OpenAI** — if `OPENAI_API_KEY` is set
3. **Mistral** — if `MISTRAL_API_KEY` is set
4. **Ollama** — if `OLLAMA_BASE_URL` is set
5. **Simple** — fallback when nothing else is configured
#### Ollama (Recommended for self-hosted)
Use a local Ollama instance for embeddings:
@@ -422,9 +429,52 @@ OLLAMA_EMBEDDING_MODEL=nomic-embed-text # Default model
OLLAMA_VERIFY_SSL=true # Verify SSL certificates
```
#### OpenAI
Hosted OpenAI embeddings (or any OpenAI-compatible API via `OPENAI_BASE_URL`):
```dotenv
OPENAI_API_KEY=sk-...
OPENAI_EMBEDDING_MODEL=text-embedding-3-small # default
# OPENAI_BASE_URL=https://models.github.ai/inference # optional
```
#### Mistral
Hosted Mistral embeddings. Requires a Mistral API key from
[console.mistral.ai](https://console.mistral.ai). Currently embeddings only
(no text generation).
```dotenv
MISTRAL_API_KEY=...
MISTRAL_EMBEDDING_MODEL=mistral-embed # default; produces 1024-dim vectors
# MISTRAL_BASE_URL=https://api.mistral.ai # optional override (proxies, on-prem)
```
Switching to or from Mistral forces a new Qdrant collection because the
collection name encodes the model (see "Qdrant Collection Naming" above).
#### Amazon Bedrock
Bedrock provides hosted embedding models (Titan, Cohere) and uses the AWS
credential chain (env vars, profiles, or IAM role):
```dotenv
AWS_REGION=us-east-1
BEDROCK_EMBEDDING_MODEL=amazon.titan-embed-text-v2:0
# AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY are optional — boto3 will use
# the standard credential chain if not set.
```
#### Simple Embedding Provider (Fallback)
If `OLLAMA_BASE_URL` is not set, the server uses a simple random embedding provider for testing. This is **not suitable for production** as it generates random embeddings with no semantic meaning.
If no provider env var is set, the server falls back to a simple deterministic
embedding provider for testing. This is **not suitable for production** as
its embeddings have no semantic meaning.
```dotenv
SIMPLE_EMBEDDING_DIMENSION=384 # optional; default 384
```
### Document Chunking Configuration
@@ -533,7 +583,21 @@ equivalent.** Operators who need a runtime toggle should open an issue.
| `VECTOR_SYNC_QUEUE_MAX_SIZE` | ⚠️ Optional | `10000` | Max queued documents |
| `OLLAMA_BASE_URL` | ⚠️ Optional | - | Ollama API endpoint for embeddings |
| `OLLAMA_EMBEDDING_MODEL` | ⚠️ Optional | `nomic-embed-text` | Embedding model to use |
| `OLLAMA_GENERATION_MODEL` | ⚠️ Optional | - | Ollama model for text generation |
| `OLLAMA_VERIFY_SSL` | ⚠️ Optional | `true` | Verify SSL certificates |
| `OPENAI_API_KEY` | ⚠️ Optional | - | OpenAI API key (selects OpenAI provider) |
| `OPENAI_BASE_URL` | ⚠️ Optional | - | OpenAI base URL override (for compatible APIs) |
| `OPENAI_EMBEDDING_MODEL` | ⚠️ Optional | `text-embedding-3-small` | OpenAI embedding model |
| `OPENAI_GENERATION_MODEL` | ⚠️ Optional | - | OpenAI model for text generation |
| `MISTRAL_API_KEY` | ⚠️ Optional | - | Mistral API key (selects Mistral provider) |
| `MISTRAL_EMBEDDING_MODEL` | ⚠️ Optional | `mistral-embed` | Mistral embedding model (1024-dim) |
| `MISTRAL_BASE_URL` | ⚠️ Optional | - | Mistral base URL override (proxies, on-prem) |
| `AWS_REGION` | ⚠️ Optional | - | AWS region (selects Bedrock provider) |
| `AWS_ACCESS_KEY_ID` | ⚠️ Optional | - | AWS access key (boto3 credential chain fallback) |
| `AWS_SECRET_ACCESS_KEY` | ⚠️ Optional | - | AWS secret key (boto3 credential chain fallback) |
| `BEDROCK_EMBEDDING_MODEL` | ⚠️ Optional | - | Bedrock embedding model ID |
| `BEDROCK_GENERATION_MODEL` | ⚠️ Optional | - | Bedrock generation model ID |
| `SIMPLE_EMBEDDING_DIMENSION` | ⚠️ Optional | `384` | Dimension for the fallback Simple provider |
| `DOCUMENT_CHUNK_SIZE` | ⚠️ Optional | `512` | Words per chunk for document embedding |
| `DOCUMENT_CHUNK_OVERLAP` | ⚠️ Optional | `50` | Overlapping words between chunks (must be < chunk size) |
+11 -18
View File
@@ -565,9 +565,10 @@ async def get_chunk_context(request: Request) -> JSONResponse:
status_code=404,
)
# For PDF files, also fetch the highlighted page image from Qdrant if available
# This is useful for clients that want to show a pre-rendered image
highlighted_page_image = None
# For PDF files, also fetch the chunk's bounding box from Qdrant if
# available so the client can overlay a highlight on top of a
# render-on-demand page image (Deck #76).
chunk_bbox = None
page_number = chunk_context.page_number
if doc_type == "file":
@@ -575,7 +576,7 @@ async def get_chunk_context(request: Request) -> JSONResponse:
settings = get_settings()
qdrant_client = await get_qdrant_client()
# Prefer chunk_index for the highlighted-image lookup (always indexed);
# Prefer chunk_index for the chunk-bbox lookup (always indexed);
# fall back to (chunk_start_offset, chunk_end_offset) when not provided.
if chunk_index is not None:
points_response = await qdrant_client.scroll(
@@ -589,10 +590,6 @@ async def get_chunk_context(request: Request) -> JSONResponse:
FieldCondition(
key="user_id", match=MatchValue(value=user_id)
),
FieldCondition(
key="doc_type",
match=MatchValue(value=doc_type),
),
FieldCondition(
key="chunk_index",
match=MatchValue(value=chunk_index),
@@ -601,7 +598,7 @@ async def get_chunk_context(request: Request) -> JSONResponse:
),
limit=1,
with_vectors=False,
with_payload=["highlighted_page_image", "page_number"],
with_payload=["chunk_bbox", "page_number"],
)
else:
points_response = await qdrant_client.scroll(
@@ -615,10 +612,6 @@ async def get_chunk_context(request: Request) -> JSONResponse:
FieldCondition(
key="user_id", match=MatchValue(value=user_id)
),
FieldCondition(
key="doc_type",
match=MatchValue(value=doc_type),
),
FieldCondition(
key="chunk_start_offset",
match=MatchValue(value=start),
@@ -631,19 +624,19 @@ async def get_chunk_context(request: Request) -> JSONResponse:
),
limit=1,
with_vectors=False,
with_payload=["highlighted_page_image", "page_number"],
with_payload=["chunk_bbox", "page_number"],
)
if points_response[0]:
payload = points_response[0][0].payload
if payload:
highlighted_page_image = payload.get("highlighted_page_image")
chunk_bbox = payload.get("chunk_bbox")
# Trust Qdrant page number if available (might be more accurate than context expansion logic)
if payload.get("page_number") is not None:
page_number = payload.get("page_number")
except Exception as e:
logger.warning(f"Failed to fetch highlighted image: {e}")
logger.warning(f"Failed to fetch chunk bbox: {e}")
# Build response
response_data = {
@@ -658,8 +651,8 @@ async def get_chunk_context(request: Request) -> JSONResponse:
"total_chunks": chunk_context.total_chunks,
}
if highlighted_page_image:
response_data["highlighted_page_image"] = highlighted_page_image
if chunk_bbox:
response_data["chunk_bbox"] = chunk_bbox
return JSONResponse(response_data)
+2 -1
View File
@@ -125,12 +125,13 @@ from nextcloud_mcp_server.server import (
)
from nextcloud_mcp_server.server.auth_tools import register_auth_tools
from nextcloud_mcp_server.server.oauth_tools import register_oauth_tools
from nextcloud_mcp_server.vector import processor_task, scanner_task
from nextcloud_mcp_server.vector.oauth_sync import (
oauth_processor_task,
user_manager_task,
)
from nextcloud_mcp_server.vector.processor import processor_task
from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client
from nextcloud_mcp_server.vector.scanner import scanner_task
from nextcloud_mcp_server.vector.webhook_receiver import handle_nextcloud_webhook
logger = logging.getLogger(__name__)
+14 -22
View File
@@ -623,8 +623,10 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse:
f"after_len={len(chunk_context.after_context)}"
)
# For PDF files, also fetch the highlighted page image from Qdrant
highlighted_page_image = None
# For PDF files, also fetch the chunk bbox from Qdrant so the client
# can overlay a highlight on top of a render-on-demand page image
# (Deck #76).
chunk_bbox = None
page_number = chunk_context.page_number
if doc_type == "file":
try:
@@ -632,7 +634,7 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse:
qdrant_client = await get_qdrant_client()
username = request.user.display_name
# Prefer chunk_index for the highlighted-image lookup (always indexed);
# Prefer chunk_index for the chunk-bbox lookup (always indexed);
# fall back to (chunk_start_offset, chunk_end_offset) when not provided.
if chunk_index is not None:
points_response = await qdrant_client.scroll(
@@ -646,10 +648,6 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse:
FieldCondition(
key="user_id", match=MatchValue(value=username)
),
FieldCondition(
key="doc_type",
match=MatchValue(value=doc_type),
),
FieldCondition(
key="chunk_index",
match=MatchValue(value=chunk_index),
@@ -658,7 +656,7 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse:
),
limit=1,
with_vectors=False,
with_payload=["highlighted_page_image", "page_number"],
with_payload=["chunk_bbox", "page_number"],
)
else:
points_response = await qdrant_client.scroll(
@@ -672,10 +670,6 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse:
FieldCondition(
key="user_id", match=MatchValue(value=username)
),
FieldCondition(
key="doc_type",
match=MatchValue(value=doc_type),
),
FieldCondition(
key="chunk_start_offset",
match=MatchValue(value=start),
@@ -688,22 +682,20 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse:
),
limit=1,
with_vectors=False,
with_payload=["highlighted_page_image", "page_number"],
with_payload=["chunk_bbox", "page_number"],
)
points = points_response[0]
if points and points[0].payload:
highlighted_page_image = points[0].payload.get(
"highlighted_page_image"
)
chunk_bbox = points[0].payload.get("chunk_bbox")
page_number = points[0].payload.get("page_number")
if highlighted_page_image:
if chunk_bbox:
logger.info(
f"Found highlighted image for chunk: "
f"page={page_number}, image_size={len(highlighted_page_image)}"
f"Found chunk bbox: page={page_number}, "
f"rects={len(chunk_bbox)}"
)
except Exception as e:
logger.warning(f"Failed to fetch highlighted image: {e}")
logger.warning(f"Failed to fetch chunk bbox: {e}")
# Return response compatible with frontend expectations
response_data: dict = {
@@ -718,8 +710,8 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse:
"total_chunks": chunk_context.total_chunks,
}
if highlighted_page_image:
response_data["highlighted_page_image"] = highlighted_page_image
if chunk_bbox:
response_data["chunk_bbox"] = chunk_bbox
return JSONResponse(response_data)
+63 -9
View File
@@ -77,11 +77,25 @@ _DEFAULTS: dict[str, Any] = {
# Ollama
"ollama_base_url": None,
"ollama_embedding_model": "nomic-embed-text",
"ollama_generation_model": None,
"ollama_verify_ssl": True,
# OpenAI
"openai_api_key": None,
"openai_base_url": None,
"openai_embedding_model": "text-embedding-3-small",
"openai_generation_model": None,
# Bedrock (AWS)
"aws_region": None,
"aws_access_key_id": None,
"aws_secret_access_key": None,
"bedrock_embedding_model": None,
"bedrock_generation_model": None,
# Mistral
"mistral_api_key": None,
"mistral_embedding_model": "mistral-embed",
"mistral_base_url": None,
# Simple (fallback) embedding dimension
"simple_embedding_dimension": 384,
# Document chunking
"document_chunk_size": 2048,
"document_chunk_overlap": 200,
@@ -486,15 +500,32 @@ class Settings:
qdrant_api_key: str | None = None
qdrant_collection: str = "nextcloud_content"
# Ollama settings (for embeddings)
# Ollama settings (embeddings + optional generation)
ollama_base_url: str | None = None
ollama_embedding_model: str = "nomic-embed-text"
ollama_generation_model: str | None = None
ollama_verify_ssl: bool = True
# OpenAI settings (for embeddings)
# OpenAI settings (embeddings + optional generation)
openai_api_key: str | None = None
openai_base_url: str | None = None
openai_embedding_model: str = "text-embedding-3-small"
openai_generation_model: str | None = None
# Bedrock (AWS) settings — boto3 also reads these from its credential chain
aws_region: str | None = None
aws_access_key_id: str | None = None
aws_secret_access_key: str | None = None
bedrock_embedding_model: str | None = None
bedrock_generation_model: str | None = None
# Mistral settings (embeddings only)
mistral_api_key: str | None = None
mistral_embedding_model: str = "mistral-embed"
mistral_base_url: str | None = None
# Simple (fallback) provider — dimension when no real provider configured
simple_embedding_dimension: int = 384
# Document chunking settings (for vector embeddings)
document_chunk_size: int = 2048 # Characters per chunk
@@ -573,23 +604,32 @@ class Settings:
Get the active embedding model name based on provider priority.
Priority order (same as ProviderRegistry):
1. OpenAI - if OPENAI_API_KEY is set
2. Ollama - if OLLAMA_BASE_URL is set
3. Simple - fallback (returns "simple-384")
1. Bedrock - if AWS_REGION or BEDROCK_EMBEDDING_MODEL is set
2. OpenAI - if OPENAI_API_KEY is set
3. Mistral - if MISTRAL_API_KEY is set
4. Ollama - if OLLAMA_BASE_URL is set
5. Simple - fallback (returns "simple-{dimension}")
Returns:
Active embedding model name
"""
# Check OpenAI first (higher priority than Ollama in registry)
if (
self.aws_region
or self.bedrock_embedding_model
or self.bedrock_generation_model
):
return self.bedrock_embedding_model or "bedrock-default"
if self.openai_api_key:
return self.openai_embedding_model
# Check Ollama
if self.mistral_api_key:
return self.mistral_embedding_model
if self.ollama_base_url:
return self.ollama_embedding_model
# Fallback to simple provider indicator
return "simple-384"
return f"simple-{self.simple_embedding_dimension}"
def get_collection_name(self) -> str:
"""
@@ -835,11 +875,25 @@ def get_settings() -> Settings:
# Ollama settings
"ollama_base_url": "OLLAMA_BASE_URL",
"ollama_embedding_model": "OLLAMA_EMBEDDING_MODEL",
"ollama_generation_model": "OLLAMA_GENERATION_MODEL",
"ollama_verify_ssl": "OLLAMA_VERIFY_SSL",
# OpenAI settings
"openai_api_key": "OPENAI_API_KEY",
"openai_base_url": "OPENAI_BASE_URL",
"openai_embedding_model": "OPENAI_EMBEDDING_MODEL",
"openai_generation_model": "OPENAI_GENERATION_MODEL",
# Bedrock (AWS) settings
"aws_region": "AWS_REGION",
"aws_access_key_id": "AWS_ACCESS_KEY_ID",
"aws_secret_access_key": "AWS_SECRET_ACCESS_KEY",
"bedrock_embedding_model": "BEDROCK_EMBEDDING_MODEL",
"bedrock_generation_model": "BEDROCK_GENERATION_MODEL",
# Mistral settings
"mistral_api_key": "MISTRAL_API_KEY",
"mistral_embedding_model": "MISTRAL_EMBEDDING_MODEL",
"mistral_base_url": "MISTRAL_BASE_URL",
# Simple provider
"simple_embedding_dimension": "SIMPLE_EMBEDDING_DIMENSION",
# Document chunking settings
"document_chunk_size": "DOCUMENT_CHUNK_SIZE",
"document_chunk_overlap": "DOCUMENT_CHUNK_OVERLAP",
@@ -3,6 +3,7 @@
from .anthropic import AnthropicProvider
from .base import Provider
from .bedrock import BedrockProvider
from .mistral import MistralProvider
from .ollama import OllamaProvider
from .openai import OpenAIProvider
from .registry import get_provider, reset_provider
@@ -13,6 +14,7 @@ __all__ = [
"OllamaProvider",
"OpenAIProvider",
"AnthropicProvider",
"MistralProvider",
"SimpleProvider",
"BedrockProvider",
"get_provider",
+78
View File
@@ -0,0 +1,78 @@
"""Shared rate-limit retry helper for provider modules.
OpenAI and Mistral both retry on 429 with the same exponential-backoff curve;
extracting the loop here keeps the two provider modules thin and lets future
providers (Bedrock throttling, etc.) reuse the same primitive.
"""
from __future__ import annotations
import logging
from collections.abc import Awaitable, Callable
from functools import wraps
from typing import Any, TypeVar
import anyio
logger = logging.getLogger(__name__)
MAX_RETRIES = 5
INITIAL_RETRY_DELAY = 2.0
MAX_RETRY_DELAY = 60.0
T = TypeVar("T")
def retry_on_rate_limit(
exception_type: type[BaseException],
is_rate_limit: Callable[[BaseException], bool] = lambda _exc: True,
*,
provider_name: str = "provider",
) -> Callable[[Callable[..., Awaitable[T]]], Callable[..., Awaitable[T]]]:
"""Build a decorator that retries on rate-limit exceptions.
Args:
exception_type: Catch this exception class (e.g. ``openai.RateLimitError``,
``mistralai.client.errors.SDKError``).
is_rate_limit: Predicate that decides whether a caught exception is
actually a rate-limit (vs. some other error of the same class).
Defaults to "always True" — appropriate when ``exception_type`` is
already a rate-limit-specific class.
provider_name: Used in log messages so operators can tell which
provider exhausted retries.
"""
def decorator(func: Callable[..., Awaitable[T]]) -> Callable[..., Awaitable[T]]:
@wraps(func)
async def wrapper(*args: Any, **kwargs: Any) -> T:
retry_delay = INITIAL_RETRY_DELAY
last_error: BaseException | None = None
for attempt in range(1, MAX_RETRIES + 1):
try:
return await func(*args, **kwargs)
except exception_type as e:
if not is_rate_limit(e):
raise
last_error = e
if attempt < MAX_RETRIES:
logger.warning(
"%s rate limit hit (attempt %d/%d), retrying in %.1fs...",
provider_name,
attempt,
MAX_RETRIES,
retry_delay,
)
await anyio.sleep(retry_delay)
retry_delay = min(retry_delay * 2, MAX_RETRY_DELAY)
logger.error(
"%s rate limit exceeded after %d attempts", provider_name, MAX_RETRIES
)
if last_error is None: # pragma: no cover — loop above always sets this
raise RuntimeError("retry loop exited without capturing an error")
raise last_error
return wrapper
return decorator
+199
View File
@@ -0,0 +1,199 @@
"""Mistral provider for embeddings.
Currently supports embeddings only (``mistral-embed``, 1024-dim). Generation
can be added later if needed; see ADR-015.
"""
import logging
# mistralai 2.x ships no top-level __init__.py, so `from mistralai import …`
# raises ImportError. The canonical public paths are `mistralai.client` (which
# re-exports the SDK class via `client/__init__.py`) and `mistralai.client.errors`
# (which lazy-loads SDKError). There is no `mistralai.models` subpackage either.
from mistralai.client import Mistral
from mistralai.client.errors import SDKError
from ._retry import retry_on_rate_limit
from .base import Provider
logger = logging.getLogger(__name__)
# Well-known Mistral embedding model dimensions
MISTRAL_EMBEDDING_DIMENSIONS: dict[str, int] = {
"mistral-embed": 1024,
}
# Conservative chunk size for batch embeddings. Mistral allows large batches,
# but we keep this in line with sibling providers (OpenAI=100, Ollama=32).
BATCH_SIZE = 64
_NO_EMBEDDING_MODEL_MSG = "Embedding not supported - no embedding_model configured"
def _is_rate_limit(exc: BaseException) -> bool:
"""True only for HTTP 429 SDKErrors."""
return getattr(exc, "status_code", None) == 429
_retry_429 = retry_on_rate_limit(
SDKError, is_rate_limit=_is_rate_limit, provider_name="Mistral"
)
class MistralProvider(Provider):
"""
Mistral provider — embeddings only.
Uses the official ``mistralai`` SDK. Lazy dimension detection mirrors the
OpenAI provider: known models populate the cached dimension at construction
time; unknown models get their dimension detected on the first ``embed()``
call.
"""
def __init__(
self,
api_key: str,
embedding_model: str | None = "mistral-embed",
base_url: str | None = None,
):
"""
Initialize the Mistral provider.
Args:
api_key: Mistral API key.
embedding_model: Embedding model ID (default: ``mistral-embed``).
Pass ``None`` to disable embeddings (the provider will then
support no capabilities, which is mostly useful for tests).
base_url: Optional base URL override (e.g. proxies, on-prem).
"""
self.embedding_model = embedding_model
self._dimension: int | None = None
self.client = Mistral(api_key=api_key, server_url=base_url)
if embedding_model and embedding_model in MISTRAL_EMBEDDING_DIMENSIONS:
self._dimension = MISTRAL_EMBEDDING_DIMENSIONS[embedding_model]
logger.info(
"Initialized Mistral provider: base_url=%s, embedding_model=%s, "
"dimension=%s",
base_url or "default",
embedding_model,
self._dimension,
)
@property
def supports_embeddings(self) -> bool:
return self.embedding_model is not None
@property
def supports_generation(self) -> bool:
return False
@_retry_429
async def embed(self, text: str) -> list[float]:
"""Generate an embedding for a single text."""
if not self.supports_embeddings:
raise NotImplementedError(_NO_EMBEDDING_MODEL_MSG)
assert self.embedding_model is not None
response = await self.client.embeddings.create_async(
model=self.embedding_model,
inputs=[text],
)
if not response.data or response.data[0].embedding is None:
raise RuntimeError(
f"Mistral embeddings API returned no embedding for model "
f"{self.embedding_model}"
)
embedding = response.data[0].embedding
if self._dimension is None:
self._dimension = len(embedding)
logger.info(
"Detected embedding dimension: %d for model %s",
self._dimension,
self.embedding_model,
)
return embedding
async def embed_batch(self, texts: list[str]) -> list[list[float]]:
"""Generate embeddings for multiple texts, chunking by ``BATCH_SIZE``."""
if not self.supports_embeddings:
raise NotImplementedError(_NO_EMBEDDING_MODEL_MSG)
if not texts:
return []
all_embeddings: list[list[float]] = []
for i in range(0, len(texts), BATCH_SIZE):
batch = texts[i : i + BATCH_SIZE]
batch_embeddings = await self._embed_batch_request(batch)
all_embeddings.extend(batch_embeddings)
if self._dimension is None and batch_embeddings:
self._dimension = len(batch_embeddings[0])
logger.info(
"Detected embedding dimension: %d for model %s",
self._dimension,
self.embedding_model,
)
return all_embeddings
@_retry_429
async def _embed_batch_request(self, batch: list[str]) -> list[list[float]]:
"""Single batch request with rate-limit retry."""
assert self.embedding_model is not None
response = await self.client.embeddings.create_async(
model=self.embedding_model,
inputs=batch,
)
# Defensive: response.data items have Optional fields. Sort by index
# (default 0 if missing) and reject None embeddings explicitly.
sorted_data = sorted(response.data or [], key=lambda x: x.index or 0)
result: list[list[float]] = []
for item in sorted_data:
if item.embedding is None:
raise RuntimeError(
f"Mistral embeddings API returned a null embedding for "
f"model {self.embedding_model}"
)
result.append(item.embedding)
if len(result) != len(batch):
raise RuntimeError(
f"Mistral embeddings API returned {len(result)} embeddings "
f"for {len(batch)} inputs"
)
return result
def get_dimension(self) -> int:
if not self.supports_embeddings:
raise NotImplementedError(_NO_EMBEDDING_MODEL_MSG)
if self._dimension is None:
raise RuntimeError(
f"Embedding dimension not detected yet for model "
f"{self.embedding_model}. Call embed() first or use a known "
"model."
)
return self._dimension
async def generate(self, prompt: str, max_tokens: int = 500) -> str:
raise NotImplementedError(
"MistralProvider does not support generation. "
"Use OpenAI, Anthropic, or Bedrock for text generation."
)
async def close(self) -> None:
# The mistralai 2.x client (Speakeasy-generated) does not expose a
# public close()/aclose() — only the async-context-manager protocol
# (__aenter__/__aexit__). Calling __aexit__ directly is internal API
# and brittle across SDK patch versions; the underlying httpx client
# is closed during garbage collection, so we leave this as a no-op.
return None
+19 -43
View File
@@ -7,46 +7,17 @@ Supports:
"""
import logging
from functools import wraps
import anyio
from openai import AsyncOpenAI, RateLimitError
from ._retry import retry_on_rate_limit
from .base import Provider
logger = logging.getLogger(__name__)
# Rate limit retry configuration
MAX_RETRIES = 5
INITIAL_RETRY_DELAY = 2.0 # seconds
MAX_RETRY_DELAY = 60.0 # seconds
def retry_on_rate_limit(func):
"""Decorator to retry on OpenAI rate limit errors with exponential backoff."""
@wraps(func)
async def wrapper(*args, **kwargs):
retry_delay = INITIAL_RETRY_DELAY
last_error: Exception | None = None
for attempt in range(1, MAX_RETRIES + 1):
try:
return await func(*args, **kwargs)
except RateLimitError as e:
last_error = e
if attempt < MAX_RETRIES:
logger.warning(
f"Rate limit hit (attempt {attempt}/{MAX_RETRIES}), "
f"retrying in {retry_delay:.1f}s..."
)
await anyio.sleep(retry_delay)
retry_delay = min(retry_delay * 2, MAX_RETRY_DELAY)
logger.error(f"Rate limit exceeded after {MAX_RETRIES} attempts")
raise last_error # type: ignore[misc]
return wrapper
# OpenAI's RateLimitError is itself a 429-specific class, so the default
# is_rate_limit predicate ("always True") matches the previous behavior.
_retry_429 = retry_on_rate_limit(RateLimitError, provider_name="OpenAI")
# Well-known embedding dimensions for OpenAI models
@@ -106,9 +77,12 @@ class OpenAIProvider(Provider):
self._dimension = OPENAI_EMBEDDING_DIMENSIONS[embedding_model]
logger.info(
f"Initialized OpenAI provider: base_url={base_url or 'default'} "
f"(embedding_model={embedding_model}, generation_model={generation_model}, "
f"dimension={self._dimension})"
"Initialized OpenAI provider: base_url=%s "
"(embedding_model=%s, generation_model=%s, dimension=%s)",
base_url or "default",
embedding_model,
generation_model,
self._dimension,
)
@property
@@ -121,7 +95,7 @@ class OpenAIProvider(Provider):
"""Whether this provider supports text generation."""
return self.generation_model is not None
@retry_on_rate_limit
@_retry_429
async def embed(self, text: str) -> list[float]:
"""
Generate embedding vector for text.
@@ -152,8 +126,9 @@ class OpenAIProvider(Provider):
if self._dimension is None:
self._dimension = len(embedding)
logger.info(
f"Detected embedding dimension: {self._dimension} "
f"for model {self.embedding_model}"
"Detected embedding dimension: %d for model %s",
self._dimension,
self.embedding_model,
)
return embedding
@@ -196,13 +171,14 @@ class OpenAIProvider(Provider):
if self._dimension is None and batch_embeddings:
self._dimension = len(batch_embeddings[0])
logger.info(
f"Detected embedding dimension: {self._dimension} "
f"for model {self.embedding_model}"
"Detected embedding dimension: %d for model %s",
self._dimension,
self.embedding_model,
)
return all_embeddings
@retry_on_rate_limit
@_retry_429
async def _embed_batch_request(self, batch: list[str]) -> list[list[float]]:
"""Make a single batch embedding request with retry logic."""
assert self.embedding_model is not None # Type narrowing
@@ -237,7 +213,7 @@ class OpenAIProvider(Provider):
)
return self._dimension
@retry_on_rate_limit
@_retry_429
async def generate(self, prompt: str, max_tokens: int = 500) -> str:
"""
Generate text from a prompt.
+78 -82
View File
@@ -1,10 +1,11 @@
"""Provider registry and factory for auto-detection and instantiation."""
import logging
import os
from ..config import get_settings
from .base import Provider
from .bedrock import BedrockProvider
from .mistral import MistralProvider
from .ollama import OllamaProvider
from .openai import OpenAIProvider
from .simple import SimpleProvider
@@ -16,117 +17,112 @@ class ProviderRegistry:
"""
Registry for provider auto-detection and instantiation.
Checks environment variables in priority order and creates appropriate provider:
1. Bedrock (AWS_REGION + BEDROCK_*_MODEL)
2. OpenAI (OPENAI_API_KEY)
3. Ollama (OLLAMA_BASE_URL)
4. Simple (fallback for testing/development)
Reads configuration via dynaconf-backed Settings (see ``config.py``).
Checks provider settings in priority order and creates the appropriate
provider:
1. Bedrock (``AWS_REGION`` or ``BEDROCK_*_MODEL``)
2. OpenAI (``OPENAI_API_KEY``)
3. Mistral (``MISTRAL_API_KEY``)
4. Ollama (``OLLAMA_BASE_URL``)
5. Simple (fallback for testing/development)
"""
@staticmethod
def create_provider() -> Provider:
"""
Auto-detect and create provider based on environment variables.
Auto-detect and create provider based on configured settings.
Settings are sourced via :func:`nextcloud_mcp_server.config.get_settings`,
which reads from settings files and environment variables (env vars
always win, see ADR-024/025).
Priority order:
1. Bedrock - if AWS_REGION or BEDROCK_EMBEDDING_MODEL is set
2. OpenAI - if OPENAI_API_KEY is set
3. Ollama - if OLLAMA_BASE_URL is set
4. Simple - fallback for testing/development
1. Bedrock - if ``aws_region`` or ``bedrock_embedding_model`` is set
2. OpenAI - if ``openai_api_key`` is set
3. Mistral - if ``mistral_api_key`` is set
4. Ollama - if ``ollama_base_url`` is set
5. Simple - fallback for testing/development
Returns:
Provider instance
Environment Variables:
Bedrock:
- AWS_REGION: AWS region (e.g., "us-east-1")
- AWS_ACCESS_KEY_ID: AWS access key (optional, uses credential chain)
- AWS_SECRET_ACCESS_KEY: AWS secret key (optional)
- BEDROCK_EMBEDDING_MODEL: Model ID for embeddings (e.g., "amazon.titan-embed-text-v2:0")
- BEDROCK_GENERATION_MODEL: Model ID for text generation (e.g., "anthropic.claude-3-sonnet-20240229-v1:0")
OpenAI:
- OPENAI_API_KEY: OpenAI API key (or GITHUB_TOKEN for GitHub Models)
- OPENAI_BASE_URL: Base URL override (e.g., "https://models.github.ai/inference")
- OPENAI_EMBEDDING_MODEL: Model for embeddings (default: "text-embedding-3-small")
- OPENAI_GENERATION_MODEL: Model for text generation (e.g., "gpt-4o-mini")
Ollama:
- OLLAMA_BASE_URL: Ollama API base URL (e.g., "http://localhost:11434")
- OLLAMA_EMBEDDING_MODEL: Model for embeddings (default: "nomic-embed-text")
- OLLAMA_GENERATION_MODEL: Model for text generation (e.g., "llama3.2:1b")
- OLLAMA_VERIFY_SSL: Verify SSL certificates (default: "true")
Simple (no configuration needed, fallback):
- SIMPLE_EMBEDDING_DIMENSION: Embedding dimension (default: 384)
"""
# 1. Check for Bedrock
aws_region = os.getenv("AWS_REGION")
bedrock_embedding_model = os.getenv("BEDROCK_EMBEDDING_MODEL")
bedrock_generation_model = os.getenv("BEDROCK_GENERATION_MODEL")
settings = get_settings()
if aws_region or bedrock_embedding_model or bedrock_generation_model:
# 1. Bedrock
if (
settings.aws_region
or settings.bedrock_embedding_model
or settings.bedrock_generation_model
):
logger.info(
f"Using Bedrock provider: region={aws_region}, "
f"embedding_model={bedrock_embedding_model}, "
f"generation_model={bedrock_generation_model}"
"Using Bedrock provider: region=%s, embedding_model=%s, "
"generation_model=%s",
settings.aws_region,
settings.bedrock_embedding_model,
settings.bedrock_generation_model,
)
return BedrockProvider(
region_name=aws_region,
embedding_model=bedrock_embedding_model,
generation_model=bedrock_generation_model,
aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"),
aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"),
region_name=settings.aws_region,
embedding_model=settings.bedrock_embedding_model,
generation_model=settings.bedrock_generation_model,
aws_access_key_id=settings.aws_access_key_id,
aws_secret_access_key=settings.aws_secret_access_key,
)
# 2. Check for OpenAI
openai_api_key = os.getenv("OPENAI_API_KEY")
if openai_api_key:
base_url = os.getenv("OPENAI_BASE_URL")
embedding_model = os.getenv(
"OPENAI_EMBEDDING_MODEL", "text-embedding-3-small"
)
generation_model = os.getenv("OPENAI_GENERATION_MODEL")
# 2. OpenAI
if settings.openai_api_key:
logger.info(
f"Using OpenAI provider: base_url={base_url or 'default'}, "
f"embedding_model={embedding_model}, "
f"generation_model={generation_model}"
"Using OpenAI provider: base_url=%s, embedding_model=%s, "
"generation_model=%s",
settings.openai_base_url or "default",
settings.openai_embedding_model,
settings.openai_generation_model,
)
return OpenAIProvider(
api_key=openai_api_key,
base_url=base_url,
embedding_model=embedding_model,
generation_model=generation_model,
api_key=settings.openai_api_key,
base_url=settings.openai_base_url,
embedding_model=settings.openai_embedding_model,
generation_model=settings.openai_generation_model,
)
# 3. Check for Ollama (local LLM)
ollama_url = os.getenv("OLLAMA_BASE_URL")
if ollama_url:
embedding_model = os.getenv("OLLAMA_EMBEDDING_MODEL", "nomic-embed-text")
generation_model = os.getenv("OLLAMA_GENERATION_MODEL")
verify_ssl = os.getenv("OLLAMA_VERIFY_SSL", "true").lower() == "true"
# 3. Mistral
if settings.mistral_api_key:
logger.info(
f"Using Ollama provider: {ollama_url}, "
f"embedding_model={embedding_model}, "
f"generation_model={generation_model}"
"Using Mistral provider: base_url=%s, embedding_model=%s",
settings.mistral_base_url or "default",
settings.mistral_embedding_model,
)
return MistralProvider(
api_key=settings.mistral_api_key,
base_url=settings.mistral_base_url,
embedding_model=settings.mistral_embedding_model,
)
# 4. Ollama
if settings.ollama_base_url:
logger.info(
"Using Ollama provider: %s, embedding_model=%s, generation_model=%s",
settings.ollama_base_url,
settings.ollama_embedding_model,
settings.ollama_generation_model,
)
return OllamaProvider(
base_url=ollama_url,
embedding_model=embedding_model,
generation_model=generation_model,
verify_ssl=verify_ssl,
base_url=settings.ollama_base_url,
embedding_model=settings.ollama_embedding_model,
generation_model=settings.ollama_generation_model,
verify_ssl=settings.ollama_verify_ssl,
)
# 4. Fallback to Simple provider for development/testing
dimension = int(os.getenv("SIMPLE_EMBEDDING_DIMENSION", "384"))
# 5. Simple (fallback)
logger.warning(
"No provider configured (AWS_REGION, OPENAI_API_KEY, OLLAMA_BASE_URL not set). "
"No provider configured (AWS_REGION, OPENAI_API_KEY, "
"MISTRAL_API_KEY, OLLAMA_BASE_URL not set). "
"Using SimpleProvider for testing/development. "
"For production, configure Bedrock, OpenAI, or Ollama."
"For production, configure Bedrock, OpenAI, Mistral, or Ollama."
)
return SimpleProvider(dimension=dimension)
return SimpleProvider(dimension=settings.simple_embedding_dimension)
# Singleton instance
@@ -691,6 +691,122 @@ class PDFHighlighter:
f"Failed to delete temp directory {temp_pdf_path.parent}: {e}"
)
@staticmethod
def compute_chunk_bboxes_batch(
pdf_bytes: bytes,
chunks: list[tuple[int, int, int, int | None, str]],
page_boundaries: list[dict],
full_text: str,
) -> dict[int, tuple[list[tuple[float, float, float, float]], int]]:
"""Compute normalized bounding boxes for chunks without rendering.
Lightweight alternative to highlight_chunks_batch — opens the PDF,
locates each chunk on its assigned page using the same text-search
path as the highlighter (`_find_chunk_bbox`), and returns
page-normalized rectangles. Skips the get_pixmap + PIL pipeline
entirely, so no PNG bytes are produced.
Args:
pdf_bytes: PDF file bytes.
chunks: List of (chunk_index, start_offset, end_offset,
stored_page_number, chunk_text). chunk_index is the dict key.
page_boundaries: Pre-computed page boundaries from the document
processor; each entry is {"page", "start_offset", "end_offset"}.
full_text: Full document text (for cross-page chunk handling).
Returns:
dict mapping chunk_index to (normalized_bboxes, page_number).
Each bbox is (x0, y0, x1, y1) in [0, 1] relative to page width
and height, top-left origin. Chunks whose bbox cannot be located
are omitted from the result.
"""
results: dict[int, tuple[list[tuple[float, float, float, float]], int]] = {}
if not chunks:
return results
temp_pdf_path = None
doc = None
try:
temp_dir = Path(tempfile.mkdtemp(prefix="pdf_bbox_batch_"))
temp_pdf_path = temp_dir / "pdf.pdf"
temp_pdf_path.write_bytes(pdf_bytes)
doc = pymupdf.open(temp_pdf_path)
for (
chunk_index,
start_offset,
end_offset,
_,
_,
) in chunks:
chunk_page_info = PDFHighlighter.find_chunk_page(
start_offset, end_offset, page_boundaries
)
if not chunk_page_info:
logger.debug("Chunk %s: not found on any page", chunk_index)
continue
page_num = chunk_page_info["page_num"]
page_boundary = next(
(b for b in page_boundaries if b["page"] == page_num), None
)
if page_boundary is None:
logger.debug(
"Chunk %s: page %s not found in boundaries",
chunk_index,
page_num,
)
continue
page_text_length = (
page_boundary["end_offset"] - page_boundary["start_offset"]
)
# Page-relative slice (handles chunks that span page boundaries)
chunk_start_on_page = max(start_offset, page_boundary["start_offset"])
chunk_end_on_page = min(end_offset, page_boundary["end_offset"])
page_relative_text = full_text[chunk_start_on_page:chunk_end_on_page]
page = doc[page_num - 1]
bbox = PDFHighlighter._find_chunk_bbox(
page,
page_relative_text,
chunk_page_info["page_relative_start"],
chunk_page_info["page_relative_end"],
page_text_length,
)
if bbox is None:
continue
page_rect = page.rect
w = page_rect.width or 1.0
h = page_rect.height or 1.0
normalized = (
bbox[0] / w,
bbox[1] / h,
bbox[2] / w,
bbox[3] / h,
)
results[chunk_index] = ([normalized], page_num)
logger.info(f"Computed bboxes for {len(results)}/{len(chunks)} chunks")
return results
except Exception as e:
logger.error(f"Error computing chunk bboxes: {e}", exc_info=True)
return results
finally:
if doc is not None:
doc.close()
if temp_pdf_path and temp_pdf_path.parent.exists():
try:
shutil.rmtree(temp_pdf_path.parent)
except Exception as e:
logger.warning(f"Failed to clean up temp dir: {e}")
@staticmethod
def highlight_chunks_batch(
pdf_bytes: bytes,
+9 -8
View File
@@ -1,16 +1,17 @@
"""Vector database and background sync package."""
"""Vector database and background sync package.
`processor` and `scanner` are intentionally NOT re-exported from this
package init: they transitively import `server.semantic` ->
`search.bm25_hybrid`, which forms an import cycle with
`search.algorithms` -> `vector.placeholder` -> `vector/__init__`.
Consumers that need those symbols import them from their submodules
directly (e.g. `from nextcloud_mcp_server.vector.processor import ...`).
"""
from .document_chunker import DocumentChunker
from .processor import process_document, processor_task
from .qdrant_client import get_qdrant_client
from .scanner import DocumentTask, scan_user_documents, scanner_task
__all__ = [
"get_qdrant_client",
"DocumentChunker",
"scanner_task",
"scan_user_documents",
"DocumentTask",
"processor_task",
"process_document",
]
+26 -49
View File
@@ -3,7 +3,6 @@
Processes documents from stream: fetches content, generates embeddings, stores in Qdrant.
"""
import base64
import logging
import time
import uuid
@@ -538,7 +537,11 @@ async def _index_document(
# Initialize results containers
dense_embeddings: list = []
sparse_embeddings: list = []
chunk_images: dict[int, dict] = {}
# chunk_index -> list[(x0, y0, x1, y1)] of normalized rectangles
# in [0, 1] relative to page width/height. The page is taken from
# `chunk.page_number` (offset-based) and stored as `page_number`
# in the Qdrant payload, so we don't carry an `actual_page_num` here.
chunk_bboxes: dict[int, list[tuple[float, float, float, float]]] = {}
# Determine if we need PDF highlighting
is_pdf = doc_task.doc_type == "file" and content_type == "application/pdf"
@@ -570,8 +573,8 @@ async def _index_document(
sparse_embeddings = await bm25_service.encode_batch(chunk_texts)
async def generate_highlights():
"""Generate highlighted page images for PDF chunks (CPU-bound)."""
nonlocal chunk_images
"""Compute chunk bounding boxes for PDF chunks (CPU-bound, no rendering)."""
nonlocal chunk_bboxes
if not is_pdf:
return
@@ -579,64 +582,42 @@ async def _index_document(
assert content_bytes is not None
with trace_operation(
"vector_sync.generate_highlights",
"vector_sync.compute_chunk_bboxes",
attributes={
"vector_sync.chunk_count": len(chunks),
"vector_sync.pdf_size": len(content_bytes),
},
):
# Build chunk data for batch processing
# Format: (chunk_index, start_offset, end_offset, page_number, chunk_text)
chunk_data: list[tuple[int, int, int, int | None, str]] = [
(i, chunk.start_offset, chunk.end_offset, chunk.page_number, chunk.text)
for i, chunk in enumerate(chunks)
if chunk.page_number is not None
]
# Get pre-computed page boundaries from document processor
page_boundaries = file_metadata.get("page_boundaries")
if not page_boundaries:
logger.warning("No page boundaries available, skipping highlighting")
logger.warning(
"No page boundaries available, skipping bbox computation"
)
return
# Type narrowing: page_boundaries is guaranteed to be list[dict] here
page_boundaries_list = cast(list[dict[str, Any]], page_boundaries)
logger.info(
f"Batch generating highlighted page images for {len(chunk_data)} PDF chunks"
)
logger.info(f"Computing chunk bboxes for {len(chunk_data)} PDF chunks")
# Run CPU-bound highlighting in thread pool
# Pass pre-computed page boundaries and full text to avoid re-processing the PDF
batch_results = await anyio.to_thread.run_sync( # type: ignore[attr-defined]
lambda: PDFHighlighter.highlight_chunks_batch(
lambda: PDFHighlighter.compute_chunk_bboxes_batch(
pdf_bytes=content_bytes,
chunks=chunk_data,
page_boundaries=page_boundaries_list,
full_text=content,
color="yellow",
zoom=2.0,
)
)
# Convert results to storage format
for chunk_index, (
png_bytes,
actual_page_num,
highlight_count,
) in batch_results.items():
image_base64 = base64.b64encode(png_bytes).decode("utf-8")
chunk_images[chunk_index] = {
"image": image_base64,
"page": actual_page_num,
"highlights": highlight_count,
"size": len(png_bytes),
}
for chunk_index, (bboxes, _) in batch_results.items():
chunk_bboxes[chunk_index] = bboxes
logger.info(
f"Generated {len(chunk_images)}/{len(chunks)} highlighted page images "
f"(avg {sum(img['size'] for img in chunk_images.values()) // max(len(chunk_images), 1):,} bytes)"
)
logger.info(f"Computed bboxes for {len(chunk_bboxes)}/{len(chunks)} chunks")
# Run all embedding/highlighting operations in parallel
# - Dense embeddings: I/O bound (API call)
@@ -752,16 +733,11 @@ async def _index_document(
if doc_task.doc_type == "deck_card"
else {}
),
# Highlighted page image (PDF only)
**(
{
"highlighted_page_image": chunk_images[i]["image"],
"highlighted_page_number": chunk_images[i]["page"],
"highlight_count": chunk_images[i]["highlights"],
}
if i in chunk_images
else {}
),
# Chunk bbox (PDF only) — normalized rectangles in [0,1]
# relative to page width/height. Replaces the legacy
# `highlighted_page_image` (Deck #76). The page number
# comes from `page_number` (set above for PDF chunks).
**({"chunk_bbox": chunk_bboxes[i]} if i in chunk_bboxes else {}),
},
)
)
@@ -780,15 +756,16 @@ async def _index_document(
f"Failed to delete placeholder for {doc_task.doc_type}_{doc_task.doc_id}: {e}"
)
# Upsert to Qdrant in batches to avoid timeout with large payloads
# Each batch is limited to avoid WriteTimeout when sending large image payloads
BATCH_SIZE = 10 # ~2MB per batch with images
# Upsert to Qdrant in batches. Now that we no longer embed PNG payloads,
# per-point payloads are small (chunk text + small metadata), so we can
# safely use a larger batch size.
BATCH_SIZE = 100
with trace_operation(
"vector_sync.qdrant_upsert",
attributes={
"vector_sync.point_count": len(points),
"vector_sync.collection": settings.get_collection_name(),
"vector_sync.images_count": len(chunk_images),
"vector_sync.bboxes_count": len(chunk_bboxes),
"vector_sync.batch_size": BATCH_SIZE,
},
):
+2 -1
View File
@@ -1,6 +1,6 @@
[project]
name = "nextcloud-mcp-server"
version = "0.81.0"
version = "0.83.0"
description = "Model Context Protocol (MCP) server for Nextcloud integration - enables AI assistants to interact with Nextcloud data"
authors = [
{name = "Chris Coutinho", email = "chris@coutinho.io"}
@@ -43,6 +43,7 @@ dependencies = [
"pymupdf4llm>=0.2.2",
"openai>=2.8.1",
"dynaconf>=3.2.13,<4.0",
"mistralai>=2.4.5",
]
classifiers = [
"Development Status :: 4 - Beta",
+137
View File
@@ -0,0 +1,137 @@
#!/usr/bin/env python3
"""Purge legacy `highlighted_page_image` payloads from Qdrant (Deck #76).
Iterates all points in the configured Qdrant collection and deletes the
legacy payload keys `highlighted_page_image`, `highlighted_page_number`,
and `highlight_count`. This relieves disk pressure caused by inline
base64 PNGs that the new code path no longer writes.
Idempotent: deleting non-existent keys is a no-op, so re-runs are safe.
Usage:
uv run python scripts/purge_page_images.py [--dry-run] [--batch-size 256]
Connection settings (Qdrant URL/API key, collection name) are read from
the same `Settings` object the server uses.
"""
from __future__ import annotations
import argparse
import logging
import sys
from functools import partial
import anyio
from qdrant_client import AsyncQdrantClient
from nextcloud_mcp_server.config import get_settings
logger = logging.getLogger("purge_page_images")
LEGACY_FIELDS = [
"highlighted_page_image",
"highlighted_page_number",
"highlight_count",
]
async def purge(dry_run: bool, batch_size: int) -> None:
settings = get_settings()
if not settings.qdrant_url:
raise SystemExit(
"qdrant_url is not configured. Set QDRANT_URL (and QDRANT_API_KEY "
"if required) before running this script."
)
collection = settings.get_collection_name()
# AsyncQdrantClient doesn't implement __aenter__/__aexit__, so use
# try/finally to guarantee the underlying aiohttp session is closed.
client = AsyncQdrantClient(
url=settings.qdrant_url,
api_key=settings.qdrant_api_key,
timeout=60,
)
try:
next_offset = None
total_seen = 0
total_updated = 0
logger.info(
"Scanning collection %s; will delete keys %s%s",
collection,
LEGACY_FIELDS,
" (dry run)" if dry_run else "",
)
while True:
points, next_offset = await client.scroll(
collection_name=collection,
limit=batch_size,
offset=next_offset,
with_payload=False,
with_vectors=False,
)
if not points:
break
ids = [p.id for p in points]
total_seen += len(ids)
if not dry_run:
await client.delete_payload(
collection_name=collection,
keys=LEGACY_FIELDS,
points=ids,
)
total_updated += len(ids)
logger.info(
"Batch: ids=%d total_seen=%d total_updated=%d",
len(ids),
total_seen,
total_updated,
)
if next_offset is None:
break
logger.info(
"Done. total_seen=%d total_updated=%d%s",
total_seen,
total_updated,
" (dry run, no writes)" if dry_run else "",
)
finally:
await client.close()
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--dry-run",
action="store_true",
help="Scan only; do not write any changes.",
)
parser.add_argument(
"--batch-size",
type=int,
default=256,
help="Points per scroll/update batch (default: 256).",
)
parser.add_argument(
"-v", "--verbose", action="store_true", help="Enable debug logging."
)
args = parser.parse_args()
logging.basicConfig(
level=logging.DEBUG if args.verbose else logging.INFO,
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
)
anyio.run(partial(purge, dry_run=args.dry_run, batch_size=args.batch_size))
return 0
if __name__ == "__main__":
sys.exit(main())
+269
View File
@@ -0,0 +1,269 @@
"""Unit tests for Mistral provider."""
from unittest.mock import AsyncMock, MagicMock
import pytest
from mistralai.client.errors import SDKError
from nextcloud_mcp_server.providers.mistral import (
BATCH_SIZE,
MISTRAL_EMBEDDING_DIMENSIONS,
MistralProvider,
_is_rate_limit,
)
def _make_data(embedding: list[float], index: int) -> MagicMock:
"""Build a mock EmbeddingResponseData entry."""
item = MagicMock()
item.embedding = embedding
item.index = index
return item
def _make_response(embeddings: list[list[float]]) -> MagicMock:
"""Build a mock EmbeddingResponse with `embeddings` indexed in order."""
response = MagicMock()
response.data = [_make_data(emb, i) for i, emb in enumerate(embeddings)]
return response
@pytest.fixture
def mock_mistral_client(mocker):
"""Mock the Mistral SDK constructor."""
mock_client = MagicMock()
mock_client.embeddings = MagicMock()
mocker.patch(
"nextcloud_mcp_server.providers.mistral.Mistral", return_value=mock_client
)
return mock_client
@pytest.mark.unit
async def test_mistral_embedding_single(mock_mistral_client):
"""Single text embed: round-trip through SDK with correct kwargs."""
mock_mistral_client.embeddings.create_async = AsyncMock(
return_value=_make_response([[0.1, 0.2, 0.3]])
)
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
embedding = await provider.embed("hello world")
assert embedding == [0.1, 0.2, 0.3]
mock_mistral_client.embeddings.create_async.assert_awaited_once_with(
model="mistral-embed",
inputs=["hello world"],
)
@pytest.mark.unit
async def test_mistral_embedding_batch_single_call(mock_mistral_client):
"""Batch smaller than BATCH_SIZE issues a single API call."""
mock_mistral_client.embeddings.create_async = AsyncMock(
return_value=_make_response([[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]])
)
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
embeddings = await provider.embed_batch(["a", "b", "c"])
assert embeddings == [[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]]
assert mock_mistral_client.embeddings.create_async.await_count == 1
@pytest.mark.unit
async def test_mistral_embedding_batch_chunking(mock_mistral_client):
"""Batches exceeding BATCH_SIZE are split into multiple API calls."""
# Each call returns one embedding per input it received; capture by side
# effect so we can inspect lengths per chunk.
def _side_effect(*, model, inputs, **_kwargs):
return _make_response([[float(i)] for i in range(len(inputs))])
mock_mistral_client.embeddings.create_async = AsyncMock(side_effect=_side_effect)
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
total = BATCH_SIZE * 2 + 5 # forces three chunks: 64, 64, 5 (with default)
embeddings = await provider.embed_batch([f"text-{i}" for i in range(total)])
assert len(embeddings) == total
assert mock_mistral_client.embeddings.create_async.await_count == 3
# Verify the chunk sizes the SDK was actually called with.
chunk_sizes = [
len(call.kwargs["inputs"])
for call in mock_mistral_client.embeddings.create_async.await_args_list
]
assert chunk_sizes == [BATCH_SIZE, BATCH_SIZE, 5]
@pytest.mark.unit
async def test_mistral_embedding_batch_order_preserved(mock_mistral_client):
"""Out-of-order index in response data is sorted before returning."""
response = MagicMock()
response.data = [
_make_data([0.3, 0.3], 2),
_make_data([0.1, 0.1], 0),
_make_data([0.2, 0.2], 1),
]
mock_mistral_client.embeddings.create_async = AsyncMock(return_value=response)
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
embeddings = await provider.embed_batch(["x", "y", "z"])
assert embeddings == [[0.1, 0.1], [0.2, 0.2], [0.3, 0.3]]
@pytest.mark.unit
async def test_mistral_supports_capabilities(mock_mistral_client):
"""Mistral provider advertises embeddings only."""
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
assert provider.supports_embeddings is True
assert provider.supports_generation is False
@pytest.mark.unit
async def test_mistral_generate_not_implemented(mock_mistral_client):
"""generate() always raises NotImplementedError."""
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
with pytest.raises(NotImplementedError, match="does not support generation"):
await provider.generate("test prompt")
@pytest.mark.unit
async def test_mistral_get_dimension_known_model(mock_mistral_client):
"""Known model: dimension available without an API call."""
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
assert provider.get_dimension() == MISTRAL_EMBEDDING_DIMENSIONS["mistral-embed"]
mock_mistral_client.embeddings.create_async.assert_not_called()
@pytest.mark.unit
async def test_mistral_get_dimension_unknown_model_detected(mock_mistral_client):
"""Unknown model: dimension detected on first embed() call."""
mock_mistral_client.embeddings.create_async = AsyncMock(
return_value=_make_response([[0.1] * 768])
)
provider = MistralProvider(api_key="test-key", embedding_model="custom-mistral")
with pytest.raises(RuntimeError, match="not detected yet"):
provider.get_dimension()
await provider.embed("test")
assert provider.get_dimension() == 768
@pytest.mark.unit
async def test_mistral_no_embeddings_disabled(mock_mistral_client):
"""Setting embedding_model=None disables the embedding capability."""
provider = MistralProvider(api_key="test-key", embedding_model=None)
assert provider.supports_embeddings is False
with pytest.raises(NotImplementedError, match="no embedding_model configured"):
await provider.embed("test")
with pytest.raises(NotImplementedError, match="no embedding_model configured"):
await provider.embed_batch(["test"])
with pytest.raises(NotImplementedError, match="no embedding_model configured"):
provider.get_dimension()
@pytest.mark.unit
async def test_mistral_empty_batch(mock_mistral_client):
"""An empty batch returns [] without calling the API."""
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
assert await provider.embed_batch([]) == []
mock_mistral_client.embeddings.create_async.assert_not_called()
@pytest.mark.unit
async def test_mistral_close_no_error(mock_mistral_client):
"""close() is best-effort and does not raise."""
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
# No __aexit__ on the mock by default → close() should silently no-op.
await provider.close()
@pytest.mark.unit
async def test_mistral_base_url_passed_to_sdk(mocker):
"""base_url is forwarded as server_url to the Mistral SDK constructor."""
mock_ctor = mocker.patch(
"nextcloud_mcp_server.providers.mistral.Mistral", return_value=MagicMock()
)
MistralProvider(
api_key="test-key",
embedding_model="mistral-embed",
base_url="https://example.com/mistral",
)
mock_ctor.assert_called_once_with(
api_key="test-key",
server_url="https://example.com/mistral",
)
@pytest.mark.unit
async def test_mistral_embed_raises_on_empty_response_data(mock_mistral_client):
"""embed(): empty response.data triggers the defensive RuntimeError guard."""
empty_response = MagicMock()
empty_response.data = []
mock_mistral_client.embeddings.create_async = AsyncMock(return_value=empty_response)
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
with pytest.raises(RuntimeError, match="returned no embedding"):
await provider.embed("test")
@pytest.mark.unit
async def test_mistral_embed_raises_on_null_embedding(mock_mistral_client):
"""embed(): a single response item with embedding=None is rejected."""
null_item = MagicMock()
null_item.embedding = None
null_item.index = 0
null_response = MagicMock()
null_response.data = [null_item]
mock_mistral_client.embeddings.create_async = AsyncMock(return_value=null_response)
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
with pytest.raises(RuntimeError, match="returned no embedding"):
await provider.embed("test")
@pytest.mark.unit
async def test_mistral_batch_raises_on_null_embedding(mock_mistral_client):
"""_embed_batch_request: a null embedding inside a batch raises explicitly."""
good = _make_data([0.1, 0.2], 0)
bad = MagicMock()
bad.embedding = None
bad.index = 1
response = MagicMock()
response.data = [good, bad]
mock_mistral_client.embeddings.create_async = AsyncMock(return_value=response)
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
with pytest.raises(RuntimeError, match="null embedding"):
await provider.embed_batch(["a", "b"])
@pytest.mark.unit
async def test_mistral_batch_raises_on_count_mismatch(mock_mistral_client):
"""_embed_batch_request: fewer embeddings returned than inputs sent."""
# Two inputs sent, one embedding returned.
response = _make_response([[0.1, 0.2]])
mock_mistral_client.embeddings.create_async = AsyncMock(return_value=response)
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
with pytest.raises(RuntimeError, match="returned 1 embeddings for 2 inputs"):
await provider.embed_batch(["a", "b"])
@pytest.mark.unit
def test_mistral_is_rate_limit_predicate():
"""_is_rate_limit returns True only for SDKErrors with status_code == 429."""
err_429 = MagicMock(spec=SDKError)
err_429.status_code = 429
err_500 = MagicMock(spec=SDKError)
err_500.status_code = 500
assert _is_rate_limit(err_429) is True
assert _is_rate_limit(err_500) is False
# ValueError has no status_code attr → getattr returns None → False.
assert _is_rate_limit(ValueError()) is False
+136
View File
@@ -0,0 +1,136 @@
"""Unit tests for ProviderRegistry — dynaconf-driven auto-detection."""
import pytest
from nextcloud_mcp_server.config import _reload_config
from nextcloud_mcp_server.providers import (
BedrockProvider,
MistralProvider,
OllamaProvider,
OpenAIProvider,
SimpleProvider,
get_provider,
reset_provider,
)
from nextcloud_mcp_server.providers.bedrock import BOTO3_AVAILABLE
def _clear_provider_envs(monkeypatch: pytest.MonkeyPatch) -> None:
"""Strip every provider-selection env var so each test starts clean."""
for name in (
"AWS_REGION",
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
"BEDROCK_EMBEDDING_MODEL",
"BEDROCK_GENERATION_MODEL",
"OPENAI_API_KEY",
"OPENAI_BASE_URL",
"OPENAI_EMBEDDING_MODEL",
"OPENAI_GENERATION_MODEL",
"MISTRAL_API_KEY",
"MISTRAL_BASE_URL",
"MISTRAL_EMBEDDING_MODEL",
"OLLAMA_BASE_URL",
"OLLAMA_EMBEDDING_MODEL",
"OLLAMA_GENERATION_MODEL",
"OLLAMA_VERIFY_SSL",
"SIMPLE_EMBEDDING_DIMENSION",
):
monkeypatch.delenv(name, raising=False)
@pytest.fixture
def clean_provider_env(monkeypatch):
"""Reset provider singleton + dynaconf cache around each test."""
_clear_provider_envs(monkeypatch)
reset_provider()
_reload_config()
yield monkeypatch
reset_provider()
@pytest.mark.unit
def test_registry_falls_back_to_simple(clean_provider_env):
"""No provider env set → SimpleProvider (with default dimension)."""
provider = get_provider()
assert isinstance(provider, SimpleProvider)
assert provider.get_dimension() == 384
@pytest.mark.unit
def test_registry_picks_simple_with_custom_dimension(clean_provider_env):
"""SIMPLE_EMBEDDING_DIMENSION flows through dynaconf to SimpleProvider."""
clean_provider_env.setenv("SIMPLE_EMBEDDING_DIMENSION", "512")
_reload_config()
provider = get_provider()
assert isinstance(provider, SimpleProvider)
assert provider.get_dimension() == 512
@pytest.mark.unit
def test_registry_picks_mistral_when_api_key_set(clean_provider_env, mocker):
"""MISTRAL_API_KEY alone is enough to select MistralProvider."""
# MistralProvider eagerly constructs the SDK client in __init__; stub it
# so the test doesn't depend on the SDK accepting arbitrary keys.
mocker.patch("nextcloud_mcp_server.providers.mistral.Mistral")
clean_provider_env.setenv("MISTRAL_API_KEY", "test-key")
_reload_config()
provider = get_provider()
assert isinstance(provider, MistralProvider)
@pytest.mark.unit
def test_registry_picks_ollama_when_base_url_set(clean_provider_env, mocker):
"""OLLAMA_BASE_URL selects OllamaProvider."""
# OllamaProvider eagerly probes /api/tags in __init__; stub it out.
mocker.patch(
"nextcloud_mcp_server.providers.ollama.OllamaProvider._check_model_is_loaded"
)
clean_provider_env.setenv("OLLAMA_BASE_URL", "http://localhost:11434")
_reload_config()
provider = get_provider()
assert isinstance(provider, OllamaProvider)
@pytest.mark.unit
def test_registry_openai_wins_over_mistral_and_ollama(clean_provider_env):
"""OpenAI takes priority when multiple provider env vars are set."""
clean_provider_env.setenv("OPENAI_API_KEY", "openai-key")
clean_provider_env.setenv("MISTRAL_API_KEY", "mistral-key")
clean_provider_env.setenv("OLLAMA_BASE_URL", "http://localhost:11434")
_reload_config()
provider = get_provider()
assert isinstance(provider, OpenAIProvider)
@pytest.mark.unit
def test_registry_mistral_wins_over_ollama(clean_provider_env, mocker):
"""Mistral takes priority over Ollama when both are configured."""
# Stub the Mistral SDK constructor for the same reason as the sibling
# picker test — keeps the registry test independent of SDK key validation.
mocker.patch("nextcloud_mcp_server.providers.mistral.Mistral")
clean_provider_env.setenv("MISTRAL_API_KEY", "mistral-key")
clean_provider_env.setenv("OLLAMA_BASE_URL", "http://localhost:11434")
_reload_config()
provider = get_provider()
assert isinstance(provider, MistralProvider)
@pytest.mark.unit
def test_registry_bedrock_wins_when_aws_region_set(clean_provider_env):
"""AWS_REGION alone routes to Bedrock, even with other providers configured."""
if not BOTO3_AVAILABLE:
pytest.skip("boto3 not installed")
clean_provider_env.setenv("AWS_REGION", "us-east-1")
clean_provider_env.setenv("OPENAI_API_KEY", "openai-key")
clean_provider_env.setenv("MISTRAL_API_KEY", "mistral-key")
_reload_config()
provider = get_provider()
assert isinstance(provider, BedrockProvider)
+103
View File
@@ -0,0 +1,103 @@
"""Unit tests for the shared rate-limit retry decorator."""
from unittest.mock import AsyncMock
import pytest
from nextcloud_mcp_server.providers import _retry
class _FakeError(Exception):
"""Stand-in for an SDK exception with an HTTP status code attached."""
def __init__(self, status_code: int):
super().__init__(f"status {status_code}")
self.status_code = status_code
@pytest.fixture(autouse=True)
def _no_real_sleep(monkeypatch):
"""Replace anyio.sleep with an awaitable no-op so retries don't waste time."""
monkeypatch.setattr(_retry.anyio, "sleep", AsyncMock(return_value=None))
@pytest.mark.unit
async def test_retry_succeeds_after_429():
"""A 429 followed by success returns the success value."""
calls = {"n": 0}
@_retry.retry_on_rate_limit(
_FakeError, is_rate_limit=lambda e: e.status_code == 429
)
async def flaky():
calls["n"] += 1
if calls["n"] < 3:
raise _FakeError(429)
return "ok"
result = await flaky()
assert result == "ok"
assert calls["n"] == 3
@pytest.mark.unit
async def test_retry_reraises_non_rate_limit_immediately():
"""A non-rate-limit error of the same class is re-raised on first hit."""
calls = {"n": 0}
@_retry.retry_on_rate_limit(
_FakeError, is_rate_limit=lambda e: e.status_code == 429
)
async def boom():
calls["n"] += 1
raise _FakeError(500)
with pytest.raises(_FakeError, match="status 500"):
await boom()
assert calls["n"] == 1 # No retries on non-429.
@pytest.mark.unit
async def test_retry_gives_up_after_max_retries():
"""After MAX_RETRIES failed attempts the last error is re-raised."""
calls = {"n": 0}
@_retry.retry_on_rate_limit(
_FakeError, is_rate_limit=lambda e: e.status_code == 429
)
async def always_429():
calls["n"] += 1
raise _FakeError(429)
with pytest.raises(_FakeError, match="status 429"):
await always_429()
assert calls["n"] == _retry.MAX_RETRIES
@pytest.mark.unit
async def test_retry_default_predicate_treats_all_as_rate_limit():
"""Default predicate (`lambda _: True`) retries every caught exception."""
calls = {"n": 0}
@_retry.retry_on_rate_limit(_FakeError)
async def fail_once():
calls["n"] += 1
if calls["n"] < 2:
raise _FakeError(503)
return "recovered"
result = await fail_once()
assert result == "recovered"
assert calls["n"] == 2
@pytest.mark.unit
async def test_retry_does_not_catch_unrelated_exceptions():
"""Exceptions of a different class bypass the decorator entirely."""
@_retry.retry_on_rate_limit(_FakeError)
async def value_error():
raise ValueError("nope")
with pytest.raises(ValueError, match="nope"):
await value_error()
@@ -0,0 +1,232 @@
"""Unit tests for PDFHighlighter.compute_chunk_bboxes_batch (Deck #76).
Replaces the legacy `highlight_chunks_batch`-+-base64 pipeline that inflated
Qdrant payloads with per-chunk PNG screenshots. The new path returns
normalized bounding boxes only.
"""
from __future__ import annotations
import pymupdf
import pytest
from nextcloud_mcp_server.search.pdf_highlighter import PDFHighlighter
def _make_pdf(pages: list[str]) -> bytes:
"""Build an in-memory PDF whose pages contain the given text."""
doc = pymupdf.open()
for body in pages:
page = doc.new_page(width=595, height=842) # A4
page.insert_text((50, 50), body)
pdf_bytes = doc.tobytes()
doc.close()
return pdf_bytes
def _page_boundaries(pages: list[str]) -> tuple[list[dict], str]:
"""Build (page_boundaries, full_text) compatible with the highlighter API."""
boundaries: list[dict] = []
cursor = 0
parts: list[str] = []
for i, body in enumerate(pages, start=1):
end = cursor + len(body)
boundaries.append({"page": i, "start_offset": cursor, "end_offset": end})
parts.append(body)
cursor = end
return boundaries, "".join(parts)
@pytest.mark.unit
def test_compute_chunk_bboxes_returns_normalized_rects():
"""Each returned bbox should be 4 floats in [0, 1] tagged with the page."""
pages = [
"Chapter 1: Introduction. Nextcloud is a self-hosted collaboration platform "
"covering installation, configuration and maintenance topics.",
"Chapter 2: Installation. Download the package, extract it to the web "
"server directory, and configure the database connection.",
]
pdf_bytes = _make_pdf(pages)
boundaries, full_text = _page_boundaries(pages)
chunks = [
(
0,
0,
len(pages[0]),
1,
"Chapter 1: Introduction. Nextcloud is a self-hosted collaboration platform.",
),
(
1,
len(pages[0]),
len(pages[0]) + len(pages[1]),
2,
"Chapter 2: Installation. Download the package.",
),
]
results = PDFHighlighter.compute_chunk_bboxes_batch(
pdf_bytes=pdf_bytes,
chunks=chunks,
page_boundaries=boundaries,
full_text=full_text,
)
assert set(results) == {0, 1}
bboxes_p1, page_p1 = results[0]
bboxes_p2, page_p2 = results[1]
assert page_p1 == 1
assert page_p2 == 2
for rects in (bboxes_p1, bboxes_p2):
assert len(rects) >= 1
for rect in rects:
assert len(rect) == 4
x0, y0, x1, y1 = rect
assert 0.0 <= x0 < x1 <= 1.0
assert 0.0 <= y0 < y1 <= 1.0
@pytest.mark.unit
def test_compute_chunk_bboxes_empty_input():
assert (
PDFHighlighter.compute_chunk_bboxes_batch(
pdf_bytes=b"",
chunks=[],
page_boundaries=[],
full_text="",
)
== {}
)
@pytest.mark.unit
def test_compute_chunk_bboxes_omits_when_offsets_out_of_range():
"""Chunks whose offsets fall outside every page boundary are omitted.
Verifies the docstring contract: *"Chunks whose bbox cannot be located
are omitted from the result."* (path: ``find_chunk_page`` returns None).
"""
pages = ["Page one body text content here for the test."]
pdf_bytes = _make_pdf(pages)
boundaries, full_text = _page_boundaries(pages)
# Offsets way beyond the document end — no page boundary matches.
out_of_range_start = len(full_text) + 1000
out_of_range_end = out_of_range_start + 50
chunks = [(0, out_of_range_start, out_of_range_end, 1, "irrelevant")]
results = PDFHighlighter.compute_chunk_bboxes_batch(
pdf_bytes=pdf_bytes,
chunks=chunks,
page_boundaries=boundaries,
full_text=full_text,
)
assert results == {}
@pytest.mark.unit
def test_compute_chunk_bboxes_omits_when_text_not_in_pdf():
"""Chunks whose page-relative text isn't on the page are omitted.
Verifies the second omission path: ``_find_chunk_bbox`` returns None
when the supplied text cannot be located on the rendered page.
"""
pages = ["Hello world."]
pdf_bytes = _make_pdf(pages)
# Build boundaries from the real text but pass a *different* full_text
# so the page-relative slice is content that does not exist in the PDF.
boundaries, _ = _page_boundaries(pages)
bogus_full_text = "Z" * len(pages[0])
chunks = [(0, 0, len(pages[0]), 1, "ignored")]
results = PDFHighlighter.compute_chunk_bboxes_batch(
pdf_bytes=pdf_bytes,
chunks=chunks,
page_boundaries=boundaries,
full_text=bogus_full_text,
)
assert results == {}
@pytest.mark.unit
@pytest.mark.parametrize("page_index", [0, 1])
def test_compute_chunk_bboxes_assigns_correct_page(page_index: int):
"""Verify the page number returned matches the page the chunk lives on."""
pages = [
"Page one talks about apples and oranges in detail.",
"Page two discusses bananas and grapes thoroughly.",
]
pdf_bytes = _make_pdf(pages)
boundaries, full_text = _page_boundaries(pages)
if page_index == 0:
chunk_text = "apples and oranges"
offsets = (0, len(pages[0]))
else:
chunk_text = "bananas and grapes"
offsets = (len(pages[0]), len(pages[0]) + len(pages[1]))
chunks = [(0, offsets[0], offsets[1], page_index + 1, chunk_text)]
results = PDFHighlighter.compute_chunk_bboxes_batch(
pdf_bytes=pdf_bytes,
chunks=chunks,
page_boundaries=boundaries,
full_text=full_text,
)
assert results, "expected a bbox for the chunk"
_, page_num = results[0]
assert page_num == page_index + 1
@pytest.mark.unit
def test_compute_chunk_bboxes_handles_unordered_page_boundaries():
"""Page lookup must match by ``page`` key, not by list position.
Regression guard: an earlier implementation indexed
``page_boundaries[page_num - 1]``, which silently produces a wrong
bbox if boundaries are passed out of order. Reverse the boundaries
and assert the result is identical to the in-order case.
"""
pages = [
"Page one talks about apples and oranges in detail.",
"Page two discusses bananas and grapes thoroughly.",
]
pdf_bytes = _make_pdf(pages)
boundaries, full_text = _page_boundaries(pages)
chunks = [
(0, 0, len(pages[0]), 1, "apples and oranges"),
(
1,
len(pages[0]),
len(pages[0]) + len(pages[1]),
2,
"bananas and grapes",
),
]
in_order = PDFHighlighter.compute_chunk_bboxes_batch(
pdf_bytes=pdf_bytes,
chunks=chunks,
page_boundaries=boundaries,
full_text=full_text,
)
reversed_order = PDFHighlighter.compute_chunk_bboxes_batch(
pdf_bytes=pdf_bytes,
chunks=chunks,
page_boundaries=list(reversed(boundaries)),
full_text=full_text,
)
assert in_order == reversed_order
assert reversed_order[0][1] == 1
assert reversed_order[1][1] == 2
Generated
+82 -43
View File
@@ -769,6 +769,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/97/43/11d6e5d2c00bf000b5329717c74563bf76a9193f4a41cb0c4ef277dde4fa/dynaconf-3.2.13-py2.py3-none-any.whl", hash = "sha256:4305527aef4834bdba3e39479b23c005186e83fb85f65bcaa4bcea58fa26759b", size = 238041, upload-time = "2026-03-17T19:38:45.337Z" },
]
[[package]]
name = "eval-type-backport"
version = "0.3.1"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/fb/a3/cafafb4558fd638aadfe4121dc6cefb8d743368c085acb2f521df0f3d9d7/eval_type_backport-0.3.1.tar.gz", hash = "sha256:57e993f7b5b69d271e37482e62f74e76a0276c82490cf8e4f0dffeb6b332d5ed", size = 9445, upload-time = "2025-12-02T11:51:42.987Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/cf/22/fdc2e30d43ff853720042fa15baa3e6122722be1a7950a98233ebb55cd71/eval_type_backport-0.3.1-py3-none-any.whl", hash = "sha256:279ab641905e9f11129f56a8a78f493518515b83402b860f6f06dd7c011fdfa8", size = 6063, upload-time = "2025-12-02T11:51:41.665Z" },
]
[[package]]
name = "executing"
version = "2.2.1"
@@ -1478,6 +1487,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/73/07/02e16ed01e04a374e644b575638ec7987ae846d25ad97bcc9945a3ee4b0e/jsonpatch-1.33-py2.py3-none-any.whl", hash = "sha256:0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade", size = 12898, upload-time = "2023-06-16T21:01:28.466Z" },
]
[[package]]
name = "jsonpath-python"
version = "1.1.6"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/98/18/4ca8742534a5993ff383f7602e325ce2d5d7cc93d72ac5e1cdedbea8a458/jsonpath_python-1.1.6.tar.gz", hash = "sha256:dded9932b4ec41fb8726e09c83afa4e6be618f938c2db287cc2a81723c639671", size = 88178, upload-time = "2026-05-07T01:26:34.482Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/55/8a/1270a6803bd821cbfcdda387eaa13cb41a7b1f7b9bd145979b3bfb9d6cb7/jsonpath_python-1.1.6-py3-none-any.whl", hash = "sha256:a1c50afd8d3fbbaf47a4873bc890dcb3c15da96f5c020327977d844d8731a2d4", size = 14453, upload-time = "2026-05-07T01:26:33.306Z" },
]
[[package]]
name = "jsonpointer"
version = "3.0.0"
@@ -1842,6 +1860,25 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" },
]
[[package]]
name = "mistralai"
version = "2.4.5"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "eval-type-backport" },
{ name = "httpx" },
{ name = "jsonpath-python" },
{ name = "opentelemetry-api" },
{ name = "opentelemetry-semantic-conventions" },
{ name = "pydantic" },
{ name = "python-dateutil" },
{ name = "typing-inspection" },
]
sdist = { url = "https://files.pythonhosted.org/packages/8e/3f/5624d57c5897c83c55d3e4c7dd4127de42ad14fd3183e26566cdc7dca1bf/mistralai-2.4.5.tar.gz", hash = "sha256:ef165bb004ec4423cbf19a440bf0983ca0c3fc92ab12a35ebca097bdf418e33a", size = 424611, upload-time = "2026-05-07T11:46:43.888Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/1b/48/2c5c4f853dec32a625c1a3d23809b80cf2e135c3441fe1764f72910dfea9/mistralai-2.4.5-py3-none-any.whl", hash = "sha256:bf3b6550258ab16dec8547b90e9c18bebf9099f55b7fc25a884bf0bbeffced0f", size = 995999, upload-time = "2026-05-07T11:46:41.915Z" },
]
[[package]]
name = "mmh3"
version = "5.2.0"
@@ -2086,7 +2123,7 @@ wheels = [
[[package]]
name = "nextcloud-mcp-server"
version = "0.81.0"
version = "0.83.0"
source = { editable = "." }
dependencies = [
{ name = "aiosqlite" },
@@ -2104,6 +2141,7 @@ dependencies = [
{ name = "langchain-text-splitters" },
{ name = "markdownify" },
{ name = "mcp", extra = ["cli"] },
{ name = "mistralai" },
{ name = "openai" },
{ name = "opentelemetry-api" },
{ name = "opentelemetry-exporter-otlp-proto-grpc" },
@@ -2157,6 +2195,7 @@ requires-dist = [
{ name = "langchain-text-splitters", specifier = ">=1.0.0" },
{ name = "markdownify", specifier = ">=0.14.1" },
{ name = "mcp", extras = ["cli"], specifier = ">=1.27,<1.28" },
{ name = "mistralai", specifier = ">=2.4.5" },
{ name = "openai", specifier = ">=2.8.1" },
{ name = "opentelemetry-api", specifier = ">=1.28.2" },
{ name = "opentelemetry-exporter-otlp-proto-grpc", specifier = ">=1.28.2" },
@@ -2341,45 +2380,45 @@ wheels = [
[[package]]
name = "opentelemetry-api"
version = "1.39.0"
version = "1.39.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "importlib-metadata" },
{ name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/c0/0b/e5428c009d4d9af0515b0a8371a8aaae695371af291f45e702f7969dce6b/opentelemetry_api-1.39.0.tar.gz", hash = "sha256:6130644268c5ac6bdffaf660ce878f10906b3e789f7e2daa5e169b047a2933b9", size = 65763, upload-time = "2025-12-03T13:19:56.378Z" }
sdist = { url = "https://files.pythonhosted.org/packages/97/b9/3161be15bb8e3ad01be8be5a968a9237c3027c5be504362ff800fca3e442/opentelemetry_api-1.39.1.tar.gz", hash = "sha256:fbde8c80e1b937a2c61f20347e91c0c18a1940cecf012d62e65a7caf08967c9c", size = 65767, upload-time = "2025-12-11T13:32:39.182Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/05/85/d831a9bc0a9e0e1a304ff3d12c1489a5fbc9bf6690a15dcbdae372bbca45/opentelemetry_api-1.39.0-py3-none-any.whl", hash = "sha256:3c3b3ca5c5687b1b5b37e5c5027ff68eacea8675241b29f13110a8ffbb8f0459", size = 66357, upload-time = "2025-12-03T13:19:33.043Z" },
{ url = "https://files.pythonhosted.org/packages/cf/df/d3f1ddf4bb4cb50ed9b1139cc7b1c54c34a1e7ce8fd1b9a37c0d1551a6bd/opentelemetry_api-1.39.1-py3-none-any.whl", hash = "sha256:2edd8463432a7f8443edce90972169b195e7d6a05500cd29e6d13898187c9950", size = 66356, upload-time = "2025-12-11T13:32:17.304Z" },
]
[[package]]
name = "opentelemetry-exporter-otlp"
version = "1.39.0"
version = "1.39.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "opentelemetry-exporter-otlp-proto-grpc" },
{ name = "opentelemetry-exporter-otlp-proto-http" },
]
sdist = { url = "https://files.pythonhosted.org/packages/13/be/0e9d889f47e55cadc4041e5b53d4e0cc688f9a74811134fb0ba7cbee6905/opentelemetry_exporter_otlp-1.39.0.tar.gz", hash = "sha256:b405da0287b895fe4e2450dedb2a5b072debba1dfcfed5bdb3d1d183d8daa296", size = 6146, upload-time = "2025-12-03T13:19:58.381Z" }
sdist = { url = "https://files.pythonhosted.org/packages/30/9c/3ab1db90f32da200dba332658f2bbe602369e3d19f6aba394031a42635be/opentelemetry_exporter_otlp-1.39.1.tar.gz", hash = "sha256:7cf7470e9fd0060c8a38a23e4f695ac686c06a48ad97f8d4867bc9b420180b9c", size = 6147, upload-time = "2025-12-11T13:32:40.309Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/fc/35/212d2cae4fa9a2c02e74438612268b640ab577b8ccb04590371eb4e0f542/opentelemetry_exporter_otlp-1.39.0-py3-none-any.whl", hash = "sha256:fe155d6968d581b325574ad6dc267c8de299397b18d11feeda2206d0a47928a9", size = 7017, upload-time = "2025-12-03T13:19:35.686Z" },
{ url = "https://files.pythonhosted.org/packages/00/6c/bdc82a066e6fb1dcf9e8cc8d4e026358fe0f8690700cc6369a6bf9bd17a7/opentelemetry_exporter_otlp-1.39.1-py3-none-any.whl", hash = "sha256:68ae69775291f04f000eb4b698ff16ff685fdebe5cb52871bc4e87938a7b00fe", size = 7019, upload-time = "2025-12-11T13:32:19.387Z" },
]
[[package]]
name = "opentelemetry-exporter-otlp-proto-common"
version = "1.39.0"
version = "1.39.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "opentelemetry-proto" },
]
sdist = { url = "https://files.pythonhosted.org/packages/11/cb/3a29ce606b10c76d413d6edd42d25a654af03e73e50696611e757d2602f3/opentelemetry_exporter_otlp_proto_common-1.39.0.tar.gz", hash = "sha256:a135fceed1a6d767f75be65bd2845da344dd8b9258eeed6bc48509d02b184409", size = 20407, upload-time = "2025-12-03T13:19:59.003Z" }
sdist = { url = "https://files.pythonhosted.org/packages/e9/9d/22d241b66f7bbde88a3bfa6847a351d2c46b84de23e71222c6aae25c7050/opentelemetry_exporter_otlp_proto_common-1.39.1.tar.gz", hash = "sha256:763370d4737a59741c89a67b50f9e39271639ee4afc999dadfe768541c027464", size = 20409, upload-time = "2025-12-11T13:32:40.885Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/ef/c6/215edba62d13a3948c718b289539f70e40965bc37fc82ecd55bb0b749c1a/opentelemetry_exporter_otlp_proto_common-1.39.0-py3-none-any.whl", hash = "sha256:3d77be7c4bdf90f1a76666c934368b8abed730b5c6f0547a2ec57feb115849ac", size = 18367, upload-time = "2025-12-03T13:19:36.906Z" },
{ url = "https://files.pythonhosted.org/packages/8c/02/ffc3e143d89a27ac21fd557365b98bd0653b98de8a101151d5805b5d4c33/opentelemetry_exporter_otlp_proto_common-1.39.1-py3-none-any.whl", hash = "sha256:08f8a5862d64cc3435105686d0216c1365dc5701f86844a8cd56597d0c764fde", size = 18366, upload-time = "2025-12-11T13:32:20.2Z" },
]
[[package]]
name = "opentelemetry-exporter-otlp-proto-grpc"
version = "1.39.0"
version = "1.39.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "googleapis-common-protos" },
@@ -2390,14 +2429,14 @@ dependencies = [
{ name = "opentelemetry-sdk" },
{ name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/7e/62/4db083ee9620da3065eeb559e9fc128f41a1d15e7c48d7c83aafbccd354c/opentelemetry_exporter_otlp_proto_grpc-1.39.0.tar.gz", hash = "sha256:7e7bb3f436006836c0e0a42ac619097746ad5553ad7128a5bd4d3e727f37fc06", size = 24650, upload-time = "2025-12-03T13:20:00.06Z" }
sdist = { url = "https://files.pythonhosted.org/packages/53/48/b329fed2c610c2c32c9366d9dc597202c9d1e58e631c137ba15248d8850f/opentelemetry_exporter_otlp_proto_grpc-1.39.1.tar.gz", hash = "sha256:772eb1c9287485d625e4dbe9c879898e5253fea111d9181140f51291b5fec3ad", size = 24650, upload-time = "2025-12-11T13:32:41.429Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/56/e8/d420b94ffddfd8cff85bb4aa5d98da26ce7935dc3cf3eca6b83cd39ab436/opentelemetry_exporter_otlp_proto_grpc-1.39.0-py3-none-any.whl", hash = "sha256:758641278050de9bb895738f35ff8840e4a47685b7e6ef4a201fe83196ba7a05", size = 19765, upload-time = "2025-12-03T13:19:38.143Z" },
{ url = "https://files.pythonhosted.org/packages/81/a3/cc9b66575bd6597b98b886a2067eea2693408d2d5f39dad9ab7fc264f5f3/opentelemetry_exporter_otlp_proto_grpc-1.39.1-py3-none-any.whl", hash = "sha256:fa1c136a05c7e9b4c09f739469cbdb927ea20b34088ab1d959a849b5cc589c18", size = 19766, upload-time = "2025-12-11T13:32:21.027Z" },
]
[[package]]
name = "opentelemetry-exporter-otlp-proto-http"
version = "1.39.0"
version = "1.39.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "googleapis-common-protos" },
@@ -2408,14 +2447,14 @@ dependencies = [
{ name = "requests" },
{ name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/81/dc/1e9bf3f6a28e29eba516bc0266e052996d02bc7e92675f3cd38169607609/opentelemetry_exporter_otlp_proto_http-1.39.0.tar.gz", hash = "sha256:28d78fc0eb82d5a71ae552263d5012fa3ebad18dfd189bf8d8095ba0e65ee1ed", size = 17287, upload-time = "2025-12-03T13:20:01.134Z" }
sdist = { url = "https://files.pythonhosted.org/packages/80/04/2a08fa9c0214ae38880df01e8bfae12b067ec0793446578575e5080d6545/opentelemetry_exporter_otlp_proto_http-1.39.1.tar.gz", hash = "sha256:31bdab9745c709ce90a49a0624c2bd445d31a28ba34275951a6a362d16a0b9cb", size = 17288, upload-time = "2025-12-11T13:32:42.029Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/bc/46/e4a102e17205bb05a50dbf24ef0e92b66b648cd67db9a68865af06a242fd/opentelemetry_exporter_otlp_proto_http-1.39.0-py3-none-any.whl", hash = "sha256:5789cb1375a8b82653328c0ce13a054d285f774099faf9d068032a49de4c7862", size = 19639, upload-time = "2025-12-03T13:19:39.536Z" },
{ url = "https://files.pythonhosted.org/packages/95/f1/b27d3e2e003cd9a3592c43d099d2ed8d0a947c15281bf8463a256db0b46c/opentelemetry_exporter_otlp_proto_http-1.39.1-py3-none-any.whl", hash = "sha256:d9f5207183dd752a412c4cd564ca8875ececba13be6e9c6c370ffb752fd59985", size = 19641, upload-time = "2025-12-11T13:32:22.248Z" },
]
[[package]]
name = "opentelemetry-instrumentation"
version = "0.60b0"
version = "0.60b1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "opentelemetry-api" },
@@ -2423,14 +2462,14 @@ dependencies = [
{ name = "packaging" },
{ name = "wrapt" },
]
sdist = { url = "https://files.pythonhosted.org/packages/55/3c/bd53dbb42eff93d18e3047c7be11224aa9966ce98ac4cc5bfb860a32c95a/opentelemetry_instrumentation-0.60b0.tar.gz", hash = "sha256:4e9fec930f283a2677a2217754b40aaf9ef76edae40499c165bc7f1d15366a74", size = 31707, upload-time = "2025-12-03T13:22:00.352Z" }
sdist = { url = "https://files.pythonhosted.org/packages/41/0f/7e6b713ac117c1f5e4e3300748af699b9902a2e5e34c9cf443dde25a01fa/opentelemetry_instrumentation-0.60b1.tar.gz", hash = "sha256:57ddc7974c6eb35865af0426d1a17132b88b2ed8586897fee187fd5b8944bd6a", size = 31706, upload-time = "2025-12-11T13:36:42.515Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/5c/7b/5b5b9f8cfe727a28553acf9cd287b1d7f706f5c0a00d6e482df55b169483/opentelemetry_instrumentation-0.60b0-py3-none-any.whl", hash = "sha256:aaafa1483543a402819f1bdfb06af721c87d60dd109501f9997332862a35c76a", size = 33096, upload-time = "2025-12-03T13:20:51.785Z" },
{ url = "https://files.pythonhosted.org/packages/77/d2/6788e83c5c86a2690101681aeef27eeb2a6bf22df52d3f263a22cee20915/opentelemetry_instrumentation-0.60b1-py3-none-any.whl", hash = "sha256:04480db952b48fb1ed0073f822f0ee26012b7be7c3eac1a3793122737c78632d", size = 33096, upload-time = "2025-12-11T13:35:33.067Z" },
]
[[package]]
name = "opentelemetry-instrumentation-asgi"
version = "0.60b0"
version = "0.60b1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "asgiref" },
@@ -2439,14 +2478,14 @@ dependencies = [
{ name = "opentelemetry-semantic-conventions" },
{ name = "opentelemetry-util-http" },
]
sdist = { url = "https://files.pythonhosted.org/packages/b0/0a/715ea7044708d3c215385fb2a1c6ffe429aacb3cd23a348060aaeda52834/opentelemetry_instrumentation_asgi-0.60b0.tar.gz", hash = "sha256:928731218050089dca69f0fe980b8bfe109f384be8b89802d7337372ddb67b91", size = 26083, upload-time = "2025-12-03T13:22:05.672Z" }
sdist = { url = "https://files.pythonhosted.org/packages/77/db/851fa88db7441da82d50bd80f2de5ee55213782e25dc858e04d0c9961d60/opentelemetry_instrumentation_asgi-0.60b1.tar.gz", hash = "sha256:16bfbe595cd24cda309a957456d0fc2523f41bc7b076d1f2d7e98a1ad9876d6f", size = 26107, upload-time = "2025-12-11T13:36:47.015Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/9b/8c/c6c59127fd996107243ca45669355665a7daff578ddafb86d6d2d3b01428/opentelemetry_instrumentation_asgi-0.60b0-py3-none-any.whl", hash = "sha256:9d76a541269452c718a0384478f3291feb650c5a3f29e578fdc6613ea3729cf3", size = 16907, upload-time = "2025-12-03T13:20:58.962Z" },
{ url = "https://files.pythonhosted.org/packages/76/76/1fb94367cef64420d2171157a6b9509582873bd09a6afe08a78a8d1f59d9/opentelemetry_instrumentation_asgi-0.60b1-py3-none-any.whl", hash = "sha256:d48def2dbed10294c99cfcf41ebbd0c414d390a11773a41f472d20000fcddc25", size = 16933, upload-time = "2025-12-11T13:35:40.462Z" },
]
[[package]]
name = "opentelemetry-instrumentation-httpx"
version = "0.60b0"
version = "0.60b1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "opentelemetry-api" },
@@ -2455,70 +2494,70 @@ dependencies = [
{ name = "opentelemetry-util-http" },
{ name = "wrapt" },
]
sdist = { url = "https://files.pythonhosted.org/packages/09/71/9dc0bc5ab14122251f520ffe9fc8dd892ca688d7d591482b5b1843d685d2/opentelemetry_instrumentation_httpx-0.60b0.tar.gz", hash = "sha256:fcf349a92fb0b941a2a18bec65141f4ba62cbf7a457a1aa580794bad44dc477c", size = 20612, upload-time = "2025-12-03T13:22:20.5Z" }
sdist = { url = "https://files.pythonhosted.org/packages/86/08/11208bcfcab4fc2023252c3f322aa397fd9ad948355fea60f5fc98648603/opentelemetry_instrumentation_httpx-0.60b1.tar.gz", hash = "sha256:a506ebaf28c60112cbe70ad4f0338f8603f148938cb7b6794ce1051cd2b270ae", size = 20611, upload-time = "2025-12-11T13:37:01.661Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/3a/22/a340c8bfad6f31bfd6a15b0b24d5e68e05e3975e4dbdc3cea6ec4f96e060/opentelemetry_instrumentation_httpx-0.60b0-py3-none-any.whl", hash = "sha256:3f5e6fc4ddf1d9de2aaddb5255110827154dbd4de9187da906c8c2a3cc2219e9", size = 15702, upload-time = "2025-12-03T13:21:20.667Z" },
{ url = "https://files.pythonhosted.org/packages/43/59/b98e84eebf745ffc75397eaad4763795bff8a30cbf2373a50ed4e70646c5/opentelemetry_instrumentation_httpx-0.60b1-py3-none-any.whl", hash = "sha256:f37636dd742ad2af83d896ba69601ed28da51fa4e25d1ab62fde89ce413e275b", size = 15701, upload-time = "2025-12-11T13:36:04.56Z" },
]
[[package]]
name = "opentelemetry-instrumentation-logging"
version = "0.60b0"
version = "0.60b1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "opentelemetry-api" },
{ name = "opentelemetry-instrumentation" },
]
sdist = { url = "https://files.pythonhosted.org/packages/5b/8b/a1aed0b695a58a1f0bdcf90fae5e468cc0fd7de2b74b9816b17e45c38e43/opentelemetry_instrumentation_logging-0.60b0.tar.gz", hash = "sha256:6d87840666669cbbcd53d2230c7a33476862d0bf7f1adc67a95519c6ccfc8281", size = 9969, upload-time = "2025-12-03T13:22:22.608Z" }
sdist = { url = "https://files.pythonhosted.org/packages/60/a6/4515895b383113677fd2ad21813df5e56108a2df14ebb7916c962c9a0234/opentelemetry_instrumentation_logging-0.60b1.tar.gz", hash = "sha256:98f4b9c7aeb9314a30feee7c002c7ea9abea07c90df5f97fb058b850bc45b89a", size = 9968, upload-time = "2025-12-11T13:37:03.974Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/00/89/6d4f5d8d03376637bff5b8d9e91356104a6f1ce9a34a2099bb2f4bf5e2b5/opentelemetry_instrumentation_logging-0.60b0-py3-none-any.whl", hash = "sha256:af75b3020911b9b6a1b4b19819a165eb131ce9bfdd313062d578fc2dc9a5cd0f", size = 12576, upload-time = "2025-12-03T13:21:24.797Z" },
{ url = "https://files.pythonhosted.org/packages/f1/f9/8a4ce3901bc52277794e4b18c4ac43dc5929806eff01d22812364132f45f/opentelemetry_instrumentation_logging-0.60b1-py3-none-any.whl", hash = "sha256:f2e18cbc7e1dd3628c80e30d243897fdc93c5b7e0c8ae60abd2b9b6a99f82343", size = 12577, upload-time = "2025-12-11T13:36:08.123Z" },
]
[[package]]
name = "opentelemetry-proto"
version = "1.39.0"
version = "1.39.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "protobuf" },
]
sdist = { url = "https://files.pythonhosted.org/packages/48/b5/64d2f8c3393cd13ea2092106118f7b98461ba09333d40179a31444c6f176/opentelemetry_proto-1.39.0.tar.gz", hash = "sha256:c1fa48678ad1a1624258698e59be73f990b7fc1f39e73e16a9d08eef65dd838c", size = 46153, upload-time = "2025-12-03T13:20:08.729Z" }
sdist = { url = "https://files.pythonhosted.org/packages/49/1d/f25d76d8260c156c40c97c9ed4511ec0f9ce353f8108ca6e7561f82a06b2/opentelemetry_proto-1.39.1.tar.gz", hash = "sha256:6c8e05144fc0d3ed4d22c2289c6b126e03bcd0e6a7da0f16cedd2e1c2772e2c8", size = 46152, upload-time = "2025-12-11T13:32:48.681Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/e3/4d/d500e1862beed68318705732d1976c390f4a72ca8009c4983ff627acff20/opentelemetry_proto-1.39.0-py3-none-any.whl", hash = "sha256:1e086552ac79acb501485ff0ce75533f70f3382d43d0a30728eeee594f7bf818", size = 72534, upload-time = "2025-12-03T13:19:50.251Z" },
{ url = "https://files.pythonhosted.org/packages/51/95/b40c96a7b5203005a0b03d8ce8cd212ff23f1793d5ba289c87a097571b18/opentelemetry_proto-1.39.1-py3-none-any.whl", hash = "sha256:22cdc78efd3b3765d09e68bfbd010d4fc254c9818afd0b6b423387d9dee46007", size = 72535, upload-time = "2025-12-11T13:32:33.866Z" },
]
[[package]]
name = "opentelemetry-sdk"
version = "1.39.0"
version = "1.39.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "opentelemetry-api" },
{ name = "opentelemetry-semantic-conventions" },
{ name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/51/e3/7cd989003e7cde72e0becfe830abff0df55c69d237ee7961a541e0167833/opentelemetry_sdk-1.39.0.tar.gz", hash = "sha256:c22204f12a0529e07aa4d985f1bca9d6b0e7b29fe7f03e923548ae52e0e15dde", size = 171322, upload-time = "2025-12-03T13:20:09.651Z" }
sdist = { url = "https://files.pythonhosted.org/packages/eb/fb/c76080c9ba07e1e8235d24cdcc4d125ef7aa3edf23eb4e497c2e50889adc/opentelemetry_sdk-1.39.1.tar.gz", hash = "sha256:cf4d4563caf7bff906c9f7967e2be22d0d6b349b908be0d90fb21c8e9c995cc6", size = 171460, upload-time = "2025-12-11T13:32:49.369Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/a4/b4/2adc8bc83eb1055ecb592708efb6f0c520cc2eb68970b02b0f6ecda149cf/opentelemetry_sdk-1.39.0-py3-none-any.whl", hash = "sha256:90cfb07600dfc0d2de26120cebc0c8f27e69bf77cd80ef96645232372709a514", size = 132413, upload-time = "2025-12-03T13:19:51.364Z" },
{ url = "https://files.pythonhosted.org/packages/7c/98/e91cf858f203d86f4eccdf763dcf01cf03f1dae80c3750f7e635bfa206b6/opentelemetry_sdk-1.39.1-py3-none-any.whl", hash = "sha256:4d5482c478513ecb0a5d938dcc61394e647066e0cc2676bee9f3af3f3f45f01c", size = 132565, upload-time = "2025-12-11T13:32:35.069Z" },
]
[[package]]
name = "opentelemetry-semantic-conventions"
version = "0.60b0"
version = "0.60b1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "opentelemetry-api" },
{ name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/71/0e/176a7844fe4e3cb5de604212094dffaed4e18b32f1c56b5258bcbcba85c2/opentelemetry_semantic_conventions-0.60b0.tar.gz", hash = "sha256:227d7aa73cbb8a2e418029d6b6465553aa01cf7e78ec9d0bc3255c7b3ac5bf8f", size = 137935, upload-time = "2025-12-03T13:20:12.395Z" }
sdist = { url = "https://files.pythonhosted.org/packages/91/df/553f93ed38bf22f4b999d9be9c185adb558982214f33eae539d3b5cd0858/opentelemetry_semantic_conventions-0.60b1.tar.gz", hash = "sha256:87c228b5a0669b748c76d76df6c364c369c28f1c465e50f661e39737e84bc953", size = 137935, upload-time = "2025-12-11T13:32:50.487Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/d0/56/af0306666f91bae47db14d620775604688361f0f76a872e0005277311131/opentelemetry_semantic_conventions-0.60b0-py3-none-any.whl", hash = "sha256:069530852691136018087b52688857d97bba61cd641d0f8628d2d92788c4f78a", size = 219981, upload-time = "2025-12-03T13:19:53.585Z" },
{ url = "https://files.pythonhosted.org/packages/7a/5e/5958555e09635d09b75de3c4f8b9cae7335ca545d77392ffe7331534c402/opentelemetry_semantic_conventions-0.60b1-py3-none-any.whl", hash = "sha256:9fa8c8b0c110da289809292b0591220d3a7b53c1526a23021e977d68597893fb", size = 219982, upload-time = "2025-12-11T13:32:36.955Z" },
]
[[package]]
name = "opentelemetry-util-http"
version = "0.60b0"
version = "0.60b1"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/38/0d/786a713445cf338131fef3a84fab1378e4b2ef3c3ea348eeb0c915eb804a/opentelemetry_util_http-0.60b0.tar.gz", hash = "sha256:e42b7bb49bba43b6f34390327d97e5016eb1c47949ceaf37c4795472a4e3a82d", size = 10576, upload-time = "2025-12-03T13:22:41.224Z" }
sdist = { url = "https://files.pythonhosted.org/packages/50/fc/c47bb04a1d8a941a4061307e1eddfa331ed4d0ab13d8a9781e6db256940a/opentelemetry_util_http-0.60b1.tar.gz", hash = "sha256:0d97152ca8c8a41ced7172d29d3622a219317f74ae6bb3027cfbdcf22c3cc0d6", size = 11053, upload-time = "2025-12-11T13:37:25.115Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/53/5d/a448862f6d10c95685ed0e703596b6bd1784074e7ad90bffdc550abb7b68/opentelemetry_util_http-0.60b0-py3-none-any.whl", hash = "sha256:4f366f1a48adb74ffa6f80aee26f96882e767e01b03cd1cfb948b6e1020341fe", size = 8742, upload-time = "2025-12-03T13:21:54.553Z" },
{ url = "https://files.pythonhosted.org/packages/16/5c/d3f1733665f7cd582ef0842fb1d2ed0bc1fba10875160593342d22bba375/opentelemetry_util_http-0.60b1-py3-none-any.whl", hash = "sha256:66381ba28550c91bee14dcba8979ace443444af1ed609226634596b4b0faf199", size = 8947, upload-time = "2025-12-11T13:36:37.151Z" },
]
[[package]]
@@ -3314,7 +3353,7 @@ wheels = [
[[package]]
name = "pytest-otel"
version = "2.0.1"
version = "2.0.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "opentelemetry-api" },
@@ -3322,9 +3361,9 @@ dependencies = [
{ name = "opentelemetry-sdk" },
{ name = "pytest" },
]
sdist = { url = "https://files.pythonhosted.org/packages/ca/5e/771f8dbdf55ae57603d89bb26e046c13e8bee3492c74cb3afb6044166e9a/pytest_otel-2.0.1.tar.gz", hash = "sha256:3d529dc34105862cca39fd1258d00dd3d17f3b7d92ebf0953d55326bb017af3d", size = 17880, upload-time = "2025-12-08T14:56:40.393Z" }
sdist = { url = "https://files.pythonhosted.org/packages/98/14/e4f7cb90c4c93dbdd85034fd65e7a16da328bd71c6ca5270e4f8d37fa4ba/pytest_otel-2.0.3.tar.gz", hash = "sha256:782985fef50acc6922db5e69da4584b384d785c863a96ebc11eaf2a7fef5189c", size = 18587, upload-time = "2026-01-02T16:12:20.394Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/16/fa/0cfe23bac571f68b4f427ad7ae313ddd8de881ec2ff49dcea196a2a96592/pytest_otel-2.0.1-py2.py3-none-any.whl", hash = "sha256:501f36f02f55578ca34c3ccbe55e81cc1e14bb130da13f7cf0b6480d54a9e5db", size = 14530, upload-time = "2025-12-08T14:56:41.731Z" },
{ url = "https://files.pythonhosted.org/packages/ac/f8/d9b93b41b299c7e14b47887eb31fb4cb77d771c2e9f3e0bcac0750f62699/pytest_otel-2.0.3-py2.py3-none-any.whl", hash = "sha256:3c6c331e943609ad7df7c718714090dee91211340f133a8798217e72e3b2cd67", size = 14914, upload-time = "2026-01-02T16:12:21.946Z" },
]
[[package]]