Merge remote-tracking branch 'origin/master' into fix/chunk-context-indexed-lookup
# Conflicts: # nextcloud_mcp_server/api/visualization.py # nextcloud_mcp_server/auth/viz_routes.py
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
#!/bin/bash
|
||||
if ! command -v sonar &> /dev/null; then
|
||||
exit 0
|
||||
fi
|
||||
sonar hook claude-pre-tool-use
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/bash
|
||||
if ! command -v sonar &> /dev/null; then
|
||||
exit 0
|
||||
fi
|
||||
sonar hook claude-prompt-submit
|
||||
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"hooks": {
|
||||
"PreToolUse": [
|
||||
{
|
||||
"matcher": "Read",
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": ".claude/hooks/sonar-secrets/build-scripts/pretool-secrets.sh",
|
||||
"timeout": 60
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"UserPromptSubmit": [
|
||||
{
|
||||
"matcher": "*",
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": ".claude/hooks/sonar-secrets/build-scripts/prompt-secrets.sh",
|
||||
"timeout": 60
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ description: |
|
||||
in this repo's automated PR reviews. Use when the user is about to push, says "ready
|
||||
to push", "review my work", "check before PR", or invokes /pre-push-review.
|
||||
Report-only — does not modify code.
|
||||
model: sonnet
|
||||
allowed-tools:
|
||||
- Bash
|
||||
- Read
|
||||
|
||||
@@ -5,6 +5,30 @@ All notable changes to the Nextcloud MCP Server will be documented in this file.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
||||
and this project adheres to [PEP 440](https://peps.python.org/pep-0440/).
|
||||
|
||||
## v0.83.0 (2026-05-08)
|
||||
|
||||
### Feat
|
||||
|
||||
- **vector**: replace inline page-image payloads with chunk_bbox (Deck #76)
|
||||
|
||||
### Refactor
|
||||
|
||||
- **vector**: address PR #775 review round 3 — fix unused var, harden boundary lookup, rename trace span
|
||||
- **vector**: address PR #775 review round 2 — drop dead page field, add omission tests
|
||||
- **vector**: address PR #775 review — drop unused payload key, fix resource leaks
|
||||
|
||||
## v0.82.0 (2026-05-08)
|
||||
|
||||
### Feat
|
||||
|
||||
- **providers**: add Mistral embedding provider, route registry through dynaconf
|
||||
|
||||
### Refactor
|
||||
|
||||
- **providers**: address PR #772 review round 3 — hermetic test, lazy logging, defensive-guard tests
|
||||
- **providers**: address PR #772 review round 2 — guard, naming, docs, tests
|
||||
- **providers**: address PR #772 review — shared retry, cleaner imports, no-op close
|
||||
|
||||
## v0.81.0 (2026-05-07)
|
||||
|
||||
### Feat
|
||||
|
||||
+1
-1
@@ -334,7 +334,7 @@ services:
|
||||
- claude-funnel
|
||||
|
||||
qdrant:
|
||||
image: docker.io/qdrant/qdrant:v1.17.1@sha256:94728574965d17c6485dd361aa3c0818b325b9016dac5ea6afec7b4b2700865f
|
||||
image: docker.io/qdrant/qdrant:v1.18.0@sha256:1cf3e07c9f269030c7cfed0d5c0beea7bb081848a88e2ae13b35a613d4dd5019
|
||||
restart: always
|
||||
ports:
|
||||
- 127.0.0.1:6333:6333 # REST API
|
||||
|
||||
@@ -118,10 +118,16 @@ class ProviderRegistry:
|
||||
@staticmethod
|
||||
def create_provider() -> Provider:
|
||||
# 1. Bedrock (AWS_REGION or BEDROCK_*_MODEL)
|
||||
# 2. Ollama (OLLAMA_BASE_URL)
|
||||
# 3. Simple (fallback)
|
||||
# 2. OpenAI (OPENAI_API_KEY)
|
||||
# 3. Mistral (MISTRAL_API_KEY)
|
||||
# 4. Ollama (OLLAMA_BASE_URL)
|
||||
# 5. Simple (fallback)
|
||||
```
|
||||
|
||||
Configuration is sourced via the dynaconf-backed `Settings` dataclass in
|
||||
`config.py`; the registry reads `get_settings()` rather than `os.getenv`
|
||||
directly, so settings files and env vars share one resolution path.
|
||||
|
||||
**Environment Variables:**
|
||||
|
||||
**Bedrock:**
|
||||
@@ -131,6 +137,17 @@ class ProviderRegistry:
|
||||
- `BEDROCK_EMBEDDING_MODEL`: Model ID for embeddings (e.g., "amazon.titan-embed-text-v2:0")
|
||||
- `BEDROCK_GENERATION_MODEL`: Model ID for text generation (e.g., "anthropic.claude-3-sonnet-20240229-v1:0")
|
||||
|
||||
**OpenAI:**
|
||||
- `OPENAI_API_KEY`: OpenAI API key (or `GITHUB_TOKEN` for GitHub Models)
|
||||
- `OPENAI_BASE_URL`: Optional base URL override for OpenAI-compatible APIs
|
||||
- `OPENAI_EMBEDDING_MODEL`: Embedding model (default: "text-embedding-3-small")
|
||||
- `OPENAI_GENERATION_MODEL`: Generation model (e.g., "gpt-4o-mini")
|
||||
|
||||
**Mistral (embeddings only):**
|
||||
- `MISTRAL_API_KEY`: Mistral API key from console.mistral.ai
|
||||
- `MISTRAL_EMBEDDING_MODEL`: Embedding model (default: "mistral-embed", 1024-dim)
|
||||
- `MISTRAL_BASE_URL`: Optional server URL override (proxies, on-prem)
|
||||
|
||||
**Ollama:**
|
||||
- `OLLAMA_BASE_URL`: Ollama API base URL (e.g., "http://localhost:11434")
|
||||
- `OLLAMA_EMBEDDING_MODEL`: Model for embeddings (default: "nomic-embed-text")
|
||||
|
||||
+67
-3
@@ -410,9 +410,16 @@ DOCUMENT_CHUNK_OVERLAP=50 # Overlapping words between chunks (defaul
|
||||
|
||||
### Embedding Service Configuration
|
||||
|
||||
The server uses an embedding service to generate vector representations. Two options are available:
|
||||
The server picks an embedding provider via auto-detection. Priority order
|
||||
(see `nextcloud_mcp_server/providers/registry.py`):
|
||||
|
||||
#### Ollama (Recommended)
|
||||
1. **Bedrock** — if `AWS_REGION` or `BEDROCK_EMBEDDING_MODEL` is set
|
||||
2. **OpenAI** — if `OPENAI_API_KEY` is set
|
||||
3. **Mistral** — if `MISTRAL_API_KEY` is set
|
||||
4. **Ollama** — if `OLLAMA_BASE_URL` is set
|
||||
5. **Simple** — fallback when nothing else is configured
|
||||
|
||||
#### Ollama (Recommended for self-hosted)
|
||||
|
||||
Use a local Ollama instance for embeddings:
|
||||
|
||||
@@ -422,9 +429,52 @@ OLLAMA_EMBEDDING_MODEL=nomic-embed-text # Default model
|
||||
OLLAMA_VERIFY_SSL=true # Verify SSL certificates
|
||||
```
|
||||
|
||||
#### OpenAI
|
||||
|
||||
Hosted OpenAI embeddings (or any OpenAI-compatible API via `OPENAI_BASE_URL`):
|
||||
|
||||
```dotenv
|
||||
OPENAI_API_KEY=sk-...
|
||||
OPENAI_EMBEDDING_MODEL=text-embedding-3-small # default
|
||||
# OPENAI_BASE_URL=https://models.github.ai/inference # optional
|
||||
```
|
||||
|
||||
#### Mistral
|
||||
|
||||
Hosted Mistral embeddings. Requires a Mistral API key from
|
||||
[console.mistral.ai](https://console.mistral.ai). Currently embeddings only
|
||||
(no text generation).
|
||||
|
||||
```dotenv
|
||||
MISTRAL_API_KEY=...
|
||||
MISTRAL_EMBEDDING_MODEL=mistral-embed # default; produces 1024-dim vectors
|
||||
# MISTRAL_BASE_URL=https://api.mistral.ai # optional override (proxies, on-prem)
|
||||
```
|
||||
|
||||
Switching to or from Mistral forces a new Qdrant collection because the
|
||||
collection name encodes the model (see "Qdrant Collection Naming" above).
|
||||
|
||||
#### Amazon Bedrock
|
||||
|
||||
Bedrock provides hosted embedding models (Titan, Cohere) and uses the AWS
|
||||
credential chain (env vars, profiles, or IAM role):
|
||||
|
||||
```dotenv
|
||||
AWS_REGION=us-east-1
|
||||
BEDROCK_EMBEDDING_MODEL=amazon.titan-embed-text-v2:0
|
||||
# AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY are optional — boto3 will use
|
||||
# the standard credential chain if not set.
|
||||
```
|
||||
|
||||
#### Simple Embedding Provider (Fallback)
|
||||
|
||||
If `OLLAMA_BASE_URL` is not set, the server uses a simple random embedding provider for testing. This is **not suitable for production** as it generates random embeddings with no semantic meaning.
|
||||
If no provider env var is set, the server falls back to a simple deterministic
|
||||
embedding provider for testing. This is **not suitable for production** as
|
||||
its embeddings have no semantic meaning.
|
||||
|
||||
```dotenv
|
||||
SIMPLE_EMBEDDING_DIMENSION=384 # optional; default 384
|
||||
```
|
||||
|
||||
### Document Chunking Configuration
|
||||
|
||||
@@ -533,7 +583,21 @@ equivalent.** Operators who need a runtime toggle should open an issue.
|
||||
| `VECTOR_SYNC_QUEUE_MAX_SIZE` | ⚠️ Optional | `10000` | Max queued documents |
|
||||
| `OLLAMA_BASE_URL` | ⚠️ Optional | - | Ollama API endpoint for embeddings |
|
||||
| `OLLAMA_EMBEDDING_MODEL` | ⚠️ Optional | `nomic-embed-text` | Embedding model to use |
|
||||
| `OLLAMA_GENERATION_MODEL` | ⚠️ Optional | - | Ollama model for text generation |
|
||||
| `OLLAMA_VERIFY_SSL` | ⚠️ Optional | `true` | Verify SSL certificates |
|
||||
| `OPENAI_API_KEY` | ⚠️ Optional | - | OpenAI API key (selects OpenAI provider) |
|
||||
| `OPENAI_BASE_URL` | ⚠️ Optional | - | OpenAI base URL override (for compatible APIs) |
|
||||
| `OPENAI_EMBEDDING_MODEL` | ⚠️ Optional | `text-embedding-3-small` | OpenAI embedding model |
|
||||
| `OPENAI_GENERATION_MODEL` | ⚠️ Optional | - | OpenAI model for text generation |
|
||||
| `MISTRAL_API_KEY` | ⚠️ Optional | - | Mistral API key (selects Mistral provider) |
|
||||
| `MISTRAL_EMBEDDING_MODEL` | ⚠️ Optional | `mistral-embed` | Mistral embedding model (1024-dim) |
|
||||
| `MISTRAL_BASE_URL` | ⚠️ Optional | - | Mistral base URL override (proxies, on-prem) |
|
||||
| `AWS_REGION` | ⚠️ Optional | - | AWS region (selects Bedrock provider) |
|
||||
| `AWS_ACCESS_KEY_ID` | ⚠️ Optional | - | AWS access key (boto3 credential chain fallback) |
|
||||
| `AWS_SECRET_ACCESS_KEY` | ⚠️ Optional | - | AWS secret key (boto3 credential chain fallback) |
|
||||
| `BEDROCK_EMBEDDING_MODEL` | ⚠️ Optional | - | Bedrock embedding model ID |
|
||||
| `BEDROCK_GENERATION_MODEL` | ⚠️ Optional | - | Bedrock generation model ID |
|
||||
| `SIMPLE_EMBEDDING_DIMENSION` | ⚠️ Optional | `384` | Dimension for the fallback Simple provider |
|
||||
| `DOCUMENT_CHUNK_SIZE` | ⚠️ Optional | `512` | Words per chunk for document embedding |
|
||||
| `DOCUMENT_CHUNK_OVERLAP` | ⚠️ Optional | `50` | Overlapping words between chunks (must be < chunk size) |
|
||||
|
||||
|
||||
@@ -565,9 +565,10 @@ async def get_chunk_context(request: Request) -> JSONResponse:
|
||||
status_code=404,
|
||||
)
|
||||
|
||||
# For PDF files, also fetch the highlighted page image from Qdrant if available
|
||||
# This is useful for clients that want to show a pre-rendered image
|
||||
highlighted_page_image = None
|
||||
# For PDF files, also fetch the chunk's bounding box from Qdrant if
|
||||
# available so the client can overlay a highlight on top of a
|
||||
# render-on-demand page image (Deck #76).
|
||||
chunk_bbox = None
|
||||
page_number = chunk_context.page_number
|
||||
|
||||
if doc_type == "file":
|
||||
@@ -575,7 +576,7 @@ async def get_chunk_context(request: Request) -> JSONResponse:
|
||||
settings = get_settings()
|
||||
qdrant_client = await get_qdrant_client()
|
||||
|
||||
# Prefer chunk_index for the highlighted-image lookup (always indexed);
|
||||
# Prefer chunk_index for the chunk-bbox lookup (always indexed);
|
||||
# fall back to (chunk_start_offset, chunk_end_offset) when not provided.
|
||||
if chunk_index is not None:
|
||||
points_response = await qdrant_client.scroll(
|
||||
@@ -589,10 +590,6 @@ async def get_chunk_context(request: Request) -> JSONResponse:
|
||||
FieldCondition(
|
||||
key="user_id", match=MatchValue(value=user_id)
|
||||
),
|
||||
FieldCondition(
|
||||
key="doc_type",
|
||||
match=MatchValue(value=doc_type),
|
||||
),
|
||||
FieldCondition(
|
||||
key="chunk_index",
|
||||
match=MatchValue(value=chunk_index),
|
||||
@@ -601,7 +598,7 @@ async def get_chunk_context(request: Request) -> JSONResponse:
|
||||
),
|
||||
limit=1,
|
||||
with_vectors=False,
|
||||
with_payload=["highlighted_page_image", "page_number"],
|
||||
with_payload=["chunk_bbox", "page_number"],
|
||||
)
|
||||
else:
|
||||
points_response = await qdrant_client.scroll(
|
||||
@@ -615,10 +612,6 @@ async def get_chunk_context(request: Request) -> JSONResponse:
|
||||
FieldCondition(
|
||||
key="user_id", match=MatchValue(value=user_id)
|
||||
),
|
||||
FieldCondition(
|
||||
key="doc_type",
|
||||
match=MatchValue(value=doc_type),
|
||||
),
|
||||
FieldCondition(
|
||||
key="chunk_start_offset",
|
||||
match=MatchValue(value=start),
|
||||
@@ -631,19 +624,19 @@ async def get_chunk_context(request: Request) -> JSONResponse:
|
||||
),
|
||||
limit=1,
|
||||
with_vectors=False,
|
||||
with_payload=["highlighted_page_image", "page_number"],
|
||||
with_payload=["chunk_bbox", "page_number"],
|
||||
)
|
||||
|
||||
if points_response[0]:
|
||||
payload = points_response[0][0].payload
|
||||
if payload:
|
||||
highlighted_page_image = payload.get("highlighted_page_image")
|
||||
chunk_bbox = payload.get("chunk_bbox")
|
||||
# Trust Qdrant page number if available (might be more accurate than context expansion logic)
|
||||
if payload.get("page_number") is not None:
|
||||
page_number = payload.get("page_number")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to fetch highlighted image: {e}")
|
||||
logger.warning(f"Failed to fetch chunk bbox: {e}")
|
||||
|
||||
# Build response
|
||||
response_data = {
|
||||
@@ -658,8 +651,8 @@ async def get_chunk_context(request: Request) -> JSONResponse:
|
||||
"total_chunks": chunk_context.total_chunks,
|
||||
}
|
||||
|
||||
if highlighted_page_image:
|
||||
response_data["highlighted_page_image"] = highlighted_page_image
|
||||
if chunk_bbox:
|
||||
response_data["chunk_bbox"] = chunk_bbox
|
||||
|
||||
return JSONResponse(response_data)
|
||||
|
||||
|
||||
@@ -125,12 +125,13 @@ from nextcloud_mcp_server.server import (
|
||||
)
|
||||
from nextcloud_mcp_server.server.auth_tools import register_auth_tools
|
||||
from nextcloud_mcp_server.server.oauth_tools import register_oauth_tools
|
||||
from nextcloud_mcp_server.vector import processor_task, scanner_task
|
||||
from nextcloud_mcp_server.vector.oauth_sync import (
|
||||
oauth_processor_task,
|
||||
user_manager_task,
|
||||
)
|
||||
from nextcloud_mcp_server.vector.processor import processor_task
|
||||
from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client
|
||||
from nextcloud_mcp_server.vector.scanner import scanner_task
|
||||
from nextcloud_mcp_server.vector.webhook_receiver import handle_nextcloud_webhook
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -623,8 +623,10 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse:
|
||||
f"after_len={len(chunk_context.after_context)}"
|
||||
)
|
||||
|
||||
# For PDF files, also fetch the highlighted page image from Qdrant
|
||||
highlighted_page_image = None
|
||||
# For PDF files, also fetch the chunk bbox from Qdrant so the client
|
||||
# can overlay a highlight on top of a render-on-demand page image
|
||||
# (Deck #76).
|
||||
chunk_bbox = None
|
||||
page_number = chunk_context.page_number
|
||||
if doc_type == "file":
|
||||
try:
|
||||
@@ -632,7 +634,7 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse:
|
||||
qdrant_client = await get_qdrant_client()
|
||||
username = request.user.display_name
|
||||
|
||||
# Prefer chunk_index for the highlighted-image lookup (always indexed);
|
||||
# Prefer chunk_index for the chunk-bbox lookup (always indexed);
|
||||
# fall back to (chunk_start_offset, chunk_end_offset) when not provided.
|
||||
if chunk_index is not None:
|
||||
points_response = await qdrant_client.scroll(
|
||||
@@ -646,10 +648,6 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse:
|
||||
FieldCondition(
|
||||
key="user_id", match=MatchValue(value=username)
|
||||
),
|
||||
FieldCondition(
|
||||
key="doc_type",
|
||||
match=MatchValue(value=doc_type),
|
||||
),
|
||||
FieldCondition(
|
||||
key="chunk_index",
|
||||
match=MatchValue(value=chunk_index),
|
||||
@@ -658,7 +656,7 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse:
|
||||
),
|
||||
limit=1,
|
||||
with_vectors=False,
|
||||
with_payload=["highlighted_page_image", "page_number"],
|
||||
with_payload=["chunk_bbox", "page_number"],
|
||||
)
|
||||
else:
|
||||
points_response = await qdrant_client.scroll(
|
||||
@@ -672,10 +670,6 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse:
|
||||
FieldCondition(
|
||||
key="user_id", match=MatchValue(value=username)
|
||||
),
|
||||
FieldCondition(
|
||||
key="doc_type",
|
||||
match=MatchValue(value=doc_type),
|
||||
),
|
||||
FieldCondition(
|
||||
key="chunk_start_offset",
|
||||
match=MatchValue(value=start),
|
||||
@@ -688,22 +682,20 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse:
|
||||
),
|
||||
limit=1,
|
||||
with_vectors=False,
|
||||
with_payload=["highlighted_page_image", "page_number"],
|
||||
with_payload=["chunk_bbox", "page_number"],
|
||||
)
|
||||
|
||||
points = points_response[0]
|
||||
if points and points[0].payload:
|
||||
highlighted_page_image = points[0].payload.get(
|
||||
"highlighted_page_image"
|
||||
)
|
||||
chunk_bbox = points[0].payload.get("chunk_bbox")
|
||||
page_number = points[0].payload.get("page_number")
|
||||
if highlighted_page_image:
|
||||
if chunk_bbox:
|
||||
logger.info(
|
||||
f"Found highlighted image for chunk: "
|
||||
f"page={page_number}, image_size={len(highlighted_page_image)}"
|
||||
f"Found chunk bbox: page={page_number}, "
|
||||
f"rects={len(chunk_bbox)}"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to fetch highlighted image: {e}")
|
||||
logger.warning(f"Failed to fetch chunk bbox: {e}")
|
||||
|
||||
# Return response compatible with frontend expectations
|
||||
response_data: dict = {
|
||||
@@ -718,8 +710,8 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse:
|
||||
"total_chunks": chunk_context.total_chunks,
|
||||
}
|
||||
|
||||
if highlighted_page_image:
|
||||
response_data["highlighted_page_image"] = highlighted_page_image
|
||||
if chunk_bbox:
|
||||
response_data["chunk_bbox"] = chunk_bbox
|
||||
|
||||
return JSONResponse(response_data)
|
||||
|
||||
|
||||
@@ -77,11 +77,25 @@ _DEFAULTS: dict[str, Any] = {
|
||||
# Ollama
|
||||
"ollama_base_url": None,
|
||||
"ollama_embedding_model": "nomic-embed-text",
|
||||
"ollama_generation_model": None,
|
||||
"ollama_verify_ssl": True,
|
||||
# OpenAI
|
||||
"openai_api_key": None,
|
||||
"openai_base_url": None,
|
||||
"openai_embedding_model": "text-embedding-3-small",
|
||||
"openai_generation_model": None,
|
||||
# Bedrock (AWS)
|
||||
"aws_region": None,
|
||||
"aws_access_key_id": None,
|
||||
"aws_secret_access_key": None,
|
||||
"bedrock_embedding_model": None,
|
||||
"bedrock_generation_model": None,
|
||||
# Mistral
|
||||
"mistral_api_key": None,
|
||||
"mistral_embedding_model": "mistral-embed",
|
||||
"mistral_base_url": None,
|
||||
# Simple (fallback) embedding dimension
|
||||
"simple_embedding_dimension": 384,
|
||||
# Document chunking
|
||||
"document_chunk_size": 2048,
|
||||
"document_chunk_overlap": 200,
|
||||
@@ -486,15 +500,32 @@ class Settings:
|
||||
qdrant_api_key: str | None = None
|
||||
qdrant_collection: str = "nextcloud_content"
|
||||
|
||||
# Ollama settings (for embeddings)
|
||||
# Ollama settings (embeddings + optional generation)
|
||||
ollama_base_url: str | None = None
|
||||
ollama_embedding_model: str = "nomic-embed-text"
|
||||
ollama_generation_model: str | None = None
|
||||
ollama_verify_ssl: bool = True
|
||||
|
||||
# OpenAI settings (for embeddings)
|
||||
# OpenAI settings (embeddings + optional generation)
|
||||
openai_api_key: str | None = None
|
||||
openai_base_url: str | None = None
|
||||
openai_embedding_model: str = "text-embedding-3-small"
|
||||
openai_generation_model: str | None = None
|
||||
|
||||
# Bedrock (AWS) settings — boto3 also reads these from its credential chain
|
||||
aws_region: str | None = None
|
||||
aws_access_key_id: str | None = None
|
||||
aws_secret_access_key: str | None = None
|
||||
bedrock_embedding_model: str | None = None
|
||||
bedrock_generation_model: str | None = None
|
||||
|
||||
# Mistral settings (embeddings only)
|
||||
mistral_api_key: str | None = None
|
||||
mistral_embedding_model: str = "mistral-embed"
|
||||
mistral_base_url: str | None = None
|
||||
|
||||
# Simple (fallback) provider — dimension when no real provider configured
|
||||
simple_embedding_dimension: int = 384
|
||||
|
||||
# Document chunking settings (for vector embeddings)
|
||||
document_chunk_size: int = 2048 # Characters per chunk
|
||||
@@ -573,23 +604,32 @@ class Settings:
|
||||
Get the active embedding model name based on provider priority.
|
||||
|
||||
Priority order (same as ProviderRegistry):
|
||||
1. OpenAI - if OPENAI_API_KEY is set
|
||||
2. Ollama - if OLLAMA_BASE_URL is set
|
||||
3. Simple - fallback (returns "simple-384")
|
||||
1. Bedrock - if AWS_REGION or BEDROCK_EMBEDDING_MODEL is set
|
||||
2. OpenAI - if OPENAI_API_KEY is set
|
||||
3. Mistral - if MISTRAL_API_KEY is set
|
||||
4. Ollama - if OLLAMA_BASE_URL is set
|
||||
5. Simple - fallback (returns "simple-{dimension}")
|
||||
|
||||
Returns:
|
||||
Active embedding model name
|
||||
"""
|
||||
# Check OpenAI first (higher priority than Ollama in registry)
|
||||
if (
|
||||
self.aws_region
|
||||
or self.bedrock_embedding_model
|
||||
or self.bedrock_generation_model
|
||||
):
|
||||
return self.bedrock_embedding_model or "bedrock-default"
|
||||
|
||||
if self.openai_api_key:
|
||||
return self.openai_embedding_model
|
||||
|
||||
# Check Ollama
|
||||
if self.mistral_api_key:
|
||||
return self.mistral_embedding_model
|
||||
|
||||
if self.ollama_base_url:
|
||||
return self.ollama_embedding_model
|
||||
|
||||
# Fallback to simple provider indicator
|
||||
return "simple-384"
|
||||
return f"simple-{self.simple_embedding_dimension}"
|
||||
|
||||
def get_collection_name(self) -> str:
|
||||
"""
|
||||
@@ -835,11 +875,25 @@ def get_settings() -> Settings:
|
||||
# Ollama settings
|
||||
"ollama_base_url": "OLLAMA_BASE_URL",
|
||||
"ollama_embedding_model": "OLLAMA_EMBEDDING_MODEL",
|
||||
"ollama_generation_model": "OLLAMA_GENERATION_MODEL",
|
||||
"ollama_verify_ssl": "OLLAMA_VERIFY_SSL",
|
||||
# OpenAI settings
|
||||
"openai_api_key": "OPENAI_API_KEY",
|
||||
"openai_base_url": "OPENAI_BASE_URL",
|
||||
"openai_embedding_model": "OPENAI_EMBEDDING_MODEL",
|
||||
"openai_generation_model": "OPENAI_GENERATION_MODEL",
|
||||
# Bedrock (AWS) settings
|
||||
"aws_region": "AWS_REGION",
|
||||
"aws_access_key_id": "AWS_ACCESS_KEY_ID",
|
||||
"aws_secret_access_key": "AWS_SECRET_ACCESS_KEY",
|
||||
"bedrock_embedding_model": "BEDROCK_EMBEDDING_MODEL",
|
||||
"bedrock_generation_model": "BEDROCK_GENERATION_MODEL",
|
||||
# Mistral settings
|
||||
"mistral_api_key": "MISTRAL_API_KEY",
|
||||
"mistral_embedding_model": "MISTRAL_EMBEDDING_MODEL",
|
||||
"mistral_base_url": "MISTRAL_BASE_URL",
|
||||
# Simple provider
|
||||
"simple_embedding_dimension": "SIMPLE_EMBEDDING_DIMENSION",
|
||||
# Document chunking settings
|
||||
"document_chunk_size": "DOCUMENT_CHUNK_SIZE",
|
||||
"document_chunk_overlap": "DOCUMENT_CHUNK_OVERLAP",
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
from .anthropic import AnthropicProvider
|
||||
from .base import Provider
|
||||
from .bedrock import BedrockProvider
|
||||
from .mistral import MistralProvider
|
||||
from .ollama import OllamaProvider
|
||||
from .openai import OpenAIProvider
|
||||
from .registry import get_provider, reset_provider
|
||||
@@ -13,6 +14,7 @@ __all__ = [
|
||||
"OllamaProvider",
|
||||
"OpenAIProvider",
|
||||
"AnthropicProvider",
|
||||
"MistralProvider",
|
||||
"SimpleProvider",
|
||||
"BedrockProvider",
|
||||
"get_provider",
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Shared rate-limit retry helper for provider modules.
|
||||
|
||||
OpenAI and Mistral both retry on 429 with the same exponential-backoff curve;
|
||||
extracting the loop here keeps the two provider modules thin and lets future
|
||||
providers (Bedrock throttling, etc.) reuse the same primitive.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable
|
||||
from functools import wraps
|
||||
from typing import Any, TypeVar
|
||||
|
||||
import anyio
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MAX_RETRIES = 5
|
||||
INITIAL_RETRY_DELAY = 2.0
|
||||
MAX_RETRY_DELAY = 60.0
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
def retry_on_rate_limit(
|
||||
exception_type: type[BaseException],
|
||||
is_rate_limit: Callable[[BaseException], bool] = lambda _exc: True,
|
||||
*,
|
||||
provider_name: str = "provider",
|
||||
) -> Callable[[Callable[..., Awaitable[T]]], Callable[..., Awaitable[T]]]:
|
||||
"""Build a decorator that retries on rate-limit exceptions.
|
||||
|
||||
Args:
|
||||
exception_type: Catch this exception class (e.g. ``openai.RateLimitError``,
|
||||
``mistralai.client.errors.SDKError``).
|
||||
is_rate_limit: Predicate that decides whether a caught exception is
|
||||
actually a rate-limit (vs. some other error of the same class).
|
||||
Defaults to "always True" — appropriate when ``exception_type`` is
|
||||
already a rate-limit-specific class.
|
||||
provider_name: Used in log messages so operators can tell which
|
||||
provider exhausted retries.
|
||||
"""
|
||||
|
||||
def decorator(func: Callable[..., Awaitable[T]]) -> Callable[..., Awaitable[T]]:
|
||||
@wraps(func)
|
||||
async def wrapper(*args: Any, **kwargs: Any) -> T:
|
||||
retry_delay = INITIAL_RETRY_DELAY
|
||||
last_error: BaseException | None = None
|
||||
|
||||
for attempt in range(1, MAX_RETRIES + 1):
|
||||
try:
|
||||
return await func(*args, **kwargs)
|
||||
except exception_type as e:
|
||||
if not is_rate_limit(e):
|
||||
raise
|
||||
last_error = e
|
||||
if attempt < MAX_RETRIES:
|
||||
logger.warning(
|
||||
"%s rate limit hit (attempt %d/%d), retrying in %.1fs...",
|
||||
provider_name,
|
||||
attempt,
|
||||
MAX_RETRIES,
|
||||
retry_delay,
|
||||
)
|
||||
await anyio.sleep(retry_delay)
|
||||
retry_delay = min(retry_delay * 2, MAX_RETRY_DELAY)
|
||||
|
||||
logger.error(
|
||||
"%s rate limit exceeded after %d attempts", provider_name, MAX_RETRIES
|
||||
)
|
||||
if last_error is None: # pragma: no cover — loop above always sets this
|
||||
raise RuntimeError("retry loop exited without capturing an error")
|
||||
raise last_error
|
||||
|
||||
return wrapper
|
||||
|
||||
return decorator
|
||||
@@ -0,0 +1,199 @@
|
||||
"""Mistral provider for embeddings.
|
||||
|
||||
Currently supports embeddings only (``mistral-embed``, 1024-dim). Generation
|
||||
can be added later if needed; see ADR-015.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
# mistralai 2.x ships no top-level __init__.py, so `from mistralai import …`
|
||||
# raises ImportError. The canonical public paths are `mistralai.client` (which
|
||||
# re-exports the SDK class via `client/__init__.py`) and `mistralai.client.errors`
|
||||
# (which lazy-loads SDKError). There is no `mistralai.models` subpackage either.
|
||||
from mistralai.client import Mistral
|
||||
from mistralai.client.errors import SDKError
|
||||
|
||||
from ._retry import retry_on_rate_limit
|
||||
from .base import Provider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Well-known Mistral embedding model dimensions
|
||||
MISTRAL_EMBEDDING_DIMENSIONS: dict[str, int] = {
|
||||
"mistral-embed": 1024,
|
||||
}
|
||||
|
||||
# Conservative chunk size for batch embeddings. Mistral allows large batches,
|
||||
# but we keep this in line with sibling providers (OpenAI=100, Ollama=32).
|
||||
BATCH_SIZE = 64
|
||||
|
||||
_NO_EMBEDDING_MODEL_MSG = "Embedding not supported - no embedding_model configured"
|
||||
|
||||
|
||||
def _is_rate_limit(exc: BaseException) -> bool:
|
||||
"""True only for HTTP 429 SDKErrors."""
|
||||
return getattr(exc, "status_code", None) == 429
|
||||
|
||||
|
||||
_retry_429 = retry_on_rate_limit(
|
||||
SDKError, is_rate_limit=_is_rate_limit, provider_name="Mistral"
|
||||
)
|
||||
|
||||
|
||||
class MistralProvider(Provider):
|
||||
"""
|
||||
Mistral provider — embeddings only.
|
||||
|
||||
Uses the official ``mistralai`` SDK. Lazy dimension detection mirrors the
|
||||
OpenAI provider: known models populate the cached dimension at construction
|
||||
time; unknown models get their dimension detected on the first ``embed()``
|
||||
call.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
api_key: str,
|
||||
embedding_model: str | None = "mistral-embed",
|
||||
base_url: str | None = None,
|
||||
):
|
||||
"""
|
||||
Initialize the Mistral provider.
|
||||
|
||||
Args:
|
||||
api_key: Mistral API key.
|
||||
embedding_model: Embedding model ID (default: ``mistral-embed``).
|
||||
Pass ``None`` to disable embeddings (the provider will then
|
||||
support no capabilities, which is mostly useful for tests).
|
||||
base_url: Optional base URL override (e.g. proxies, on-prem).
|
||||
"""
|
||||
self.embedding_model = embedding_model
|
||||
self._dimension: int | None = None
|
||||
|
||||
self.client = Mistral(api_key=api_key, server_url=base_url)
|
||||
|
||||
if embedding_model and embedding_model in MISTRAL_EMBEDDING_DIMENSIONS:
|
||||
self._dimension = MISTRAL_EMBEDDING_DIMENSIONS[embedding_model]
|
||||
|
||||
logger.info(
|
||||
"Initialized Mistral provider: base_url=%s, embedding_model=%s, "
|
||||
"dimension=%s",
|
||||
base_url or "default",
|
||||
embedding_model,
|
||||
self._dimension,
|
||||
)
|
||||
|
||||
@property
|
||||
def supports_embeddings(self) -> bool:
|
||||
return self.embedding_model is not None
|
||||
|
||||
@property
|
||||
def supports_generation(self) -> bool:
|
||||
return False
|
||||
|
||||
@_retry_429
|
||||
async def embed(self, text: str) -> list[float]:
|
||||
"""Generate an embedding for a single text."""
|
||||
if not self.supports_embeddings:
|
||||
raise NotImplementedError(_NO_EMBEDDING_MODEL_MSG)
|
||||
|
||||
assert self.embedding_model is not None
|
||||
response = await self.client.embeddings.create_async(
|
||||
model=self.embedding_model,
|
||||
inputs=[text],
|
||||
)
|
||||
|
||||
if not response.data or response.data[0].embedding is None:
|
||||
raise RuntimeError(
|
||||
f"Mistral embeddings API returned no embedding for model "
|
||||
f"{self.embedding_model}"
|
||||
)
|
||||
|
||||
embedding = response.data[0].embedding
|
||||
|
||||
if self._dimension is None:
|
||||
self._dimension = len(embedding)
|
||||
logger.info(
|
||||
"Detected embedding dimension: %d for model %s",
|
||||
self._dimension,
|
||||
self.embedding_model,
|
||||
)
|
||||
|
||||
return embedding
|
||||
|
||||
async def embed_batch(self, texts: list[str]) -> list[list[float]]:
|
||||
"""Generate embeddings for multiple texts, chunking by ``BATCH_SIZE``."""
|
||||
if not self.supports_embeddings:
|
||||
raise NotImplementedError(_NO_EMBEDDING_MODEL_MSG)
|
||||
|
||||
if not texts:
|
||||
return []
|
||||
|
||||
all_embeddings: list[list[float]] = []
|
||||
for i in range(0, len(texts), BATCH_SIZE):
|
||||
batch = texts[i : i + BATCH_SIZE]
|
||||
batch_embeddings = await self._embed_batch_request(batch)
|
||||
all_embeddings.extend(batch_embeddings)
|
||||
|
||||
if self._dimension is None and batch_embeddings:
|
||||
self._dimension = len(batch_embeddings[0])
|
||||
logger.info(
|
||||
"Detected embedding dimension: %d for model %s",
|
||||
self._dimension,
|
||||
self.embedding_model,
|
||||
)
|
||||
|
||||
return all_embeddings
|
||||
|
||||
@_retry_429
|
||||
async def _embed_batch_request(self, batch: list[str]) -> list[list[float]]:
|
||||
"""Single batch request with rate-limit retry."""
|
||||
assert self.embedding_model is not None
|
||||
response = await self.client.embeddings.create_async(
|
||||
model=self.embedding_model,
|
||||
inputs=batch,
|
||||
)
|
||||
|
||||
# Defensive: response.data items have Optional fields. Sort by index
|
||||
# (default 0 if missing) and reject None embeddings explicitly.
|
||||
sorted_data = sorted(response.data or [], key=lambda x: x.index or 0)
|
||||
result: list[list[float]] = []
|
||||
for item in sorted_data:
|
||||
if item.embedding is None:
|
||||
raise RuntimeError(
|
||||
f"Mistral embeddings API returned a null embedding for "
|
||||
f"model {self.embedding_model}"
|
||||
)
|
||||
result.append(item.embedding)
|
||||
|
||||
if len(result) != len(batch):
|
||||
raise RuntimeError(
|
||||
f"Mistral embeddings API returned {len(result)} embeddings "
|
||||
f"for {len(batch)} inputs"
|
||||
)
|
||||
return result
|
||||
|
||||
def get_dimension(self) -> int:
|
||||
if not self.supports_embeddings:
|
||||
raise NotImplementedError(_NO_EMBEDDING_MODEL_MSG)
|
||||
|
||||
if self._dimension is None:
|
||||
raise RuntimeError(
|
||||
f"Embedding dimension not detected yet for model "
|
||||
f"{self.embedding_model}. Call embed() first or use a known "
|
||||
"model."
|
||||
)
|
||||
return self._dimension
|
||||
|
||||
async def generate(self, prompt: str, max_tokens: int = 500) -> str:
|
||||
raise NotImplementedError(
|
||||
"MistralProvider does not support generation. "
|
||||
"Use OpenAI, Anthropic, or Bedrock for text generation."
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
# The mistralai 2.x client (Speakeasy-generated) does not expose a
|
||||
# public close()/aclose() — only the async-context-manager protocol
|
||||
# (__aenter__/__aexit__). Calling __aexit__ directly is internal API
|
||||
# and brittle across SDK patch versions; the underlying httpx client
|
||||
# is closed during garbage collection, so we leave this as a no-op.
|
||||
return None
|
||||
@@ -7,46 +7,17 @@ Supports:
|
||||
"""
|
||||
|
||||
import logging
|
||||
from functools import wraps
|
||||
|
||||
import anyio
|
||||
from openai import AsyncOpenAI, RateLimitError
|
||||
|
||||
from ._retry import retry_on_rate_limit
|
||||
from .base import Provider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Rate limit retry configuration
|
||||
MAX_RETRIES = 5
|
||||
INITIAL_RETRY_DELAY = 2.0 # seconds
|
||||
MAX_RETRY_DELAY = 60.0 # seconds
|
||||
|
||||
|
||||
def retry_on_rate_limit(func):
|
||||
"""Decorator to retry on OpenAI rate limit errors with exponential backoff."""
|
||||
|
||||
@wraps(func)
|
||||
async def wrapper(*args, **kwargs):
|
||||
retry_delay = INITIAL_RETRY_DELAY
|
||||
last_error: Exception | None = None
|
||||
|
||||
for attempt in range(1, MAX_RETRIES + 1):
|
||||
try:
|
||||
return await func(*args, **kwargs)
|
||||
except RateLimitError as e:
|
||||
last_error = e
|
||||
if attempt < MAX_RETRIES:
|
||||
logger.warning(
|
||||
f"Rate limit hit (attempt {attempt}/{MAX_RETRIES}), "
|
||||
f"retrying in {retry_delay:.1f}s..."
|
||||
)
|
||||
await anyio.sleep(retry_delay)
|
||||
retry_delay = min(retry_delay * 2, MAX_RETRY_DELAY)
|
||||
|
||||
logger.error(f"Rate limit exceeded after {MAX_RETRIES} attempts")
|
||||
raise last_error # type: ignore[misc]
|
||||
|
||||
return wrapper
|
||||
# OpenAI's RateLimitError is itself a 429-specific class, so the default
|
||||
# is_rate_limit predicate ("always True") matches the previous behavior.
|
||||
_retry_429 = retry_on_rate_limit(RateLimitError, provider_name="OpenAI")
|
||||
|
||||
|
||||
# Well-known embedding dimensions for OpenAI models
|
||||
@@ -106,9 +77,12 @@ class OpenAIProvider(Provider):
|
||||
self._dimension = OPENAI_EMBEDDING_DIMENSIONS[embedding_model]
|
||||
|
||||
logger.info(
|
||||
f"Initialized OpenAI provider: base_url={base_url or 'default'} "
|
||||
f"(embedding_model={embedding_model}, generation_model={generation_model}, "
|
||||
f"dimension={self._dimension})"
|
||||
"Initialized OpenAI provider: base_url=%s "
|
||||
"(embedding_model=%s, generation_model=%s, dimension=%s)",
|
||||
base_url or "default",
|
||||
embedding_model,
|
||||
generation_model,
|
||||
self._dimension,
|
||||
)
|
||||
|
||||
@property
|
||||
@@ -121,7 +95,7 @@ class OpenAIProvider(Provider):
|
||||
"""Whether this provider supports text generation."""
|
||||
return self.generation_model is not None
|
||||
|
||||
@retry_on_rate_limit
|
||||
@_retry_429
|
||||
async def embed(self, text: str) -> list[float]:
|
||||
"""
|
||||
Generate embedding vector for text.
|
||||
@@ -152,8 +126,9 @@ class OpenAIProvider(Provider):
|
||||
if self._dimension is None:
|
||||
self._dimension = len(embedding)
|
||||
logger.info(
|
||||
f"Detected embedding dimension: {self._dimension} "
|
||||
f"for model {self.embedding_model}"
|
||||
"Detected embedding dimension: %d for model %s",
|
||||
self._dimension,
|
||||
self.embedding_model,
|
||||
)
|
||||
|
||||
return embedding
|
||||
@@ -196,13 +171,14 @@ class OpenAIProvider(Provider):
|
||||
if self._dimension is None and batch_embeddings:
|
||||
self._dimension = len(batch_embeddings[0])
|
||||
logger.info(
|
||||
f"Detected embedding dimension: {self._dimension} "
|
||||
f"for model {self.embedding_model}"
|
||||
"Detected embedding dimension: %d for model %s",
|
||||
self._dimension,
|
||||
self.embedding_model,
|
||||
)
|
||||
|
||||
return all_embeddings
|
||||
|
||||
@retry_on_rate_limit
|
||||
@_retry_429
|
||||
async def _embed_batch_request(self, batch: list[str]) -> list[list[float]]:
|
||||
"""Make a single batch embedding request with retry logic."""
|
||||
assert self.embedding_model is not None # Type narrowing
|
||||
@@ -237,7 +213,7 @@ class OpenAIProvider(Provider):
|
||||
)
|
||||
return self._dimension
|
||||
|
||||
@retry_on_rate_limit
|
||||
@_retry_429
|
||||
async def generate(self, prompt: str, max_tokens: int = 500) -> str:
|
||||
"""
|
||||
Generate text from a prompt.
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
"""Provider registry and factory for auto-detection and instantiation."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
from ..config import get_settings
|
||||
from .base import Provider
|
||||
from .bedrock import BedrockProvider
|
||||
from .mistral import MistralProvider
|
||||
from .ollama import OllamaProvider
|
||||
from .openai import OpenAIProvider
|
||||
from .simple import SimpleProvider
|
||||
@@ -16,117 +17,112 @@ class ProviderRegistry:
|
||||
"""
|
||||
Registry for provider auto-detection and instantiation.
|
||||
|
||||
Checks environment variables in priority order and creates appropriate provider:
|
||||
1. Bedrock (AWS_REGION + BEDROCK_*_MODEL)
|
||||
2. OpenAI (OPENAI_API_KEY)
|
||||
3. Ollama (OLLAMA_BASE_URL)
|
||||
4. Simple (fallback for testing/development)
|
||||
Reads configuration via dynaconf-backed Settings (see ``config.py``).
|
||||
Checks provider settings in priority order and creates the appropriate
|
||||
provider:
|
||||
|
||||
1. Bedrock (``AWS_REGION`` or ``BEDROCK_*_MODEL``)
|
||||
2. OpenAI (``OPENAI_API_KEY``)
|
||||
3. Mistral (``MISTRAL_API_KEY``)
|
||||
4. Ollama (``OLLAMA_BASE_URL``)
|
||||
5. Simple (fallback for testing/development)
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def create_provider() -> Provider:
|
||||
"""
|
||||
Auto-detect and create provider based on environment variables.
|
||||
Auto-detect and create provider based on configured settings.
|
||||
|
||||
Settings are sourced via :func:`nextcloud_mcp_server.config.get_settings`,
|
||||
which reads from settings files and environment variables (env vars
|
||||
always win, see ADR-024/025).
|
||||
|
||||
Priority order:
|
||||
1. Bedrock - if AWS_REGION or BEDROCK_EMBEDDING_MODEL is set
|
||||
2. OpenAI - if OPENAI_API_KEY is set
|
||||
3. Ollama - if OLLAMA_BASE_URL is set
|
||||
4. Simple - fallback for testing/development
|
||||
|
||||
1. Bedrock - if ``aws_region`` or ``bedrock_embedding_model`` is set
|
||||
2. OpenAI - if ``openai_api_key`` is set
|
||||
3. Mistral - if ``mistral_api_key`` is set
|
||||
4. Ollama - if ``ollama_base_url`` is set
|
||||
5. Simple - fallback for testing/development
|
||||
|
||||
Returns:
|
||||
Provider instance
|
||||
|
||||
Environment Variables:
|
||||
Bedrock:
|
||||
- AWS_REGION: AWS region (e.g., "us-east-1")
|
||||
- AWS_ACCESS_KEY_ID: AWS access key (optional, uses credential chain)
|
||||
- AWS_SECRET_ACCESS_KEY: AWS secret key (optional)
|
||||
- BEDROCK_EMBEDDING_MODEL: Model ID for embeddings (e.g., "amazon.titan-embed-text-v2:0")
|
||||
- BEDROCK_GENERATION_MODEL: Model ID for text generation (e.g., "anthropic.claude-3-sonnet-20240229-v1:0")
|
||||
|
||||
OpenAI:
|
||||
- OPENAI_API_KEY: OpenAI API key (or GITHUB_TOKEN for GitHub Models)
|
||||
- OPENAI_BASE_URL: Base URL override (e.g., "https://models.github.ai/inference")
|
||||
- OPENAI_EMBEDDING_MODEL: Model for embeddings (default: "text-embedding-3-small")
|
||||
- OPENAI_GENERATION_MODEL: Model for text generation (e.g., "gpt-4o-mini")
|
||||
|
||||
Ollama:
|
||||
- OLLAMA_BASE_URL: Ollama API base URL (e.g., "http://localhost:11434")
|
||||
- OLLAMA_EMBEDDING_MODEL: Model for embeddings (default: "nomic-embed-text")
|
||||
- OLLAMA_GENERATION_MODEL: Model for text generation (e.g., "llama3.2:1b")
|
||||
- OLLAMA_VERIFY_SSL: Verify SSL certificates (default: "true")
|
||||
|
||||
Simple (no configuration needed, fallback):
|
||||
- SIMPLE_EMBEDDING_DIMENSION: Embedding dimension (default: 384)
|
||||
"""
|
||||
# 1. Check for Bedrock
|
||||
aws_region = os.getenv("AWS_REGION")
|
||||
bedrock_embedding_model = os.getenv("BEDROCK_EMBEDDING_MODEL")
|
||||
bedrock_generation_model = os.getenv("BEDROCK_GENERATION_MODEL")
|
||||
settings = get_settings()
|
||||
|
||||
if aws_region or bedrock_embedding_model or bedrock_generation_model:
|
||||
# 1. Bedrock
|
||||
if (
|
||||
settings.aws_region
|
||||
or settings.bedrock_embedding_model
|
||||
or settings.bedrock_generation_model
|
||||
):
|
||||
logger.info(
|
||||
f"Using Bedrock provider: region={aws_region}, "
|
||||
f"embedding_model={bedrock_embedding_model}, "
|
||||
f"generation_model={bedrock_generation_model}"
|
||||
"Using Bedrock provider: region=%s, embedding_model=%s, "
|
||||
"generation_model=%s",
|
||||
settings.aws_region,
|
||||
settings.bedrock_embedding_model,
|
||||
settings.bedrock_generation_model,
|
||||
)
|
||||
return BedrockProvider(
|
||||
region_name=aws_region,
|
||||
embedding_model=bedrock_embedding_model,
|
||||
generation_model=bedrock_generation_model,
|
||||
aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"),
|
||||
aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"),
|
||||
region_name=settings.aws_region,
|
||||
embedding_model=settings.bedrock_embedding_model,
|
||||
generation_model=settings.bedrock_generation_model,
|
||||
aws_access_key_id=settings.aws_access_key_id,
|
||||
aws_secret_access_key=settings.aws_secret_access_key,
|
||||
)
|
||||
|
||||
# 2. Check for OpenAI
|
||||
openai_api_key = os.getenv("OPENAI_API_KEY")
|
||||
if openai_api_key:
|
||||
base_url = os.getenv("OPENAI_BASE_URL")
|
||||
embedding_model = os.getenv(
|
||||
"OPENAI_EMBEDDING_MODEL", "text-embedding-3-small"
|
||||
)
|
||||
generation_model = os.getenv("OPENAI_GENERATION_MODEL")
|
||||
|
||||
# 2. OpenAI
|
||||
if settings.openai_api_key:
|
||||
logger.info(
|
||||
f"Using OpenAI provider: base_url={base_url or 'default'}, "
|
||||
f"embedding_model={embedding_model}, "
|
||||
f"generation_model={generation_model}"
|
||||
"Using OpenAI provider: base_url=%s, embedding_model=%s, "
|
||||
"generation_model=%s",
|
||||
settings.openai_base_url or "default",
|
||||
settings.openai_embedding_model,
|
||||
settings.openai_generation_model,
|
||||
)
|
||||
return OpenAIProvider(
|
||||
api_key=openai_api_key,
|
||||
base_url=base_url,
|
||||
embedding_model=embedding_model,
|
||||
generation_model=generation_model,
|
||||
api_key=settings.openai_api_key,
|
||||
base_url=settings.openai_base_url,
|
||||
embedding_model=settings.openai_embedding_model,
|
||||
generation_model=settings.openai_generation_model,
|
||||
)
|
||||
|
||||
# 3. Check for Ollama (local LLM)
|
||||
ollama_url = os.getenv("OLLAMA_BASE_URL")
|
||||
if ollama_url:
|
||||
embedding_model = os.getenv("OLLAMA_EMBEDDING_MODEL", "nomic-embed-text")
|
||||
generation_model = os.getenv("OLLAMA_GENERATION_MODEL")
|
||||
verify_ssl = os.getenv("OLLAMA_VERIFY_SSL", "true").lower() == "true"
|
||||
|
||||
# 3. Mistral
|
||||
if settings.mistral_api_key:
|
||||
logger.info(
|
||||
f"Using Ollama provider: {ollama_url}, "
|
||||
f"embedding_model={embedding_model}, "
|
||||
f"generation_model={generation_model}"
|
||||
"Using Mistral provider: base_url=%s, embedding_model=%s",
|
||||
settings.mistral_base_url or "default",
|
||||
settings.mistral_embedding_model,
|
||||
)
|
||||
return MistralProvider(
|
||||
api_key=settings.mistral_api_key,
|
||||
base_url=settings.mistral_base_url,
|
||||
embedding_model=settings.mistral_embedding_model,
|
||||
)
|
||||
|
||||
# 4. Ollama
|
||||
if settings.ollama_base_url:
|
||||
logger.info(
|
||||
"Using Ollama provider: %s, embedding_model=%s, generation_model=%s",
|
||||
settings.ollama_base_url,
|
||||
settings.ollama_embedding_model,
|
||||
settings.ollama_generation_model,
|
||||
)
|
||||
return OllamaProvider(
|
||||
base_url=ollama_url,
|
||||
embedding_model=embedding_model,
|
||||
generation_model=generation_model,
|
||||
verify_ssl=verify_ssl,
|
||||
base_url=settings.ollama_base_url,
|
||||
embedding_model=settings.ollama_embedding_model,
|
||||
generation_model=settings.ollama_generation_model,
|
||||
verify_ssl=settings.ollama_verify_ssl,
|
||||
)
|
||||
|
||||
# 4. Fallback to Simple provider for development/testing
|
||||
dimension = int(os.getenv("SIMPLE_EMBEDDING_DIMENSION", "384"))
|
||||
# 5. Simple (fallback)
|
||||
logger.warning(
|
||||
"No provider configured (AWS_REGION, OPENAI_API_KEY, OLLAMA_BASE_URL not set). "
|
||||
"No provider configured (AWS_REGION, OPENAI_API_KEY, "
|
||||
"MISTRAL_API_KEY, OLLAMA_BASE_URL not set). "
|
||||
"Using SimpleProvider for testing/development. "
|
||||
"For production, configure Bedrock, OpenAI, or Ollama."
|
||||
"For production, configure Bedrock, OpenAI, Mistral, or Ollama."
|
||||
)
|
||||
return SimpleProvider(dimension=dimension)
|
||||
return SimpleProvider(dimension=settings.simple_embedding_dimension)
|
||||
|
||||
|
||||
# Singleton instance
|
||||
|
||||
@@ -691,6 +691,122 @@ class PDFHighlighter:
|
||||
f"Failed to delete temp directory {temp_pdf_path.parent}: {e}"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def compute_chunk_bboxes_batch(
|
||||
pdf_bytes: bytes,
|
||||
chunks: list[tuple[int, int, int, int | None, str]],
|
||||
page_boundaries: list[dict],
|
||||
full_text: str,
|
||||
) -> dict[int, tuple[list[tuple[float, float, float, float]], int]]:
|
||||
"""Compute normalized bounding boxes for chunks without rendering.
|
||||
|
||||
Lightweight alternative to highlight_chunks_batch — opens the PDF,
|
||||
locates each chunk on its assigned page using the same text-search
|
||||
path as the highlighter (`_find_chunk_bbox`), and returns
|
||||
page-normalized rectangles. Skips the get_pixmap + PIL pipeline
|
||||
entirely, so no PNG bytes are produced.
|
||||
|
||||
Args:
|
||||
pdf_bytes: PDF file bytes.
|
||||
chunks: List of (chunk_index, start_offset, end_offset,
|
||||
stored_page_number, chunk_text). chunk_index is the dict key.
|
||||
page_boundaries: Pre-computed page boundaries from the document
|
||||
processor; each entry is {"page", "start_offset", "end_offset"}.
|
||||
full_text: Full document text (for cross-page chunk handling).
|
||||
|
||||
Returns:
|
||||
dict mapping chunk_index to (normalized_bboxes, page_number).
|
||||
Each bbox is (x0, y0, x1, y1) in [0, 1] relative to page width
|
||||
and height, top-left origin. Chunks whose bbox cannot be located
|
||||
are omitted from the result.
|
||||
"""
|
||||
results: dict[int, tuple[list[tuple[float, float, float, float]], int]] = {}
|
||||
|
||||
if not chunks:
|
||||
return results
|
||||
|
||||
temp_pdf_path = None
|
||||
doc = None
|
||||
try:
|
||||
temp_dir = Path(tempfile.mkdtemp(prefix="pdf_bbox_batch_"))
|
||||
temp_pdf_path = temp_dir / "pdf.pdf"
|
||||
temp_pdf_path.write_bytes(pdf_bytes)
|
||||
|
||||
doc = pymupdf.open(temp_pdf_path)
|
||||
|
||||
for (
|
||||
chunk_index,
|
||||
start_offset,
|
||||
end_offset,
|
||||
_,
|
||||
_,
|
||||
) in chunks:
|
||||
chunk_page_info = PDFHighlighter.find_chunk_page(
|
||||
start_offset, end_offset, page_boundaries
|
||||
)
|
||||
if not chunk_page_info:
|
||||
logger.debug("Chunk %s: not found on any page", chunk_index)
|
||||
continue
|
||||
|
||||
page_num = chunk_page_info["page_num"]
|
||||
page_boundary = next(
|
||||
(b for b in page_boundaries if b["page"] == page_num), None
|
||||
)
|
||||
if page_boundary is None:
|
||||
logger.debug(
|
||||
"Chunk %s: page %s not found in boundaries",
|
||||
chunk_index,
|
||||
page_num,
|
||||
)
|
||||
continue
|
||||
page_text_length = (
|
||||
page_boundary["end_offset"] - page_boundary["start_offset"]
|
||||
)
|
||||
|
||||
# Page-relative slice (handles chunks that span page boundaries)
|
||||
chunk_start_on_page = max(start_offset, page_boundary["start_offset"])
|
||||
chunk_end_on_page = min(end_offset, page_boundary["end_offset"])
|
||||
page_relative_text = full_text[chunk_start_on_page:chunk_end_on_page]
|
||||
|
||||
page = doc[page_num - 1]
|
||||
bbox = PDFHighlighter._find_chunk_bbox(
|
||||
page,
|
||||
page_relative_text,
|
||||
chunk_page_info["page_relative_start"],
|
||||
chunk_page_info["page_relative_end"],
|
||||
page_text_length,
|
||||
)
|
||||
|
||||
if bbox is None:
|
||||
continue
|
||||
|
||||
page_rect = page.rect
|
||||
w = page_rect.width or 1.0
|
||||
h = page_rect.height or 1.0
|
||||
normalized = (
|
||||
bbox[0] / w,
|
||||
bbox[1] / h,
|
||||
bbox[2] / w,
|
||||
bbox[3] / h,
|
||||
)
|
||||
results[chunk_index] = ([normalized], page_num)
|
||||
|
||||
logger.info(f"Computed bboxes for {len(results)}/{len(chunks)} chunks")
|
||||
return results
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error computing chunk bboxes: {e}", exc_info=True)
|
||||
return results
|
||||
|
||||
finally:
|
||||
if doc is not None:
|
||||
doc.close()
|
||||
if temp_pdf_path and temp_pdf_path.parent.exists():
|
||||
try:
|
||||
shutil.rmtree(temp_pdf_path.parent)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to clean up temp dir: {e}")
|
||||
|
||||
@staticmethod
|
||||
def highlight_chunks_batch(
|
||||
pdf_bytes: bytes,
|
||||
|
||||
@@ -1,16 +1,17 @@
|
||||
"""Vector database and background sync package."""
|
||||
"""Vector database and background sync package.
|
||||
|
||||
`processor` and `scanner` are intentionally NOT re-exported from this
|
||||
package init: they transitively import `server.semantic` ->
|
||||
`search.bm25_hybrid`, which forms an import cycle with
|
||||
`search.algorithms` -> `vector.placeholder` -> `vector/__init__`.
|
||||
Consumers that need those symbols import them from their submodules
|
||||
directly (e.g. `from nextcloud_mcp_server.vector.processor import ...`).
|
||||
"""
|
||||
|
||||
from .document_chunker import DocumentChunker
|
||||
from .processor import process_document, processor_task
|
||||
from .qdrant_client import get_qdrant_client
|
||||
from .scanner import DocumentTask, scan_user_documents, scanner_task
|
||||
|
||||
__all__ = [
|
||||
"get_qdrant_client",
|
||||
"DocumentChunker",
|
||||
"scanner_task",
|
||||
"scan_user_documents",
|
||||
"DocumentTask",
|
||||
"processor_task",
|
||||
"process_document",
|
||||
]
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
Processes documents from stream: fetches content, generates embeddings, stores in Qdrant.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
@@ -538,7 +537,11 @@ async def _index_document(
|
||||
# Initialize results containers
|
||||
dense_embeddings: list = []
|
||||
sparse_embeddings: list = []
|
||||
chunk_images: dict[int, dict] = {}
|
||||
# chunk_index -> list[(x0, y0, x1, y1)] of normalized rectangles
|
||||
# in [0, 1] relative to page width/height. The page is taken from
|
||||
# `chunk.page_number` (offset-based) and stored as `page_number`
|
||||
# in the Qdrant payload, so we don't carry an `actual_page_num` here.
|
||||
chunk_bboxes: dict[int, list[tuple[float, float, float, float]]] = {}
|
||||
|
||||
# Determine if we need PDF highlighting
|
||||
is_pdf = doc_task.doc_type == "file" and content_type == "application/pdf"
|
||||
@@ -570,8 +573,8 @@ async def _index_document(
|
||||
sparse_embeddings = await bm25_service.encode_batch(chunk_texts)
|
||||
|
||||
async def generate_highlights():
|
||||
"""Generate highlighted page images for PDF chunks (CPU-bound)."""
|
||||
nonlocal chunk_images
|
||||
"""Compute chunk bounding boxes for PDF chunks (CPU-bound, no rendering)."""
|
||||
nonlocal chunk_bboxes
|
||||
if not is_pdf:
|
||||
return
|
||||
|
||||
@@ -579,64 +582,42 @@ async def _index_document(
|
||||
assert content_bytes is not None
|
||||
|
||||
with trace_operation(
|
||||
"vector_sync.generate_highlights",
|
||||
"vector_sync.compute_chunk_bboxes",
|
||||
attributes={
|
||||
"vector_sync.chunk_count": len(chunks),
|
||||
"vector_sync.pdf_size": len(content_bytes),
|
||||
},
|
||||
):
|
||||
# Build chunk data for batch processing
|
||||
# Format: (chunk_index, start_offset, end_offset, page_number, chunk_text)
|
||||
chunk_data: list[tuple[int, int, int, int | None, str]] = [
|
||||
(i, chunk.start_offset, chunk.end_offset, chunk.page_number, chunk.text)
|
||||
for i, chunk in enumerate(chunks)
|
||||
if chunk.page_number is not None
|
||||
]
|
||||
|
||||
# Get pre-computed page boundaries from document processor
|
||||
page_boundaries = file_metadata.get("page_boundaries")
|
||||
if not page_boundaries:
|
||||
logger.warning("No page boundaries available, skipping highlighting")
|
||||
logger.warning(
|
||||
"No page boundaries available, skipping bbox computation"
|
||||
)
|
||||
return
|
||||
|
||||
# Type narrowing: page_boundaries is guaranteed to be list[dict] here
|
||||
page_boundaries_list = cast(list[dict[str, Any]], page_boundaries)
|
||||
|
||||
logger.info(
|
||||
f"Batch generating highlighted page images for {len(chunk_data)} PDF chunks"
|
||||
)
|
||||
logger.info(f"Computing chunk bboxes for {len(chunk_data)} PDF chunks")
|
||||
|
||||
# Run CPU-bound highlighting in thread pool
|
||||
# Pass pre-computed page boundaries and full text to avoid re-processing the PDF
|
||||
batch_results = await anyio.to_thread.run_sync( # type: ignore[attr-defined]
|
||||
lambda: PDFHighlighter.highlight_chunks_batch(
|
||||
lambda: PDFHighlighter.compute_chunk_bboxes_batch(
|
||||
pdf_bytes=content_bytes,
|
||||
chunks=chunk_data,
|
||||
page_boundaries=page_boundaries_list,
|
||||
full_text=content,
|
||||
color="yellow",
|
||||
zoom=2.0,
|
||||
)
|
||||
)
|
||||
|
||||
# Convert results to storage format
|
||||
for chunk_index, (
|
||||
png_bytes,
|
||||
actual_page_num,
|
||||
highlight_count,
|
||||
) in batch_results.items():
|
||||
image_base64 = base64.b64encode(png_bytes).decode("utf-8")
|
||||
chunk_images[chunk_index] = {
|
||||
"image": image_base64,
|
||||
"page": actual_page_num,
|
||||
"highlights": highlight_count,
|
||||
"size": len(png_bytes),
|
||||
}
|
||||
for chunk_index, (bboxes, _) in batch_results.items():
|
||||
chunk_bboxes[chunk_index] = bboxes
|
||||
|
||||
logger.info(
|
||||
f"Generated {len(chunk_images)}/{len(chunks)} highlighted page images "
|
||||
f"(avg {sum(img['size'] for img in chunk_images.values()) // max(len(chunk_images), 1):,} bytes)"
|
||||
)
|
||||
logger.info(f"Computed bboxes for {len(chunk_bboxes)}/{len(chunks)} chunks")
|
||||
|
||||
# Run all embedding/highlighting operations in parallel
|
||||
# - Dense embeddings: I/O bound (API call)
|
||||
@@ -752,16 +733,11 @@ async def _index_document(
|
||||
if doc_task.doc_type == "deck_card"
|
||||
else {}
|
||||
),
|
||||
# Highlighted page image (PDF only)
|
||||
**(
|
||||
{
|
||||
"highlighted_page_image": chunk_images[i]["image"],
|
||||
"highlighted_page_number": chunk_images[i]["page"],
|
||||
"highlight_count": chunk_images[i]["highlights"],
|
||||
}
|
||||
if i in chunk_images
|
||||
else {}
|
||||
),
|
||||
# Chunk bbox (PDF only) — normalized rectangles in [0,1]
|
||||
# relative to page width/height. Replaces the legacy
|
||||
# `highlighted_page_image` (Deck #76). The page number
|
||||
# comes from `page_number` (set above for PDF chunks).
|
||||
**({"chunk_bbox": chunk_bboxes[i]} if i in chunk_bboxes else {}),
|
||||
},
|
||||
)
|
||||
)
|
||||
@@ -780,15 +756,16 @@ async def _index_document(
|
||||
f"Failed to delete placeholder for {doc_task.doc_type}_{doc_task.doc_id}: {e}"
|
||||
)
|
||||
|
||||
# Upsert to Qdrant in batches to avoid timeout with large payloads
|
||||
# Each batch is limited to avoid WriteTimeout when sending large image payloads
|
||||
BATCH_SIZE = 10 # ~2MB per batch with images
|
||||
# Upsert to Qdrant in batches. Now that we no longer embed PNG payloads,
|
||||
# per-point payloads are small (chunk text + small metadata), so we can
|
||||
# safely use a larger batch size.
|
||||
BATCH_SIZE = 100
|
||||
with trace_operation(
|
||||
"vector_sync.qdrant_upsert",
|
||||
attributes={
|
||||
"vector_sync.point_count": len(points),
|
||||
"vector_sync.collection": settings.get_collection_name(),
|
||||
"vector_sync.images_count": len(chunk_images),
|
||||
"vector_sync.bboxes_count": len(chunk_bboxes),
|
||||
"vector_sync.batch_size": BATCH_SIZE,
|
||||
},
|
||||
):
|
||||
|
||||
+2
-1
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "nextcloud-mcp-server"
|
||||
version = "0.81.0"
|
||||
version = "0.83.0"
|
||||
description = "Model Context Protocol (MCP) server for Nextcloud integration - enables AI assistants to interact with Nextcloud data"
|
||||
authors = [
|
||||
{name = "Chris Coutinho", email = "chris@coutinho.io"}
|
||||
@@ -43,6 +43,7 @@ dependencies = [
|
||||
"pymupdf4llm>=0.2.2",
|
||||
"openai>=2.8.1",
|
||||
"dynaconf>=3.2.13,<4.0",
|
||||
"mistralai>=2.4.5",
|
||||
]
|
||||
classifiers = [
|
||||
"Development Status :: 4 - Beta",
|
||||
|
||||
Executable
+137
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Purge legacy `highlighted_page_image` payloads from Qdrant (Deck #76).
|
||||
|
||||
Iterates all points in the configured Qdrant collection and deletes the
|
||||
legacy payload keys `highlighted_page_image`, `highlighted_page_number`,
|
||||
and `highlight_count`. This relieves disk pressure caused by inline
|
||||
base64 PNGs that the new code path no longer writes.
|
||||
|
||||
Idempotent: deleting non-existent keys is a no-op, so re-runs are safe.
|
||||
|
||||
Usage:
|
||||
uv run python scripts/purge_page_images.py [--dry-run] [--batch-size 256]
|
||||
|
||||
Connection settings (Qdrant URL/API key, collection name) are read from
|
||||
the same `Settings` object the server uses.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
from functools import partial
|
||||
|
||||
import anyio
|
||||
from qdrant_client import AsyncQdrantClient
|
||||
|
||||
from nextcloud_mcp_server.config import get_settings
|
||||
|
||||
logger = logging.getLogger("purge_page_images")
|
||||
|
||||
LEGACY_FIELDS = [
|
||||
"highlighted_page_image",
|
||||
"highlighted_page_number",
|
||||
"highlight_count",
|
||||
]
|
||||
|
||||
|
||||
async def purge(dry_run: bool, batch_size: int) -> None:
|
||||
settings = get_settings()
|
||||
if not settings.qdrant_url:
|
||||
raise SystemExit(
|
||||
"qdrant_url is not configured. Set QDRANT_URL (and QDRANT_API_KEY "
|
||||
"if required) before running this script."
|
||||
)
|
||||
collection = settings.get_collection_name()
|
||||
|
||||
# AsyncQdrantClient doesn't implement __aenter__/__aexit__, so use
|
||||
# try/finally to guarantee the underlying aiohttp session is closed.
|
||||
client = AsyncQdrantClient(
|
||||
url=settings.qdrant_url,
|
||||
api_key=settings.qdrant_api_key,
|
||||
timeout=60,
|
||||
)
|
||||
try:
|
||||
next_offset = None
|
||||
total_seen = 0
|
||||
total_updated = 0
|
||||
|
||||
logger.info(
|
||||
"Scanning collection %s; will delete keys %s%s",
|
||||
collection,
|
||||
LEGACY_FIELDS,
|
||||
" (dry run)" if dry_run else "",
|
||||
)
|
||||
|
||||
while True:
|
||||
points, next_offset = await client.scroll(
|
||||
collection_name=collection,
|
||||
limit=batch_size,
|
||||
offset=next_offset,
|
||||
with_payload=False,
|
||||
with_vectors=False,
|
||||
)
|
||||
if not points:
|
||||
break
|
||||
|
||||
ids = [p.id for p in points]
|
||||
total_seen += len(ids)
|
||||
|
||||
if not dry_run:
|
||||
await client.delete_payload(
|
||||
collection_name=collection,
|
||||
keys=LEGACY_FIELDS,
|
||||
points=ids,
|
||||
)
|
||||
total_updated += len(ids)
|
||||
|
||||
logger.info(
|
||||
"Batch: ids=%d total_seen=%d total_updated=%d",
|
||||
len(ids),
|
||||
total_seen,
|
||||
total_updated,
|
||||
)
|
||||
|
||||
if next_offset is None:
|
||||
break
|
||||
|
||||
logger.info(
|
||||
"Done. total_seen=%d total_updated=%d%s",
|
||||
total_seen,
|
||||
total_updated,
|
||||
" (dry run, no writes)" if dry_run else "",
|
||||
)
|
||||
finally:
|
||||
await client.close()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Scan only; do not write any changes.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--batch-size",
|
||||
type=int,
|
||||
default=256,
|
||||
help="Points per scroll/update batch (default: 256).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-v", "--verbose", action="store_true", help="Enable debug logging."
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG if args.verbose else logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
||||
)
|
||||
|
||||
anyio.run(partial(purge, dry_run=args.dry_run, batch_size=args.batch_size))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,269 @@
|
||||
"""Unit tests for Mistral provider."""
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from mistralai.client.errors import SDKError
|
||||
|
||||
from nextcloud_mcp_server.providers.mistral import (
|
||||
BATCH_SIZE,
|
||||
MISTRAL_EMBEDDING_DIMENSIONS,
|
||||
MistralProvider,
|
||||
_is_rate_limit,
|
||||
)
|
||||
|
||||
|
||||
def _make_data(embedding: list[float], index: int) -> MagicMock:
|
||||
"""Build a mock EmbeddingResponseData entry."""
|
||||
item = MagicMock()
|
||||
item.embedding = embedding
|
||||
item.index = index
|
||||
return item
|
||||
|
||||
|
||||
def _make_response(embeddings: list[list[float]]) -> MagicMock:
|
||||
"""Build a mock EmbeddingResponse with `embeddings` indexed in order."""
|
||||
response = MagicMock()
|
||||
response.data = [_make_data(emb, i) for i, emb in enumerate(embeddings)]
|
||||
return response
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_mistral_client(mocker):
|
||||
"""Mock the Mistral SDK constructor."""
|
||||
mock_client = MagicMock()
|
||||
mock_client.embeddings = MagicMock()
|
||||
mocker.patch(
|
||||
"nextcloud_mcp_server.providers.mistral.Mistral", return_value=mock_client
|
||||
)
|
||||
return mock_client
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_mistral_embedding_single(mock_mistral_client):
|
||||
"""Single text embed: round-trip through SDK with correct kwargs."""
|
||||
mock_mistral_client.embeddings.create_async = AsyncMock(
|
||||
return_value=_make_response([[0.1, 0.2, 0.3]])
|
||||
)
|
||||
|
||||
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
|
||||
embedding = await provider.embed("hello world")
|
||||
|
||||
assert embedding == [0.1, 0.2, 0.3]
|
||||
mock_mistral_client.embeddings.create_async.assert_awaited_once_with(
|
||||
model="mistral-embed",
|
||||
inputs=["hello world"],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_mistral_embedding_batch_single_call(mock_mistral_client):
|
||||
"""Batch smaller than BATCH_SIZE issues a single API call."""
|
||||
mock_mistral_client.embeddings.create_async = AsyncMock(
|
||||
return_value=_make_response([[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]])
|
||||
)
|
||||
|
||||
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
|
||||
embeddings = await provider.embed_batch(["a", "b", "c"])
|
||||
|
||||
assert embeddings == [[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]]
|
||||
assert mock_mistral_client.embeddings.create_async.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_mistral_embedding_batch_chunking(mock_mistral_client):
|
||||
"""Batches exceeding BATCH_SIZE are split into multiple API calls."""
|
||||
|
||||
# Each call returns one embedding per input it received; capture by side
|
||||
# effect so we can inspect lengths per chunk.
|
||||
def _side_effect(*, model, inputs, **_kwargs):
|
||||
return _make_response([[float(i)] for i in range(len(inputs))])
|
||||
|
||||
mock_mistral_client.embeddings.create_async = AsyncMock(side_effect=_side_effect)
|
||||
|
||||
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
|
||||
total = BATCH_SIZE * 2 + 5 # forces three chunks: 64, 64, 5 (with default)
|
||||
embeddings = await provider.embed_batch([f"text-{i}" for i in range(total)])
|
||||
|
||||
assert len(embeddings) == total
|
||||
assert mock_mistral_client.embeddings.create_async.await_count == 3
|
||||
# Verify the chunk sizes the SDK was actually called with.
|
||||
chunk_sizes = [
|
||||
len(call.kwargs["inputs"])
|
||||
for call in mock_mistral_client.embeddings.create_async.await_args_list
|
||||
]
|
||||
assert chunk_sizes == [BATCH_SIZE, BATCH_SIZE, 5]
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_mistral_embedding_batch_order_preserved(mock_mistral_client):
|
||||
"""Out-of-order index in response data is sorted before returning."""
|
||||
response = MagicMock()
|
||||
response.data = [
|
||||
_make_data([0.3, 0.3], 2),
|
||||
_make_data([0.1, 0.1], 0),
|
||||
_make_data([0.2, 0.2], 1),
|
||||
]
|
||||
mock_mistral_client.embeddings.create_async = AsyncMock(return_value=response)
|
||||
|
||||
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
|
||||
embeddings = await provider.embed_batch(["x", "y", "z"])
|
||||
|
||||
assert embeddings == [[0.1, 0.1], [0.2, 0.2], [0.3, 0.3]]
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_mistral_supports_capabilities(mock_mistral_client):
|
||||
"""Mistral provider advertises embeddings only."""
|
||||
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
|
||||
assert provider.supports_embeddings is True
|
||||
assert provider.supports_generation is False
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_mistral_generate_not_implemented(mock_mistral_client):
|
||||
"""generate() always raises NotImplementedError."""
|
||||
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
|
||||
with pytest.raises(NotImplementedError, match="does not support generation"):
|
||||
await provider.generate("test prompt")
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_mistral_get_dimension_known_model(mock_mistral_client):
|
||||
"""Known model: dimension available without an API call."""
|
||||
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
|
||||
assert provider.get_dimension() == MISTRAL_EMBEDDING_DIMENSIONS["mistral-embed"]
|
||||
mock_mistral_client.embeddings.create_async.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_mistral_get_dimension_unknown_model_detected(mock_mistral_client):
|
||||
"""Unknown model: dimension detected on first embed() call."""
|
||||
mock_mistral_client.embeddings.create_async = AsyncMock(
|
||||
return_value=_make_response([[0.1] * 768])
|
||||
)
|
||||
|
||||
provider = MistralProvider(api_key="test-key", embedding_model="custom-mistral")
|
||||
|
||||
with pytest.raises(RuntimeError, match="not detected yet"):
|
||||
provider.get_dimension()
|
||||
|
||||
await provider.embed("test")
|
||||
assert provider.get_dimension() == 768
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_mistral_no_embeddings_disabled(mock_mistral_client):
|
||||
"""Setting embedding_model=None disables the embedding capability."""
|
||||
provider = MistralProvider(api_key="test-key", embedding_model=None)
|
||||
assert provider.supports_embeddings is False
|
||||
|
||||
with pytest.raises(NotImplementedError, match="no embedding_model configured"):
|
||||
await provider.embed("test")
|
||||
with pytest.raises(NotImplementedError, match="no embedding_model configured"):
|
||||
await provider.embed_batch(["test"])
|
||||
with pytest.raises(NotImplementedError, match="no embedding_model configured"):
|
||||
provider.get_dimension()
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_mistral_empty_batch(mock_mistral_client):
|
||||
"""An empty batch returns [] without calling the API."""
|
||||
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
|
||||
assert await provider.embed_batch([]) == []
|
||||
mock_mistral_client.embeddings.create_async.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_mistral_close_no_error(mock_mistral_client):
|
||||
"""close() is best-effort and does not raise."""
|
||||
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
|
||||
# No __aexit__ on the mock by default → close() should silently no-op.
|
||||
await provider.close()
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_mistral_base_url_passed_to_sdk(mocker):
|
||||
"""base_url is forwarded as server_url to the Mistral SDK constructor."""
|
||||
mock_ctor = mocker.patch(
|
||||
"nextcloud_mcp_server.providers.mistral.Mistral", return_value=MagicMock()
|
||||
)
|
||||
|
||||
MistralProvider(
|
||||
api_key="test-key",
|
||||
embedding_model="mistral-embed",
|
||||
base_url="https://example.com/mistral",
|
||||
)
|
||||
mock_ctor.assert_called_once_with(
|
||||
api_key="test-key",
|
||||
server_url="https://example.com/mistral",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_mistral_embed_raises_on_empty_response_data(mock_mistral_client):
|
||||
"""embed(): empty response.data triggers the defensive RuntimeError guard."""
|
||||
empty_response = MagicMock()
|
||||
empty_response.data = []
|
||||
mock_mistral_client.embeddings.create_async = AsyncMock(return_value=empty_response)
|
||||
|
||||
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
|
||||
with pytest.raises(RuntimeError, match="returned no embedding"):
|
||||
await provider.embed("test")
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_mistral_embed_raises_on_null_embedding(mock_mistral_client):
|
||||
"""embed(): a single response item with embedding=None is rejected."""
|
||||
null_item = MagicMock()
|
||||
null_item.embedding = None
|
||||
null_item.index = 0
|
||||
null_response = MagicMock()
|
||||
null_response.data = [null_item]
|
||||
mock_mistral_client.embeddings.create_async = AsyncMock(return_value=null_response)
|
||||
|
||||
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
|
||||
with pytest.raises(RuntimeError, match="returned no embedding"):
|
||||
await provider.embed("test")
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_mistral_batch_raises_on_null_embedding(mock_mistral_client):
|
||||
"""_embed_batch_request: a null embedding inside a batch raises explicitly."""
|
||||
good = _make_data([0.1, 0.2], 0)
|
||||
bad = MagicMock()
|
||||
bad.embedding = None
|
||||
bad.index = 1
|
||||
response = MagicMock()
|
||||
response.data = [good, bad]
|
||||
mock_mistral_client.embeddings.create_async = AsyncMock(return_value=response)
|
||||
|
||||
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
|
||||
with pytest.raises(RuntimeError, match="null embedding"):
|
||||
await provider.embed_batch(["a", "b"])
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_mistral_batch_raises_on_count_mismatch(mock_mistral_client):
|
||||
"""_embed_batch_request: fewer embeddings returned than inputs sent."""
|
||||
# Two inputs sent, one embedding returned.
|
||||
response = _make_response([[0.1, 0.2]])
|
||||
mock_mistral_client.embeddings.create_async = AsyncMock(return_value=response)
|
||||
|
||||
provider = MistralProvider(api_key="test-key", embedding_model="mistral-embed")
|
||||
with pytest.raises(RuntimeError, match="returned 1 embeddings for 2 inputs"):
|
||||
await provider.embed_batch(["a", "b"])
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_mistral_is_rate_limit_predicate():
|
||||
"""_is_rate_limit returns True only for SDKErrors with status_code == 429."""
|
||||
err_429 = MagicMock(spec=SDKError)
|
||||
err_429.status_code = 429
|
||||
err_500 = MagicMock(spec=SDKError)
|
||||
err_500.status_code = 500
|
||||
|
||||
assert _is_rate_limit(err_429) is True
|
||||
assert _is_rate_limit(err_500) is False
|
||||
# ValueError has no status_code attr → getattr returns None → False.
|
||||
assert _is_rate_limit(ValueError()) is False
|
||||
@@ -0,0 +1,136 @@
|
||||
"""Unit tests for ProviderRegistry — dynaconf-driven auto-detection."""
|
||||
|
||||
import pytest
|
||||
|
||||
from nextcloud_mcp_server.config import _reload_config
|
||||
from nextcloud_mcp_server.providers import (
|
||||
BedrockProvider,
|
||||
MistralProvider,
|
||||
OllamaProvider,
|
||||
OpenAIProvider,
|
||||
SimpleProvider,
|
||||
get_provider,
|
||||
reset_provider,
|
||||
)
|
||||
from nextcloud_mcp_server.providers.bedrock import BOTO3_AVAILABLE
|
||||
|
||||
|
||||
def _clear_provider_envs(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Strip every provider-selection env var so each test starts clean."""
|
||||
for name in (
|
||||
"AWS_REGION",
|
||||
"AWS_ACCESS_KEY_ID",
|
||||
"AWS_SECRET_ACCESS_KEY",
|
||||
"BEDROCK_EMBEDDING_MODEL",
|
||||
"BEDROCK_GENERATION_MODEL",
|
||||
"OPENAI_API_KEY",
|
||||
"OPENAI_BASE_URL",
|
||||
"OPENAI_EMBEDDING_MODEL",
|
||||
"OPENAI_GENERATION_MODEL",
|
||||
"MISTRAL_API_KEY",
|
||||
"MISTRAL_BASE_URL",
|
||||
"MISTRAL_EMBEDDING_MODEL",
|
||||
"OLLAMA_BASE_URL",
|
||||
"OLLAMA_EMBEDDING_MODEL",
|
||||
"OLLAMA_GENERATION_MODEL",
|
||||
"OLLAMA_VERIFY_SSL",
|
||||
"SIMPLE_EMBEDDING_DIMENSION",
|
||||
):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def clean_provider_env(monkeypatch):
|
||||
"""Reset provider singleton + dynaconf cache around each test."""
|
||||
_clear_provider_envs(monkeypatch)
|
||||
reset_provider()
|
||||
_reload_config()
|
||||
yield monkeypatch
|
||||
reset_provider()
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_registry_falls_back_to_simple(clean_provider_env):
|
||||
"""No provider env set → SimpleProvider (with default dimension)."""
|
||||
provider = get_provider()
|
||||
assert isinstance(provider, SimpleProvider)
|
||||
assert provider.get_dimension() == 384
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_registry_picks_simple_with_custom_dimension(clean_provider_env):
|
||||
"""SIMPLE_EMBEDDING_DIMENSION flows through dynaconf to SimpleProvider."""
|
||||
clean_provider_env.setenv("SIMPLE_EMBEDDING_DIMENSION", "512")
|
||||
_reload_config()
|
||||
|
||||
provider = get_provider()
|
||||
assert isinstance(provider, SimpleProvider)
|
||||
assert provider.get_dimension() == 512
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_registry_picks_mistral_when_api_key_set(clean_provider_env, mocker):
|
||||
"""MISTRAL_API_KEY alone is enough to select MistralProvider."""
|
||||
# MistralProvider eagerly constructs the SDK client in __init__; stub it
|
||||
# so the test doesn't depend on the SDK accepting arbitrary keys.
|
||||
mocker.patch("nextcloud_mcp_server.providers.mistral.Mistral")
|
||||
clean_provider_env.setenv("MISTRAL_API_KEY", "test-key")
|
||||
_reload_config()
|
||||
|
||||
provider = get_provider()
|
||||
assert isinstance(provider, MistralProvider)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_registry_picks_ollama_when_base_url_set(clean_provider_env, mocker):
|
||||
"""OLLAMA_BASE_URL selects OllamaProvider."""
|
||||
# OllamaProvider eagerly probes /api/tags in __init__; stub it out.
|
||||
mocker.patch(
|
||||
"nextcloud_mcp_server.providers.ollama.OllamaProvider._check_model_is_loaded"
|
||||
)
|
||||
clean_provider_env.setenv("OLLAMA_BASE_URL", "http://localhost:11434")
|
||||
_reload_config()
|
||||
|
||||
provider = get_provider()
|
||||
assert isinstance(provider, OllamaProvider)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_registry_openai_wins_over_mistral_and_ollama(clean_provider_env):
|
||||
"""OpenAI takes priority when multiple provider env vars are set."""
|
||||
clean_provider_env.setenv("OPENAI_API_KEY", "openai-key")
|
||||
clean_provider_env.setenv("MISTRAL_API_KEY", "mistral-key")
|
||||
clean_provider_env.setenv("OLLAMA_BASE_URL", "http://localhost:11434")
|
||||
_reload_config()
|
||||
|
||||
provider = get_provider()
|
||||
assert isinstance(provider, OpenAIProvider)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_registry_mistral_wins_over_ollama(clean_provider_env, mocker):
|
||||
"""Mistral takes priority over Ollama when both are configured."""
|
||||
# Stub the Mistral SDK constructor for the same reason as the sibling
|
||||
# picker test — keeps the registry test independent of SDK key validation.
|
||||
mocker.patch("nextcloud_mcp_server.providers.mistral.Mistral")
|
||||
clean_provider_env.setenv("MISTRAL_API_KEY", "mistral-key")
|
||||
clean_provider_env.setenv("OLLAMA_BASE_URL", "http://localhost:11434")
|
||||
_reload_config()
|
||||
|
||||
provider = get_provider()
|
||||
assert isinstance(provider, MistralProvider)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_registry_bedrock_wins_when_aws_region_set(clean_provider_env):
|
||||
"""AWS_REGION alone routes to Bedrock, even with other providers configured."""
|
||||
if not BOTO3_AVAILABLE:
|
||||
pytest.skip("boto3 not installed")
|
||||
|
||||
clean_provider_env.setenv("AWS_REGION", "us-east-1")
|
||||
clean_provider_env.setenv("OPENAI_API_KEY", "openai-key")
|
||||
clean_provider_env.setenv("MISTRAL_API_KEY", "mistral-key")
|
||||
_reload_config()
|
||||
|
||||
provider = get_provider()
|
||||
assert isinstance(provider, BedrockProvider)
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Unit tests for the shared rate-limit retry decorator."""
|
||||
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from nextcloud_mcp_server.providers import _retry
|
||||
|
||||
|
||||
class _FakeError(Exception):
|
||||
"""Stand-in for an SDK exception with an HTTP status code attached."""
|
||||
|
||||
def __init__(self, status_code: int):
|
||||
super().__init__(f"status {status_code}")
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_real_sleep(monkeypatch):
|
||||
"""Replace anyio.sleep with an awaitable no-op so retries don't waste time."""
|
||||
monkeypatch.setattr(_retry.anyio, "sleep", AsyncMock(return_value=None))
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_retry_succeeds_after_429():
|
||||
"""A 429 followed by success returns the success value."""
|
||||
calls = {"n": 0}
|
||||
|
||||
@_retry.retry_on_rate_limit(
|
||||
_FakeError, is_rate_limit=lambda e: e.status_code == 429
|
||||
)
|
||||
async def flaky():
|
||||
calls["n"] += 1
|
||||
if calls["n"] < 3:
|
||||
raise _FakeError(429)
|
||||
return "ok"
|
||||
|
||||
result = await flaky()
|
||||
assert result == "ok"
|
||||
assert calls["n"] == 3
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_retry_reraises_non_rate_limit_immediately():
|
||||
"""A non-rate-limit error of the same class is re-raised on first hit."""
|
||||
calls = {"n": 0}
|
||||
|
||||
@_retry.retry_on_rate_limit(
|
||||
_FakeError, is_rate_limit=lambda e: e.status_code == 429
|
||||
)
|
||||
async def boom():
|
||||
calls["n"] += 1
|
||||
raise _FakeError(500)
|
||||
|
||||
with pytest.raises(_FakeError, match="status 500"):
|
||||
await boom()
|
||||
assert calls["n"] == 1 # No retries on non-429.
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_retry_gives_up_after_max_retries():
|
||||
"""After MAX_RETRIES failed attempts the last error is re-raised."""
|
||||
calls = {"n": 0}
|
||||
|
||||
@_retry.retry_on_rate_limit(
|
||||
_FakeError, is_rate_limit=lambda e: e.status_code == 429
|
||||
)
|
||||
async def always_429():
|
||||
calls["n"] += 1
|
||||
raise _FakeError(429)
|
||||
|
||||
with pytest.raises(_FakeError, match="status 429"):
|
||||
await always_429()
|
||||
assert calls["n"] == _retry.MAX_RETRIES
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_retry_default_predicate_treats_all_as_rate_limit():
|
||||
"""Default predicate (`lambda _: True`) retries every caught exception."""
|
||||
calls = {"n": 0}
|
||||
|
||||
@_retry.retry_on_rate_limit(_FakeError)
|
||||
async def fail_once():
|
||||
calls["n"] += 1
|
||||
if calls["n"] < 2:
|
||||
raise _FakeError(503)
|
||||
return "recovered"
|
||||
|
||||
result = await fail_once()
|
||||
assert result == "recovered"
|
||||
assert calls["n"] == 2
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_retry_does_not_catch_unrelated_exceptions():
|
||||
"""Exceptions of a different class bypass the decorator entirely."""
|
||||
|
||||
@_retry.retry_on_rate_limit(_FakeError)
|
||||
async def value_error():
|
||||
raise ValueError("nope")
|
||||
|
||||
with pytest.raises(ValueError, match="nope"):
|
||||
await value_error()
|
||||
@@ -0,0 +1,232 @@
|
||||
"""Unit tests for PDFHighlighter.compute_chunk_bboxes_batch (Deck #76).
|
||||
|
||||
Replaces the legacy `highlight_chunks_batch`-+-base64 pipeline that inflated
|
||||
Qdrant payloads with per-chunk PNG screenshots. The new path returns
|
||||
normalized bounding boxes only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pymupdf
|
||||
import pytest
|
||||
|
||||
from nextcloud_mcp_server.search.pdf_highlighter import PDFHighlighter
|
||||
|
||||
|
||||
def _make_pdf(pages: list[str]) -> bytes:
|
||||
"""Build an in-memory PDF whose pages contain the given text."""
|
||||
doc = pymupdf.open()
|
||||
for body in pages:
|
||||
page = doc.new_page(width=595, height=842) # A4
|
||||
page.insert_text((50, 50), body)
|
||||
pdf_bytes = doc.tobytes()
|
||||
doc.close()
|
||||
return pdf_bytes
|
||||
|
||||
|
||||
def _page_boundaries(pages: list[str]) -> tuple[list[dict], str]:
|
||||
"""Build (page_boundaries, full_text) compatible with the highlighter API."""
|
||||
boundaries: list[dict] = []
|
||||
cursor = 0
|
||||
parts: list[str] = []
|
||||
for i, body in enumerate(pages, start=1):
|
||||
end = cursor + len(body)
|
||||
boundaries.append({"page": i, "start_offset": cursor, "end_offset": end})
|
||||
parts.append(body)
|
||||
cursor = end
|
||||
return boundaries, "".join(parts)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_compute_chunk_bboxes_returns_normalized_rects():
|
||||
"""Each returned bbox should be 4 floats in [0, 1] tagged with the page."""
|
||||
pages = [
|
||||
"Chapter 1: Introduction. Nextcloud is a self-hosted collaboration platform "
|
||||
"covering installation, configuration and maintenance topics.",
|
||||
"Chapter 2: Installation. Download the package, extract it to the web "
|
||||
"server directory, and configure the database connection.",
|
||||
]
|
||||
pdf_bytes = _make_pdf(pages)
|
||||
boundaries, full_text = _page_boundaries(pages)
|
||||
|
||||
chunks = [
|
||||
(
|
||||
0,
|
||||
0,
|
||||
len(pages[0]),
|
||||
1,
|
||||
"Chapter 1: Introduction. Nextcloud is a self-hosted collaboration platform.",
|
||||
),
|
||||
(
|
||||
1,
|
||||
len(pages[0]),
|
||||
len(pages[0]) + len(pages[1]),
|
||||
2,
|
||||
"Chapter 2: Installation. Download the package.",
|
||||
),
|
||||
]
|
||||
|
||||
results = PDFHighlighter.compute_chunk_bboxes_batch(
|
||||
pdf_bytes=pdf_bytes,
|
||||
chunks=chunks,
|
||||
page_boundaries=boundaries,
|
||||
full_text=full_text,
|
||||
)
|
||||
|
||||
assert set(results) == {0, 1}
|
||||
|
||||
bboxes_p1, page_p1 = results[0]
|
||||
bboxes_p2, page_p2 = results[1]
|
||||
|
||||
assert page_p1 == 1
|
||||
assert page_p2 == 2
|
||||
|
||||
for rects in (bboxes_p1, bboxes_p2):
|
||||
assert len(rects) >= 1
|
||||
for rect in rects:
|
||||
assert len(rect) == 4
|
||||
x0, y0, x1, y1 = rect
|
||||
assert 0.0 <= x0 < x1 <= 1.0
|
||||
assert 0.0 <= y0 < y1 <= 1.0
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_compute_chunk_bboxes_empty_input():
|
||||
assert (
|
||||
PDFHighlighter.compute_chunk_bboxes_batch(
|
||||
pdf_bytes=b"",
|
||||
chunks=[],
|
||||
page_boundaries=[],
|
||||
full_text="",
|
||||
)
|
||||
== {}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_compute_chunk_bboxes_omits_when_offsets_out_of_range():
|
||||
"""Chunks whose offsets fall outside every page boundary are omitted.
|
||||
|
||||
Verifies the docstring contract: *"Chunks whose bbox cannot be located
|
||||
are omitted from the result."* (path: ``find_chunk_page`` returns None).
|
||||
"""
|
||||
pages = ["Page one body text content here for the test."]
|
||||
pdf_bytes = _make_pdf(pages)
|
||||
boundaries, full_text = _page_boundaries(pages)
|
||||
|
||||
# Offsets way beyond the document end — no page boundary matches.
|
||||
out_of_range_start = len(full_text) + 1000
|
||||
out_of_range_end = out_of_range_start + 50
|
||||
chunks = [(0, out_of_range_start, out_of_range_end, 1, "irrelevant")]
|
||||
|
||||
results = PDFHighlighter.compute_chunk_bboxes_batch(
|
||||
pdf_bytes=pdf_bytes,
|
||||
chunks=chunks,
|
||||
page_boundaries=boundaries,
|
||||
full_text=full_text,
|
||||
)
|
||||
|
||||
assert results == {}
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_compute_chunk_bboxes_omits_when_text_not_in_pdf():
|
||||
"""Chunks whose page-relative text isn't on the page are omitted.
|
||||
|
||||
Verifies the second omission path: ``_find_chunk_bbox`` returns None
|
||||
when the supplied text cannot be located on the rendered page.
|
||||
"""
|
||||
pages = ["Hello world."]
|
||||
pdf_bytes = _make_pdf(pages)
|
||||
# Build boundaries from the real text but pass a *different* full_text
|
||||
# so the page-relative slice is content that does not exist in the PDF.
|
||||
boundaries, _ = _page_boundaries(pages)
|
||||
bogus_full_text = "Z" * len(pages[0])
|
||||
|
||||
chunks = [(0, 0, len(pages[0]), 1, "ignored")]
|
||||
|
||||
results = PDFHighlighter.compute_chunk_bboxes_batch(
|
||||
pdf_bytes=pdf_bytes,
|
||||
chunks=chunks,
|
||||
page_boundaries=boundaries,
|
||||
full_text=bogus_full_text,
|
||||
)
|
||||
|
||||
assert results == {}
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@pytest.mark.parametrize("page_index", [0, 1])
|
||||
def test_compute_chunk_bboxes_assigns_correct_page(page_index: int):
|
||||
"""Verify the page number returned matches the page the chunk lives on."""
|
||||
pages = [
|
||||
"Page one talks about apples and oranges in detail.",
|
||||
"Page two discusses bananas and grapes thoroughly.",
|
||||
]
|
||||
pdf_bytes = _make_pdf(pages)
|
||||
boundaries, full_text = _page_boundaries(pages)
|
||||
|
||||
if page_index == 0:
|
||||
chunk_text = "apples and oranges"
|
||||
offsets = (0, len(pages[0]))
|
||||
else:
|
||||
chunk_text = "bananas and grapes"
|
||||
offsets = (len(pages[0]), len(pages[0]) + len(pages[1]))
|
||||
|
||||
chunks = [(0, offsets[0], offsets[1], page_index + 1, chunk_text)]
|
||||
|
||||
results = PDFHighlighter.compute_chunk_bboxes_batch(
|
||||
pdf_bytes=pdf_bytes,
|
||||
chunks=chunks,
|
||||
page_boundaries=boundaries,
|
||||
full_text=full_text,
|
||||
)
|
||||
|
||||
assert results, "expected a bbox for the chunk"
|
||||
_, page_num = results[0]
|
||||
assert page_num == page_index + 1
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_compute_chunk_bboxes_handles_unordered_page_boundaries():
|
||||
"""Page lookup must match by ``page`` key, not by list position.
|
||||
|
||||
Regression guard: an earlier implementation indexed
|
||||
``page_boundaries[page_num - 1]``, which silently produces a wrong
|
||||
bbox if boundaries are passed out of order. Reverse the boundaries
|
||||
and assert the result is identical to the in-order case.
|
||||
"""
|
||||
pages = [
|
||||
"Page one talks about apples and oranges in detail.",
|
||||
"Page two discusses bananas and grapes thoroughly.",
|
||||
]
|
||||
pdf_bytes = _make_pdf(pages)
|
||||
boundaries, full_text = _page_boundaries(pages)
|
||||
|
||||
chunks = [
|
||||
(0, 0, len(pages[0]), 1, "apples and oranges"),
|
||||
(
|
||||
1,
|
||||
len(pages[0]),
|
||||
len(pages[0]) + len(pages[1]),
|
||||
2,
|
||||
"bananas and grapes",
|
||||
),
|
||||
]
|
||||
|
||||
in_order = PDFHighlighter.compute_chunk_bboxes_batch(
|
||||
pdf_bytes=pdf_bytes,
|
||||
chunks=chunks,
|
||||
page_boundaries=boundaries,
|
||||
full_text=full_text,
|
||||
)
|
||||
reversed_order = PDFHighlighter.compute_chunk_bboxes_batch(
|
||||
pdf_bytes=pdf_bytes,
|
||||
chunks=chunks,
|
||||
page_boundaries=list(reversed(boundaries)),
|
||||
full_text=full_text,
|
||||
)
|
||||
|
||||
assert in_order == reversed_order
|
||||
assert reversed_order[0][1] == 1
|
||||
assert reversed_order[1][1] == 2
|
||||
@@ -769,6 +769,15 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/97/43/11d6e5d2c00bf000b5329717c74563bf76a9193f4a41cb0c4ef277dde4fa/dynaconf-3.2.13-py2.py3-none-any.whl", hash = "sha256:4305527aef4834bdba3e39479b23c005186e83fb85f65bcaa4bcea58fa26759b", size = 238041, upload-time = "2026-03-17T19:38:45.337Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "eval-type-backport"
|
||||
version = "0.3.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/fb/a3/cafafb4558fd638aadfe4121dc6cefb8d743368c085acb2f521df0f3d9d7/eval_type_backport-0.3.1.tar.gz", hash = "sha256:57e993f7b5b69d271e37482e62f74e76a0276c82490cf8e4f0dffeb6b332d5ed", size = 9445, upload-time = "2025-12-02T11:51:42.987Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/cf/22/fdc2e30d43ff853720042fa15baa3e6122722be1a7950a98233ebb55cd71/eval_type_backport-0.3.1-py3-none-any.whl", hash = "sha256:279ab641905e9f11129f56a8a78f493518515b83402b860f6f06dd7c011fdfa8", size = 6063, upload-time = "2025-12-02T11:51:41.665Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "executing"
|
||||
version = "2.2.1"
|
||||
@@ -1478,6 +1487,15 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/73/07/02e16ed01e04a374e644b575638ec7987ae846d25ad97bcc9945a3ee4b0e/jsonpatch-1.33-py2.py3-none-any.whl", hash = "sha256:0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade", size = 12898, upload-time = "2023-06-16T21:01:28.466Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jsonpath-python"
|
||||
version = "1.1.6"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/98/18/4ca8742534a5993ff383f7602e325ce2d5d7cc93d72ac5e1cdedbea8a458/jsonpath_python-1.1.6.tar.gz", hash = "sha256:dded9932b4ec41fb8726e09c83afa4e6be618f938c2db287cc2a81723c639671", size = 88178, upload-time = "2026-05-07T01:26:34.482Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/55/8a/1270a6803bd821cbfcdda387eaa13cb41a7b1f7b9bd145979b3bfb9d6cb7/jsonpath_python-1.1.6-py3-none-any.whl", hash = "sha256:a1c50afd8d3fbbaf47a4873bc890dcb3c15da96f5c020327977d844d8731a2d4", size = 14453, upload-time = "2026-05-07T01:26:33.306Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jsonpointer"
|
||||
version = "3.0.0"
|
||||
@@ -1842,6 +1860,25 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mistralai"
|
||||
version = "2.4.5"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "eval-type-backport" },
|
||||
{ name = "httpx" },
|
||||
{ name = "jsonpath-python" },
|
||||
{ name = "opentelemetry-api" },
|
||||
{ name = "opentelemetry-semantic-conventions" },
|
||||
{ name = "pydantic" },
|
||||
{ name = "python-dateutil" },
|
||||
{ name = "typing-inspection" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/8e/3f/5624d57c5897c83c55d3e4c7dd4127de42ad14fd3183e26566cdc7dca1bf/mistralai-2.4.5.tar.gz", hash = "sha256:ef165bb004ec4423cbf19a440bf0983ca0c3fc92ab12a35ebca097bdf418e33a", size = 424611, upload-time = "2026-05-07T11:46:43.888Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/1b/48/2c5c4f853dec32a625c1a3d23809b80cf2e135c3441fe1764f72910dfea9/mistralai-2.4.5-py3-none-any.whl", hash = "sha256:bf3b6550258ab16dec8547b90e9c18bebf9099f55b7fc25a884bf0bbeffced0f", size = 995999, upload-time = "2026-05-07T11:46:41.915Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mmh3"
|
||||
version = "5.2.0"
|
||||
@@ -2086,7 +2123,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "nextcloud-mcp-server"
|
||||
version = "0.81.0"
|
||||
version = "0.83.0"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "aiosqlite" },
|
||||
@@ -2104,6 +2141,7 @@ dependencies = [
|
||||
{ name = "langchain-text-splitters" },
|
||||
{ name = "markdownify" },
|
||||
{ name = "mcp", extra = ["cli"] },
|
||||
{ name = "mistralai" },
|
||||
{ name = "openai" },
|
||||
{ name = "opentelemetry-api" },
|
||||
{ name = "opentelemetry-exporter-otlp-proto-grpc" },
|
||||
@@ -2157,6 +2195,7 @@ requires-dist = [
|
||||
{ name = "langchain-text-splitters", specifier = ">=1.0.0" },
|
||||
{ name = "markdownify", specifier = ">=0.14.1" },
|
||||
{ name = "mcp", extras = ["cli"], specifier = ">=1.27,<1.28" },
|
||||
{ name = "mistralai", specifier = ">=2.4.5" },
|
||||
{ name = "openai", specifier = ">=2.8.1" },
|
||||
{ name = "opentelemetry-api", specifier = ">=1.28.2" },
|
||||
{ name = "opentelemetry-exporter-otlp-proto-grpc", specifier = ">=1.28.2" },
|
||||
@@ -2341,45 +2380,45 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry-api"
|
||||
version = "1.39.0"
|
||||
version = "1.39.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "importlib-metadata" },
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/c0/0b/e5428c009d4d9af0515b0a8371a8aaae695371af291f45e702f7969dce6b/opentelemetry_api-1.39.0.tar.gz", hash = "sha256:6130644268c5ac6bdffaf660ce878f10906b3e789f7e2daa5e169b047a2933b9", size = 65763, upload-time = "2025-12-03T13:19:56.378Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/97/b9/3161be15bb8e3ad01be8be5a968a9237c3027c5be504362ff800fca3e442/opentelemetry_api-1.39.1.tar.gz", hash = "sha256:fbde8c80e1b937a2c61f20347e91c0c18a1940cecf012d62e65a7caf08967c9c", size = 65767, upload-time = "2025-12-11T13:32:39.182Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/05/85/d831a9bc0a9e0e1a304ff3d12c1489a5fbc9bf6690a15dcbdae372bbca45/opentelemetry_api-1.39.0-py3-none-any.whl", hash = "sha256:3c3b3ca5c5687b1b5b37e5c5027ff68eacea8675241b29f13110a8ffbb8f0459", size = 66357, upload-time = "2025-12-03T13:19:33.043Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cf/df/d3f1ddf4bb4cb50ed9b1139cc7b1c54c34a1e7ce8fd1b9a37c0d1551a6bd/opentelemetry_api-1.39.1-py3-none-any.whl", hash = "sha256:2edd8463432a7f8443edce90972169b195e7d6a05500cd29e6d13898187c9950", size = 66356, upload-time = "2025-12-11T13:32:17.304Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry-exporter-otlp"
|
||||
version = "1.39.0"
|
||||
version = "1.39.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "opentelemetry-exporter-otlp-proto-grpc" },
|
||||
{ name = "opentelemetry-exporter-otlp-proto-http" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/13/be/0e9d889f47e55cadc4041e5b53d4e0cc688f9a74811134fb0ba7cbee6905/opentelemetry_exporter_otlp-1.39.0.tar.gz", hash = "sha256:b405da0287b895fe4e2450dedb2a5b072debba1dfcfed5bdb3d1d183d8daa296", size = 6146, upload-time = "2025-12-03T13:19:58.381Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/30/9c/3ab1db90f32da200dba332658f2bbe602369e3d19f6aba394031a42635be/opentelemetry_exporter_otlp-1.39.1.tar.gz", hash = "sha256:7cf7470e9fd0060c8a38a23e4f695ac686c06a48ad97f8d4867bc9b420180b9c", size = 6147, upload-time = "2025-12-11T13:32:40.309Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/fc/35/212d2cae4fa9a2c02e74438612268b640ab577b8ccb04590371eb4e0f542/opentelemetry_exporter_otlp-1.39.0-py3-none-any.whl", hash = "sha256:fe155d6968d581b325574ad6dc267c8de299397b18d11feeda2206d0a47928a9", size = 7017, upload-time = "2025-12-03T13:19:35.686Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/00/6c/bdc82a066e6fb1dcf9e8cc8d4e026358fe0f8690700cc6369a6bf9bd17a7/opentelemetry_exporter_otlp-1.39.1-py3-none-any.whl", hash = "sha256:68ae69775291f04f000eb4b698ff16ff685fdebe5cb52871bc4e87938a7b00fe", size = 7019, upload-time = "2025-12-11T13:32:19.387Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry-exporter-otlp-proto-common"
|
||||
version = "1.39.0"
|
||||
version = "1.39.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "opentelemetry-proto" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/11/cb/3a29ce606b10c76d413d6edd42d25a654af03e73e50696611e757d2602f3/opentelemetry_exporter_otlp_proto_common-1.39.0.tar.gz", hash = "sha256:a135fceed1a6d767f75be65bd2845da344dd8b9258eeed6bc48509d02b184409", size = 20407, upload-time = "2025-12-03T13:19:59.003Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/e9/9d/22d241b66f7bbde88a3bfa6847a351d2c46b84de23e71222c6aae25c7050/opentelemetry_exporter_otlp_proto_common-1.39.1.tar.gz", hash = "sha256:763370d4737a59741c89a67b50f9e39271639ee4afc999dadfe768541c027464", size = 20409, upload-time = "2025-12-11T13:32:40.885Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ef/c6/215edba62d13a3948c718b289539f70e40965bc37fc82ecd55bb0b749c1a/opentelemetry_exporter_otlp_proto_common-1.39.0-py3-none-any.whl", hash = "sha256:3d77be7c4bdf90f1a76666c934368b8abed730b5c6f0547a2ec57feb115849ac", size = 18367, upload-time = "2025-12-03T13:19:36.906Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8c/02/ffc3e143d89a27ac21fd557365b98bd0653b98de8a101151d5805b5d4c33/opentelemetry_exporter_otlp_proto_common-1.39.1-py3-none-any.whl", hash = "sha256:08f8a5862d64cc3435105686d0216c1365dc5701f86844a8cd56597d0c764fde", size = 18366, upload-time = "2025-12-11T13:32:20.2Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry-exporter-otlp-proto-grpc"
|
||||
version = "1.39.0"
|
||||
version = "1.39.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "googleapis-common-protos" },
|
||||
@@ -2390,14 +2429,14 @@ dependencies = [
|
||||
{ name = "opentelemetry-sdk" },
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/7e/62/4db083ee9620da3065eeb559e9fc128f41a1d15e7c48d7c83aafbccd354c/opentelemetry_exporter_otlp_proto_grpc-1.39.0.tar.gz", hash = "sha256:7e7bb3f436006836c0e0a42ac619097746ad5553ad7128a5bd4d3e727f37fc06", size = 24650, upload-time = "2025-12-03T13:20:00.06Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/53/48/b329fed2c610c2c32c9366d9dc597202c9d1e58e631c137ba15248d8850f/opentelemetry_exporter_otlp_proto_grpc-1.39.1.tar.gz", hash = "sha256:772eb1c9287485d625e4dbe9c879898e5253fea111d9181140f51291b5fec3ad", size = 24650, upload-time = "2025-12-11T13:32:41.429Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/56/e8/d420b94ffddfd8cff85bb4aa5d98da26ce7935dc3cf3eca6b83cd39ab436/opentelemetry_exporter_otlp_proto_grpc-1.39.0-py3-none-any.whl", hash = "sha256:758641278050de9bb895738f35ff8840e4a47685b7e6ef4a201fe83196ba7a05", size = 19765, upload-time = "2025-12-03T13:19:38.143Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/81/a3/cc9b66575bd6597b98b886a2067eea2693408d2d5f39dad9ab7fc264f5f3/opentelemetry_exporter_otlp_proto_grpc-1.39.1-py3-none-any.whl", hash = "sha256:fa1c136a05c7e9b4c09f739469cbdb927ea20b34088ab1d959a849b5cc589c18", size = 19766, upload-time = "2025-12-11T13:32:21.027Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry-exporter-otlp-proto-http"
|
||||
version = "1.39.0"
|
||||
version = "1.39.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "googleapis-common-protos" },
|
||||
@@ -2408,14 +2447,14 @@ dependencies = [
|
||||
{ name = "requests" },
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/81/dc/1e9bf3f6a28e29eba516bc0266e052996d02bc7e92675f3cd38169607609/opentelemetry_exporter_otlp_proto_http-1.39.0.tar.gz", hash = "sha256:28d78fc0eb82d5a71ae552263d5012fa3ebad18dfd189bf8d8095ba0e65ee1ed", size = 17287, upload-time = "2025-12-03T13:20:01.134Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/80/04/2a08fa9c0214ae38880df01e8bfae12b067ec0793446578575e5080d6545/opentelemetry_exporter_otlp_proto_http-1.39.1.tar.gz", hash = "sha256:31bdab9745c709ce90a49a0624c2bd445d31a28ba34275951a6a362d16a0b9cb", size = 17288, upload-time = "2025-12-11T13:32:42.029Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/bc/46/e4a102e17205bb05a50dbf24ef0e92b66b648cd67db9a68865af06a242fd/opentelemetry_exporter_otlp_proto_http-1.39.0-py3-none-any.whl", hash = "sha256:5789cb1375a8b82653328c0ce13a054d285f774099faf9d068032a49de4c7862", size = 19639, upload-time = "2025-12-03T13:19:39.536Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/95/f1/b27d3e2e003cd9a3592c43d099d2ed8d0a947c15281bf8463a256db0b46c/opentelemetry_exporter_otlp_proto_http-1.39.1-py3-none-any.whl", hash = "sha256:d9f5207183dd752a412c4cd564ca8875ececba13be6e9c6c370ffb752fd59985", size = 19641, upload-time = "2025-12-11T13:32:22.248Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry-instrumentation"
|
||||
version = "0.60b0"
|
||||
version = "0.60b1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "opentelemetry-api" },
|
||||
@@ -2423,14 +2462,14 @@ dependencies = [
|
||||
{ name = "packaging" },
|
||||
{ name = "wrapt" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/55/3c/bd53dbb42eff93d18e3047c7be11224aa9966ce98ac4cc5bfb860a32c95a/opentelemetry_instrumentation-0.60b0.tar.gz", hash = "sha256:4e9fec930f283a2677a2217754b40aaf9ef76edae40499c165bc7f1d15366a74", size = 31707, upload-time = "2025-12-03T13:22:00.352Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/41/0f/7e6b713ac117c1f5e4e3300748af699b9902a2e5e34c9cf443dde25a01fa/opentelemetry_instrumentation-0.60b1.tar.gz", hash = "sha256:57ddc7974c6eb35865af0426d1a17132b88b2ed8586897fee187fd5b8944bd6a", size = 31706, upload-time = "2025-12-11T13:36:42.515Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/5c/7b/5b5b9f8cfe727a28553acf9cd287b1d7f706f5c0a00d6e482df55b169483/opentelemetry_instrumentation-0.60b0-py3-none-any.whl", hash = "sha256:aaafa1483543a402819f1bdfb06af721c87d60dd109501f9997332862a35c76a", size = 33096, upload-time = "2025-12-03T13:20:51.785Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/77/d2/6788e83c5c86a2690101681aeef27eeb2a6bf22df52d3f263a22cee20915/opentelemetry_instrumentation-0.60b1-py3-none-any.whl", hash = "sha256:04480db952b48fb1ed0073f822f0ee26012b7be7c3eac1a3793122737c78632d", size = 33096, upload-time = "2025-12-11T13:35:33.067Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry-instrumentation-asgi"
|
||||
version = "0.60b0"
|
||||
version = "0.60b1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "asgiref" },
|
||||
@@ -2439,14 +2478,14 @@ dependencies = [
|
||||
{ name = "opentelemetry-semantic-conventions" },
|
||||
{ name = "opentelemetry-util-http" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/b0/0a/715ea7044708d3c215385fb2a1c6ffe429aacb3cd23a348060aaeda52834/opentelemetry_instrumentation_asgi-0.60b0.tar.gz", hash = "sha256:928731218050089dca69f0fe980b8bfe109f384be8b89802d7337372ddb67b91", size = 26083, upload-time = "2025-12-03T13:22:05.672Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/77/db/851fa88db7441da82d50bd80f2de5ee55213782e25dc858e04d0c9961d60/opentelemetry_instrumentation_asgi-0.60b1.tar.gz", hash = "sha256:16bfbe595cd24cda309a957456d0fc2523f41bc7b076d1f2d7e98a1ad9876d6f", size = 26107, upload-time = "2025-12-11T13:36:47.015Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/9b/8c/c6c59127fd996107243ca45669355665a7daff578ddafb86d6d2d3b01428/opentelemetry_instrumentation_asgi-0.60b0-py3-none-any.whl", hash = "sha256:9d76a541269452c718a0384478f3291feb650c5a3f29e578fdc6613ea3729cf3", size = 16907, upload-time = "2025-12-03T13:20:58.962Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/76/76/1fb94367cef64420d2171157a6b9509582873bd09a6afe08a78a8d1f59d9/opentelemetry_instrumentation_asgi-0.60b1-py3-none-any.whl", hash = "sha256:d48def2dbed10294c99cfcf41ebbd0c414d390a11773a41f472d20000fcddc25", size = 16933, upload-time = "2025-12-11T13:35:40.462Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry-instrumentation-httpx"
|
||||
version = "0.60b0"
|
||||
version = "0.60b1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "opentelemetry-api" },
|
||||
@@ -2455,70 +2494,70 @@ dependencies = [
|
||||
{ name = "opentelemetry-util-http" },
|
||||
{ name = "wrapt" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/09/71/9dc0bc5ab14122251f520ffe9fc8dd892ca688d7d591482b5b1843d685d2/opentelemetry_instrumentation_httpx-0.60b0.tar.gz", hash = "sha256:fcf349a92fb0b941a2a18bec65141f4ba62cbf7a457a1aa580794bad44dc477c", size = 20612, upload-time = "2025-12-03T13:22:20.5Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/86/08/11208bcfcab4fc2023252c3f322aa397fd9ad948355fea60f5fc98648603/opentelemetry_instrumentation_httpx-0.60b1.tar.gz", hash = "sha256:a506ebaf28c60112cbe70ad4f0338f8603f148938cb7b6794ce1051cd2b270ae", size = 20611, upload-time = "2025-12-11T13:37:01.661Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/3a/22/a340c8bfad6f31bfd6a15b0b24d5e68e05e3975e4dbdc3cea6ec4f96e060/opentelemetry_instrumentation_httpx-0.60b0-py3-none-any.whl", hash = "sha256:3f5e6fc4ddf1d9de2aaddb5255110827154dbd4de9187da906c8c2a3cc2219e9", size = 15702, upload-time = "2025-12-03T13:21:20.667Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/43/59/b98e84eebf745ffc75397eaad4763795bff8a30cbf2373a50ed4e70646c5/opentelemetry_instrumentation_httpx-0.60b1-py3-none-any.whl", hash = "sha256:f37636dd742ad2af83d896ba69601ed28da51fa4e25d1ab62fde89ce413e275b", size = 15701, upload-time = "2025-12-11T13:36:04.56Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry-instrumentation-logging"
|
||||
version = "0.60b0"
|
||||
version = "0.60b1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "opentelemetry-api" },
|
||||
{ name = "opentelemetry-instrumentation" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/5b/8b/a1aed0b695a58a1f0bdcf90fae5e468cc0fd7de2b74b9816b17e45c38e43/opentelemetry_instrumentation_logging-0.60b0.tar.gz", hash = "sha256:6d87840666669cbbcd53d2230c7a33476862d0bf7f1adc67a95519c6ccfc8281", size = 9969, upload-time = "2025-12-03T13:22:22.608Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/60/a6/4515895b383113677fd2ad21813df5e56108a2df14ebb7916c962c9a0234/opentelemetry_instrumentation_logging-0.60b1.tar.gz", hash = "sha256:98f4b9c7aeb9314a30feee7c002c7ea9abea07c90df5f97fb058b850bc45b89a", size = 9968, upload-time = "2025-12-11T13:37:03.974Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/00/89/6d4f5d8d03376637bff5b8d9e91356104a6f1ce9a34a2099bb2f4bf5e2b5/opentelemetry_instrumentation_logging-0.60b0-py3-none-any.whl", hash = "sha256:af75b3020911b9b6a1b4b19819a165eb131ce9bfdd313062d578fc2dc9a5cd0f", size = 12576, upload-time = "2025-12-03T13:21:24.797Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f1/f9/8a4ce3901bc52277794e4b18c4ac43dc5929806eff01d22812364132f45f/opentelemetry_instrumentation_logging-0.60b1-py3-none-any.whl", hash = "sha256:f2e18cbc7e1dd3628c80e30d243897fdc93c5b7e0c8ae60abd2b9b6a99f82343", size = 12577, upload-time = "2025-12-11T13:36:08.123Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry-proto"
|
||||
version = "1.39.0"
|
||||
version = "1.39.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "protobuf" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/48/b5/64d2f8c3393cd13ea2092106118f7b98461ba09333d40179a31444c6f176/opentelemetry_proto-1.39.0.tar.gz", hash = "sha256:c1fa48678ad1a1624258698e59be73f990b7fc1f39e73e16a9d08eef65dd838c", size = 46153, upload-time = "2025-12-03T13:20:08.729Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/49/1d/f25d76d8260c156c40c97c9ed4511ec0f9ce353f8108ca6e7561f82a06b2/opentelemetry_proto-1.39.1.tar.gz", hash = "sha256:6c8e05144fc0d3ed4d22c2289c6b126e03bcd0e6a7da0f16cedd2e1c2772e2c8", size = 46152, upload-time = "2025-12-11T13:32:48.681Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e3/4d/d500e1862beed68318705732d1976c390f4a72ca8009c4983ff627acff20/opentelemetry_proto-1.39.0-py3-none-any.whl", hash = "sha256:1e086552ac79acb501485ff0ce75533f70f3382d43d0a30728eeee594f7bf818", size = 72534, upload-time = "2025-12-03T13:19:50.251Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/51/95/b40c96a7b5203005a0b03d8ce8cd212ff23f1793d5ba289c87a097571b18/opentelemetry_proto-1.39.1-py3-none-any.whl", hash = "sha256:22cdc78efd3b3765d09e68bfbd010d4fc254c9818afd0b6b423387d9dee46007", size = 72535, upload-time = "2025-12-11T13:32:33.866Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry-sdk"
|
||||
version = "1.39.0"
|
||||
version = "1.39.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "opentelemetry-api" },
|
||||
{ name = "opentelemetry-semantic-conventions" },
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/51/e3/7cd989003e7cde72e0becfe830abff0df55c69d237ee7961a541e0167833/opentelemetry_sdk-1.39.0.tar.gz", hash = "sha256:c22204f12a0529e07aa4d985f1bca9d6b0e7b29fe7f03e923548ae52e0e15dde", size = 171322, upload-time = "2025-12-03T13:20:09.651Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/eb/fb/c76080c9ba07e1e8235d24cdcc4d125ef7aa3edf23eb4e497c2e50889adc/opentelemetry_sdk-1.39.1.tar.gz", hash = "sha256:cf4d4563caf7bff906c9f7967e2be22d0d6b349b908be0d90fb21c8e9c995cc6", size = 171460, upload-time = "2025-12-11T13:32:49.369Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/a4/b4/2adc8bc83eb1055ecb592708efb6f0c520cc2eb68970b02b0f6ecda149cf/opentelemetry_sdk-1.39.0-py3-none-any.whl", hash = "sha256:90cfb07600dfc0d2de26120cebc0c8f27e69bf77cd80ef96645232372709a514", size = 132413, upload-time = "2025-12-03T13:19:51.364Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7c/98/e91cf858f203d86f4eccdf763dcf01cf03f1dae80c3750f7e635bfa206b6/opentelemetry_sdk-1.39.1-py3-none-any.whl", hash = "sha256:4d5482c478513ecb0a5d938dcc61394e647066e0cc2676bee9f3af3f3f45f01c", size = 132565, upload-time = "2025-12-11T13:32:35.069Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry-semantic-conventions"
|
||||
version = "0.60b0"
|
||||
version = "0.60b1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "opentelemetry-api" },
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/71/0e/176a7844fe4e3cb5de604212094dffaed4e18b32f1c56b5258bcbcba85c2/opentelemetry_semantic_conventions-0.60b0.tar.gz", hash = "sha256:227d7aa73cbb8a2e418029d6b6465553aa01cf7e78ec9d0bc3255c7b3ac5bf8f", size = 137935, upload-time = "2025-12-03T13:20:12.395Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/91/df/553f93ed38bf22f4b999d9be9c185adb558982214f33eae539d3b5cd0858/opentelemetry_semantic_conventions-0.60b1.tar.gz", hash = "sha256:87c228b5a0669b748c76d76df6c364c369c28f1c465e50f661e39737e84bc953", size = 137935, upload-time = "2025-12-11T13:32:50.487Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/d0/56/af0306666f91bae47db14d620775604688361f0f76a872e0005277311131/opentelemetry_semantic_conventions-0.60b0-py3-none-any.whl", hash = "sha256:069530852691136018087b52688857d97bba61cd641d0f8628d2d92788c4f78a", size = 219981, upload-time = "2025-12-03T13:19:53.585Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7a/5e/5958555e09635d09b75de3c4f8b9cae7335ca545d77392ffe7331534c402/opentelemetry_semantic_conventions-0.60b1-py3-none-any.whl", hash = "sha256:9fa8c8b0c110da289809292b0591220d3a7b53c1526a23021e977d68597893fb", size = 219982, upload-time = "2025-12-11T13:32:36.955Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry-util-http"
|
||||
version = "0.60b0"
|
||||
version = "0.60b1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/38/0d/786a713445cf338131fef3a84fab1378e4b2ef3c3ea348eeb0c915eb804a/opentelemetry_util_http-0.60b0.tar.gz", hash = "sha256:e42b7bb49bba43b6f34390327d97e5016eb1c47949ceaf37c4795472a4e3a82d", size = 10576, upload-time = "2025-12-03T13:22:41.224Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/50/fc/c47bb04a1d8a941a4061307e1eddfa331ed4d0ab13d8a9781e6db256940a/opentelemetry_util_http-0.60b1.tar.gz", hash = "sha256:0d97152ca8c8a41ced7172d29d3622a219317f74ae6bb3027cfbdcf22c3cc0d6", size = 11053, upload-time = "2025-12-11T13:37:25.115Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/53/5d/a448862f6d10c95685ed0e703596b6bd1784074e7ad90bffdc550abb7b68/opentelemetry_util_http-0.60b0-py3-none-any.whl", hash = "sha256:4f366f1a48adb74ffa6f80aee26f96882e767e01b03cd1cfb948b6e1020341fe", size = 8742, upload-time = "2025-12-03T13:21:54.553Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/16/5c/d3f1733665f7cd582ef0842fb1d2ed0bc1fba10875160593342d22bba375/opentelemetry_util_http-0.60b1-py3-none-any.whl", hash = "sha256:66381ba28550c91bee14dcba8979ace443444af1ed609226634596b4b0faf199", size = 8947, upload-time = "2025-12-11T13:36:37.151Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3314,7 +3353,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "pytest-otel"
|
||||
version = "2.0.1"
|
||||
version = "2.0.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "opentelemetry-api" },
|
||||
@@ -3322,9 +3361,9 @@ dependencies = [
|
||||
{ name = "opentelemetry-sdk" },
|
||||
{ name = "pytest" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/ca/5e/771f8dbdf55ae57603d89bb26e046c13e8bee3492c74cb3afb6044166e9a/pytest_otel-2.0.1.tar.gz", hash = "sha256:3d529dc34105862cca39fd1258d00dd3d17f3b7d92ebf0953d55326bb017af3d", size = 17880, upload-time = "2025-12-08T14:56:40.393Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/98/14/e4f7cb90c4c93dbdd85034fd65e7a16da328bd71c6ca5270e4f8d37fa4ba/pytest_otel-2.0.3.tar.gz", hash = "sha256:782985fef50acc6922db5e69da4584b384d785c863a96ebc11eaf2a7fef5189c", size = 18587, upload-time = "2026-01-02T16:12:20.394Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/16/fa/0cfe23bac571f68b4f427ad7ae313ddd8de881ec2ff49dcea196a2a96592/pytest_otel-2.0.1-py2.py3-none-any.whl", hash = "sha256:501f36f02f55578ca34c3ccbe55e81cc1e14bb130da13f7cf0b6480d54a9e5db", size = 14530, upload-time = "2025-12-08T14:56:41.731Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ac/f8/d9b93b41b299c7e14b47887eb31fb4cb77d771c2e9f3e0bcac0750f62699/pytest_otel-2.0.3-py2.py3-none-any.whl", hash = "sha256:3c6c331e943609ad7df7c718714090dee91211340f133a8798217e72e3b2cd67", size = 14914, upload-time = "2026-01-02T16:12:21.946Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
Reference in New Issue
Block a user