Merge remote-tracking branch 'origin/master' into fix/309-embed-resilience
# Conflicts: # nextcloud_mcp_server/vector/processor.py
This commit is contained in:
@@ -6,6 +6,7 @@ import click
|
||||
import uvicorn
|
||||
|
||||
from nextcloud_mcp_server.config import (
|
||||
Settings,
|
||||
get_database_url,
|
||||
get_settings,
|
||||
is_ephemeral_token_db,
|
||||
@@ -17,7 +18,12 @@ from nextcloud_mcp_server.migrations import (
|
||||
show_migration_history,
|
||||
upgrade_database,
|
||||
)
|
||||
from nextcloud_mcp_server.observability import get_uvicorn_logging_config
|
||||
from nextcloud_mcp_server.observability import (
|
||||
get_uvicorn_logging_config,
|
||||
setup_logging,
|
||||
setup_metrics,
|
||||
setup_tracing,
|
||||
)
|
||||
from nextcloud_mcp_server.server import AVAILABLE_APPS
|
||||
|
||||
from .app import get_app
|
||||
@@ -284,6 +290,41 @@ def run(
|
||||
)
|
||||
|
||||
|
||||
def _init_worker_observability(settings: Settings) -> None:
|
||||
"""Configure logging, metrics, and tracing for the standalone ingest worker."""
|
||||
# Mirrors app.py's lifespan bootstrap; without it the worker's astrolabe_*
|
||||
# metrics and document_processor.parse spans are invisible in external mode.
|
||||
# Structured logging first, so every subsequent startup line is JSON like
|
||||
# the API's — the worker entrypoint never went through uvicorn's log_config.
|
||||
setup_logging(
|
||||
log_format=settings.log_format,
|
||||
log_level=settings.log_level,
|
||||
include_trace_context=settings.log_include_trace_context,
|
||||
)
|
||||
|
||||
if settings.metrics_enabled:
|
||||
setup_metrics(port=settings.metrics_port)
|
||||
logger.info(
|
||||
"Prometheus metrics enabled on dedicated port %s", settings.metrics_port
|
||||
)
|
||||
|
||||
if settings.otel_exporter_otlp_endpoint:
|
||||
setup_tracing(
|
||||
service_name=settings.otel_service_name,
|
||||
otlp_endpoint=settings.otel_exporter_otlp_endpoint,
|
||||
otlp_verify_ssl=settings.otel_exporter_verify_ssl,
|
||||
sampling_rate=settings.otel_traces_sampler_arg,
|
||||
)
|
||||
logger.info(
|
||||
"OpenTelemetry tracing enabled (endpoint: %s)",
|
||||
settings.otel_exporter_otlp_endpoint,
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"OpenTelemetry tracing disabled (set OTEL_EXPORTER_OTLP_ENDPOINT to enable)"
|
||||
)
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option(
|
||||
"--concurrency",
|
||||
@@ -319,6 +360,11 @@ def worker(concurrency: int | None):
|
||||
f"resolved INGEST_QUEUE={settings.ingest_queue!r}"
|
||||
)
|
||||
|
||||
# Initialize observability here, not in a lifespan — the worker never runs
|
||||
# uvicorn, so it skips app.py's bootstrap (the WHY lives in the helper's
|
||||
# docstring). Done after the queue check so a misconfig fails fast.
|
||||
_init_worker_observability(settings)
|
||||
|
||||
from nextcloud_mcp_server.vector.queue.procrastinate import ( # noqa: PLC0415
|
||||
INGEST_QUEUE_NAME,
|
||||
apply_ingest_queue_schema,
|
||||
|
||||
@@ -5,7 +5,7 @@ import mimetypes
|
||||
import xml.etree.ElementTree as ET
|
||||
from email.utils import parsedate_to_datetime
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from urllib.parse import unquote
|
||||
from urllib.parse import quote, unquote
|
||||
from xml.sax.saxutils import escape as xml_escape
|
||||
|
||||
from httpx import HTTPStatusError
|
||||
@@ -26,11 +26,42 @@ WEBDAV_SEARCH_PAGE_SIZE = 500
|
||||
WEBDAV_SEARCH_MAX_RESULTS = 50000
|
||||
|
||||
|
||||
def _encode_dav_path(path: str) -> str:
|
||||
"""Percent-encode a *decoded* DAV path for use in a request URL/header.
|
||||
|
||||
Paths flow through this client already URL-decoded (e.g. ``unquote`` on the
|
||||
``<d:href>`` of a PROPFIND/REPORT response, or raw user-supplied paths from
|
||||
MCP tools), so characters like ``#``, ``,`` and spaces reach httpx verbatim.
|
||||
An unencoded ``#`` is parsed as a URL fragment and silently truncates the
|
||||
request path → spurious 404 on otherwise-valid files (issue: OHR-Bench
|
||||
ingest, card 309). ``quote`` with ``safe="/"`` encodes the unsafe characters
|
||||
while preserving the path separators; ASCII-only paths are unchanged.
|
||||
|
||||
Encode exactly once: the input is decoded, so a literal ``%`` becomes
|
||||
``%25`` (correct) rather than being mistaken for an existing escape.
|
||||
"""
|
||||
return quote(path, safe="/")
|
||||
|
||||
|
||||
class WebDAVClient(BaseNextcloudClient):
|
||||
"""Client for Nextcloud WebDAV operations."""
|
||||
|
||||
app_name = "webdav"
|
||||
|
||||
def _webdav_path(self, path: str) -> str:
|
||||
"""Build the request path for ``path`` under the user's DAV root.
|
||||
|
||||
Percent-encodes the caller-supplied portion (see ``_encode_dav_path``)
|
||||
so names with ``#``, commas, or spaces don't truncate/404; the base
|
||||
``/remote.php/dav/files/<user>`` segment is left as-is.
|
||||
|
||||
Precondition: ``path`` is a **decoded** path (the convention everywhere
|
||||
in this client — PROPFIND/REPORT hrefs are ``unquote``d before storage,
|
||||
and MCP-tool inputs are raw). It is encoded exactly once, so passing an
|
||||
already-encoded path would double-encode it (``%20`` → ``%2520``).
|
||||
"""
|
||||
return f"{self._get_webdav_base_path()}/{_encode_dav_path(path.lstrip('/'))}"
|
||||
|
||||
async def delete_resource(self, path: str) -> Dict[str, Any]:
|
||||
"""Delete a resource (file or directory) via WebDAV DELETE."""
|
||||
# Ensure path ends with a slash if it's a directory
|
||||
@@ -39,7 +70,7 @@ class WebDAVClient(BaseNextcloudClient):
|
||||
else:
|
||||
path_with_slash = path
|
||||
|
||||
webdav_path = f"{self._get_webdav_base_path()}/{path_with_slash.lstrip('/')}"
|
||||
webdav_path = self._webdav_path(path_with_slash)
|
||||
logger.debug("Deleting WebDAV resource: %s", webdav_path)
|
||||
|
||||
headers = {"OCS-APIRequest": "true"}
|
||||
@@ -124,15 +155,15 @@ class WebDAVClient(BaseNextcloudClient):
|
||||
mime_type: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Add/Update an attachment to a note via WebDAV PUT."""
|
||||
# Construct paths based on provided category
|
||||
webdav_base = self._get_webdav_base_path()
|
||||
# Construct paths based on provided category. Encode via _webdav_path so
|
||||
# categories/filenames with '#', commas or spaces don't truncate/404.
|
||||
category_path_part = f"{category}/" if category else ""
|
||||
attachment_dir_segment = f".attachments.{note_id}"
|
||||
parent_dir_webdav_rel_path = (
|
||||
f"Notes/{category_path_part}{attachment_dir_segment}"
|
||||
)
|
||||
parent_dir_path = f"{webdav_base}/{parent_dir_webdav_rel_path}"
|
||||
attachment_path = f"{parent_dir_path}/{filename}"
|
||||
parent_dir_path = self._webdav_path(parent_dir_webdav_rel_path)
|
||||
attachment_path = self._webdav_path(f"{parent_dir_webdav_rel_path}/{filename}")
|
||||
|
||||
logger.debug("Uploading attachment '%s' for note %s", filename, note_id)
|
||||
|
||||
@@ -144,7 +175,7 @@ class WebDAVClient(BaseNextcloudClient):
|
||||
headers = {"Content-Type": mime_type, "OCS-APIRequest": "true"}
|
||||
try:
|
||||
# First check if we can access WebDAV at all
|
||||
notes_dir_path = f"{webdav_base}/Notes"
|
||||
notes_dir_path = self._webdav_path("Notes")
|
||||
propfind_headers = {"Depth": "0", "OCS-APIRequest": "true"}
|
||||
notes_dir_response = await self._make_request(
|
||||
"PROPFIND", notes_dir_path, headers=propfind_headers
|
||||
@@ -209,10 +240,11 @@ class WebDAVClient(BaseNextcloudClient):
|
||||
self, note_id: int, filename: str, category: Optional[str] = None
|
||||
) -> Tuple[bytes, str]:
|
||||
"""Fetch a specific attachment from a note via WebDAV GET."""
|
||||
webdav_base = self._get_webdav_base_path()
|
||||
category_path_part = f"{category}/" if category else ""
|
||||
attachment_dir_segment = f".attachments.{note_id}"
|
||||
attachment_path = f"{webdav_base}/Notes/{category_path_part}{attachment_dir_segment}/{filename}"
|
||||
attachment_path = self._webdav_path(
|
||||
f"Notes/{category_path_part}{attachment_dir_segment}/{filename}"
|
||||
)
|
||||
|
||||
logger.debug("Fetching attachment '%s' for note %s", filename, note_id)
|
||||
|
||||
@@ -252,7 +284,7 @@ class WebDAVClient(BaseNextcloudClient):
|
||||
|
||||
async def list_directory(self, path: str = "") -> List[Dict[str, Any]]:
|
||||
"""List files and directories in the specified path via WebDAV PROPFIND."""
|
||||
webdav_path = f"{self._get_webdav_base_path()}/{path.lstrip('/')}"
|
||||
webdav_path = self._webdav_path(path)
|
||||
if not webdav_path.endswith("/"):
|
||||
webdav_path += "/"
|
||||
|
||||
@@ -352,7 +384,7 @@ class WebDAVClient(BaseNextcloudClient):
|
||||
|
||||
async def read_file(self, path: str) -> Tuple[bytes, str]:
|
||||
"""Read a file's content via WebDAV GET."""
|
||||
webdav_path = f"{self._get_webdav_base_path()}/{path.lstrip('/')}"
|
||||
webdav_path = self._webdav_path(path)
|
||||
|
||||
logger.debug("Reading file: %s", path)
|
||||
|
||||
@@ -379,7 +411,7 @@ class WebDAVClient(BaseNextcloudClient):
|
||||
self, path: str, content: bytes, content_type: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""Write content to a file via WebDAV PUT."""
|
||||
webdav_path = f"{self._get_webdav_base_path()}/{path.lstrip('/')}"
|
||||
webdav_path = self._webdav_path(path)
|
||||
|
||||
logger.debug("Writing file: %s", path)
|
||||
|
||||
@@ -410,7 +442,7 @@ class WebDAVClient(BaseNextcloudClient):
|
||||
self, path: str, recursive: bool = False
|
||||
) -> Dict[str, Any]:
|
||||
"""Create a directory via WebDAV MKCOL."""
|
||||
webdav_path = f"{self._get_webdav_base_path()}/{path.lstrip('/')}"
|
||||
webdav_path = self._webdav_path(path)
|
||||
if not webdav_path.endswith("/"):
|
||||
webdav_path += "/"
|
||||
|
||||
@@ -468,10 +500,8 @@ class WebDAVClient(BaseNextcloudClient):
|
||||
Returns:
|
||||
Dict with status_code and optional message
|
||||
"""
|
||||
source_webdav_path = f"{self._get_webdav_base_path()}/{source_path.lstrip('/')}"
|
||||
destination_webdav_path = (
|
||||
f"{self._get_webdav_base_path()}/{destination_path.lstrip('/')}"
|
||||
)
|
||||
source_webdav_path = self._webdav_path(source_path)
|
||||
destination_webdav_path = self._webdav_path(destination_path)
|
||||
|
||||
# Ensure paths have consistent trailing slashes for directories
|
||||
if source_path.endswith("/") and not destination_path.endswith("/"):
|
||||
@@ -552,10 +582,8 @@ class WebDAVClient(BaseNextcloudClient):
|
||||
Returns:
|
||||
Dict with status_code and optional message
|
||||
"""
|
||||
source_webdav_path = f"{self._get_webdav_base_path()}/{source_path.lstrip('/')}"
|
||||
destination_webdav_path = (
|
||||
f"{self._get_webdav_base_path()}/{destination_path.lstrip('/')}"
|
||||
)
|
||||
source_webdav_path = self._webdav_path(source_path)
|
||||
destination_webdav_path = self._webdav_path(destination_path)
|
||||
|
||||
# Ensure paths have consistent trailing slashes for directories
|
||||
if source_path.endswith("/") and not destination_path.endswith("/"):
|
||||
@@ -1678,7 +1706,7 @@ class WebDAVClient(BaseNextcloudClient):
|
||||
distinguish a definitive absence (HTTP 404) from a
|
||||
brittle response (None).
|
||||
"""
|
||||
webdav_path = f"{self._get_webdav_base_path()}/{path.lstrip('/')}"
|
||||
webdav_path = self._webdav_path(path)
|
||||
|
||||
propfind_body = """<?xml version="1.0"?>
|
||||
<d:propfind xmlns:d="DAV:" xmlns:oc="http://owncloud.org/ns">
|
||||
|
||||
@@ -146,6 +146,10 @@ _DEFAULTS: dict[str, Any] = {
|
||||
"document_pdf_graphics_limit": 1000,
|
||||
"document_parse_timeout_seconds": 120.0,
|
||||
"document_parse_mem_limit_mb": 1536,
|
||||
# Pre-parse size cap (MB): PDFs larger than this fail fast with reason
|
||||
# "oversize" instead of burning the OCR timeout to 0 chars on a pathological
|
||||
# file. 0 disables the guard.
|
||||
"document_max_pdf_size_mb": 50.0,
|
||||
# Tier-0 classifier (records classification metrics on the tiered path)
|
||||
"document_classify_enabled": True,
|
||||
# Tiered PDF pipeline: pypdfium2 is the default/only hot-path extractor;
|
||||
@@ -168,6 +172,10 @@ _DEFAULTS: dict[str, Any] = {
|
||||
"document_ocr_page_fraction": 0.5,
|
||||
"document_ocr_min_page_chars": 16,
|
||||
"document_ocr_detect_scanned": True,
|
||||
# OCR backend request timeout (seconds). Slow scanned newspapers can take
|
||||
# 20-60s; raise/lower per tenant. Configurable so a tenant isn't stuck with
|
||||
# the 180s default when its gateway has its own shorter ceiling.
|
||||
"document_ocr_timeout_seconds": 180.0,
|
||||
# Observability
|
||||
"metrics_enabled": True,
|
||||
"metrics_port": 9090,
|
||||
@@ -319,7 +327,10 @@ _dynaconf = Dynaconf(
|
||||
Validator("VERIFICATION_CONCURRENCY", gte=1),
|
||||
Validator("DOCUMENT_CHUNK_SIZE", gte=1),
|
||||
Validator("DOCUMENT_PARSE_TIMEOUT_SECONDS", gte=1),
|
||||
Validator("DOCUMENT_OCR_TIMEOUT_SECONDS", gte=1),
|
||||
Validator("DOCUMENT_PARSE_MEM_LIMIT_MB", gte=128),
|
||||
# 0 disables the pre-parse PDF size cap; otherwise it must be positive.
|
||||
Validator("DOCUMENT_MAX_PDF_SIZE_MB", gte=0),
|
||||
# >=1: pymupdf4llm treats graphics_limit=0 as "no cap", which would
|
||||
# re-expose the OOM this guards against.
|
||||
Validator("DOCUMENT_PDF_GRAPHICS_LIMIT", gte=1),
|
||||
@@ -782,6 +793,11 @@ class Settings:
|
||||
# float so a fractional DOCUMENT_PARSE_TIMEOUT_SECONDS is honoured, matching
|
||||
# anyio.move_on_after's float seconds.
|
||||
document_parse_timeout_seconds: float = 120.0
|
||||
# Pre-parse PDF size cap (MB). A PDF larger than this fails fast with
|
||||
# parse_failed_reason="oversize" (placeholder marked "failed") rather than
|
||||
# being handed to the fast/OCR tiers, where a pathological large file burns
|
||||
# the OCR timeout for 0 chars. 0 disables the guard.
|
||||
document_max_pdf_size_mb: float = 50.0
|
||||
# RLIMIT_AS in the parse subprocess (below the pod limit). Applied once per
|
||||
# worker for its lifetime, so changing it needs a pod restart.
|
||||
document_parse_mem_limit_mb: int = 1536
|
||||
@@ -801,6 +817,10 @@ class Settings:
|
||||
# gateway routes on the "<provider>/" prefix; the direct mistral backend
|
||||
# strips it.
|
||||
document_ocr_model: str = "mistral/mistral-ocr-latest"
|
||||
# OCR backend HTTP request timeout (seconds). float for parity with the
|
||||
# parse timeout / httpx.Timeout; per-tenant tunable so a gateway with a
|
||||
# shorter ceiling isn't masked by the 180s default.
|
||||
document_ocr_timeout_seconds: float = 180.0
|
||||
# OCR escalation triggers (tier-0), per-tenant tunable. A page is OCR-worthy
|
||||
# if near-empty (< min_page_chars) OR low text-quality (< min_text_quality)
|
||||
# OR (when detect_scanned, image-analysis only runs when OCR is enabled)
|
||||
@@ -1431,12 +1451,14 @@ def get_settings() -> Settings:
|
||||
"document_chunk_page_aware": "DOCUMENT_CHUNK_PAGE_AWARE",
|
||||
"document_pdf_graphics_limit": "DOCUMENT_PDF_GRAPHICS_LIMIT",
|
||||
"document_parse_timeout_seconds": "DOCUMENT_PARSE_TIMEOUT_SECONDS",
|
||||
"document_max_pdf_size_mb": "DOCUMENT_MAX_PDF_SIZE_MB",
|
||||
"document_parse_mem_limit_mb": "DOCUMENT_PARSE_MEM_LIMIT_MB",
|
||||
"document_classify_enabled": "DOCUMENT_CLASSIFY_ENABLED",
|
||||
"document_tier1_engine": "DOCUMENT_TIER1_ENGINE",
|
||||
"document_ocr_enabled": "DOCUMENT_OCR_ENABLED",
|
||||
"document_ocr_provider": "DOCUMENT_OCR_PROVIDER",
|
||||
"document_ocr_model": "DOCUMENT_OCR_MODEL",
|
||||
"document_ocr_timeout_seconds": "DOCUMENT_OCR_TIMEOUT_SECONDS",
|
||||
"document_ocr_min_text_quality": "DOCUMENT_OCR_MIN_TEXT_QUALITY",
|
||||
"document_ocr_page_fraction": "DOCUMENT_OCR_PAGE_FRACTION",
|
||||
"document_ocr_min_page_chars": "DOCUMENT_OCR_MIN_PAGE_CHARS",
|
||||
|
||||
@@ -31,7 +31,9 @@ from .base import DocumentProcessor, ProcessingResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_OCR_TIMEOUT_SECONDS = 180.0
|
||||
# Connect timeout for the OCR backend request. The overall (read) timeout is
|
||||
# configurable via DOCUMENT_OCR_TIMEOUT_SECONDS and resolved per call.
|
||||
_OCR_CONNECT_TIMEOUT_SECONDS = 10.0
|
||||
|
||||
|
||||
def _pages_to_text(
|
||||
@@ -93,8 +95,12 @@ class _GatewayOcrBackend(_OcrBackend):
|
||||
"document_b64": base64.b64encode(content).decode("ascii"),
|
||||
"mime_type": mime_type,
|
||||
}
|
||||
# Resolved per call (get_settings builds fresh) so test monkeypatching is
|
||||
# honoured; a live tenant change still needs a restart because the backend
|
||||
# instance itself is cached for the pod's lifetime.
|
||||
ocr_timeout = get_settings().document_ocr_timeout_seconds
|
||||
async with httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(_OCR_TIMEOUT_SECONDS, connect=10.0)
|
||||
timeout=httpx.Timeout(ocr_timeout, connect=_OCR_CONNECT_TIMEOUT_SECONDS)
|
||||
) as client:
|
||||
resp = await client.post(self._url, json=payload, headers=headers)
|
||||
resp.raise_for_status()
|
||||
@@ -120,10 +126,17 @@ class _MistralOcrBackend(_OcrBackend):
|
||||
data_url = (
|
||||
f"data:{mime_type};base64,{base64.b64encode(content).decode('ascii')}"
|
||||
)
|
||||
resp = await self._client.ocr.process_async(
|
||||
model=self._model,
|
||||
document={"type": "document_url", "document_url": data_url},
|
||||
)
|
||||
# Apply DOCUMENT_OCR_TIMEOUT_SECONDS uniformly with the gateway backend.
|
||||
# The Mistral SDK manages its own httpx client, so wrap the call in an
|
||||
# anyio cancel-scope timeout rather than threading a per-request timeout
|
||||
# through the SDK; on expiry this raises TimeoutError, which the
|
||||
# OcrProcessor turns into a clean parse failure.
|
||||
ocr_timeout = get_settings().document_ocr_timeout_seconds
|
||||
with anyio.fail_after(ocr_timeout):
|
||||
resp = await self._client.ocr.process_async(
|
||||
model=self._model,
|
||||
document={"type": "document_url", "document_url": data_url},
|
||||
)
|
||||
pages = [(p.index, p.markdown or "") for p in (resp.pages or [])]
|
||||
return _pages_to_text(pages)
|
||||
|
||||
@@ -251,6 +264,24 @@ class OcrProcessor(DocumentProcessor):
|
||||
text, boundaries = await backend.ocr(
|
||||
content, content_type.split(";")[0].strip().lower()
|
||||
)
|
||||
except (TimeoutError, httpx.TimeoutException):
|
||||
# Two timeout shapes reach here: the Mistral backend's
|
||||
# anyio.fail_after raises the builtin TimeoutError, while the gateway
|
||||
# backend's httpx.Timeout raises httpx.ReadTimeout (a
|
||||
# httpx.TimeoutException, NOT a TimeoutError). Catch both so a
|
||||
# too-low DOCUMENT_OCR_TIMEOUT_SECONDS lands in its own reason bucket
|
||||
# rather than being conflated with provider errors.
|
||||
timeout = settings.document_ocr_timeout_seconds
|
||||
logger.warning(
|
||||
"OCR timed out for %s after %.1fs", filename or "<bytes>", timeout
|
||||
)
|
||||
return ProcessingResult(
|
||||
text="",
|
||||
metadata={"parse_failed_reason": "timeout"},
|
||||
processor=self.name,
|
||||
success=False,
|
||||
error=f"OCR timed out after {timeout:.1f}s",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("OCR failed for %s: %s", filename or "<bytes>", e)
|
||||
return ProcessingResult(
|
||||
|
||||
@@ -202,6 +202,33 @@ class ProcessorRegistry:
|
||||
"""
|
||||
settings = get_settings()
|
||||
|
||||
# Pre-parse size guard: a pathologically large PDF (e.g. a 42 MB scanned
|
||||
# DUDE) burns the OCR timeout for 0 chars. Fail fast with an explicit
|
||||
# reason so the caller marks the placeholder "failed" instead of
|
||||
# retrying. 0 disables the cap. This lives on the auto-tiered path only:
|
||||
# an explicit processor_name="ocr" override (registry.process) bypasses
|
||||
# _process_pdf entirely and is intentionally not size-gated (power-user
|
||||
# escape hatch). Returning here also skips _run_processor, so the
|
||||
# rejection is counted on astrolabe_document_parse_failed_total{oversize}
|
||||
# (via vector/processor.py) but deliberately not on the parse-duration
|
||||
# histogram -- there is no parse to time.
|
||||
max_pdf_mb = settings.document_max_pdf_size_mb
|
||||
if max_pdf_mb > 0 and len(content) > max_pdf_mb * 1024 * 1024:
|
||||
size_mb = len(content) / (1024 * 1024)
|
||||
logger.warning(
|
||||
"PDF %s is %.1f MB (> %.1f MB cap); failing fast as oversize",
|
||||
filename or "<bytes>",
|
||||
size_mb,
|
||||
max_pdf_mb,
|
||||
)
|
||||
return ProcessingResult(
|
||||
text="",
|
||||
metadata={"parse_failed_reason": "oversize"},
|
||||
processor="size_guard",
|
||||
success=False,
|
||||
error=(f"PDF exceeds size cap: {size_mb:.1f} MB > {max_pdf_mb:.1f} MB"),
|
||||
)
|
||||
|
||||
if settings.document_tier1_engine == "pymupdf":
|
||||
processor = self._pdf_processor_for_tier("structured")
|
||||
if processor is None:
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Error-formatting helpers for the vector-sync pipeline.
|
||||
|
||||
Vector-sync work runs inside anyio task groups, so a failure in a child task
|
||||
surfaces as a ``BaseExceptionGroup`` whose default ``str()`` is the useless
|
||||
``"unhandled errors in a TaskGroup (N sub-exception)"`` -- it hides the real
|
||||
``ConnectError`` / ``APIConnectionError`` that operators need to triage embed
|
||||
drops (card 309). ``format_exception_group`` flattens the group to the leaf
|
||||
exceptions so log lines name the actual cause; pair it with ``exc_info=True`` to
|
||||
keep the full traceback.
|
||||
"""
|
||||
|
||||
|
||||
def format_exception_group(exc: BaseException) -> str:
|
||||
"""Return a concise, leaf-naming string for ``exc``.
|
||||
|
||||
For a (possibly nested) ``BaseExceptionGroup`` this joins the ``repr`` of
|
||||
each leaf exception; for an ordinary exception it returns its ``repr``. The
|
||||
result is meant for the human-readable portion of a log message, not for
|
||||
parsing.
|
||||
"""
|
||||
if not isinstance(exc, BaseExceptionGroup):
|
||||
return repr(exc)
|
||||
leaves = _flatten(exc)
|
||||
noun = "sub-exception" if len(leaves) == 1 else "sub-exceptions"
|
||||
return f"{len(leaves)} {noun}: " + "; ".join(repr(e) for e in leaves)
|
||||
|
||||
|
||||
def _flatten(exc: BaseException) -> list[BaseException]:
|
||||
"""Depth-first list of the leaf exceptions within ``exc``."""
|
||||
if isinstance(exc, BaseExceptionGroup):
|
||||
leaves: list[BaseException] = []
|
||||
for sub in exc.exceptions:
|
||||
leaves.extend(_flatten(sub))
|
||||
return leaves
|
||||
return [exc]
|
||||
@@ -30,6 +30,7 @@ from httpx import BasicAuth, HTTPStatusError
|
||||
from nextcloud_mcp_server.auth.storage import RefreshTokenStorage
|
||||
from nextcloud_mcp_server.client import NextcloudClient
|
||||
from nextcloud_mcp_server.config import get_settings
|
||||
from nextcloud_mcp_server.vector._errors import format_exception_group
|
||||
from nextcloud_mcp_server.vector.processor import process_document
|
||||
from nextcloud_mcp_server.vector.queue.ports import TaskProducer
|
||||
from nextcloud_mcp_server.vector.scanner import DocumentTask, scan_user_documents
|
||||
@@ -257,7 +258,7 @@ async def user_scanner_task(
|
||||
logger.error(
|
||||
"[BasicAuth] Scanner error for %s: %s (%s/%s)",
|
||||
user_id,
|
||||
e,
|
||||
format_exception_group(e),
|
||||
consecutive_errors,
|
||||
max_consecutive_errors,
|
||||
exc_info=True,
|
||||
@@ -331,7 +332,7 @@ async def multi_user_processor_task(
|
||||
break
|
||||
|
||||
except NotProvisionedError:
|
||||
if doc_task:
|
||||
if doc_task is not None:
|
||||
logger.warning(
|
||||
"[BasicAuth] User %s not provisioned, skipping %s_%s",
|
||||
doc_task.user_id,
|
||||
@@ -341,18 +342,21 @@ async def multi_user_processor_task(
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
if doc_task:
|
||||
if doc_task is not None:
|
||||
logger.error(
|
||||
"[BasicAuth] Processor %s error processing %s_%s: %s",
|
||||
worker_id,
|
||||
doc_task.doc_type,
|
||||
doc_task.doc_id,
|
||||
e,
|
||||
format_exception_group(e),
|
||||
exc_info=True,
|
||||
)
|
||||
else:
|
||||
logger.error(
|
||||
"[BasicAuth] Processor %s error: %s", worker_id, e, exc_info=True
|
||||
"[BasicAuth] Processor %s error: %s",
|
||||
worker_id,
|
||||
format_exception_group(e),
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
finally:
|
||||
@@ -481,7 +485,11 @@ async def user_manager_task(
|
||||
logger.info("[BasicAuth] Stopped %s scanner(s)", len(revoked_users))
|
||||
|
||||
except Exception as e:
|
||||
logger.error("[BasicAuth] User manager error: %s", e, exc_info=True)
|
||||
logger.error(
|
||||
"[BasicAuth] User manager error: %s",
|
||||
format_exception_group(e),
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
# Sleep until next poll
|
||||
try:
|
||||
|
||||
@@ -33,6 +33,7 @@ from nextcloud_mcp_server.observability.tracing import trace_operation
|
||||
from nextcloud_mcp_server.search.pdf_highlighter import PDFHighlighter
|
||||
from nextcloud_mcp_server.usage import UsageEventStore
|
||||
from nextcloud_mcp_server.vector import payload_keys
|
||||
from nextcloud_mcp_server.vector._errors import format_exception_group
|
||||
from nextcloud_mcp_server.vector.document_chunker import (
|
||||
DocumentChunker,
|
||||
PageAwareChunker,
|
||||
@@ -284,6 +285,11 @@ async def processor_task(
|
||||
# Signal that the task has started and is ready
|
||||
task_status.started()
|
||||
|
||||
# Initialised before the loop so the broad except handler below can't hit an
|
||||
# unbound name if receive() itself raises a non-TimeoutError/EndOfStream
|
||||
# exception on the very first iteration (mirrors multi_user_processor_task).
|
||||
doc_task: DocumentTask | None = None
|
||||
|
||||
while not shutdown_event.is_set():
|
||||
try:
|
||||
# Get document with timeout (allows checking shutdown)
|
||||
@@ -313,14 +319,22 @@ async def processor_task(
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Processor %s error processing %s_%s: %s",
|
||||
worker_id,
|
||||
doc_task.doc_type,
|
||||
doc_task.doc_id,
|
||||
e,
|
||||
exc_info=True,
|
||||
)
|
||||
if doc_task is not None:
|
||||
logger.error(
|
||||
"Processor %s error processing %s_%s: %s",
|
||||
worker_id,
|
||||
doc_task.doc_type,
|
||||
doc_task.doc_id,
|
||||
format_exception_group(e),
|
||||
exc_info=True,
|
||||
)
|
||||
else:
|
||||
logger.error(
|
||||
"Processor %s error: %s",
|
||||
worker_id,
|
||||
format_exception_group(e),
|
||||
exc_info=True,
|
||||
)
|
||||
# Continue to next document (no task_done() needed with streams)
|
||||
|
||||
logger.info("Processor %s stopped", worker_id)
|
||||
@@ -499,7 +513,7 @@ async def process_document(
|
||||
max_retries,
|
||||
doc_task.doc_type,
|
||||
doc_task.doc_id,
|
||||
e,
|
||||
format_exception_group(e),
|
||||
extra={
|
||||
"doc_id": doc_task.doc_id,
|
||||
"doc_type": doc_task.doc_type,
|
||||
@@ -518,7 +532,7 @@ async def process_document(
|
||||
doc_task.doc_id,
|
||||
max_retries,
|
||||
reason,
|
||||
e,
|
||||
format_exception_group(e),
|
||||
extra={
|
||||
"doc_id": doc_task.doc_id,
|
||||
"doc_type": doc_task.doc_type,
|
||||
|
||||
Reference in New Issue
Block a user