admin/mcp-nextcloud
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #836 from cbcoutinho/feat/183-procrastinate-ingest-queue

Browse Source 
feat: replace NATS ingest with procrastinate Postgres queue (#183)
This commit is contained in:
Chris Coutinho
2026-06-03 23:44:06 +02:00
committed by GitHub
parent aa2f01cd4b 5affbbcaa6
commit 4e983e98f6
31 changed files with 1643 additions and 1154 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nextcloud_mcp_server/api/management.py
+23 -51
View File
@@ -293,52 +293,21 @@ async def get_vector_sync_status(request: Request) -> JSONResponse:
)
try:
# Bus status backend (INGEST_MODE=external): there is no in-process
# queue; pending/terminal state comes from the NATS status subscriber's
# store. indexed_documents stays the mode-independent Qdrant count.
if settings.status_backend == "bus":
store = getattr(request.app.state, "status_store", None)
indexed_count = 0
try:
qdrant_client = await get_qdrant_client()
count_result = await qdrant_client.count(
collection_name=settings.get_collection_name(),
count_filter=Filter(must=[get_placeholder_filter()]),
)
indexed_count = count_result.count
except Exception as e:
logger.warning("Failed to query Qdrant for indexed count: %s", e)
return JSONResponse(
{
"status": "idle",
"indexed_documents": indexed_count,
"pending_documents": 0,
"status_backend": "bus",
"recent_states": store.counts() if store is not None else {},
}
)
# Get document receive stream from app state (set by starlette_lifespan in app.py)
document_receive_stream = getattr(
request.app.state, "document_receive_stream", None
# Outstanding-work view depends on the queue backend (Deck #183):
# memory → stream buffer depth; postgres → procrastinate job counts.
from nextcloud_mcp_server.vector.ingest_status import ( # noqa: PLC0415
get_ingest_pending,
)
if document_receive_stream is None:
logger.debug("document_receive_stream not available in app state")
return JSONResponse(
{
"status": "unknown",
"indexed_documents": 0,
"pending_documents": 0,
"message": "Vector sync stream not initialized",
}
)
pending = await get_ingest_pending(
task_producer=getattr(request.app.state, "task_producer", None),
document_receive_stream=getattr(
request.app.state, "document_receive_stream", None
),
ingest_queue=settings.ingest_queue,
)
# Get pending count from stream statistics
stream_stats = document_receive_stream.statistics()
pending_count = stream_stats.current_buffer_used
# Get Qdrant client and query indexed count
# Get Qdrant client and query indexed count (backend-independent)
indexed_count = 0
try:
qdrant_client = await get_qdrant_client()
@@ -355,15 +324,18 @@ async def get_vector_sync_status(request: Request) -> JSONResponse:
# Continue with indexed_count = 0
# Determine status
status = "syncing" if pending_count > 0 else "idle"
status = "syncing" if pending.pending > 0 else "idle"
return JSONResponse(
{
"status": status,
"indexed_documents": indexed_count,
"pending_documents": pending_count,
}
)
body: dict[str, object] = {
"status": status,
"indexed_documents": indexed_count,
"pending_documents": pending.pending,
"ingest_queue": settings.ingest_queue,
}
if pending.job_counts is not None:
# Per-status breakdown (todo/doing/failed/…) on the postgres backend.
body["job_counts"] = pending.job_counts
return JSONResponse(body)
except Exception as e:
error_msg = _sanitize_error_for_client(e, "get_vector_sync_status")
nextcloud_mcp_server/app.py
+68 -105
View File
@@ -136,9 +136,8 @@ from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client
from nextcloud_mcp_server.vector.queue import (
MemoryTaskProducer,
TaskProducer,
build_external_producer,
build_producer,
)
from nextcloud_mcp_server.vector.queue.status import NatsStatusSubscriber, StatusStore
from nextcloud_mcp_server.vector.scanner import DocumentTask, scanner_task
from nextcloud_mcp_server.vector.webhook_receiver import handle_nextcloud_webhook
@@ -332,12 +331,10 @@ class VectorSyncState:
document_send_stream: MemoryObjectSendStream | None = None
document_receive_stream: MemoryObjectReceiveStream | None = None
# Ingest producer the scanner/webhook send to: the in-memory send stream
# (local mode) or the NATS bus producer (external mode). The webhook reads
# this; in local mode it is the same object as document_send_stream.
# (INGEST_QUEUE=memory) or the procrastinate producer (INGEST_QUEUE=postgres,
# Deck #183). The webhook reads this; in memory mode it is the same object as
# document_send_stream.
task_producer: "TaskProducer | None" = None
# Bus status store (STATUS_BACKEND=bus): populated by the NATS status
# subscriber, read by the vector-sync status endpoint. None in local mode.
status_store: "StatusStore | None" = None
shutdown_event: anyio.Event | None = None
scanner_wake_event: anyio.Event | None = None
# Long-lived task group used for fire-and-forget background work spawned
@@ -350,33 +347,6 @@ class VectorSyncState:
_vector_sync_state = VectorSyncState()
async def _build_status_subscriber(
settings: "Settings",
) -> "tuple[StatusStore | None, NatsStatusSubscriber | None]":
"""Build the bus status store + subscriber when STATUS_BACKEND=bus.
Returns ``(None, None)`` for local status (the status endpoint reads the
in-memory stream buffer instead). __post_init__ guarantees that bus status
only pairs with external ingest, so ingest_bus_url/tenant_id are set.
"""
if not (settings.ingest_mode == "external" and settings.status_backend == "bus"):
return None, None
# Defence-in-depth (robust under ``python -O``, which strips asserts):
# __post_init__ already guarantees these when status_backend == "bus".
if settings.ingest_bus_url is None or settings.tenant_id is None:
raise ValueError(
"STATUS_BACKEND=bus requires INGEST_BUS_URL and TENANT_ID "
"(guaranteed by Settings validation)"
)
store = StatusStore(max_size=settings.vector_sync_queue_max_size)
subscriber = await NatsStatusSubscriber.connect(
url=settings.ingest_bus_url,
tenant_id=settings.tenant_id,
store=store,
)
return store, subscriber
@dataclass
class AppContext:
"""Application context for BasicAuth mode."""
@@ -385,10 +355,18 @@ class AppContext:
storage: "RefreshTokenStorage | None" = None
document_send_stream: MemoryObjectSendStream | None = None
document_receive_stream: MemoryObjectReceiveStream | None = None
task_producer: "TaskProducer | None" = None
shutdown_event: anyio.Event | None = None
scanner_wake_event: anyio.Event | None = None
@property
def task_producer(self) -> "TaskProducer | None":
# Read dynamically from the module-level singleton (like
# eviction_task_group) rather than snapshotting at yield time — that way
# a session can't observe a stale ``None`` and the per-session yields
# can't forget to forward it (the bug this property replaces). The
# vector-sync status tool reads this for postgres-backend job counts.
return _vector_sync_state.task_producer
@property
def eviction_task_group(self) -> TaskGroup | None:
# Read dynamically from the module-level singleton instead of
@@ -411,10 +389,14 @@ class OAuthAppContext:
server_client_id: str | None = None # MCP server's OAuth client ID (static or DCR)
document_send_stream: MemoryObjectSendStream | None = None
document_receive_stream: MemoryObjectReceiveStream | None = None
task_producer: "TaskProducer | None" = None
shutdown_event: anyio.Event | None = None
scanner_wake_event: anyio.Event | None = None
@property
def task_producer(self) -> "TaskProducer | None":
# See AppContext.task_producer for rationale.
return _vector_sync_state.task_producer
@property
def eviction_task_group(self) -> TaskGroup | None:
# See AppContext.eviction_task_group for rationale.
@@ -636,8 +618,8 @@ async def app_lifespan_basic(server: FastMCP) -> AsyncIterator[AppContext]:
document_receive_stream=_vector_sync_state.document_receive_stream,
shutdown_event=_vector_sync_state.shutdown_event,
scanner_wake_event=_vector_sync_state.scanner_wake_event,
# eviction_task_group is exposed via @property (reads
# _vector_sync_state at access time, not snapshot).
# task_producer and eviction_task_group are exposed via @property
# (read _vector_sync_state at access time, not snapshot).
)
finally:
logger.info("Shutting down BasicAuth session")
@@ -1269,8 +1251,8 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
document_receive_stream=_vector_sync_state.document_receive_stream,
shutdown_event=_vector_sync_state.shutdown_event,
scanner_wake_event=_vector_sync_state.scanner_wake_event,
# eviction_task_group is exposed via @property (reads
# _vector_sync_state at access time, not snapshot).
# task_producer and eviction_task_group are exposed via
# @property (read _vector_sync_state at access time).
)
finally:
logger.info("Shutting down MCP server")
@@ -1681,39 +1663,38 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
# Orphan-sweep before scanner starts — card #101.
await _sweep_orphan_placeholders_if_enabled()
# Initialize shared state. INGEST_MODE selects the transport
# (design §10.1): local uses the in-memory anyio stream + the
# in-process processor pool; external publishes to NATS and runs no
# in-process consumer (the document-processor consumes).
external = settings.ingest_mode == "external"
# Initialize shared state. INGEST_QUEUE selects the transport
# (Deck #183): ``memory`` uses the in-process anyio stream + the
# in-process processor pool (SQLite/dev); ``postgres`` defers jobs to
# the per-tenant Postgres via procrastinate and runs no in-process
# consumer (the separate ``worker`` role drains the queue).
use_postgres = settings.ingest_queue == "postgres"
shutdown_event = anyio.Event()
scanner_wake_event = anyio.Event()
send_stream = None
receive_stream = None
task_producer: TaskProducer
if external:
task_producer = await build_external_producer(settings)
logger.info(
"Ingest mode external: publishing to %s", settings.ingest_bus_url
)
if use_postgres:
# Open the connector once (build_producer) and reuse it to create
# procrastinate's tables before the scanner can defer — a single
# open/close cycle, matching the worker command.
producer = await build_producer(settings)
await producer.ensure_schema()
task_producer = producer
logger.info("Ingest queue: postgres (procrastinate); worker drains it")
else:
send_stream, receive_stream = anyio.create_memory_object_stream[
DocumentTask
](max_buffer_size=settings.vector_sync_queue_max_size)
task_producer = MemoryTaskProducer(send_stream)
# Bus status backend: subscribe to mcp.document.* into a store the
# status endpoint reads (STATUS_BACKEND=bus; external mode only).
status_store, status_subscriber = await _build_status_subscriber(settings)
# Store in app state for access from routes (ADR-007). In external
# Store in app state for access from routes (ADR-007). In postgres
# mode there is no in-memory stream, so document_send/receive_stream
# stay None; task_producer is the bus producer.
# stay None; task_producer is the procrastinate producer.
app.state.document_send_stream = send_stream
app.state.document_receive_stream = receive_stream
app.state.task_producer = task_producer
app.state.status_store = status_store
app.state.shutdown_event = shutdown_event
app.state.scanner_wake_event = scanner_wake_event
@@ -1721,7 +1702,6 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
_vector_sync_state.document_send_stream = send_stream
_vector_sync_state.document_receive_stream = receive_stream
_vector_sync_state.task_producer = task_producer
_vector_sync_state.status_store = status_store
_vector_sync_state.shutdown_event = shutdown_event
_vector_sync_state.scanner_wake_event = scanner_wake_event
logger.info("Vector sync state stored in module singleton")
@@ -1733,7 +1713,6 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
browser_app.state.document_send_stream = send_stream
browser_app.state.document_receive_stream = receive_stream
browser_app.state.task_producer = task_producer
browser_app.state.status_store = status_store
browser_app.state.shutdown_event = shutdown_event
browser_app.state.scanner_wake_event = scanner_wake_event
logger.info("Vector sync state shared with browser_app for /app")
@@ -1751,9 +1730,9 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
username,
)
# The in-process processor pool runs only in local mode; in
# external mode the document-processor service is the consumer.
if not external:
# The in-process processor pool runs only in memory mode; in
# postgres mode the out-of-process worker is the consumer.
if not use_postgres:
assert receive_stream is not None
for i in range(settings.vector_sync_processor_workers):
await tg.start(
@@ -1765,10 +1744,6 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
username,
)
# Bus status subscriber (STATUS_BACKEND=bus).
if status_subscriber is not None:
await tg.start(status_subscriber.run, shutdown_event)
# Expose this long-lived task group to request-path code that
# wants to spawn background work (e.g. ADR-019 verify-on-read
# eviction). Eviction coroutines have their own try/except, so
@@ -1776,9 +1751,9 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
_vector_sync_state.eviction_task_group = tg
logger.info(
"Background sync tasks started: 1 scanner + %s processors (ingest=%s)",
0 if external else settings.vector_sync_processor_workers,
settings.ingest_mode,
"Background sync tasks started: 1 scanner + %s processors (queue=%s)",
0 if use_postgres else settings.vector_sync_processor_workers,
settings.ingest_queue,
)
# Run MCP session manager and yield
@@ -1791,12 +1766,10 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
shutdown_event.set()
# Request path must not spawn into a cancelling group.
_vector_sync_state.eviction_task_group = None
# Drain the shared bus connection (external mode only).
# Close the procrastinate connector pool (postgres mode).
_drain = getattr(task_producer, "drain", None)
if external and _drain is not None:
if use_postgres and _drain is not None:
await _drain()
if status_subscriber is not None:
await status_subscriber.aclose()
await client.close()
# TaskGroup automatically cancels all tasks on exit
@@ -1904,11 +1877,12 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
except Exception as e:
logger.warning("App password cleanup failed (non-fatal): %s", e)
# Initialize shared state. INGEST_MODE selects the transport
# (design §10.1): local uses the in-memory anyio stream + the
# in-process processor pool; external publishes to NATS and runs
# no in-process consumer (the document-processor consumes).
external = settings.ingest_mode == "external"
# Initialize shared state. INGEST_QUEUE selects the transport
# (Deck #183): ``memory`` uses the in-process anyio stream + the
# in-process processor pool; ``postgres`` defers jobs via
# procrastinate and runs no in-process consumer (the separate
# ``worker`` role drains the queue).
use_postgres = settings.ingest_queue == "postgres"
shutdown_event = anyio.Event()
scanner_wake_event = anyio.Event()
@@ -1918,11 +1892,15 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
send_stream = None
receive_stream = None
task_producer: TaskProducer
if external:
task_producer = await build_external_producer(settings)
if use_postgres:
# Single open/close cycle: build_producer opens the connector
# and ensure_schema reuses it to create procrastinate's tables
# before any scanner defers (matches the worker command).
producer = await build_producer(settings)
await producer.ensure_schema()
task_producer = producer
logger.info(
"Ingest mode external: publishing to %s",
settings.ingest_bus_url,
"Ingest queue: postgres (procrastinate); worker drains it"
)
else:
send_stream, receive_stream = anyio.create_memory_object_stream[
@@ -1930,17 +1908,10 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
](max_buffer_size=settings.vector_sync_queue_max_size)
task_producer = MemoryTaskProducer(send_stream)
# Bus status backend: subscribe to mcp.document.* into a store
# the status endpoint reads (STATUS_BACKEND=bus; external only).
status_store, status_subscriber = await _build_status_subscriber(
settings
)
# Store in app state for access from routes (ADR-007)
app.state.document_send_stream = send_stream
app.state.document_receive_stream = receive_stream
app.state.task_producer = task_producer
app.state.status_store = status_store
app.state.shutdown_event = shutdown_event
app.state.scanner_wake_event = scanner_wake_event
@@ -1948,7 +1919,6 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
_vector_sync_state.document_send_stream = send_stream
_vector_sync_state.document_receive_stream = receive_stream
_vector_sync_state.task_producer = task_producer
_vector_sync_state.status_store = status_store
_vector_sync_state.shutdown_event = shutdown_event
_vector_sync_state.scanner_wake_event = scanner_wake_event
logger.info("Vector sync state stored in module singleton")
@@ -1960,7 +1930,6 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
browser_app.state.document_send_stream = send_stream
browser_app.state.document_receive_stream = receive_stream
browser_app.state.task_producer = task_producer
browser_app.state.status_store = status_store
browser_app.state.shutdown_event = shutdown_event
browser_app.state.scanner_wake_event = scanner_wake_event
logger.info(
@@ -1991,9 +1960,9 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
tg,
)
# In-process processor pool runs only in local mode; in
# external mode the document-processor service consumes.
if not external:
# In-process processor pool runs only in memory mode; in
# postgres mode the out-of-process worker consumes.
if not use_postgres:
assert receive_stream is not None
for i in range(settings.vector_sync_processor_workers):
await tg.start(
@@ -2004,10 +1973,6 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
nextcloud_host_for_sync,
)
# Bus status subscriber (STATUS_BACKEND=bus).
if status_subscriber is not None:
await tg.start(status_subscriber.run, shutdown_event)
# Expose this long-lived task group to request-path code
# that wants to spawn background work (e.g. ADR-019
# verify-on-read eviction). Eviction coroutines have their
@@ -2015,9 +1980,9 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
_vector_sync_state.eviction_task_group = tg
logger.info(
"Background sync tasks started: 1 user manager + %s processors (ingest=%s)",
0 if external else settings.vector_sync_processor_workers,
settings.ingest_mode,
"Background sync tasks started: 1 user manager + %s processors (queue=%s)",
0 if use_postgres else settings.vector_sync_processor_workers,
settings.ingest_queue,
)
# Run MCP session manager and yield
@@ -2030,12 +1995,10 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
shutdown_event.set()
# Request path must not spawn into a cancelling group.
_vector_sync_state.eviction_task_group = None
# Drain the shared bus connection (external only).
# Close the procrastinate connector pool (postgres).
_drain = getattr(task_producer, "drain", None)
if external and _drain is not None:
if use_postgres and _drain is not None:
await _drain()
if status_subscriber is not None:
await status_subscriber.aclose()
# Close token broker HTTP client
if token_broker._http_client:
await token_broker._http_client.aclose()
nextcloud_mcp_server/auth/userinfo_routes.py
+14 -11
View File
@@ -115,17 +115,20 @@ async def _get_processing_status(request: Request) -> dict[str, Any] | None:
return None
try:
# Get document receive stream from app state
document_receive_stream = getattr(
request.app.state, "document_receive_stream", None
# Outstanding-work view depends on the queue backend (Deck #183):
# memory → stream buffer depth; postgres → procrastinate job counts (the
# in-memory stream is absent in postgres mode, so don't early-return on it).
from nextcloud_mcp_server.vector.ingest_status import ( # noqa: PLC0415
get_ingest_pending,
)
if document_receive_stream is None:
logger.debug("document_receive_stream not available in app state")
return None
# Get pending count from stream statistics
stats = document_receive_stream.statistics()
pending_count = stats.current_buffer_used
pending = await get_ingest_pending(
task_producer=getattr(request.app.state, "task_producer", None),
document_receive_stream=getattr(
request.app.state, "document_receive_stream", None
),
ingest_queue=settings.ingest_queue,
)
# Get Qdrant client and query indexed count
indexed_count = 0
@@ -147,11 +150,11 @@ async def _get_processing_status(request: Request) -> dict[str, Any] | None:
# Continue with indexed_count = 0
# Determine status
status = "syncing" if pending_count > 0 else "idle"
status = "syncing" if pending.pending > 0 else "idle"
return {
"indexed_count": indexed_count,
"pending_count": pending_count,
"pending_count": pending.pending,
"status": status,
}
nextcloud_mcp_server/canonical.py
+5 -6
View File
@@ -2,9 +2,9 @@
The Astrolabe Cloud decomposition (design §2.3) fixes a single canonical JSON
encoding so hashes computed here match those computed independently by the
external document-processor and embedding-gateway services. Any drift in
separators, key ordering, or unicode handling would break NATS dedup keys,
Qdrant point-ID idempotency, and ACL-hash compatibility.
external embedding-gateway service. Any drift in separators, key ordering, or
unicode handling would break Qdrant point-ID idempotency and ACL-hash
compatibility.
"""
from __future__ import annotations
@@ -17,9 +17,8 @@ def canonical_json(obj: Any) -> bytes:
"""Encode ``obj`` to canonical JSON bytes.
Deterministic across implementations: sorted keys, no inter-token
whitespace, non-ASCII preserved (UTF-8). Consumers: the NATS
``Nats-Msg-Id`` dedup header (vector/queue/nats.py), Qdrant point IDs
(vector/payload_keys.py), and ACL hashes (acl_hash.py).
whitespace, non-ASCII preserved (UTF-8). Consumers: Qdrant point IDs
(vector/payload_keys.py) and ACL hashes (acl_hash.py).
"""
return json.dumps(
obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False
nextcloud_mcp_server/cli.py
+101
View File
@@ -1,3 +1,4 @@
import logging
import os
from importlib.metadata import version
@@ -21,6 +22,8 @@ from nextcloud_mcp_server.server import AVAILABLE_APPS
from .app import get_app
logger = logging.getLogger(__name__)
@click.command()
@click.option(
@@ -281,6 +284,88 @@ def run(
)
@click.command()
@click.option(
"--concurrency",
"-c",
type=int,
default=None,
help="Max concurrent jobs. Defaults to VECTOR_SYNC_PROCESSOR_WORKERS.",
)
def worker(concurrency: int | None):
"""Run the ingest worker (Deck #183).
\b
Drains the per-tenant Postgres ingest queue (procrastinate): for each
deferred document it fetches the content as the owning user, parses, chunks,
embeds, and upserts into Qdrant. This is the scale-to-zero ``worker`` role of
the api/worker split; run it as a separate Deployment from the API pod.
\b
Requires INGEST_QUEUE=postgres (a PostgreSQL DATABASE_URL); procrastinate is
Postgres-only.
\b
Example:
$ export DATABASE_URL=postgresql+asyncpg://mcp:mcp@db/mcp
$ nextcloud-mcp-server worker -c 4
"""
import anyio # noqa: PLC0415
settings = get_settings()
if settings.ingest_queue != "postgres":
raise click.ClickException(
"worker requires INGEST_QUEUE=postgres (a PostgreSQL DATABASE_URL); "
f"resolved INGEST_QUEUE={settings.ingest_queue!r}"
)
from nextcloud_mcp_server.vector.queue.procrastinate import ( # noqa: PLC0415
INGEST_QUEUE_NAME,
apply_ingest_queue_schema,
get_procrastinate_app,
)
workers = concurrency or settings.vector_sync_processor_workers
app = get_procrastinate_app()
# Register the configured document processors (Unstructured / Tesseract /
# custom HTTP) in the worker process. The always-on API pod does this in its
# lifespan; the worker has its own startup path, so without this the worker
# would silently fall back to the import-time-registered PyMuPDF only.
from nextcloud_mcp_server.app import initialize_document_processors # noqa: PLC0415
initialize_document_processors()
async def _run() -> None:
# Open the connector pool once and reuse it for both the defensive
# schema apply (the always-on API pod is the authoritative applier) and
# the worker loop — manage_connection=False avoids a redundant
# open/close cycle on startup.
async with app.open_async():
await apply_ingest_queue_schema(app, manage_connection=False)
# Structured log (not click.echo) so it lands in the JSON / OTel
# pipeline like every other startup message.
logger.info(
"Ingest worker started: queue=%s concurrency=%s delete_succeeded=%s",
INGEST_QUEUE_NAME,
workers,
settings.ingest_delete_succeeded_jobs,
)
await app.run_worker_async(
queues=[INGEST_QUEUE_NAME],
concurrency=workers,
install_signal_handlers=True,
# Drop succeeded jobs (default) so the queue table stays lean and
# the KEDA queue-depth metric reflects only outstanding work; set
# INGEST_DELETE_SUCCEEDED_JOBS=false to retain them for audit.
delete_jobs="successful"
if settings.ingest_delete_succeeded_jobs
else "never",
)
anyio.run(_run)
@click.group()
def db():
"""Database migration management commands."""
@@ -374,6 +459,21 @@ def upgrade(database_url: str | None, database_path: str | None, revision: str):
try:
click.echo(f"Upgrading database to revision: {revision}")
upgrade_database(url, revision)
# Apply procrastinate's ingest-queue schema on Postgres so a one-shot
# migration/init job provisions everything the api + worker roles need
# (Deck #183). Idempotent + lazy import (Postgres-only extra).
from nextcloud_mcp_server.config import is_sqlite_url # noqa: PLC0415
if not is_sqlite_url(url):
import anyio # noqa: PLC0415
from nextcloud_mcp_server.vector.queue.procrastinate import ( # noqa: PLC0415
apply_ingest_queue_schema,
build_app_for_url,
)
anyio.run(apply_ingest_queue_schema, build_app_for_url(url))
click.echo(click.style("✓ Ingest queue schema applied", fg="green"))
click.echo(click.style("✓ Database upgraded successfully", fg="green"))
except Exception as e:
click.echo(click.style(f"✗ Upgrade failed: {e}", fg="red"), err=True)
@@ -483,6 +583,7 @@ def cli():
cli.add_command(run)
cli.add_command(worker)
cli.add_command(db)
nextcloud_mcp_server/config.py
+150 -54
View File
@@ -11,6 +11,8 @@ from typing import Any
from dynaconf import Dynaconf, Validator
logger = logging.getLogger(__name__)
# Sentinel for "key not in dynaconf at all" vs "explicitly set to None".
_UNSET = object()
@@ -169,14 +171,27 @@ _DEFAULTS: dict[str, Any] = {
# the current monolithic behavior; self-hosters who set none are
# unaffected. See docs/architecture/mcp-decomposition.md (sibling repo).
"embedding_provider": "autodetect", # autodetect | gateway
"ingest_mode": "local", # local | external
"status_backend": "local", # local | bus
# Ingest queue backend (Deck #183). None → auto: ``postgres`` (procrastinate)
# when DATABASE_URL is Postgres, else ``memory`` (the in-process anyio queue
# for SQLite/dev). procrastinate requires PostgreSQL.
"ingest_queue": None, # memory | postgres
# Process role for the per-tenant two-pod model (Deck #183). ``api`` runs the
# MCP/query server + scanner (defers jobs); the ``worker`` role is the
# `nextcloud-mcp-server worker` process that drains the queue. ``all`` keeps
# the monolithic behaviour (API + in-process SQLite pool).
"mcp_role": "all", # api | worker | all
# Reclaim an ingest job orphaned in ``doing`` by a crashed worker once its
# worker heartbeat is this many seconds stale (Deck #183). Default is well
# above the longest expected document; raise it for slow embedding backends.
"ingest_stalled_job_seconds": 300,
# Delete succeeded ingest jobs (keeps the queue table lean + the KEDA
# queue-depth metric clean). Set false to retain succeeded rows for audit
# (note: indexing success is also recorded in logs/metrics regardless).
"ingest_delete_succeeded_jobs": True,
"collection_metadata_source": "qdrant", # qdrant | api
# CP base URL for COLLECTION_METADATA_SOURCE=api (e.g. http://control-plane).
# Required only when the source is api.
"collection_metadata_api_url": None,
"fact_event_emitter": "none", # none | nats | stdout
"ingest_bus_url": None, # required when ingest_mode=external
"embedding_gateway_url": None, # required when embedding_provider=gateway
# Provider-namespaced model the gateway serves, "<provider>/<model>"
# (the gateway routes on the "/"-prefix; mistral/mistral-embed → Mistral
@@ -191,8 +206,7 @@ _DEFAULTS: dict[str, Any] = {
"embedding_gateway_client_id": None,
"embedding_gateway_client_secret": None,
"embedding_gateway_scope": None, # e.g. astrolabe-embedding-gateway/embed
"tenant_id": None, # NATS per-tenant subject token (UUID form)
"ingest_bus_num_replicas": 1, # JetStream stream replicas (prod: 3)
"tenant_id": None, # per-tenant identity (UUID form); see vector/payload_keys
# Query-side ACL pre-filter (design §11). OFF by default: a Qdrant
# `match any` on `acl_hash` excludes points missing the key, so enabling
# this before a real ACL backfill would silently drop legacy results.
@@ -251,6 +265,7 @@ _dynaconf = Dynaconf(
# Port ranges
Validator("METRICS_PORT", gte=1, lte=65535),
# Positive integers
Validator("INGEST_STALLED_JOB_SECONDS", gte=1),
Validator("VECTOR_SYNC_SCAN_INTERVAL", gte=1),
Validator("VECTOR_SYNC_PROCESSOR_WORKERS", gte=1),
Validator("VECTOR_SYNC_QUEUE_MAX_SIZE", gte=1),
@@ -699,12 +714,14 @@ class Settings:
# MCP decomposition hook points (design §10, opt-in). All defaults
# reproduce the current monolith; validated in __post_init__.
embedding_provider: str = "autodetect" # autodetect | gateway
ingest_mode: str = "local" # local | external
status_backend: str = "local" # local | bus
# Ingest queue backend (Deck #183). None → resolved in __post_init__ to
# ``postgres`` when DATABASE_URL is Postgres, else ``memory``.
ingest_queue: str | None = None # memory | postgres
mcp_role: str = "all" # api | worker | all (Deck #183 two-pod model)
ingest_stalled_job_seconds: int = 300 # crashed-worker reclaim threshold
ingest_delete_succeeded_jobs: bool = True # drop succeeded ingest jobs
collection_metadata_source: str = "qdrant" # qdrant | api
collection_metadata_api_url: str | None = None # CP URL when source=api
fact_event_emitter: str = "none" # none | nats | stdout
ingest_bus_url: str | None = None # required when ingest_mode=external
embedding_gateway_url: str | None = None # required when provider=gateway
embedding_gateway_model: str = (
"mistral/mistral-embed" # provider-namespaced id the gateway routes on
@@ -714,13 +731,11 @@ class Settings:
embedding_gateway_client_id: str | None = None
embedding_gateway_client_secret: str | None = None
embedding_gateway_scope: str | None = None
tenant_id: str | None = None # NATS per-tenant subject token (UUID form)
ingest_bus_num_replicas: int = 1 # JetStream stream replicas (prod: 3)
tenant_id: str | None = None # per-tenant identity (UUID form)
acl_prefilter_enabled: bool = False # query-side ACL pre-filter (§11); OFF
def __post_init__(self):
"""Validate configuration and set defaults."""
logger = logging.getLogger(__name__)
# Validate SSL/TLS configuration
if not self.nextcloud_verify_ssl:
@@ -803,10 +818,8 @@ class Settings:
# the monolith, so deployments that set none of these pass through.
_enum_fields = {
"embedding_provider": {"autodetect", "gateway"},
"ingest_mode": {"local", "external"},
"status_backend": {"local", "bus"},
"mcp_role": {"api", "worker", "all"},
"collection_metadata_source": {"qdrant", "api"},
"fact_event_emitter": {"none", "nats", "stdout"},
}
for _field, _allowed in _enum_fields.items():
_val = (getattr(self, _field) or "").strip().lower()
@@ -816,21 +829,25 @@ class Settings:
f"{_field.upper()} must be one of {sorted(_allowed)}; got {_val!r}"
)
# Fail-fast: external ingest sources its status from the bus. With the
# in-process state machine empty, STATUS_BACKEND=local would leave
# status streams silently empty — crash loudly instead (design §10.1).
if self.status_backend == "local" and self.ingest_mode == "external":
raise RuntimeError(
"STATUS_BACKEND=local is incompatible with INGEST_MODE=external; "
"set STATUS_BACKEND=bus"
# Ingest queue backend (Deck #183). Unset → auto-derive from the
# database backend: procrastinate needs PostgreSQL, so SQLite/dev falls
# back to the in-process anyio queue. An explicit ``postgres`` against a
# SQLite DATABASE_URL is a misconfiguration — fail loudly.
_queue = (self.ingest_queue or "").strip().lower()
if not _queue:
_queue = "memory" if is_sqlite_url(get_database_url()) else "postgres"
if _queue not in {"memory", "postgres"}:
raise ValueError(
f"INGEST_QUEUE must be one of ['memory', 'postgres']; got {_queue!r}"
)
self.ingest_queue = _queue
if self.ingest_queue == "postgres" and is_sqlite_url(get_database_url()):
raise ValueError(
"INGEST_QUEUE=postgres requires a PostgreSQL DATABASE_URL "
"(procrastinate is Postgres-only); use INGEST_QUEUE=memory for "
"SQLite/dev"
)
# Conditional-required settings for the active hook points.
if self.ingest_mode == "external":
if not self.ingest_bus_url:
raise ValueError("INGEST_BUS_URL is required when INGEST_MODE=external")
if not self.tenant_id:
raise ValueError("TENANT_ID is required when INGEST_MODE=external")
if self.embedding_provider == "gateway" and not self.embedding_gateway_url:
raise ValueError(
"EMBEDDING_GATEWAY_URL is required when EMBEDDING_PROVIDER=gateway"
@@ -859,23 +876,6 @@ class Settings:
"client-credentials) or all left unset (unauthenticated gateway)"
)
# TENANT_ID is a NATS subject token; '.', '*', '>', and whitespace are
# reserved/illegal there and would silently break subscriptions (§3.4).
if self.tenant_id and (
any(c in self.tenant_id for c in ".*>")
or any(c.isspace() for c in self.tenant_id)
):
raise ValueError(
"TENANT_ID must not contain '.', '*', '>', or whitespace "
"(it is used as a NATS subject token)"
)
if self.ingest_bus_num_replicas < 1:
raise ValueError(
f"INGEST_BUS_NUM_REPLICAS must be >= 1; "
f"got {self.ingest_bus_num_replicas}"
)
# --- ADR-022 follow-up: deployment mode is the single source of truth ---
# The ENABLE_MULTI_USER_BASIC_AUTH and ENABLE_LOGIN_FLOW env vars were
# removed in favour of MCP_DEPLOYMENT_MODE. We do TWO things here:
@@ -1065,8 +1065,6 @@ def _get_semantic_search_enabled() -> bool:
Returns:
True if semantic search should be enabled
"""
logger = logging.getLogger(__name__)
new_value = _dynaconf.get("ENABLE_SEMANTIC_SEARCH", False)
old_value = _dynaconf.get("VECTOR_SYNC_ENABLED", False)
@@ -1147,7 +1145,6 @@ def _log_bg_ops_advisories_once(
return
_bg_ops_advisories_logged = True
logger = logging.getLogger(__name__)
if explicit and legacy:
logger.warning(
"Both ENABLE_BACKGROUND_OPERATIONS and ENABLE_OFFLINE_ACCESS are set. "
@@ -1316,12 +1313,12 @@ def get_settings() -> Settings:
"excluded_tags": "EXCLUDED_TAGS",
# MCP decomposition hook points (design §10)
"embedding_provider": "EMBEDDING_PROVIDER",
"ingest_mode": "INGEST_MODE",
"status_backend": "STATUS_BACKEND",
"ingest_queue": "INGEST_QUEUE",
"mcp_role": "MCP_ROLE",
"ingest_stalled_job_seconds": "INGEST_STALLED_JOB_SECONDS",
"ingest_delete_succeeded_jobs": "INGEST_DELETE_SUCCEEDED_JOBS",
"collection_metadata_source": "COLLECTION_METADATA_SOURCE",
"collection_metadata_api_url": "COLLECTION_METADATA_API_URL",
"fact_event_emitter": "FACT_EVENT_EMITTER",
"ingest_bus_url": "INGEST_BUS_URL",
"embedding_gateway_url": "EMBEDDING_GATEWAY_URL",
"embedding_gateway_model": "EMBEDDING_GATEWAY_MODEL",
"embedding_gateway_token_url": "EMBEDDING_GATEWAY_TOKEN_URL",
@@ -1329,7 +1326,6 @@ def get_settings() -> Settings:
"embedding_gateway_client_secret": "EMBEDDING_GATEWAY_CLIENT_SECRET",
"embedding_gateway_scope": "EMBEDDING_GATEWAY_SCOPE",
"tenant_id": "TENANT_ID",
"ingest_bus_num_replicas": "INGEST_BUS_NUM_REPLICAS",
"acl_prefilter_enabled": "ACL_PREFILTER_ENABLED",
}
@@ -1405,3 +1401,103 @@ def get_database_ssl() -> bool | ssl.SSLContext | None:
if settings.database_verify_ssl is True:
return True
return None
def _pg_ssl_params() -> dict[str, str]:
"""Map the DATABASE_VERIFY_SSL / DATABASE_CA_BUNDLE settings to libpq
keyword params for psycopg3 (used by procrastinate, Deck #183).
psycopg/libpq takes ``sslmode`` (and ``sslrootcert``) rather than an
``ssl.SSLContext`` like asyncpg, so we translate :func:`get_database_ssl`'s
intent into the equivalent libpq settings:
- ``None`` (both unset) → ``{}`` (omit; libpq default ``prefer``,
matching the asyncpg default for cluster-local Postgres without TLS).
- ``False`` (DATABASE_VERIFY_SSL=false) → ``sslmode=require`` (encrypt but
do not verify the certificate).
- CA bundle set → ``sslmode=verify-full`` + ``sslrootcert``.
- ``True`` (verify, no bundle) → ``sslmode=verify-full`` (system trust).
"""
ssl_setting = get_database_ssl()
if ssl_setting is None:
return {}
if ssl_setting is False:
return {"sslmode": "require"}
settings = get_settings()
if settings.database_ca_bundle:
return {"sslmode": "verify-full", "sslrootcert": settings.database_ca_bundle}
return {"sslmode": "verify-full"}
def get_procrastinate_conninfo(database_url: str | None = None) -> str:
"""Build a libpq conninfo string for procrastinate's psycopg3 connector.
Derives the connection from ``DATABASE_URL`` (a SQLAlchemy URL such as
``postgresql+asyncpg://user:pass@host/db``): the SQLAlchemy driver suffix
(``+asyncpg``/``+psycopg``) is stripped and the parts are rendered via
:func:`psycopg.conninfo.make_conninfo`, which quotes values correctly (never
f-string the password). TLS settings are appended from :func:`_pg_ssl_params`.
This is driver-agnostic on purpose: procrastinate uses psycopg3 regardless of
which SQLAlchemy driver the app's own engine uses, so it works whether
``DATABASE_URL`` carries ``+asyncpg`` or ``+psycopg``.
TODO(Deck #183 follow-up, out-of-tree): unify the app's SQLAlchemy engine on
psycopg3 too (``postgresql+psycopg://``) and drop asyncpg, so the deployment
ships a single Postgres driver. This belongs in the rendered Helm chart
(set ``DATABASE_URL`` to a ``+psycopg`` URL) rather than rewriting the driver
in code — see charts repo, not this repo.
Only the host/port/dbname/user/password components are forwarded, plus
``connect_timeout`` (a libpq keyword) honored from the URL query string or
defaulted to 10s so a slow/unreachable DB can't hang worker/API startup
indefinitely. Any *other* ``?key=value`` query parameters are **dropped**
(TLS is set separately via :func:`_pg_ssl_params`, and SQLAlchemy-specific
options don't map cleanly to libpq keywords); a warning lists them.
Raises ``ValueError`` for a non-Postgres URL — procrastinate is Postgres-only.
"""
from psycopg.conninfo import make_conninfo # noqa: PLC0415
from sqlalchemy.engine.url import make_url # noqa: PLC0415
url = make_url(database_url or get_database_url())
if not url.drivername.startswith("postgresql"):
raise ValueError(
"get_procrastinate_conninfo requires a PostgreSQL DATABASE_URL; "
f"got driver {url.drivername!r}"
)
# ``connect_timeout`` is forwarded (libpq keyword); everything else in the
# query string is dropped with a warning.
dropped = sorted(k for k in url.query if k != "connect_timeout")
if dropped:
logger.warning(
"Dropping DATABASE_URL query parameters not forwarded to the "
"procrastinate connector: %s",
", ".join(dropped),
)
params: dict[str, str] = {}
if url.host:
params["host"] = url.host
if url.port:
params["port"] = str(url.port)
if url.database:
params["dbname"] = url.database
if url.username:
params["user"] = url.username
if url.password:
params["password"] = url.password
# Honor an operator-supplied connect_timeout, else default to 10s. (make_url
# query values are str or a tuple of strs when repeated; take the last.)
# ``connect_timeout=0`` (disable) is preserved — only a missing/empty value
# falls back to the default, and an explicit-but-empty value is flagged.
_ct = url.query.get("connect_timeout")
if isinstance(_ct, (list, tuple)):
_ct = _ct[-1] if _ct else None
if _ct == "":
logger.warning("DATABASE_URL has an empty connect_timeout=; using default 10s")
params["connect_timeout"] = _ct if _ct else "10"
params.update(_pg_ssl_params())
return make_conninfo(**params)
nextcloud_mcp_server/config_validators.py
-2
View File
@@ -187,8 +187,6 @@ def detect_auth_mode(settings: Settings) -> AuthMode:
ValueError: If explicit deployment_mode is unrecognised.
"""
logger = logging.getLogger(__name__)
# ADR-021: explicit deployment mode wins
if settings.deployment_mode:
mode_str = settings.deployment_mode.lower().strip()
nextcloud_mcp_server/models/semantic.py
+11
View File
@@ -170,6 +170,17 @@ class VectorSyncStatusResponse(BaseResponse):
description='Sync status: "idle", "syncing", or "disabled"',
)
enabled: bool = Field(default=False, description="Whether vector sync is enabled")
ingest_queue: str | None = Field(
default=None,
description='Ingest queue backend: "memory" or "postgres" (Deck #183)',
)
job_counts: dict[str, int] | None = Field(
default=None,
description=(
"Per-status ingest job counts (todo/doing/failed/…) on the postgres "
"queue backend; None on the in-memory backend"
),
)
__all__ = [
nextcloud_mcp_server/server/semantic.py
+18 -18
View File
@@ -966,23 +966,21 @@ def configure_semantic_tools(mcp: FastMCP):
# missing attribute is a typo that should fail loudly. The
# value itself can legitimately be ``None`` before sync starts,
# which the check below handles.
# Outstanding-work view depends on the queue backend (Deck #183):
# memory → stream buffer depth; postgres → procrastinate job counts.
# Direct attribute access matches the eviction_task_group pattern at
# ``nc_semantic_search``: both AppContext and OAuthAppContext define
# these, so a missing attribute is a typo that should fail loudly.
from nextcloud_mcp_server.vector.ingest_status import ( # noqa: PLC0415
get_ingest_pending,
)
lifespan_ctx = ctx.request_context.lifespan_context
document_receive_stream = lifespan_ctx.document_receive_stream
if document_receive_stream is None:
logger.debug(
"document_receive_stream not available in lifespan context"
)
return VectorSyncStatusResponse(
indexed_count=0,
pending_count=0,
status="unknown",
enabled=True,
)
# Get pending count from stream statistics
stream_stats = document_receive_stream.statistics()
pending_count = stream_stats.current_buffer_used
pending = await get_ingest_pending(
task_producer=lifespan_ctx.task_producer,
document_receive_stream=lifespan_ctx.document_receive_stream,
ingest_queue=settings.ingest_queue,
)
# Get Qdrant client and query indexed count
indexed_count = 0
@@ -1002,13 +1000,15 @@ def configure_semantic_tools(mcp: FastMCP):
# Continue with indexed_count = 0
# Determine status
status = "syncing" if pending_count > 0 else "idle"
status = "syncing" if pending.pending > 0 else "idle"
return VectorSyncStatusResponse(
indexed_count=indexed_count,
pending_count=pending_count,
pending_count=pending.pending,
status=status,
enabled=True,
ingest_queue=settings.ingest_queue,
job_counts=pending.job_counts,
)
except Exception as e:
nextcloud_mcp_server/vector/ingest_status.py
+62
View File
@@ -0,0 +1,62 @@
"""Shared read model for the vector-sync status surface (Deck #183).
The status endpoints (``/api/v1/vector-sync/status``, the userinfo route, and
the ``nc_get_vector_sync_status`` MCP tool) all need the same "how much work is
outstanding" figure, computed differently per ``INGEST_QUEUE`` backend:
- ``memory`` — the in-process anyio stream's buffer depth (today's behavior).
- ``postgres`` — procrastinate job counts read from the per-tenant Postgres
(``todo`` + ``doing``), plus the per-status breakdown for observability.
``indexed_documents`` (the Qdrant placeholder count) is backend-independent and
stays at each call site.
"""
from __future__ import annotations
import logging
from dataclasses import dataclass
from typing import Any
logger = logging.getLogger(__name__)
@dataclass
class IngestPending:
"""Outstanding-work view for the active ingest queue backend."""
pending: int
# Per-status counts (todo/doing/failed/…) on the postgres backend; None on
# the memory backend, which has no durable per-status breakdown.
job_counts: dict[str, int] | None = None
async def get_ingest_pending(
*, task_producer: Any, document_receive_stream: Any, ingest_queue: str | None
) -> IngestPending:
"""Compute outstanding ingest work for the configured queue backend.
``task_producer`` and ``document_receive_stream`` are intentionally typed
``Any``: they're duck-typed across backends. Only ``ProcrastinateTaskProducer``
exposes ``job_counts`` (the ``TaskProducer`` protocol doesn't), and the memory
backend reads the anyio stream's ``statistics()`` — so no single concrete type
or Protocol fits both branches, and we probe with ``hasattr`` instead.
Never raises — a status surface must stay available even if the queue is
unreachable; failures degrade to ``pending=0``.
"""
if ingest_queue == "postgres":
counts: dict[str, int] = {}
if task_producer is not None and hasattr(task_producer, "job_counts"):
try:
counts = await task_producer.job_counts()
except Exception as e:
logger.warning("Failed to read ingest job counts: %s", e)
pending = counts.get("todo", 0) + counts.get("doing", 0)
return IngestPending(pending=pending, job_counts=counts)
if document_receive_stream is None:
return IngestPending(pending=0)
return IngestPending(
pending=document_receive_stream.statistics().current_buffer_used
)
nextcloud_mcp_server/vector/processor.py
+7 -2
View File
@@ -154,7 +154,9 @@ async def processor_task(
logger.info("Processor %s stopped", worker_id)
async def process_document(doc_task: DocumentTask, nc_client: NextcloudClient):
async def process_document(
doc_task: DocumentTask, nc_client: NextcloudClient, *, max_retries: int = 3
):
"""
Process a single document: fetch, tokenize, embed, store in Qdrant.
@@ -163,6 +165,10 @@ async def process_document(doc_task: DocumentTask, nc_client: NextcloudClient):
Args:
doc_task: Document task to process
nc_client: Authenticated Nextcloud client
max_retries: In-process indexing attempts before re-raising. The default
(3) suits the in-process SQLite pool, which has no durable retry. The
procrastinate worker passes ``1`` so durable retry is owned by the
queue (and survives worker crashes), avoiding compounding 3×N retries.
"""
start_time = time.time()
@@ -230,7 +236,6 @@ async def process_document(doc_task: DocumentTask, nc_client: NextcloudClient):
return
# Handle indexing with retry
max_retries = 3
retry_delay = 1.0
for attempt in range(max_retries):
nextcloud_mcp_server/vector/queue/__init__.py
+3 -3
View File
@@ -1,7 +1,7 @@
"""Ingest-path ports & adapters (design §10, hexagonal)."""
"""Ingest-path ports & adapters (design §10, hexagonal; Deck #183)."""
from .factory import build_external_producer
from .factory import build_producer
from .memory import MemoryTaskProducer
from .ports import TaskProducer
__all__ = ["MemoryTaskProducer", "TaskProducer", "build_external_producer"]
__all__ = ["MemoryTaskProducer", "TaskProducer", "build_producer"]
nextcloud_mcp_server/vector/queue/factory.py
+24 -42
View File
@@ -1,60 +1,42 @@
"""Composition root for the ingest producer (design §10).
"""Composition root for the ingest producer (Deck #183).
``build_external_producer`` is called by the lifespan only when
``INGEST_MODE=external``; local mode uses the in-memory stream directly (it
already satisfies :class:`TaskProducer`). The transport under ``external`` is
selected from the ``INGEST_BUS_URL`` scheme (``nats://`` now, ``postgres://``
later) so moving the external processor to Postgres needs no new INGEST_MODE.
The transport is selected from ``INGEST_QUEUE``:
- ``postgres`` → :class:`ProcrastinateTaskProducer`, which defers jobs into the
per-tenant Postgres for the out-of-process ``worker`` role to drain.
- ``memory`` (SQLite/dev default) → the in-process anyio stream, built inline by
the server lifespan (it owns both the send and receive ends), so it is not
produced here.
"""
from __future__ import annotations
import logging
from urllib.parse import urlsplit
from typing import TYPE_CHECKING
from ...config import Settings
from .ports import TaskProducer
if TYPE_CHECKING:
from .procrastinate import ProcrastinateTaskProducer
logger = logging.getLogger(__name__)
def _transport_for(url: str) -> str:
scheme = urlsplit(url).scheme.lower()
if scheme.startswith("postgres"):
return "postgres"
if not scheme.startswith("nats"):
logger.warning(
"INGEST_BUS_URL scheme %r is neither nats:// nor postgres://; "
"defaulting to the NATS transport",
scheme,
)
return "nats"
async def build_producer(settings: Settings) -> ProcrastinateTaskProducer:
"""Build the Postgres (procrastinate) ingest producer.
async def build_external_producer(settings: Settings) -> TaskProducer:
"""Build the external-ingest producer for the configured transport.
Precondition: ``settings.ingest_mode == "external"`` (so __post_init__ has
guaranteed ``ingest_bus_url`` and ``tenant_id`` are set).
Returns the concrete :class:`ProcrastinateTaskProducer` (not just the
``TaskProducer`` protocol) so the lifespan can call ``ensure_schema()`` on
the open connector. Precondition: ``settings.ingest_queue == "postgres"``
(the memory transport is constructed inline by the lifespan because it needs
the paired receive stream for the in-process processor pool).
"""
# Defence-in-depth (robust under ``python -O``, which strips asserts):
# __post_init__ already guarantees these when ingest_mode == external.
if settings.ingest_bus_url is None or settings.tenant_id is None:
if settings.ingest_queue != "postgres":
raise ValueError(
"build_external_producer requires INGEST_BUS_URL and TENANT_ID "
"(guaranteed by Settings validation when INGEST_MODE=external)"
"build_producer is only for INGEST_QUEUE=postgres; the memory "
f"transport is built inline by the lifespan (got {settings.ingest_queue!r})"
)
transport = _transport_for(settings.ingest_bus_url)
if transport == "postgres":
from .postgres import PostgresTaskProducer # noqa: PLC0415
from .procrastinate import ProcrastinateTaskProducer # noqa: PLC0415
return await PostgresTaskProducer.connect(settings)
from .nats import NatsTaskProducer # noqa: PLC0415
return await NatsTaskProducer.connect(
url=settings.ingest_bus_url,
tenant_id=settings.tenant_id,
num_replicas=settings.ingest_bus_num_replicas,
)
return await ProcrastinateTaskProducer.connect()
nextcloud_mcp_server/vector/queue/nats.py
-173
View File
@@ -1,173 +0,0 @@
"""NATS JetStream ``TaskProducer`` — external ingest transport (design §3.4).
Publishes ``mcp.ingest.requested.{tenant_id}`` for the external
document-processor to consume. Translates the in-process ``DocumentTask`` into
the wire ``IngestMessage`` schema (mirrored in astrolabe-cloud-website's
``bus/messages.py``), with the JetStream ``Nats-Msg-Id`` dedup header per §3.4.
This server is only the *producer* on this transport; the document-processor
owns the consumer. ``nats-py`` is imported lazily so deployments that never
enable external ingest don't pay the import.
"""
from __future__ import annotations
import hashlib
import logging
from datetime import datetime, timezone
from types import TracebackType
from typing import TYPE_CHECKING, Any
from ...canonical import canonical_json
if TYPE_CHECKING:
from ..scanner import DocumentTask
logger = logging.getLogger(__name__)
STREAM_NAME = "mcp"
INGEST_SUBJECT_PREFIX = "mcp.ingest.requested"
def warn_if_insecure_nats_url(url: str) -> None:
"""Log a warning when the bus URL is not TLS-encrypted.
``nats://`` (and ``ws://``) carry tenant document metadata in cleartext;
production deployments should use ``tls://`` (or ``wss://``). We connect
regardless — this is an operator alert, not a hard failure.
"""
scheme = url.split("://", 1)[0].lower()
if scheme not in ("tls", "wss"):
logger.warning(
"NATS bus URL uses unencrypted transport (scheme=%s://); "
"use tls:// in production to protect document metadata in transit",
scheme,
)
def _modified_at_rfc3339(modified_at: int) -> str:
"""DocumentTask.modified_at is an epoch int (0 for deletes)."""
return datetime.fromtimestamp(int(modified_at), tz=timezone.utc).isoformat()
def _content_hash(task: DocumentTask) -> str:
"""etag is the change-detection token; fall back to modified_at when it is
absent (e.g. deletes, or sources whose etag we don't thread through).
TODO(follow-up, PR #814 review): thread etags for file / deck_card /
news_item scans too (only note scans pass etag today). Until then their
JetStream Nats-Msg-Id dedup keys off modified_at, which misses content
changes that leave modified_at unchanged (e.g. a file move/rename).
"""
return task.etag or str(task.modified_at)
def msg_id(tenant_id: str, doc_id: str, modified_at_rfc3339: str) -> str:
"""JetStream dedup header per §3.4. SHA-256 over canonical JSON (NOT the
BLAKE2b helper) — it is an opaque external header, not a stored field."""
return hashlib.sha256(
canonical_json(
{
"tenant_id": tenant_id,
"doc_id": doc_id,
"modified_at": modified_at_rfc3339,
}
)
).hexdigest()
class NatsTaskProducer:
"""Publishes ingest requests to NATS JetStream."""
def __init__(self, nc: Any, js: Any, tenant_id: str):
self._nc = nc
self._js = js
self.tenant_id = tenant_id
@classmethod
async def connect(
cls, *, url: str, tenant_id: str, num_replicas: int = 1
) -> NatsTaskProducer:
import nats # noqa: PLC0415 (lazy: optional dependency for external mode)
warn_if_insecure_nats_url(url)
nc = await nats.connect(url)
js = nc.jetstream()
await cls._ensure_stream(js, num_replicas)
logger.info("Connected NATS ingest producer: url=%s, tenant=%s", url, tenant_id)
return cls(nc, js, tenant_id)
@staticmethod
async def _ensure_stream(js: Any, num_replicas: int) -> None:
# noqa: PLC0415 — nats.js types are only importable once nats-py is present.
from nats.js.api import RetentionPolicy, StreamConfig # noqa: PLC0415
config = StreamConfig(
name=STREAM_NAME,
subjects=["mcp.>"],
retention=RetentionPolicy.LIMITS,
num_replicas=num_replicas,
)
try:
await js.add_stream(config=config)
logger.info("nats.stream_created stream=%s", STREAM_NAME)
except Exception as exc:
# add_stream is idempotent in spirit but errors when the stream
# already exists; treat as benign (mirrors the processor's
# ensure_stream). A genuinely broken broker surfaces on publish.
logger.info("nats.stream_exists_or_unavailable detail=%s", exc)
def ingest_message(self, task: DocumentTask) -> dict[str, Any]:
"""DocumentTask → wire IngestMessage dict (mirrors the sibling schema)."""
return {
"tenant_id": self.tenant_id,
"doc_id": task.doc_id,
"content_hash": _content_hash(task),
"modified_at": _modified_at_rfc3339(task.modified_at),
"doc_type": task.doc_type,
"operation": task.operation,
"user_id": task.user_id,
"file_path": task.file_path,
}
async def send(self, task: DocumentTask) -> None:
message = self.ingest_message(task)
subject = f"{INGEST_SUBJECT_PREFIX}.{self.tenant_id}"
headers = {
"Nats-Msg-Id": msg_id(self.tenant_id, task.doc_id, message["modified_at"])
}
await self._js.publish(subject, canonical_json(message), headers=headers)
# The scanner/oauth_sync use the producer as a clone-able async context
# manager (memory-stream semantics). The bus connection is owned by the
# lifespan, so cloning shares it and __aexit__ is a no-op (close happens via
# aclose() on shutdown).
def clone(self) -> NatsTaskProducer:
return self
async def __aenter__(self) -> NatsTaskProducer:
return self
async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc: BaseException | None,
tb: TracebackType | None,
) -> None:
return None
# The bare suppression marker silences S7503 (async method without await):
# ``async def`` is required by the TaskProducer protocol, but this handle
# close is a genuine no-op.
async def aclose(self) -> None: # NOSONAR
# Per-handle close (e.g. a per-user scanner clone exiting). The bus
# connection is shared and owned by the lifespan, so this is a no-op;
# the connection is torn down once via ``drain()`` on shutdown.
return None
async def drain(self) -> None:
"""Drain + close the shared NATS connection (lifespan shutdown only)."""
try:
await self._nc.drain()
except Exception:
logger.warning("NATS drain on shutdown failed", exc_info=True)
nextcloud_mcp_server/vector/queue/ports.py
+17 -11
View File
@@ -1,18 +1,18 @@
"""Ingest-path ports (design §10, hexagonal).
"""Ingest-path ports (design §10, hexagonal; Deck #183).
A ``TaskProducer`` is where the scanner + webhook receiver send a
``DocumentTask``. The transport behind it is swappable:
- the in-process anyio ``MemoryObjectSendStream`` (local ingest — the default),
- ``NatsTaskProducer`` (external ingest → the document-processor), and
- a future Postgres-queue producer (seam only; the *external* processor owns the
consume side — see ``postgres.py``).
- ``MemoryTaskProducer`` over the in-process anyio ``MemoryObjectSendStream``
(``INGEST_QUEUE=memory`` — the SQLite/dev default), and
- ``ProcrastinateTaskProducer`` (``INGEST_QUEUE=postgres``), which defers jobs
into the per-tenant Postgres for the out-of-process ``worker`` role to drain.
The protocol is exactly the surface the scanner/oauth_sync already use on the
memory stream (``send`` + ``clone`` + ``async with``), so both adapters drop in
with only a type-annotation change at the call sites. There is intentionally NO
consumer port: the MCP server's only in-process consumer is the memory stream;
when ingest is external the document-processor is the consumer, not this server.
with only a type-annotation change at the call sites. There is no consumer port:
in memory mode the in-process processor pool is the consumer; in postgres mode
the procrastinate worker is.
"""
from __future__ import annotations
@@ -52,8 +52,14 @@ class TaskProducer(Protocol):
async def aclose(self) -> None:
"""Close *this* handle (e.g. a per-user clone when its scanner exits).
For the memory stream this closes the clone; for the shared bus
connection it is a no-op (the connection is owned by the lifespan,
which drains it once on shutdown).
For the memory stream this closes the clone; for a shared connection it
is a no-op (the connection is owned by the lifespan, which tears it down
once on shutdown).
"""
...
# Note: this protocol deliberately omits ``drain()``. An implementation that
# owns a long-lived shared connection (e.g. ProcrastinateTaskProducer's
# connector pool) may additionally provide ``async def drain()`` for the
# lifespan to close that pool once on shutdown; the lifespan probes for it
# with ``getattr(task_producer, "drain", None)``, so it stays optional.
nextcloud_mcp_server/vector/queue/postgres.py
-52
View File
@@ -1,52 +0,0 @@
"""Postgres-queue ``TaskProducer`` — documented seam, not implemented.
The external document-processor may later drain a Postgres-backed queue instead
of NATS to limit NATS operational overhead. Processing stays *external*; only
the transport changes — so on this server it would be a drop-in producer swap.
The consume side + the queue-table migration belong to that processor-side
refactor (cross-repo), NOT here. This stub exists so the transport value and the
``TaskProducer`` Protocol conformance are testable today.
"""
from __future__ import annotations
from types import TracebackType
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from ..scanner import DocumentTask
_NOT_IMPLEMENTED = (
"Postgres ingest transport is a documented seam. The external "
"document-processor owns the Postgres-drain refactor (transport swap only; "
"processing stays external). Use INGEST_BUS_URL=nats://… for now."
)
class PostgresTaskProducer:
@classmethod
async def connect(cls, settings: Any) -> PostgresTaskProducer:
raise NotImplementedError(_NOT_IMPLEMENTED)
async def send(self, task: DocumentTask) -> None: # pragma: no cover
raise NotImplementedError(_NOT_IMPLEMENTED)
def clone(self) -> PostgresTaskProducer: # pragma: no cover
return self
async def __aenter__(self) -> PostgresTaskProducer: # pragma: no cover
return self
async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc: BaseException | None,
tb: TracebackType | None,
) -> None: # pragma: no cover
return None
# The bare suppression marker silences S7503 (async method without await):
# ``async def`` is required by the TaskProducer protocol; this stub is a
# no-op until the Postgres transport lands.
async def aclose(self) -> None: # NOSONAR # pragma: no cover
return None
nextcloud_mcp_server/vector/queue/procrastinate.py
+383
View File
@@ -0,0 +1,383 @@
"""Procrastinate-backed ingest queue — the Postgres ``TaskProducer`` + worker
(Deck #183).
This replaces NATS JetStream and the old Postgres-queue stub. The MCP server now
owns *both* sides of ingest:
- **Producer** (API role / scanner) — :class:`ProcrastinateTaskProducer.send`
*defers* one ``ingest:process_document`` job per changed document into the
per-tenant Postgres (the same app DB; procrastinate manages its own tables).
- **Consumer** (worker role) — ``nextcloud-mcp-server worker`` runs
:func:`procrastinate.App.run_worker`, which drains the ``ingest`` queue and
invokes the existing :func:`process_document` pipeline.
Design notes:
- **No execution ``lock``, only ``queueing_lock``.** procrastinate does NOT
auto-reclaim ``doing`` jobs, so a per-doc execution lock would permanently
deadlock a document if a worker crashed mid-job. The Qdrant upsert is
idempotent (deterministic ``uuid5`` point IDs), so a concurrent/re-run is
harmless; ``queueing_lock`` (partial-unique on ``status='todo'``) is enough to
dedupe enqueues, and :func:`reclaim_stalled_ingest_jobs` retries jobs orphaned
in ``doing`` by a crash.
- procrastinate is Postgres-only and uses asyncio; ``anyio`` runs natively on the
asyncio backend, so the worker can call the anyio-based pipeline directly.
- Tasks are defined on a :class:`procrastinate.Blueprint` so the connector is
decoupled from the task registry: production binds a real
:class:`PsycopgConnector`; unit tests bind ``testing.InMemoryConnector``.
"""
from __future__ import annotations
import logging
from dataclasses import asdict
from datetime import datetime, timezone
from types import TracebackType
from typing import TYPE_CHECKING
from procrastinate import App, Blueprint, JobContext, PsycopgConnector, RetryStrategy
from procrastinate.connector import BaseConnector
from procrastinate.exceptions import AlreadyEnqueued
from ...config import get_procrastinate_conninfo, get_settings
from ..scanner import DocumentTask
if TYPE_CHECKING:
from ...client import NextcloudClient
logger = logging.getLogger(__name__)
# Single queue for document ingest. KEDA scales the worker Deployment on the
# depth of this queue (``SELECT count(*) FROM procrastinate_jobs WHERE
# queue_name='ingest' AND status='todo'``).
INGEST_QUEUE_NAME = "ingest"
# Blueprint namespace → registered task names are prefixed ``ingest:``.
_NAMESPACE = "ingest"
INGEST_TASK_NAME = f"{_NAMESPACE}:process_document"
# A crashed worker leaves its job in ``doing``; reclaim it once its (per-worker)
# heartbeat is this many seconds stale. The default is sized well above the
# longest expected ``process_document`` (PDF render + embedding) so a slow-but-
# live worker — whose heartbeat stays current during a long job — is never
# reclaimed out from under itself. Operators on slow embedding backends can tune
# it via INGEST_STALLED_JOB_SECONDS (read per-run in reclaim_stalled_ingest_jobs).
# Tasks are defined as plain functions and registered onto a *fresh* Blueprint
# per app (see _build_ingest_blueprint). procrastinate's add_tasks_from mutates
# the blueprint's task names in place (namespace prefixing), so a single shared
# Blueprint cannot be added to more than one App — which the tests (in-memory +
# real Postgres) and any re-init path require.
async def process_document_task(
*,
user_id: str,
doc_id: str,
doc_type: str,
operation: str,
modified_at: int,
file_path: str | None = None,
metadata: dict[str, int | str] | None = None,
etag: str | None = None,
owner_id: str | None = None,
) -> None:
"""Worker entry: rebuild the DocumentTask, resolve creds, run the pipeline."""
# Local imports avoid a heavy import chain at blueprint-definition time
# (this module is also imported by the API pod just to defer jobs).
from ..oauth_sync import NotProvisionedError # noqa: PLC0415
from ..processor import process_document # noqa: PLC0415
task = DocumentTask(
user_id=user_id,
doc_id=doc_id,
doc_type=doc_type,
operation=operation,
modified_at=modified_at,
file_path=file_path,
metadata=metadata,
etag=etag,
owner_id=owner_id,
)
try:
nc_client = await _resolve_client(user_id)
except NotProvisionedError:
# A deprovisioned user must not pin a worker slot retrying forever.
# Finish the job as a no-op; the next scan re-enqueues once the user
# re-provisions an app password. Other errors (transient DB/network)
# propagate so procrastinate's retry strategy handles them.
logger.warning(
"ingest.skip_no_credentials user=%s doc=%s:%s", user_id, doc_type, doc_id
)
return
try:
# Durable retry is procrastinate's job; disable the in-process loop.
await process_document(task, nc_client, max_retries=1)
finally:
await nc_client.close()
async def reclaim_stalled_ingest_jobs(context: JobContext, timestamp: int) -> None:
"""Re-queue ingest jobs orphaned in ``doing`` by a crashed worker.
procrastinate prunes dead *workers* but does not reset their in-flight jobs;
without this they'd sit in ``doing`` forever. ``timestamp`` is procrastinate's
periodic-run marker (unused).
"""
manager = context.app.job_manager
retry_at = datetime.now(tz=timezone.utc)
stalled_after = get_settings().ingest_stalled_job_seconds
reclaimed = 0
for job in await manager.get_stalled_jobs(
queue=INGEST_QUEUE_NAME, seconds_since_heartbeat=stalled_after
):
if job.id is None:
continue
await manager.retry_job_by_id_async(job_id=job.id, retry_at=retry_at)
reclaimed += 1
if reclaimed:
logger.warning("ingest.reclaimed_stalled_jobs count=%d", reclaimed)
else:
# Visible heartbeat under verbose logging when debugging a suspected
# reclaim failure, without noising up production logs.
logger.debug("ingest.reclaim_check stalled=0")
async def _resolve_client(user_id: str) -> NextcloudClient:
"""Build an authenticated NextcloudClient for ``user_id`` in the worker.
Single-user BasicAuth uses the shared env credentials; every multi-user mode
resolves the user's locally-stored app password (BasicAuth).
"""
from ...client import NextcloudClient # noqa: PLC0415
from ...config_validators import AuthMode, detect_auth_mode # noqa: PLC0415
settings = get_settings()
if detect_auth_mode(settings) == AuthMode.SINGLE_USER_BASIC:
return NextcloudClient.from_env()
from ..oauth_sync import get_user_client_basic_auth # noqa: PLC0415
host = settings.nextcloud_host
if not host:
raise ValueError("NEXTCLOUD_HOST is required for multi-user ingest")
return await get_user_client_basic_auth(user_id, host)
def _build_ingest_blueprint() -> Blueprint:
"""Create a fresh Blueprint with the ingest tasks registered.
Fresh per call because ``add_tasks_from`` mutates the blueprint's task names
(namespace prefixing), so the same Blueprint cannot be reused across Apps.
"""
bp = Blueprint()
# Durable retry owned by the queue (survives worker crashes); the in-process
# retry loop in process_document is disabled on this path via max_retries=1.
bp.task(
name="process_document",
queue=INGEST_QUEUE_NAME,
retry=RetryStrategy(max_attempts=5, exponential_wait=4),
)(process_document_task)
reclaim = bp.task(
name="reclaim_stalled_jobs", queue=INGEST_QUEUE_NAME, pass_context=True
)(reclaim_stalled_ingest_jobs)
bp.periodic(cron="*/5 * * * *", periodic_id="reclaim_stalled_ingest")(reclaim)
return bp
def build_app(connector: BaseConnector) -> App:
"""Build an App for the given connector with the ingest tasks registered.
Shared by production (:func:`get_procrastinate_app`) and tests (which pass a
``testing.InMemoryConnector``).
"""
app = App(connector=connector)
app.add_tasks_from(_build_ingest_blueprint(), namespace=_NAMESPACE)
return app
def build_app_for_url(database_url: str) -> App:
"""Build an App bound to an explicit Postgres URL (for the CLI, which may
target a ``--database-url`` that differs from the ``DATABASE_URL`` env)."""
return build_app(
PsycopgConnector(conninfo=get_procrastinate_conninfo(database_url))
)
_app: App | None = None
def get_procrastinate_app() -> App:
"""Process-wide procrastinate App bound to the Postgres app database."""
global _app
if _app is None:
_app = build_app(PsycopgConnector(conninfo=get_procrastinate_conninfo()))
return _app
async def _ingest_schema_present(app: App) -> bool:
row = await app.connector.execute_query_one_async(
"SELECT to_regclass('procrastinate_jobs') IS NOT NULL AS present"
)
return bool(row["present"])
async def _apply_ingest_queue_schema_open(app: App) -> None:
"""Apply the ingest-queue schema on an already-open connector (apply-if-absent).
procrastinate's ``schema.sql`` uses bare ``CREATE TYPE``/``CREATE TABLE``
(not ``IF NOT EXISTS``), so it errors if re-applied — it is meant to run
once on a fresh DB. We skip when ``procrastinate_jobs`` already exists;
*version* upgrades use procrastinate's own migration files (operator-run, a
lineage independent of the app's Alembic schema).
Safe to call concurrently across rolling-update pods without an advisory
lock: Postgres DDL is transactional and procrastinate applies the whole
schema in one transaction, so a pod that loses the race rolls back cleanly
and we treat the resulting error as benign once the schema is present.
"""
if await _ingest_schema_present(app):
logger.debug("ingest queue schema already present; skipping apply")
return
try:
await app.schema_manager.apply_schema_async()
logger.info("Applied procrastinate ingest queue schema")
except Exception:
# The apply runs in a single transaction, so any failure rolls back
# atomically (no partial schema). The only benign case is losing the
# create race to another pod — confirmed by re-checking presence. Any
# other failure (network, auth, …) leaves the schema absent, so this
# branch re-raises it rather than masking it.
if await _ingest_schema_present(app):
logger.info("Ingest queue schema applied concurrently by another pod")
return
raise
async def apply_ingest_queue_schema(
app: App | None = None, *, manage_connection: bool = True
) -> None:
"""Create procrastinate's tables on a fresh database (apply-if-absent).
By default opens a short-lived connection, so it is safe to call standalone
from the CLI ``db upgrade`` path. Pass ``manage_connection=False`` when the
caller already holds an open connector (the ``worker`` command opens the App
once and reuses it) to avoid a redundant open/close cycle.
"""
app = app or get_procrastinate_app()
if not manage_connection:
await _apply_ingest_queue_schema_open(app)
return
async with app.open_async():
await _apply_ingest_queue_schema_open(app)
# Job-status keys procrastinate flattens into each list_queues row (alongside
# ``name`` and ``jobs_count``). ``aborting`` is legacy/unused since v3.0.0.
_JOB_STATUSES = ("todo", "doing", "succeeded", "failed", "cancelled", "aborted")
async def get_ingest_job_counts(app: App | None = None) -> dict[str, int]:
"""Return ingest job counts by status (``todo``/``doing``/``failed``/…).
Reads procrastinate's per-queue stats via the manager API (not hand-written
SQL) so a future schema bump doesn't silently break the status surface. The
manager flattens its per-status ``stats`` into top-level row keys, so we read
the known status keys directly. Assumes the app's connector is already open.
"""
app = app or get_procrastinate_app()
counts: dict[str, int] = {}
for row in await app.job_manager.list_queues_async(queue=INGEST_QUEUE_NAME):
for status in _JOB_STATUSES:
if status in row:
counts[status] = counts.get(status, 0) + int(row[status])
return counts
def _doc_queueing_lock(task: DocumentTask) -> str:
"""Per-document enqueue-dedup key (partial-unique on ``status='todo'``).
Collision-safe with a raw ``:`` delimiter because the first two segments can
never themselves contain ``:``:
- ``user_id`` — a Nextcloud username/UID; Nextcloud rejects ``:`` in
usernames (Web UI + provisioning validation), so it is colon-free.
- ``doc_type`` — a controlled enum (``note``/``file``/``deck_card``/
``news_item``); a future ``doc_type`` containing ``:`` would break this
invariant, so keep doc_type colon-free.
The trailing ``doc_id`` may contain anything — it's the final unambiguous
segment.
"""
return f"{task.user_id}:{task.doc_type}:{task.doc_id}"
class ProcrastinateTaskProducer:
"""``TaskProducer`` that defers ingest jobs into Postgres via procrastinate.
The App's connector pool is owned by the server lifespan (opened once,
closed on shutdown), so ``clone``/``aenter``/``aexit``/``aclose`` are no-ops
— there is no per-handle resource like the memory stream's clones.
"""
def __init__(self, app: App):
self._app = app
@classmethod
async def connect(cls) -> ProcrastinateTaskProducer:
app = get_procrastinate_app()
# ``App.open_async()`` returns procrastinate's dual-mode AwaitableContext:
# ``await``-ing it opens the connector pool and leaves it open (vs the
# ``async with`` form, which closes on block exit). The producer's pool is
# long-lived — owned by the server lifespan and torn down once in
# ``drain()`` (close_async) on shutdown — so the bare ``await`` is correct
# here, unlike the scoped ``async with`` used for one-shot schema apply.
await app.open_async()
return cls(app)
async def send(self, task: DocumentTask, /) -> None:
key = _doc_queueing_lock(task)
deferrer = self._app.configure_task(INGEST_TASK_NAME, queueing_lock=key)
try:
await deferrer.defer_async(**asdict(task))
except AlreadyEnqueued:
# A todo job already exists for this doc; the next periodic scan
# re-evaluates freshness (placeholder/Qdrant modified_at only
# advances after a successful index), so this is not a lost update.
logger.debug("ingest.already_enqueued key=%s", key)
async def ensure_schema(self) -> None:
"""Apply the ingest-queue schema on the producer's already-open pool.
Lets the API lifespan provision the schema without a second open/close
cycle (it already opened the connector to build this producer) — the
``worker`` command shares the same single-open pattern.
"""
await _apply_ingest_queue_schema_open(self._app)
async def job_counts(self) -> dict[str, int]:
"""Ingest job counts by status (for the vector-sync status surface)."""
return await get_ingest_job_counts(self._app)
def clone(self) -> ProcrastinateTaskProducer:
return self
async def __aenter__(self) -> ProcrastinateTaskProducer:
return self
async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc: BaseException | None,
tb: TracebackType | None,
) -> None:
return None
# The bare suppression marker silences S7503 (async method without await):
# ``async def`` is required by the TaskProducer protocol; per-handle close is
# a no-op (the pool is owned by the lifespan, drained once on shutdown).
async def aclose(self) -> None: # NOSONAR
return None
async def drain(self) -> None:
"""Close the shared connector pool (lifespan shutdown only)."""
await self._app.close_async()
nextcloud_mcp_server/vector/queue/status.py
-195
View File
@@ -1,195 +0,0 @@
"""Status surface for ingest jobs (design §10.1, ``STATUS_BACKEND``).
- ``local``: in-process job state — the memory-stream buffer (today's behavior,
read directly by the status endpoint).
- ``bus``: a background subscriber consumes
``mcp.document.{ready,failed,reparsed}.{tenant_id}`` into a bounded in-process
:class:`StatusStore` that the status endpoint / ``nc_get_vector_sync_status``
read.
**Honest constraint (design §10.2 / decision):** MCP progress notifications
(``ctx.report_progress``) can only be emitted inside an *active tool-call
request*; a background subscriber has no ``ctx`` and the MCP SDK exposes no
out-of-band push. So "surface events as MCP progress notifications" is delivered
via this store (polled by the status endpoint / a tool), not an unsolicited
server push. True server-initiated progress / SSE is a follow-up — the
``on_event`` callback seam is left in place for it.
"""
from __future__ import annotations
import logging
from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Callable
if TYPE_CHECKING:
import anyio
from anyio.abc import TaskStatus
logger = logging.getLogger(__name__)
# Terminal/intermediate document states carried on mcp.document.* subjects.
_VALID_STATES = {"ready", "failed", "reparsed"}
class StatusStore:
"""Bounded LRU of recent document states keyed by ``doc_id``."""
def __init__(self, max_size: int = 10_000):
self._entries: OrderedDict[str, dict[str, Any]] = OrderedDict()
self._max = max_size
def record(
self,
doc_id: str,
state: str,
*,
content_hash: str | None = None,
transitioned_at: str | None = None,
) -> None:
self._entries[doc_id] = {
"state": state,
"content_hash": content_hash,
"transitioned_at": transitioned_at,
}
self._entries.move_to_end(doc_id)
while len(self._entries) > self._max:
self._entries.popitem(last=False)
def get(self, doc_id: str) -> dict[str, Any] | None:
return self._entries.get(doc_id)
def counts(self) -> dict[str, int]:
out: dict[str, int] = {}
for entry in self._entries.values():
out[entry["state"]] = out.get(entry["state"], 0) + 1
return out
def __len__(self) -> int:
return len(self._entries)
def state_from_subject(subject: str) -> str | None:
"""``mcp.document.<state>.<tenant_id>`` → ``<state>`` (or None if unknown)."""
parts = subject.split(".")
if len(parts) >= 4 and parts[0] == "mcp" and parts[1] == "document":
state = parts[2]
if state in _VALID_STATES:
return state
return None
class NatsStatusSubscriber:
"""Consumes ``mcp.document.*.{tenant_id}`` into a :class:`StatusStore`."""
def __init__(
self,
nc: Any,
js: Any,
tenant_id: str,
store: StatusStore,
on_event: Callable[[str, str], None] | None = None,
):
self._nc = nc
self._js = js
self.tenant_id = tenant_id
self.store = store
# on_event(doc_id, state) — seam for a future SSE / progress bridge.
self._on_event = on_event
def handle_message(self, subject: str, data: bytes) -> None:
"""Parse one status message into the store. Unit-testable without NATS."""
import json # noqa: PLC0415
state = state_from_subject(subject)
if state is None:
logger.warning("status.unknown_subject subject=%s", subject)
return
try:
payload = json.loads(data)
doc_id = payload["doc_id"]
except Exception:
logger.warning("status.bad_message subject=%s", subject, exc_info=True)
return
self.store.record(
doc_id,
state,
content_hash=payload.get("content_hash"),
transitioned_at=payload.get("transitioned_at"),
)
if self._on_event is not None:
self._on_event(doc_id, state)
@classmethod
async def connect(
cls, *, url: str, tenant_id: str, store: StatusStore
) -> NatsStatusSubscriber:
import nats # noqa: PLC0415
from .nats import warn_if_insecure_nats_url # noqa: PLC0415
warn_if_insecure_nats_url(url)
nc = await nats.connect(url)
js = nc.jetstream()
return cls(nc, js, tenant_id, store)
async def run(
self,
shutdown_event: anyio.Event,
*,
task_status: TaskStatus | None = None,
) -> None:
"""Durable pull-consumer loop. Requires a live broker (integration)."""
import anyio # noqa: PLC0415
import nats.errors # noqa: PLC0415
subject = f"mcp.document.*.{self.tenant_id}"
# Signal "task running" *before* the first (fallible) subscribe: bus
# status is a non-critical observability path, so a broker that isn't
# ready at startup should retry below rather than crash the lifespan.
# ``started()`` therefore means "the subscriber loop is running", not
# "the subscription succeeded".
if task_status is not None:
task_status.started()
sub = None
while not shutdown_event.is_set():
if sub is None:
try:
sub = await self._js.pull_subscribe(
subject, durable=f"mcp-status-{self.tenant_id}"
)
except Exception:
# Broker not ready / transient connect error: back off and
# retry the subscribe instead of giving up.
logger.warning(
"NATS status subscribe failed; retrying", exc_info=True
)
await anyio.sleep(5)
continue
try:
msgs = await sub.fetch(batch=16, timeout=5)
except nats.errors.TimeoutError:
# Expected when idle: no messages within the fetch window. Loop
# straight back to re-check shutdown — no log, no extra sleep.
continue
except Exception:
# Real broker error (disconnect, auth failure, stream deleted):
# drop the (possibly dead) subscription, back off, and
# re-subscribe on the next iteration rather than hot-spinning.
logger.warning(
"NATS status subscriber fetch failed; re-subscribing",
exc_info=True,
)
sub = None
await anyio.sleep(5)
continue
for msg in msgs:
self.handle_message(msg.subject, msg.data)
await msg.ack()
async def aclose(self) -> None:
try:
await self._nc.drain()
except Exception:
logger.warning("NATS status subscriber drain failed", exc_info=True)
nextcloud_mcp_server/vector/webhook_receiver.py
+4 -3
View File
@@ -42,9 +42,10 @@ async def handle_nextcloud_webhook(request: Request) -> JSONResponse:
"""Receive a Nextcloud webhook and queue a DocumentTask for vector sync.
Returns quickly so NC's webhook worker is not blocked. The task producer is
read from ``request.app.state.task_producer`` (the in-memory send stream in
local mode, or the NATS bus producer in external mode); when vector sync
isn't running we return 503 so NC retries delivery.
read from ``request.app.state.task_producer`` (the in-memory send stream when
``INGEST_QUEUE=memory``, or the procrastinate producer when
``INGEST_QUEUE=postgres``); when vector sync isn't running we return 503 so
NC retries delivery.
When ``WEBHOOK_SECRET`` is set, the request must carry
``Authorization: Bearer <secret>`` (registered via ``authData`` so NC