admin/mcp-nextcloud
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: replace NATS ingest with procrastinate Postgres queue (#183)

Browse Source 
Re-architect document ingest from the shared NATS-glued document-processor to a
per-tenant, in-process model owned by nextcloud-mcp-server (Deck #183). The MCP
server now owns both sides of ingest:

- Producer (api role): the scanner defers one job per changed document into the
  app's Postgres via procrastinate (queueing_lock dedup; no execution lock, so a
  crashed worker can't deadlock a doc — Qdrant upserts are idempotent).
- Consumer (worker role): `nextcloud-mcp-server worker` drains the queue and runs
  the existing process_document pipeline; a periodic task reclaims jobs orphaned
  in `doing` by a crash.

INGEST_QUEUE selects the transport (auto: postgres when DATABASE_URL is Postgres,
else the in-process anyio queue for SQLite/dev). procrastinate manages its own
tables (applied on a fresh DB at startup and by `db upgrade`). The vector-sync
status surface reads job counts from Postgres in postgres mode. procrastinate +
psycopg3 ship in the [postgres] extra; the app's own engine still uses asyncpg
(driver unification is a follow-up handled in the rendered Helm chart).

NATS JetStream, the Postgres-queue stub, the bus status subscriber, and nats-py
are removed.

BREAKING CHANGE: the external-NATS-ingest env vars are removed
(INGEST_MODE, STATUS_BACKEND, INGEST_BUS_URL, INGEST_BUS_NUM_REPLICAS,
FACT_EVENT_EMITTER). Use INGEST_QUEUE (memory|postgres) and the `worker`
command instead. TENANT_ID is retained (no longer NATS-subject-charset-validated).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
2026-06-03 04:11:11 +02:00
co-authored by Claude Opus 4.8
parent b91af923d2
commit 21b7922bac
27 changed files with 1369 additions and 1120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nextcloud_mcp_server/api/management.py
+23 -51
View File
@@ -293,52 +293,21 @@ async def get_vector_sync_status(request: Request) -> JSONResponse:
)
try:
# Bus status backend (INGEST_MODE=external): there is no in-process
# queue; pending/terminal state comes from the NATS status subscriber's
# store. indexed_documents stays the mode-independent Qdrant count.
if settings.status_backend == "bus":
store = getattr(request.app.state, "status_store", None)
indexed_count = 0
try:
qdrant_client = await get_qdrant_client()
count_result = await qdrant_client.count(
collection_name=settings.get_collection_name(),
count_filter=Filter(must=[get_placeholder_filter()]),
)
indexed_count = count_result.count
except Exception as e:
logger.warning("Failed to query Qdrant for indexed count: %s", e)
return JSONResponse(
{
"status": "idle",
"indexed_documents": indexed_count,
"pending_documents": 0,
"status_backend": "bus",
"recent_states": store.counts() if store is not None else {},
}
)
# Get document receive stream from app state (set by starlette_lifespan in app.py)
document_receive_stream = getattr(
request.app.state, "document_receive_stream", None
# Outstanding-work view depends on the queue backend (Deck #183):
# memory → stream buffer depth; postgres → procrastinate job counts.
from nextcloud_mcp_server.vector.ingest_status import ( # noqa: PLC0415
get_ingest_pending,
)
if document_receive_stream is None:
logger.debug("document_receive_stream not available in app state")
return JSONResponse(
{
"status": "unknown",
"indexed_documents": 0,
"pending_documents": 0,
"message": "Vector sync stream not initialized",
}
)
pending = await get_ingest_pending(
task_producer=getattr(request.app.state, "task_producer", None),
document_receive_stream=getattr(
request.app.state, "document_receive_stream", None
),
ingest_queue=settings.ingest_queue,
)
# Get pending count from stream statistics
stream_stats = document_receive_stream.statistics()
pending_count = stream_stats.current_buffer_used
# Get Qdrant client and query indexed count
# Get Qdrant client and query indexed count (backend-independent)
indexed_count = 0
try:
qdrant_client = await get_qdrant_client()
@@ -355,15 +324,18 @@ async def get_vector_sync_status(request: Request) -> JSONResponse:
# Continue with indexed_count = 0
# Determine status
status = "syncing" if pending_count > 0 else "idle"
status = "syncing" if pending.pending > 0 else "idle"
return JSONResponse(
{
"status": status,
"indexed_documents": indexed_count,
"pending_documents": pending_count,
}
)
body: dict[str, object] = {
"status": status,
"indexed_documents": indexed_count,
"pending_documents": pending.pending,
"ingest_queue": settings.ingest_queue,
}
if pending.job_counts is not None:
# Per-status breakdown (todo/doing/failed/…) on the postgres backend.
body["job_counts"] = pending.job_counts
return JSONResponse(body)
except Exception as e:
error_msg = _sanitize_error_for_client(e, "get_vector_sync_status")
nextcloud_mcp_server/app.py
+53 -98
View File
@@ -136,9 +136,8 @@ from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client
from nextcloud_mcp_server.vector.queue import (
MemoryTaskProducer,
TaskProducer,
build_external_producer,
build_producer,
)
from nextcloud_mcp_server.vector.queue.status import NatsStatusSubscriber, StatusStore
from nextcloud_mcp_server.vector.scanner import DocumentTask, scanner_task
from nextcloud_mcp_server.vector.webhook_receiver import handle_nextcloud_webhook
@@ -332,12 +331,10 @@ class VectorSyncState:
document_send_stream: MemoryObjectSendStream | None = None
document_receive_stream: MemoryObjectReceiveStream | None = None
# Ingest producer the scanner/webhook send to: the in-memory send stream
# (local mode) or the NATS bus producer (external mode). The webhook reads
# this; in local mode it is the same object as document_send_stream.
# (INGEST_QUEUE=memory) or the procrastinate producer (INGEST_QUEUE=postgres,
# Deck #183). The webhook reads this; in memory mode it is the same object as
# document_send_stream.
task_producer: "TaskProducer | None" = None
# Bus status store (STATUS_BACKEND=bus): populated by the NATS status
# subscriber, read by the vector-sync status endpoint. None in local mode.
status_store: "StatusStore | None" = None
shutdown_event: anyio.Event | None = None
scanner_wake_event: anyio.Event | None = None
# Long-lived task group used for fire-and-forget background work spawned
@@ -350,33 +347,6 @@ class VectorSyncState:
_vector_sync_state = VectorSyncState()
async def _build_status_subscriber(
settings: "Settings",
) -> "tuple[StatusStore | None, NatsStatusSubscriber | None]":
"""Build the bus status store + subscriber when STATUS_BACKEND=bus.
Returns ``(None, None)`` for local status (the status endpoint reads the
in-memory stream buffer instead). __post_init__ guarantees that bus status
only pairs with external ingest, so ingest_bus_url/tenant_id are set.
"""
if not (settings.ingest_mode == "external" and settings.status_backend == "bus"):
return None, None
# Defence-in-depth (robust under ``python -O``, which strips asserts):
# __post_init__ already guarantees these when status_backend == "bus".
if settings.ingest_bus_url is None or settings.tenant_id is None:
raise ValueError(
"STATUS_BACKEND=bus requires INGEST_BUS_URL and TENANT_ID "
"(guaranteed by Settings validation)"
)
store = StatusStore(max_size=settings.vector_sync_queue_max_size)
subscriber = await NatsStatusSubscriber.connect(
url=settings.ingest_bus_url,
tenant_id=settings.tenant_id,
store=store,
)
return store, subscriber
@dataclass
class AppContext:
"""Application context for BasicAuth mode."""
@@ -1681,39 +1651,40 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
# Orphan-sweep before scanner starts — card #101.
await _sweep_orphan_placeholders_if_enabled()
# Initialize shared state. INGEST_MODE selects the transport
# (design §10.1): local uses the in-memory anyio stream + the
# in-process processor pool; external publishes to NATS and runs no
# in-process consumer (the document-processor consumes).
external = settings.ingest_mode == "external"
# Initialize shared state. INGEST_QUEUE selects the transport
# (Deck #183): ``memory`` uses the in-process anyio stream + the
# in-process processor pool (SQLite/dev); ``postgres`` defers jobs to
# the per-tenant Postgres via procrastinate and runs no in-process
# consumer (the separate ``worker`` role drains the queue).
use_postgres = settings.ingest_queue == "postgres"
shutdown_event = anyio.Event()
scanner_wake_event = anyio.Event()
send_stream = None
receive_stream = None
task_producer: TaskProducer
if external:
task_producer = await build_external_producer(settings)
logger.info(
"Ingest mode external: publishing to %s", settings.ingest_bus_url
if use_postgres:
# Create procrastinate's tables before the scanner can defer.
# Lazy import: the procrastinate lib is a Postgres-only extra.
from nextcloud_mcp_server.vector.queue.procrastinate import ( # noqa: PLC0415
apply_ingest_queue_schema,
)
await apply_ingest_queue_schema()
task_producer = await build_producer(settings)
logger.info("Ingest queue: postgres (procrastinate); worker drains it")
else:
send_stream, receive_stream = anyio.create_memory_object_stream[
DocumentTask
](max_buffer_size=settings.vector_sync_queue_max_size)
task_producer = MemoryTaskProducer(send_stream)
# Bus status backend: subscribe to mcp.document.* into a store the
# status endpoint reads (STATUS_BACKEND=bus; external mode only).
status_store, status_subscriber = await _build_status_subscriber(settings)
# Store in app state for access from routes (ADR-007). In external
# Store in app state for access from routes (ADR-007). In postgres
# mode there is no in-memory stream, so document_send/receive_stream
# stay None; task_producer is the bus producer.
# stay None; task_producer is the procrastinate producer.
app.state.document_send_stream = send_stream
app.state.document_receive_stream = receive_stream
app.state.task_producer = task_producer
app.state.status_store = status_store
app.state.shutdown_event = shutdown_event
app.state.scanner_wake_event = scanner_wake_event
@@ -1721,7 +1692,6 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
_vector_sync_state.document_send_stream = send_stream
_vector_sync_state.document_receive_stream = receive_stream
_vector_sync_state.task_producer = task_producer
_vector_sync_state.status_store = status_store
_vector_sync_state.shutdown_event = shutdown_event
_vector_sync_state.scanner_wake_event = scanner_wake_event
logger.info("Vector sync state stored in module singleton")
@@ -1733,7 +1703,6 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
browser_app.state.document_send_stream = send_stream
browser_app.state.document_receive_stream = receive_stream
browser_app.state.task_producer = task_producer
browser_app.state.status_store = status_store
browser_app.state.shutdown_event = shutdown_event
browser_app.state.scanner_wake_event = scanner_wake_event
logger.info("Vector sync state shared with browser_app for /app")
@@ -1751,9 +1720,9 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
username,
)
# The in-process processor pool runs only in local mode; in
# external mode the document-processor service is the consumer.
if not external:
# The in-process processor pool runs only in memory mode; in
# postgres mode the out-of-process worker is the consumer.
if not use_postgres:
assert receive_stream is not None
for i in range(settings.vector_sync_processor_workers):
await tg.start(
@@ -1765,10 +1734,6 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
username,
)
# Bus status subscriber (STATUS_BACKEND=bus).
if status_subscriber is not None:
await tg.start(status_subscriber.run, shutdown_event)
# Expose this long-lived task group to request-path code that
# wants to spawn background work (e.g. ADR-019 verify-on-read
# eviction). Eviction coroutines have their own try/except, so
@@ -1776,9 +1741,9 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
_vector_sync_state.eviction_task_group = tg
logger.info(
"Background sync tasks started: 1 scanner + %s processors (ingest=%s)",
0 if external else settings.vector_sync_processor_workers,
settings.ingest_mode,
"Background sync tasks started: 1 scanner + %s processors (queue=%s)",
0 if use_postgres else settings.vector_sync_processor_workers,
settings.ingest_queue,
)
# Run MCP session manager and yield
@@ -1791,12 +1756,10 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
shutdown_event.set()
# Request path must not spawn into a cancelling group.
_vector_sync_state.eviction_task_group = None
# Drain the shared bus connection (external mode only).
# Close the procrastinate connector pool (postgres mode).
_drain = getattr(task_producer, "drain", None)
if external and _drain is not None:
if use_postgres and _drain is not None:
await _drain()
if status_subscriber is not None:
await status_subscriber.aclose()
await client.close()
# TaskGroup automatically cancels all tasks on exit
@@ -1904,11 +1867,12 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
except Exception as e:
logger.warning("App password cleanup failed (non-fatal): %s", e)
# Initialize shared state. INGEST_MODE selects the transport
# (design §10.1): local uses the in-memory anyio stream + the
# in-process processor pool; external publishes to NATS and runs
# no in-process consumer (the document-processor consumes).
external = settings.ingest_mode == "external"
# Initialize shared state. INGEST_QUEUE selects the transport
# (Deck #183): ``memory`` uses the in-process anyio stream + the
# in-process processor pool; ``postgres`` defers jobs via
# procrastinate and runs no in-process consumer (the separate
# ``worker`` role drains the queue).
use_postgres = settings.ingest_queue == "postgres"
shutdown_event = anyio.Event()
scanner_wake_event = anyio.Event()
@@ -1918,11 +1882,17 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
send_stream = None
receive_stream = None
task_producer: TaskProducer
if external:
task_producer = await build_external_producer(settings)
if use_postgres:
# Create procrastinate's tables before any scanner defers.
# Lazy import: procrastinate is a Postgres-only extra.
from nextcloud_mcp_server.vector.queue.procrastinate import ( # noqa: PLC0415
apply_ingest_queue_schema,
)
await apply_ingest_queue_schema()
task_producer = await build_producer(settings)
logger.info(
"Ingest mode external: publishing to %s",
settings.ingest_bus_url,
"Ingest queue: postgres (procrastinate); worker drains it"
)
else:
send_stream, receive_stream = anyio.create_memory_object_stream[
@@ -1930,17 +1900,10 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
](max_buffer_size=settings.vector_sync_queue_max_size)
task_producer = MemoryTaskProducer(send_stream)
# Bus status backend: subscribe to mcp.document.* into a store
# the status endpoint reads (STATUS_BACKEND=bus; external only).
status_store, status_subscriber = await _build_status_subscriber(
settings
)
# Store in app state for access from routes (ADR-007)
app.state.document_send_stream = send_stream
app.state.document_receive_stream = receive_stream
app.state.task_producer = task_producer
app.state.status_store = status_store
app.state.shutdown_event = shutdown_event
app.state.scanner_wake_event = scanner_wake_event
@@ -1948,7 +1911,6 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
_vector_sync_state.document_send_stream = send_stream
_vector_sync_state.document_receive_stream = receive_stream
_vector_sync_state.task_producer = task_producer
_vector_sync_state.status_store = status_store
_vector_sync_state.shutdown_event = shutdown_event
_vector_sync_state.scanner_wake_event = scanner_wake_event
logger.info("Vector sync state stored in module singleton")
@@ -1960,7 +1922,6 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
browser_app.state.document_send_stream = send_stream
browser_app.state.document_receive_stream = receive_stream
browser_app.state.task_producer = task_producer
browser_app.state.status_store = status_store
browser_app.state.shutdown_event = shutdown_event
browser_app.state.scanner_wake_event = scanner_wake_event
logger.info(
@@ -1991,9 +1952,9 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
tg,
)
# In-process processor pool runs only in local mode; in
# external mode the document-processor service consumes.
if not external:
# In-process processor pool runs only in memory mode; in
# postgres mode the out-of-process worker consumes.
if not use_postgres:
assert receive_stream is not None
for i in range(settings.vector_sync_processor_workers):
await tg.start(
@@ -2004,10 +1965,6 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
nextcloud_host_for_sync,
)
# Bus status subscriber (STATUS_BACKEND=bus).
if status_subscriber is not None:
await tg.start(status_subscriber.run, shutdown_event)
# Expose this long-lived task group to request-path code
# that wants to spawn background work (e.g. ADR-019
# verify-on-read eviction). Eviction coroutines have their
@@ -2015,9 +1972,9 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
_vector_sync_state.eviction_task_group = tg
logger.info(
"Background sync tasks started: 1 user manager + %s processors (ingest=%s)",
0 if external else settings.vector_sync_processor_workers,
settings.ingest_mode,
"Background sync tasks started: 1 user manager + %s processors (queue=%s)",
0 if use_postgres else settings.vector_sync_processor_workers,
settings.ingest_queue,
)
# Run MCP session manager and yield
@@ -2030,12 +1987,10 @@ def get_app(transport: str = "streamable-http", enabled_apps: list[str] | None =
shutdown_event.set()
# Request path must not spawn into a cancelling group.
_vector_sync_state.eviction_task_group = None
# Drain the shared bus connection (external only).
# Close the procrastinate connector pool (postgres).
_drain = getattr(task_producer, "drain", None)
if external and _drain is not None:
if use_postgres and _drain is not None:
await _drain()
if status_subscriber is not None:
await status_subscriber.aclose()
# Close token broker HTTP client
if token_broker._http_client:
await token_broker._http_client.aclose()
nextcloud_mcp_server/auth/userinfo_routes.py
+14 -11
View File
@@ -115,17 +115,20 @@ async def _get_processing_status(request: Request) -> dict[str, Any] | None:
return None
try:
# Get document receive stream from app state
document_receive_stream = getattr(
request.app.state, "document_receive_stream", None
# Outstanding-work view depends on the queue backend (Deck #183):
# memory → stream buffer depth; postgres → procrastinate job counts (the
# in-memory stream is absent in postgres mode, so don't early-return on it).
from nextcloud_mcp_server.vector.ingest_status import ( # noqa: PLC0415
get_ingest_pending,
)
if document_receive_stream is None:
logger.debug("document_receive_stream not available in app state")
return None
# Get pending count from stream statistics
stats = document_receive_stream.statistics()
pending_count = stats.current_buffer_used
pending = await get_ingest_pending(
task_producer=getattr(request.app.state, "task_producer", None),
document_receive_stream=getattr(
request.app.state, "document_receive_stream", None
),
ingest_queue=settings.ingest_queue,
)
# Get Qdrant client and query indexed count
indexed_count = 0
@@ -147,11 +150,11 @@ async def _get_processing_status(request: Request) -> dict[str, Any] | None:
# Continue with indexed_count = 0
# Determine status
status = "syncing" if pending_count > 0 else "idle"
status = "syncing" if pending.pending > 0 else "idle"
return {
"indexed_count": indexed_count,
"pending_count": pending_count,
"pending_count": pending.pending,
"status": status,
}
nextcloud_mcp_server/canonical.py
+5 -6
View File
@@ -2,9 +2,9 @@
The Astrolabe Cloud decomposition (design §2.3) fixes a single canonical JSON
encoding so hashes computed here match those computed independently by the
external document-processor and embedding-gateway services. Any drift in
separators, key ordering, or unicode handling would break NATS dedup keys,
Qdrant point-ID idempotency, and ACL-hash compatibility.
external embedding-gateway service. Any drift in separators, key ordering, or
unicode handling would break Qdrant point-ID idempotency and ACL-hash
compatibility.
"""
from __future__ import annotations
@@ -17,9 +17,8 @@ def canonical_json(obj: Any) -> bytes:
"""Encode ``obj`` to canonical JSON bytes.
Deterministic across implementations: sorted keys, no inter-token
whitespace, non-ASCII preserved (UTF-8). Consumers: the NATS
``Nats-Msg-Id`` dedup header (vector/queue/nats.py), Qdrant point IDs
(vector/payload_keys.py), and ACL hashes (acl_hash.py).
whitespace, non-ASCII preserved (UTF-8). Consumers: Qdrant point IDs
(vector/payload_keys.py) and ACL hashes (acl_hash.py).
"""
return json.dumps(
obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False
nextcloud_mcp_server/cli.py
+79
View File
@@ -281,6 +281,69 @@ def run(
)
@click.command()
@click.option(
"--concurrency",
"-c",
type=int,
default=None,
help="Max concurrent jobs. Defaults to VECTOR_SYNC_PROCESSOR_WORKERS.",
)
def worker(concurrency: int | None):
"""Run the ingest worker (Deck #183).
\b
Drains the per-tenant Postgres ingest queue (procrastinate): for each
deferred document it fetches the content as the owning user, parses, chunks,
embeds, and upserts into Qdrant. This is the scale-to-zero ``worker`` role of
the api/worker split; run it as a separate Deployment from the API pod.
\b
Requires INGEST_QUEUE=postgres (a PostgreSQL DATABASE_URL); procrastinate is
Postgres-only.
\b
Example:
$ export DATABASE_URL=postgresql+asyncpg://mcp:mcp@db/mcp
$ nextcloud-mcp-server worker -c 4
"""
import anyio # noqa: PLC0415
settings = get_settings()
if settings.ingest_queue != "postgres":
raise click.ClickException(
"worker requires INGEST_QUEUE=postgres (a PostgreSQL DATABASE_URL); "
f"resolved INGEST_QUEUE={settings.ingest_queue!r}"
)
from nextcloud_mcp_server.vector.queue.procrastinate import ( # noqa: PLC0415
INGEST_QUEUE_NAME,
apply_ingest_queue_schema,
get_procrastinate_app,
)
workers = concurrency or settings.vector_sync_processor_workers
app = get_procrastinate_app()
async def _run() -> None:
# Defensive apply (the always-on API pod is the authoritative applier).
await apply_ingest_queue_schema(app)
async with app.open_async():
click.echo(
f"Ingest worker started: queue={INGEST_QUEUE_NAME} concurrency={workers}"
)
await app.run_worker_async(
queues=[INGEST_QUEUE_NAME],
concurrency=workers,
install_signal_handlers=True,
# Drop succeeded jobs so the queue table stays lean and the KEDA
# queue-depth metric reflects only outstanding work.
delete_jobs="successful",
)
anyio.run(_run)
@click.group()
def db():
"""Database migration management commands."""
@@ -374,6 +437,21 @@ def upgrade(database_url: str | None, database_path: str | None, revision: str):
try:
click.echo(f"Upgrading database to revision: {revision}")
upgrade_database(url, revision)
# Apply procrastinate's ingest-queue schema on Postgres so a one-shot
# migration/init job provisions everything the api + worker roles need
# (Deck #183). Idempotent + lazy import (Postgres-only extra).
from nextcloud_mcp_server.config import is_sqlite_url # noqa: PLC0415
if not is_sqlite_url(url):
import anyio # noqa: PLC0415
from nextcloud_mcp_server.vector.queue.procrastinate import ( # noqa: PLC0415
apply_ingest_queue_schema,
build_app_for_url,
)
anyio.run(apply_ingest_queue_schema, build_app_for_url(url))
click.echo(click.style("✓ Ingest queue schema applied", fg="green"))
click.echo(click.style("✓ Database upgraded successfully", fg="green"))
except Exception as e:
click.echo(click.style(f"✗ Upgrade failed: {e}", fg="red"), err=True)
@@ -483,6 +561,7 @@ def cli():
cli.add_command(run)
cli.add_command(worker)
cli.add_command(db)
nextcloud_mcp_server/config.py
+108 -50
View File
@@ -169,14 +169,19 @@ _DEFAULTS: dict[str, Any] = {
# the current monolithic behavior; self-hosters who set none are
# unaffected. See docs/architecture/mcp-decomposition.md (sibling repo).
"embedding_provider": "autodetect", # autodetect | gateway
"ingest_mode": "local", # local | external
"status_backend": "local", # local | bus
# Ingest queue backend (Deck #183). None → auto: ``postgres`` (procrastinate)
# when DATABASE_URL is Postgres, else ``memory`` (the in-process anyio queue
# for SQLite/dev). procrastinate requires PostgreSQL.
"ingest_queue": None, # memory | postgres
# Process role for the per-tenant two-pod model (Deck #183). ``api`` runs the
# MCP/query server + scanner (defers jobs); the ``worker`` role is the
# `nextcloud-mcp-server worker` process that drains the queue. ``all`` keeps
# the monolithic behaviour (API + in-process SQLite pool).
"mcp_role": "all", # api | worker | all
"collection_metadata_source": "qdrant", # qdrant | api
# CP base URL for COLLECTION_METADATA_SOURCE=api (e.g. http://control-plane).
# Required only when the source is api.
"collection_metadata_api_url": None,
"fact_event_emitter": "none", # none | nats | stdout
"ingest_bus_url": None, # required when ingest_mode=external
"embedding_gateway_url": None, # required when embedding_provider=gateway
# Provider-namespaced model the gateway serves, "<provider>/<model>"
# (the gateway routes on the "/"-prefix; mistral/mistral-embed → Mistral
@@ -191,8 +196,7 @@ _DEFAULTS: dict[str, Any] = {
"embedding_gateway_client_id": None,
"embedding_gateway_client_secret": None,
"embedding_gateway_scope": None, # e.g. astrolabe-embedding-gateway/embed
"tenant_id": None, # NATS per-tenant subject token (UUID form)
"ingest_bus_num_replicas": 1, # JetStream stream replicas (prod: 3)
"tenant_id": None, # per-tenant identity (UUID form); see vector/payload_keys
# Query-side ACL pre-filter (design §11). OFF by default: a Qdrant
# `match any` on `acl_hash` excludes points missing the key, so enabling
# this before a real ACL backfill would silently drop legacy results.
@@ -699,12 +703,12 @@ class Settings:
# MCP decomposition hook points (design §10, opt-in). All defaults
# reproduce the current monolith; validated in __post_init__.
embedding_provider: str = "autodetect" # autodetect | gateway
ingest_mode: str = "local" # local | external
status_backend: str = "local" # local | bus
# Ingest queue backend (Deck #183). None → resolved in __post_init__ to
# ``postgres`` when DATABASE_URL is Postgres, else ``memory``.
ingest_queue: str | None = None # memory | postgres
mcp_role: str = "all" # api | worker | all (Deck #183 two-pod model)
collection_metadata_source: str = "qdrant" # qdrant | api
collection_metadata_api_url: str | None = None # CP URL when source=api
fact_event_emitter: str = "none" # none | nats | stdout
ingest_bus_url: str | None = None # required when ingest_mode=external
embedding_gateway_url: str | None = None # required when provider=gateway
embedding_gateway_model: str = (
"mistral/mistral-embed" # provider-namespaced id the gateway routes on
@@ -714,8 +718,7 @@ class Settings:
embedding_gateway_client_id: str | None = None
embedding_gateway_client_secret: str | None = None
embedding_gateway_scope: str | None = None
tenant_id: str | None = None # NATS per-tenant subject token (UUID form)
ingest_bus_num_replicas: int = 1 # JetStream stream replicas (prod: 3)
tenant_id: str | None = None # per-tenant identity (UUID form)
acl_prefilter_enabled: bool = False # query-side ACL pre-filter (§11); OFF
def __post_init__(self):
@@ -803,10 +806,8 @@ class Settings:
# the monolith, so deployments that set none of these pass through.
_enum_fields = {
"embedding_provider": {"autodetect", "gateway"},
"ingest_mode": {"local", "external"},
"status_backend": {"local", "bus"},
"mcp_role": {"api", "worker", "all"},
"collection_metadata_source": {"qdrant", "api"},
"fact_event_emitter": {"none", "nats", "stdout"},
}
for _field, _allowed in _enum_fields.items():
_val = (getattr(self, _field) or "").strip().lower()
@@ -816,21 +817,25 @@ class Settings:
f"{_field.upper()} must be one of {sorted(_allowed)}; got {_val!r}"
)
# Fail-fast: external ingest sources its status from the bus. With the
# in-process state machine empty, STATUS_BACKEND=local would leave
# status streams silently empty — crash loudly instead (design §10.1).
if self.status_backend == "local" and self.ingest_mode == "external":
raise RuntimeError(
"STATUS_BACKEND=local is incompatible with INGEST_MODE=external; "
"set STATUS_BACKEND=bus"
# Ingest queue backend (Deck #183). Unset → auto-derive from the
# database backend: procrastinate needs PostgreSQL, so SQLite/dev falls
# back to the in-process anyio queue. An explicit ``postgres`` against a
# SQLite DATABASE_URL is a misconfiguration — fail loudly.
_queue = (self.ingest_queue or "").strip().lower()
if not _queue:
_queue = "memory" if is_sqlite_url(get_database_url()) else "postgres"
if _queue not in {"memory", "postgres"}:
raise ValueError(
f"INGEST_QUEUE must be one of ['memory', 'postgres']; got {_queue!r}"
)
self.ingest_queue = _queue
if self.ingest_queue == "postgres" and is_sqlite_url(get_database_url()):
raise ValueError(
"INGEST_QUEUE=postgres requires a PostgreSQL DATABASE_URL "
"(procrastinate is Postgres-only); use INGEST_QUEUE=memory for "
"SQLite/dev"
)
# Conditional-required settings for the active hook points.
if self.ingest_mode == "external":
if not self.ingest_bus_url:
raise ValueError("INGEST_BUS_URL is required when INGEST_MODE=external")
if not self.tenant_id:
raise ValueError("TENANT_ID is required when INGEST_MODE=external")
if self.embedding_provider == "gateway" and not self.embedding_gateway_url:
raise ValueError(
"EMBEDDING_GATEWAY_URL is required when EMBEDDING_PROVIDER=gateway"
@@ -859,23 +864,6 @@ class Settings:
"client-credentials) or all left unset (unauthenticated gateway)"
)
# TENANT_ID is a NATS subject token; '.', '*', '>', and whitespace are
# reserved/illegal there and would silently break subscriptions (§3.4).
if self.tenant_id and (
any(c in self.tenant_id for c in ".*>")
or any(c.isspace() for c in self.tenant_id)
):
raise ValueError(
"TENANT_ID must not contain '.', '*', '>', or whitespace "
"(it is used as a NATS subject token)"
)
if self.ingest_bus_num_replicas < 1:
raise ValueError(
f"INGEST_BUS_NUM_REPLICAS must be >= 1; "
f"got {self.ingest_bus_num_replicas}"
)
# --- ADR-022 follow-up: deployment mode is the single source of truth ---
# The ENABLE_MULTI_USER_BASIC_AUTH and ENABLE_LOGIN_FLOW env vars were
# removed in favour of MCP_DEPLOYMENT_MODE. We do TWO things here:
@@ -1316,12 +1304,10 @@ def get_settings() -> Settings:
"excluded_tags": "EXCLUDED_TAGS",
# MCP decomposition hook points (design §10)
"embedding_provider": "EMBEDDING_PROVIDER",
"ingest_mode": "INGEST_MODE",
"status_backend": "STATUS_BACKEND",
"ingest_queue": "INGEST_QUEUE",
"mcp_role": "MCP_ROLE",
"collection_metadata_source": "COLLECTION_METADATA_SOURCE",
"collection_metadata_api_url": "COLLECTION_METADATA_API_URL",
"fact_event_emitter": "FACT_EVENT_EMITTER",
"ingest_bus_url": "INGEST_BUS_URL",
"embedding_gateway_url": "EMBEDDING_GATEWAY_URL",
"embedding_gateway_model": "EMBEDDING_GATEWAY_MODEL",
"embedding_gateway_token_url": "EMBEDDING_GATEWAY_TOKEN_URL",
@@ -1329,7 +1315,6 @@ def get_settings() -> Settings:
"embedding_gateway_client_secret": "EMBEDDING_GATEWAY_CLIENT_SECRET",
"embedding_gateway_scope": "EMBEDDING_GATEWAY_SCOPE",
"tenant_id": "TENANT_ID",
"ingest_bus_num_replicas": "INGEST_BUS_NUM_REPLICAS",
"acl_prefilter_enabled": "ACL_PREFILTER_ENABLED",
}
@@ -1405,3 +1390,76 @@ def get_database_ssl() -> bool | ssl.SSLContext | None:
if settings.database_verify_ssl is True:
return True
return None
def _pg_ssl_params() -> dict[str, str]:
"""Map the DATABASE_VERIFY_SSL / DATABASE_CA_BUNDLE settings to libpq
keyword params for psycopg3 (used by procrastinate, Deck #183).
psycopg/libpq takes ``sslmode`` (and ``sslrootcert``) rather than an
``ssl.SSLContext`` like asyncpg, so we translate :func:`get_database_ssl`'s
intent into the equivalent libpq settings:
- ``None`` (both unset) → ``{}`` (omit; libpq default ``prefer``,
matching the asyncpg default for cluster-local Postgres without TLS).
- ``False`` (DATABASE_VERIFY_SSL=false) → ``sslmode=require`` (encrypt but
do not verify the certificate).
- CA bundle set → ``sslmode=verify-full`` + ``sslrootcert``.
- ``True`` (verify, no bundle) → ``sslmode=verify-full`` (system trust).
"""
ssl_setting = get_database_ssl()
if ssl_setting is None:
return {}
if ssl_setting is False:
return {"sslmode": "require"}
settings = get_settings()
if settings.database_ca_bundle:
return {"sslmode": "verify-full", "sslrootcert": settings.database_ca_bundle}
return {"sslmode": "verify-full"}
def get_procrastinate_conninfo(database_url: str | None = None) -> str:
"""Build a libpq conninfo string for procrastinate's psycopg3 connector.
Derives the connection from ``DATABASE_URL`` (a SQLAlchemy URL such as
``postgresql+asyncpg://user:pass@host/db``): the SQLAlchemy driver suffix
(``+asyncpg``/``+psycopg``) is stripped and the parts are rendered via
:func:`psycopg.conninfo.make_conninfo`, which quotes values correctly (never
f-string the password). TLS settings are appended from :func:`_pg_ssl_params`.
This is driver-agnostic on purpose: procrastinate uses psycopg3 regardless of
which SQLAlchemy driver the app's own engine uses, so it works whether
``DATABASE_URL`` carries ``+asyncpg`` or ``+psycopg``.
TODO(Deck #183 follow-up, out-of-tree): unify the app's SQLAlchemy engine on
psycopg3 too (``postgresql+psycopg://``) and drop asyncpg, so the deployment
ships a single Postgres driver. This belongs in the rendered Helm chart
(set ``DATABASE_URL`` to a ``+psycopg`` URL) rather than rewriting the driver
in code — see charts repo, not this repo.
Raises ``ValueError`` for a non-Postgres URL — procrastinate is Postgres-only.
"""
from psycopg.conninfo import make_conninfo # noqa: PLC0415
from sqlalchemy.engine.url import make_url # noqa: PLC0415
url = make_url(database_url or get_database_url())
if not url.drivername.startswith("postgresql"):
raise ValueError(
"get_procrastinate_conninfo requires a PostgreSQL DATABASE_URL; "
f"got driver {url.drivername!r}"
)
params: dict[str, str] = {}
if url.host:
params["host"] = url.host
if url.port:
params["port"] = str(url.port)
if url.database:
params["dbname"] = url.database
if url.username:
params["user"] = url.username
if url.password:
params["password"] = url.password
params.update(_pg_ssl_params())
return make_conninfo(**params)
nextcloud_mcp_server/models/semantic.py
+11
View File
@@ -170,6 +170,17 @@ class VectorSyncStatusResponse(BaseResponse):
description='Sync status: "idle", "syncing", or "disabled"',
)
enabled: bool = Field(default=False, description="Whether vector sync is enabled")
ingest_queue: str | None = Field(
default=None,
description='Ingest queue backend: "memory" or "postgres" (Deck #183)',
)
job_counts: dict[str, int] | None = Field(
default=None,
description=(
"Per-status ingest job counts (todo/doing/failed/…) on the postgres "
"queue backend; None on the in-memory backend"
),
)
__all__ = [
nextcloud_mcp_server/server/semantic.py
+18 -18
View File
@@ -945,23 +945,21 @@ def configure_semantic_tools(mcp: FastMCP):
# missing attribute is a typo that should fail loudly. The
# value itself can legitimately be ``None`` before sync starts,
# which the check below handles.
# Outstanding-work view depends on the queue backend (Deck #183):
# memory → stream buffer depth; postgres → procrastinate job counts.
# Direct attribute access matches the eviction_task_group pattern at
# ``nc_semantic_search``: both AppContext and OAuthAppContext define
# these, so a missing attribute is a typo that should fail loudly.
from nextcloud_mcp_server.vector.ingest_status import ( # noqa: PLC0415
get_ingest_pending,
)
lifespan_ctx = ctx.request_context.lifespan_context
document_receive_stream = lifespan_ctx.document_receive_stream
if document_receive_stream is None:
logger.debug(
"document_receive_stream not available in lifespan context"
)
return VectorSyncStatusResponse(
indexed_count=0,
pending_count=0,
status="unknown",
enabled=True,
)
# Get pending count from stream statistics
stream_stats = document_receive_stream.statistics()
pending_count = stream_stats.current_buffer_used
pending = await get_ingest_pending(
task_producer=lifespan_ctx.task_producer,
document_receive_stream=lifespan_ctx.document_receive_stream,
ingest_queue=settings.ingest_queue,
)
# Get Qdrant client and query indexed count
indexed_count = 0
@@ -981,13 +979,15 @@ def configure_semantic_tools(mcp: FastMCP):
# Continue with indexed_count = 0
# Determine status
status = "syncing" if pending_count > 0 else "idle"
status = "syncing" if pending.pending > 0 else "idle"
return VectorSyncStatusResponse(
indexed_count=indexed_count,
pending_count=pending_count,
pending_count=pending.pending,
status=status,
enabled=True,
ingest_queue=settings.ingest_queue,
job_counts=pending.job_counts,
)
except Exception as e:
nextcloud_mcp_server/vector/ingest_status.py
+56
View File
@@ -0,0 +1,56 @@
"""Shared read model for the vector-sync status surface (Deck #183).
The status endpoints (``/api/v1/vector-sync/status``, the userinfo route, and
the ``nc_get_vector_sync_status`` MCP tool) all need the same "how much work is
outstanding" figure, computed differently per ``INGEST_QUEUE`` backend:
- ``memory`` — the in-process anyio stream's buffer depth (today's behavior).
- ``postgres`` — procrastinate job counts read from the per-tenant Postgres
(``todo`` + ``doing``), plus the per-status breakdown for observability.
``indexed_documents`` (the Qdrant placeholder count) is backend-independent and
stays at each call site.
"""
from __future__ import annotations
import logging
from dataclasses import dataclass
from typing import Any
logger = logging.getLogger(__name__)
@dataclass
class IngestPending:
"""Outstanding-work view for the active ingest queue backend."""
pending: int
# Per-status counts (todo/doing/failed/…) on the postgres backend; None on
# the memory backend, which has no durable per-status breakdown.
job_counts: dict[str, int] | None = None
async def get_ingest_pending(
*, task_producer: Any, document_receive_stream: Any, ingest_queue: str | None
) -> IngestPending:
"""Compute outstanding ingest work for the configured queue backend.
Never raises — a status surface must stay available even if the queue is
unreachable; failures degrade to ``pending=0``.
"""
if ingest_queue == "postgres":
counts: dict[str, int] = {}
if task_producer is not None and hasattr(task_producer, "job_counts"):
try:
counts = await task_producer.job_counts()
except Exception as e:
logger.warning("Failed to read ingest job counts: %s", e)
pending = counts.get("todo", 0) + counts.get("doing", 0)
return IngestPending(pending=pending, job_counts=counts)
if document_receive_stream is None:
return IngestPending(pending=0)
return IngestPending(
pending=document_receive_stream.statistics().current_buffer_used
)
nextcloud_mcp_server/vector/processor.py
+7 -2
View File
@@ -154,7 +154,9 @@ async def processor_task(
logger.info("Processor %s stopped", worker_id)
async def process_document(doc_task: DocumentTask, nc_client: NextcloudClient):
async def process_document(
doc_task: DocumentTask, nc_client: NextcloudClient, *, max_retries: int = 3
):
"""
Process a single document: fetch, tokenize, embed, store in Qdrant.
@@ -163,6 +165,10 @@ async def process_document(doc_task: DocumentTask, nc_client: NextcloudClient):
Args:
doc_task: Document task to process
nc_client: Authenticated Nextcloud client
max_retries: In-process indexing attempts before re-raising. The default
(3) suits the in-process SQLite pool, which has no durable retry. The
procrastinate worker passes ``1`` so durable retry is owned by the
queue (and survives worker crashes), avoiding compounding 3×N retries.
"""
start_time = time.time()
@@ -230,7 +236,6 @@ async def process_document(doc_task: DocumentTask, nc_client: NextcloudClient):
return
# Handle indexing with retry
max_retries = 3
retry_delay = 1.0
for attempt in range(max_retries):
nextcloud_mcp_server/vector/queue/__init__.py
+3 -3
View File
@@ -1,7 +1,7 @@
"""Ingest-path ports & adapters (design §10, hexagonal)."""
"""Ingest-path ports & adapters (design §10, hexagonal; Deck #183)."""
from .factory import build_external_producer
from .factory import build_producer
from .memory import MemoryTaskProducer
from .ports import TaskProducer
__all__ = ["MemoryTaskProducer", "TaskProducer", "build_external_producer"]
__all__ = ["MemoryTaskProducer", "TaskProducer", "build_producer"]
nextcloud_mcp_server/vector/queue/factory.py
+18 -41
View File
@@ -1,16 +1,17 @@
"""Composition root for the ingest producer (design §10).
"""Composition root for the ingest producer (Deck #183).
``build_external_producer`` is called by the lifespan only when
``INGEST_MODE=external``; local mode uses the in-memory stream directly (it
already satisfies :class:`TaskProducer`). The transport under ``external`` is
selected from the ``INGEST_BUS_URL`` scheme (``nats://`` now, ``postgres://``
later) so moving the external processor to Postgres needs no new INGEST_MODE.
The transport is selected from ``INGEST_QUEUE``:
- ``postgres`` → :class:`ProcrastinateTaskProducer`, which defers jobs into the
per-tenant Postgres for the out-of-process ``worker`` role to drain.
- ``memory`` (SQLite/dev default) → the in-process anyio stream, built inline by
the server lifespan (it owns both the send and receive ends), so it is not
produced here.
"""
from __future__ import annotations
import logging
from urllib.parse import urlsplit
from ...config import Settings
from .ports import TaskProducer
@@ -18,43 +19,19 @@ from .ports import TaskProducer
logger = logging.getLogger(__name__)
def _transport_for(url: str) -> str:
scheme = urlsplit(url).scheme.lower()
if scheme.startswith("postgres"):
return "postgres"
if not scheme.startswith("nats"):
logger.warning(
"INGEST_BUS_URL scheme %r is neither nats:// nor postgres://; "
"defaulting to the NATS transport",
scheme,
)
return "nats"
async def build_producer(settings: Settings) -> TaskProducer:
"""Build the Postgres (procrastinate) ingest producer.
async def build_external_producer(settings: Settings) -> TaskProducer:
"""Build the external-ingest producer for the configured transport.
Precondition: ``settings.ingest_mode == "external"`` (so __post_init__ has
guaranteed ``ingest_bus_url`` and ``tenant_id`` are set).
Precondition: ``settings.ingest_queue == "postgres"`` (the memory transport
is constructed inline by the lifespan because it needs the paired receive
stream for the in-process processor pool).
"""
# Defence-in-depth (robust under ``python -O``, which strips asserts):
# __post_init__ already guarantees these when ingest_mode == external.
if settings.ingest_bus_url is None or settings.tenant_id is None:
if settings.ingest_queue != "postgres":
raise ValueError(
"build_external_producer requires INGEST_BUS_URL and TENANT_ID "
"(guaranteed by Settings validation when INGEST_MODE=external)"
"build_producer is only for INGEST_QUEUE=postgres; the memory "
f"transport is built inline by the lifespan (got {settings.ingest_queue!r})"
)
transport = _transport_for(settings.ingest_bus_url)
if transport == "postgres":
from .postgres import PostgresTaskProducer # noqa: PLC0415
from .procrastinate import ProcrastinateTaskProducer # noqa: PLC0415
return await PostgresTaskProducer.connect(settings)
from .nats import NatsTaskProducer # noqa: PLC0415
return await NatsTaskProducer.connect(
url=settings.ingest_bus_url,
tenant_id=settings.tenant_id,
num_replicas=settings.ingest_bus_num_replicas,
)
return await ProcrastinateTaskProducer.connect()
nextcloud_mcp_server/vector/queue/nats.py
-173
View File
@@ -1,173 +0,0 @@
"""NATS JetStream ``TaskProducer`` — external ingest transport (design §3.4).
Publishes ``mcp.ingest.requested.{tenant_id}`` for the external
document-processor to consume. Translates the in-process ``DocumentTask`` into
the wire ``IngestMessage`` schema (mirrored in astrolabe-cloud-website's
``bus/messages.py``), with the JetStream ``Nats-Msg-Id`` dedup header per §3.4.
This server is only the *producer* on this transport; the document-processor
owns the consumer. ``nats-py`` is imported lazily so deployments that never
enable external ingest don't pay the import.
"""
from __future__ import annotations
import hashlib
import logging
from datetime import datetime, timezone
from types import TracebackType
from typing import TYPE_CHECKING, Any
from ...canonical import canonical_json
if TYPE_CHECKING:
from ..scanner import DocumentTask
logger = logging.getLogger(__name__)
STREAM_NAME = "mcp"
INGEST_SUBJECT_PREFIX = "mcp.ingest.requested"
def warn_if_insecure_nats_url(url: str) -> None:
"""Log a warning when the bus URL is not TLS-encrypted.
``nats://`` (and ``ws://``) carry tenant document metadata in cleartext;
production deployments should use ``tls://`` (or ``wss://``). We connect
regardless — this is an operator alert, not a hard failure.
"""
scheme = url.split("://", 1)[0].lower()
if scheme not in ("tls", "wss"):
logger.warning(
"NATS bus URL uses unencrypted transport (scheme=%s://); "
"use tls:// in production to protect document metadata in transit",
scheme,
)
def _modified_at_rfc3339(modified_at: int) -> str:
"""DocumentTask.modified_at is an epoch int (0 for deletes)."""
return datetime.fromtimestamp(int(modified_at), tz=timezone.utc).isoformat()
def _content_hash(task: DocumentTask) -> str:
"""etag is the change-detection token; fall back to modified_at when it is
absent (e.g. deletes, or sources whose etag we don't thread through).
TODO(follow-up, PR #814 review): thread etags for file / deck_card /
news_item scans too (only note scans pass etag today). Until then their
JetStream Nats-Msg-Id dedup keys off modified_at, which misses content
changes that leave modified_at unchanged (e.g. a file move/rename).
"""
return task.etag or str(task.modified_at)
def msg_id(tenant_id: str, doc_id: str, modified_at_rfc3339: str) -> str:
"""JetStream dedup header per §3.4. SHA-256 over canonical JSON (NOT the
BLAKE2b helper) — it is an opaque external header, not a stored field."""
return hashlib.sha256(
canonical_json(
{
"tenant_id": tenant_id,
"doc_id": doc_id,
"modified_at": modified_at_rfc3339,
}
)
).hexdigest()
class NatsTaskProducer:
"""Publishes ingest requests to NATS JetStream."""
def __init__(self, nc: Any, js: Any, tenant_id: str):
self._nc = nc
self._js = js
self.tenant_id = tenant_id
@classmethod
async def connect(
cls, *, url: str, tenant_id: str, num_replicas: int = 1
) -> NatsTaskProducer:
import nats # noqa: PLC0415 (lazy: optional dependency for external mode)
warn_if_insecure_nats_url(url)
nc = await nats.connect(url)
js = nc.jetstream()
await cls._ensure_stream(js, num_replicas)
logger.info("Connected NATS ingest producer: url=%s, tenant=%s", url, tenant_id)
return cls(nc, js, tenant_id)
@staticmethod
async def _ensure_stream(js: Any, num_replicas: int) -> None:
# noqa: PLC0415 — nats.js types are only importable once nats-py is present.
from nats.js.api import RetentionPolicy, StreamConfig # noqa: PLC0415
config = StreamConfig(
name=STREAM_NAME,
subjects=["mcp.>"],
retention=RetentionPolicy.LIMITS,
num_replicas=num_replicas,
)
try:
await js.add_stream(config=config)
logger.info("nats.stream_created stream=%s", STREAM_NAME)
except Exception as exc:
# add_stream is idempotent in spirit but errors when the stream
# already exists; treat as benign (mirrors the processor's
# ensure_stream). A genuinely broken broker surfaces on publish.
logger.info("nats.stream_exists_or_unavailable detail=%s", exc)
def ingest_message(self, task: DocumentTask) -> dict[str, Any]:
"""DocumentTask → wire IngestMessage dict (mirrors the sibling schema)."""
return {
"tenant_id": self.tenant_id,
"doc_id": task.doc_id,
"content_hash": _content_hash(task),
"modified_at": _modified_at_rfc3339(task.modified_at),
"doc_type": task.doc_type,
"operation": task.operation,
"user_id": task.user_id,
"file_path": task.file_path,
}
async def send(self, task: DocumentTask) -> None:
message = self.ingest_message(task)
subject = f"{INGEST_SUBJECT_PREFIX}.{self.tenant_id}"
headers = {
"Nats-Msg-Id": msg_id(self.tenant_id, task.doc_id, message["modified_at"])
}
await self._js.publish(subject, canonical_json(message), headers=headers)
# The scanner/oauth_sync use the producer as a clone-able async context
# manager (memory-stream semantics). The bus connection is owned by the
# lifespan, so cloning shares it and __aexit__ is a no-op (close happens via
# aclose() on shutdown).
def clone(self) -> NatsTaskProducer:
return self
async def __aenter__(self) -> NatsTaskProducer:
return self
async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc: BaseException | None,
tb: TracebackType | None,
) -> None:
return None
# The bare suppression marker silences S7503 (async method without await):
# ``async def`` is required by the TaskProducer protocol, but this handle
# close is a genuine no-op.
async def aclose(self) -> None: # NOSONAR
# Per-handle close (e.g. a per-user scanner clone exiting). The bus
# connection is shared and owned by the lifespan, so this is a no-op;
# the connection is torn down once via ``drain()`` on shutdown.
return None
async def drain(self) -> None:
"""Drain + close the shared NATS connection (lifespan shutdown only)."""
try:
await self._nc.drain()
except Exception:
logger.warning("NATS drain on shutdown failed", exc_info=True)
nextcloud_mcp_server/vector/queue/ports.py
+8 -8
View File
@@ -1,18 +1,18 @@
"""Ingest-path ports (design §10, hexagonal).
"""Ingest-path ports (design §10, hexagonal; Deck #183).
A ``TaskProducer`` is where the scanner + webhook receiver send a
``DocumentTask``. The transport behind it is swappable:
- the in-process anyio ``MemoryObjectSendStream`` (local ingest — the default),
- ``NatsTaskProducer`` (external ingest → the document-processor), and
- a future Postgres-queue producer (seam only; the *external* processor owns the
consume side — see ``postgres.py``).
- ``MemoryTaskProducer`` over the in-process anyio ``MemoryObjectSendStream``
(``INGEST_QUEUE=memory`` — the SQLite/dev default), and
- ``ProcrastinateTaskProducer`` (``INGEST_QUEUE=postgres``), which defers jobs
into the per-tenant Postgres for the out-of-process ``worker`` role to drain.
The protocol is exactly the surface the scanner/oauth_sync already use on the
memory stream (``send`` + ``clone`` + ``async with``), so both adapters drop in
with only a type-annotation change at the call sites. There is intentionally NO
consumer port: the MCP server's only in-process consumer is the memory stream;
when ingest is external the document-processor is the consumer, not this server.
with only a type-annotation change at the call sites. There is no consumer port:
in memory mode the in-process processor pool is the consumer; in postgres mode
the procrastinate worker is.
"""
from __future__ import annotations
nextcloud_mcp_server/vector/queue/postgres.py
-52
View File
@@ -1,52 +0,0 @@
"""Postgres-queue ``TaskProducer`` — documented seam, not implemented.
The external document-processor may later drain a Postgres-backed queue instead
of NATS to limit NATS operational overhead. Processing stays *external*; only
the transport changes — so on this server it would be a drop-in producer swap.
The consume side + the queue-table migration belong to that processor-side
refactor (cross-repo), NOT here. This stub exists so the transport value and the
``TaskProducer`` Protocol conformance are testable today.
"""
from __future__ import annotations
from types import TracebackType
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from ..scanner import DocumentTask
_NOT_IMPLEMENTED = (
"Postgres ingest transport is a documented seam. The external "
"document-processor owns the Postgres-drain refactor (transport swap only; "
"processing stays external). Use INGEST_BUS_URL=nats://… for now."
)
class PostgresTaskProducer:
@classmethod
async def connect(cls, settings: Any) -> PostgresTaskProducer:
raise NotImplementedError(_NOT_IMPLEMENTED)
async def send(self, task: DocumentTask) -> None: # pragma: no cover
raise NotImplementedError(_NOT_IMPLEMENTED)
def clone(self) -> PostgresTaskProducer: # pragma: no cover
return self
async def __aenter__(self) -> PostgresTaskProducer: # pragma: no cover
return self
async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc: BaseException | None,
tb: TracebackType | None,
) -> None: # pragma: no cover
return None
# The bare suppression marker silences S7503 (async method without await):
# ``async def`` is required by the TaskProducer protocol; this stub is a
# no-op until the Postgres transport lands.
async def aclose(self) -> None: # NOSONAR # pragma: no cover
return None
nextcloud_mcp_server/vector/queue/procrastinate.py
+335
View File
@@ -0,0 +1,335 @@
"""Procrastinate-backed ingest queue — the Postgres ``TaskProducer`` + worker
(Deck #183).
This replaces NATS JetStream and the old Postgres-queue stub. The MCP server now
owns *both* sides of ingest:
- **Producer** (API role / scanner) — :class:`ProcrastinateTaskProducer.send`
*defers* one ``ingest:process_document`` job per changed document into the
per-tenant Postgres (the same app DB; procrastinate manages its own tables).
- **Consumer** (worker role) — ``nextcloud-mcp-server worker`` runs
:func:`procrastinate.App.run_worker`, which drains the ``ingest`` queue and
invokes the existing :func:`process_document` pipeline.
Design notes:
- **No execution ``lock``, only ``queueing_lock``.** procrastinate does NOT
auto-reclaim ``doing`` jobs, so a per-doc execution lock would permanently
deadlock a document if a worker crashed mid-job. The Qdrant upsert is
idempotent (deterministic ``uuid5`` point IDs), so a concurrent/re-run is
harmless; ``queueing_lock`` (partial-unique on ``status='todo'``) is enough to
dedupe enqueues, and :func:`reclaim_stalled_ingest_jobs` retries jobs orphaned
in ``doing`` by a crash.
- procrastinate is Postgres-only and uses asyncio; ``anyio`` runs natively on the
asyncio backend, so the worker can call the anyio-based pipeline directly.
- Tasks are defined on a :class:`procrastinate.Blueprint` so the connector is
decoupled from the task registry: production binds a real
:class:`PsycopgConnector`; unit tests bind ``testing.InMemoryConnector``.
"""
from __future__ import annotations
import logging
from dataclasses import asdict
from datetime import datetime, timezone
from types import TracebackType
from typing import TYPE_CHECKING
from procrastinate import App, Blueprint, JobContext, PsycopgConnector, RetryStrategy
from procrastinate.connector import BaseConnector
from procrastinate.exceptions import AlreadyEnqueued
from ...config import get_procrastinate_conninfo, get_settings
from ..scanner import DocumentTask
if TYPE_CHECKING:
from ...client import NextcloudClient
logger = logging.getLogger(__name__)
# Single queue for document ingest. KEDA scales the worker Deployment on the
# depth of this queue (``SELECT count(*) FROM procrastinate_jobs WHERE
# queue_name='ingest' AND status='todo'``).
INGEST_QUEUE_NAME = "ingest"
# Blueprint namespace → registered task names are prefixed ``ingest:``.
_NAMESPACE = "ingest"
INGEST_TASK_NAME = f"{_NAMESPACE}:process_document"
_RECLAIM_TASK_NAME = f"{_NAMESPACE}:reclaim_stalled_jobs"
# A crashed worker leaves its job in ``doing``; reclaim it once its (per-worker)
# heartbeat is this many seconds stale. Sized well above the longest expected
# ``process_document`` (PDF render + embedding) so a slow-but-live worker — whose
# heartbeat stays current during a long job — is never reclaimed out from under
# itself.
_STALLED_AFTER_SECONDS = 300
# Tasks are defined as plain functions and registered onto a *fresh* Blueprint
# per app (see _build_ingest_blueprint). procrastinate's add_tasks_from mutates
# the blueprint's task names in place (namespace prefixing), so a single shared
# Blueprint cannot be added to more than one App — which the tests (in-memory +
# real Postgres) and any re-init path require.
async def process_document_task(
*,
user_id: str,
doc_id: str,
doc_type: str,
operation: str,
modified_at: int,
file_path: str | None = None,
metadata: dict[str, int | str] | None = None,
etag: str | None = None,
owner_id: str | None = None,
) -> None:
"""Worker entry: rebuild the DocumentTask, resolve creds, run the pipeline."""
# Local imports avoid a heavy import chain at blueprint-definition time
# (this module is also imported by the API pod just to defer jobs).
from ..oauth_sync import NotProvisionedError # noqa: PLC0415
from ..processor import process_document # noqa: PLC0415
task = DocumentTask(
user_id=user_id,
doc_id=doc_id,
doc_type=doc_type,
operation=operation,
modified_at=modified_at,
file_path=file_path,
metadata=metadata,
etag=etag,
owner_id=owner_id,
)
try:
nc_client = await _resolve_client(user_id)
except NotProvisionedError:
# A deprovisioned user must not pin a worker slot retrying forever.
# Finish the job as a no-op; the next scan re-enqueues once the user
# re-provisions an app password. Other errors (transient DB/network)
# propagate so procrastinate's retry strategy handles them.
logger.warning(
"ingest.skip_no_credentials user=%s doc=%s:%s", user_id, doc_type, doc_id
)
return
try:
# Durable retry is procrastinate's job; disable the in-process loop.
await process_document(task, nc_client, max_retries=1)
finally:
await nc_client.close()
async def reclaim_stalled_ingest_jobs(context: JobContext, timestamp: int) -> None:
"""Re-queue ingest jobs orphaned in ``doing`` by a crashed worker.
procrastinate prunes dead *workers* but does not reset their in-flight jobs;
without this they'd sit in ``doing`` forever. ``timestamp`` is procrastinate's
periodic-run marker (unused).
"""
manager = context.app.job_manager
retry_at = datetime.now(tz=timezone.utc)
reclaimed = 0
for job in await manager.get_stalled_jobs(
queue=INGEST_QUEUE_NAME, seconds_since_heartbeat=_STALLED_AFTER_SECONDS
):
if job.id is None:
continue
await manager.retry_job_by_id_async(job_id=job.id, retry_at=retry_at)
reclaimed += 1
if reclaimed:
logger.warning("ingest.reclaimed_stalled_jobs count=%d", reclaimed)
async def _resolve_client(user_id: str) -> NextcloudClient:
"""Build an authenticated NextcloudClient for ``user_id`` in the worker.
Single-user BasicAuth uses the shared env credentials; every multi-user mode
resolves the user's locally-stored app password (BasicAuth).
"""
from ...client import NextcloudClient # noqa: PLC0415
from ...config_validators import AuthMode, detect_auth_mode # noqa: PLC0415
settings = get_settings()
if detect_auth_mode(settings) == AuthMode.SINGLE_USER_BASIC:
return NextcloudClient.from_env()
from ..oauth_sync import get_user_client_basic_auth # noqa: PLC0415
host = settings.nextcloud_host
if not host:
raise ValueError("NEXTCLOUD_HOST is required for multi-user ingest")
return await get_user_client_basic_auth(user_id, host)
def _build_ingest_blueprint() -> Blueprint:
"""Create a fresh Blueprint with the ingest tasks registered.
Fresh per call because ``add_tasks_from`` mutates the blueprint's task names
(namespace prefixing), so the same Blueprint cannot be reused across Apps.
"""
bp = Blueprint()
# Durable retry owned by the queue (survives worker crashes); the in-process
# retry loop in process_document is disabled on this path via max_retries=1.
bp.task(
name="process_document",
queue=INGEST_QUEUE_NAME,
retry=RetryStrategy(max_attempts=5, exponential_wait=4),
)(process_document_task)
reclaim = bp.task(
name="reclaim_stalled_jobs", queue=INGEST_QUEUE_NAME, pass_context=True
)(reclaim_stalled_ingest_jobs)
bp.periodic(cron="*/5 * * * *", periodic_id="reclaim_stalled_ingest")(reclaim)
return bp
def build_app(connector: BaseConnector) -> App:
"""Build an App for the given connector with the ingest tasks registered.
Shared by production (:func:`get_procrastinate_app`) and tests (which pass a
``testing.InMemoryConnector``).
"""
app = App(connector=connector)
app.add_tasks_from(_build_ingest_blueprint(), namespace=_NAMESPACE)
return app
def build_app_for_url(database_url: str) -> App:
"""Build an App bound to an explicit Postgres URL (for the CLI, which may
target a ``--database-url`` that differs from the ``DATABASE_URL`` env)."""
return build_app(
PsycopgConnector(conninfo=get_procrastinate_conninfo(database_url))
)
_app: App | None = None
def get_procrastinate_app() -> App:
"""Process-wide procrastinate App bound to the Postgres app database."""
global _app
if _app is None:
_app = build_app(PsycopgConnector(conninfo=get_procrastinate_conninfo()))
return _app
async def _ingest_schema_present(app: App) -> bool:
row = await app.connector.execute_query_one_async(
"SELECT to_regclass('procrastinate_jobs') IS NOT NULL AS present"
)
return bool(row["present"])
async def apply_ingest_queue_schema(app: App | None = None) -> None:
"""Create procrastinate's tables on a fresh database (apply-if-absent).
procrastinate's ``schema.sql`` uses bare ``CREATE TYPE``/``CREATE TABLE``
(not ``IF NOT EXISTS``), so it errors if re-applied — it is meant to run
once on a fresh DB. We skip when ``procrastinate_jobs`` already exists;
*version* upgrades use procrastinate's own migration files (operator-run, a
lineage independent of the app's Alembic schema).
Safe to call concurrently across rolling-update pods without an advisory
lock: Postgres DDL is transactional and procrastinate applies the whole
schema in one transaction, so a pod that loses the race rolls back cleanly
and we treat the resulting error as benign once the schema is present.
Opens a short-lived connection, so it is safe to call from the CLI
(``db upgrade`` / worker startup).
"""
app = app or get_procrastinate_app()
async with app.open_async():
if await _ingest_schema_present(app):
logger.debug("ingest queue schema already present; skipping apply")
return
try:
await app.schema_manager.apply_schema_async()
logger.info("Applied procrastinate ingest queue schema")
except Exception:
# A racing pod likely committed the schema while our transaction
# rolled back atomically. Benign iff the schema is now present.
if await _ingest_schema_present(app):
logger.info("Ingest queue schema applied concurrently by another pod")
return
raise
# Job-status keys procrastinate flattens into each list_queues row (alongside
# ``name`` and ``jobs_count``). ``aborting`` is legacy/unused since v3.0.0.
_JOB_STATUSES = ("todo", "doing", "succeeded", "failed", "cancelled", "aborted")
async def get_ingest_job_counts(app: App | None = None) -> dict[str, int]:
"""Return ingest job counts by status (``todo``/``doing``/``failed``/…).
Reads procrastinate's per-queue stats via the manager API (not hand-written
SQL) so a future schema bump doesn't silently break the status surface. The
manager flattens its per-status ``stats`` into top-level row keys, so we read
the known status keys directly. Assumes the app's connector is already open.
"""
app = app or get_procrastinate_app()
counts: dict[str, int] = {}
for row in await app.job_manager.list_queues_async(queue=INGEST_QUEUE_NAME):
for status in _JOB_STATUSES:
if status in row:
counts[status] = counts.get(status, 0) + int(row[status])
return counts
def _doc_queueing_lock(task: DocumentTask) -> str:
"""Per-document enqueue-dedup key (partial-unique on ``status='todo'``)."""
return f"{task.user_id}:{task.doc_type}:{task.doc_id}"
class ProcrastinateTaskProducer:
"""``TaskProducer`` that defers ingest jobs into Postgres via procrastinate.
The App's connector pool is owned by the server lifespan (opened once,
closed on shutdown), so ``clone``/``aenter``/``aexit``/``aclose`` are no-ops
— there is no per-handle resource like the memory stream's clones.
"""
def __init__(self, app: App):
self._app = app
@classmethod
async def connect(cls) -> ProcrastinateTaskProducer:
app = get_procrastinate_app()
await app.open_async()
return cls(app)
async def send(self, task: DocumentTask, /) -> None:
key = _doc_queueing_lock(task)
deferrer = self._app.configure_task(INGEST_TASK_NAME, queueing_lock=key)
try:
await deferrer.defer_async(**asdict(task))
except AlreadyEnqueued:
# A todo job already exists for this doc; the next periodic scan
# re-evaluates freshness (placeholder/Qdrant modified_at only
# advances after a successful index), so this is not a lost update.
logger.debug("ingest.already_enqueued key=%s", key)
async def job_counts(self) -> dict[str, int]:
"""Ingest job counts by status (for the vector-sync status surface)."""
return await get_ingest_job_counts(self._app)
def clone(self) -> ProcrastinateTaskProducer:
return self
async def __aenter__(self) -> ProcrastinateTaskProducer:
return self
async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc: BaseException | None,
tb: TracebackType | None,
) -> None:
return None
# The bare suppression marker silences S7503 (async method without await):
# ``async def`` is required by the TaskProducer protocol; per-handle close is
# a no-op (the pool is owned by the lifespan, drained once on shutdown).
async def aclose(self) -> None: # NOSONAR
return None
async def drain(self) -> None:
"""Close the shared connector pool (lifespan shutdown only)."""
await self._app.close_async()
nextcloud_mcp_server/vector/queue/status.py
-195
View File
@@ -1,195 +0,0 @@
"""Status surface for ingest jobs (design §10.1, ``STATUS_BACKEND``).
- ``local``: in-process job state — the memory-stream buffer (today's behavior,
read directly by the status endpoint).
- ``bus``: a background subscriber consumes
``mcp.document.{ready,failed,reparsed}.{tenant_id}`` into a bounded in-process
:class:`StatusStore` that the status endpoint / ``nc_get_vector_sync_status``
read.
**Honest constraint (design §10.2 / decision):** MCP progress notifications
(``ctx.report_progress``) can only be emitted inside an *active tool-call
request*; a background subscriber has no ``ctx`` and the MCP SDK exposes no
out-of-band push. So "surface events as MCP progress notifications" is delivered
via this store (polled by the status endpoint / a tool), not an unsolicited
server push. True server-initiated progress / SSE is a follow-up — the
``on_event`` callback seam is left in place for it.
"""
from __future__ import annotations
import logging
from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Callable
if TYPE_CHECKING:
import anyio
from anyio.abc import TaskStatus
logger = logging.getLogger(__name__)
# Terminal/intermediate document states carried on mcp.document.* subjects.
_VALID_STATES = {"ready", "failed", "reparsed"}
class StatusStore:
"""Bounded LRU of recent document states keyed by ``doc_id``."""
def __init__(self, max_size: int = 10_000):
self._entries: OrderedDict[str, dict[str, Any]] = OrderedDict()
self._max = max_size
def record(
self,
doc_id: str,
state: str,
*,
content_hash: str | None = None,
transitioned_at: str | None = None,
) -> None:
self._entries[doc_id] = {
"state": state,
"content_hash": content_hash,
"transitioned_at": transitioned_at,
}
self._entries.move_to_end(doc_id)
while len(self._entries) > self._max:
self._entries.popitem(last=False)
def get(self, doc_id: str) -> dict[str, Any] | None:
return self._entries.get(doc_id)
def counts(self) -> dict[str, int]:
out: dict[str, int] = {}
for entry in self._entries.values():
out[entry["state"]] = out.get(entry["state"], 0) + 1
return out
def __len__(self) -> int:
return len(self._entries)
def state_from_subject(subject: str) -> str | None:
"""``mcp.document.<state>.<tenant_id>`` → ``<state>`` (or None if unknown)."""
parts = subject.split(".")
if len(parts) >= 4 and parts[0] == "mcp" and parts[1] == "document":
state = parts[2]
if state in _VALID_STATES:
return state
return None
class NatsStatusSubscriber:
"""Consumes ``mcp.document.*.{tenant_id}`` into a :class:`StatusStore`."""
def __init__(
self,
nc: Any,
js: Any,
tenant_id: str,
store: StatusStore,
on_event: Callable[[str, str], None] | None = None,
):
self._nc = nc
self._js = js
self.tenant_id = tenant_id
self.store = store
# on_event(doc_id, state) — seam for a future SSE / progress bridge.
self._on_event = on_event
def handle_message(self, subject: str, data: bytes) -> None:
"""Parse one status message into the store. Unit-testable without NATS."""
import json # noqa: PLC0415
state = state_from_subject(subject)
if state is None:
logger.warning("status.unknown_subject subject=%s", subject)
return
try:
payload = json.loads(data)
doc_id = payload["doc_id"]
except Exception:
logger.warning("status.bad_message subject=%s", subject, exc_info=True)
return
self.store.record(
doc_id,
state,
content_hash=payload.get("content_hash"),
transitioned_at=payload.get("transitioned_at"),
)
if self._on_event is not None:
self._on_event(doc_id, state)
@classmethod
async def connect(
cls, *, url: str, tenant_id: str, store: StatusStore
) -> NatsStatusSubscriber:
import nats # noqa: PLC0415
from .nats import warn_if_insecure_nats_url # noqa: PLC0415
warn_if_insecure_nats_url(url)
nc = await nats.connect(url)
js = nc.jetstream()
return cls(nc, js, tenant_id, store)
async def run(
self,
shutdown_event: anyio.Event,
*,
task_status: TaskStatus | None = None,
) -> None:
"""Durable pull-consumer loop. Requires a live broker (integration)."""
import anyio # noqa: PLC0415
import nats.errors # noqa: PLC0415
subject = f"mcp.document.*.{self.tenant_id}"
# Signal "task running" *before* the first (fallible) subscribe: bus
# status is a non-critical observability path, so a broker that isn't
# ready at startup should retry below rather than crash the lifespan.
# ``started()`` therefore means "the subscriber loop is running", not
# "the subscription succeeded".
if task_status is not None:
task_status.started()
sub = None
while not shutdown_event.is_set():
if sub is None:
try:
sub = await self._js.pull_subscribe(
subject, durable=f"mcp-status-{self.tenant_id}"
)
except Exception:
# Broker not ready / transient connect error: back off and
# retry the subscribe instead of giving up.
logger.warning(
"NATS status subscribe failed; retrying", exc_info=True
)
await anyio.sleep(5)
continue
try:
msgs = await sub.fetch(batch=16, timeout=5)
except nats.errors.TimeoutError:
# Expected when idle: no messages within the fetch window. Loop
# straight back to re-check shutdown — no log, no extra sleep.
continue
except Exception:
# Real broker error (disconnect, auth failure, stream deleted):
# drop the (possibly dead) subscription, back off, and
# re-subscribe on the next iteration rather than hot-spinning.
logger.warning(
"NATS status subscriber fetch failed; re-subscribing",
exc_info=True,
)
sub = None
await anyio.sleep(5)
continue
for msg in msgs:
self.handle_message(msg.subject, msg.data)
await msg.ack()
async def aclose(self) -> None:
try:
await self._nc.drain()
except Exception:
logger.warning("NATS status subscriber drain failed", exc_info=True)
nextcloud_mcp_server/vector/webhook_receiver.py
+4 -3
View File
@@ -42,9 +42,10 @@ async def handle_nextcloud_webhook(request: Request) -> JSONResponse:
"""Receive a Nextcloud webhook and queue a DocumentTask for vector sync.
Returns quickly so NC's webhook worker is not blocked. The task producer is
read from ``request.app.state.task_producer`` (the in-memory send stream in
local mode, or the NATS bus producer in external mode); when vector sync
isn't running we return 503 so NC retries delivery.
read from ``request.app.state.task_producer`` (the in-memory send stream when
``INGEST_QUEUE=memory``, or the procrastinate producer when
``INGEST_QUEUE=postgres``); when vector sync isn't running we return 503 so
NC retries delivery.
When ``WEBHOOK_SECRET`` is set, the request must carry
``Authorization: Bearer <secret>`` (registered via ``authData`` so NC