diff --git a/docs/configuration.md b/docs/configuration.md index ae6d229b..ce535de7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -799,9 +799,10 @@ EMBEDDING_GATEWAY_TOKEN_URL=... EMBEDDING_GATEWAY_CLIENT_ID=... EMBEDDING_GATEWAY_CLIENT_SECRET=... -# Ingest queue backend. Default (unset) auto-derives from DATABASE_URL: -# - PostgreSQL DATABASE_URL → "postgres" (the procrastinate queue) -# - SQLite / unset → "memory" (the in-process anyio queue) +# Ingest queue backend. Default (unset) is "memory" — the in-process anyio +# queue — *regardless of DATABASE_URL*. procrastinate is strictly opt-in: set +# INGEST_QUEUE=postgres to split ingest into a separate worker (requires a +# PostgreSQL DATABASE_URL). A Postgres DATABASE_URL alone never enables it. INGEST_QUEUE=postgres # memory | postgres # Process role (informational; the worker is launched via the `worker` command): MCP_ROLE=all # api | worker | all (default) @@ -810,8 +811,13 @@ TENANT_ID= # per-tenant identity (used in collection naming) ### Postgres ingest queue + worker (api/worker split) -When `INGEST_QUEUE=postgres` (a PostgreSQL `DATABASE_URL`), the scanner **defers** -one job per changed document into the app's Postgres via +This is **opt-in**. By default (`INGEST_QUEUE=memory`) the scanner processes +changed documents in-process via anyio task groups in the API pod — no +procrastinate, no separate worker, even when `DATABASE_URL` is Postgres. + +When you explicitly set `INGEST_QUEUE=postgres` (against a PostgreSQL +`DATABASE_URL`), the scanner instead **defers** one job per changed document +into the app's Postgres via [procrastinate](https://procrastinate.readthedocs.io); a separate **worker** process drains the queue (fetch → chunk → embed → upsert Qdrant). Run the two roles as separate Deployments from the same image: diff --git a/nextcloud_mcp_server/config.py b/nextcloud_mcp_server/config.py index 37516c13..96cce29b 100644 --- a/nextcloud_mcp_server/config.py +++ b/nextcloud_mcp_server/config.py @@ -171,9 +171,10 @@ _DEFAULTS: dict[str, Any] = { # the current monolithic behavior; self-hosters who set none are # unaffected. See docs/architecture/mcp-decomposition.md (sibling repo). "embedding_provider": "autodetect", # autodetect | gateway - # Ingest queue backend (Deck #183). None → auto: ``postgres`` (procrastinate) - # when DATABASE_URL is Postgres, else ``memory`` (the in-process anyio queue - # for SQLite/dev). procrastinate requires PostgreSQL. + # Ingest queue backend (Deck #183). None → ``memory`` (the in-process anyio + # queue): procrastinate is strictly opt-in, even on a Postgres DATABASE_URL. + # Set ``postgres`` explicitly to split ingest into a procrastinate worker; + # that requires a PostgreSQL DATABASE_URL. "ingest_queue": None, # memory | postgres # Process role for the per-tenant two-pod model (Deck #183). ``api`` runs the # MCP/query server + scanner (defers jobs); the ``worker`` role is the @@ -829,13 +830,15 @@ class Settings: f"{_field.upper()} must be one of {sorted(_allowed)}; got {_val!r}" ) - # Ingest queue backend (Deck #183). Unset → auto-derive from the - # database backend: procrastinate needs PostgreSQL, so SQLite/dev falls - # back to the in-process anyio queue. An explicit ``postgres`` against a - # SQLite DATABASE_URL is a misconfiguration — fail loudly. + # Ingest queue backend (Deck #183). Procrastinate is opt-in: unset → + # ``memory`` (the in-process anyio queue) regardless of DB backend, so a + # Postgres DATABASE_URL alone never silently spins up a procrastinate + # worker. ``postgres`` must be set explicitly, and an explicit + # ``postgres`` against a SQLite DATABASE_URL is a misconfiguration — + # fail loudly below. _queue = (self.ingest_queue or "").strip().lower() if not _queue: - _queue = "memory" if is_sqlite_url(get_database_url()) else "postgres" + _queue = "memory" if _queue not in {"memory", "postgres"}: raise ValueError( f"INGEST_QUEUE must be one of ['memory', 'postgres']; got {_queue!r}" diff --git a/tests/unit/test_decomposition_config.py b/tests/unit/test_decomposition_config.py index 9a2869a0..7f4b31a1 100644 --- a/tests/unit/test_decomposition_config.py +++ b/tests/unit/test_decomposition_config.py @@ -59,13 +59,24 @@ class TestIngestQueueResolution: with pytest.raises(ValueError, match="INGEST_QUEUE=postgres requires"): Settings(ingest_queue="postgres") - def test_auto_postgres_when_database_url_is_postgres(self, monkeypatch): + def test_memory_default_even_on_postgres_url(self, monkeypatch): + # Procrastinate is opt-in: a Postgres DATABASE_URL with INGEST_QUEUE + # unset must NOT silently enable procrastinate. Default → memory. monkeypatch.setattr( config_module, "get_database_url", lambda: "postgresql+asyncpg://mcp:mcp@db/mcp", ) - assert Settings().ingest_queue == "postgres" + assert Settings().ingest_queue == "memory" + + def test_explicit_postgres_on_postgres_url(self, monkeypatch): + # Opting in explicitly against a Postgres URL is the supported path. + monkeypatch.setattr( + config_module, + "get_database_url", + lambda: "postgresql+asyncpg://mcp:mcp@db/mcp", + ) + assert Settings(ingest_queue="postgres").ingest_queue == "postgres" def test_explicit_memory_on_postgres_url(self, monkeypatch): monkeypatch.setattr(