Adds a `DATABASE_URL` setting that lets `RefreshTokenStorage` run against
any SQLAlchemy async backend, primarily `postgresql+asyncpg://...` for
HA k8s deployments. Default behavior is unchanged: when `DATABASE_URL` is
unset the server falls back to the existing `TOKEN_STORAGE_DB` path /
ephemeral SQLite tempfile.
Why
---
Today every MCP pod needs its own PVC to hold the SQLite file, which
pins the Deployment to one replica and blocks horizontal scaling. With
this change, operators can point all replicas at a shared Postgres
(CNPG, RDS, etc.) and the pods become stateless. Encryption stays in
Python (Fernet); the database only sees ciphertext.
What changed
------------
- `config.get_database_url()` resolves DATABASE_URL → TOKEN_STORAGE_DB →
ephemeral tempfile in that priority order.
- `RefreshTokenStorage` builds a process-shared `AsyncEngine` in
`initialize()`. SQLite gets NullPool; Postgres gets pool_size=10,
max_overflow=20, pool_pre_ping=True. 30 aiosqlite call sites adapted
via a thin `_DBConn` / `_Cursor` / `_Row` / `_ExecuteCtx` shim so
existing method bodies need no churn beyond the connection
context-manager swap.
- 7 `INSERT OR REPLACE` statements rewritten as portable
`INSERT ... ON CONFLICT (...) DO UPDATE` (SQLite ≥ 3.24, Postgres ≥ 9.5).
- `sqlite_master` legacy-detection lookup replaced with SQLAlchemy
inspector so the path works against either backend.
- File-permission hardening + parent-dir creation gated on
`is_sqlite_url(...)` — centralized backends manage their own filesystem.
- Alembic migrations 001/002/003/005 converted from raw `op.execute(SQL)`
to portable `op.create_table()` / `op.create_index()` with SQLAlchemy
types. All timestamp columns are `sa.BigInteger` so Postgres allocates
BIGINT (unix epochs don't fit in INT4). SQLite treats BIGINT as
INTEGER, so existing deployments at revision 006 see no schema drift.
- `migrations.py` + CLI take URLs; `db {upgrade,downgrade,current,history}`
gain `--database-url / -u` alongside the legacy `--database-path / -d`.
`get_current_revision()` uses SQLAlchemy inspector instead of raw
sqlite3, so the CLI works against Postgres too.
- `docker-compose.yml` adds a `postgres-test` service under the
`postgres` profile (pinned `postgres:16-alpine` digest) for
integration testing.
- Unit storage tests parametrized over backends via shared
`tests/fixtures/storage_backend.py` — every test in
`test_app_password_storage.py` and `test_webhook_storage.py` runs
once per backend that is available. Postgres is opted in by
`TEST_DATABASE_URL`.
- New `tests/integration/test_storage_postgres.py` (5 tests, marked
`postgres` + `integration`) covers refresh-token, app-password,
OAuth-session, webhook, and audit-log paths end-to-end on Postgres.
- New `docs/ADR-026-pluggable-database-backend.md` records the decision;
`docs/configuration.md` documents `DATABASE_URL` with examples.
Out of scope
------------
- No SQLite → Postgres data migration tool (clean cutover; tokens reissue
on next login, webhooks re-register on next sync tick).
- This repo does not provision Postgres. The matching helm chart change
lives in cbcoutinho/helm-charts (database.url / existingSecret values).
Verification
------------
- `uv run pytest tests/unit/` — 1012 passed, SQLite path unchanged.
- `docker compose --profile postgres up -d postgres-test`
- `TEST_DATABASE_URL=... uv run pytest tests/integration/test_storage_postgres.py -m postgres -v`
— 5 passed.
- `TEST_DATABASE_URL=... uv run pytest tests/unit/test_app_password_storage.py
tests/unit/test_webhook_storage.py` — 50 passed (25 per backend).
- `uv run ruff check && uv run ruff format --check && uv run ty check -- nextcloud_mcp_server` — clean.
Tracked on Astrolabe Cloud POC board, card #99.
---
_This PR was generated with the help of AI, and reviewed by a Human_
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
165 lines
5.8 KiB
Python
165 lines
5.8 KiB
Python
"""End-to-end Postgres backend smoke for RefreshTokenStorage (ADR-026).
|
|
|
|
Exercises every storage method touched by the SQLAlchemy / asyncpg port
|
|
against a fresh Postgres schema. The test is opt-in: it requires the
|
|
``postgres-test`` docker-compose service to be running and
|
|
``TEST_DATABASE_URL`` to be exported.
|
|
|
|
Bring up the dependency once::
|
|
|
|
docker compose --profile postgres up -d postgres-test
|
|
export TEST_DATABASE_URL=postgresql+asyncpg://mcp:mcp@localhost:5433/mcp
|
|
|
|
Then run::
|
|
|
|
uv run pytest tests/integration/test_storage_postgres.py -v -m postgres
|
|
|
|
When ``TEST_DATABASE_URL`` is unset (or the service is unreachable) the
|
|
test is skipped so the full suite still passes locally without Docker.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import socket
|
|
from urllib.parse import urlparse
|
|
|
|
import pytest
|
|
from cryptography.fernet import Fernet
|
|
|
|
from nextcloud_mcp_server.auth.storage import RefreshTokenStorage
|
|
|
|
pytestmark = [pytest.mark.integration, pytest.mark.postgres]
|
|
|
|
|
|
def _postgres_url() -> str | None:
|
|
return os.environ.get("TEST_DATABASE_URL") or None
|
|
|
|
|
|
def _reachable(url: str) -> bool:
|
|
parsed = urlparse(url)
|
|
try:
|
|
with socket.create_connection(
|
|
(parsed.hostname or "localhost", parsed.port or 5432), timeout=1.0
|
|
):
|
|
return True
|
|
except OSError:
|
|
return False
|
|
|
|
|
|
@pytest.fixture
|
|
def postgres_url() -> str:
|
|
url = _postgres_url()
|
|
if not url:
|
|
pytest.skip(
|
|
"TEST_DATABASE_URL not set — run "
|
|
"`docker compose --profile postgres up -d postgres-test` and export "
|
|
"TEST_DATABASE_URL=postgresql+asyncpg://mcp:mcp@localhost:5433/mcp"
|
|
)
|
|
if not _reachable(url):
|
|
pytest.skip(f"Postgres at {url} is not reachable")
|
|
return url
|
|
|
|
|
|
@pytest.fixture
|
|
async def reset_schema(postgres_url: str):
|
|
"""Drop+recreate the public schema before and after each test."""
|
|
from sqlalchemy import text
|
|
from sqlalchemy.ext.asyncio import create_async_engine
|
|
|
|
async def _reset() -> None:
|
|
engine = create_async_engine(postgres_url, future=True)
|
|
try:
|
|
async with engine.begin() as conn:
|
|
await conn.execute(text("DROP SCHEMA public CASCADE"))
|
|
await conn.execute(text("CREATE SCHEMA public"))
|
|
finally:
|
|
await engine.dispose()
|
|
|
|
await _reset()
|
|
yield
|
|
await _reset()
|
|
|
|
|
|
@pytest.fixture
|
|
async def storage(postgres_url: str, reset_schema):
|
|
key = Fernet.generate_key()
|
|
s = RefreshTokenStorage(database_url=postgres_url, encryption_key=key)
|
|
await s.initialize()
|
|
yield s
|
|
|
|
|
|
async def test_refresh_token_roundtrip(storage: RefreshTokenStorage):
|
|
"""Store + retrieve + upsert + delete a refresh token end-to-end."""
|
|
await storage.store_refresh_token(
|
|
user_id="alice", refresh_token="rt-1", expires_at=9_999_999_999
|
|
)
|
|
tok = await storage.get_refresh_token("alice")
|
|
assert tok is not None
|
|
assert tok["refresh_token"] == "rt-1"
|
|
assert tok["expires_at"] == 9_999_999_999
|
|
|
|
# Upsert preserves user_id, swaps token contents.
|
|
await storage.store_refresh_token(
|
|
user_id="alice", refresh_token="rt-2", expires_at=9_999_999_999
|
|
)
|
|
tok = await storage.get_refresh_token("alice")
|
|
assert tok is not None and tok["refresh_token"] == "rt-2"
|
|
|
|
assert await storage.delete_refresh_token("alice") is True
|
|
assert await storage.get_refresh_token("alice") is None
|
|
|
|
|
|
async def test_app_password_roundtrip(storage: RefreshTokenStorage):
|
|
"""Store + retrieve + replace + delete a scoped app password."""
|
|
await storage.store_app_password(user_id="bob", app_password="pw-1")
|
|
assert await storage.get_app_password("bob") == "pw-1"
|
|
|
|
# Replace path exercises the ON CONFLICT DO UPDATE on the singleton row.
|
|
await storage.store_app_password(user_id="bob", app_password="pw-2")
|
|
assert await storage.get_app_password("bob") == "pw-2"
|
|
|
|
assert await storage.delete_app_password("bob") is True
|
|
assert await storage.get_app_password("bob") is None
|
|
|
|
|
|
async def test_oauth_session_lifecycle(storage: RefreshTokenStorage):
|
|
"""Cover the ADR-004 progressive-consent session table."""
|
|
await storage.store_oauth_session(
|
|
session_id="sess-1",
|
|
client_redirect_uri="http://localhost:12345/callback",
|
|
mcp_authorization_code="mcp-code-abc",
|
|
flow_type="hybrid",
|
|
ttl_seconds=600,
|
|
)
|
|
fetched = await storage.get_oauth_session("sess-1")
|
|
assert fetched is not None
|
|
assert fetched["mcp_authorization_code"] == "mcp-code-abc"
|
|
|
|
by_code = await storage.get_oauth_session_by_mcp_code("mcp-code-abc")
|
|
assert by_code is not None and by_code["session_id"] == "sess-1"
|
|
|
|
|
|
async def test_webhook_tracking(storage: RefreshTokenStorage):
|
|
"""Tracks webhook ↔ preset mappings via ON CONFLICT upserts."""
|
|
await storage.store_webhook(webhook_id=101, preset_id="notes_sync")
|
|
await storage.store_webhook(webhook_id=202, preset_id="notes_sync")
|
|
await storage.store_webhook(webhook_id=303, preset_id="calendar_sync")
|
|
|
|
assert sorted(await storage.get_webhooks_by_preset("notes_sync")) == [101, 202]
|
|
assert await storage.get_webhooks_by_preset("calendar_sync") == [303]
|
|
|
|
# Re-storing the same webhook_id is a no-op upsert.
|
|
await storage.store_webhook(webhook_id=101, preset_id="notes_sync")
|
|
assert sorted(await storage.get_webhooks_by_preset("notes_sync")) == [101, 202]
|
|
|
|
assert await storage.delete_webhook(webhook_id=101) is True
|
|
assert await storage.get_webhooks_by_preset("notes_sync") == [202]
|
|
|
|
|
|
async def test_audit_log_capture(storage: RefreshTokenStorage):
|
|
"""Audit events from upstream methods land in audit_logs."""
|
|
await storage.store_app_password(user_id="carol", app_password="x")
|
|
logs = await storage.get_audit_logs(user_id="carol", limit=10)
|
|
assert any(entry["event"] == "store_app_password" for entry in logs)
|