diff --git a/.gitignore b/.gitignore index d64fb5d5..d527b495 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ __pycache__/ # Dynaconf (ADR-024) .secrets.toml +settings.toml settings.local.toml # Git diff --git a/alembic.ini b/alembic.ini index 1a2971ee..8f60afd9 100644 --- a/alembic.ini +++ b/alembic.ini @@ -40,11 +40,12 @@ path_separator = os # Default: utf-8 # output_encoding = utf-8 -# Database URL - can be overridden by: -# 1. Passing -x database_url=... to alembic commands -# 2. Setting in environment via get_database_url() in env.py -# Default: sqlite:///app/data/tokens.db -sqlalchemy.url = sqlite+aiosqlite:////app/data/tokens.db +# Database URL - placeholder only. +# Runtime code (migrations.get_alembic_config / alembic/env.get_database_url) +# overrides this with config.get_token_db_path() which defaults to an +# ephemeral per-process tempfile unless TOKEN_STORAGE_DB is set. +# This value only matters for manual `alembic` invocations from the repo root. +sqlalchemy.url = sqlite+aiosqlite:///./tokens.db [post_write_hooks] # Post-write hooks allow you to run scripts after generating migration files diff --git a/docker-compose.yml b/docker-compose.yml index 7134df2d..62fa8e02 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -86,13 +86,17 @@ services: - 127.0.0.1:9090:9090 volumes: - mcp-data:/app/data - - ./settings.toml:/app/settings.toml:ro + - ./settings.toml.example:/app/settings.toml:ro environment: - NEXTCLOUD_HOST=http://app:80 - NEXTCLOUD_USERNAME=admin - NEXTCLOUD_PASSWORD=admin - NEXTCLOUD_PUBLIC_ISSUER_URL=http://localhost:8080 + # Pin the token DB inside the mcp-data volume so the runtime default + # (ephemeral tempfile) doesn't silently apply inside containers. + - TOKEN_STORAGE_DB=/app/data/tokens.db + # Semantic search configuration (ADR-007, ADR-021) - ENABLE_SEMANTIC_SEARCH=true - VECTOR_SYNC_SCAN_INTERVAL=5 @@ -167,7 +171,7 @@ services: # NO admin credentials - credentials come from client Authorization header volumes: - multi-user-basic-data:/app/data - - ./settings.toml:/app/settings.toml:ro + - ./settings.toml.example:/app/settings.toml:ro profiles: - multi-user-basic @@ -243,7 +247,7 @@ services: volumes: - keycloak-tokens:/app/data - keycloak-oauth-storage:/app/.oauth - - ./settings.toml:/app/settings.toml:ro + - ./settings.toml.example:/app/settings.toml:ro profiles: - keycloak @@ -282,7 +286,7 @@ services: volumes: - login-flow-data:/app/data - login-flow-oauth-storage:/app/.oauth - - ./settings.toml:/app/settings.toml:ro + - ./settings.toml.example:/app/settings.toml:ro profiles: - login-flow diff --git a/nextcloud_mcp_server/alembic/env.py b/nextcloud_mcp_server/alembic/env.py index 3026174a..ffcc2d2b 100644 --- a/nextcloud_mcp_server/alembic/env.py +++ b/nextcloud_mcp_server/alembic/env.py @@ -51,8 +51,14 @@ def get_database_url() -> str: url = config.get_main_option("sqlalchemy.url") if not url: - # Default to /app/data/tokens.db for Docker deployments - db_path = Path("/app/data/tokens.db") + # Fall back to the same resolver the runtime uses (ephemeral tempfile + # unless TOKEN_STORAGE_DB is set). Imported lazily to avoid pulling + # the full config module into offline alembic invocations. + from nextcloud_mcp_server.config import ( # noqa: PLC0415 + get_token_db_path, + ) + + db_path = Path(get_token_db_path()) url = f"sqlite+aiosqlite:///{db_path}" logger.warning( f"No database URL configured, using default: {url}. " diff --git a/nextcloud_mcp_server/auth/storage.py b/nextcloud_mcp_server/auth/storage.py index ec2f2bc4..ed6675df 100644 --- a/nextcloud_mcp_server/auth/storage.py +++ b/nextcloud_mcp_server/auth/storage.py @@ -39,6 +39,7 @@ import httpx from anyio import to_thread from cryptography.fernet import Fernet +from nextcloud_mcp_server.config import get_token_db_path, is_ephemeral_token_db from nextcloud_mcp_server.migrations import stamp_database, upgrade_database from nextcloud_mcp_server.observability.metrics import record_db_operation @@ -82,7 +83,10 @@ class RefreshTokenStorage: Create storage instance from environment variables. Environment variables: - TOKEN_STORAGE_DB: Path to database file (default: /app/data/tokens.db) + TOKEN_STORAGE_DB: Path to database file. If unset, a per-process + tempfile is allocated and deleted at interpreter exit — + tokens are ephemeral and wiped on restart. Set this to a + filesystem path to persist tokens across restarts. TOKEN_ENCRYPTION_KEY: Optional base64-encoded Fernet key (required for token storage) Returns: @@ -92,7 +96,13 @@ class RefreshTokenStorage: If TOKEN_ENCRYPTION_KEY is not set, token storage operations will fail, but webhook tracking will still work. """ - db_path = os.getenv("TOKEN_STORAGE_DB", "/app/data/tokens.db") + db_path = get_token_db_path() + if is_ephemeral_token_db(db_path): + logger.info( + "Using ephemeral token storage at %s " + "(set TOKEN_STORAGE_DB to persist tokens across restarts)", + db_path, + ) encryption_key_b64 = os.getenv("TOKEN_ENCRYPTION_KEY") encryption_key = None diff --git a/nextcloud_mcp_server/cli.py b/nextcloud_mcp_server/cli.py index 5e995ce6..ee6b045c 100644 --- a/nextcloud_mcp_server/cli.py +++ b/nextcloud_mcp_server/cli.py @@ -6,6 +6,8 @@ import uvicorn from nextcloud_mcp_server.config import ( get_settings, + get_token_db_path, + is_ephemeral_token_db, ) from nextcloud_mcp_server.migrations import ( create_migration, @@ -285,13 +287,25 @@ def db(): pass +def _warn_if_ephemeral(database_path: str) -> None: + if is_ephemeral_token_db(database_path): + click.echo( + click.style( + f"⚠ Using ephemeral tempfile {database_path}; changes " + "will be lost on exit. Pass --database-path or set " + "TOKEN_STORAGE_DB to operate on a persistent database.", + fg="yellow", + ), + err=True, + ) + + @db.command() @click.option( "--database-path", "-d", envvar="TOKEN_STORAGE_DB", - default="/app/data/tokens.db", - show_default=True, + default=None, help="Path to token storage database (can also use TOKEN_STORAGE_DB env var)", ) @click.option( @@ -301,7 +315,7 @@ def db(): show_default=True, help="Target revision (default: head for latest)", ) -def upgrade(database_path: str, revision: str): +def upgrade(database_path: str | None, revision: str): """Upgrade database to a specific revision. \b @@ -315,6 +329,8 @@ def upgrade(database_path: str, revision: str): # Use custom database path $ nextcloud-mcp-server db upgrade -d /path/to/tokens.db """ + database_path = database_path or get_token_db_path() + _warn_if_ephemeral(database_path) try: click.echo(f"Upgrading database to revision: {revision}") upgrade_database(database_path, revision) @@ -329,8 +345,7 @@ def upgrade(database_path: str, revision: str): "--database-path", "-d", envvar="TOKEN_STORAGE_DB", - default="/app/data/tokens.db", - show_default=True, + default=None, help="Path to token storage database", ) @click.option( @@ -343,7 +358,7 @@ def upgrade(database_path: str, revision: str): @click.confirmation_option( prompt="Are you sure you want to downgrade the database? This may result in data loss." ) -def downgrade(database_path: str, revision: str): +def downgrade(database_path: str | None, revision: str): """Downgrade database to a specific revision. WARNING: This may result in data loss! Use with caution. @@ -359,6 +374,8 @@ def downgrade(database_path: str, revision: str): # Downgrade to base (empty database) $ nextcloud-mcp-server db downgrade --revision base """ + database_path = database_path or get_token_db_path() + _warn_if_ephemeral(database_path) try: click.echo(f"Downgrading database to revision: {revision}") downgrade_database(database_path, revision) @@ -373,17 +390,18 @@ def downgrade(database_path: str, revision: str): "--database-path", "-d", envvar="TOKEN_STORAGE_DB", - default="/app/data/tokens.db", - show_default=True, + default=None, help="Path to token storage database", ) -def current(database_path: str): +def current(database_path: str | None): """Show current database revision. \b Example: $ nextcloud-mcp-server db current """ + database_path = database_path or get_token_db_path() + _warn_if_ephemeral(database_path) try: revision = get_current_revision(database_path) if revision: @@ -406,17 +424,18 @@ def current(database_path: str): "--database-path", "-d", envvar="TOKEN_STORAGE_DB", - default="/app/data/tokens.db", - show_default=True, + default=None, help="Path to token storage database", ) -def history(database_path: str): +def history(database_path: str | None): """Show migration history. \b Example: $ nextcloud-mcp-server db history """ + database_path = database_path or get_token_db_path() + _warn_if_ephemeral(database_path) try: click.echo("Migration history:") show_migration_history(database_path) diff --git a/nextcloud_mcp_server/config.py b/nextcloud_mcp_server/config.py index 9114d018..9e820b00 100644 --- a/nextcloud_mcp_server/config.py +++ b/nextcloud_mcp_server/config.py @@ -1,34 +1,152 @@ +import atexit import logging import logging.config import os import socket import ssl +import tempfile from dataclasses import dataclass from pathlib import Path from typing import Any from dynaconf import Dynaconf, Validator -# Resolve root_path for dynaconf settings files. -# Editable installs: parent.parent is the project root (has settings.toml). -# Non-editable installs (Docker): settings.toml is mounted at WORKDIR (/app/). -_config_root = Path(__file__).parent.parent -if not (_config_root / "settings.toml").exists(): - _config_root = Path.cwd() - # Sentinel for "key not in dynaconf at all" vs "explicitly set to None". _UNSET = object() -# Dynaconf instance — loads settings.toml + .secrets.toml + env vars. -# Env vars always win (12-factor). See ADR-024 for architecture. +# Built-in defaults — declared in Python so env vars work without any settings +# file being present (e.g., `uvx` / `pip install` deployments). Mirrors the +# [default] section that used to live in settings.toml. Keys set here are +# "known" to dynaconf, which is required because we run with +# ignore_unknown_envvars=True. See ADR-024/025. +_DEFAULTS: dict[str, Any] = { + # Deployment mode (ADR-021) + "mcp_deployment_mode": None, + # Nextcloud core + "nextcloud_host": None, + "nextcloud_username": None, + "nextcloud_password": None, + "nextcloud_app_password": None, + "nextcloud_verify_ssl": True, + "nextcloud_ca_bundle": None, + "nextcloud_mcp_server_url": None, + "nextcloud_resource_uri": None, + # OAuth/OIDC + "oidc_discovery_url": None, + "nextcloud_oidc_client_id": None, + "nextcloud_oidc_client_secret": None, + "oidc_issuer": None, + "jwks_uri": None, + "introspection_uri": None, + "userinfo_uri": None, + "oidc_resource_server_id": None, + # Mode flags + "enable_multi_user_basic_auth": False, + "enable_login_flow": False, + "enable_semantic_search": False, + "enable_background_operations": False, + "vector_sync_enabled": False, + "enable_offline_access": False, + "enable_token_exchange": False, + # Token storage + "token_encryption_key": None, + # None = ephemeral per-process tempfile (see get_token_db_path()). + # Set TOKEN_STORAGE_DB to persist tokens across restarts. + "token_storage_db": None, + # Vector sync + "vector_sync_scan_interval": 300, + "vector_sync_processor_workers": 3, + "vector_sync_queue_max_size": 10000, + "vector_sync_user_poll_interval": 60, + # Qdrant + "qdrant_url": None, + "qdrant_location": None, + "qdrant_api_key": None, + "qdrant_collection": "nextcloud_content", + # Ollama + "ollama_base_url": None, + "ollama_embedding_model": "nomic-embed-text", + "ollama_verify_ssl": True, + # OpenAI + "openai_api_key": None, + "openai_base_url": None, + "openai_embedding_model": "text-embedding-3-small", + # Document chunking + "document_chunk_size": 2048, + "document_chunk_overlap": 200, + # Observability + "metrics_enabled": True, + "metrics_port": 9090, + "otel_exporter_otlp_endpoint": None, + "otel_exporter_verify_ssl": False, + "otel_service_name": "nextcloud-mcp-server", + "otel_traces_sampler": "always_on", + "otel_traces_sampler_arg": 1.0, + "log_format": "text", + "log_level": "INFO", + "log_include_trace_context": True, + # Document processing + "enable_document_processing": False, + "document_processor": "unstructured", + "enable_unstructured": False, + "unstructured_api_url": "http://unstructured:8000", + "unstructured_timeout": 120, + "unstructured_strategy": "auto", + "unstructured_languages": "eng,deu", + "progress_interval": 10, + "enable_tesseract": False, + "tesseract_cmd": None, + "tesseract_lang": "eng", + "enable_pymupdf": True, + "pymupdf_extract_images": True, + "pymupdf_image_dir": None, + "enable_custom_processor": False, + "custom_processor_url": None, + "custom_processor_types": "application/pdf", + "custom_processor_name": "custom", + "custom_processor_api_key": None, + "custom_processor_timeout": 60, +} + + +def _resolve_settings_files() -> list[str]: + """Find optional external settings files. + + Priority: + 1. NEXTCLOUD_MCP_SETTINGS_FILE env var (absolute or relative path) + 2. ./settings.toml in cwd (for docker / dev workflows) + 3. .secrets.toml alongside whichever settings.toml was found + + Returns an empty list if no files are present — that's fine, defaults and + env vars still apply. + """ + files: list[str] = [] + explicit = os.environ.get("NEXTCLOUD_MCP_SETTINGS_FILE") + if explicit: + p = Path(explicit) + if p.exists(): + files.append(str(p)) + else: + cwd_settings = Path.cwd() / "settings.toml" + if cwd_settings.exists(): + files.append(str(cwd_settings)) + cwd_secrets = Path.cwd() / ".secrets.toml" + if cwd_secrets.exists(): + files.append(str(cwd_secrets)) + return files + + +# Dynaconf instance — env vars always win (12-factor). Settings files are +# optional; when absent the defaults above provide the full key schema so +# env vars still override correctly. See ADR-024/025 for architecture. _dynaconf = Dynaconf( - settings_files=["settings.toml", ".secrets.toml"], + settings_files=_resolve_settings_files(), environments=True, envvar_prefix=False, env_switcher="MCP_DEPLOYMENT_MODE", ignore_unknown_envvars=True, - root_path=str(_config_root), load_dotenv=False, + **_DEFAULTS, validators=[ # Port ranges Validator("METRICS_PORT", gte=1, lte=65535), @@ -73,6 +191,47 @@ def _reload_config(): _dynaconf.validators.validate_all() +_ephemeral_db_path: str | None = None + + +def get_token_db_path() -> str: + """Resolve the token SQLite database path. + + Priority: + 1. TOKEN_STORAGE_DB env var / dynaconf setting if explicitly set — + docker-compose pins /app/data/tokens.db this way. + 2. Otherwise a per-process tempfile under tempfile.gettempdir(), + allocated lazily and deleted at interpreter exit via atexit. + Ephemeral: tokens are wiped on restart, matching the Qdrant + ":memory:" default pattern used elsewhere in this project. + """ + explicit = os.environ.get("TOKEN_STORAGE_DB") or _dynaconf.get("TOKEN_STORAGE_DB") + if explicit: + return str(explicit) + global _ephemeral_db_path + if _ephemeral_db_path is None: + fd, path = tempfile.mkstemp( + prefix=f"nextcloud-mcp-tokens-{os.getpid()}-", suffix=".db" + ) + os.close(fd) + _ephemeral_db_path = path + + def _cleanup(p: str = path) -> None: + try: + if os.path.exists(p): + os.unlink(p) + except OSError: + pass + + atexit.register(_cleanup) + return _ephemeral_db_path + + +def is_ephemeral_token_db(path: str) -> bool: + """Return True if the given path is the process-local ephemeral tempfile.""" + return path == _ephemeral_db_path + + LOGGING_CONFIG = { "version": 1, "disable_existing_loggers": False, diff --git a/nextcloud_mcp_server/migrations.py b/nextcloud_mcp_server/migrations.py index 7e919809..0964f453 100644 --- a/nextcloud_mcp_server/migrations.py +++ b/nextcloud_mcp_server/migrations.py @@ -13,6 +13,7 @@ from alembic.config import Config import nextcloud_mcp_server.alembic as alembic_package from alembic import command +from nextcloud_mcp_server.config import get_token_db_path logger = logging.getLogger(__name__) @@ -25,8 +26,9 @@ def get_alembic_config(database_path: str | Path | None = None) -> Config: package location instead of alembic.ini file. Args: - database_path: Path to SQLite database file. If None, uses default - (/app/data/tokens.db for Docker) + database_path: Path to SQLite database file. If None, resolves via + config.get_token_db_path() (ephemeral tempfile unless + TOKEN_STORAGE_DB is set). Returns: Alembic Config object configured for the specified database @@ -45,7 +47,7 @@ def get_alembic_config(database_path: str | Path | None = None) -> Config: if database_path: db_path = Path(database_path).resolve() else: - db_path = Path("/app/data/tokens.db") # Default for Docker + db_path = Path(get_token_db_path()).resolve() url = f"sqlite+aiosqlite:///{db_path}" config.set_main_option("sqlalchemy.url", url) @@ -100,7 +102,7 @@ def get_current_revision(database_path: str | Path | None = None) -> str | None: """ if database_path is None: - database_path = "/app/data/tokens.db" + database_path = get_token_db_path() db_path = Path(database_path).resolve() diff --git a/settings.toml b/settings.toml.example similarity index 100% rename from settings.toml rename to settings.toml.example