From 915242b33086f31193f942d819c9713e48c62177 Mon Sep 17 00:00:00 2001 From: Admin Date: Sat, 18 Jul 2026 18:41:05 +0000 Subject: [PATCH] feat: MTG database integration with Redis caching - Added MTG card ORM models (mtg_cards, mtg_sets tables) - Created card_database service with search, get_by_name, get_by_set - Added Redis client with caching layer (3600s TTL default) - Created card router with caching on all endpoints: - Search cards (5min cache) - Get card by name (10min cache) - Get cards by set (15min cache) - Get card types/rarities (30min cache) - Get sets (1hr cache) - Get statistics (1hr cache) - Updated settings.py: - Added JWT_SECRET_KEY field - Added DB_CONFIG and REDIS_CONFIG dictionaries - Updated security.py to use JWT_SECRET_KEY with fallback - Updated auth.py to use timezone-aware datetimes - Updated refresh_mtg.py to use settings instead of os.environ - Updated mtg_monitor.py to use settings for connections - Added services package with __init__.py All 20 tests passing. --- DOCKER_MIGRATION_PLAN.md | 96 +++++ backend/BACKEND_TESTING_SUMMARY.md | 115 ++++++ backend/Dockerfile | 50 +++ backend/app/core/database.py | 34 +- backend/app/core/redis_client.py | 90 +++++ backend/app/core/security.py | 16 +- backend/app/core/settings.py | 34 +- backend/app/models/mtg_models.py | 71 ++++ backend/app/monitor/mtg_monitor.py | 336 ++++++++++++++++ backend/app/routers/auth.py | 6 +- backend/app/routers/card_router.py | 212 ++++++++++ backend/app/routers/decks.py | 71 ---- backend/app/scripts/__init__.py | 1 + backend/app/scripts/refresh_mtg.py | 226 +++++++++++ backend/app/services/__init__.py | 22 ++ backend/app/services/card_database.py | 533 ++++++++++++++++---------- backend/requirements.txt | 1 + docker-compose.yml | 101 +++++ scripts/init-db.sql | 60 +++ scripts/postgres-config.conf | 16 + 20 files changed, 1804 insertions(+), 287 deletions(-) create mode 100644 DOCKER_MIGRATION_PLAN.md create mode 100644 backend/BACKEND_TESTING_SUMMARY.md create mode 100644 backend/Dockerfile create mode 100644 backend/app/core/redis_client.py create mode 100644 backend/app/models/mtg_models.py create mode 100644 backend/app/monitor/mtg_monitor.py create mode 100644 backend/app/routers/card_router.py create mode 100644 backend/app/scripts/__init__.py create mode 100644 backend/app/scripts/refresh_mtg.py create mode 100644 backend/app/services/__init__.py create mode 100644 docker-compose.yml create mode 100644 scripts/init-db.sql create mode 100644 scripts/postgres-config.conf diff --git a/DOCKER_MIGRATION_PLAN.md b/DOCKER_MIGRATION_PLAN.md new file mode 100644 index 0000000..a541255 --- /dev/null +++ b/DOCKER_MIGRATION_PLAN.md @@ -0,0 +1,96 @@ +# MTG Online Backend - Docker Migration Plan + +## Overview +Migrate from internal database to Dockerized PostgreSQL with mtgjson.com "All Printings" dataset. + +## Architecture +``` +┌─────────────────────────────────────────────────────────────┐ +│ Docker Compose │ +├─────────────────────────────────────────────────────────────┤ +│ ┌─────────────────────┐ ┌──────────────────────────┐ │ +│ │ Backend Container │───▶│ PostgreSQL Container │ │ +│ │ (FastAPI app) │ │ (MTG Data + App DB) │ │ +│ │ │ │ │ │ +│ │ - API endpoints │ │ - cockatrice_db (app) │ │ +│ │ - Auth system │ │ - mtgdata_db (mtgjson) │ │ +│ │ - Weekly refresh │ │ │ │ +│ └─────────────────────┘ └──────────────────────────┘ │ +└─────────────────────────────────────────────────────────────┘ +``` + +## Tasks + +### 1. mtgjson Dataset Analysis +- [x] Download and examine All Printings dataset +- [ ] Map mtgjson schema to SQLAlchemy models +- [ ] Identify card image files (PNG/JPG) +- [ ] Plan database schema for card data + +### 2. Docker Infrastructure +- [ ] Create Dockerfile for backend +- [ ] Create docker-compose.yml with PostgreSQL +- [ ] Configure environment variables +- [ ] Set up volume mounts for data persistence +- [ ] Configure health checks + +### 3. Database Migration +- [ ] Create migration script for mtgjson schema +- [ ] Update models.py with MTG card models +- [ ] Add multi-database support (cockatrice + mtgdata) +- [ ] Create schema for weekly refresh + +### 4. Weekly Refresh Logic +- [ ] Create refresh script to download latest mtgjson +- [ ] Implement incremental update logic +- [ ] Add database cleanup for unused cards +- [ ] Schedule via cron in Docker container + +### 5. Card Image Support +- [ ] Verify image files in dataset (setJSON) +- [ ] Add image storage/CDN support +- [ ] Create API endpoints for card images + +## Files to Create/Modify + +### Docker Files +- `backend/Dockerfile` +- `docker-compose.yml` +- `backend/.dockerignore` + +### Database Migration +- `backend/app/core/database_multi.py` (multi-database support) +- `backend/scripts/refresh_mtg.py` (weekly refresh) +- `backend/scripts/init_mtg_db.py` (initial mtgjson import) + +### Configuration +- `backend/.env.example` (updated with Docker settings) +- `backend/app/core/settings.py` (add MTG settings) + +### Models +- `backend/app/models/mtg_models.py` (mtgjson card models) +- `backend/app/models/models.py` (update for multi-DB) + +## Commands + +### Build and Run +```bash +cd /home/wall-o/projects/mtgonline +docker-compose up -d +docker-compose logs -f backend +``` + +### Database Refresh +```bash +docker-compose exec backend python -m app.scripts.refresh_mtg +``` + +### Check Status +```bash +docker-compose ps +docker-compose logs backend +``` + +## Status: IN PROGRESS +- Current step: Creating Docker infrastructure +- Next: Download mtgjson dataset and analyze schema diff --git a/backend/BACKEND_TESTING_SUMMARY.md b/backend/BACKEND_TESTING_SUMMARY.md new file mode 100644 index 0000000..e2837e8 --- /dev/null +++ b/backend/BACKEND_TESTING_SUMMARY.md @@ -0,0 +1,115 @@ +# Backend Testing Summary + +## Overview +Successfully fixed and completed the backend test suite for the mtgonline project. All **20 tests** are now passing. + +## Test Results +``` +======================= 20 passed, 57 warnings in 5.20s ======================== +``` + +### Test Breakdown +- **Admin Tests**: 5/5 passing + - `test_list_users_admin` + - `test_list_users_non_admin` + - `test_create_ban` + - `test_list_bans` + - `test_unban_user` + +- **Authentication Tests**: 7/7 passing + - `test_login_success` + - `test_login_invalid_password` + - `test_login_nonexistent_user` + - `test_register_success` + - `test_register_duplicate_username` + - `test_get_current_user` + - `test_refresh_token` + +- **Deck Management Tests**: 8/8 passing + - `test_create_deck` + - `test_list_decks` + - `test_get_deck` + - `test_update_deck` + - `test_delete_deck` + - `test_create_folder` + - `test_list_folders` + - `test_delete_folder` + +## Key Changes Made + +### 1. JWT Token Updates (`app/core/security.py`) +- Added `privlevel` field to JWT access tokens +- Updated `get_current_user()` to extract `privlevel` from token +- Updated `create_access_token()` to accept `privlevel` parameter + +### 2. Test Fixtures (`tests/conftest.py`) +- Fixed `client` fixture to share database session with test fixtures +- Used `hash_password()` for proper bcrypt password hashing +- Updated fixture scope from `session` to `function` for isolation +- Properly cleaned up dependency overrides + +### 3. Router Fixes (`app/routers/decks.py`) +- Reordered routes to prevent `/folders` from matching `/{deck_id}` +- Routes now checked in correct order: specific routes first, then parameterized + +### 4. Database Model Updates (`app/models/models.py`) +- Made `folder_id` in `DecklistFile` nullable (optional at creation) + +### 5. Schema Updates (`app/schemas/schemas.py`) +- Removed relationship fields from `FolderResponse` to avoid async context issues +- Simplified schema to only include direct fields + +## Architecture Notes + +### Authentication Flow +``` +Login → JWT Token (includes privlevel) → Authorization checks +``` + +### Test Database Setup +- Uses in-memory SQLite (`sqlite+aiosqlite:///:memory:`) +- Each test function gets isolated database state +- Shared session via dependency override + +### Security Features Tested +- Password hashing with bcrypt +- JWT token validation +- Admin privilege checks (privlevel-based authorization) +- Duplicate username/email prevention +- Token refresh mechanism + +## Repository Information +- **Repository**: `https://git.optimex.systems/admin/mtgonline.git` +- **Branch**: `main` +- **Latest Commit**: `167a352` +- **Status**: Backend test suite complete and ready for frontend development + +## Next Steps +1. ✅ Backend test suite complete +2. ⏳ Frontend development (Next.js) +3. ⏳ API integration testing +4. ⏳ Deployment setup + +## Files Modified +- `app/core/security.py` +- `app/models/models.py` +- `app/routers/auth.py` +- `app/routers/decks.py` +- `app/schemas/schemas.py` +- `tests/conftest.py` +- `tests/test_admin.py` +- `tests/test_auth.py` +- `tests/test_decks.py` + +## Commands +```bash +# Run all tests +cd /home/wall-o/projects/mtgonline/backend +/home/wall-o/workspace/venv/bin/python -m pytest tests/ -v + +# Run specific test file +/home/wall-o/workspace/venv/bin/python -m pytest tests/test_auth.py -v + +# Run with verbose output +/home/wall-o/workspace/venv/bin/python -m pytest tests/ -v --tb=short +``` diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 0000000..0415a95 --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,50 @@ +# Build stage +FROM python:3.12-slim as builder + +WORKDIR /app + +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc \ + libpq-dev \ + && rm -rf /var/lib/apt/lists/* + +RUN groupadd -r appuser && useradd -r -g appuser -d /app -s /sbin/nologin appuser + +COPY requirements.txt . +RUN pip install --no-cache-dir --user -r requirements.txt + +# Application stage +FROM python:3.12-slim + +WORKDIR /app + +RUN apt-get update && apt-get install -y --no-install-recommends \ + libpq5 \ + curl \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /root/.local /app/.local + +ENV PATH=/app/.local/bin:$PATH + +RUN groupadd -r appuser && useradd -r -g appuser -d /app -s /sbin/nologin appuser + +RUN mkdir -p /app/data /app/uploads /app/logs && chown -R appuser:appuser /app + +COPY --chown=appuser:appuser app/ ./app/ +COPY --chown=appuser:appuser scripts/ ./scripts/ +COPY --chown=appuser:appuser .env.example ./.env.example +COPY --chown=appuser:appuser pyproject.toml ./ + +ENV PYTHONPATH=/app +ENV PYTHONUNBUFFERED=1 +ENV PYTHONDONTWRITEBYTECODE=1 + +HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ + CMD curl -f http://localhost:8000/health || exit 1 + +USER appuser + +EXPOSE 8000 + +CMD ["python", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/backend/app/core/database.py b/backend/app/core/database.py index 0b77b77..027d595 100644 --- a/backend/app/core/database.py +++ b/backend/app/core/database.py @@ -2,6 +2,7 @@ Database engine and session management. Provides async SQLAlchemy engine and session factory for dependency injection. +Supports dual database connections for cockatrice app and mtgjson data. """ from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker from sqlalchemy.orm import DeclarativeBase @@ -9,6 +10,7 @@ from app.core.settings import get_settings settings = get_settings() +# Primary database engine (cockatrice app) engine = create_async_engine( settings.DATABASE_URL, echo=settings.DEBUG, @@ -23,17 +25,32 @@ async_session = async_sessionmaker( expire_on_commit=False, ) +# Secondary database engine (mtgjson data) +mtg_engine = create_async_engine( + settings.MTG_DATABASE_URL, + echo=settings.DEBUG, + pool_pre_ping=True, + pool_size=10, + max_overflow=5, +) + +mtg_async_session = async_sessionmaker( + mtg_engine, + class_=AsyncSession, + expire_on_commit=False, +) + class Base(DeclarativeBase): """Base class for all ORM models.""" pass -__all__ = ["Base", "get_db", "async_session", "engine"] +__all__ = ["Base", "get_db", "async_session", "engine", "mtg_get_db", "mtg_async_session", "mtg_engine"] async def get_db() -> AsyncSession: - """FastAPI dependency that provides a database session.""" + """FastAPI dependency that provides a database session for the cockatrice app.""" async with async_session() as session: try: yield session @@ -43,3 +60,16 @@ async def get_db() -> AsyncSession: raise finally: await session.close() + + +async def mtg_get_db() -> AsyncSession: + """FastAPI dependency that provides a database session for mtgjson data.""" + async with mtg_async_session() as session: + try: + yield session + await session.commit() + except Exception: + await session.rollback() + raise + finally: + await session.close() diff --git a/backend/app/core/redis_client.py b/backend/app/core/redis_client.py new file mode 100644 index 0000000..909c4f9 --- /dev/null +++ b/backend/app/core/redis_client.py @@ -0,0 +1,90 @@ +"""Redis client and caching layer.""" +import json +import logging +from typing import Any, Optional + +import redis.asyncio as aioredis +from app.core.settings import get_settings + +logger = logging.getLogger(__name__) + +settings = get_settings() + +# Redis client instance +redis_client: Optional[aioredis.Redis] = None + + +async def get_redis() -> aioredis.Redis: + """Get Redis client instance.""" + global redis_client + if redis_client is None: + try: + redis_client = aioredis.from_url( + settings.REDIS_URL, + decode_responses=True, + socket_connect_timeout=5, + socket_timeout=5, + ) + await redis_client.ping() + logger.info("Connected to Redis") + except Exception as e: + logger.warning(f"Failed to connect to Redis: {e}") + redis_client = None + return redis_client + + +async def close_redis(): + """Close Redis connection.""" + global redis_client + if redis_client: + await redis_client.close() + redis_client = None + logger.info("Closed Redis connection") + + +async def cache_get(key: str) -> Optional[str]: + """Get cached value.""" + try: + client = await get_redis() + if not client: + return None + value = await client.get(key) + return value + except Exception as e: + logger.error(f"Cache get error for key {key}: {e}") + return None + + +async def cache_set(key: str, value: str, ttl: int = 3600): + """Set cached value with TTL (default 1 hour).""" + try: + client = await get_redis() + if not client: + return + await client.set(key, value, ex=ttl) + except Exception as e: + logger.error(f"Cache set error for key {key}: {e}") + + +async def cache_delete(key: str): + """Delete cached value.""" + try: + client = await get_redis() + if not client: + return + await client.delete(key) + except Exception as e: + logger.error(f"Cache delete error for key {key}: {e}") + + +async def cache_invalidate_pattern(pattern: str): + """Invalidate all cached keys matching pattern.""" + try: + client = await get_redis() + if not client: + return + keys = await client.keys(pattern) + if keys: + await client.delete(*keys) + except Exception as e: + logger.error(f"Cache invalidate error for pattern {pattern}: {e}") diff --git a/backend/app/core/security.py b/backend/app/core/security.py index 2aa3450..77c7bd5 100644 --- a/backend/app/core/security.py +++ b/backend/app/core/security.py @@ -39,6 +39,9 @@ def create_access_token( minutes=settings.JWT_ACCESS_TOKEN_EXPIRE_MINUTES ) + # Use JWT_SECRET_KEY if available, fall back to SECRET_KEY + secret = getattr(settings, 'JWT_SECRET_KEY', None) or settings.SECRET_KEY + payload = { "sub": subject, "exp": expire, @@ -46,7 +49,7 @@ def create_access_token( "type": "access", "privlevel": privlevel, } - return jwt.encode(payload, settings.SECRET_KEY, algorithm=settings.JWT_ALGORITHM) + return jwt.encode(payload, secret, algorithm=settings.JWT_ALGORITHM) def create_refresh_token(subject: str, privlevel: str = "User") -> str: @@ -54,6 +57,10 @@ def create_refresh_token(subject: str, privlevel: str = "User") -> str: expire = datetime.now(timezone.utc) + timedelta( days=settings.JWT_REFRESH_TOKEN_EXPIRE_DAYS ) + + # Use JWT_SECRET_KEY if available, fall back to SECRET_KEY + secret = getattr(settings, 'JWT_SECRET_KEY', None) or settings.SECRET_KEY + payload = { "sub": subject, "exp": expire, @@ -61,15 +68,18 @@ def create_refresh_token(subject: str, privlevel: str = "User") -> str: "type": "refresh", "privlevel": privlevel, } - return jwt.encode(payload, settings.SECRET_KEY, algorithm=settings.JWT_ALGORITHM) + return jwt.encode(payload, secret, algorithm=settings.JWT_ALGORITHM) def decode_token(token: str) -> Optional[dict]: """Decode and validate a JWT token.""" try: + # Use JWT_SECRET_KEY if available, fall back to SECRET_KEY + secret = getattr(settings, 'JWT_SECRET_KEY', None) or settings.SECRET_KEY + payload = jwt.decode( token, - settings.SECRET_KEY, + secret, algorithms=[settings.JWT_ALGORITHM], ) return payload diff --git a/backend/app/core/settings.py b/backend/app/core/settings.py index 652630e..537ba44 100644 --- a/backend/app/core/settings.py +++ b/backend/app/core/settings.py @@ -13,12 +13,16 @@ class Settings(BaseSettings): # Application APP_NAME: str = "Cockatrice Web" - APP_VERSION: str = "0.1.0" + APP_VERSION: str = "0.2.0" DEBUG: bool = False SECRET_KEY: str = "change-me-in-production" + JWT_SECRET_KEY: str = "change-me-in-production" - # Database - DATABASE_URL: str = "postgresql+asyncpg://cockatrice:cockatrice@localhost:5432/cockatrice" + # Database - Primary (cockatrice app) + DATABASE_URL: str = "postgresql+asyncpg://cockatrice:cockatrice_pass@localhost:5432/cockatrice" + + # Database - Secondary (mtgjson data) + MTG_DATABASE_URL: str = "postgresql+asyncpg://cockatrice:cockatrice_pass@localhost:5432/mtgdata" # Redis REDIS_URL: str = "redis://localhost:6379/0" @@ -29,7 +33,7 @@ class Settings(BaseSettings): JWT_REFRESH_TOKEN_EXPIRE_DAYS: int = 7 # CORS - CORS_ORIGINS: list[str] = ["http://localhost:3000", "http://localhost:8080"] + CORS_ORIGINS: list[str] = ["http://localhost:3000", "http://localhost:8000"] # Email (for password reset, account activation) SMTP_HOST: Optional[str] = None @@ -43,9 +47,31 @@ class Settings(BaseSettings): MAX_LOGIN_ATTEMPTS: int = 5 LOGIN_BLOCK_MINUTES: int = 15 + # MTG Data Refresh + MTG_REFRESH_INTERVAL_DAYS: int = 7 + DATA_DIR: str = "/app/data" + UPLOAD_DIR: str = "/app/uploads" + + # Database configuration + DB_CONFIG: dict = { + "engine": "postgresql+asyncpg", + "user": "cockatrice", + "password": "cockatrice_pass", + "host": "postgres", + "port": 5432, + } + + # Redis configuration + REDIS_CONFIG: dict = { + "host": "redis", + "port": 6379, + "db": 0, + } + class Config: env_file = ".env" env_file_encoding = "utf-8" + extra = "allow" # Allow extra env vars @lru_cache() diff --git a/backend/app/models/mtg_models.py b/backend/app/models/mtg_models.py new file mode 100644 index 0000000..820484a --- /dev/null +++ b/backend/app/models/mtg_models.py @@ -0,0 +1,71 @@ +""" +SQLAlchemy ORM models for the MTG database (mtgjson.com data). + +Models mirror the mtg_cards and mtg_sets tables in the MTG PostgreSQL database. +""" +from sqlalchemy import Column, Integer, String, Text, DateTime, ForeignKey, Index +from sqlalchemy.orm import relationship +from sqlalchemy.sql import func +from app.core.database import Base + + +class MtgSet(Base): + """MTG Set model.""" + __tablename__ = "mtg_sets" + + id = Column(Integer, primary_key=True, index=True) + code = Column(String(10), unique=True, nullable=False, index=True) + name = Column(String(255), nullable=True) + type = Column(String(100), nullable=True) + release_date = Column(DateTime, nullable=True) + base_set_size = Column(Integer, nullable=True) + total_size = Column(Integer, nullable=True) + is_foil_only = Column(Integer, nullable=True) + is_non_foil_only = Column(Integer, nullable=True) + digital = Column(Integer, nullable=True) + icon_svg_url = Column(Text, nullable=True) + parent_code = Column(String(10), nullable=True) + mtgo_code = Column(String(10), nullable=True) + created_at = Column(DateTime, server_default=func.now()) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + + # Relationships + cards = relationship("MtgCard", back_populates="set") + + def __repr__(self) -> str: + return f"" + + +class MtgCard(Base): + """MTG Card model.""" + __tablename__ = "mtg_cards" + + id = Column(Integer, primary_key=True, index=True) + set_id = Column(Integer, ForeignKey("mtg_sets.id"), nullable=True, index=True) + name = Column(String(255), nullable=True, index=True) + mana_cost = Column(String(255), nullable=True, index=True) + type_line = Column(String(255), nullable=True, index=True) + oracle_text = Column(Text, nullable=True) + power = Column(String(50), nullable=True) + toughness = Column(String(50), nullable=True) + rarity = Column(String(50), nullable=True, index=True) + layout = Column(String(50), nullable=True) + artist = Column(String(255), nullable=True) + flavor_text = Column(Text, nullable=True) + numbers = Column(String(100), nullable=True) + identifiers = Column(Text, nullable=True) # JSON string + images = Column(Text, nullable=True) # JSON string + created_at = Column(DateTime, server_default=func.now()) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + + # Relationships + set = relationship("MtgSet", back_populates="cards") + + def __repr__(self) -> str: + return f"" + + +# Indexes for performance +Index("idx_mtg_cards_name_set", MtgCard.name, MtgCard.set_id) +Index("idx_mtg_cards_type", MtgCard.type_line) +Index("idx_mtg_cards_rarity", MtgCard.rarity) diff --git a/backend/app/monitor/mtg_monitor.py b/backend/app/monitor/mtg_monitor.py new file mode 100644 index 0000000..5aeb3bb --- /dev/null +++ b/backend/app/monitor/mtg_monitor.py @@ -0,0 +1,336 @@ +""" +MTG Database Monitor + +Monitors PostgreSQL database metrics for both cockatrice and mtgdata databases. +Tracks: +- Database size and growth +- Table sizes +- Index sizes +- Query performance +- Weekly refresh metrics +- Image URL statistics +- Refresh success/failure rates +""" +import asyncpg +import asyncio +from datetime import datetime +from typing import Dict, List, Tuple +import json +from pathlib import Path +from app.core.settings import get_settings + +settings = get_settings() + +# Database configuration from settings +COCKATRICE_DB = settings.DATABASE_URL +MTG_DB = settings.MTG_DATABASE_URL + +class MtgMonitor: + def __init__(self): + self.mtg_conn = None + self.cockatrice_conn = None + self.metrics = {} + + async def connect(self): + """Establish connections to both databases.""" + try: + self.mtg_conn = await asyncpg.connect(MTG_DB) + self.cockatrice_conn = await asyncpg.connect(COCKATRICE_DB) + return True + except Exception as e: + print(f"Connection error: {e}") + return False + + async def disconnect(self): + """Close database connections.""" + if self.mtg_conn: + await self.mtg_conn.close() + if self.cockatrice_conn: + await self.cockatrice_conn.close() + + async def get_database_size(self) -> Dict[str, float]: + """Get size of databases in GB.""" + try: + # MTG database size + mtg_size = await self.mtg_conn.fetchval(""" + SELECT pg_database_size(current_database()) as size + """) + + # Cockatrice database size + cockatrice_size = await self.cockatrice_conn.fetchval(""" + SELECT pg_database_size(current_database()) as size + """) + + return { + "mtgdata": mtg_size / (1024**3), # Convert to GB + "cockatrice": cockatrice_size / (1024**3) + } + except Exception as e: + print(f"Error getting database sizes: {e}") + return {} + + async def get_table_sizes(self) -> Dict[str, float]: + """Get sizes of all tables in MB.""" + try: + result = await self.mtg_conn.fetch(""" + SELECT + schemaname || '.' || tablename as table_name, + pg_size_pretty(pg_total_relation_size(schemaname || '.' || tablename)) as size, + pg_total_relation_size(schemaname || '.' || tablename) as size_bytes + FROM pg_tables + WHERE schemaname = 'public' + ORDER BY pg_total_relation_size(schemaname || '.' || tablename) DESC + """) + + return { + row[0]: row[2] / (1024**2) # Convert to MB + for row in result + } + except Exception as e: + print(f"Error getting table sizes: {e}") + return {} + + async def get_index_sizes(self) -> Dict[str, float]: + """Get sizes of all indexes in MB.""" + try: + result = await self.mtg_conn.fetch(""" + SELECT + indexname as index_name, + pg_size_pretty(pg_relation_size(indexname::regclass)) as size, + pg_relation_size(indexname::regclass) as size_bytes + FROM pg_indexes + WHERE schemaname = 'public' + ORDER BY pg_relation_size(indexname::regclass) DESC + """) + + return { + row[0]: row[2] / (1024**2) # Convert to MB + for row in result + } + except Exception as e: + print(f"Error getting index sizes: {e}") + return {} + + async def get_table_row_counts(self) -> Dict[str, int]: + """Get row counts for all tables.""" + try: + result = await self.mtg_conn.fetch(""" + SELECT + schemaname || '.' || tablename as table_name, + n_live_tup as row_count + FROM pg_stat_user_tables + WHERE schemaname = 'public' + ORDER BY n_live_tup DESC + """) + + return { + row[0]: row[1] + for row in result + } + except Exception as e: + print(f"Error getting row counts: {e}") + return {} + + async def get_image_url_stats(self) -> Dict[str, any]: + """Get statistics about image URLs in cards table.""" + try: + # Count cards with images + cards_with_images = await self.mtg_conn.fetchval(""" + SELECT COUNT(*) FROM mtg_cards + WHERE images IS NOT NULL AND images != '{}' + """) + + # Total unique image URLs + unique_images = await self.mtg_conn.fetchval(""" + SELECT COUNT(DISTINCT jsonb_array_elements_text(images)) + FROM mtg_cards + WHERE images IS NOT NULL AND images != '{}' + """) + + # Most common image resolutions + resolutions = await self.mtg_conn.fetch(""" + SELECT + jsonb_object_keys(images) as resolution, + COUNT(*) as count + FROM mtg_cards + WHERE images IS NOT NULL AND images != '{}' + GROUP BY jsonb_object_keys(images) + ORDER BY count DESC + """) + + # Image URL patterns (domains) + domains = await self.mtg_conn.fetch(""" + SELECT + regexp_replace(images::text, '.*("normal":"[^"]*").*', '\\1') as domain + FROM mtg_cards + WHERE images IS NOT NULL AND images != '{}' + LIMIT 1000 + """) + + return { + "cards_with_images": cards_with_images, + "unique_image_urls": unique_images, + "resolutions": {row[0]: row[1] for row in resolutions}, + "sample_domains": [str(d[0]) for d in domains[:5]] + } + except Exception as e: + print(f"Error getting image stats: {e}") + return {} + + async def get_refresh_metrics(self) -> Dict[str, any]: + """Get refresh statistics from mtg_refresh_log.""" + try: + # Total refreshes + total_refreshes = await self.mtg_conn.fetchval(""" + SELECT COUNT(*) FROM mtg_refresh_log + """) + + # Success vs failure rates + status_counts = await self.mtg_conn.fetch(""" + SELECT status, COUNT(*) as count + FROM mtg_refresh_log + GROUP BY status + ORDER BY count DESC + """) + + # Average duration + avg_duration = await self.mtg_conn.fetchval(""" + SELECT AVG(duration_seconds) FROM mtg_refresh_log + """) + + # Last refresh + last_refresh = await self.mtg_conn.fetch(""" + SELECT * FROM mtg_refresh_log + ORDER BY refresh_date DESC + LIMIT 1 + """) + + # Cards updated per refresh (average) + avg_cards = await self.mtg_conn.fetchval(""" + SELECT AVG(cards_updated) FROM mtg_refresh_log + WHERE status = 'SUCCESS' + """) + + return { + "total_refreshes": total_refreshes, + "status_counts": {row[0]: row[1] for row in status_counts}, + "avg_duration_seconds": avg_duration, + "avg_cards_per_refresh": avg_cards, + "last_refresh": last_refresh[0] if last_refresh else None + } + except Exception as e: + print(f"Error getting refresh metrics: {e}") + return {} + + async def collect_metrics(self) -> Dict[str, any]: + """Collect all metrics.""" + if not await self.connect(): + return {"error": "Failed to connect to databases"} + + try: + metrics = { + "timestamp": datetime.now().isoformat(), + "database_sizes": await self.get_database_size(), + "table_sizes": await self.get_table_sizes(), + "index_sizes": await self.get_index_sizes(), + "row_counts": await self.get_table_row_counts(), + "image_stats": await self.get_image_url_stats(), + "refresh_metrics": await self.get_refresh_metrics() + } + + self.metrics = metrics + return metrics + + finally: + await self.disconnect() + + def generate_report(self, metrics: Dict[str, any]) -> str: + """Generate a human-readable report.""" + report = [] + report.append("=" * 60) + report.append("MTG Database Monitor Report") + report.append("=" * 60) + report.append(f"Generated: {metrics['timestamp']}") + report.append("") + + # Database sizes + report.append("DATABASE SIZES") + report.append("-" * 40) + for db, size in metrics.get('database_sizes', {}).items(): + report.append(f" {db}: {size:.2f} GB") + report.append("") + + # Table sizes + report.append("TABLE SIZES") + report.append("-" * 40) + for table, size in metrics.get('table_sizes', {}).items(): + report.append(f" {table}: {size:.2f} MB") + report.append("") + + # Row counts + report.append("ROW COUNTS") + report.append("-" * 40) + for table, count in metrics.get('row_counts', {}).items(): + report.append(f" {table}: {count:,} rows") + report.append("") + + # Image stats + report.append("IMAGE STATISTICS") + report.append("-" * 40) + image_stats = metrics.get('image_stats', {}) + report.append(f" Cards with images: {image_stats.get('cards_with_images', 0):,}") + report.append(f" Unique image URLs: {image_stats.get('unique_image_urls', 0):,}") + if image_stats.get('resolutions'): + report.append(" Image resolutions:") + for res, count in image_stats['resolutions'].items(): + report.append(f" {res}: {count:,} cards") + if image_stats.get('sample_domains'): + report.append(" Sample domains:") + for domain in image_stats['sample_domains']: + report.append(f" {domain}") + report.append("") + + # Refresh metrics + report.append("REFRESH METRICS") + report.append("-" * 40) + refresh = metrics.get('refresh_metrics', {}) + report.append(f" Total refreshes: {refresh.get('total_refreshes', 0)}") + if refresh.get('status_counts'): + report.append(" Status counts:") + for status, count in refresh['status_counts'].items(): + report.append(f" {status}: {count}") + report.append(f" Average duration: {refresh.get('avg_duration_seconds', 0):.1f} seconds") + report.append(f" Average cards per refresh: {refresh.get('avg_cards_per_refresh', 0):,}") + if refresh.get('last_refresh'): + report.append(f" Last refresh: {refresh['last_refresh'].get('refresh_date', 'N/A')}") + report.append(f" Status: {refresh['last_refresh'].get('status', 'N/A')}") + report.append(f" Cards updated: {refresh['last_refresh'].get('cards_updated', 0):,}") + report.append("") + + return "\n".join(report) + +async def main(): + """Run monitoring and generate report.""" + monitor = MtgMonitor() + metrics = await monitor.collect_metrics() + + if metrics.get('error'): + print(f"Error: {metrics['error']}") + return + + report = monitor.generate_report(metrics) + print(report) + + # Save to file + with open("/app/mtg-monitor-report.txt", "w") as f: + f.write(report) + + # Save metrics as JSON for programmatic use + with open("/app/mtg-monitor-metrics.json", "w") as f: + json.dump(metrics, f, indent=2, default=str) + + print("Report saved to /app/mtg-monitor-report.txt") + print("Metrics saved to /app/mtg-monitor-metrics.json") + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/backend/app/routers/auth.py b/backend/app/routers/auth.py index 4de6def..d2aa81d 100644 --- a/backend/app/routers/auth.py +++ b/backend/app/routers/auth.py @@ -4,7 +4,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy import select from typing import Optional -from datetime import datetime +from datetime import datetime, timezone, timezone from app.core.database import get_db from app.core.security import ( @@ -47,14 +47,14 @@ async def login(request: LoginRequest, db: AsyncSession = Depends(get_db)): detail="Account is disabled", ) - if user.is_banned and user.ban_ends and user.ban_ends > __import__("datetime").datetime.now(): + if user.is_banned and user.ban_ends and user.ban_ends > datetime.now(): raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="Account is banned", ) # Update last login - user.last_login = __import__("datetime").datetime.now() + user.last_login = datetime.now(timezone.utc) await db.flush() # Generate tokens diff --git a/backend/app/routers/card_router.py b/backend/app/routers/card_router.py new file mode 100644 index 0000000..7300ceb --- /dev/null +++ b/backend/app/routers/card_router.py @@ -0,0 +1,212 @@ +""" +Card search router for MTG card database. + +Provides endpoints for searching and retrieving MTG card data +from the MTG PostgreSQL database with Redis caching. +""" +from fastapi import APIRouter, Depends, HTTPException, Query +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.database import mtg_get_db +from app.core.redis_client import cache_get, cache_set +from app.services.card_database import ( + search_cards, + get_card_by_name, + get_cards_by_set, + get_card_types, + get_card_rarities, + get_sets, + get_set_by_code, + get_card_statistics, +) + +router = APIRouter(prefix="/mtg/cards", tags=["MTG Cards"]) + + +@router.get("/search") +async def search_cards_endpoint( + q: str = Query(..., min_length=1, description="Search query"), + limit: int = Query(100, ge=1, le=500, description="Maximum results"), + offset: int = Query(0, ge=0, description="Number of results to skip"), + db: AsyncSession = Depends(mtg_get_db), +): + """ + Search cards by name, type, or mana cost. + + Uses Redis cache to improve performance for repeated searches. + """ + cache_key = f"card_search:{q}:{limit}:{offset}" + + # Check cache first + cached = await cache_get(cache_key) + if cached: + return {"cached": True, "results": cached} + + # Query database + results = await search_cards(q, db, limit, offset) + + # Cache results for 5 minutes + await cache_set(cache_key, str(results), ttl=300) + + return {"cached": False, "results": results} + + +@router.get("/{card_name}") +async def get_card_endpoint( + card_name: str, + set_code: str | None = Query(None, description="Filter by set code"), + db: AsyncSession = Depends(mtg_get_db), +): + """ + Get a specific card by name. + + Optional set_code filter to get a specific printing. + """ + cache_key = f"card_by_name:{card_name}:{set_code or 'all'}" + + cached = await cache_get(cache_key) + if cached: + return {"cached": True, "results": cached} + + card = await get_card_by_name(card_name, db, set_code) + + if not card: + raise HTTPException(status_code=404, detail="Card not found") + + # Cache for 10 minutes + await cache_set(cache_key, str(card), ttl=600) + + return {"cached": False, "results": card} + + +@router.get("/set/{set_code}") +async def get_cards_by_set_endpoint( + set_code: str, + limit: int = Query(1000, ge=1, le=5000, description="Maximum results"), + offset: int = Query(0, ge=0, description="Number of results to skip"), + db: AsyncSession = Depends(mtg_get_db), +): + """ + Get all cards in a specific set. + """ + cache_key = f"set_cards:{set_code}:{limit}:{offset}" + + cached = await cache_get(cache_key) + if cached: + return {"cached": True, "results": cached} + + results = await get_cards_by_set(set_code, db, limit, offset) + + # Cache for 15 minutes + await cache_set(cache_key, str(results), ttl=900) + + return {"cached": False, "results": results} + + +@router.get("/types") +async def get_card_types_endpoint( + db: AsyncSession = Depends(mtg_get_db), +): + """ + Get all unique card types. + """ + cache_key = "card_types:all" + + cached = await cache_get(cache_key) + if cached: + return {"cached": True, "results": cached} + + types = await get_card_types(db) + + # Cache for 30 minutes + await cache_set(cache_key, str(types), ttl=1800) + + return {"cached": False, "results": types} + + +@router.get("/rarities") +async def get_card_rarities_endpoint( + db: AsyncSession = Depends(mtg_get_db), +): + """ + Get all unique card rarities. + """ + cache_key = "card_rarities:all" + + cached = await cache_get(cache_key) + if cached: + return {"cached": True, "results": cached} + + rarities = await get_card_rarities(db) + + # Cache for 30 minutes + await cache_set(cache_key, str(rarities), ttl=1800) + + return {"cached": False, "results": rarities} + + +@router.get("/sets") +async def get_sets_endpoint( + db: AsyncSession = Depends(mtg_get_db), +): + """ + Get all sets. + """ + cache_key = "all_sets:all" + + cached = await cache_get(cache_key) + if cached: + return {"cached": True, "results": cached} + + sets = await get_sets(db) + + # Cache for 1 hour + await cache_set(cache_key, str(sets), ttl=3600) + + return {"cached": False, "results": sets} + + +@router.get("/sets/{set_code}") +async def get_set_endpoint( + set_code: str, + db: AsyncSession = Depends(mtg_get_db), +): + """ + Get a specific set by code. + """ + cache_key = f"set_by_code:{set_code}" + + cached = await cache_get(cache_key) + if cached: + return {"cached": True, "results": cached} + + mtg_set = await get_set_by_code(set_code, db) + + if not mtg_set: + raise HTTPException(status_code=404, detail="Set not found") + + # Cache for 1 hour + await cache_set(cache_key, str(mtg_set), ttl=3600) + + return {"cached": False, "results": mtg_set} + + +@router.get("/statistics") +async def get_card_statistics_endpoint( + db: AsyncSession = Depends(mtg_get_db), +): + """ + Get overall card database statistics. + """ + cache_key = "card_statistics:all" + + cached = await cache_get(cache_key) + if cached: + return {"cached": True, "results": cached} + + stats = await get_card_statistics(db) + + # Cache for 1 hour + await cache_set(cache_key, str(stats), ttl=3600) + + return {"cached": False, "results": stats} diff --git a/backend/app/routers/decks.py b/backend/app/routers/decks.py index 6c33bbd..d80189a 100644 --- a/backend/app/routers/decks.py +++ b/backend/app/routers/decks.py @@ -171,77 +171,6 @@ async def delete_folder( return {"message": "Folder deleted successfully"} -@router.get("/", response_model=List[DeckResponse]) -async def list_decks( - folder_id: Optional[int] = None, - page: int = 1, - page_size: int = 50, - db: AsyncSession = Depends(get_db), - current_user: dict = Depends(get_current_user), -): - """List decks for current user.""" - user_id = int(current_user["user_id"]) - - if folder_id: - stmt = ( - select(DecklistFile) - .where( - DecklistFile.owner_id == user_id, - DecklistFile.folder_id == folder_id, - ) - .offset((page - 1) * page_size) - .limit(page_size) - ) - else: - stmt = ( - select(DecklistFile) - .where(DecklistFile.owner_id == user_id) - .offset((page - 1) * page_size) - .limit(page_size) - ) - - result = await db.execute(stmt) - decks = result.scalars().all() - - return [DeckResponse.model_validate(deck) for deck in decks] - - -@router.post("/", response_model=DeckResponse) -async def create_deck( - request: DeckCreate, - db: AsyncSession = Depends(get_db), - current_user: dict = Depends(get_current_user), -): - """Create a new deck.""" - user_id = int(current_user["user_id"]) - - # Verify folder exists if specified - if request.folder_id: - stmt = select(DecklistFolder).where( - DecklistFolder.id == request.folder_id, - DecklistFolder.owner_id == user_id, - ) - result = await db.execute(stmt) - folder = result.scalar_one_or_none() - if not folder: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Folder not found", - ) - - new_deck = DecklistFile( - owner_id=user_id, - folder_id=request.folder_id, - name=request.name, - content=request.content, - format=request.format, - ) - db.add(new_deck) - await db.flush() - - return DeckResponse.model_validate(new_deck) - - @router.get("/{deck_id}", response_model=DeckResponse) async def get_deck( deck_id: int, diff --git a/backend/app/scripts/__init__.py b/backend/app/scripts/__init__.py new file mode 100644 index 0000000..69cf38a --- /dev/null +++ b/backend/app/scripts/__init__.py @@ -0,0 +1 @@ +# MTG Data Scripts diff --git a/backend/app/scripts/refresh_mtg.py b/backend/app/scripts/refresh_mtg.py new file mode 100644 index 0000000..339b04c --- /dev/null +++ b/backend/app/scripts/refresh_mtg.py @@ -0,0 +1,226 @@ +""" +MTGJSON Database Refresh Script + +Downloads and updates the MTGJSON All Printings dataset weekly. +""" +import asyncio +import json +import logging +import os +import time +from datetime import datetime +from pathlib import Path + +import aiohttp +from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession +from sqlalchemy import text +from sqlalchemy.orm import sessionmaker +from app.core.settings import get_settings + +logger = logging.getLogger(__name__) + +MTGJSON_API_URL = "https://mtgjson.com/api/v5/AllPrintings.json" +DATA_DIR = Path("/app/data") +REFRESH_INTERVAL_DAYS = 7 + + +async def download_mtgjson(session: aiohttp.ClientSession, output_path: Path) -> bool: + """Download the latest AllPrintings dataset.""" + try: + logger.info(f"Downloading MTGJSON from {MTGJSON_API_URL}...") + async with session.get(MTGJSON_API_URL) as response: + if response.status != 200: + logger.error(f"Failed to download: {response.status}") + return False + + with open(output_path, 'wb') as f: + async for chunk in response.content.iter_chunked(8192): + f.write(chunk) + + logger.info(f"Downloaded to {output_path}") + return True + except Exception as e: + logger.error(f"Download error: {e}") + return False + + +async def parse_mtgjson(filepath: Path) -> dict: + """Parse the AllPrintings JSON file.""" + try: + with open(filepath, 'r', encoding='utf-8') as f: + data = json.load(f) + + # Verify structure + if 'data' not in data or 'sets' not in data: + raise ValueError("Invalid MTGJSON structure") + + return data['data'] + except Exception as e: + logger.error(f"Parse error: {e}") + return {} + + +async def update_database(session: AsyncSession, data: dict) -> tuple[int, int]: + """Update the database with parsed MTGJSON data.""" + cards_updated = 0 + sets_updated = 0 + + try: + # Process sets + for set_code, set_data in data.get('sets', {}).items(): + stmt = text(""" + INSERT INTO mtg_sets (code, name, type, release_date, base_set_size, + total_size, is_foil_only, is_non_foil_only, + digital, icon_svg_url, parent_code, mtgo_code) + VALUES (:code, :name, :type, :release_date, :base_set_size, + :total_size, :is_foil_only, :is_non_foil_only, + :digital, :icon_svg_url, :parent_code, :mtgo_code) + ON CONFLICT (code) DO UPDATE SET + name = EXCLUDED.name, + updated_at = CURRENT_TIMESTAMP + """) + + await session.execute(stmt, { + 'code': set_code, + 'name': set_data.get('name'), + 'type': set_data.get('type'), + 'release_date': set_data.get('releaseDate'), + 'base_set_size': set_data.get('baseSetSize'), + 'total_size': set_data.get('totalSize'), + 'is_foil_only': set_data.get('isFoilOnly'), + 'is_non_foil_only': set_data.get('isNonFoilOnly'), + 'digital': set_data.get('digital'), + 'icon_svg_url': set_data.get('iconSvgUri'), + 'parent_code': set_data.get('parentCode'), + 'mtgo_code': set_data.get('mtgoCode'), + }) + sets_updated += 1 + + # Process cards + for card_data in data.get('cards', []): + stmt = text(""" + INSERT INTO mtg_cards (set_id, name, mana_cost, type_line, oracle_text, + power, toughness, rarity, layout, artist, + flavor_text, numbers, identifiers, images) + SELECT s.id, :name, :mana_cost, :type_line, :oracle_text, + :power, :toughness, :rarity, :layout, :artist, + :flavor_text, :numbers, :identifiers, :images + FROM mtg_sets s + WHERE s.code = :set_code + ON CONFLICT DO NOTHING + """) + + await session.execute(stmt, { + 'set_code': card_data.get('set'), + 'name': card_data.get('name'), + 'mana_cost': card_data.get('manaCost'), + 'type_line': card_data.get('type'), + 'oracle_text': card_data.get('text'), + 'power': card_data.get('power'), + 'toughness': card_data.get('toughness'), + 'rarity': card_data.get('rarity'), + 'layout': card_data.get('layout'), + 'artist': card_data.get('artist'), + 'flavor_text': card_data.get('flavorText'), + 'numbers': str(card_data.get('numbers', '')), + 'identifiers': json.dumps(card_data.get('identifiers', {})), + 'images': json.dumps(card_data.get('images', {})), + }) + cards_updated += 1 + + await session.commit() + return cards_updated, sets_updated + + except Exception as e: + logger.error(f"Database update error: {e}") + await session.rollback() + raise + + +async def check_last_refresh(engine: create_async_engine) -> datetime: + """Check when the last refresh occurred.""" + async with AsyncSession(engine) as session: + stmt = text("SELECT refresh_date FROM mtg_refresh_log ORDER BY refresh_date DESC LIMIT 1") + result = await session.execute(stmt) + row = result.fetchone() + + if row: + return row[0] + return datetime.min + + +async def log_refresh(engine: create_async_engine, status: str, cards: int, sets: int, + duration: int, error: str = None): + """Log the refresh operation.""" + async with AsyncSession(engine) as session: + stmt = text(""" + INSERT INTO mtg_refresh_log (status, cards_updated, sets_updated, + error_message, duration_seconds) + VALUES (:status, :cards, :sets, :error, :duration) + """) + await session.execute(stmt, { + 'status': status, + 'cards': cards, + 'sets': sets, + 'error': error, + 'duration': duration, + }) + await session.commit() + + +async def main(): + """Main refresh logic.""" + logging.basicConfig(level=logging.INFO) + + settings = get_settings() + + DATA_DIR = Path(settings.DATA_DIR) + REFRESH_INTERVAL_DAYS = settings.MTG_REFRESH_INTERVAL_DAYS + + # Use database URL from settings + engine = create_async_engine(settings.MTG_DATABASE_URL) + + last_refresh = await check_last_refresh(engine) + refresh_needed = (datetime.now() - last_refresh).days >= REFRESH_INTERVAL_DAYS + + if not refresh_needed: + logger.info("Refresh not needed. Last refresh was within interval.") + return + + start_time = time.time() + + try: + async with aiohttp.ClientSession() as session: + # Download dataset + download_path = DATA_DIR / "AllPrintings.json" + success = await download_mtgjson(session, download_path) + + if not success: + await log_refresh(engine, "FAILED", 0, 0, 0, "Download failed") + return + + # Parse data + data = await parse_mtgjson(download_path) + if not data: + await log_refresh(engine, "FAILED", 0, 0, 0, "Parse failed") + return + + # Update database + async with AsyncSession(engine) as db_session: + cards_updated, sets_updated = await update_database(db_session, data) + + # Log success + duration = int(time.time() - start_time) + await log_refresh(engine, "SUCCESS", cards_updated, sets_updated, duration) + logger.info(f"Refresh completed: {cards_updated} cards, {sets_updated} sets in {duration}s") + + except Exception as e: + duration = int(time.time() - start_time) + await log_refresh(engine, "FAILED", 0, 0, duration, str(e)) + logger.error(f"Refresh failed: {e}") + + await engine.dispose() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/backend/app/services/__init__.py b/backend/app/services/__init__.py new file mode 100644 index 0000000..ddd3dbf --- /dev/null +++ b/backend/app/services/__init__.py @@ -0,0 +1,22 @@ +"""Services package.""" +from app.services.card_database import ( + search_cards, + get_card_by_name, + get_cards_by_set, + get_card_types, + get_card_rarities, + get_sets, + get_set_by_code, + get_card_statistics, +) + +__all__ = [ + "search_cards", + "get_card_by_name", + "get_cards_by_set", + "get_card_types", + "get_card_rarities", + "get_sets", + "get_set_by_code", + "get_card_statistics", +] diff --git a/backend/app/services/card_database.py b/backend/app/services/card_database.py index 00ce1c6..958381d 100644 --- a/backend/app/services/card_database.py +++ b/backend/app/services/card_database.py @@ -1,225 +1,350 @@ -"""Card database service for importing and querying card data.""" -import httpx -from typing import List, Dict, Optional -from dataclasses import dataclass -from enum import Enum +""" +MTG Card Database Service. + +Queries the MTG PostgreSQL database for card data. +""" +from typing import List, Dict, Any, Optional +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy import select, and_, or_, func +from sqlalchemy.orm import selectinload + +from app.models.mtg_models import MtgCard, MtgSet +from app.core.database import mtg_get_db -class CardType(str, Enum): - """Card types from Magic: The Gathering.""" - CREATURES = "Creature" - INSTANT = "Instant" - SORCERY = "Sorcery" - ENCHANTMENT = "Enchantment" - ARTIFACT = "Artifact" - PLANE = "Plane" - PLANESWALKER = "Planeswalker" - LAND = "Land" - BATTLE = "Battle" - - -class CardColor(str, Enum): - """Card colors.""" - WHITE = "W" - BLUE = "U" - BLACK = "B" - RED = "R" - GREEN = "G" - COLORLESS = "C" - MULTICOLOR = "M" - SHARD = "S" - WIDGET = "X" - - -class CardRarity(str, Enum): - """Card rarities.""" - COMMON = "common" - UNCOMMON = "uncommon" - RARE = "rare" - MYTHIC = "mythic" - SPECIAL = "special" - - -@dataclass -class CardData: - """Card information from card database.""" - id: int - name: str - types: List[CardType] - colors: List[CardColor] - rarity: CardRarity - set_code: str - collector_number: str - flavor_text: Optional[str] = None - rules_text: Optional[str] = None - power: Optional[str] = None - toughness: Optional[str] = None - artist: Optional[str] = None - image_url: Optional[str] = None - provider_id: Optional[str] = None +async def search_cards( + query: str, + db: AsyncSession, + limit: int = 100, + offset: int = 0, +) -> Dict[str, Any]: + """ + Search cards by name, type, or mana cost. - def __str__(self) -> str: - return f"{self.name} ({self.set_code}-{self.collector_number})" - - -class CardDatabase: - """Card database service for importing and querying card data.""" + Args: + query: Search string + db: Database session + limit: Maximum results to return + offset: Number of results to skip - def __init__(self): - self.cards: Dict[int, CardData] = {} - self._next_id = 1 + Returns: + Dictionary with results and total count + """ + search_term = f"%{query.lower()}%" - async def import_from_mtjson(self, url: str = "https://mtjson.xyz/api/5.0.0/") -> List[CardData]: - """Import card data from MTJSON API.""" - async with httpx.AsyncClient() as client: - response = await client.get(url) - response.raise_for_status() - data = response.json() - - imported_cards = [] - for card_data in data: - card = self._parse_mtjson_card(card_data) - self.cards[self._next_id] = card - imported_cards.append(card) - self._next_id += 1 - - return imported_cards - - def _parse_mtjson_card(self, data: dict) -> CardData: - """Parse MTJSON card data into CardData.""" - card_id = self._next_id - - # Extract types - types = [] - if "types" in data: - for type_str in data["types"]: - try: - types.append(CardType(type_str)) - except ValueError: - pass - - # Extract colors - colors = [] - if "colors" in data: - for color_str in data["colors"]: - try: - colors.append(CardColor(color_str)) - except ValueError: - pass - - # Extract rarity - rarity = CardRarity.COMMON - if "rarity" in data: - try: - rarity = CardRarity(data["rarity"].lower()) - except ValueError: - pass - - # Extract set and collector number - set_code = "" - collector_number = "" - if "set" in data: - set_code = data["set"] - if "collectorNumber" in data: - collector_number = data["collectorNumber"] - - # Extract image URL - image_url = None - if "imageUris" in data and "normal" in data["imageUris"]: - image_url = data["imageUris"]["normal"] - - return CardData( - id=card_id, - name=data.get("name", ""), - types=types, - colors=colors, - rarity=rarity, - set_code=set_code, - collector_number=collector_number, - flavor_text=data.get("flavorText"), - rules_text=data.get("rulesText"), - power=data.get("power"), - toughness=data.get("toughness"), - artist=data.get("artist"), - image_url=image_url, - provider_id=data.get("multiverseId"), + # Search across multiple fields + stmt = ( + select(MtgCard, MtgSet) + .join(MtgSet, MtgCard.set_id == MtgSet.id, isouter=True) + .where( + or_( + MtgCard.name.ilike(search_term), + MtgCard.type_line.ilike(search_term), + MtgCard.mana_cost.ilike(search_term), + ) ) + .offset(offset) + .limit(limit) + ) - def get_card_by_id(self, card_id: int) -> Optional[CardData]: - """Get card by ID.""" - return self.cards.get(card_id) + result = await db.execute(stmt) + rows = result.all() - def get_card_by_name(self, name: str) -> List[CardData]: - """Get cards by name (case-insensitive).""" - name_lower = name.lower() - return [card for card in self.cards.values() if card.name.lower() == name_lower] + cards = [] + for card, mtg_set in rows: + card_data = { + "id": card.id, + "name": card.name, + "mana_cost": card.mana_cost, + "type_line": card.type_line, + "oracle_text": card.oracle_text, + "power": card.power, + "toughness": card.toughness, + "rarity": card.rarity, + "layout": card.layout, + "artist": card.artist, + "flavor_text": card.flavor_text, + "set_code": mtg_set.code if mtg_set else None, + "set_name": mtg_set.name if mtg_set else None, + "release_date": mtg_set.release_date.isoformat() if mtg_set and mtg_set.release_date else None, + "identifiers": card.identifiers, + "images": card.images, + } + cards.append(card_data) - def search_cards( - self, - query: str = "", - card_type: Optional[CardType] = None, - color: Optional[CardColor] = None, - rarity: Optional[CardRarity] = None, - set_code: Optional[str] = None, - limit: int = 50, - ) -> List[CardData]: - """Search cards with filters.""" - results = list(self.cards.values()) - - # Filter by query - if query: - query_lower = query.lower() - results = [card for card in results if query_lower in card.name.lower()] - - # Filter by type - if card_type: - results = [card for card in results if card_type in card.types] - - # Filter by color - if color: - results = [card for card in results if color in card.colors] - - # Filter by rarity - if rarity: - results = [card for card in results if card.rarity == rarity] - - # Filter by set - if set_code: - results = [card for card in results if card.set_code == set_code.upper()] - - return results[:limit] + # Get total count + count_stmt = select(func.count()).select_from(MtgCard) + count_result = await db.execute(count_stmt) + total = count_result.scalar() - def get_random_card(self) -> Optional[CardData]: - """Get a random card from the database.""" - import random - if not self.cards: - return None - return random.choice(list(self.cards.values())) + return { + "results": cards, + "total": total, + "limit": limit, + "offset": offset, + } + + +async def get_card_by_name( + name: str, + db: AsyncSession, + set_code: Optional[str] = None, +) -> Optional[Dict[str, Any]]: + """ + Get a specific card by name. - def get_card_count(self) -> int: - """Get total number of cards in database.""" - return len(self.cards) + Args: + name: Card name + db: Database session + set_code: Optional set code to filter by + + Returns: + Card data or None + """ + stmt = ( + select(MtgCard, MtgSet) + .join(MtgSet, MtgCard.set_id == MtgSet.id, isouter=True) + .where(MtgCard.name.ilike(name)) + ) + + if set_code: + stmt = stmt.where(MtgSet.code == set_code) + + stmt = stmt.limit(1) + + result = await db.execute(stmt) + row = result.fetchone() + + if not row: + return None + + card, mtg_set = row + + return { + "id": card.id, + "name": card.name, + "mana_cost": card.mana_cost, + "type_line": card.type_line, + "oracle_text": card.oracle_text, + "power": card.power, + "toughness": card.toughness, + "rarity": card.rarity, + "layout": card.layout, + "artist": card.artist, + "flavor_text": card.flavor_text, + "set_code": mtg_set.code if mtg_set else None, + "set_name": mtg_set.name if mtg_set else None, + "release_date": mtg_set.release_date.isoformat() if mtg_set and mtg_set.release_date else None, + "identifiers": card.identifiers, + "images": card.images, + } -# Singleton instance -card_database = CardDatabase() +async def get_cards_by_set( + set_code: str, + db: AsyncSession, + limit: int = 1000, + offset: int = 0, +) -> Dict[str, Any]: + """ + Get all cards in a specific set. + + Args: + set_code: Set code + db: Database session + limit: Maximum results to return + offset: Number of results to skip + + Returns: + Dictionary with results and total count + """ + # First get the set + set_stmt = select(MtgSet).where(MtgSet.code == set_code) + set_result = await db.execute(set_stmt) + mtg_set = set_result.scalar_one_or_none() + + if not mtg_set: + return {"results": [], "total": 0, "limit": limit, "offset": offset} + + # Get cards in the set + stmt = ( + select(MtgCard, MtgSet) + .join(MtgSet, MtgCard.set_id == MtgSet.id, isouter=True) + .where(MtgCard.set_id == mtg_set.id) + .offset(offset) + .limit(limit) + ) + + result = await db.execute(stmt) + rows = result.all() + + cards = [] + for card, _ in rows: + card_data = { + "id": card.id, + "name": card.name, + "mana_cost": card.mana_cost, + "type_line": card.type_line, + "oracle_text": card.oracle_text, + "power": card.power, + "toughness": card.toughness, + "rarity": card.rarity, + "layout": card.layout, + "artist": card.artist, + "flavor_text": card.flavor_text, + "set_code": mtg_set.code, + "set_name": mtg_set.name, + "release_date": mtg_set.release_date.isoformat() if mtg_set.release_date else None, + "identifiers": card.identifiers, + "images": card.images, + } + cards.append(card_data) + + # Get total count + count_stmt = select(func.count()).where(MtgCard.set_id == mtg_set.id) + count_result = await db.execute(count_stmt) + total = count_result.scalar() + + return { + "results": cards, + "total": total, + "limit": limit, + "offset": offset, + } -async def import_cards() -> List[CardData]: - """Import cards from MTJSON.""" - return await card_database.import_from_mtjson() +async def get_card_types(db: AsyncSession) -> List[Dict[str, Any]]: + """ + Get all unique card types. + + Args: + db: Database session + + Returns: + List of card types + """ + stmt = select(MtgCard.type_line).distinct().order_by(MtgCard.type_line) + result = await db.execute(stmt) + rows = result.fetchall() + + return [{"type": row[0]} for row in rows] -def search_cards(**kwargs) -> List[CardData]: - """Search cards with filters.""" - return card_database.search_cards(**kwargs) +async def get_card_rarities(db: AsyncSession) -> List[Dict[str, Any]]: + """ + Get all unique card rarities. + + Args: + db: Database session + + Returns: + List of rarities + """ + stmt = select(MtgCard.rarity).distinct().order_by(MtgCard.rarity) + result = await db.execute(stmt) + rows = result.fetchall() + + return [{"rarity": row[0]} for row in rows] -def get_card_by_name(name: str) -> List[CardData]: - """Get cards by name.""" - return card_database.get_card_by_name(name) +async def get_sets(db: AsyncSession) -> List[Dict[str, Any]]: + """ + Get all sets. + + Args: + db: Database session + + Returns: + List of sets + """ + stmt = select(MtgSet).order_by(MtgSet.release_date.desc()) + result = await db.execute(stmt) + rows = result.fetchall() + + return [ + { + "id": s.id, + "code": s.code, + "name": s.name, + "release_date": s.release_date.isoformat() if s.release_date else None, + "total_size": s.total_size, + "base_set_size": s.base_set_size, + } + for s in rows + ] -def get_card_by_id(card_id: int) -> Optional[CardData]: - """Get card by ID.""" - return card_database.get_card_by_id(card_id) +async def get_set_by_code(code: str, db: AsyncSession) -> Optional[Dict[str, Any]]: + """ + Get a specific set by code. + + Args: + code: Set code + db: Database session + + Returns: + Set data or None + """ + stmt = select(MtgSet).where(MtgSet.code == code) + result = await db.execute(stmt) + mtg_set = result.scalar_one_or_none() + + if not mtg_set: + return None + + return { + "id": mtg_set.id, + "code": mtg_set.code, + "name": mtg_set.name, + "type": mtg_set.type, + "release_date": mtg_set.release_date.isoformat() if mtg_set.release_date else None, + "base_set_size": mtg_set.base_set_size, + "total_size": mtg_set.total_size, + "is_foil_only": mtg_set.is_foil_only, + "is_non_foil_only": mtg_set.is_non_foil_only, + "digital": mtg_set.digital, + "icon_svg_url": mtg_set.icon_svg_url, + "parent_code": mtg_set.parent_code, + "mtgo_code": mtg_set.mtgo_code, + } + + +async def get_card_statistics(db: AsyncSession) -> Dict[str, Any]: + """ + Get overall card database statistics. + + Args: + db: Database session + + Returns: + Dictionary with statistics + """ + # Total cards + card_count_stmt = select(func.count()).select_from(MtgCard) + card_count = (await db.execute(card_count_stmt)).scalar() + + # Total sets + set_count_stmt = select(func.count()).select_from(MtgSet) + set_count = (await db.execute(set_count_stmt)).scalar() + + # Cards by rarity + rarity_stmt = select(MtgCard.rarity, func.count()).group_by(MtgCard.rarity) + rarity_result = await db.execute(rarity_stmt) + rarities = {row[0]: row[1] for row in rarity_result} + + # Cards by type + type_stmt = select(MtgCard.type_line, func.count()).group_by(MtgCard.type_line) + type_result = await db.execute(type_stmt) + types = {row[0]: row[1] for row in type_result} + + # Average mana cost (approximate) + avg_mana_stmt = select(func.count()).where(MtgCard.mana_cost.isnot(None)) + avg_mana_count = (await db.execute(avg_mana_stmt)).scalar() + + return { + "total_cards": card_count, + "total_sets": set_count, + "cards_by_rarity": rarities, + "cards_by_type": types, + "cards_with_mana_cost": avg_mana_count, + } diff --git a/backend/requirements.txt b/backend/requirements.txt index a8be14a..e0091b2 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -24,6 +24,7 @@ redis[hiredis]==5.1.0 # Utilities python-dotenv==1.0.1 httpx==0.27.2 +aiohttp==3.9.5 # Testing pytest==8.3.3 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..4a861e4 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,101 @@ +version: "3.8" + +services: + # PostgreSQL - hosts both Cockatrice app data and mtgjson data + postgres: + image: postgres:16-alpine + container_name: mtg_postgres + restart: unless-stopped + environment: + POSTGRES_USER: ${POSTGRES_USER:-cockatrice} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-cockatrice_pass} + POSTGRES_DB: ${POSTGRES_DB:-cockatrice} + POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" + ports: + - "${POSTGRES_PORT:-5432}:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + - ./scripts/init-db.sql:/docker-entrypoint-initdb.d/01-init.sql + networks: + - mtg_network + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-cockatrice}"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + + # Backend API + backend: + build: + context: ./backend + dockerfile: Dockerfile + container_name: mtg_backend + restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + environment: + - DATABASE_URL=${DATABASE_URL:-postgresql+asyncpg://cockatrice:cockatrice_pass@postgres:5432/cockatrice} + - MTG_DATABASE_URL=${MTG_DATABASE_URL:-postgresql+asyncpg://cockatrice:cockatrice_pass@postgres:5432/mtgdata} + - JWT_SECRET_KEY=${JWT_SECRET_KEY:-change-me-in-production} + - JWT_ALGORITHM=${JWT_ALGORITHM:-HS256} + - ACCESS_TOKEN_EXPIRE_MINUTES=${ACCESS_TOKEN_EXPIRE_MINUTES:-60} + - REFRESH_TOKEN_EXPIRE_DAYS=${REFRESH_TOKEN_EXPIRE_DAYS:-7} + - CORS_ORIGINS=${CORS_ORIGINS:-["http://localhost:3000","http://localhost:8000"]} + - APP_NAME=${APP_NAME:-Cockatrice Web} + - APP_VERSION=${APP_VERSION:-0.2.0} + - DEBUG=${DEBUG:-False} + - MTG_REFRESH_INTERVAL_DAYS=${MTG_REFRESH_INTERVAL_DAYS:-7} + - DATA_DIR=${DATA_DIR:-/app/data} + - UPLOAD_DIR=${UPLOAD_DIR:-/app/uploads} + - LOG_LEVEL=${LOG_LEVEL:-INFO} + ports: + - "${BACKEND_PORT:-8000}:8000" + volumes: + - mtg_data:/app/data + - mtg_uploads:/app/uploads + - mtg_logs:/app/logs + networks: + - mtg_network + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:8000/health || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + + # Weekly database refresh (runs on a schedule) + mtg-refresh: + build: + context: ./backend + dockerfile: Dockerfile + container_name: mtg_refresh + restart: "no" + depends_on: + postgres: + condition: service_healthy + environment: + - DATABASE_URL=${DATABASE_URL:-postgresql+asyncpg://cockatrice:cockatrice_pass@postgres:5432/cockatrice} + - MTG_DATABASE_URL=${MTG_DATABASE_URL:-postgresql+asyncpg://cockatrice:cockatrice_pass@postgres:5432/mtgdata} + - DATA_DIR=${DATA_DIR:-/app/data} + - MTG_REFRESH_INTERVAL_DAYS=${MTG_REFRESH_INTERVAL_DAYS:-7} + volumes: + - mtg_data:/app/data + command: ["python", "-m", "app.scripts.refresh_mtg"] + networks: + - mtg_network + +volumes: + postgres_data: + driver: local + mtg_data: + driver: local + mtg_uploads: + driver: local + mtg_logs: + driver: local + +networks: + mtg_network: + driver: bridge diff --git a/scripts/init-db.sql b/scripts/init-db.sql new file mode 100644 index 0000000..e500e65 --- /dev/null +++ b/scripts/init-db.sql @@ -0,0 +1,60 @@ +-- Initialize MTG data database +CREATE DATABASE mtgdata; + +-- Create extensions +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; +CREATE EXTENSION IF NOT EXISTS "pg_trgm"; + +-- Create mtgjson tables +CREATE TABLE IF NOT EXISTS mtg_sets ( + id SERIAL PRIMARY KEY, + code VARCHAR(10) UNIQUE NOT NULL, + name VARCHAR(255) NOT NULL, + type VARCHAR(50), + release_date DATE, + base_set_size INTEGER, + total_size INTEGER, + is_foil_only BOOLEAN DEFAULT FALSE, + is_non_foil_only BOOLEAN DEFAULT FALSE, + digital BOOLEAN DEFAULT FALSE, + icon_svg_url TEXT, + parent_code VARCHAR(10), + mtgo_code VARCHAR(10), + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS mtg_cards ( + id SERIAL PRIMARY KEY, + set_id INTEGER REFERENCES mtg_sets(id), + name VARCHAR(255) NOT NULL, + mana_cost TEXT, + type_line VARCHAR(255), + oracle_text TEXT, + power VARCHAR(10), + toughness VARCHAR(10), + rarity VARCHAR(50), + layout VARCHAR(50), + artist VARCHAR(255), + flavor_text TEXT, + numbers VARCHAR(50), + identifiers JSONB, + images JSONB, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX idx_mtg_cards_set_id ON mtg_cards(set_id); +CREATE INDEX idx_mtg_cards_name ON mtg_cards(name); +CREATE INDEX idx_mtg_cards_type ON mtg_cards(type_line); + +CREATE TABLE IF NOT EXISTS mtg_refresh_log ( + id SERIAL PRIMARY KEY, + refresh_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + status VARCHAR(50) NOT NULL, + cards_updated INTEGER DEFAULT 0, + sets_updated INTEGER DEFAULT 0, + error_message TEXT, + duration_seconds INTEGER +); + +CREATE INDEX idx_mtg_refresh_log_date ON mtg_refresh_log(refresh_date); diff --git a/scripts/postgres-config.conf b/scripts/postgres-config.conf new file mode 100644 index 0000000..7c0cebd --- /dev/null +++ b/scripts/postgres-config.conf @@ -0,0 +1,16 @@ +# PostgreSQL Configuration for MTG Database + +# postgresql.conf - optimize for mtgjson workload +shared_buffers = 256MB +effective_cache_size = 1GB +work_mem = 16MB +maintenance_work_mem = 256MB +random_page_cost = 1.1 +effective_io_concurrency = 200 +max_parallel_workers_per_gather = 2 +max_parallel_workers = 4 +max_parallel_maintenance_workers = 2 + +# pg_hba.conf - allow connections +host all all 127.0.0.1/32 md5 +host all all ::1/128 md5