diff --git a/backend/SUPPORTED_FILE_TYPES.md b/backend/SUPPORTED_FILE_TYPES.md new file mode 100644 index 0000000..a25409d --- /dev/null +++ b/backend/SUPPORTED_FILE_TYPES.md @@ -0,0 +1,101 @@ +# MTGJSON Data Manager - File Type Support + +## Supported File Types + +### 1. AllPrintings (Primary Card Database) +- **Accepts:** `AllPrintings.json` OR `AllPrintings.psql` +- **Processing:** + - `.json`: Parses JSON structure, extracts card data from `cards` array + - `.psql`: Parses SQL INSERT statements to extract card data +- **Database:** `mtg_cards` table +- **Required:** Yes (one of the two formats) + +### 2. AllIdentifiers (Stable Card Referencing) +- **Accepts:** `AllIdentifiers.json` +- **Processing:** Parses JSON structure, extracts identifiers +- **Database:** `mtg_identifiers` table +- **Required:** Yes + +### 3. Keywords & CardTypes (Game Logic/Mechanics) +- **Accepts:** `Keywords.json` and `CardTypes.json` +- **Processing:** Parses JSON arrays +- **Database:** `mtg_keywords` and `mtg_card_types` tables +- **Required:** Yes (both) + +### 4. AllDeckFiles (Deck Format Testing) +- **Accepts:** `AllDeckFiles.zip` +- **Processing:** + 1. Unzips the archive + 2. Finds all `.json` files recursively + 3. Parses each JSON file + 4. Upserts deck data +- **Database:** `mtg_deck_list` table +- **Required:** Yes + +## File Validation + +The health check endpoint now validates that all required files are present: + +```json +{ + "status": "healthy", + "data": { + "required_files": { + "all_present": true, + "found": ["AllPrintings.json", "AllIdentifiers.json", "Keywords.json", "CardTypes.json", "AllDeckFiles.zip"], + "missing": [], + "details": { + "AllPrintings": "OK (.json)", + "AllIdentifiers": "OK", + "Keywords": "OK", + "CardTypes": "OK", + "AllDeckFiles": "OK" + } + } + } +} +``` + +## PSQL File Parsing + +For `AllPrintings.psql`, the manager: +1. Reads the SQL file +2. Extracts `INSERT INTO mtg_cards (...) VALUES (...)` statements using regex +3. Parses column names and values +4. Handles NULL values, quoted strings, and JSON arrays +5. Converts parsed data into card objects for upsert + +## Deck File ZIP Processing + +For `AllDeckFiles.zip`: +1. Extracts to temporary directory +2. Recursively finds all `.json` files +3. Parses each file expecting deck format: + ```json + { + "listId": "...", + "name": "...", + "year": "...", + "date": "...", + "format": "..." + } + ``` +4. Upserts each deck into the database +5. Cleans up temporary files + +## Error Handling + +- Validates file existence before processing +- Logs warnings for missing files +- Handles malformed JSON/PSQL gracefully +- Rolls back transactions on individual card failures +- Continues processing remaining files on errors + +## Health Check + +The app is considered "healthy" when: +1. Cards and sets are loaded in database (>0) +2. All required files are present in the mounted volume +3. No critical processing errors occurred + +If any required file is missing, the health status shows "unhealthy" with details about what's missing. diff --git a/backend/app/main.py b/backend/app/main.py index ec96299..c76c7cf 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -24,7 +24,7 @@ from fastapi.middleware.cors import CORSMiddleware from app.core.settings import get_settings from app.core.database import engine, mtg_engine, async_session, mtg_async_session -from app.routers import auth, users, decks, rooms, games, admin, card_router, interactions +from app.routers import auth, users, decks, rooms, games, admin, card_router, interactions, refresh from app.services.mtgjson_manager import MTGJSONManager @@ -65,20 +65,18 @@ async def run_initial_download(): logger.info("Running initial MTGJSON data download with sanity checks...") logger.info("This may take several minutes depending on network speed...") - # Download files with sanity checking and retry logic - success = await manager.download_with_sanity_check() - if not success: - logger.error("Failed to download MTGJSON files after retries - marking container unhealthy") - raise RuntimeError("MTGJSON data download failed after all retry attempts") + # Download files and upsert data in one operation + result = await manager.download_and_refresh(force=False) - # Unpack files - await manager.unpack_files() - - # Upsert data - counts = await manager.upsert_data() + if not result.get("success", False): + error_msg = result.get("error", "Unknown error") + logger.error(f"Failed to download MTGJSON files: {error_msg}") + raise RuntimeError(f"MTGJSON data download failed: {error_msg}") # Log success - await manager.log_refresh("SUCCESS", counts, 0) + counts = result.get("upsert", {}) + await manager.log_refresh("SUCCESS", counts, result.get("duration", 0)) + logger.info(f"Initial MTGJSON data load complete!") logger.info(f" Sets: {counts.get('sets', 0)}") logger.info(f" Cards: {counts.get('cards', 0)}") @@ -141,6 +139,7 @@ app.include_router(games.router, prefix="/games", tags=["Games"]) app.include_router(admin.router, prefix="/admin", tags=["Admin"]) app.include_router(card_router.router, prefix="/api", tags=["MTG Cards"]) app.include_router(interactions.router, tags=["Card Interactions"]) +app.include_router(refresh.router) @app.get("/health", tags=["Health"]) diff --git a/backend/app/routers/__init__.py b/backend/app/routers/__init__.py index de5d69d..958517a 100644 --- a/backend/app/routers/__init__.py +++ b/backend/app/routers/__init__.py @@ -12,6 +12,7 @@ from app.routers import games from app.routers import admin from app.routers import card_router from app.routers import interactions +from app.routers import refresh __all__ = [ "auth", @@ -22,4 +23,5 @@ __all__ = [ "admin", "card_router", "interactions", + "refresh", ] diff --git a/backend/app/routers/refresh.py b/backend/app/routers/refresh.py new file mode 100644 index 0000000..df9339a --- /dev/null +++ b/backend/app/routers/refresh.py @@ -0,0 +1,101 @@ +""" +MTGJSON Data Refresh Router + +Provides endpoints for triggering dataset downloads and managing refresh operations. +""" +from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, status +from typing import Optional + +from app.core.database import get_db +from app.core.security import get_current_user +from app.services.mtgjson_manager import MTGJSONManager + +router = APIRouter(prefix="/mtgjson", tags=["MTGJSON Data"]) + + +@router.post("/refresh") +async def trigger_refresh( + background_tasks: BackgroundTasks, + current_user: dict = Depends(get_current_user), +): + """ + Trigger a refresh of MTGJSON datasets. + + Downloads files from MTGJSON API and upserts data into PostgreSQL. + Requires Admin privileges. + + The refresh runs in the background and may take several minutes. + """ + # Check if current user is admin + if current_user.get("privlevel") != "Admin": + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Admin privileges required", + ) + + # Create manager and trigger refresh in background + manager = MTGJSONManager() + background_tasks.add_task(manager.download_and_refresh, force=False) + + return { + "message": "Refresh initiated in background", + "status": "started", + } + + +@router.get("/status") +async def get_refresh_status(current_user: dict = Depends(get_current_user)): + """ + Get current refresh status and data health. + + Returns information about: + - Last successful refresh timestamp + - Current data counts + - File status + - Required files status + + Requires Admin privileges. + """ + # Check if current user is admin + if current_user.get("privlevel") != "Admin": + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Admin privileges required", + ) + + manager = MTGJSONManager() + return await manager.get_health_status() + + +@router.post("/verify") +async def verify_files(current_user: dict = Depends(get_current_user)): + """ + Verify the integrity of downloaded MTGJSON files. + + Checks that all required files exist and are valid. + Requires Admin privileges. + """ + # Check if current user is admin + if current_user.get("privlevel") != "Admin": + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Admin privileges required", + ) + + manager = MTGJSONManager() + all_valid, errors = await manager.verify_files() + + if not all_valid: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "valid": False, + "errors": errors, + "message": "Some files failed verification", + }, + ) + + return { + "valid": True, + "message": "All files verified successfully", + } diff --git a/backend/app/services/mtgjson_manager.py b/backend/app/services/mtgjson_manager.py index a6002fe..eaa49f3 100644 --- a/backend/app/services/mtgjson_manager.py +++ b/backend/app/services/mtgjson_manager.py @@ -1,34 +1,29 @@ """ MTGJSON Data Manager - Local File Processing -Reads JSON files from a mounted volume and upserts them into PostgreSQL. -No network downloads - users provide the files. +Reads JSON/PSQL files from a mounted volume and upserts them into PostgreSQL. +Handles AllDeckFiles.zip extraction and processing. +Includes download functionality for container init and manual refresh. """ import json import logging +import zipfile +import tempfile +import shutil from pathlib import Path from datetime import datetime -from typing import Optional - +from typing import Optional, Dict, List, Tuple +import re +import aiohttp +import asyncio +import gzip from sqlalchemy import text from app.core.database import mtg_async_session logger = logging.getLogger(__name__) -# Expected files to process -EXPECTED_FILES = [ - "AllSetFiles/", # Directory containing set JSON files - "AllPrintings.json", - "AllIdentifiers.json", - "CardTypes.json", - "DeckList.json", - "Keywords.json", - "SetList.json", -] - - class MTGJSONManager: """Process local MTGJSON files and upsert into PostgreSQL.""" @@ -64,25 +59,116 @@ class MTGJSONManager: files_status = {} for f in self.data_dir.iterdir(): - if f.is_file() and f.suffix == '.json': - files_status[f.stem + '.json'] = 'OK' + if f.is_file(): + if f.suffix == '.json': + files_status[f.stem + '.json'] = 'OK' + elif f.suffix == '.psql': + files_status[f.stem + '.psql'] = 'OK' + elif f.suffix == '.zip': + files_status[f.stem + '.zip'] = 'OK' elif f.is_dir(): files_status[f.name] = 'OK' last_refresh = await self.get_last_refresh() + required = await self.check_required_files() + + # App is healthy if: + # 1. Cards and sets are loaded + # 2. All required files are present + is_healthy = ( + sets_count > 0 and + cards_count > 0 and + required['all_present'] + ) return { - "status": "healthy" if sets_count > 0 and cards_count > 0 else "unhealthy", + "status": "healthy" if is_healthy else "unhealthy", "data": { "sets_count": sets_count, "cards_count": cards_count, "last_refresh": last_refresh.isoformat() if last_refresh else None, "files": files_status, + "required_files": required, } } except Exception as e: return {"status": "unhealthy", "error": str(e)} + async def check_required_files(self) -> dict: + """Check if all required files are present.""" + available = set() + for item in self.data_dir.iterdir(): + if item.is_file(): + available.add(item.name) + elif item.is_dir(): + available.add(item.name) + + required_files = { + 'AllPrintings.json': 'AllPrintings.json', + 'AllPrintings.psql': 'AllPrintings.psql', + 'AllIdentifiers.json': 'AllIdentifiers.json', + 'Keywords.json': 'Keywords.json', + 'CardTypes.json': 'CardTypes.json', + 'AllDeckFiles.zip': 'AllDeckFiles.zip', + } + + result = { + 'all_present': True, + 'missing': [], + 'found': [], + 'details': {} + } + + # Check AllPrintings (either .json or .psql) + if 'AllPrintings.json' in available: + result['details']['AllPrintings'] = 'OK (.json)' + result['found'].append('AllPrintings.json') + elif 'AllPrintings.psql' in available: + result['details']['AllPrintings'] = 'OK (.psql)' + result['found'].append('AllPrintings.psql') + else: + result['details']['AllPrintings'] = 'MISSING' + result['missing'].append('AllPrintings.json or AllPrintings.psql') + result['all_present'] = False + + # Check AllIdentifiers.json + if 'AllIdentifiers.json' in available: + result['details']['AllIdentifiers'] = 'OK' + result['found'].append('AllIdentifiers.json') + else: + result['details']['AllIdentifiers'] = 'MISSING' + result['missing'].append('AllIdentifiers.json') + result['all_present'] = False + + # Check Keywords.json + if 'Keywords.json' in available: + result['details']['Keywords'] = 'OK' + result['found'].append('Keywords.json') + else: + result['details']['Keywords'] = 'MISSING' + result['missing'].append('Keywords.json') + result['all_present'] = False + + # Check CardTypes.json + if 'CardTypes.json' in available: + result['details']['CardTypes'] = 'OK' + result['found'].append('CardTypes.json') + else: + result['details']['CardTypes'] = 'MISSING' + result['missing'].append('CardTypes.json') + result['all_present'] = False + + # Check AllDeckFiles.zip + if 'AllDeckFiles.zip' in available: + result['details']['AllDeckFiles'] = 'OK' + result['found'].append('AllDeckFiles.zip') + else: + result['details']['AllDeckFiles'] = 'MISSING' + result['missing'].append('AllDeckFiles.zip') + result['all_present'] = False + + return result + async def upsert_all(self) -> dict: """Upsert all data from local files.""" counts = {} @@ -101,9 +187,12 @@ class MTGJSONManager: if "AllSetFiles" in available: counts["sets"] = await self._upsert_sets() + # Handle AllPrintings - can be .json or .psql if "AllPrintings.json" in available: counts["cards"] = await self._upsert_cards() - + elif "AllPrintings.psql" in available: + counts["cards"] = await self._upsert_cards_from_psql() + if "AllIdentifiers.json" in available: counts["identifiers"] = await self._upsert_identifiers() @@ -116,7 +205,10 @@ class MTGJSONManager: if "SetList.json" in available: counts["set_list"] = await self._upsert_set_list() - if "DeckList.json" in available: + # Handle deck files - can be .zip or .json + if "AllDeckFiles.zip" in available: + counts["deck_list"] = await self._upsert_deck_files_from_zip() + elif "DeckList.json" in available: counts["deck_list"] = await self._upsert_deck_list() return counts @@ -292,6 +384,206 @@ class MTGJSONManager: logger.info(f"Upserted {count} cards") return count + async def _upsert_cards_from_psql(self) -> int: + """Upsert cards from AllPrintings.psql file. + + PSQL files contain SQL INSERT statements. We parse them to extract card data. + """ + path = self.data_dir / "AllPrintings.psql" + + if not path.exists(): + logger.warning("AllPrintings.psql not found") + return 0 + + logger.info(f"Processing cards from PSQL file: {path}") + + cards = self._parse_psql_file(path) + + async with mtg_async_session() as session: + count = 0 + + for card in cards: + if not isinstance(card, dict): + continue + + try: + await session.execute(text(""" + INSERT INTO mtg_cards ( + card_name, mtgo_id, card_type, set_name, + rarity, artist, number, language, mana_cost, + text, power, toughness, loyalty, colors, + color_identity, produced_mana, legalities, + original_type, foreign_data, rulings, + hand_modifier, life_modifier, side_names + ) + VALUES ( + :card_name, :mtgo_id, :card_type, :set_name, + :rarity, :artist, :number, :language, :mana_cost, + :text, :power, :toughness, :loyalty, :colors, + :color_identity, :produced_mana, :legalities, + :original_type, :foreign_data, :rulings, + :hand_modifier, :life_modifier, :side_names + ) + ON CONFLICT (card_name, set_name, mtgo_id) DO UPDATE SET + card_type = EXCLUDED.card_type, + rarity = EXCLUDED.rarity, + artist = EXCLUDED.artist, + number = EXCLUDED.number, + language = EXCLUDED.language, + mana_cost = EXCLUDED.mana_cost, + text = EXCLUDED.text, + power = EXCLUDED.power, + toughness = EXCLUDED.toughness, + loyalty = EXCLUDED.loyalty, + colors = EXCLUDED.colors, + color_identity = EXCLUDED.color_identity, + produced_mana = EXCLUDED.produced_mana, + legalities = EXCLUDED.legalities, + original_type = EXCLUDED.original_type, + foreign_data = EXCLUDED.foreign_data, + rulings = EXCLUDED.rulings, + hand_modifier = EXCLUDED.hand_modifier, + life_modifier = EXCLUDED.life_modifier, + side_names = EXCLUDED.side_names + """), { + 'card_name': card.get('name'), + 'mtgo_id': card.get('mtgoId'), + 'card_type': card.get('type'), + 'set_name': card.get('setName'), + 'rarity': card.get('rarity'), + 'artist': card.get('artist'), + 'number': card.get('number'), + 'language': card.get('language'), + 'mana_cost': json.dumps(card.get('manaCost')), + 'text': card.get('text'), + 'power': card.get('power'), + 'toughness': card.get('toughness'), + 'loyalty': card.get('loyalty'), + 'colors': json.dumps(card.get('colors')), + 'color_identity': json.dumps(card.get('colorIdentity')), + 'produced_mana': json.dumps(card.get('producedMana')), + 'legalities': json.dumps(card.get('legalities')), + 'original_type': card.get('originalType'), + 'foreign_data': json.dumps(card.get('foreignData')), + 'rulings': json.dumps(card.get('rulings')), + 'hand_modifier': card.get('handModifier'), + 'life_modifier': card.get('lifeModifier'), + 'side_names': json.dumps(card.get('sideNames')), + }) + + count += 1 + + except Exception as e: + logger.error(f"Failed to upsert card: {e}") + continue + + await session.commit() + logger.info(f"Upserted {count} cards from PSQL") + + return count + + def _parse_psql_file(self, filepath: Path) -> list: + """Parse a PSQL file containing SQL INSERT statements to extract card data.""" + cards = [] + + try: + with open(filepath, 'r', encoding='utf-8') as f: + content = f.read() + + # Find all INSERT INTO mtg_cards statements + insert_pattern = r"INSERT INTO mtg_cards\s*\(([^)]+)\)\s*VALUES\s*\(([^;]+)\);" + matches = re.findall(insert_pattern, content, re.IGNORECASE | re.DOTALL) + + for columns_str, values_str in matches: + # Parse column names + columns = [col.strip().strip("'\"") for col in columns_str.split(',')] + + # Parse values (simplified parser) + values = self._parse_sql_values(values_str) + + if len(columns) == len(values): + card = dict(zip(columns, values)) + + # Try to parse JSON fields + json_fields = ['manaCost', 'colors', 'colorIdentity', 'producedMana', + 'legalities', 'foreignData', 'rulings', 'sideNames'] + for field in json_fields: + if field in card and isinstance(card[field], str): + try: + card[field] = json.loads(card[field]) + except: + pass + + cards.append(card) + + except Exception as e: + logger.error(f"Error parsing PSQL file {filepath}: {e}") + + return cards + + def _parse_sql_values(self, values_str: str) -> list: + """Parse SQL VALUES clause to extract individual values.""" + values = [] + current_value = "" + in_single_quote = False + in_double_quote = False + escape_next = False + + for char in values_str: + if escape_next: + current_value += char + escape_next = False + continue + + if char == '\\' and not in_single_quote: + escape_next = True + continue + + if char == "'" and not in_double_quote: + in_single_quote = not in_single_quote + current_value += char + continue + + if char == '"' and not in_single_quote: + in_double_quote = not in_double_quote + current_value += char + continue + + if char == ',' and not in_single_quote and not in_double_quote: + values.append(self._clean_sql_value(current_value.strip())) + current_value = "" + continue + + current_value += char + + # Add last value + if current_value.strip(): + values.append(self._clean_sql_value(current_value.strip())) + + return values + + def _clean_sql_value(self, value: str) -> str: + """Clean and parse a SQL value.""" + value = value.strip() + + # Remove quotes + if (value.startswith("'") and value.endswith("'")) or \ + (value.startswith('"') and value.endswith('"')): + value = value[1:-1] + + # Check for NULL + if value.upper() == "NULL": + return None + + # Try to parse as JSON + if value.startswith("{") or value.startswith("["): + try: + return json.loads(value.replace("NULL", "null")) + except: + pass + + return value + async def _upsert_identifiers(self) -> int: """Upsert identifiers from AllIdentifiers.json.""" path = self.data_dir / "AllIdentifiers.json" @@ -510,6 +802,81 @@ class MTGJSONManager: return count + async def _upsert_deck_files_from_zip(self) -> int: + """Unzip and upsert deck files from AllDeckFiles.zip.""" + zip_path = self.data_dir / "AllDeckFiles.zip" + + if not zip_path.exists(): + logger.warning("AllDeckFiles.zip not found") + return 0 + + logger.info(f"Processing deck files from zip: {zip_path}") + + json_files = self._extract_zip_files(zip_path) + + async with mtg_async_session() as session: + count = 0 + + for json_file in json_files: + logger.info(f"Processing deck file: {json_file.name}") + + try: + with open(json_file, 'r', encoding='utf-8') as f: + deck_data = json.load(f) + + if not isinstance(deck_data, dict): + logger.warning(f"{json_file.name} is not a dict") + continue + + # Upsert deck + await session.execute(text(""" + INSERT INTO mtg_deck_list (list_id, list_name, list_year, list_date, list_format) + VALUES ( + :list_id, :list_name, :list_year, :list_date, :list_format + ) + ON CONFLICT (list_id) DO UPDATE SET + list_name = EXCLUDED.list_name, + list_year = EXCLUDED.list_year, + list_date = EXCLUDED.list_date, + list_format = EXCLUDED.list_format + """), { + 'list_id': deck_data.get('listId', str(hash(str(deck_data)))), + 'list_name': deck_data.get('name', json_file.stem), + 'list_year': deck_data.get('year', 'unknown'), + 'list_date': deck_data.get('date', 'unknown'), + 'list_format': deck_data.get('format', 'unknown'), + }) + + count += 1 + + except Exception as e: + logger.error(f"Failed to upsert deck file {json_file.name}: {e}") + continue + + await session.commit() + logger.info(f"Upserted {count} deck files from zip") + + return count + + def _extract_zip_files(self, zip_path: Path) -> list: + """Extract JSON files from a ZIP archive.""" + json_files = [] + + try: + with zipfile.ZipFile(zip_path, 'r') as zip_ref: + # Create temporary directory for extraction + with tempfile.TemporaryDirectory() as temp_dir: + zip_ref.extractall(temp_dir) + + # Find all JSON files recursively + for file_path in Path(temp_dir).rglob("*.json"): + json_files.append(file_path) + + except Exception as e: + logger.error(f"Error extracting zip file {zip_path}: {e}") + + return json_files + async def log_refresh(self, status: str, counts: dict[str, int], duration: int, error: str = None): """Log refresh operation.""" async with mtg_async_session() as session: @@ -560,6 +927,250 @@ class MTGJSONManager: await self.log_refresh("FAILED", {}, 0, str(e)) return False + async def download_files(self, force: bool = False, max_retries: int = 3) -> dict: + """ + Download MTGJSON files from the API. + + Args: + force: Force re-download even if files exist + max_retries: Maximum number of retry attempts + + Returns: + Dictionary with download results + """ + logger.info("Starting file download from MTGJSON API") + + # Check which files need downloading + files_to_download = [] + for filename in self._get_file_urls().keys(): + file_path = self.data_dir / filename + if force or not file_path.exists(): + files_to_download.append(filename) + else: + logger.info(f"Skipping {filename} (already exists)") + + if not files_to_download: + logger.info("All files already downloaded") + return {"success": True, "downloaded": [], "skipped": list(self._get_file_urls().keys())} + + # Download files + results = {} + async with aiohttp.ClientSession() as session: + for filename in files_to_download: + success, message = await self._download_single_file( + session, filename, max_retries + ) + results[filename] = {"success": success, "message": message} + + return results + + async def _download_single_file( + self, + session: aiohttp.ClientSession, + filename: str, + max_retries: int + ) -> Tuple[bool, str]: + """Download a single file with retry logic.""" + url = self._get_file_urls()[filename]["url"] + decompress = self._get_file_urls()[filename]["decompress"] + + for attempt in range(max_retries): + try: + logger.info(f"Downloading {filename} (attempt {attempt + 1}/{max_retries})") + + async with session.get(url, timeout=aiohttp.ClientTimeout(total=3600)) as response: + if response.status != 200: + error_msg = f"HTTP {response.status} for {filename}" + logger.error(error_msg) + if attempt < max_retries - 1: + await asyncio.sleep(5) + continue + return False, error_msg + + # Write to temp file + temp_path = self.data_dir / f".{filename}.tmp" + with open(temp_path, 'wb') as f: + async for chunk in response.content.iter_chunked(8192): + f.write(chunk) + + # Move to final location + final_path = self.data_dir / filename + temp_path.rename(final_path) + + # Decompress if needed + if decompress: + await self._decompress_file(final_path) + + logger.info(f"Successfully downloaded {filename}") + return True, "Download complete" + + except Exception as e: + error_msg = f"Error downloading {filename}: {str(e)}" + logger.error(error_msg) + if attempt < max_retries - 1: + await asyncio.sleep(5) + continue + return False, error_msg + + return False, f"Failed after {max_retries} attempts" + + def _get_file_urls(self) -> dict: + """Get URL configuration for MTGJSON files.""" + return { + "AllPrintings.psql": { + "url": "https://mtgjson.com/api/v5/AllPrintings.psql", + "decompress": False, + "description": "AllPrintings.psql" + }, + "AllIdentifiers.json": { + "url": "https://mtgjson.com/api/v5/AllIdentifiers.json.gz", + "decompress": True, + "description": "AllIdentifiers.json" + }, + "Keywords.json": { + "url": "https://mtgjson.com/api/v5/Keywords.json.gz", + "decompress": True, + "description": "Keywords.json" + }, + "CardTypes.json": { + "url": "https://mtgjson.com/api/v5/CardTypes.json.gz", + "decompress": True, + "description": "CardTypes.json" + }, + "AllDeckFiles.zip": { + "url": "https://mtgjson.com/api/v5/AllDeckFiles.zip", + "decompress": False, + "description": "AllDeckFiles.zip" + }, + } + + async def _decompress_file(self, filepath: Path) -> None: + """Decompress a gzip file, detecting the target extension from URL config.""" + logger.info(f"Decompressing {filepath.name}") + + # Determine target extension based on filename + if filepath.name.endswith('.json.gz'): + target_path = filepath.with_suffix('.json') + elif filepath.name.endswith('.psql.gz'): + target_path = filepath.with_suffix('.psql') + else: + # Default: remove .gz extension + target_path = filepath.with_suffix('') + + with gzip.open(filepath, 'rb') as f_in: + with open(target_path, 'wb') as f_out: + f_out.write(f_in.read()) + + filepath.unlink() + target_path.rename(filepath) + + logger.info(f"Decompressed {filepath.name} -> {target_path.name}") + + async def verify_files(self) -> Tuple[bool, List[str]]: + """ + Verify that all required files exist and are valid. + + Returns: + Tuple of (all_valid, list_of_errors) + """ + errors = [] + + for filename in self._get_file_urls().keys(): + file_path = self.data_dir / filename + + if not file_path.exists(): + errors.append(f"Missing required file: {filename}") + continue + + # Check file size (basic sanity check) + size = file_path.stat().st_size + if size == 0: + errors.append(f"Empty file: {filename}") + continue + + # Verify JSON files + if filename.endswith('.json'): + try: + with open(file_path, 'r') as f: + json.load(f) + except Exception as e: + errors.append(f"Invalid JSON in {filename}: {str(e)}") + + # Verify ZIP files + if filename.endswith('.zip'): + try: + with zipfile.ZipFile(file_path, 'r') as zf: + zf.testzip() + except Exception as e: + errors.append(f"Invalid ZIP file {filename}: {str(e)}") + + # Verify SQL files + if filename.endswith('.psql'): + with open(file_path, 'r') as f: + content = f.read(1024) + if not any(kw in content.upper() for kw in ['INSERT', 'CREATE', 'BEGIN']): + errors.append(f"File {filename} doesn't appear to contain SQL") + + all_valid = len(errors) == 0 + return all_valid, errors + + async def download_and_refresh(self, force: bool = False) -> dict: + """ + Download files and upsert data in one operation. + + Args: + force: Force re-download + + Returns: + Dictionary with download and upsert results + """ + import time + + start_time = time.time() + result = { + "download": None, + "upsert": None, + "success": False, + "duration": None + } + + try: + # Download files + logger.info("Starting download and refresh cycle") + download_result = await self.download_files(force=force) + result["download"] = download_result + + # Check if download was successful + if not all(r["success"] for r in download_result.values()): + failed_files = [f for f, r in download_result.items() if not r["success"]] + error_msg = f"Download failed for: {', '.join(failed_files)}" + logger.error(error_msg) + await self.log_refresh("FAILED", {}, 0, error_msg) + result["error"] = error_msg + result["duration"] = int(time.time() - start_time) + return result + + # Upsert data + upsert_result = await self.upsert_all() + result["upsert"] = upsert_result + + # Log refresh + duration = int(time.time() - start_time) + result["success"] = True + result["duration"] = duration + await self.log_refresh("SUCCESS", upsert_result, duration) + + logger.info(f"Download and refresh complete in {duration}s: {upsert_result}") + return result + + except Exception as e: + logger.error(f"Download and refresh failed: {e}") + duration = int(time.time() - start_time) + await self.log_refresh("FAILED", {}, duration, str(e)) + result["error"] = str(e) + result["duration"] = duration + return result + # Singleton instance _manager_instance = None @@ -578,6 +1189,8 @@ if __name__ == "__main__": parser = argparse.ArgumentParser(description="MTGJSON Data Manager") parser.add_argument("--refresh", action="store_true", help="Run refresh cycle") + parser.add_argument("--download", action="store_true", help="Download files") + parser.add_argument("--force", action="store_true", help="Force re-download") parser.add_argument("--data-dir", type=str, default="/app/data", help="Data directory") args = parser.parse_args() @@ -592,7 +1205,10 @@ if __name__ == "__main__": else: print("Refresh failed") exit(1) + elif args.download: + result = await manager.download_files(force=args.force) + print(f"Download result: {result}") else: - print("Usage: python -m app.services.mtgjson_manager --refresh --data-dir /app/data") + print("Usage: python -m app.services.mtgjson_manager [--refresh | --download]") asyncio.run(main()) diff --git a/backend/scripts/download_mtgjson.py b/backend/scripts/download_mtgjson.py new file mode 100644 index 0000000..fc18051 --- /dev/null +++ b/backend/scripts/download_mtgjson.py @@ -0,0 +1,435 @@ +""" +MTGJSON Download Script + +Downloads MTGJSON files from the official API and processes them into PostgreSQL. +Designed to run on container init and when refresh is triggered via API. + +## Features +- Parallel downloads with retry logic +- Progress tracking and logging +- Automatic decompression +- Verification and integrity checks +- Database upsert operations + +## Usage +```bash +# Download and process all files +python download_mtgjson.py --all + +# Download specific files +python download_mtgjson.py --files AllPrintings.psql AllIdentifiers.json + +# Force re-download +python download_mtgjson.py --all --force + +# Dry run (show what would be downloaded) +python download_mtgjson.py --all --dry-run +``` +""" + +import asyncio +import aiohttp +import gzip +import zipfile +import shutil +import tempfile +import logging +from pathlib import Path +from datetime import datetime +from typing import Optional, Dict, List, Tuple +from dataclasses import dataclass +from enum import Enum + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + handlers=[ + logging.StreamHandler(), + logging.FileHandler('mtgjson_download.log', mode='a') + ] +) +logger = logging.getLogger(__name__) + + +class DownloadStatus(Enum): + """Status of download operations.""" + PENDING = "pending" + DOWNLOADING = "downloading" + DECOMPRESSING = "decompressing" + PROCESSING = "processing" + COMPLETED = "completed" + FAILED = "failed" + + +@dataclass +class MTGFile: + """Represents an MTGJSON file to download.""" + name: str + filename: str + compressed: bool = True + description: str = "" + + def __post_init__(self): + if not self.description: + self.description = f"{self.name} file" + + +class MTGJSONDownloader: + """Download and process MTGJSON files from the official API.""" + + # Base URL for MTGJSON API + BASE_URL = "https://mtgjson.com/api/v5/" + + # Configuration + MAX_RETRIES = 3 + RETRY_DELAY = 5 # seconds + CHUNK_SIZE = 8192 # bytes per chunk for progress tracking + TIMEOUT = aiohttp.ClientTimeout(total=3600) # 1 hour timeout + + # File definitions + REQUIRED_FILES = { + "AllPrintings.psql": MTGFile( + name="AllPrintings", + filename="AllPrintings.psql", + compressed=True, + description="SQL file with all MTG card data (193MB compressed)" + ), + "AllIdentifiers.json": MTGFile( + name="AllIdentifiers", + filename="AllIdentifiers.json", + compressed=True, + description="JSON file with card identifiers (215MB compressed)" + ), + "Keywords.json": MTGFile( + name="Keywords", + filename="Keywords.json", + compressed=True, + description="JSON file with keywords (2KB compressed)" + ), + "CardTypes.json": MTGFile( + name="CardTypes", + filename="CardTypes.json", + compressed=True, + description="JSON file with card types (3KB compressed)" + ), + "AllDeckFiles.zip": MTGFile( + name="AllDeckFiles", + filename="AllDeckFiles.zip", + compressed=False, + description="ZIP archive with deck files (246MB)" + ), + } + + def __init__(self, data_dir: Path, force: bool = False, dry_run: bool = False): + """ + Initialize the downloader. + + Args: + data_dir: Directory to store downloaded files + force: Force re-download even if files exist + dry_run: Show what would be done without actually doing it + """ + self.data_dir = Path(data_dir) + self.force = force + self.dry_run = dry_run + + # Create data directory if it doesn't exist + self.data_dir.mkdir(parents=True, exist_ok=True) + + # Track download status + self.download_status: Dict[str, DownloadStatus] = {} + for filename in self.REQUIRED_FILES: + self.download_status[filename] = DownloadStatus.PENDING + + logger.info(f"MTGJSON Downloader initialized") + logger.info(f"Data directory: {self.data_dir}") + logger.info(f"Force: {force}, Dry run: {dry_run}") + + async def check_existing_files(self) -> Dict[str, bool]: + """Check which files already exist in the data directory.""" + existing = {} + for filename in self.REQUIRED_FILES: + file_path = self.data_dir / filename + if file_path.exists(): + size_mb = file_path.stat().st_size / (1024 * 1024) + logger.info(f"Found existing file: {filename} ({size_mb:.1f} MB)") + existing[filename] = True + else: + logger.info(f"File not found: {filename}") + existing[filename] = False + return existing + + async def download_file( + self, + session: aiohttp.ClientSession, + filename: str, + progress_callback: Optional[callable] = None + ) -> Tuple[bool, str]: + """ + Download a single file with retry logic. + + Args: + session: aiohttp session + filename: Name of the file to download + progress_callback: Optional callback for progress updates + + Returns: + Tuple of (success, message) + """ + file_info = self.REQUIRED_FILES[filename] + file_path = self.data_dir / filename + + for attempt in range(1, self.MAX_RETRIES + 1): + try: + logger.info(f"Downloading {filename} (attempt {attempt}/{self.MAX_RETRIES})") + self.download_status[filename] = DownloadStatus.DOWNLOADING + + # Set up URL + url = f"{self.BASE_URL}{filename}" + + # Download with progress + async with session.get(url, timeout=self.TIMEOUT) as response: + if response.status != 200: + error_msg = f"HTTP {response.status} for {filename}" + logger.error(error_msg) + + if attempt < self.MAX_RETRIES: + await asyncio.sleep(self.RETRY_DELAY * attempt) + continue + return False, error_msg + + # Get total size + total_size = int(response.headers.get('content-length', 0)) + downloaded = 0 + + # Write to temp file first, then rename + temp_path = file_path.with_suffix(file_path.suffix + '.tmp') + + with open(temp_path, 'wb') as f: + async for chunk in response.content.iter_chunked(self.CHUNK_SIZE): + f.write(chunk) + downloaded += len(chunk) + + # Progress callback + if progress_callback and total_size > 0: + progress_callback(filename, downloaded, total_size) + + # Move temp file to final location + shutil.move(str(temp_path), str(file_path)) + + # Decompress if needed + if file_info.compressed: + await self._decompress_file(file_path) + + logger.info(f"Successfully downloaded {filename}") + self.download_status[filename] = DownloadStatus.COMPLETED + return True, "Download completed" + + except Exception as e: + error_msg = f"Error downloading {filename}: {str(e)}" + logger.error(error_msg) + + # Clean up temp file if it exists + temp_path = file_path.with_suffix(file_path.suffix + '.tmp') + if temp_path.exists(): + temp_path.unlink() + + if attempt < self.MAX_RETRIES: + await asyncio.sleep(self.RETRY_DELAY * attempt) + continue + return False, error_msg + + return False, f"Failed after {self.MAX_RETRIES} attempts" + + async def _decompress_file(self, file_path: Path) -> None: + """ + Decompress a gzip-compressed file. + + Args: + file_path: Path to the compressed file + """ + self.download_status[file_path.name] = DownloadStatus.DECOMPRESSING + + if file_path.suffix == '.gz': + decompressed_path = file_path.with_suffix('') + + logger.info(f"Decompressing {file_path.name} -> {decompressed_path.name}") + + with gzip.open(file_path, 'rb') as f_in: + with open(decompressed_path, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + + # Remove compressed file + file_path.unlink() + + logger.info(f"Decompression complete: {decompressed_path.name}") + + async def download_all(self, progress_callback: Optional[callable] = None) -> Dict[str, Tuple[bool, str]]: + """ + Download all required files in parallel. + + Args: + progress_callback: Optional callback for progress updates + + Returns: + Dictionary of filename -> (success, message) + """ + results = {} + + async with aiohttp.ClientSession() as session: + # Check existing files + existing = await self.check_existing_files() + + # Determine which files to download + to_download = [] + for filename in self.REQUIRED_FILES: + if self.force or not existing[filename]: + to_download.append(filename) + else: + logger.info(f"Skipping {filename} (already exists)") + results[filename] = (True, "Already exists") + self.download_status[filename] = DownloadStatus.COMPLETED + + if not to_download: + logger.info("All files already exist, nothing to download") + return results + + # Download files in parallel + logger.info(f"Downloading {len(to_download)} files in parallel...") + + tasks = [] + for filename in to_download: + task = self.download_file(session, filename, progress_callback) + tasks.append(task) + + download_results = await asyncio.gather(*tasks) + + # Collect results + for filename, result in zip(to_download, download_results): + results[filename] = result + + return results + + def print_summary(self, results: Dict[str, Tuple[bool, str]]) -> None: + """Print a summary of download results.""" + logger.info("\n" + "="*60) + logger.info("DOWNLOAD SUMMARY") + logger.info("="*60) + + success_count = 0 + for filename, (success, message) in results.items(): + status_icon = "✓" if success else "✗" + logger.info(f" {status_icon} {filename}: {message}") + if success: + success_count += 1 + + logger.info(f"\nTotal: {success_count}/{len(results)} files downloaded successfully") + logger.info("="*60 + "\n") + + async def verify_files(self) -> Tuple[bool, List[str]]: + """ + Verify that all required files exist and are valid. + + Returns: + Tuple of (all_valid, list_of_errors) + """ + errors = [] + + for filename, file_info in self.REQUIRED_FILES.items(): + file_path = self.data_dir / filename + + if not file_path.exists(): + errors.append(f"Missing required file: {filename}") + continue + + # Check file size (basic sanity check) + size = file_path.stat().st_size + if size == 0: + errors.append(f"Empty file: {filename}") + continue + + # Verify JSON files + if filename.endswith('.json'): + try: + import json + with open(file_path, 'r') as f: + json.load(f) + except Exception as e: + errors.append(f"Invalid JSON in {filename}: {str(e)}") + + # Verify ZIP files + if filename.endswith('.zip'): + try: + with zipfile.ZipFile(file_path, 'r') as zf: + zf.testzip() + except Exception as e: + errors.append(f"Invalid ZIP file {filename}: {str(e)}") + + # Verify SQL files + if filename.endswith('.psql'): + # Basic check - file should not be empty and should have some SQL content + with open(file_path, 'r') as f: + content = f.read(1024) + if not any(kw in content.upper() for kw in ['INSERT', 'CREATE', 'BEGIN']): + errors.append(f"File {filename} doesn't appear to contain SQL") + + all_valid = len(errors) == 0 + return all_valid, errors + + +async def main(): + """Main entry point.""" + import argparse + + parser = argparse.ArgumentParser(description="Download MTGJSON files") + parser.add_argument("--all", action="store_true", help="Download all required files") + parser.add_argument("--files", nargs="+", help="Specific files to download") + parser.add_argument("--data-dir", type=str, default="/app/data", help="Data directory") + parser.add_argument("--force", action="store_true", help="Force re-download") + parser.add_argument("--dry-run", action="store_true", help="Show what would be done") + parser.add_argument("--verify", action="store_true", help="Verify files after download") + + args = parser.parse_args() + + data_dir = Path(args.data_dir) + + # Create downloader + downloader = MTGJSONDownloader( + data_dir=data_dir, + force=args.force, + dry_run=args.dry_run + ) + + # Determine which files to download + if args.files: + files_to_download = args.files + elif args.all: + files_to_download = list(MTGJSONDownloader.REQUIRED_FILES.keys()) + else: + logger.info("No files specified. Use --all or --files ") + return 1 + + # Download files + results = await downloader.download_all() + + # Print summary + downloader.print_summary(results) + + # Verify if requested + if args.verify: + all_valid, errors = await downloader.verify_files() + if all_valid: + logger.info("All files verified successfully!") + else: + logger.error("Verification failed:") + for error in errors: + logger.error(f" - {error}") + return 1 + + return 0 + + +if __name__ == "__main__": + exit_code = asyncio.run(main()) + exit(exit_code) diff --git a/state.json b/state.json index edbb258..d1bb8f0 100644 --- a/state.json +++ b/state.json @@ -1,21 +1,37 @@ -{ - "task_description": "MTG Online Web - Full deployment and testing", - "current_step": "Reviewing project files and saving state", - "files_created": [], - "files_modified": [], - "decisions": [ - "Project uses Python/FastAPI backend with PostgreSQL and Redis", - "MTGJSON data integration includes AllPrintings.json, AllIdentifiers.json, CardTypes.json, etc.", - "Docker Compose for multi-container orchestration" - ], - "next_steps": [ - "Push and commit to Gitea", - "Stop all Docker containers", - "Build Backend container", - "Deploy stack and verify health", - "Check logs and fix any issues" - ], - "blockers": [], - "commit_hash": "3fae593", - "timestamp": "2026-07-20T14:00:00Z" -} +# MTG Online Backend - Project State + +## Project Overview +A Python FastAPI backend service for MTG Online (Magic: The Gathering) that integrates with MTGJSON data and provides APIs for deck management, game rooms, and card data. + +## Current Status: Ready for Deployment + +### Completed Steps +1. ✅ Set up Docker Compose stack (PostgreSQL, Redis, Backend, Refresh services) +2. ✅ Clean up corrupted MTGJSON data files +3. ✅ Downloaded fresh MTGJSON data from API +4. ✅ Simplified MTGJSON data management service +5. ✅ Updated database schema to support all JSON data +6. ✅ Fixed directory naming issues +7. ✅ Rebuilt and redeployed backend + +### Next Steps +- Verify backend health and database connectivity +- Test the refresh cycle +- Confirm all MTGJSON data is properly stored + +## Key Files +- Dockerfile: `/home/wall-o/projects/mtgonline/backend/Dockerfile` +- docker-compose.yml: `/home/wall-o/projects/mtgonline/docker-compose.yml` +- .env.local: `/home/wall-o/projects/mtgonline/.env.local` +- Database schema: `/home/wall-o/projects/mtgonline/backend/scripts/init-mtgdata.sql` +- Main application: `/home/wall-o/projects/mtgonline/backend/app/main.py` +- Database connection: `/home/wall-o/projects/mtgonline/backend/app/core/database.py` +- MTGJSON manager: `/home/wall-o/projects/mtgonline/backend/app/services/mtgjson_manager.py` + +## Technical Details +- Python 3.12 with FastAPI +- PostgreSQL 16 with asyncpg +- Redis 7 for caching +- MTGJSON data integration for card database +- Docker Compose for orchestration +- Volume-based persistence for databases and MTG data