""" MTGJSON Database Refresh Script Downloads and updates the MTGJSON All Printings dataset weekly. """ import asyncio import json import logging import os import time from datetime import datetime from pathlib import Path import aiohttp from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession from sqlalchemy import text from sqlalchemy.orm import sessionmaker from app.core.settings import get_settings logger = logging.getLogger(__name__) MTGJSON_API_URL = "https://mtgjson.com/api/v5/AllPrintings.json" DATA_DIR = Path("/app/data") REFRESH_INTERVAL_DAYS = 7 async def download_mtgjson(session: aiohttp.ClientSession, output_path: Path) -> bool: """Download the latest AllPrintings dataset.""" try: logger.info(f"Downloading MTGJSON from {MTGJSON_API_URL}...") async with session.get(MTGJSON_API_URL) as response: if response.status != 200: logger.error(f"Failed to download: {response.status}") return False with open(output_path, 'wb') as f: async for chunk in response.content.iter_chunked(8192): f.write(chunk) logger.info(f"Downloaded to {output_path}") return True except Exception as e: logger.error(f"Download error: {e}") return False async def parse_mtgjson(filepath: Path) -> dict: """Parse the AllPrintings JSON file.""" try: with open(filepath, 'r', encoding='utf-8') as f: data = json.load(f) # Verify structure if 'data' not in data or 'sets' not in data: raise ValueError("Invalid MTGJSON structure") return data['data'] except Exception as e: logger.error(f"Parse error: {e}") return {} async def update_database(session: AsyncSession, data: dict) -> tuple[int, int]: """Update the database with parsed MTGJSON data.""" cards_updated = 0 sets_updated = 0 try: # Process sets for set_code, set_data in data.get('sets', {}).items(): stmt = text(""" INSERT INTO mtg_sets (code, name, type, release_date, base_set_size, total_size, is_foil_only, is_non_foil_only, digital, icon_svg_url, parent_code, mtgo_code) VALUES (:code, :name, :type, :release_date, :base_set_size, :total_size, :is_foil_only, :is_non_foil_only, :digital, :icon_svg_url, :parent_code, :mtgo_code) ON CONFLICT (code) DO UPDATE SET name = EXCLUDED.name, updated_at = CURRENT_TIMESTAMP """) await session.execute(stmt, { 'code': set_code, 'name': set_data.get('name'), 'type': set_data.get('type'), 'release_date': set_data.get('releaseDate'), 'base_set_size': set_data.get('baseSetSize'), 'total_size': set_data.get('totalSize'), 'is_foil_only': set_data.get('isFoilOnly'), 'is_non_foil_only': set_data.get('isNonFoilOnly'), 'digital': set_data.get('digital'), 'icon_svg_url': set_data.get('iconSvgUri'), 'parent_code': set_data.get('parentCode'), 'mtgo_code': set_data.get('mtgoCode'), }) sets_updated += 1 # Process cards for card_data in data.get('cards', []): stmt = text(""" INSERT INTO mtg_cards (set_id, name, mana_cost, type_line, oracle_text, power, toughness, rarity, layout, artist, flavor_text, numbers, identifiers, images) SELECT s.id, :name, :mana_cost, :type_line, :oracle_text, :power, :toughness, :rarity, :layout, :artist, :flavor_text, :numbers, :identifiers, :images FROM mtg_sets s WHERE s.code = :set_code ON CONFLICT DO NOTHING """) await session.execute(stmt, { 'set_code': card_data.get('set'), 'name': card_data.get('name'), 'mana_cost': card_data.get('manaCost'), 'type_line': card_data.get('type'), 'oracle_text': card_data.get('text'), 'power': card_data.get('power'), 'toughness': card_data.get('toughness'), 'rarity': card_data.get('rarity'), 'layout': card_data.get('layout'), 'artist': card_data.get('artist'), 'flavor_text': card_data.get('flavorText'), 'numbers': str(card_data.get('numbers', '')), 'identifiers': json.dumps(card_data.get('identifiers', {})), 'images': json.dumps(card_data.get('images', {})), }) cards_updated += 1 await session.commit() return cards_updated, sets_updated except Exception as e: logger.error(f"Database update error: {e}") await session.rollback() raise async def check_last_refresh(engine: create_async_engine) -> datetime: """Check when the last refresh occurred.""" async with AsyncSession(engine) as session: stmt = text("SELECT refresh_date FROM mtg_refresh_log ORDER BY refresh_date DESC LIMIT 1") result = await session.execute(stmt) row = result.fetchone() if row: return row[0] return datetime.min async def log_refresh(engine: create_async_engine, status: str, cards: int, sets: int, duration: int, error: str = None): """Log the refresh operation.""" async with AsyncSession(engine) as session: stmt = text(""" INSERT INTO mtg_refresh_log (status, cards_updated, sets_updated, error_message, duration_seconds) VALUES (:status, :cards, :sets, :error, :duration) """) await session.execute(stmt, { 'status': status, 'cards': cards, 'sets': sets, 'error': error, 'duration': duration, }) await session.commit() async def main(): """Main refresh logic.""" logging.basicConfig(level=logging.INFO) settings = get_settings() DATA_DIR = Path(settings.DATA_DIR) REFRESH_INTERVAL_DAYS = settings.MTG_REFRESH_INTERVAL_DAYS # Use database URL from settings engine = create_async_engine(settings.MTG_DATABASE_URL) last_refresh = await check_last_refresh(engine) refresh_needed = (datetime.now() - last_refresh).days >= REFRESH_INTERVAL_DAYS if not refresh_needed: logger.info("Refresh not needed. Last refresh was within interval.") return start_time = time.time() try: async with aiohttp.ClientSession() as session: # Download dataset download_path = DATA_DIR / "AllPrintings.json" success = await download_mtgjson(session, download_path) if not success: await log_refresh(engine, "FAILED", 0, 0, 0, "Download failed") return # Parse data data = await parse_mtgjson(download_path) if not data: await log_refresh(engine, "FAILED", 0, 0, 0, "Parse failed") return # Update database async with AsyncSession(engine) as db_session: cards_updated, sets_updated = await update_database(db_session, data) # Log success duration = int(time.time() - start_time) await log_refresh(engine, "SUCCESS", cards_updated, sets_updated, duration) logger.info(f"Refresh completed: {cards_updated} cards, {sets_updated} sets in {duration}s") except Exception as e: duration = int(time.time() - start_time) await log_refresh(engine, "FAILED", 0, 0, duration, str(e)) logger.error(f"Refresh failed: {e}") await engine.dispose() if __name__ == "__main__": asyncio.run(main())