feat: MTGJSON data manager service with download, unpack, and upsert
- Created MTGJSONManager service for complete data lifecycle - Handles download, unpack (gzip/zip), and PostgreSQL upsert - ON CONFLICT DO UPDATE preserves existing data - Startup triggers initial download on first container init - Health check verifies MTG data exists in database - Weekly refresh via MTG_REFRESH_INTERVAL_DAYS setting - Updated docker-compose start_period to 600s for download time
This commit is contained in:
@@ -1,269 +1,86 @@
|
||||
"""
|
||||
MTGJSON Database Refresh Script
|
||||
|
||||
Downloads and updates the MTGJSON All Printings dataset weekly.
|
||||
Downloads and updates the MTGJSON datasets weekly.
|
||||
Uses the MTGJSONManager service for all data operations.
|
||||
|
||||
Usage:
|
||||
python -m app.scripts.refresh_mtg
|
||||
python -m app.scripts.refresh_mtg --force
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import aiohttp
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from app.core.settings import get_settings
|
||||
from app.services.mtgjson_manager import MTGJSONManager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MTGJSON_API_URL = "https://mtgjson.com/api/v5/AllPrintings.json"
|
||||
DATA_DIR = Path("/app/data")
|
||||
REFRESH_INTERVAL_DAYS = 7
|
||||
|
||||
|
||||
async def download_mtgjson(session: aiohttp.ClientSession, output_path: Path) -> bool:
|
||||
"""Download the latest AllPrintings dataset."""
|
||||
try:
|
||||
logger.info(f"Downloading MTGJSON from {MTGJSON_API_URL}...")
|
||||
async with session.get(MTGJSON_API_URL) as response:
|
||||
if response.status != 200:
|
||||
logger.error(f"Failed to download: {response.status}")
|
||||
return False
|
||||
|
||||
with open(output_path, 'wb') as f:
|
||||
async for chunk in response.content.iter_chunked(8192):
|
||||
f.write(chunk)
|
||||
|
||||
logger.info(f"Downloaded to {output_path}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Download error: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def parse_mtgjson(filepath: Path) -> tuple[dict, dict]:
|
||||
"""Parse the AllPrintings JSON file.
|
||||
|
||||
MTGJSON v5 AllPrintings structure:
|
||||
{
|
||||
"meta": {...},
|
||||
"data": {
|
||||
"10E": {"baseSetSize": 383, "block": "Core Set", "cards": [...]},
|
||||
"UNH": {...}
|
||||
}
|
||||
}
|
||||
|
||||
Returns:
|
||||
(sets_dict, cards_dict) where sets_dict maps set_code -> set_data
|
||||
and cards_dict maps set_code -> list of card dicts
|
||||
"""
|
||||
try:
|
||||
with open(filepath, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
|
||||
# Verify structure - data key is required
|
||||
if 'data' not in data:
|
||||
raise ValueError("Invalid MTGJSON structure: missing 'data' key")
|
||||
|
||||
mtg_data = data['data']
|
||||
|
||||
# MTGJSON v5: data contains set codes directly as keys
|
||||
# Each set code maps to {baseSetSize, block, cards: [...]}
|
||||
sets_dict = {}
|
||||
cards_dict = {}
|
||||
|
||||
for set_code, set_data in mtg_data.items():
|
||||
# Skip if it looks like metadata, not a set
|
||||
if isinstance(set_data, dict) and 'baseSetSize' in set_data:
|
||||
# Convert release_date string to datetime object
|
||||
release_date_str = set_data.get('releaseDate')
|
||||
if release_date_str:
|
||||
try:
|
||||
set_data['releaseDate'] = datetime.fromisoformat(release_date_str.replace('Z', '+00:00'))
|
||||
except (ValueError, AttributeError):
|
||||
pass # Keep as string if parsing fails
|
||||
|
||||
sets_dict[set_code] = set_data
|
||||
# Extract cards for this set
|
||||
if 'cards' in set_data and isinstance(set_data['cards'], list):
|
||||
cards_dict[set_code] = set_data['cards']
|
||||
|
||||
if not sets_dict:
|
||||
raise ValueError("No sets found in MTGJSON data")
|
||||
|
||||
logger.info(f"Parsed {len(sets_dict)} sets, {sum(len(c) for c in cards_dict.values())} cards")
|
||||
return sets_dict, cards_dict
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Parse error: {e}")
|
||||
return {}, {}
|
||||
|
||||
|
||||
async def update_database(session: AsyncSession, sets_dict: dict, cards_dict: dict) -> tuple[int, int]:
|
||||
"""Update the database with parsed MTGJSON data."""
|
||||
cards_updated = 0
|
||||
sets_updated = 0
|
||||
|
||||
try:
|
||||
# Process sets
|
||||
for set_code, set_data in sets_dict.items():
|
||||
stmt = text("""
|
||||
INSERT INTO mtg_sets (code, name, type, release_date, base_set_size,
|
||||
total_size, is_foil_only, is_non_foil_only,
|
||||
digital, icon_svg_url, parent_code, mtgo_code)
|
||||
VALUES (:code, :name, :type, :release_date, :base_set_size,
|
||||
:total_size, :is_foil_only, :is_non_foil_only,
|
||||
:digital, :icon_svg_url, :parent_code, :mtgo_code)
|
||||
ON CONFLICT (code) DO UPDATE SET
|
||||
name = EXCLUDED.name,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""")
|
||||
|
||||
await session.execute(stmt, {
|
||||
'code': set_code,
|
||||
'name': set_data.get('name'),
|
||||
'type': set_data.get('type'),
|
||||
'release_date': set_data.get('releaseDate'),
|
||||
'base_set_size': set_data.get('baseSetSize'),
|
||||
'total_size': set_data.get('totalSize'),
|
||||
'is_foil_only': set_data.get('isFoilOnly'),
|
||||
'is_non_foil_only': set_data.get('isNonFoilOnly'),
|
||||
'digital': set_data.get('digital'),
|
||||
'icon_svg_url': set_data.get('iconSvgUri'),
|
||||
'parent_code': set_data.get('parentCode'),
|
||||
'mtgo_code': set_data.get('mtgoCode'),
|
||||
})
|
||||
sets_updated += 1
|
||||
|
||||
# Process cards grouped by set
|
||||
for set_code, cards in cards_dict.items():
|
||||
for card_data in cards:
|
||||
stmt = text("""
|
||||
INSERT INTO mtg_cards (set_id, name, mana_cost, type_line, oracle_text,
|
||||
power, toughness, rarity, layout, artist,
|
||||
flavor_text, numbers, identifiers, images)
|
||||
SELECT s.id, :name, :mana_cost, :type_line, :oracle_text,
|
||||
:power, :toughness, :rarity, :layout, :artist,
|
||||
:flavor_text, :numbers, :identifiers, :images
|
||||
FROM mtg_sets s
|
||||
WHERE s.code = :set_code
|
||||
ON CONFLICT DO NOTHING
|
||||
""")
|
||||
|
||||
await session.execute(stmt, {
|
||||
'set_code': set_code,
|
||||
'name': card_data.get('name'),
|
||||
'mana_cost': card_data.get('manaCost'),
|
||||
'type_line': card_data.get('type'),
|
||||
'oracle_text': card_data.get('text'),
|
||||
'power': card_data.get('power'),
|
||||
'toughness': card_data.get('toughness'),
|
||||
'rarity': card_data.get('rarity'),
|
||||
'layout': card_data.get('layout'),
|
||||
'artist': card_data.get('artist'),
|
||||
'flavor_text': card_data.get('flavorText'),
|
||||
'numbers': str(card_data.get('numbers', '')),
|
||||
'identifiers': json.dumps(card_data.get('identifiers', {})),
|
||||
'images': json.dumps(card_data.get('images', {})),
|
||||
})
|
||||
cards_updated += 1
|
||||
|
||||
await session.commit()
|
||||
return cards_updated, sets_updated
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Database update error: {e}")
|
||||
await session.rollback()
|
||||
raise
|
||||
|
||||
|
||||
async def check_last_refresh(engine: create_async_engine) -> datetime:
|
||||
"""Check when the last refresh occurred."""
|
||||
async with AsyncSession(engine) as session:
|
||||
stmt = text("SELECT refresh_date FROM mtg_refresh_log ORDER BY refresh_date DESC LIMIT 1")
|
||||
result = await session.execute(stmt)
|
||||
row = result.fetchone()
|
||||
|
||||
if row:
|
||||
return row[0]
|
||||
return datetime.min
|
||||
|
||||
|
||||
async def log_refresh(engine: create_async_engine, status: str, cards: int, sets: int,
|
||||
duration: int, error: str = None):
|
||||
"""Log the refresh operation."""
|
||||
async with AsyncSession(engine) as session:
|
||||
stmt = text("""
|
||||
INSERT INTO mtg_refresh_log (status, cards_updated, sets_updated,
|
||||
error_message, duration_seconds)
|
||||
VALUES (:status, :cards, :sets, :error, :duration)
|
||||
""")
|
||||
await session.execute(stmt, {
|
||||
'status': status,
|
||||
'cards': cards,
|
||||
'sets': sets,
|
||||
'error': error,
|
||||
'duration': duration,
|
||||
})
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def main():
|
||||
"""Main refresh logic."""
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
async def run_refresh(force: bool = False):
|
||||
"""Run the refresh cycle."""
|
||||
settings = get_settings()
|
||||
|
||||
DATA_DIR = Path(settings.DATA_DIR)
|
||||
REFRESH_INTERVAL_DAYS = settings.MTG_REFRESH_INTERVAL_DAYS
|
||||
|
||||
# Use database URL from settings
|
||||
engine = create_async_engine(settings.MTG_DATABASE_URL)
|
||||
|
||||
last_refresh = await check_last_refresh(engine)
|
||||
refresh_needed = (datetime.now() - last_refresh).days >= REFRESH_INTERVAL_DAYS
|
||||
|
||||
if not refresh_needed:
|
||||
logger.info("Refresh not needed. Last refresh was within interval.")
|
||||
return
|
||||
manager = MTGJSONManager(settings.DATA_DIR)
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
# Download dataset
|
||||
download_path = DATA_DIR / "AllPrintings.json"
|
||||
success = await download_mtgjson(session, download_path)
|
||||
|
||||
# Check if refresh is needed
|
||||
last_refresh = await manager.get_last_refresh()
|
||||
|
||||
if not force and not manager.is_refresh_needed(last_refresh, settings.MTG_REFRESH_INTERVAL_DAYS):
|
||||
logger.info("Refresh not needed. Last refresh was within interval.")
|
||||
return
|
||||
|
||||
logger.info("Starting MTGJSON refresh cycle...")
|
||||
|
||||
# Download files
|
||||
if force or last_refresh is None:
|
||||
logger.info("Downloading MTGJSON files...")
|
||||
success = await manager.download_files()
|
||||
if not success:
|
||||
await log_refresh(engine, "FAILED", 0, 0, 0, "Download failed")
|
||||
logger.error("Failed to download files")
|
||||
await manager.log_refresh("FAILED_DOWNLOAD", {}, 0, "Download failed")
|
||||
return
|
||||
|
||||
# Parse data (returns sets_dict, cards_dict)
|
||||
sets_dict, cards_dict = await parse_mtgjson(download_path)
|
||||
if not sets_dict:
|
||||
await log_refresh(engine, "FAILED", 0, 0, 0, "Parse failed")
|
||||
return
|
||||
|
||||
# Update database
|
||||
async with AsyncSession(engine) as db_session:
|
||||
cards_updated, sets_updated = await update_database(db_session, sets_dict, cards_dict)
|
||||
|
||||
# Log success
|
||||
duration = int(time.time() - start_time)
|
||||
await log_refresh(engine, "SUCCESS", cards_updated, sets_updated, duration)
|
||||
logger.info(f"Refresh completed: {cards_updated} cards, {sets_updated} sets in {duration}s")
|
||||
|
||||
|
||||
# Unpack files
|
||||
logger.info("Unpacking MTGJSON files...")
|
||||
await manager.unpack_files()
|
||||
|
||||
# Upsert data
|
||||
logger.info("Upserting data into database...")
|
||||
counts = await manager.upsert_data()
|
||||
|
||||
# Log success
|
||||
duration = int(time.time() - start_time)
|
||||
await manager.log_refresh("SUCCESS", counts, duration)
|
||||
logger.info(f"Refresh completed successfully in {duration}s")
|
||||
logger.info(f" Sets: {counts.get('sets', 0)}")
|
||||
logger.info(f" Cards: {counts.get('cards', 0)}")
|
||||
|
||||
except Exception as e:
|
||||
duration = int(time.time() - start_time)
|
||||
await log_refresh(engine, "FAILED", 0, 0, duration, str(e))
|
||||
await manager.log_refresh("FAILED", {}, duration, str(e))
|
||||
logger.error(f"Refresh failed: {e}")
|
||||
raise
|
||||
|
||||
|
||||
async def main():
|
||||
"""Main entry point."""
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
await engine.dispose()
|
||||
parser = argparse.ArgumentParser(description="MTGJSON Refresh Script")
|
||||
parser.add_argument("--force", action="store_true", help="Force refresh even if not needed")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
await run_refresh(force=args.force)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user