diff --git a/backend/scripts/load_mtgdata.py b/backend/scripts/load_mtgdata.py new file mode 100644 index 0000000..063b986 --- /dev/null +++ b/backend/scripts/load_mtgdata.py @@ -0,0 +1,333 @@ +#!/usr/bin/env python3 +""" +MTGJSON to PostgreSQL Data Loader + +Converts MTGJSON v5 AllPrintings.json to PostgreSQL format +and upserts data into the mtgdata database. + +Uses synchronous psycopg2 for reliable Docker networking. +""" +import json +import logging +import sys +from datetime import datetime +from sqlalchemy import create_engine, text +from sqlalchemy.orm import sessionmaker + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + handlers=[logging.StreamHandler(sys.stdout)] +) +logger = logging.getLogger(__name__) + + +class MTGJSONDataLoader: + """Load MTGJSON data into PostgreSQL database using synchronous psycopg2.""" + + def __init__(self, db_url: str, json_path: str): + self.db_url = db_url + self.json_path = json_path + self.engine = None + self.session = None + + def connect(self): + """Establish database connection using synchronous psycopg2.""" + self.engine = create_engine(self.db_url) + self.session = sessionmaker(bind=self.engine) + logger.info("✓ Connected to database") + + def close(self): + """Close database connection.""" + if self.session: + self.session.close() + if self.engine: + self.engine.dispose() + logger.info("✓ Database connection closed") + + def load_and_upsert(self): + """Load MTGJSON data and upsert into database using single session.""" + logger.info(f"Loading MTGJSON from: {self.json_path}") + + # Load the JSON file + with open(self.json_path, 'r') as f: + mtg_data = json.load(f) + + data = mtg_data['data'] + total_sets = len(data) + logger.info(f"Loaded {total_sets} sets from MTGJSON") + + # Use single session for all operations + session = self.session() + try: + # Process each set + total_cards = 0 + processed_sets = 0 + + for set_code, set_data in data.items(): + processed_sets += 1 + logger.info(f"\nProcessing set: {set_code} ({processed_sets}/{total_sets})") + + # Upsert set data (returns set_id) + set_id = self.upsert_set(session, set_data) + + # Upsert cards in the set + if 'cards' in set_data: + cards_in_set = len(set_data['cards']) + total_cards += cards_in_set + + # Process cards in batches + batch_size = 100 + for i in range(0, cards_in_set, batch_size): + batch = set_data['cards'][i:i + batch_size] + for card_data in batch: + self.upsert_card(session, card_data, set_id) + + # Commit batch + session.commit() + logger.info(f" Progress: {i + batch_size}/{cards_in_set} cards") + + logger.info(f"✓ Processed {cards_in_set} cards in set {set_code}") + + # Final commit + session.commit() + + logger.info(f"\n{'='*60}") + logger.info(f"Data loading complete!") + logger.info(f" Sets processed: {processed_sets}") + logger.info(f" Total cards loaded: {total_cards:,}") + logger.info(f"{'='*60}") + except Exception: + session.rollback() + raise + finally: + session.close() + + def upsert_set(self, session, set_data: dict) -> int: + """Upsert a set into the database. Returns set_id.""" + try: + # Extract set data + code = set_data.get('code', '').upper() + name = set_data.get('name') + set_type = set_data.get('type') + release_date = set_data.get('releaseDate') + base_set_size = set_data.get('baseSetSize') + total_size = set_data.get('totalSize') + is_foil_only = set_data.get('isFoilOnly') + is_non_foil_only = set_data.get('isNonFoilOnly') + digital = set_data.get('digital') + icon_svg_url = set_data.get('iconSvgUri') + parent_code = set_data.get('parentCode') + mtgo_code = set_data.get('mtgoCode') + + # Try to parse release date + if release_date and isinstance(release_date, str): + try: + release_date = datetime.strptime(release_date, '%Y-%m-%dT%H:%M:%S.%fZ').date() + except: + release_date = None + + # Check if set exists + result = session.execute( + text("SELECT id FROM mtg_sets WHERE code = :code"), + {"code": code} + ) + + existing = result.fetchone() + + if existing: + # Update existing set + set_id = existing[0] + session.execute( + text(""" + UPDATE mtg_sets + SET name = :name, type = :type, release_date = :release_date, + base_set_size = :base_set_size, total_size = :total_size, + is_foil_only = :is_foil_only, is_non_foil_only = :is_non_foil_only, + digital = :digital, icon_svg_url = :icon_svg_url, + parent_code = :parent_code, mtgo_code = :mtgo_code, + updated_at = CURRENT_TIMESTAMP + WHERE id = :id + """), + { + "name": name, + "type": set_type, + "release_date": release_date, + "base_set_size": base_set_size, + "total_size": total_size, + "is_foil_only": is_foil_only, + "is_non_foil_only": is_non_foil_only, + "digital": digital, + "icon_svg_url": icon_svg_url, + "parent_code": parent_code, + "mtgo_code": mtgo_code, + "id": set_id + } + ) + logger.debug(f" Updated set: {code}") + else: + # Insert new set + result = session.execute( + text(""" + INSERT INTO mtg_sets + (code, name, type, release_date, base_set_size, total_size, + is_foil_only, is_non_foil_only, digital, icon_svg_url, + parent_code, mtgo_code) + VALUES (:code, :name, :type, :release_date, :base_set_size, :total_size, + :is_foil_only, :is_non_foil_only, :digital, :icon_svg_url, + :parent_code, :mtgo_code) + RETURNING id + """), + { + "code": code, + "name": name, + "type": set_type, + "release_date": release_date, + "base_set_size": base_set_size, + "total_size": total_size, + "is_foil_only": is_foil_only, + "is_non_foil_only": is_non_foil_only, + "digital": digital, + "icon_svg_url": icon_svg_url, + "parent_code": parent_code, + "mtgo_code": mtgo_code + } + ) + set_id = result.fetchone()[0] + logger.debug(f" Created set: {code}") + + # Commit the set insert + session.commit() + return set_id + except Exception: + session.rollback() + raise + + def upsert_card(self, session, card_data: dict, set_id: int): + """Upsert a card into the database.""" + try: + # Extract card data + name = card_data.get('name') + mana_cost = card_data.get('manaCost') + type_line = card_data.get('typeLine') + oracle_text = card_data.get('oracleText') + power = card_data.get('power') + toughness = card_data.get('toughness') + rarity = card_data.get('rarity') + layout = card_data.get('layout') + artist = card_data.get('artist') + flavor_text = card_data.get('flavorText') + numbers = card_data.get('numbers') + identifiers = card_data.get('identifiers') + images = card_data.get('images') + + # Serialize complex fields to JSON + if isinstance(identifiers, dict): + identifiers = json.dumps(identifiers) + elif identifiers: + identifiers = str(identifiers) + + if isinstance(images, dict): + images = json.dumps(images) + elif images: + images = str(images) + + # Check if card exists (by name + set_id) + result = session.execute( + text(""" + SELECT id FROM mtg_cards + WHERE name = :name AND set_id = :set_id + """), + {"name": name, "set_id": set_id} + ) + + existing = result.fetchone() + + if existing: + # Update existing card + card_id = existing[0] + session.execute( + text(""" + UPDATE mtg_cards + SET mana_cost = :mana_cost, type_line = :type_line, + oracle_text = :oracle_text, power = :power, + toughness = :toughness, rarity = :rarity, + layout = :layout, artist = :artist, + flavor_text = :flavor_text, numbers = :numbers, + identifiers = :identifiers, images = :images, + updated_at = CURRENT_TIMESTAMP + WHERE id = :id + """), + { + "mana_cost": mana_cost, + "type_line": type_line, + "oracle_text": oracle_text, + "power": power, + "toughness": toughness, + "rarity": rarity, + "layout": layout, + "artist": artist, + "flavor_text": flavor_text, + "numbers": numbers, + "identifiers": identifiers, + "images": images, + "id": card_id + } + ) + else: + # Insert new card + session.execute( + text(""" + INSERT INTO mtg_cards + (set_id, name, mana_cost, type_line, oracle_text, + power, toughness, rarity, layout, artist, + flavor_text, numbers, identifiers, images) + VALUES (:set_id, :name, :mana_cost, :type_line, :oracle_text, + :power, :toughness, :rarity, :layout, :artist, + :flavor_text, :numbers, :identifiers, :images) + """), + { + "set_id": set_id, + "name": name, + "mana_cost": mana_cost, + "type_line": type_line, + "oracle_text": oracle_text, + "power": power, + "toughness": toughness, + "rarity": rarity, + "layout": layout, + "artist": artist, + "flavor_text": flavor_text, + "numbers": numbers, + "identifiers": identifiers, + "images": images + } + ) + except Exception: + session.rollback() + raise + + +def main(): + """Main entry point.""" + # Database URL - use psycopg2 for synchronous connection + db_url = "postgresql+psycopg2://mtgonline:mtgonline_pass@172.18.0.2:5432/mtgdata" + + # JSON file path + json_path = "/app/data/AllPrintings.json" + + loader = MTGJSONDataLoader(db_url, json_path) + + try: + loader.connect() + loader.load_and_upsert() + except Exception as e: + logger.error(f"Error loading data: {e}", exc_info=True) + sys.exit(1) + finally: + loader.close() + + +if __name__ == "__main__": + main()