mtgonline/backend/scripts/load_mtgdata.py

#!/usr/bin/env python3
"""
MTGJSON to PostgreSQL Data Loader

Converts MTGJSON v5 AllPrintings.json to PostgreSQL format
and upserts data into the mtgdata database.

Uses synchronous psycopg2 for reliable Docker networking.
"""
import json
import logging
import sys
from datetime import datetime
from sqlalchemy import create_engine, text
from sqlalchemy.orm import sessionmaker

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s',
    handlers=[logging.StreamHandler(sys.stdout)]
)
logger = logging.getLogger(__name__)


class MTGJSONDataLoader:
    """Load MTGJSON data into PostgreSQL database using synchronous psycopg2."""
    
    def __init__(self, db_url: str, json_path: str):
        self.db_url = db_url
        self.json_path = json_path
        self.engine = None
        self.session = None
        
    def connect(self):
        """Establish database connection using synchronous psycopg2."""
        self.engine = create_engine(self.db_url)
        self.session = sessionmaker(bind=self.engine)
        logger.info("✓ Connected to database")
        
    def close(self):
        """Close database connection."""
        if self.session:
            self.session.close()
        if self.engine:
            self.engine.dispose()
            logger.info("✓ Database connection closed")
    
    def load_and_upsert(self):
        """Load MTGJSON data and upsert into database using single session."""
        logger.info(f"Loading MTGJSON from: {self.json_path}")
        
        # Load the JSON file
        with open(self.json_path, 'r') as f:
            mtg_data = json.load(f)
        
        data = mtg_data['data']
        total_sets = len(data)
        logger.info(f"Loaded {total_sets} sets from MTGJSON")
        
        # Use single session for all operations
        session = self.session()
        try:
            # Process each set
            total_cards = 0
            processed_sets = 0
            
            for set_code, set_data in data.items():
                processed_sets += 1
                logger.info(f"\nProcessing set: {set_code} ({processed_sets}/{total_sets})")
                
                # Upsert set data (returns set_id)
                set_id = self.upsert_set(session, set_data)
                
                # Upsert cards in the set
                if 'cards' in set_data:
                    cards_in_set = len(set_data['cards'])
                    total_cards += cards_in_set
                    
                    # Process cards in batches
                    batch_size = 100
                    for i in range(0, cards_in_set, batch_size):
                        batch = set_data['cards'][i:i + batch_size]
                        for card_data in batch:
                            self.upsert_card(session, card_data, set_id)
                        
                        # Commit batch
                        session.commit()
                        logger.info(f"  Progress: {i + batch_size}/{cards_in_set} cards")
                    
                    logger.info(f"✓ Processed {cards_in_set} cards in set {set_code}")
            
            # Final commit
            session.commit()
            
            logger.info(f"\n{'='*60}")
            logger.info(f"Data loading complete!")
            logger.info(f"  Sets processed: {processed_sets}")
            logger.info(f"  Total cards loaded: {total_cards:,}")
            logger.info(f"{'='*60}")
        except Exception:
            session.rollback()
            raise
        finally:
            session.close()
    
    def upsert_set(self, session, set_data: dict) -> int:
        """Upsert a set into the database. Returns set_id."""
        try:
            # Extract set data
            code = set_data.get('code', '').upper()
            name = set_data.get('name')
            set_type = set_data.get('type')
            release_date = set_data.get('releaseDate')
            base_set_size = set_data.get('baseSetSize')
            total_size = set_data.get('totalSize')
            is_foil_only = set_data.get('isFoilOnly')
            is_non_foil_only = set_data.get('isNonFoilOnly')
            digital = set_data.get('digital')
            icon_svg_url = set_data.get('iconSvgUri')
            parent_code = set_data.get('parentCode')
            mtgo_code = set_data.get('mtgoCode')
            
            # Try to parse release date
            if release_date and isinstance(release_date, str):
                try:
                    release_date = datetime.strptime(release_date, '%Y-%m-%dT%H:%M:%S.%fZ').date()
                except:
                    release_date = None
            
            # Check if set exists
            result = session.execute(
                text("SELECT id FROM mtg_sets WHERE code = :code"),
                {"code": code}
            )
            
            existing = result.fetchone()
            
            if existing:
                # Update existing set
                set_id = existing[0]
                session.execute(
                    text("""
                        UPDATE mtg_sets 
                        SET name = :name, type = :type, release_date = :release_date,
                            base_set_size = :base_set_size, total_size = :total_size,
                            is_foil_only = :is_foil_only, is_non_foil_only = :is_non_foil_only,
                            digital = :digital, icon_svg_url = :icon_svg_url,
                            parent_code = :parent_code, mtgo_code = :mtgo_code,
                            updated_at = CURRENT_TIMESTAMP
                        WHERE id = :id
                    """),
                    {
                        "name": name,
                        "type": set_type,
                        "release_date": release_date,
                        "base_set_size": base_set_size,
                        "total_size": total_size,
                        "is_foil_only": is_foil_only,
                        "is_non_foil_only": is_non_foil_only,
                        "digital": digital,
                        "icon_svg_url": icon_svg_url,
                        "parent_code": parent_code,
                        "mtgo_code": mtgo_code,
                        "id": set_id
                    }
                )
                logger.debug(f"  Updated set: {code}")
            else:
                # Insert new set
                result = session.execute(
                    text("""
                        INSERT INTO mtg_sets 
                        (code, name, type, release_date, base_set_size, total_size,
                         is_foil_only, is_non_foil_only, digital, icon_svg_url,
                         parent_code, mtgo_code)
                        VALUES (:code, :name, :type, :release_date, :base_set_size, :total_size,
                                :is_foil_only, :is_non_foil_only, :digital, :icon_svg_url,
                                :parent_code, :mtgo_code)
                        RETURNING id
                    """),
                    {
                        "code": code,
                        "name": name,
                        "type": set_type,
                        "release_date": release_date,
                        "base_set_size": base_set_size,
                        "total_size": total_size,
                        "is_foil_only": is_foil_only,
                        "is_non_foil_only": is_non_foil_only,
                        "digital": digital,
                        "icon_svg_url": icon_svg_url,
                        "parent_code": parent_code,
                        "mtgo_code": mtgo_code
                    }
                )
                set_id = result.fetchone()[0]
                logger.debug(f"  Created set: {code}")
            
            # Commit the set insert
            session.commit()
            return set_id
        except Exception:
            session.rollback()
            raise
    
    def upsert_card(self, session, card_data: dict, set_id: int):
        """Upsert a card into the database."""
        try:
            # Extract card data
            name = card_data.get('name')
            mana_cost = card_data.get('manaCost')
            type_line = card_data.get('typeLine')
            oracle_text = card_data.get('oracleText')
            power = card_data.get('power')
            toughness = card_data.get('toughness')
            rarity = card_data.get('rarity')
            layout = card_data.get('layout')
            artist = card_data.get('artist')
            flavor_text = card_data.get('flavorText')
            numbers = card_data.get('numbers')
            identifiers = card_data.get('identifiers')
            images = card_data.get('images')
            
            # Serialize complex fields to JSON
            if isinstance(identifiers, dict):
                identifiers = json.dumps(identifiers)
            elif identifiers:
                identifiers = str(identifiers)
            
            if isinstance(images, dict):
                images = json.dumps(images)
            elif images:
                images = str(images)
            
            # Check if card exists (by name + set_id)
            result = session.execute(
                text("""
                    SELECT id FROM mtg_cards 
                    WHERE name = :name AND set_id = :set_id
                """),
                {"name": name, "set_id": set_id}
            )
            
            existing = result.fetchone()
            
            if existing:
                # Update existing card
                card_id = existing[0]
                session.execute(
                    text("""
                        UPDATE mtg_cards 
                        SET mana_cost = :mana_cost, type_line = :type_line,
                            oracle_text = :oracle_text, power = :power,
                            toughness = :toughness, rarity = :rarity,
                            layout = :layout, artist = :artist,
                            flavor_text = :flavor_text, numbers = :numbers,
                            identifiers = :identifiers, images = :images,
                            updated_at = CURRENT_TIMESTAMP
                        WHERE id = :id
                    """),
                    {
                        "mana_cost": mana_cost,
                        "type_line": type_line,
                        "oracle_text": oracle_text,
                        "power": power,
                        "toughness": toughness,
                        "rarity": rarity,
                        "layout": layout,
                        "artist": artist,
                        "flavor_text": flavor_text,
                        "numbers": numbers,
                        "identifiers": identifiers,
                        "images": images,
                        "id": card_id
                    }
                )
            else:
                # Insert new card
                session.execute(
                    text("""
                        INSERT INTO mtg_cards 
                        (set_id, name, mana_cost, type_line, oracle_text,
                         power, toughness, rarity, layout, artist,
                         flavor_text, numbers, identifiers, images)
                        VALUES (:set_id, :name, :mana_cost, :type_line, :oracle_text,
                                :power, :toughness, :rarity, :layout, :artist,
                                :flavor_text, :numbers, :identifiers, :images)
                    """),
                    {
                        "set_id": set_id,
                        "name": name,
                        "mana_cost": mana_cost,
                        "type_line": type_line,
                        "oracle_text": oracle_text,
                        "power": power,
                        "toughness": toughness,
                        "rarity": rarity,
                        "layout": layout,
                        "artist": artist,
                        "flavor_text": flavor_text,
                        "numbers": numbers,
                        "identifiers": identifiers,
                        "images": images
                    }
                )
        except Exception:
            session.rollback()
            raise


def main():
    """Main entry point."""
    # Database URL - use psycopg2 for synchronous connection
    db_url = "postgresql+psycopg2://mtgonline:mtgonline_pass@172.18.0.2:5432/mtgdata"
    
    # JSON file path
    json_path = "/app/data/AllPrintings.json"
    
    loader = MTGJSONDataLoader(db_url, json_path)
    
    try:
        loader.connect()
        loader.load_and_upsert()
    except Exception as e:
        logger.error(f"Error loading data: {e}", exc_info=True)
        sys.exit(1)
    finally:
        loader.close()


if __name__ == "__main__":
    main()