#!/usr/bin/env python3
"""
MTGJSON Data Uploader

Downloads all MTGJSON data and upserts it into the PostgreSQL database.
This script is designed to run inside the Docker container.
"""

import asyncio
import gzip
import json
import os
import sys
import zipfile
from pathlib import Path
from datetime import datetime
from typing import Any
from urllib.request import urlretrieve

# Add parent directory to path for imports
sys.path.insert(0, str(Path(__file__).parent.parent))

from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
from sqlalchemy.orm import sessionmaker
from sqlalchemy import text, insert, update, select, and_, Table, MetaData, Column, Integer, String, Text, Boolean, DateTime, Date, ForeignKey
from sqlalchemy.dialects.postgresql import insert as pg_insert

from app.core.settings import get_settings

# Define table metadata
metadata = MetaData()

# Define table objects
mtg_sets_table = Table('mtg_sets', metadata,
    Column('id', Integer, primary_key=True),
    Column('code', String(10), unique=True, nullable=False),
    Column('name', String(255)),
    Column('type', String(100)),
    Column('release_date', Date),
    Column('base_set_size', Integer),
    Column('total_size', Integer),
    Column('is_foil_only', Boolean),
    Column('is_non_foil_only', Boolean),
    Column('digital', Boolean),
    Column('icon_svg_url', Text),
    Column('parent_code', String(10)),
    Column('mtgo_code', String(10)),
    Column('card_count', Integer),
    Column('image_url', Text),
    Column('created_at', DateTime, default=datetime.utcnow),
    Column('updated_at', DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
)

mtg_cards_table = Table('mtg_cards', metadata,
    Column('id', Integer, primary_key=True),
    Column('set_id', Integer, ForeignKey('mtg_sets.id')),
    Column('name', String(255)),
    Column('mana_cost', String(255)),
    Column('type_line', String(255)),
    Column('oracle_text', Text),
    Column('power', String(50)),
    Column('toughness', String(50)),
    Column('rarity', String(50)),
    Column('layout', String(50)),
    Column('artist', String(255)),
    Column('flavor_text', Text),
    Column('numbers', String(100)),
    Column('identifiers', Text),
    Column('images', Text),
    Column('created_at', DateTime, default=datetime.utcnow),
    Column('updated_at', DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
)

card_identifiers_table = Table('card_identifiers', metadata,
    Column('id', Integer, primary_key=True),
    Column('uuid', String, unique=True, nullable=False),
    Column('name', String(255)),
    Column('mana_cost', String(255)),
    Column('type_line', String(255)),
    Column('oracle_text', Text),
    Column('power', String(50)),
    Column('toughness', String(50)),
    Column('rarity', String(50)),
    Column('layout', String(50)),
    Column('artist', String(255)),
    Column('flavor_text', Text),
    Column('set_code', String(10)),
    Column('created_at', DateTime, default=datetime.utcnow),
    Column('updated_at', DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
)

decks_table = Table('decks', metadata,
    Column('id', Integer, primary_key=True),
    Column('name', String(255), nullable=False),
    Column('description', Text),
    Column('format', String(50)),
    Column('command', Text),
    Column('commander', Text),
    Column('creation_date', DateTime, default=datetime.utcnow),
    Column('updated_at', DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
)

card_types_table = Table('card_types', metadata,
    Column('id', Integer, primary_key=True),
    Column('type', String(100), unique=True, nullable=False),
    Column('description', Text),
    Column('created_at', DateTime, default=datetime.utcnow),
    Column('updated_at', DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
)

deck_list_table = Table('deck_list', metadata,
    Column('id', Integer, primary_key=True),
    Column('deck_id', String(100), unique=True, nullable=False),
    Column('name', String(255)),
    Column('description', Text),
    Column('format', String(50)),
    Column('command', Text),
    Column('commander', Text),
    Column('total_cards', Integer),
    Column('created_at', DateTime, default=datetime.utcnow),
    Column('updated_at', DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
)

card_keywords_table = Table('card_keywords', metadata,
    Column('id', Integer, primary_key=True),
    Column('keyword', String(100), unique=True, nullable=False),
    Column('description', Text),
    Column('created_at', DateTime, default=datetime.utcnow),
    Column('updated_at', DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
)

set_list_table = Table('set_list', metadata,
    Column('id', Integer, primary_key=True),
    Column('set_code', String(10), unique=True, nullable=False),
    Column('set_name', String(255)),
    Column('set_type', String(100)),
    Column('release_date', Date),
    Column('base_set_size', Integer),
    Column('total_size', Integer),
    Column('created_at', DateTime, default=datetime.utcnow),
    Column('updated_at', DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
)

# MTGJSON API v5 base URL
MTGJSON_API_V5 = "https://mtgjson.com/api/v5"

# Files to download
MTGJSON_FILES = [
    "AllPrintings.psql.gz",
    "AllSetFiles.zip",
    "AllDeckFiles.zip",
    "AllIdentifiers.json.gz",
    "CardTypes.json.gz",
    "DeckList.json.gz",
    "Keywords.json.gz",
    "SetList.json.gz",
]


async def create_tables(engine: create_async_engine) -> None:
    """Create all required tables."""
    async with engine.begin() as conn:
        # Cards table (from AllPrintings)
        await conn.execute(text("""
            CREATE TABLE IF NOT EXISTS cards (
                id SERIAL PRIMARY KEY,
                artist TEXT,
                asciiName TEXT,
                attractionLights TEXT,
                availability TEXT,
                boosterTypes TEXT,
                borderColor TEXT,
                cardParts TEXT,
                colorIdentity TEXT,
                colorIndicator TEXT,
                colors TEXT,
                defense TEXT,
                duelDeck TEXT,
                edhrecRank INTEGER,
                edhrecSaltiness FLOAT,
                faceConvertedManaCost FLOAT,
                faceFlavorName TEXT,
                faceManaValue FLOAT,
                faceName TEXT,
                facePrintedName TEXT,
                finishes TEXT,
                flavorName TEXT,
                flavorText TEXT,
                frameEffects TEXT,
                frameVersion TEXT,
                hand TEXT,
                hasAlternativeDeckLimit BOOLEAN,
                hasContentWarning BOOLEAN,
                isAlternative BOOLEAN,
                isFullArt BOOLEAN,
                isFunny BOOLEAN,
                isGameChanger BOOLEAN,
                isOnlineOnly BOOLEAN,
                isOversized BOOLEAN,
                isPromo BOOLEAN,
                isRebalanced BOOLEAN,
                isReprint BOOLEAN,
                isReserved BOOLEAN,
                isStorySpotlight BOOLEAN,
                isTextless BOOLEAN,
                isTimeshifted BOOLEAN,
                keywords TEXT,
                language TEXT,
                layout TEXT,
                leadershipSkills TEXT,
                life TEXT,
                loyalty TEXT,
                manaCost TEXT,
                manaValue FLOAT,
                name TEXT,
                number TEXT,
                originalPrintings TEXT,
                originalReleaseDate TEXT,
                originalText TEXT,
                otherFaceIds TEXT,
                power TEXT,
                printedName TEXT,
                printedText TEXT,
                printedType TEXT,
                printings TEXT,
                producedMana TEXT,
                promoTypes TEXT,
                rarity TEXT,
                rebalancedPrintings TEXT,
                relatedCards TEXT,
                securityStamp TEXT,
                setCode TEXT,
                side TEXT,
                signature TEXT,
                skuIds TEXT,
                sourceProducts TEXT,
                subsets TEXT,
                subtypes TEXT,
                supertypes TEXT,
                text TEXT,
                toughness TEXT,
                type TEXT,
                types TEXT,
                uuid TEXT,
                variations TEXT,
                watermark TEXT,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
            )
        """))
        
        # MTG Sets table
        await conn.execute(text("""
            CREATE TABLE IF NOT EXISTS mtg_sets (
                id SERIAL PRIMARY KEY,
                code VARCHAR(10) UNIQUE NOT NULL,
                name VARCHAR(255),
                type VARCHAR(100),
                release_date DATE,
                base_set_size INTEGER,
                total_size INTEGER,
                is_foil_only BOOLEAN,
                is_non_foil_only BOOLEAN,
                digital BOOLEAN,
                icon_svg_url TEXT,
                parent_code VARCHAR(10),
                mtgo_code VARCHAR(10),
                card_count INTEGER,
                image_url TEXT,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
            )
        """))
        
        # MTG Cards table (from set files)
        await conn.execute(text("""
            CREATE TABLE IF NOT EXISTS mtg_cards (
                id SERIAL PRIMARY KEY,
                set_id INTEGER REFERENCES mtg_sets(id) ON DELETE CASCADE,
                name VARCHAR(255),
                mana_cost VARCHAR(255),
                type_line VARCHAR(255),
                oracle_text TEXT,
                power VARCHAR(50),
                toughness VARCHAR(50),
                rarity VARCHAR(50),
                layout VARCHAR(50),
                artist VARCHAR(255),
                flavor_text TEXT,
                numbers VARCHAR(100),
                identifiers TEXT,
                images TEXT,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
            )
        """))
        
        # Card Identifiers table
        await conn.execute(text("""
            CREATE TABLE IF NOT EXISTS card_identifiers (
                id SERIAL PRIMARY KEY,
                uuid TEXT UNIQUE NOT NULL,
                name VARCHAR(255),
                mana_cost VARCHAR(255),
                type_line VARCHAR(255),
                oracle_text TEXT,
                power VARCHAR(50),
                toughness VARCHAR(50),
                rarity VARCHAR(50),
                layout VARCHAR(50),
                artist VARCHAR(255),
                flavor_text TEXT,
                set_code VARCHAR(10),
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
            )
        """))
        
        # Decks table
        await conn.execute(text("""
            CREATE TABLE IF NOT EXISTS decks (
                id SERIAL PRIMARY KEY,
                name VARCHAR(255) NOT NULL,
                description TEXT,
                format VARCHAR(50),
                command TEXT,
                commander TEXT,
                creation_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
            )
        """))
        
        # Card Types table
        await conn.execute(text("""
            CREATE TABLE IF NOT EXISTS card_types (
                id SERIAL PRIMARY KEY,
                type VARCHAR(100) UNIQUE NOT NULL,
                description TEXT,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
            )
        """))
        
        # Deck List table
        await conn.execute(text("""
            CREATE TABLE IF NOT EXISTS deck_list (
                id SERIAL PRIMARY KEY,
                deck_id VARCHAR(100) UNIQUE NOT NULL,
                name VARCHAR(255),
                description TEXT,
                format VARCHAR(50),
                command TEXT,
                commander TEXT,
                total_cards INTEGER,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
            )
        """))
        
        # Card Keywords table
        await conn.execute(text("""
            CREATE TABLE IF NOT EXISTS card_keywords (
                id SERIAL PRIMARY KEY,
                keyword VARCHAR(100) UNIQUE NOT NULL,
                description TEXT,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
            )
        """))
        
        # Set List table
        await conn.execute(text("""
            CREATE TABLE IF NOT EXISTS set_list (
                id SERIAL PRIMARY KEY,
                set_code VARCHAR(10) UNIQUE NOT NULL,
                set_name VARCHAR(255),
                set_type VARCHAR(100),
                release_date DATE,
                base_set_size INTEGER,
                total_size INTEGER,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
            )
        """))
        
        await conn.commit()
        print("✓ Database tables created")


async def import_all_printings_psql(engine: create_async_engine, psql_file: Path) -> int:
    """Import AllPrintings.psql.gz file."""
    if not psql_file.exists():
        print("✗ AllPrintings.psql.gz not found")
        return 0
    
    print(f"Importing {psql_file.name}...")
    
    # Decompress
    psql_content = gzip.decompress(psql_file.read_bytes())
    psql_path = psql_file.with_suffix('.psql')
    psql_path.write_bytes(psql_content)
    
    # Use psql command to import
    import subprocess
    
    settings = get_settings()
    db_url = settings.MTG_DATABASE_URL
    
    # Parse database URL to extract connection details
    # Format: postgresql+asyncpg://user:pass@host:port/database
    url_parts = db_url.replace('postgresql+asyncpg://', '').split('@')
    user_pass = url_parts[0].split('//')[1]
    host_db = url_parts[1]
    
    user, password = user_pass.split(':')
    host, port_db = host_db.split(':')
    database = port_db.split('/')[1]
    
    cmd = [
        'psql', '-h', host, '-p', port_db.split('/')[0],
        '-U', user, '-d', database,
        '-f', str(psql_path)
    ]
    
    result = subprocess.run(cmd, capture_output=True, text=True)
    
    if result.returncode != 0:
        print(f"✗ Import failed: {result.stderr[:500]}")
        psql_path.unlink()
        return 0
    
    # Count records
    async with engine.connect() as conn:
        result = await conn.execute(text("SELECT COUNT(*) FROM cards"))
        count = result.scalar()
        print(f"✓ Imported {count:,} cards")
    
    # Clean up
    psql_path.unlink()
    return count


async def import_all_set_files(engine: create_async_engine, set_files_dir: Path) -> tuple[int, int]:
    """Import AllSetFiles.zip - sets and cards."""
    if not set_files_dir.exists():
        print("✗ AllSetFiles directory not found")
        return 0, 0
    
    set_count = 0
    card_count = 0
    
    async with engine.begin() as conn:
        # Import sets
        for set_file in sorted(set_files_dir.glob('*.json')):
            data = json.loads(set_file.read_text())
            
            if 'code' in data:
                image_url = None
                if 'image' in data and data['image']:
                    image_url = data['image'].get('normal')
                
                # Upsert set
                result = await conn.execute(
                    pg_insert(mtg_sets_table).values(
                        code=data.get('code'),
                        name=data.get('name'),
                        type=data.get('type'),
                        release_date=data.get('releaseDate'),
                        base_set_size=data.get('baseSetSize'),
                        total_size=data.get('totalSetSize'),
                        is_foil_only=data.get('isFoilOnly'),
                        is_non_foil_only=data.get('isNonFoilOnly'),
                        digital=data.get('digital'),
                        icon_svg_url=data.get('iconSvgUrl'),
                        parent_code=data.get('parentCode'),
                        mtgo_code=data.get('mtgoCode'),
                        card_count=data.get('cardCount'),
                        image_url=image_url,
                    ).on_conflict_do_update(
                        index_elements=['code'],
                        set_={
                            'name': data.get('name'),
                            'type': data.get('type'),
                            'release_date': data.get('releaseDate'),
                            'base_set_size': data.get('baseSetSize'),
                            'total_size': data.get('totalSetSize'),
                            'is_foil_only': data.get('isFoilOnly'),
                            'is_non_foil_only': data.get('isNonFoilOnly'),
                            'digital': data.get('digital'),
                            'icon_svg_url': data.get('iconSvgUrl'),
                            'parent_code': data.get('parentCode'),
                            'mtgo_code': data.get('mtgoCode'),
                            'card_count': data.get('cardCount'),
                            'image_url': image_url,
                            'updated_at': datetime.utcnow(),
                        }
                    ).returning(mtg_sets_table.id),
                    execution_options={"autocommit": True}
                )
                set_id = result.scalar()
                
                if 'cards' in data:
                    for card_data in data['cards']:
                        identifiers = {
                            'multiId': card_data.get('multiverseIds'),
                            'tcgplayerProductId': card_data.get('tcgplayerProductId'),
                            'cardmarketId': card_data.get('cardmarketId'),
                        }
                        
                        images = {}
                        if 'image_uris' in card_data:
                            images = {
                                'small': card_data['image_uris'].get('small'),
                                'normal': card_data['image_uris'].get('normal'),
                                'large': card_data['image_uris'].get('large'),
                                'png': card_data['image_uris'].get('png'),
                                'art_crop': card_data['image_uris'].get('art_crop'),
                            }
                        
                        # Upsert card
                        await conn.execute(
                            pg_insert(mtg_cards_table).values(
                                set_id=set_id,
                                name=card_data.get('name'),
                                mana_cost=card_data.get('manaCost'),
                                type_line=card_data.get('typeLine'),
                                oracle_text=card_data.get('oracleText'),
                                power=card_data.get('power'),
                                toughness=card_data.get('toughness'),
                                rarity=card_data.get('rarity'),
                                layout=card_data.get('layout'),
                                artist=card_data.get('artist'),
                                flavor_text=card_data.get('flavorText'),
                                numbers=str(card_data.get('number')),
                                identifiers=json.dumps(identifiers),
                                images=json.dumps(images),
                            ).on_conflict_do_nothing(),
                            execution_options={"autocommit": True}
                        )
                        card_count += 1
                
                set_count += 1
    
    print(f"✓ Imported {set_count} sets and {card_count} cards from sets")
    return set_count, card_count


async def import_all_identifiers(engine: create_async_engine, file_path: Path) -> int:
    """Import AllIdentifiers.json.gz."""
    if not file_path.exists():
        print("✗ AllIdentifiers.json.gz not found")
        return 0
    
    data = json.loads(gzip.decompress(file_path.read_bytes()))
    
    count = 0
    async with engine.begin() as conn:
        for uuid, card_data in data.items():
            await conn.execute(
                pg_insert(card_identifiers_table).values(
                    uuid=uuid,
                    name=card_data.get('name'),
                    mana_cost=card_data.get('manaCost'),
                    type_line=card_data.get('typeLine'),
                    oracle_text=card_data.get('oracleText'),
                    power=card_data.get('power'),
                    toughness=card_data.get('toughness'),
                    rarity=card_data.get('rarity'),
                    layout=card_data.get('layout'),
                    artist=card_data.get('artist'),
                    flavor_text=card_data.get('flavorText'),
                    set_code=card_data.get('setCode'),
                ).on_conflict_do_update(
                    index_elements=['uuid'],
                    set_={
                        'name': card_data.get('name'),
                        'mana_cost': card_data.get('manaCost'),
                        'type_line': card_data.get('typeLine'),
                        'oracle_text': card_data.get('oracleText'),
                        'power': card_data.get('power'),
                        'toughness': card_data.get('toughness'),
                        'rarity': card_data.get('rarity'),
                        'layout': card_data.get('layout'),
                        'artist': card_data.get('artist'),
                        'flavor_text': card_data.get('flavorText'),
                        'set_code': card_data.get('setCode'),
                        'updated_at': datetime.utcnow(),
                    }
                ),
                execution_options={"autocommit": True}
            )
            count += 1
    
    print(f"✓ Imported {count} identifiers")
    return count


async def import_all_deck_files(engine: create_async_engine, deck_files_dir: Path) -> int:
    """Import AllDeckFiles.zip."""
    if not deck_files_dir.exists():
        print("✗ AllDeckFiles directory not found")
        return 0
    
    count = 0
    async with engine.begin() as conn:
        for deck_file in sorted(deck_files_dir.glob('*.json')):
            data = json.loads(deck_file.read_text())
            
            if 'name' in data and 'cards' in data:
                await conn.execute(
                    pg_insert(decks_table).values(
                        name=data.get('name'),
                        description=data.get('description'),
                        format=data.get('format'),
                        command=data.get('command'),
                        commander=data.get('commander'),
                    ).on_conflict_do_update(
                        index_elements=['name'],
                        set_={
                            'description': data.get('description'),
                            'format': data.get('format'),
                            'command': data.get('command'),
                            'commander': data.get('commander'),
                            'updated_at': datetime.utcnow(),
                        }
                    ),
                    execution_options={"autocommit": True}
                )
                count += 1
    
    print(f"✓ Imported {count} decks")
    return count


async def import_json_files(engine: create_async_engine, file_path: Path, 
                           table_name: str, name_field: str, id_field: str) -> int:
    """Import a JSON.gz file into a table."""
    if not file_path.exists():
        return 0
    
    data = json.loads(gzip.decompress(file_path.read_bytes()))
    
    count = 0
    async with engine.begin() as conn:
        for item in data:
            values = {k: v for k, v in item.items() if k != id_field}
            await conn.execute(
                pg_insert(text(f'{table_name}_table')).values(values),
                execution_options={"autocommit": True}
            )
            count += 1
    
    print(f"✓ Imported {count} items into {table_name}")
    return count


def download_file(url: str, destination: Path) -> bool:
    """Download a file from URL to destination."""
    try:
        print(f"Downloading {url}...")
        urlretrieve(url, destination)
        size_mb = destination.stat().st_size / (1024 * 1024)
        print(f"  ✓ Downloaded to {destination} ({size_mb:.1f} MB)")
        return True
    except Exception as e:
        print(f"  ✗ Failed to download {url}: {e}")
        return False


def extract_zip(zip_file: Path, extract_dir: Path) -> None:
    """Extract a zip file."""
    if not zip_file.exists():
        print(f"✗ {zip_file.name} not found")
        return
    
    if extract_dir.exists():
        import shutil
        shutil.rmtree(extract_dir)
    
    print(f"Extracting {zip_file.name}...")
    with zipfile.ZipFile(zip_file, 'r') as zip_ref:
        zip_ref.extractall(extract_dir)
    
    file_count = len(list(extract_dir.glob('*.json')))
    print(f"✓ Extracted to {extract_dir} ({file_count} files)")


async def main():
    """Main function to download and import all MTGJSON data."""
    settings = get_settings()
    
    # Setup data directory
    data_dir = Path(settings.DATA_DIR) / "mtgjson"
    downloads_dir = data_dir / "downloads"
    downloads_dir.mkdir(parents=True, exist_ok=True)
    
    # Create engine
    engine = create_async_engine(settings.MTG_DATABASE_URL)
    
    # Create tables
    print("=== Creating Database Tables ===")
    await create_tables(engine)
    
    # Download AllPrintings.psql.gz
    print("\n=== Downloading AllPrintings ===")
    psql_file = downloads_dir / "AllPrintings.psql.gz"
    if not psql_file.exists():
        download_file(f"{MTGJSON_API_V5}/AllPrintings.psql.gz", psql_file)
    
    # Import AllPrintings
    await import_all_printings_psql(engine, psql_file)
    
    # Download and extract AllSetFiles
    print("\n=== Importing AllSetFiles ===")
    set_files_zip = downloads_dir / "AllSetFiles.zip"
    set_files_dir = downloads_dir / "AllSetFiles"
    
    if not set_files_dir.exists():
        if not set_files_zip.exists():
            download_file(f"{MTGJSON_API_V5}/AllSetFiles.zip", set_files_zip)
        extract_zip(set_files_zip, set_files_dir)
    
    await import_all_set_files(engine, set_files_dir)
    
    # Download and extract AllDeckFiles
    print("\n=== Importing AllDeckFiles ===")
    deck_files_zip = downloads_dir / "AllDeckFiles.zip"
    deck_files_dir = downloads_dir / "AllDeckFiles"
    
    if not deck_files_dir.exists():
        if not deck_files_zip.exists():
            download_file(f"{MTGJSON_API_V5}/AllDeckFiles.zip", deck_files_zip)
        extract_zip(deck_files_zip, deck_files_dir)
    
    await import_all_deck_files(engine, deck_files_dir)
    
    # Download and import AllIdentifiers
    print("\n=== Importing AllIdentifiers ===")
    identifiers_file = downloads_dir / "AllIdentifiers.json.gz"
    if not identifiers_file.exists():
        download_file(f"{MTGJSON_API_V5}/AllIdentifiers.json.gz", identifiers_file)
    
    await import_all_identifiers(engine, identifiers_file)
    
    # Download and import CardTypes
    print("\n=== Importing CardTypes ===")
    card_types_file = downloads_dir / "CardTypes.json.gz"
    if not card_types_file.exists():
        download_file(f"{MTGJSON_API_V5}/CardTypes.json.gz", card_types_file)
    
    # Import CardTypes (simplified)
    if card_types_file.exists():
        data = json.loads(gzip.decompress(card_types_file.read_bytes()))
        async with engine.begin() as conn:
            count = 0
            for card_type in data:
                await conn.execute(
                    pg_insert(card_types_table).values(
                        type=card_type.get('type'),
                        description=card_type.get('description'),
                    ).on_conflict_do_nothing(),
                    execution_options={"autocommit": True}
                )
                count += 1
            print(f"✓ Imported {count} card types")
    
    # Download and import DeckList
    print("\n=== Importing DeckList ===")
    deck_list_file = downloads_dir / "DeckList.json.gz"
    if not deck_list_file.exists():
        download_file(f"{MTGJSON_API_V5}/DeckList.json.gz", deck_list_file)
    
    # Import DeckList (simplified)
    if deck_list_file.exists():
        data = json.loads(gzip.decompress(deck_list_file.read_bytes()))
        async with engine.begin() as conn:
            count = 0
            for deck in data:
                await conn.execute(
                    pg_insert(deck_list_table).values(
                        deck_id=deck.get('id'),
                        name=deck.get('name'),
                        description=deck.get('description'),
                        format=deck.get('format'),
                        command=deck.get('command'),
                        commander=deck.get('commander'),
                        total_cards=deck.get('totalCards'),
                    ).on_conflict_do_nothing(),
                    execution_options={"autocommit": True}
                )
                count += 1
            print(f"✓ Imported {count} deck list entries")
    
    # Download and import Keywords
    print("\n=== Importing Keywords ===")
    keywords_file = downloads_dir / "Keywords.json.gz"
    if not keywords_file.exists():
        download_file(f"{MTGJSON_API_V5}/Keywords.json.gz", keywords_file)
    
    # Import Keywords (simplified)
    if keywords_file.exists():
        data = json.loads(gzip.decompress(keywords_file.read_bytes()))
        async with engine.begin() as conn:
            count = 0
            for keyword in data:
                await conn.execute(
                    pg_insert(card_keywords_table).values(
                        keyword=keyword.get('keyword'),
                        description=keyword.get('description'),
                    ).on_conflict_do_nothing(),
                    execution_options={"autocommit": True}
                )
                count += 1
            print(f"✓ Imported {count} keywords")
    
    # Download and import SetList
    print("\n=== Importing SetList ===")
    set_list_file = downloads_dir / "SetList.json.gz"
    if not set_list_file.exists():
        download_file(f"{MTGJSON_API_V5}/SetList.json.gz", set_list_file)
    
    # Import SetList (simplified)
    if set_list_file.exists():
        data = json.loads(gzip.decompress(set_list_file.read_bytes()))
        async with engine.begin() as conn:
            count = 0
            for set_data in data:
                await conn.execute(
                    pg_insert(set_list_table).values(
                        set_code=set_data.get('code'),
                        set_name=set_data.get('name'),
                        set_type=set_data.get('type'),
                        release_date=set_data.get('releaseDate'),
                        base_set_size=set_data.get('baseSetSize'),
                        total_size=set_data.get('totalSetSize'),
                    ).on_conflict_do_nothing(),
                    execution_options={"autocommit": True}
                )
                count += 1
            print(f"✓ Imported {count} set list entries")
    
    # Create indexes
    print("\n=== Creating Indexes ===")
    async with engine.begin() as conn:
        indexes = [
            "CREATE INDEX IF NOT EXISTS idx_cards_name ON cards(name)",
            "CREATE INDEX IF NOT EXISTS idx_cards_mana_cost ON cards(manaCost)",
            "CREATE INDEX IF NOT EXISTS idx_cards_type ON cards(type)",
            "CREATE INDEX IF NOT EXISTS idx_cards_rarity ON cards(rarity)",
            "CREATE INDEX IF NOT EXISTS idx_cards_set_code ON cards(setCode)",
            "CREATE INDEX IF NOT EXISTS idx_cards_uuid ON cards(uuid)",
            "CREATE INDEX IF NOT EXISTS idx_mtg_sets_code ON mtg_sets(code)",
            "CREATE INDEX IF NOT EXISTS idx_mtg_cards_name ON mtg_cards(name)",
            "CREATE INDEX IF NOT EXISTS idx_card_identifiers_uuid ON card_identifiers(uuid)",
            "CREATE INDEX IF NOT EXISTS idx_card_identifiers_name ON card_identifiers(name)",
            "CREATE INDEX IF NOT EXISTS idx_deck_list_deck_id ON deck_list(deck_id)",
        ]
        
        for idx in indexes:
            await conn.execute(text(idx))
        
        await conn.commit()
        print("✓ Indexes created")
    
    # Show summary
    print("\n=== Import Summary ===")
    async with engine.connect() as conn:
        tables = [
            'cards', 'mtg_sets', 'mtg_cards', 'card_identifiers',
            'decks', 'card_types', 'deck_list', 'card_keywords', 'set_list'
        ]
        
        for table in tables:
            result = await conn.execute(text(f"SELECT COUNT(*) FROM {table}"))
            count = result.scalar()
            print(f"  {table:20} {count:>10,} records")
    
    await engine.dispose()
    print("\n✓ All MTGJSON data imported successfully!")


if __name__ == "__main__":
    asyncio.run(main())
