884 lines
33 KiB
Python
884 lines
33 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
MTGJSON Data Uploader
|
|
|
|
Downloads all MTGJSON data and upserts it into the PostgreSQL database.
|
|
This script is designed to run inside the Docker container.
|
|
"""
|
|
|
|
import asyncio
|
|
import gzip
|
|
import json
|
|
import os
|
|
import sys
|
|
import zipfile
|
|
from pathlib import Path
|
|
from datetime import datetime
|
|
from typing import Any
|
|
from urllib.request import urlretrieve
|
|
|
|
# Add parent directory to path for imports
|
|
sys.path.insert(0, str(Path(__file__).parent.parent))
|
|
|
|
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
|
from sqlalchemy.orm import sessionmaker
|
|
from sqlalchemy import text, insert, update, select, and_, Table, MetaData, Column, Integer, String, Text, Boolean, DateTime, Date, ForeignKey
|
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
|
|
|
from app.core.settings import get_settings
|
|
|
|
# Define table metadata
|
|
metadata = MetaData()
|
|
|
|
# Define table objects
|
|
mtg_sets_table = Table('mtg_sets', metadata,
|
|
Column('id', Integer, primary_key=True),
|
|
Column('code', String(10), unique=True, nullable=False),
|
|
Column('name', String(255)),
|
|
Column('type', String(100)),
|
|
Column('release_date', Date),
|
|
Column('base_set_size', Integer),
|
|
Column('total_size', Integer),
|
|
Column('is_foil_only', Boolean),
|
|
Column('is_non_foil_only', Boolean),
|
|
Column('digital', Boolean),
|
|
Column('icon_svg_url', Text),
|
|
Column('parent_code', String(10)),
|
|
Column('mtgo_code', String(10)),
|
|
Column('card_count', Integer),
|
|
Column('image_url', Text),
|
|
Column('created_at', DateTime, default=datetime.utcnow),
|
|
Column('updated_at', DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
|
)
|
|
|
|
mtg_cards_table = Table('mtg_cards', metadata,
|
|
Column('id', Integer, primary_key=True),
|
|
Column('set_id', Integer, ForeignKey('mtg_sets.id')),
|
|
Column('name', String(255)),
|
|
Column('mana_cost', String(255)),
|
|
Column('type_line', String(255)),
|
|
Column('oracle_text', Text),
|
|
Column('power', String(50)),
|
|
Column('toughness', String(50)),
|
|
Column('rarity', String(50)),
|
|
Column('layout', String(50)),
|
|
Column('artist', String(255)),
|
|
Column('flavor_text', Text),
|
|
Column('numbers', String(100)),
|
|
Column('identifiers', Text),
|
|
Column('images', Text),
|
|
Column('created_at', DateTime, default=datetime.utcnow),
|
|
Column('updated_at', DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
|
)
|
|
|
|
card_identifiers_table = Table('card_identifiers', metadata,
|
|
Column('id', Integer, primary_key=True),
|
|
Column('uuid', String, unique=True, nullable=False),
|
|
Column('name', String(255)),
|
|
Column('mana_cost', String(255)),
|
|
Column('type_line', String(255)),
|
|
Column('oracle_text', Text),
|
|
Column('power', String(50)),
|
|
Column('toughness', String(50)),
|
|
Column('rarity', String(50)),
|
|
Column('layout', String(50)),
|
|
Column('artist', String(255)),
|
|
Column('flavor_text', Text),
|
|
Column('set_code', String(10)),
|
|
Column('created_at', DateTime, default=datetime.utcnow),
|
|
Column('updated_at', DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
|
)
|
|
|
|
decks_table = Table('decks', metadata,
|
|
Column('id', Integer, primary_key=True),
|
|
Column('name', String(255), nullable=False),
|
|
Column('description', Text),
|
|
Column('format', String(50)),
|
|
Column('command', Text),
|
|
Column('commander', Text),
|
|
Column('creation_date', DateTime, default=datetime.utcnow),
|
|
Column('updated_at', DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
|
)
|
|
|
|
card_types_table = Table('card_types', metadata,
|
|
Column('id', Integer, primary_key=True),
|
|
Column('type', String(100), unique=True, nullable=False),
|
|
Column('description', Text),
|
|
Column('created_at', DateTime, default=datetime.utcnow),
|
|
Column('updated_at', DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
|
)
|
|
|
|
deck_list_table = Table('deck_list', metadata,
|
|
Column('id', Integer, primary_key=True),
|
|
Column('deck_id', String(100), unique=True, nullable=False),
|
|
Column('name', String(255)),
|
|
Column('description', Text),
|
|
Column('format', String(50)),
|
|
Column('command', Text),
|
|
Column('commander', Text),
|
|
Column('total_cards', Integer),
|
|
Column('created_at', DateTime, default=datetime.utcnow),
|
|
Column('updated_at', DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
|
)
|
|
|
|
card_keywords_table = Table('card_keywords', metadata,
|
|
Column('id', Integer, primary_key=True),
|
|
Column('keyword', String(100), unique=True, nullable=False),
|
|
Column('description', Text),
|
|
Column('created_at', DateTime, default=datetime.utcnow),
|
|
Column('updated_at', DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
|
)
|
|
|
|
set_list_table = Table('set_list', metadata,
|
|
Column('id', Integer, primary_key=True),
|
|
Column('set_code', String(10), unique=True, nullable=False),
|
|
Column('set_name', String(255)),
|
|
Column('set_type', String(100)),
|
|
Column('release_date', Date),
|
|
Column('base_set_size', Integer),
|
|
Column('total_size', Integer),
|
|
Column('created_at', DateTime, default=datetime.utcnow),
|
|
Column('updated_at', DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
|
)
|
|
|
|
# MTGJSON API v5 base URL
|
|
MTGJSON_API_V5 = "https://mtgjson.com/api/v5"
|
|
|
|
# Files to download
|
|
MTGJSON_FILES = [
|
|
"AllPrintings.psql.gz",
|
|
"AllSetFiles.zip",
|
|
"AllDeckFiles.zip",
|
|
"AllIdentifiers.json.gz",
|
|
"CardTypes.json.gz",
|
|
"DeckList.json.gz",
|
|
"Keywords.json.gz",
|
|
"SetList.json.gz",
|
|
]
|
|
|
|
|
|
async def create_tables(engine: create_async_engine) -> None:
|
|
"""Create all required tables."""
|
|
async with engine.begin() as conn:
|
|
# Cards table (from AllPrintings)
|
|
await conn.execute(text("""
|
|
CREATE TABLE IF NOT EXISTS cards (
|
|
id SERIAL PRIMARY KEY,
|
|
artist TEXT,
|
|
asciiName TEXT,
|
|
attractionLights TEXT,
|
|
availability TEXT,
|
|
boosterTypes TEXT,
|
|
borderColor TEXT,
|
|
cardParts TEXT,
|
|
colorIdentity TEXT,
|
|
colorIndicator TEXT,
|
|
colors TEXT,
|
|
defense TEXT,
|
|
duelDeck TEXT,
|
|
edhrecRank INTEGER,
|
|
edhrecSaltiness FLOAT,
|
|
faceConvertedManaCost FLOAT,
|
|
faceFlavorName TEXT,
|
|
faceManaValue FLOAT,
|
|
faceName TEXT,
|
|
facePrintedName TEXT,
|
|
finishes TEXT,
|
|
flavorName TEXT,
|
|
flavorText TEXT,
|
|
frameEffects TEXT,
|
|
frameVersion TEXT,
|
|
hand TEXT,
|
|
hasAlternativeDeckLimit BOOLEAN,
|
|
hasContentWarning BOOLEAN,
|
|
isAlternative BOOLEAN,
|
|
isFullArt BOOLEAN,
|
|
isFunny BOOLEAN,
|
|
isGameChanger BOOLEAN,
|
|
isOnlineOnly BOOLEAN,
|
|
isOversized BOOLEAN,
|
|
isPromo BOOLEAN,
|
|
isRebalanced BOOLEAN,
|
|
isReprint BOOLEAN,
|
|
isReserved BOOLEAN,
|
|
isStorySpotlight BOOLEAN,
|
|
isTextless BOOLEAN,
|
|
isTimeshifted BOOLEAN,
|
|
keywords TEXT,
|
|
language TEXT,
|
|
layout TEXT,
|
|
leadershipSkills TEXT,
|
|
life TEXT,
|
|
loyalty TEXT,
|
|
manaCost TEXT,
|
|
manaValue FLOAT,
|
|
name TEXT,
|
|
number TEXT,
|
|
originalPrintings TEXT,
|
|
originalReleaseDate TEXT,
|
|
originalText TEXT,
|
|
otherFaceIds TEXT,
|
|
power TEXT,
|
|
printedName TEXT,
|
|
printedText TEXT,
|
|
printedType TEXT,
|
|
printings TEXT,
|
|
producedMana TEXT,
|
|
promoTypes TEXT,
|
|
rarity TEXT,
|
|
rebalancedPrintings TEXT,
|
|
relatedCards TEXT,
|
|
securityStamp TEXT,
|
|
setCode TEXT,
|
|
side TEXT,
|
|
signature TEXT,
|
|
skuIds TEXT,
|
|
sourceProducts TEXT,
|
|
subsets TEXT,
|
|
subtypes TEXT,
|
|
supertypes TEXT,
|
|
text TEXT,
|
|
toughness TEXT,
|
|
type TEXT,
|
|
types TEXT,
|
|
uuid TEXT,
|
|
variations TEXT,
|
|
watermark TEXT,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
"""))
|
|
|
|
# MTG Sets table
|
|
await conn.execute(text("""
|
|
CREATE TABLE IF NOT EXISTS mtg_sets (
|
|
id SERIAL PRIMARY KEY,
|
|
code VARCHAR(10) UNIQUE NOT NULL,
|
|
name VARCHAR(255),
|
|
type VARCHAR(100),
|
|
release_date DATE,
|
|
base_set_size INTEGER,
|
|
total_size INTEGER,
|
|
is_foil_only BOOLEAN,
|
|
is_non_foil_only BOOLEAN,
|
|
digital BOOLEAN,
|
|
icon_svg_url TEXT,
|
|
parent_code VARCHAR(10),
|
|
mtgo_code VARCHAR(10),
|
|
card_count INTEGER,
|
|
image_url TEXT,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
"""))
|
|
|
|
# MTG Cards table (from set files)
|
|
await conn.execute(text("""
|
|
CREATE TABLE IF NOT EXISTS mtg_cards (
|
|
id SERIAL PRIMARY KEY,
|
|
set_id INTEGER REFERENCES mtg_sets(id) ON DELETE CASCADE,
|
|
name VARCHAR(255),
|
|
mana_cost VARCHAR(255),
|
|
type_line VARCHAR(255),
|
|
oracle_text TEXT,
|
|
power VARCHAR(50),
|
|
toughness VARCHAR(50),
|
|
rarity VARCHAR(50),
|
|
layout VARCHAR(50),
|
|
artist VARCHAR(255),
|
|
flavor_text TEXT,
|
|
numbers VARCHAR(100),
|
|
identifiers TEXT,
|
|
images TEXT,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
"""))
|
|
|
|
# Card Identifiers table
|
|
await conn.execute(text("""
|
|
CREATE TABLE IF NOT EXISTS card_identifiers (
|
|
id SERIAL PRIMARY KEY,
|
|
uuid TEXT UNIQUE NOT NULL,
|
|
name VARCHAR(255),
|
|
mana_cost VARCHAR(255),
|
|
type_line VARCHAR(255),
|
|
oracle_text TEXT,
|
|
power VARCHAR(50),
|
|
toughness VARCHAR(50),
|
|
rarity VARCHAR(50),
|
|
layout VARCHAR(50),
|
|
artist VARCHAR(255),
|
|
flavor_text TEXT,
|
|
set_code VARCHAR(10),
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
"""))
|
|
|
|
# Decks table
|
|
await conn.execute(text("""
|
|
CREATE TABLE IF NOT EXISTS decks (
|
|
id SERIAL PRIMARY KEY,
|
|
name VARCHAR(255) NOT NULL,
|
|
description TEXT,
|
|
format VARCHAR(50),
|
|
command TEXT,
|
|
commander TEXT,
|
|
creation_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
"""))
|
|
|
|
# Card Types table
|
|
await conn.execute(text("""
|
|
CREATE TABLE IF NOT EXISTS card_types (
|
|
id SERIAL PRIMARY KEY,
|
|
type VARCHAR(100) UNIQUE NOT NULL,
|
|
description TEXT,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
"""))
|
|
|
|
# Deck List table
|
|
await conn.execute(text("""
|
|
CREATE TABLE IF NOT EXISTS deck_list (
|
|
id SERIAL PRIMARY KEY,
|
|
deck_id VARCHAR(100) UNIQUE NOT NULL,
|
|
name VARCHAR(255),
|
|
description TEXT,
|
|
format VARCHAR(50),
|
|
command TEXT,
|
|
commander TEXT,
|
|
total_cards INTEGER,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
"""))
|
|
|
|
# Card Keywords table
|
|
await conn.execute(text("""
|
|
CREATE TABLE IF NOT EXISTS card_keywords (
|
|
id SERIAL PRIMARY KEY,
|
|
keyword VARCHAR(100) UNIQUE NOT NULL,
|
|
description TEXT,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
"""))
|
|
|
|
# Set List table
|
|
await conn.execute(text("""
|
|
CREATE TABLE IF NOT EXISTS set_list (
|
|
id SERIAL PRIMARY KEY,
|
|
set_code VARCHAR(10) UNIQUE NOT NULL,
|
|
set_name VARCHAR(255),
|
|
set_type VARCHAR(100),
|
|
release_date DATE,
|
|
base_set_size INTEGER,
|
|
total_size INTEGER,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
"""))
|
|
|
|
await conn.commit()
|
|
print("✓ Database tables created")
|
|
|
|
|
|
async def import_all_printings_psql(engine: create_async_engine, psql_file: Path) -> int:
|
|
"""Import AllPrintings.psql.gz file."""
|
|
if not psql_file.exists():
|
|
print("✗ AllPrintings.psql.gz not found")
|
|
return 0
|
|
|
|
print(f"Importing {psql_file.name}...")
|
|
|
|
# Decompress
|
|
psql_content = gzip.decompress(psql_file.read_bytes())
|
|
psql_path = psql_file.with_suffix('.psql')
|
|
psql_path.write_bytes(psql_content)
|
|
|
|
# Use psql command to import
|
|
import subprocess
|
|
|
|
settings = get_settings()
|
|
db_url = settings.MTG_DATABASE_URL
|
|
|
|
# Parse database URL to extract connection details
|
|
# Format: postgresql+asyncpg://user:pass@host:port/database
|
|
url_parts = db_url.replace('postgresql+asyncpg://', '').split('@')
|
|
user_pass = url_parts[0].split('//')[1]
|
|
host_db = url_parts[1]
|
|
|
|
user, password = user_pass.split(':')
|
|
host, port_db = host_db.split(':')
|
|
database = port_db.split('/')[1]
|
|
|
|
cmd = [
|
|
'psql', '-h', host, '-p', port_db.split('/')[0],
|
|
'-U', user, '-d', database,
|
|
'-f', str(psql_path)
|
|
]
|
|
|
|
result = subprocess.run(cmd, capture_output=True, text=True)
|
|
|
|
if result.returncode != 0:
|
|
print(f"✗ Import failed: {result.stderr[:500]}")
|
|
psql_path.unlink()
|
|
return 0
|
|
|
|
# Count records
|
|
async with engine.connect() as conn:
|
|
result = await conn.execute(text("SELECT COUNT(*) FROM cards"))
|
|
count = result.scalar()
|
|
print(f"✓ Imported {count:,} cards")
|
|
|
|
# Clean up
|
|
psql_path.unlink()
|
|
return count
|
|
|
|
|
|
async def import_all_set_files(engine: create_async_engine, set_files_dir: Path) -> tuple[int, int]:
|
|
"""Import AllSetFiles.zip - sets and cards."""
|
|
if not set_files_dir.exists():
|
|
print("✗ AllSetFiles directory not found")
|
|
return 0, 0
|
|
|
|
set_count = 0
|
|
card_count = 0
|
|
|
|
async with engine.begin() as conn:
|
|
# Import sets
|
|
for set_file in sorted(set_files_dir.glob('*.json')):
|
|
data = json.loads(set_file.read_text())
|
|
|
|
if 'code' in data:
|
|
image_url = None
|
|
if 'image' in data and data['image']:
|
|
image_url = data['image'].get('normal')
|
|
|
|
# Upsert set
|
|
result = await conn.execute(
|
|
pg_insert(mtg_sets_table).values(
|
|
code=data.get('code'),
|
|
name=data.get('name'),
|
|
type=data.get('type'),
|
|
release_date=data.get('releaseDate'),
|
|
base_set_size=data.get('baseSetSize'),
|
|
total_size=data.get('totalSetSize'),
|
|
is_foil_only=data.get('isFoilOnly'),
|
|
is_non_foil_only=data.get('isNonFoilOnly'),
|
|
digital=data.get('digital'),
|
|
icon_svg_url=data.get('iconSvgUrl'),
|
|
parent_code=data.get('parentCode'),
|
|
mtgo_code=data.get('mtgoCode'),
|
|
card_count=data.get('cardCount'),
|
|
image_url=image_url,
|
|
).on_conflict_do_update(
|
|
index_elements=['code'],
|
|
set_={
|
|
'name': data.get('name'),
|
|
'type': data.get('type'),
|
|
'release_date': data.get('releaseDate'),
|
|
'base_set_size': data.get('baseSetSize'),
|
|
'total_size': data.get('totalSetSize'),
|
|
'is_foil_only': data.get('isFoilOnly'),
|
|
'is_non_foil_only': data.get('isNonFoilOnly'),
|
|
'digital': data.get('digital'),
|
|
'icon_svg_url': data.get('iconSvgUrl'),
|
|
'parent_code': data.get('parentCode'),
|
|
'mtgo_code': data.get('mtgoCode'),
|
|
'card_count': data.get('cardCount'),
|
|
'image_url': image_url,
|
|
'updated_at': datetime.utcnow(),
|
|
}
|
|
).returning(mtg_sets_table.id),
|
|
execution_options={"autocommit": True}
|
|
)
|
|
set_id = result.scalar()
|
|
|
|
if 'cards' in data:
|
|
for card_data in data['cards']:
|
|
identifiers = {
|
|
'multiId': card_data.get('multiverseIds'),
|
|
'tcgplayerProductId': card_data.get('tcgplayerProductId'),
|
|
'cardmarketId': card_data.get('cardmarketId'),
|
|
}
|
|
|
|
images = {}
|
|
if 'image_uris' in card_data:
|
|
images = {
|
|
'small': card_data['image_uris'].get('small'),
|
|
'normal': card_data['image_uris'].get('normal'),
|
|
'large': card_data['image_uris'].get('large'),
|
|
'png': card_data['image_uris'].get('png'),
|
|
'art_crop': card_data['image_uris'].get('art_crop'),
|
|
}
|
|
|
|
# Upsert card
|
|
await conn.execute(
|
|
pg_insert(mtg_cards_table).values(
|
|
set_id=set_id,
|
|
name=card_data.get('name'),
|
|
mana_cost=card_data.get('manaCost'),
|
|
type_line=card_data.get('typeLine'),
|
|
oracle_text=card_data.get('oracleText'),
|
|
power=card_data.get('power'),
|
|
toughness=card_data.get('toughness'),
|
|
rarity=card_data.get('rarity'),
|
|
layout=card_data.get('layout'),
|
|
artist=card_data.get('artist'),
|
|
flavor_text=card_data.get('flavorText'),
|
|
numbers=str(card_data.get('number')),
|
|
identifiers=json.dumps(identifiers),
|
|
images=json.dumps(images),
|
|
).on_conflict_do_nothing(),
|
|
execution_options={"autocommit": True}
|
|
)
|
|
card_count += 1
|
|
|
|
set_count += 1
|
|
|
|
print(f"✓ Imported {set_count} sets and {card_count} cards from sets")
|
|
return set_count, card_count
|
|
|
|
|
|
async def import_all_identifiers(engine: create_async_engine, file_path: Path) -> int:
|
|
"""Import AllIdentifiers.json.gz."""
|
|
if not file_path.exists():
|
|
print("✗ AllIdentifiers.json.gz not found")
|
|
return 0
|
|
|
|
data = json.loads(gzip.decompress(file_path.read_bytes()))
|
|
|
|
count = 0
|
|
async with engine.begin() as conn:
|
|
for uuid, card_data in data.items():
|
|
await conn.execute(
|
|
pg_insert(card_identifiers_table).values(
|
|
uuid=uuid,
|
|
name=card_data.get('name'),
|
|
mana_cost=card_data.get('manaCost'),
|
|
type_line=card_data.get('typeLine'),
|
|
oracle_text=card_data.get('oracleText'),
|
|
power=card_data.get('power'),
|
|
toughness=card_data.get('toughness'),
|
|
rarity=card_data.get('rarity'),
|
|
layout=card_data.get('layout'),
|
|
artist=card_data.get('artist'),
|
|
flavor_text=card_data.get('flavorText'),
|
|
set_code=card_data.get('setCode'),
|
|
).on_conflict_do_update(
|
|
index_elements=['uuid'],
|
|
set_={
|
|
'name': card_data.get('name'),
|
|
'mana_cost': card_data.get('manaCost'),
|
|
'type_line': card_data.get('typeLine'),
|
|
'oracle_text': card_data.get('oracleText'),
|
|
'power': card_data.get('power'),
|
|
'toughness': card_data.get('toughness'),
|
|
'rarity': card_data.get('rarity'),
|
|
'layout': card_data.get('layout'),
|
|
'artist': card_data.get('artist'),
|
|
'flavor_text': card_data.get('flavorText'),
|
|
'set_code': card_data.get('setCode'),
|
|
'updated_at': datetime.utcnow(),
|
|
}
|
|
),
|
|
execution_options={"autocommit": True}
|
|
)
|
|
count += 1
|
|
|
|
print(f"✓ Imported {count} identifiers")
|
|
return count
|
|
|
|
|
|
async def import_all_deck_files(engine: create_async_engine, deck_files_dir: Path) -> int:
|
|
"""Import AllDeckFiles.zip."""
|
|
if not deck_files_dir.exists():
|
|
print("✗ AllDeckFiles directory not found")
|
|
return 0
|
|
|
|
count = 0
|
|
async with engine.begin() as conn:
|
|
for deck_file in sorted(deck_files_dir.glob('*.json')):
|
|
data = json.loads(deck_file.read_text())
|
|
|
|
if 'name' in data and 'cards' in data:
|
|
await conn.execute(
|
|
pg_insert(decks_table).values(
|
|
name=data.get('name'),
|
|
description=data.get('description'),
|
|
format=data.get('format'),
|
|
command=data.get('command'),
|
|
commander=data.get('commander'),
|
|
).on_conflict_do_update(
|
|
index_elements=['name'],
|
|
set_={
|
|
'description': data.get('description'),
|
|
'format': data.get('format'),
|
|
'command': data.get('command'),
|
|
'commander': data.get('commander'),
|
|
'updated_at': datetime.utcnow(),
|
|
}
|
|
),
|
|
execution_options={"autocommit": True}
|
|
)
|
|
count += 1
|
|
|
|
print(f"✓ Imported {count} decks")
|
|
return count
|
|
|
|
|
|
async def import_json_files(engine: create_async_engine, file_path: Path,
|
|
table_name: str, name_field: str, id_field: str) -> int:
|
|
"""Import a JSON.gz file into a table."""
|
|
if not file_path.exists():
|
|
return 0
|
|
|
|
data = json.loads(gzip.decompress(file_path.read_bytes()))
|
|
|
|
count = 0
|
|
async with engine.begin() as conn:
|
|
for item in data:
|
|
values = {k: v for k, v in item.items() if k != id_field}
|
|
await conn.execute(
|
|
pg_insert(text(f'{table_name}_table')).values(values),
|
|
execution_options={"autocommit": True}
|
|
)
|
|
count += 1
|
|
|
|
print(f"✓ Imported {count} items into {table_name}")
|
|
return count
|
|
|
|
|
|
def download_file(url: str, destination: Path) -> bool:
|
|
"""Download a file from URL to destination."""
|
|
try:
|
|
print(f"Downloading {url}...")
|
|
urlretrieve(url, destination)
|
|
size_mb = destination.stat().st_size / (1024 * 1024)
|
|
print(f" ✓ Downloaded to {destination} ({size_mb:.1f} MB)")
|
|
return True
|
|
except Exception as e:
|
|
print(f" ✗ Failed to download {url}: {e}")
|
|
return False
|
|
|
|
|
|
def extract_zip(zip_file: Path, extract_dir: Path) -> None:
|
|
"""Extract a zip file."""
|
|
if not zip_file.exists():
|
|
print(f"✗ {zip_file.name} not found")
|
|
return
|
|
|
|
if extract_dir.exists():
|
|
import shutil
|
|
shutil.rmtree(extract_dir)
|
|
|
|
print(f"Extracting {zip_file.name}...")
|
|
with zipfile.ZipFile(zip_file, 'r') as zip_ref:
|
|
zip_ref.extractall(extract_dir)
|
|
|
|
file_count = len(list(extract_dir.glob('*.json')))
|
|
print(f"✓ Extracted to {extract_dir} ({file_count} files)")
|
|
|
|
|
|
async def main():
|
|
"""Main function to download and import all MTGJSON data."""
|
|
settings = get_settings()
|
|
|
|
# Setup data directory
|
|
data_dir = Path(settings.DATA_DIR) / "mtgjson"
|
|
downloads_dir = data_dir / "downloads"
|
|
downloads_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Create engine
|
|
engine = create_async_engine(settings.MTG_DATABASE_URL)
|
|
|
|
# Create tables
|
|
print("=== Creating Database Tables ===")
|
|
await create_tables(engine)
|
|
|
|
# Download AllPrintings.psql.gz
|
|
print("\n=== Downloading AllPrintings ===")
|
|
psql_file = downloads_dir / "AllPrintings.psql.gz"
|
|
if not psql_file.exists():
|
|
download_file(f"{MTGJSON_API_V5}/AllPrintings.psql.gz", psql_file)
|
|
|
|
# Import AllPrintings
|
|
await import_all_printings_psql(engine, psql_file)
|
|
|
|
# Download and extract AllSetFiles
|
|
print("\n=== Importing AllSetFiles ===")
|
|
set_files_zip = downloads_dir / "AllSetFiles.zip"
|
|
set_files_dir = downloads_dir / "AllSetFiles"
|
|
|
|
if not set_files_dir.exists():
|
|
if not set_files_zip.exists():
|
|
download_file(f"{MTGJSON_API_V5}/AllSetFiles.zip", set_files_zip)
|
|
extract_zip(set_files_zip, set_files_dir)
|
|
|
|
await import_all_set_files(engine, set_files_dir)
|
|
|
|
# Download and extract AllDeckFiles
|
|
print("\n=== Importing AllDeckFiles ===")
|
|
deck_files_zip = downloads_dir / "AllDeckFiles.zip"
|
|
deck_files_dir = downloads_dir / "AllDeckFiles"
|
|
|
|
if not deck_files_dir.exists():
|
|
if not deck_files_zip.exists():
|
|
download_file(f"{MTGJSON_API_V5}/AllDeckFiles.zip", deck_files_zip)
|
|
extract_zip(deck_files_zip, deck_files_dir)
|
|
|
|
await import_all_deck_files(engine, deck_files_dir)
|
|
|
|
# Download and import AllIdentifiers
|
|
print("\n=== Importing AllIdentifiers ===")
|
|
identifiers_file = downloads_dir / "AllIdentifiers.json.gz"
|
|
if not identifiers_file.exists():
|
|
download_file(f"{MTGJSON_API_V5}/AllIdentifiers.json.gz", identifiers_file)
|
|
|
|
await import_all_identifiers(engine, identifiers_file)
|
|
|
|
# Download and import CardTypes
|
|
print("\n=== Importing CardTypes ===")
|
|
card_types_file = downloads_dir / "CardTypes.json.gz"
|
|
if not card_types_file.exists():
|
|
download_file(f"{MTGJSON_API_V5}/CardTypes.json.gz", card_types_file)
|
|
|
|
# Import CardTypes (simplified)
|
|
if card_types_file.exists():
|
|
data = json.loads(gzip.decompress(card_types_file.read_bytes()))
|
|
async with engine.begin() as conn:
|
|
count = 0
|
|
for card_type in data:
|
|
await conn.execute(
|
|
pg_insert(card_types_table).values(
|
|
type=card_type.get('type'),
|
|
description=card_type.get('description'),
|
|
).on_conflict_do_nothing(),
|
|
execution_options={"autocommit": True}
|
|
)
|
|
count += 1
|
|
print(f"✓ Imported {count} card types")
|
|
|
|
# Download and import DeckList
|
|
print("\n=== Importing DeckList ===")
|
|
deck_list_file = downloads_dir / "DeckList.json.gz"
|
|
if not deck_list_file.exists():
|
|
download_file(f"{MTGJSON_API_V5}/DeckList.json.gz", deck_list_file)
|
|
|
|
# Import DeckList (simplified)
|
|
if deck_list_file.exists():
|
|
data = json.loads(gzip.decompress(deck_list_file.read_bytes()))
|
|
async with engine.begin() as conn:
|
|
count = 0
|
|
for deck in data:
|
|
await conn.execute(
|
|
pg_insert(deck_list_table).values(
|
|
deck_id=deck.get('id'),
|
|
name=deck.get('name'),
|
|
description=deck.get('description'),
|
|
format=deck.get('format'),
|
|
command=deck.get('command'),
|
|
commander=deck.get('commander'),
|
|
total_cards=deck.get('totalCards'),
|
|
).on_conflict_do_nothing(),
|
|
execution_options={"autocommit": True}
|
|
)
|
|
count += 1
|
|
print(f"✓ Imported {count} deck list entries")
|
|
|
|
# Download and import Keywords
|
|
print("\n=== Importing Keywords ===")
|
|
keywords_file = downloads_dir / "Keywords.json.gz"
|
|
if not keywords_file.exists():
|
|
download_file(f"{MTGJSON_API_V5}/Keywords.json.gz", keywords_file)
|
|
|
|
# Import Keywords (simplified)
|
|
if keywords_file.exists():
|
|
data = json.loads(gzip.decompress(keywords_file.read_bytes()))
|
|
async with engine.begin() as conn:
|
|
count = 0
|
|
for keyword in data:
|
|
await conn.execute(
|
|
pg_insert(card_keywords_table).values(
|
|
keyword=keyword.get('keyword'),
|
|
description=keyword.get('description'),
|
|
).on_conflict_do_nothing(),
|
|
execution_options={"autocommit": True}
|
|
)
|
|
count += 1
|
|
print(f"✓ Imported {count} keywords")
|
|
|
|
# Download and import SetList
|
|
print("\n=== Importing SetList ===")
|
|
set_list_file = downloads_dir / "SetList.json.gz"
|
|
if not set_list_file.exists():
|
|
download_file(f"{MTGJSON_API_V5}/SetList.json.gz", set_list_file)
|
|
|
|
# Import SetList (simplified)
|
|
if set_list_file.exists():
|
|
data = json.loads(gzip.decompress(set_list_file.read_bytes()))
|
|
async with engine.begin() as conn:
|
|
count = 0
|
|
for set_data in data:
|
|
await conn.execute(
|
|
pg_insert(set_list_table).values(
|
|
set_code=set_data.get('code'),
|
|
set_name=set_data.get('name'),
|
|
set_type=set_data.get('type'),
|
|
release_date=set_data.get('releaseDate'),
|
|
base_set_size=set_data.get('baseSetSize'),
|
|
total_size=set_data.get('totalSetSize'),
|
|
).on_conflict_do_nothing(),
|
|
execution_options={"autocommit": True}
|
|
)
|
|
count += 1
|
|
print(f"✓ Imported {count} set list entries")
|
|
|
|
# Create indexes
|
|
print("\n=== Creating Indexes ===")
|
|
async with engine.begin() as conn:
|
|
indexes = [
|
|
"CREATE INDEX IF NOT EXISTS idx_cards_name ON cards(name)",
|
|
"CREATE INDEX IF NOT EXISTS idx_cards_mana_cost ON cards(manaCost)",
|
|
"CREATE INDEX IF NOT EXISTS idx_cards_type ON cards(type)",
|
|
"CREATE INDEX IF NOT EXISTS idx_cards_rarity ON cards(rarity)",
|
|
"CREATE INDEX IF NOT EXISTS idx_cards_set_code ON cards(setCode)",
|
|
"CREATE INDEX IF NOT EXISTS idx_cards_uuid ON cards(uuid)",
|
|
"CREATE INDEX IF NOT EXISTS idx_mtg_sets_code ON mtg_sets(code)",
|
|
"CREATE INDEX IF NOT EXISTS idx_mtg_cards_name ON mtg_cards(name)",
|
|
"CREATE INDEX IF NOT EXISTS idx_card_identifiers_uuid ON card_identifiers(uuid)",
|
|
"CREATE INDEX IF NOT EXISTS idx_card_identifiers_name ON card_identifiers(name)",
|
|
"CREATE INDEX IF NOT EXISTS idx_deck_list_deck_id ON deck_list(deck_id)",
|
|
]
|
|
|
|
for idx in indexes:
|
|
await conn.execute(text(idx))
|
|
|
|
await conn.commit()
|
|
print("✓ Indexes created")
|
|
|
|
# Show summary
|
|
print("\n=== Import Summary ===")
|
|
async with engine.connect() as conn:
|
|
tables = [
|
|
'cards', 'mtg_sets', 'mtg_cards', 'card_identifiers',
|
|
'decks', 'card_types', 'deck_list', 'card_keywords', 'set_list'
|
|
]
|
|
|
|
for table in tables:
|
|
result = await conn.execute(text(f"SELECT COUNT(*) FROM {table}"))
|
|
count = result.scalar()
|
|
print(f" {table:20} {count:>10,} records")
|
|
|
|
await engine.dispose()
|
|
print("\n✓ All MTGJSON data imported successfully!")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|