822 lines
29 KiB
Python
822 lines
29 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
MTGJSON Complete Data Import Script
|
|
|
|
Downloads and imports all MTGJSON data into PostgreSQL:
|
|
- AllPrintings.psql (main cards database)
|
|
- AllSetFiles (set and card data)
|
|
- AllDeckFiles (deck data)
|
|
- AllIdentifiers (card identifiers)
|
|
- CardTypes (card types)
|
|
- DeckList (deck list metadata)
|
|
- Keywords (card keywords)
|
|
- SetList (set list metadata)
|
|
|
|
This script is designed to run inside the Docker container
|
|
where SQLAlchemy and other dependencies are installed.
|
|
"""
|
|
|
|
import asyncio
|
|
import gzip
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import zipfile
|
|
from pathlib import Path
|
|
|
|
# Configuration
|
|
MTGDATA_DIR = Path('/app/mtgdata')
|
|
TEMP_DIR = MTGDATA_DIR / 'temp'
|
|
BASE_URL = 'https://mtgjson.com/api/v5'
|
|
|
|
FILES = [
|
|
'AllPrintings.psql.gz',
|
|
'AllSetFiles.zip',
|
|
'AllDeckFiles.zip',
|
|
'AllIdentifiers.json.gz',
|
|
'CardTypes.json.gz',
|
|
'DeckList.json.gz',
|
|
'Keywords.json.gz',
|
|
'SetList.json.gz'
|
|
]
|
|
|
|
|
|
def check_files():
|
|
"""Verify all files are downloaded."""
|
|
print("=== Checking Downloaded Files ===\n")
|
|
|
|
all_found = True
|
|
for file in FILES:
|
|
file_path = MTGDATA_DIR / file
|
|
if file_path.exists():
|
|
size_mb = file_path.stat().st_size / (1024 * 1024)
|
|
print(f"✓ {file} ({size_mb:.1f} MB)")
|
|
else:
|
|
print(f"✗ {file} NOT FOUND")
|
|
all_found = False
|
|
|
|
if all_found:
|
|
total_size = sum(
|
|
(MTGDATA_DIR / file).stat().st_size for file in FILES
|
|
) / (1024 * 1024)
|
|
print(f"\n✓ All files present ({total_size:.1f} MB total)")
|
|
else:
|
|
print("\n⚠ Some files missing. Run download_mtgdata.sh first.")
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def extract_files():
|
|
"""Extract zip files."""
|
|
print("\n=== Extracting Zip Files ===\n")
|
|
|
|
zip_files = {
|
|
'AllSetFiles.zip': 'AllSetFiles',
|
|
'AllDeckFiles.zip': 'AllDeckFiles'
|
|
}
|
|
|
|
for zip_file, extract_dir in zip_files.items():
|
|
zip_path = MTGDATA_DIR / zip_file
|
|
extract_path = MTGDATA_DIR / extract_dir
|
|
|
|
if zip_path.exists():
|
|
if extract_path.exists():
|
|
shutil.rmtree(extract_path)
|
|
|
|
print(f"Extracting {zip_file}...")
|
|
with zipfile.ZipFile(zip_path, 'r') as zip_ref:
|
|
zip_ref.extractall(MTGDATA_DIR)
|
|
|
|
file_count = len(list(extract_path.glob('*.json')))
|
|
print(f"✓ Extracted to {extract_dir}/ ({file_count} files)\n")
|
|
|
|
|
|
async def create_database_schema(engine):
|
|
"""Create all required database tables."""
|
|
print("=== Creating Database Schema ===\n")
|
|
|
|
async with engine.connect() as conn:
|
|
# Cards table (from AllPrintings)
|
|
await conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS cards (
|
|
id SERIAL PRIMARY KEY,
|
|
artist TEXT,
|
|
asciiName TEXT,
|
|
attractionLights TEXT,
|
|
availability TEXT,
|
|
boosterTypes TEXT,
|
|
borderColor TEXT,
|
|
cardParts TEXT,
|
|
colorIdentity TEXT,
|
|
colorIndicator TEXT,
|
|
colors TEXT,
|
|
defense TEXT,
|
|
duelDeck TEXT,
|
|
edhrecRank INTEGER,
|
|
edhrecSaltiness FLOAT,
|
|
faceConvertedManaCost FLOAT,
|
|
faceFlavorName TEXT,
|
|
faceManaValue FLOAT,
|
|
faceName TEXT,
|
|
facePrintedName TEXT,
|
|
finishes TEXT,
|
|
flavorName TEXT,
|
|
flavorText TEXT,
|
|
frameEffects TEXT,
|
|
frameVersion TEXT,
|
|
hand TEXT,
|
|
hasAlternativeDeckLimit BOOLEAN,
|
|
hasContentWarning BOOLEAN,
|
|
isAlternative BOOLEAN,
|
|
isFullArt BOOLEAN,
|
|
isFunny BOOLEAN,
|
|
isGameChanger BOOLEAN,
|
|
isOnlineOnly BOOLEAN,
|
|
isOversized BOOLEAN,
|
|
isPromo BOOLEAN,
|
|
isRebalanced BOOLEAN,
|
|
isReprint BOOLEAN,
|
|
isReserved BOOLEAN,
|
|
isStorySpotlight BOOLEAN,
|
|
isTextless BOOLEAN,
|
|
isTimeshifted BOOLEAN,
|
|
keywords TEXT,
|
|
language TEXT,
|
|
layout TEXT,
|
|
leadershipSkills TEXT,
|
|
life TEXT,
|
|
loyalty TEXT,
|
|
manaCost TEXT,
|
|
manaValue FLOAT,
|
|
name TEXT,
|
|
number TEXT,
|
|
originalPrintings TEXT,
|
|
originalReleaseDate TEXT,
|
|
originalText TEXT,
|
|
otherFaceIds TEXT,
|
|
power TEXT,
|
|
printedName TEXT,
|
|
printedText TEXT,
|
|
printedType TEXT,
|
|
printings TEXT,
|
|
producedMana TEXT,
|
|
promoTypes TEXT,
|
|
rarity TEXT,
|
|
rebalancedPrintings TEXT,
|
|
relatedCards TEXT,
|
|
securityStamp TEXT,
|
|
setCode TEXT,
|
|
side TEXT,
|
|
signature TEXT,
|
|
skuIds TEXT,
|
|
sourceProducts TEXT,
|
|
subsets TEXT,
|
|
subtypes TEXT,
|
|
supertypes TEXT,
|
|
text TEXT,
|
|
toughness TEXT,
|
|
type TEXT,
|
|
types TEXT,
|
|
uuid TEXT,
|
|
variations TEXT,
|
|
watermark TEXT,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
""")
|
|
|
|
# MTG Sets table
|
|
await conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS mtg_sets (
|
|
id SERIAL PRIMARY KEY,
|
|
code VARCHAR(10) UNIQUE NOT NULL,
|
|
name VARCHAR(255),
|
|
type VARCHAR(100),
|
|
release_date DATE,
|
|
base_set_size INTEGER,
|
|
total_size INTEGER,
|
|
is_foil_only BOOLEAN,
|
|
is_non_foil_only BOOLEAN,
|
|
digital BOOLEAN,
|
|
icon_svg_url TEXT,
|
|
parent_code VARCHAR(10),
|
|
mtgo_code VARCHAR(10),
|
|
image_url TEXT,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
""")
|
|
|
|
# MTG Cards table (from set files)
|
|
await conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS mtg_cards (
|
|
id SERIAL PRIMARY KEY,
|
|
set_id INTEGER REFERENCES mtg_sets(id) ON DELETE CASCADE,
|
|
name VARCHAR(255),
|
|
mana_cost VARCHAR(255),
|
|
type_line VARCHAR(255),
|
|
oracle_text TEXT,
|
|
power VARCHAR(50),
|
|
toughness VARCHAR(50),
|
|
rarity VARCHAR(50),
|
|
layout VARCHAR(50),
|
|
artist VARCHAR(255),
|
|
flavor_text TEXT,
|
|
numbers VARCHAR(100),
|
|
identifiers TEXT,
|
|
images TEXT,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
""")
|
|
|
|
# Card Identifiers table
|
|
await conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS card_identifiers (
|
|
id SERIAL PRIMARY KEY,
|
|
uuid TEXT UNIQUE NOT NULL,
|
|
name VARCHAR(255),
|
|
mana_cost VARCHAR(255),
|
|
type_line VARCHAR(255),
|
|
oracle_text TEXT,
|
|
power VARCHAR(50),
|
|
toughness VARCHAR(50),
|
|
rarity VARCHAR(50),
|
|
layout VARCHAR(50),
|
|
artist VARCHAR(255),
|
|
flavor_text TEXT,
|
|
set_code VARCHAR(10),
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
""")
|
|
|
|
# Decks table
|
|
await conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS decks (
|
|
id SERIAL PRIMARY KEY,
|
|
name VARCHAR(255) NOT NULL,
|
|
description TEXT,
|
|
format VARCHAR(50),
|
|
command TEXT,
|
|
commander TEXT,
|
|
creation_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
""")
|
|
|
|
# Card Types table
|
|
await conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS card_types (
|
|
id SERIAL PRIMARY KEY,
|
|
type VARCHAR(100) UNIQUE NOT NULL,
|
|
description TEXT,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
""")
|
|
|
|
# Deck List table
|
|
await conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS deck_list (
|
|
id SERIAL PRIMARY KEY,
|
|
deck_id VARCHAR(100) UNIQUE NOT NULL,
|
|
name VARCHAR(255),
|
|
description TEXT,
|
|
format VARCHAR(50),
|
|
command TEXT,
|
|
commander TEXT,
|
|
total_cards INTEGER,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
""")
|
|
|
|
# Card Keywords table
|
|
await conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS card_keywords (
|
|
id SERIAL PRIMARY KEY,
|
|
keyword VARCHAR(100) UNIQUE NOT NULL,
|
|
description TEXT,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
""")
|
|
|
|
# Set List table
|
|
await conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS set_list (
|
|
id SERIAL PRIMARY KEY,
|
|
set_code VARCHAR(10) UNIQUE NOT NULL,
|
|
set_name VARCHAR(255),
|
|
set_type VARCHAR(100),
|
|
release_date DATE,
|
|
base_set_size INTEGER,
|
|
total_size INTEGER,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
""")
|
|
|
|
await conn.commit()
|
|
print("✓ Database schema created\n")
|
|
|
|
|
|
async def import_all_printings_psql(engine):
|
|
"""Import AllPrintings.psql.gz file using psql command."""
|
|
print("=== Importing AllPrintings.psql.gz ===\n")
|
|
|
|
psql_file = MTGDATA_DIR / 'AllPrintings.psql.gz'
|
|
|
|
if not psql_file.exists():
|
|
print("✗ AllPrintings.psql.gz not found\n")
|
|
return
|
|
|
|
print("Decompressing...")
|
|
psql_content = gzip.decompress(psql_file.read_bytes())
|
|
psql_path = MTGDATA_DIR / 'AllPrintings.psql'
|
|
psql_path.write_bytes(psql_content)
|
|
|
|
print("Importing into PostgreSQL...")
|
|
|
|
# Import using psql command
|
|
# Use environment variables to get database connection info
|
|
db_user = os.environ.get('POSTGRES_USER', 'cockatrice')
|
|
db_password = os.environ.get('POSTGRES_PASSWORD', 'cockatrice_pass')
|
|
db_host = os.environ.get('POSTGRES_HOST', 'mtgonline_postgres_mtgdata')
|
|
db_port = os.environ.get('POSTGRES_PORT', '5432')
|
|
db_name = os.environ.get('POSTGRES_DB', 'mtgdata')
|
|
|
|
cmd = [
|
|
'psql', '-h', db_host, '-p', db_port, '-U', db_user, '-d', db_name,
|
|
'-f', str(psql_path)
|
|
]
|
|
|
|
result = subprocess.run(cmd, capture_output=True, text=True)
|
|
|
|
if result.returncode != 0:
|
|
print(f"✗ Import failed: {result.stderr[:500]}\n")
|
|
return
|
|
|
|
# Count records
|
|
async with engine.connect() as conn:
|
|
result = await conn.execute("SELECT COUNT(*) FROM cards")
|
|
count = result.scalar()
|
|
print(f"✓ Imported {count:,} cards\n")
|
|
|
|
# Clean up
|
|
psql_path.unlink()
|
|
|
|
|
|
async def import_all_set_files(engine):
|
|
"""Import AllSetFiles.zip - sets and cards."""
|
|
print("=== Importing AllSetFiles ===\n")
|
|
|
|
extract_dir = MTGDATA_DIR / 'AllSetFiles'
|
|
|
|
if not extract_dir.exists():
|
|
print("✗ AllSetFiles directory not found\n")
|
|
return
|
|
|
|
async with engine.connect() as conn:
|
|
# Import sets
|
|
set_count = 0
|
|
for set_file in extract_dir.glob('*.json'):
|
|
data = json.loads(set_file.read_text())
|
|
|
|
if 'code' in data:
|
|
image_url = None
|
|
if 'image' in data and data['image']:
|
|
image_url = data['image'].get('normal')
|
|
|
|
await conn.execute("""
|
|
INSERT INTO mtg_sets (code, name, type, release_date,
|
|
base_set_size, total_size, is_foil_only,
|
|
is_non_foil_only, digital, icon_svg_url,
|
|
parent_code, mtgo_code, image_url)
|
|
VALUES (:code, :name, :type, :release_date,
|
|
:base_set_size, :total_size, :is_foil_only,
|
|
:is_non_foil_only, :digital, :icon_svg_url,
|
|
:parent_code, :mtgo_code, :image_url)
|
|
ON CONFLICT (code) DO UPDATE SET
|
|
name = EXCLUDED.name, type = EXCLUDED.type,
|
|
release_date = EXCLUDED.release_date,
|
|
base_set_size = EXCLUDED.base_set_size,
|
|
total_size = EXCLUDED.total_size,
|
|
is_foil_only = EXCLUDED.is_foil_only,
|
|
is_non_foil_only = EXCLUDED.is_non_foil_only,
|
|
digital = EXCLUDED.digital,
|
|
icon_svg_url = EXCLUDED.icon_svg_url,
|
|
parent_code = EXCLUDED.parent_code,
|
|
mtgo_code = EXCLUDED.mtgo_code,
|
|
image_url = EXCLUDED.image_url
|
|
""", {
|
|
'code': data.get('code'),
|
|
'name': data.get('name'),
|
|
'type': data.get('type'),
|
|
'release_date': data.get('releaseDate'),
|
|
'base_set_size': data.get('baseSetSize'),
|
|
'total_size': data.get('totalSetSize'),
|
|
'is_foil_only': data.get('isFoilOnly'),
|
|
'is_non_foil_only': data.get('isNonFoilOnly'),
|
|
'digital': data.get('digital'),
|
|
'icon_svg_url': data.get('iconSvgUrl'),
|
|
'parent_code': data.get('parentCode'),
|
|
'mtgo_code': data.get('mtgoCode'),
|
|
'image_url': image_url,
|
|
})
|
|
set_count += 1
|
|
|
|
await conn.commit()
|
|
print(f"✓ Imported {set_count} sets")
|
|
|
|
# Import cards
|
|
card_count = 0
|
|
for set_file in extract_dir.glob('*.json'):
|
|
data = json.loads(set_file.read_text())
|
|
|
|
if 'cards' in data:
|
|
set_id = (await conn.execute(
|
|
"SELECT id FROM mtg_sets WHERE code = :code",
|
|
{'code': data['code']}
|
|
)).scalar()
|
|
|
|
if set_id:
|
|
for card_data in data['cards']:
|
|
identifiers = {
|
|
'multiId': card_data.get('multiverseIds'),
|
|
'tcgplayerProductId': card_data.get('tcgplayerProductId'),
|
|
'cardmarketId': card_data.get('cardmarketId'),
|
|
}
|
|
|
|
images = {}
|
|
if 'image_uris' in card_data:
|
|
images = {
|
|
'small': card_data['image_uris'].get('small'),
|
|
'normal': card_data['image_uris'].get('normal'),
|
|
'large': card_data['image_uris'].get('large'),
|
|
'png': card_data['image_uris'].get('png'),
|
|
'art_crop': card_data['image_uris'].get('art_crop'),
|
|
}
|
|
|
|
await conn.execute("""
|
|
INSERT INTO mtg_cards (set_id, name, mana_cost,
|
|
type_line, oracle_text, power,
|
|
toughness, rarity, layout, artist,
|
|
flavor_text, numbers, identifiers, images)
|
|
VALUES (:set_id, :name, :mana_cost, :type_line,
|
|
:oracle_text, :power, :toughness, :rarity,
|
|
:layout, :artist, :flavor_text, :numbers,
|
|
:identifiers, :images)
|
|
""", {
|
|
'set_id': set_id,
|
|
'name': card_data.get('name'),
|
|
'mana_cost': card_data.get('manaCost'),
|
|
'type_line': card_data.get('typeLine'),
|
|
'oracle_text': card_data.get('oracleText'),
|
|
'power': card_data.get('power'),
|
|
'toughness': card_data.get('toughness'),
|
|
'rarity': card_data.get('rarity'),
|
|
'layout': card_data.get('layout'),
|
|
'artist': card_data.get('artist'),
|
|
'flavor_text': card_data.get('flavorText'),
|
|
'numbers': str(card_data.get('number')),
|
|
'identifiers': json.dumps(identifiers),
|
|
'images': json.dumps(images),
|
|
})
|
|
card_count += 1
|
|
|
|
await conn.commit()
|
|
print(f"✓ Imported {card_count} cards from sets\n")
|
|
|
|
|
|
async def import_all_deck_files(engine):
|
|
"""Import AllDeckFiles.zip."""
|
|
print("=== Importing AllDeckFiles ===\n")
|
|
|
|
extract_dir = MTGDATA_DIR / 'AllDeckFiles'
|
|
|
|
if not extract_dir.exists():
|
|
print("✗ AllDeckFiles directory not found\n")
|
|
return
|
|
|
|
async with engine.connect() as conn:
|
|
deck_count = 0
|
|
for deck_file in extract_dir.glob('*.json'):
|
|
data = json.loads(deck_file.read_text())
|
|
|
|
if 'name' in data and 'cards' in data:
|
|
await conn.execute("""
|
|
INSERT INTO decks (name, description, format, command, commander)
|
|
VALUES (:name, :description, :format, :command, :commander)
|
|
ON CONFLICT (name) DO UPDATE SET
|
|
description = EXCLUDED.description,
|
|
format = EXCLUDED.format,
|
|
command = EXCLUDED.command,
|
|
commander = EXCLUDED.commander
|
|
""", {
|
|
'name': data.get('name'),
|
|
'description': data.get('description'),
|
|
'format': data.get('format'),
|
|
'command': data.get('command'),
|
|
'commander': data.get('commander'),
|
|
})
|
|
deck_count += 1
|
|
|
|
await conn.commit()
|
|
print(f"✓ Imported {deck_count} decks\n")
|
|
|
|
|
|
async def import_all_identifiers(engine):
|
|
"""Import AllIdentifiers.json.gz."""
|
|
print("=== Importing AllIdentifiers ===\n")
|
|
|
|
file_path = MTGDATA_DIR / 'AllIdentifiers.json.gz'
|
|
|
|
if not file_path.exists():
|
|
print("✗ File not found\n")
|
|
return
|
|
|
|
data = json.loads(gzip.decompress(file_path.read_bytes()))
|
|
|
|
async with engine.connect() as conn:
|
|
identifier_count = 0
|
|
for uuid, card_data in data.items():
|
|
await conn.execute("""
|
|
INSERT INTO card_identifiers (uuid, name, mana_cost, type_line,
|
|
oracle_text, power, toughness, rarity,
|
|
layout, artist, flavor_text, set_code)
|
|
VALUES (:uuid, :name, :mana_cost, :type_line, :oracle_text,
|
|
:power, :toughness, :rarity, :layout, :artist,
|
|
:flavor_text, :set_code)
|
|
ON CONFLICT (uuid) DO UPDATE SET
|
|
name = EXCLUDED.name, mana_cost = EXCLUDED.mana_cost,
|
|
type_line = EXCLUDED.type_line,
|
|
oracle_text = EXCLUDED.oracle_text,
|
|
power = EXCLUDED.power, toughness = EXCLUDED.toughness,
|
|
rarity = EXCLUDED.rarity, layout = EXCLUDED.layout,
|
|
artist = EXCLUDED.artist,
|
|
flavor_text = EXCLUDED.flavor_text,
|
|
set_code = EXCLUDED.set_code
|
|
""", {
|
|
'uuid': uuid,
|
|
'name': card_data.get('name'),
|
|
'mana_cost': card_data.get('manaCost'),
|
|
'type_line': card_data.get('typeLine'),
|
|
'oracle_text': card_data.get('oracleText'),
|
|
'power': card_data.get('power'),
|
|
'toughness': card_data.get('toughness'),
|
|
'rarity': card_data.get('rarity'),
|
|
'layout': card_data.get('layout'),
|
|
'artist': card_data.get('artist'),
|
|
'flavor_text': card_data.get('flavorText'),
|
|
'set_code': card_data.get('setCode'),
|
|
})
|
|
identifier_count += 1
|
|
|
|
await conn.commit()
|
|
print(f"✓ Imported {identifier_count} identifiers\n")
|
|
|
|
|
|
async def import_card_types(engine):
|
|
"""Import CardTypes.json.gz."""
|
|
print("=== Importing CardTypes ===\n")
|
|
|
|
file_path = MTGDATA_DIR / 'CardTypes.json.gz'
|
|
|
|
if not file_path.exists():
|
|
print("✗ File not found\n")
|
|
return
|
|
|
|
data = json.loads(gzip.decompress(file_path.read_bytes()))
|
|
|
|
async with engine.connect() as conn:
|
|
type_count = 0
|
|
for card_type in data:
|
|
await conn.execute("""
|
|
INSERT INTO card_types (type, description)
|
|
VALUES (:type, :description)
|
|
ON CONFLICT (type) DO UPDATE SET
|
|
description = EXCLUDED.description
|
|
""", {
|
|
'type': card_type.get('type'),
|
|
'description': card_type.get('description'),
|
|
})
|
|
type_count += 1
|
|
|
|
await conn.commit()
|
|
print(f"✓ Imported {type_count} card types\n")
|
|
|
|
|
|
async def import_deck_list(engine):
|
|
"""Import DeckList.json.gz."""
|
|
print("=== Importing DeckList ===\n")
|
|
|
|
file_path = MTGDATA_DIR / 'DeckList.json.gz'
|
|
|
|
if not file_path.exists():
|
|
print("✗ File not found\n")
|
|
return
|
|
|
|
data = json.loads(gzip.decompress(file_path.read_bytes()))
|
|
|
|
async with engine.connect() as conn:
|
|
deck_list_count = 0
|
|
for deck in data:
|
|
await conn.execute("""
|
|
INSERT INTO deck_list (deck_id, name, description, format,
|
|
command, commander, total_cards)
|
|
VALUES (:deck_id, :name, :description, :format,
|
|
:command, :commander, :total_cards)
|
|
ON CONFLICT (deck_id) DO UPDATE SET
|
|
name = EXCLUDED.name, description = EXCLUDED.description,
|
|
format = EXCLUDED.format, command = EXCLUDED.command,
|
|
commander = EXCLUDED.commander,
|
|
total_cards = EXCLUDED.total_cards
|
|
""", {
|
|
'deck_id': deck.get('id'),
|
|
'name': deck.get('name'),
|
|
'description': deck.get('description'),
|
|
'format': deck.get('format'),
|
|
'command': deck.get('command'),
|
|
'commander': deck.get('commander'),
|
|
'total_cards': deck.get('totalCards'),
|
|
})
|
|
deck_list_count += 1
|
|
|
|
await conn.commit()
|
|
print(f"✓ Imported {deck_list_count} deck list entries\n")
|
|
|
|
|
|
async def import_keywords(engine):
|
|
"""Import Keywords.json.gz."""
|
|
print("=== Importing Keywords ===\n")
|
|
|
|
file_path = MTGDATA_DIR / 'Keywords.json.gz'
|
|
|
|
if not file_path.exists():
|
|
print("✗ File not found\n")
|
|
return
|
|
|
|
data = json.loads(gzip.decompress(file_path.read_bytes()))
|
|
|
|
async with engine.connect() as conn:
|
|
keyword_count = 0
|
|
for keyword in data:
|
|
await conn.execute("""
|
|
INSERT INTO card_keywords (keyword, description)
|
|
VALUES (:keyword, :description)
|
|
ON CONFLICT (keyword) DO UPDATE SET
|
|
description = EXCLUDED.description
|
|
""", {
|
|
'keyword': keyword.get('keyword'),
|
|
'description': keyword.get('description'),
|
|
})
|
|
keyword_count += 1
|
|
|
|
await conn.commit()
|
|
print(f"✓ Imported {keyword_count} keywords\n")
|
|
|
|
|
|
async def import_set_list(engine):
|
|
"""Import SetList.json.gz."""
|
|
print("=== Importing SetList ===\n")
|
|
|
|
file_path = MTGDATA_DIR / 'SetList.json.gz'
|
|
|
|
if not file_path.exists():
|
|
print("✗ File not found\n")
|
|
return
|
|
|
|
data = json.loads(gzip.decompress(file_path.read_bytes()))
|
|
|
|
async with engine.connect() as conn:
|
|
set_list_count = 0
|
|
for set_data in data:
|
|
await conn.execute("""
|
|
INSERT INTO set_list (set_code, set_name, set_type, release_date,
|
|
base_set_size, total_size)
|
|
VALUES (:set_code, :set_name, :set_type, :release_date,
|
|
:base_set_size, :total_size)
|
|
ON CONFLICT (set_code) DO UPDATE SET
|
|
set_name = EXCLUDED.set_name, set_type = EXCLUDED.set_type,
|
|
release_date = EXCLUDED.release_date,
|
|
base_set_size = EXCLUDED.base_set_size,
|
|
total_size = EXCLUDED.total_size
|
|
""", {
|
|
'set_code': set_data.get('code'),
|
|
'set_name': set_data.get('name'),
|
|
'set_type': set_data.get('type'),
|
|
'release_date': set_data.get('releaseDate'),
|
|
'base_set_size': set_data.get('baseSetSize'),
|
|
'total_size': set_data.get('totalSetSize'),
|
|
})
|
|
set_list_count += 1
|
|
|
|
await conn.commit()
|
|
print(f"✓ Imported {set_list_count} set list entries\n")
|
|
|
|
|
|
async def create_indexes(engine):
|
|
"""Create indexes for better performance."""
|
|
print("=== Creating Indexes ===\n")
|
|
|
|
async with engine.connect() as conn:
|
|
indexes = [
|
|
"CREATE INDEX IF NOT EXISTS idx_cards_name ON cards(name)",
|
|
"CREATE INDEX IF NOT EXISTS idx_cards_mana_cost ON cards(manaCost)",
|
|
"CREATE INDEX IF NOT EXISTS idx_cards_type ON cards(type)",
|
|
"CREATE INDEX IF NOT EXISTS idx_cards_rarity ON cards(rarity)",
|
|
"CREATE INDEX IF NOT EXISTS idx_cards_set_code ON cards(setCode)",
|
|
"CREATE INDEX IF NOT EXISTS idx_cards_uuid ON cards(uuid)",
|
|
"CREATE INDEX IF NOT EXISTS idx_mtg_sets_code ON mtg_sets(code)",
|
|
"CREATE INDEX IF NOT EXISTS idx_mtg_cards_name ON mtg_cards(name)",
|
|
"CREATE INDEX IF NOT EXISTS idx_card_identifiers_uuid ON card_identifiers(uuid)",
|
|
"CREATE INDEX IF NOT EXISTS idx_card_identifiers_name ON card_identifiers(name)",
|
|
"CREATE INDEX IF NOT EXISTS idx_deck_list_deck_id ON deck_list(deck_id)",
|
|
]
|
|
|
|
for idx in indexes:
|
|
await conn.execute(idx)
|
|
|
|
await conn.commit()
|
|
print("✓ Indexes created\n")
|
|
|
|
|
|
async def show_summary(engine):
|
|
"""Show import summary."""
|
|
print("=== Import Summary ===\n")
|
|
|
|
async with engine.connect() as conn:
|
|
tables = [
|
|
'cards', 'mtg_sets', 'mtg_cards', 'card_identifiers',
|
|
'decks', 'card_types', 'deck_list', 'card_keywords', 'set_list'
|
|
]
|
|
|
|
for table in tables:
|
|
result = await conn.execute(f"SELECT COUNT(*) FROM {table}")
|
|
count = result.scalar()
|
|
print(f" {table:20} {count:>10,} records")
|
|
|
|
print()
|
|
|
|
|
|
async def main():
|
|
"""Main import function."""
|
|
print("=== MTGJSON Complete Data Import ===\n")
|
|
|
|
# Check files
|
|
if not check_files():
|
|
return
|
|
|
|
# Extract zip files
|
|
extract_files()
|
|
|
|
# Connect to database
|
|
from sqlalchemy.ext.asyncio import create_async_engine
|
|
from app.core.settings import get_settings
|
|
settings = get_settings()
|
|
engine = create_async_engine(settings.MTG_DATABASE_URL)
|
|
|
|
# Create schema
|
|
await create_database_schema(engine)
|
|
|
|
# Import AllPrintings
|
|
await import_all_printings_psql(engine)
|
|
|
|
# Import AllSetFiles
|
|
await import_all_set_files(engine)
|
|
|
|
# Import AllDeckFiles
|
|
await import_all_deck_files(engine)
|
|
|
|
# Import AllIdentifiers
|
|
await import_all_identifiers(engine)
|
|
|
|
# Import CardTypes
|
|
await import_card_types(engine)
|
|
|
|
# Import DeckList
|
|
await import_deck_list(engine)
|
|
|
|
# Import Keywords
|
|
await import_keywords(engine)
|
|
|
|
# Import SetList
|
|
await import_set_list(engine)
|
|
|
|
# Create indexes
|
|
await create_indexes(engine)
|
|
|
|
# Show summary
|
|
await show_summary(engine)
|
|
|
|
print("✓ All data imported successfully!\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|