Final project commit: MTGJSON data integration and backend API
This commit is contained in:
wall-o
@@ -0,0 +1,821 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
MTGJSON Complete Data Import Script
|
||||
|
||||
Downloads and imports all MTGJSON data into PostgreSQL:
|
||||
- AllPrintings.psql (main cards database)
|
||||
- AllSetFiles (set and card data)
|
||||
- AllDeckFiles (deck data)
|
||||
- AllIdentifiers (card identifiers)
|
||||
- CardTypes (card types)
|
||||
- DeckList (deck list metadata)
|
||||
- Keywords (card keywords)
|
||||
- SetList (set list metadata)
|
||||
|
||||
This script is designed to run inside the Docker container
|
||||
where SQLAlchemy and other dependencies are installed.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import gzip
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
# Configuration
|
||||
MTGDATA_DIR = Path('/app/mtgdata')
|
||||
TEMP_DIR = MTGDATA_DIR / 'temp'
|
||||
BASE_URL = 'https://mtgjson.com/api/v5'
|
||||
|
||||
FILES = [
|
||||
'AllPrintings.psql.gz',
|
||||
'AllSetFiles.zip',
|
||||
'AllDeckFiles.zip',
|
||||
'AllIdentifiers.json.gz',
|
||||
'CardTypes.json.gz',
|
||||
'DeckList.json.gz',
|
||||
'Keywords.json.gz',
|
||||
'SetList.json.gz'
|
||||
]
|
||||
|
||||
|
||||
def check_files():
|
||||
"""Verify all files are downloaded."""
|
||||
print("=== Checking Downloaded Files ===\n")
|
||||
|
||||
all_found = True
|
||||
for file in FILES:
|
||||
file_path = MTGDATA_DIR / file
|
||||
if file_path.exists():
|
||||
size_mb = file_path.stat().st_size / (1024 * 1024)
|
||||
print(f"✓ {file} ({size_mb:.1f} MB)")
|
||||
else:
|
||||
print(f"✗ {file} NOT FOUND")
|
||||
all_found = False
|
||||
|
||||
if all_found:
|
||||
total_size = sum(
|
||||
(MTGDATA_DIR / file).stat().st_size for file in FILES
|
||||
) / (1024 * 1024)
|
||||
print(f"\n✓ All files present ({total_size:.1f} MB total)")
|
||||
else:
|
||||
print("\n⚠ Some files missing. Run download_mtgdata.sh first.")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def extract_files():
|
||||
"""Extract zip files."""
|
||||
print("\n=== Extracting Zip Files ===\n")
|
||||
|
||||
zip_files = {
|
||||
'AllSetFiles.zip': 'AllSetFiles',
|
||||
'AllDeckFiles.zip': 'AllDeckFiles'
|
||||
}
|
||||
|
||||
for zip_file, extract_dir in zip_files.items():
|
||||
zip_path = MTGDATA_DIR / zip_file
|
||||
extract_path = MTGDATA_DIR / extract_dir
|
||||
|
||||
if zip_path.exists():
|
||||
if extract_path.exists():
|
||||
shutil.rmtree(extract_path)
|
||||
|
||||
print(f"Extracting {zip_file}...")
|
||||
with zipfile.ZipFile(zip_path, 'r') as zip_ref:
|
||||
zip_ref.extractall(MTGDATA_DIR)
|
||||
|
||||
file_count = len(list(extract_path.glob('*.json')))
|
||||
print(f"✓ Extracted to {extract_dir}/ ({file_count} files)\n")
|
||||
|
||||
|
||||
async def create_database_schema(engine):
|
||||
"""Create all required database tables."""
|
||||
print("=== Creating Database Schema ===\n")
|
||||
|
||||
async with engine.connect() as conn:
|
||||
# Cards table (from AllPrintings)
|
||||
await conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS cards (
|
||||
id SERIAL PRIMARY KEY,
|
||||
artist TEXT,
|
||||
asciiName TEXT,
|
||||
attractionLights TEXT,
|
||||
availability TEXT,
|
||||
boosterTypes TEXT,
|
||||
borderColor TEXT,
|
||||
cardParts TEXT,
|
||||
colorIdentity TEXT,
|
||||
colorIndicator TEXT,
|
||||
colors TEXT,
|
||||
defense TEXT,
|
||||
duelDeck TEXT,
|
||||
edhrecRank INTEGER,
|
||||
edhrecSaltiness FLOAT,
|
||||
faceConvertedManaCost FLOAT,
|
||||
faceFlavorName TEXT,
|
||||
faceManaValue FLOAT,
|
||||
faceName TEXT,
|
||||
facePrintedName TEXT,
|
||||
finishes TEXT,
|
||||
flavorName TEXT,
|
||||
flavorText TEXT,
|
||||
frameEffects TEXT,
|
||||
frameVersion TEXT,
|
||||
hand TEXT,
|
||||
hasAlternativeDeckLimit BOOLEAN,
|
||||
hasContentWarning BOOLEAN,
|
||||
isAlternative BOOLEAN,
|
||||
isFullArt BOOLEAN,
|
||||
isFunny BOOLEAN,
|
||||
isGameChanger BOOLEAN,
|
||||
isOnlineOnly BOOLEAN,
|
||||
isOversized BOOLEAN,
|
||||
isPromo BOOLEAN,
|
||||
isRebalanced BOOLEAN,
|
||||
isReprint BOOLEAN,
|
||||
isReserved BOOLEAN,
|
||||
isStorySpotlight BOOLEAN,
|
||||
isTextless BOOLEAN,
|
||||
isTimeshifted BOOLEAN,
|
||||
keywords TEXT,
|
||||
language TEXT,
|
||||
layout TEXT,
|
||||
leadershipSkills TEXT,
|
||||
life TEXT,
|
||||
loyalty TEXT,
|
||||
manaCost TEXT,
|
||||
manaValue FLOAT,
|
||||
name TEXT,
|
||||
number TEXT,
|
||||
originalPrintings TEXT,
|
||||
originalReleaseDate TEXT,
|
||||
originalText TEXT,
|
||||
otherFaceIds TEXT,
|
||||
power TEXT,
|
||||
printedName TEXT,
|
||||
printedText TEXT,
|
||||
printedType TEXT,
|
||||
printings TEXT,
|
||||
producedMana TEXT,
|
||||
promoTypes TEXT,
|
||||
rarity TEXT,
|
||||
rebalancedPrintings TEXT,
|
||||
relatedCards TEXT,
|
||||
securityStamp TEXT,
|
||||
setCode TEXT,
|
||||
side TEXT,
|
||||
signature TEXT,
|
||||
skuIds TEXT,
|
||||
sourceProducts TEXT,
|
||||
subsets TEXT,
|
||||
subtypes TEXT,
|
||||
supertypes TEXT,
|
||||
text TEXT,
|
||||
toughness TEXT,
|
||||
type TEXT,
|
||||
types TEXT,
|
||||
uuid TEXT,
|
||||
variations TEXT,
|
||||
watermark TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
""")
|
||||
|
||||
# MTG Sets table
|
||||
await conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS mtg_sets (
|
||||
id SERIAL PRIMARY KEY,
|
||||
code VARCHAR(10) UNIQUE NOT NULL,
|
||||
name VARCHAR(255),
|
||||
type VARCHAR(100),
|
||||
release_date DATE,
|
||||
base_set_size INTEGER,
|
||||
total_size INTEGER,
|
||||
is_foil_only BOOLEAN,
|
||||
is_non_foil_only BOOLEAN,
|
||||
digital BOOLEAN,
|
||||
icon_svg_url TEXT,
|
||||
parent_code VARCHAR(10),
|
||||
mtgo_code VARCHAR(10),
|
||||
image_url TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
""")
|
||||
|
||||
# MTG Cards table (from set files)
|
||||
await conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS mtg_cards (
|
||||
id SERIAL PRIMARY KEY,
|
||||
set_id INTEGER REFERENCES mtg_sets(id) ON DELETE CASCADE,
|
||||
name VARCHAR(255),
|
||||
mana_cost VARCHAR(255),
|
||||
type_line VARCHAR(255),
|
||||
oracle_text TEXT,
|
||||
power VARCHAR(50),
|
||||
toughness VARCHAR(50),
|
||||
rarity VARCHAR(50),
|
||||
layout VARCHAR(50),
|
||||
artist VARCHAR(255),
|
||||
flavor_text TEXT,
|
||||
numbers VARCHAR(100),
|
||||
identifiers TEXT,
|
||||
images TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
""")
|
||||
|
||||
# Card Identifiers table
|
||||
await conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS card_identifiers (
|
||||
id SERIAL PRIMARY KEY,
|
||||
uuid TEXT UNIQUE NOT NULL,
|
||||
name VARCHAR(255),
|
||||
mana_cost VARCHAR(255),
|
||||
type_line VARCHAR(255),
|
||||
oracle_text TEXT,
|
||||
power VARCHAR(50),
|
||||
toughness VARCHAR(50),
|
||||
rarity VARCHAR(50),
|
||||
layout VARCHAR(50),
|
||||
artist VARCHAR(255),
|
||||
flavor_text TEXT,
|
||||
set_code VARCHAR(10),
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
""")
|
||||
|
||||
# Decks table
|
||||
await conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS decks (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
description TEXT,
|
||||
format VARCHAR(50),
|
||||
command TEXT,
|
||||
commander TEXT,
|
||||
creation_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
""")
|
||||
|
||||
# Card Types table
|
||||
await conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS card_types (
|
||||
id SERIAL PRIMARY KEY,
|
||||
type VARCHAR(100) UNIQUE NOT NULL,
|
||||
description TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
""")
|
||||
|
||||
# Deck List table
|
||||
await conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS deck_list (
|
||||
id SERIAL PRIMARY KEY,
|
||||
deck_id VARCHAR(100) UNIQUE NOT NULL,
|
||||
name VARCHAR(255),
|
||||
description TEXT,
|
||||
format VARCHAR(50),
|
||||
command TEXT,
|
||||
commander TEXT,
|
||||
total_cards INTEGER,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
""")
|
||||
|
||||
# Card Keywords table
|
||||
await conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS card_keywords (
|
||||
id SERIAL PRIMARY KEY,
|
||||
keyword VARCHAR(100) UNIQUE NOT NULL,
|
||||
description TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
""")
|
||||
|
||||
# Set List table
|
||||
await conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS set_list (
|
||||
id SERIAL PRIMARY KEY,
|
||||
set_code VARCHAR(10) UNIQUE NOT NULL,
|
||||
set_name VARCHAR(255),
|
||||
set_type VARCHAR(100),
|
||||
release_date DATE,
|
||||
base_set_size INTEGER,
|
||||
total_size INTEGER,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
""")
|
||||
|
||||
await conn.commit()
|
||||
print("✓ Database schema created\n")
|
||||
|
||||
|
||||
async def import_all_printings_psql(engine):
|
||||
"""Import AllPrintings.psql.gz file using psql command."""
|
||||
print("=== Importing AllPrintings.psql.gz ===\n")
|
||||
|
||||
psql_file = MTGDATA_DIR / 'AllPrintings.psql.gz'
|
||||
|
||||
if not psql_file.exists():
|
||||
print("✗ AllPrintings.psql.gz not found\n")
|
||||
return
|
||||
|
||||
print("Decompressing...")
|
||||
psql_content = gzip.decompress(psql_file.read_bytes())
|
||||
psql_path = MTGDATA_DIR / 'AllPrintings.psql'
|
||||
psql_path.write_bytes(psql_content)
|
||||
|
||||
print("Importing into PostgreSQL...")
|
||||
|
||||
# Import using psql command
|
||||
# Use environment variables to get database connection info
|
||||
db_user = os.environ.get('POSTGRES_USER', 'cockatrice')
|
||||
db_password = os.environ.get('POSTGRES_PASSWORD', 'cockatrice_pass')
|
||||
db_host = os.environ.get('POSTGRES_HOST', 'mtgonline_postgres_mtgdata')
|
||||
db_port = os.environ.get('POSTGRES_PORT', '5432')
|
||||
db_name = os.environ.get('POSTGRES_DB', 'mtgdata')
|
||||
|
||||
cmd = [
|
||||
'psql', '-h', db_host, '-p', db_port, '-U', db_user, '-d', db_name,
|
||||
'-f', str(psql_path)
|
||||
]
|
||||
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
|
||||
if result.returncode != 0:
|
||||
print(f"✗ Import failed: {result.stderr[:500]}\n")
|
||||
return
|
||||
|
||||
# Count records
|
||||
async with engine.connect() as conn:
|
||||
result = await conn.execute("SELECT COUNT(*) FROM cards")
|
||||
count = result.scalar()
|
||||
print(f"✓ Imported {count:,} cards\n")
|
||||
|
||||
# Clean up
|
||||
psql_path.unlink()
|
||||
|
||||
|
||||
async def import_all_set_files(engine):
|
||||
"""Import AllSetFiles.zip - sets and cards."""
|
||||
print("=== Importing AllSetFiles ===\n")
|
||||
|
||||
extract_dir = MTGDATA_DIR / 'AllSetFiles'
|
||||
|
||||
if not extract_dir.exists():
|
||||
print("✗ AllSetFiles directory not found\n")
|
||||
return
|
||||
|
||||
async with engine.connect() as conn:
|
||||
# Import sets
|
||||
set_count = 0
|
||||
for set_file in extract_dir.glob('*.json'):
|
||||
data = json.loads(set_file.read_text())
|
||||
|
||||
if 'code' in data:
|
||||
image_url = None
|
||||
if 'image' in data and data['image']:
|
||||
image_url = data['image'].get('normal')
|
||||
|
||||
await conn.execute("""
|
||||
INSERT INTO mtg_sets (code, name, type, release_date,
|
||||
base_set_size, total_size, is_foil_only,
|
||||
is_non_foil_only, digital, icon_svg_url,
|
||||
parent_code, mtgo_code, image_url)
|
||||
VALUES (:code, :name, :type, :release_date,
|
||||
:base_set_size, :total_size, :is_foil_only,
|
||||
:is_non_foil_only, :digital, :icon_svg_url,
|
||||
:parent_code, :mtgo_code, :image_url)
|
||||
ON CONFLICT (code) DO UPDATE SET
|
||||
name = EXCLUDED.name, type = EXCLUDED.type,
|
||||
release_date = EXCLUDED.release_date,
|
||||
base_set_size = EXCLUDED.base_set_size,
|
||||
total_size = EXCLUDED.total_size,
|
||||
is_foil_only = EXCLUDED.is_foil_only,
|
||||
is_non_foil_only = EXCLUDED.is_non_foil_only,
|
||||
digital = EXCLUDED.digital,
|
||||
icon_svg_url = EXCLUDED.icon_svg_url,
|
||||
parent_code = EXCLUDED.parent_code,
|
||||
mtgo_code = EXCLUDED.mtgo_code,
|
||||
image_url = EXCLUDED.image_url
|
||||
""", {
|
||||
'code': data.get('code'),
|
||||
'name': data.get('name'),
|
||||
'type': data.get('type'),
|
||||
'release_date': data.get('releaseDate'),
|
||||
'base_set_size': data.get('baseSetSize'),
|
||||
'total_size': data.get('totalSetSize'),
|
||||
'is_foil_only': data.get('isFoilOnly'),
|
||||
'is_non_foil_only': data.get('isNonFoilOnly'),
|
||||
'digital': data.get('digital'),
|
||||
'icon_svg_url': data.get('iconSvgUrl'),
|
||||
'parent_code': data.get('parentCode'),
|
||||
'mtgo_code': data.get('mtgoCode'),
|
||||
'image_url': image_url,
|
||||
})
|
||||
set_count += 1
|
||||
|
||||
await conn.commit()
|
||||
print(f"✓ Imported {set_count} sets")
|
||||
|
||||
# Import cards
|
||||
card_count = 0
|
||||
for set_file in extract_dir.glob('*.json'):
|
||||
data = json.loads(set_file.read_text())
|
||||
|
||||
if 'cards' in data:
|
||||
set_id = (await conn.execute(
|
||||
"SELECT id FROM mtg_sets WHERE code = :code",
|
||||
{'code': data['code']}
|
||||
)).scalar()
|
||||
|
||||
if set_id:
|
||||
for card_data in data['cards']:
|
||||
identifiers = {
|
||||
'multiId': card_data.get('multiverseIds'),
|
||||
'tcgplayerProductId': card_data.get('tcgplayerProductId'),
|
||||
'cardmarketId': card_data.get('cardmarketId'),
|
||||
}
|
||||
|
||||
images = {}
|
||||
if 'image_uris' in card_data:
|
||||
images = {
|
||||
'small': card_data['image_uris'].get('small'),
|
||||
'normal': card_data['image_uris'].get('normal'),
|
||||
'large': card_data['image_uris'].get('large'),
|
||||
'png': card_data['image_uris'].get('png'),
|
||||
'art_crop': card_data['image_uris'].get('art_crop'),
|
||||
}
|
||||
|
||||
await conn.execute("""
|
||||
INSERT INTO mtg_cards (set_id, name, mana_cost,
|
||||
type_line, oracle_text, power,
|
||||
toughness, rarity, layout, artist,
|
||||
flavor_text, numbers, identifiers, images)
|
||||
VALUES (:set_id, :name, :mana_cost, :type_line,
|
||||
:oracle_text, :power, :toughness, :rarity,
|
||||
:layout, :artist, :flavor_text, :numbers,
|
||||
:identifiers, :images)
|
||||
""", {
|
||||
'set_id': set_id,
|
||||
'name': card_data.get('name'),
|
||||
'mana_cost': card_data.get('manaCost'),
|
||||
'type_line': card_data.get('typeLine'),
|
||||
'oracle_text': card_data.get('oracleText'),
|
||||
'power': card_data.get('power'),
|
||||
'toughness': card_data.get('toughness'),
|
||||
'rarity': card_data.get('rarity'),
|
||||
'layout': card_data.get('layout'),
|
||||
'artist': card_data.get('artist'),
|
||||
'flavor_text': card_data.get('flavorText'),
|
||||
'numbers': str(card_data.get('number')),
|
||||
'identifiers': json.dumps(identifiers),
|
||||
'images': json.dumps(images),
|
||||
})
|
||||
card_count += 1
|
||||
|
||||
await conn.commit()
|
||||
print(f"✓ Imported {card_count} cards from sets\n")
|
||||
|
||||
|
||||
async def import_all_deck_files(engine):
|
||||
"""Import AllDeckFiles.zip."""
|
||||
print("=== Importing AllDeckFiles ===\n")
|
||||
|
||||
extract_dir = MTGDATA_DIR / 'AllDeckFiles'
|
||||
|
||||
if not extract_dir.exists():
|
||||
print("✗ AllDeckFiles directory not found\n")
|
||||
return
|
||||
|
||||
async with engine.connect() as conn:
|
||||
deck_count = 0
|
||||
for deck_file in extract_dir.glob('*.json'):
|
||||
data = json.loads(deck_file.read_text())
|
||||
|
||||
if 'name' in data and 'cards' in data:
|
||||
await conn.execute("""
|
||||
INSERT INTO decks (name, description, format, command, commander)
|
||||
VALUES (:name, :description, :format, :command, :commander)
|
||||
ON CONFLICT (name) DO UPDATE SET
|
||||
description = EXCLUDED.description,
|
||||
format = EXCLUDED.format,
|
||||
command = EXCLUDED.command,
|
||||
commander = EXCLUDED.commander
|
||||
""", {
|
||||
'name': data.get('name'),
|
||||
'description': data.get('description'),
|
||||
'format': data.get('format'),
|
||||
'command': data.get('command'),
|
||||
'commander': data.get('commander'),
|
||||
})
|
||||
deck_count += 1
|
||||
|
||||
await conn.commit()
|
||||
print(f"✓ Imported {deck_count} decks\n")
|
||||
|
||||
|
||||
async def import_all_identifiers(engine):
|
||||
"""Import AllIdentifiers.json.gz."""
|
||||
print("=== Importing AllIdentifiers ===\n")
|
||||
|
||||
file_path = MTGDATA_DIR / 'AllIdentifiers.json.gz'
|
||||
|
||||
if not file_path.exists():
|
||||
print("✗ File not found\n")
|
||||
return
|
||||
|
||||
data = json.loads(gzip.decompress(file_path.read_bytes()))
|
||||
|
||||
async with engine.connect() as conn:
|
||||
identifier_count = 0
|
||||
for uuid, card_data in data.items():
|
||||
await conn.execute("""
|
||||
INSERT INTO card_identifiers (uuid, name, mana_cost, type_line,
|
||||
oracle_text, power, toughness, rarity,
|
||||
layout, artist, flavor_text, set_code)
|
||||
VALUES (:uuid, :name, :mana_cost, :type_line, :oracle_text,
|
||||
:power, :toughness, :rarity, :layout, :artist,
|
||||
:flavor_text, :set_code)
|
||||
ON CONFLICT (uuid) DO UPDATE SET
|
||||
name = EXCLUDED.name, mana_cost = EXCLUDED.mana_cost,
|
||||
type_line = EXCLUDED.type_line,
|
||||
oracle_text = EXCLUDED.oracle_text,
|
||||
power = EXCLUDED.power, toughness = EXCLUDED.toughness,
|
||||
rarity = EXCLUDED.rarity, layout = EXCLUDED.layout,
|
||||
artist = EXCLUDED.artist,
|
||||
flavor_text = EXCLUDED.flavor_text,
|
||||
set_code = EXCLUDED.set_code
|
||||
""", {
|
||||
'uuid': uuid,
|
||||
'name': card_data.get('name'),
|
||||
'mana_cost': card_data.get('manaCost'),
|
||||
'type_line': card_data.get('typeLine'),
|
||||
'oracle_text': card_data.get('oracleText'),
|
||||
'power': card_data.get('power'),
|
||||
'toughness': card_data.get('toughness'),
|
||||
'rarity': card_data.get('rarity'),
|
||||
'layout': card_data.get('layout'),
|
||||
'artist': card_data.get('artist'),
|
||||
'flavor_text': card_data.get('flavorText'),
|
||||
'set_code': card_data.get('setCode'),
|
||||
})
|
||||
identifier_count += 1
|
||||
|
||||
await conn.commit()
|
||||
print(f"✓ Imported {identifier_count} identifiers\n")
|
||||
|
||||
|
||||
async def import_card_types(engine):
|
||||
"""Import CardTypes.json.gz."""
|
||||
print("=== Importing CardTypes ===\n")
|
||||
|
||||
file_path = MTGDATA_DIR / 'CardTypes.json.gz'
|
||||
|
||||
if not file_path.exists():
|
||||
print("✗ File not found\n")
|
||||
return
|
||||
|
||||
data = json.loads(gzip.decompress(file_path.read_bytes()))
|
||||
|
||||
async with engine.connect() as conn:
|
||||
type_count = 0
|
||||
for card_type in data:
|
||||
await conn.execute("""
|
||||
INSERT INTO card_types (type, description)
|
||||
VALUES (:type, :description)
|
||||
ON CONFLICT (type) DO UPDATE SET
|
||||
description = EXCLUDED.description
|
||||
""", {
|
||||
'type': card_type.get('type'),
|
||||
'description': card_type.get('description'),
|
||||
})
|
||||
type_count += 1
|
||||
|
||||
await conn.commit()
|
||||
print(f"✓ Imported {type_count} card types\n")
|
||||
|
||||
|
||||
async def import_deck_list(engine):
|
||||
"""Import DeckList.json.gz."""
|
||||
print("=== Importing DeckList ===\n")
|
||||
|
||||
file_path = MTGDATA_DIR / 'DeckList.json.gz'
|
||||
|
||||
if not file_path.exists():
|
||||
print("✗ File not found\n")
|
||||
return
|
||||
|
||||
data = json.loads(gzip.decompress(file_path.read_bytes()))
|
||||
|
||||
async with engine.connect() as conn:
|
||||
deck_list_count = 0
|
||||
for deck in data:
|
||||
await conn.execute("""
|
||||
INSERT INTO deck_list (deck_id, name, description, format,
|
||||
command, commander, total_cards)
|
||||
VALUES (:deck_id, :name, :description, :format,
|
||||
:command, :commander, :total_cards)
|
||||
ON CONFLICT (deck_id) DO UPDATE SET
|
||||
name = EXCLUDED.name, description = EXCLUDED.description,
|
||||
format = EXCLUDED.format, command = EXCLUDED.command,
|
||||
commander = EXCLUDED.commander,
|
||||
total_cards = EXCLUDED.total_cards
|
||||
""", {
|
||||
'deck_id': deck.get('id'),
|
||||
'name': deck.get('name'),
|
||||
'description': deck.get('description'),
|
||||
'format': deck.get('format'),
|
||||
'command': deck.get('command'),
|
||||
'commander': deck.get('commander'),
|
||||
'total_cards': deck.get('totalCards'),
|
||||
})
|
||||
deck_list_count += 1
|
||||
|
||||
await conn.commit()
|
||||
print(f"✓ Imported {deck_list_count} deck list entries\n")
|
||||
|
||||
|
||||
async def import_keywords(engine):
|
||||
"""Import Keywords.json.gz."""
|
||||
print("=== Importing Keywords ===\n")
|
||||
|
||||
file_path = MTGDATA_DIR / 'Keywords.json.gz'
|
||||
|
||||
if not file_path.exists():
|
||||
print("✗ File not found\n")
|
||||
return
|
||||
|
||||
data = json.loads(gzip.decompress(file_path.read_bytes()))
|
||||
|
||||
async with engine.connect() as conn:
|
||||
keyword_count = 0
|
||||
for keyword in data:
|
||||
await conn.execute("""
|
||||
INSERT INTO card_keywords (keyword, description)
|
||||
VALUES (:keyword, :description)
|
||||
ON CONFLICT (keyword) DO UPDATE SET
|
||||
description = EXCLUDED.description
|
||||
""", {
|
||||
'keyword': keyword.get('keyword'),
|
||||
'description': keyword.get('description'),
|
||||
})
|
||||
keyword_count += 1
|
||||
|
||||
await conn.commit()
|
||||
print(f"✓ Imported {keyword_count} keywords\n")
|
||||
|
||||
|
||||
async def import_set_list(engine):
|
||||
"""Import SetList.json.gz."""
|
||||
print("=== Importing SetList ===\n")
|
||||
|
||||
file_path = MTGDATA_DIR / 'SetList.json.gz'
|
||||
|
||||
if not file_path.exists():
|
||||
print("✗ File not found\n")
|
||||
return
|
||||
|
||||
data = json.loads(gzip.decompress(file_path.read_bytes()))
|
||||
|
||||
async with engine.connect() as conn:
|
||||
set_list_count = 0
|
||||
for set_data in data:
|
||||
await conn.execute("""
|
||||
INSERT INTO set_list (set_code, set_name, set_type, release_date,
|
||||
base_set_size, total_size)
|
||||
VALUES (:set_code, :set_name, :set_type, :release_date,
|
||||
:base_set_size, :total_size)
|
||||
ON CONFLICT (set_code) DO UPDATE SET
|
||||
set_name = EXCLUDED.set_name, set_type = EXCLUDED.set_type,
|
||||
release_date = EXCLUDED.release_date,
|
||||
base_set_size = EXCLUDED.base_set_size,
|
||||
total_size = EXCLUDED.total_size
|
||||
""", {
|
||||
'set_code': set_data.get('code'),
|
||||
'set_name': set_data.get('name'),
|
||||
'set_type': set_data.get('type'),
|
||||
'release_date': set_data.get('releaseDate'),
|
||||
'base_set_size': set_data.get('baseSetSize'),
|
||||
'total_size': set_data.get('totalSetSize'),
|
||||
})
|
||||
set_list_count += 1
|
||||
|
||||
await conn.commit()
|
||||
print(f"✓ Imported {set_list_count} set list entries\n")
|
||||
|
||||
|
||||
async def create_indexes(engine):
|
||||
"""Create indexes for better performance."""
|
||||
print("=== Creating Indexes ===\n")
|
||||
|
||||
async with engine.connect() as conn:
|
||||
indexes = [
|
||||
"CREATE INDEX IF NOT EXISTS idx_cards_name ON cards(name)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_cards_mana_cost ON cards(manaCost)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_cards_type ON cards(type)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_cards_rarity ON cards(rarity)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_cards_set_code ON cards(setCode)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_cards_uuid ON cards(uuid)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_mtg_sets_code ON mtg_sets(code)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_mtg_cards_name ON mtg_cards(name)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_card_identifiers_uuid ON card_identifiers(uuid)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_card_identifiers_name ON card_identifiers(name)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_deck_list_deck_id ON deck_list(deck_id)",
|
||||
]
|
||||
|
||||
for idx in indexes:
|
||||
await conn.execute(idx)
|
||||
|
||||
await conn.commit()
|
||||
print("✓ Indexes created\n")
|
||||
|
||||
|
||||
async def show_summary(engine):
|
||||
"""Show import summary."""
|
||||
print("=== Import Summary ===\n")
|
||||
|
||||
async with engine.connect() as conn:
|
||||
tables = [
|
||||
'cards', 'mtg_sets', 'mtg_cards', 'card_identifiers',
|
||||
'decks', 'card_types', 'deck_list', 'card_keywords', 'set_list'
|
||||
]
|
||||
|
||||
for table in tables:
|
||||
result = await conn.execute(f"SELECT COUNT(*) FROM {table}")
|
||||
count = result.scalar()
|
||||
print(f" {table:20} {count:>10,} records")
|
||||
|
||||
print()
|
||||
|
||||
|
||||
async def main():
|
||||
"""Main import function."""
|
||||
print("=== MTGJSON Complete Data Import ===\n")
|
||||
|
||||
# Check files
|
||||
if not check_files():
|
||||
return
|
||||
|
||||
# Extract zip files
|
||||
extract_files()
|
||||
|
||||
# Connect to database
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
from app.core.settings import get_settings
|
||||
settings = get_settings()
|
||||
engine = create_async_engine(settings.MTG_DATABASE_URL)
|
||||
|
||||
# Create schema
|
||||
await create_database_schema(engine)
|
||||
|
||||
# Import AllPrintings
|
||||
await import_all_printings_psql(engine)
|
||||
|
||||
# Import AllSetFiles
|
||||
await import_all_set_files(engine)
|
||||
|
||||
# Import AllDeckFiles
|
||||
await import_all_deck_files(engine)
|
||||
|
||||
# Import AllIdentifiers
|
||||
await import_all_identifiers(engine)
|
||||
|
||||
# Import CardTypes
|
||||
await import_card_types(engine)
|
||||
|
||||
# Import DeckList
|
||||
await import_deck_list(engine)
|
||||
|
||||
# Import Keywords
|
||||
await import_keywords(engine)
|
||||
|
||||
# Import SetList
|
||||
await import_set_list(engine)
|
||||
|
||||
# Create indexes
|
||||
await create_indexes(engine)
|
||||
|
||||
# Show summary
|
||||
await show_summary(engine)
|
||||
|
||||
print("✓ All data imported successfully!\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user