460 lines
17 KiB
Python
460 lines
17 KiB
Python
"""
|
|
MTGJSON Database Migration Strategy
|
|
|
|
Comprehensive mapping of MTGJSON data model to PostgreSQL schema.
|
|
|
|
Strategy for Nested JSON Arrays:
|
|
1. Direct Columns: Simple scalar values (strings, numbers, booleans)
|
|
2. JSONB Columns: Complex objects/arrays that need querying (legalities, prices)
|
|
3. Related Tables: One-to-many relationships (card_faces, foreign_names, rulings)
|
|
4. Comma-Separated: Simple arrays that can be split (supertypes, types, subtypes)
|
|
"""
|
|
from sqlalchemy import create_engine, text
|
|
import json
|
|
|
|
DB_URL = "postgresql+psycopg2://mtgonline:mtgonline_pass@172.18.0.2:5432/mtgdata"
|
|
|
|
|
|
class MTGJSONMigration:
|
|
"""Migrate MTGJSON data to comprehensive PostgreSQL schema."""
|
|
|
|
def __init__(self):
|
|
self.engine = create_engine(DB_URL)
|
|
self.conn = None
|
|
|
|
def connect(self):
|
|
"""Connect to database."""
|
|
self.conn = self.engine.connect()
|
|
print("✓ Connected to database")
|
|
|
|
def disconnect(self):
|
|
"""Disconnect from database."""
|
|
if self.conn:
|
|
self.conn.close()
|
|
self.engine.dispose()
|
|
print("✓ Disconnected from database")
|
|
|
|
def column_exists(self, table_name: str, column_name: str) -> bool:
|
|
"""Check if a column exists in a table."""
|
|
result = self.conn.execute(text("""
|
|
SELECT column_name
|
|
FROM information_schema.columns
|
|
WHERE table_name = :table AND column_name = :column
|
|
"""), {"table": table_name, "column": column_name})
|
|
return result.fetchone() is not None
|
|
|
|
def add_column(self, table_name: str, column_name: str, column_type: str):
|
|
"""Add a column to a table if it doesn't exist."""
|
|
if not self.column_exists(table_name, column_name):
|
|
self.conn.execute(text(f"ALTER TABLE {table_name} ADD COLUMN {column_name} {column_type}"))
|
|
print(f" ✓ Added: {table_name}.{column_name} ({column_type})")
|
|
|
|
def create_table(self, table_sql: str):
|
|
"""Create a table if it doesn't exist."""
|
|
self.conn.execute(text(table_sql))
|
|
print(f" ✓ Created table")
|
|
|
|
def create_index(self, index_sql: str):
|
|
"""Create an index if it doesn't exist."""
|
|
self.conn.execute(text(f"CREATE INDEX IF NOT EXISTS {index_sql.split(' ON ')[1].split(' ')[0]} ON {index_sql.split(' ON ')[1].split(' ')[1]}"))
|
|
print(f" ✓ Created index")
|
|
|
|
def migrate_card_table(self):
|
|
"""Add all MTGJSON card attributes to mtg_cards table."""
|
|
print("\n📊 Migrating mtg_cards table...")
|
|
|
|
# ========================
|
|
# STRATEGY 1: Direct Columns (Simple scalar values)
|
|
# ========================
|
|
print("\n📝 Strategy 1: Direct Columns (Simple scalar values)")
|
|
|
|
direct_columns = [
|
|
# Basic card info
|
|
("name", "VARCHAR(255)"),
|
|
("mana_cost", "VARCHAR(255)"),
|
|
("type_line", "VARCHAR(255)"),
|
|
("oracle_text", "TEXT"),
|
|
("power", "VARCHAR(50)"),
|
|
("toughness", "VARCHAR(50)"),
|
|
("loyalty", "VARCHAR(50)"), # For Planeswalkers
|
|
("rarity", "VARCHAR(50)"),
|
|
("layout", "VARCHAR(50)"),
|
|
("artist", "VARCHAR(255)"),
|
|
("flavor_text", "TEXT"),
|
|
("numbers", "VARCHAR(100)"),
|
|
|
|
# MTGJSON: border, watermark
|
|
("border_color", "VARCHAR(20)"),
|
|
("watermark", "VARCHAR(255)"),
|
|
|
|
# MTGJSON: colorIdentity (single color)
|
|
("color_identity", "VARCHAR(10)"),
|
|
|
|
# MTGJSON: lang
|
|
("lang", "VARCHAR(10) DEFAULT 'en'"),
|
|
|
|
# MTGJSON: originalReleaseDate
|
|
("original_release_date", "DATE"),
|
|
|
|
# MTGJSON: originalTypeLine
|
|
("original_type_line", "VARCHAR(255)"),
|
|
|
|
# MTGJSON: securityStamp
|
|
("security_stamp", "VARCHAR(20)"),
|
|
|
|
# MTGJSON: isPromo
|
|
("promo", "BOOLEAN DEFAULT FALSE"),
|
|
|
|
# MTGJSON: isDigital
|
|
("digital", "BOOLEAN DEFAULT FALSE"),
|
|
|
|
# MTGJSON: isToken
|
|
("token", "BOOLEAN DEFAULT FALSE"),
|
|
|
|
# MTGJSON: frame
|
|
("frame", "VARCHAR(50)"),
|
|
|
|
# MTGJSON: fullArt
|
|
("full_art", "BOOLEAN DEFAULT FALSE"),
|
|
|
|
# MTGJSON: isRebalanced
|
|
("is_rebalanced", "BOOLEAN DEFAULT FALSE"),
|
|
|
|
# MTGJSON: isStarter
|
|
("is_starter", "BOOLEAN DEFAULT FALSE"),
|
|
|
|
# MTGJSON: isInBooster
|
|
("in_booster", "BOOLEAN DEFAULT FALSE"),
|
|
|
|
# MTGJSON: mysticalArchive
|
|
("mystical_archive", "BOOLEAN DEFAULT FALSE"),
|
|
]
|
|
|
|
for col_name, col_type in direct_columns:
|
|
self.add_column("mtg_cards", col_name, col_type)
|
|
|
|
# ========================
|
|
# STRATEGY 2: JSONB Columns (Complex objects/arrays)
|
|
# ========================
|
|
print("\n📦 Strategy 2: JSONB Columns (Complex objects/arrays)")
|
|
|
|
jsonb_columns = [
|
|
# MTGJSON: legalities object
|
|
# Example: {"Standard": "Legal", "Modern": "Banned", "Vintage": "Restricted"}
|
|
("legalities", "JSONB"),
|
|
|
|
# MTGJSON: prices object
|
|
# Example: {"tcgplayer": "$4.99", "low": 2.5, "mid": 4.0, "high": 6.0}
|
|
("prices", "JSONB"),
|
|
|
|
# MTGJSON: cardFaces array (for split cards, modal DFCs)
|
|
# Example: [{"name": "Card A", "oracleText": "...", "power": "2"}, {"name": "Card B", ...}]
|
|
("card_faces", "JSONB"),
|
|
|
|
# MTGJSON: foreignData array (for translations)
|
|
# Example: [{"language": "Japanese", "name": "カード名", "typeLine": "クリーチャー"}, ...]
|
|
("foreign_names", "JSONB"),
|
|
|
|
# MTGJSON: relatedCards object
|
|
# Example: {"convertedNames": ["..."], "commanderCounterparts": [...]}
|
|
("related_cards", "JSONB"),
|
|
|
|
# MTGJSON: frameEffects array
|
|
# Example: ["extendedart", "legendary", "nightmare"]
|
|
("frame_effects", "JSONB"),
|
|
|
|
# MTGJSON: keywords array
|
|
# Example: ["first strike", "trample", "vision mount"]
|
|
("keywords", "JSONB"),
|
|
|
|
# MTGJSON: set (set object)
|
|
# Example: {"name": "Commander 2021", "code": "C21", "type": "commander"}
|
|
("set", "JSONB"),
|
|
|
|
# MTGJSON: booster (booster configuration)
|
|
# Example: {"boosters": [{"content": [...], "type": "main"}]}
|
|
("booster", "JSONB"),
|
|
]
|
|
|
|
for col_name, col_type in jsonb_columns:
|
|
self.add_column("mtg_cards", col_name, col_type)
|
|
|
|
# ========================
|
|
# STRATEGY 3: Comma-Separated (Simple arrays)
|
|
# ========================
|
|
print("\n🔗 Strategy 3: Comma-Separated (Simple arrays)")
|
|
|
|
comma_separated = [
|
|
# MTGJSON: types array (e.g., ["Creature", "Human"])
|
|
("types", "VARCHAR(255)"),
|
|
|
|
# MTGJSON: subtypes array (e.g., ["Elf", "Rogue"])
|
|
("subtypes", "VARCHAR(255)"),
|
|
|
|
# MTGJSON: supertypes array (e.g., ["Legendary"])
|
|
("supertypes", "VARCHAR(100)"),
|
|
|
|
# MTGJSON: colors array (e.g., ["W", "G"]) - stored as comma-separated
|
|
("colors", "VARCHAR(20)"),
|
|
]
|
|
|
|
for col_name, col_type in comma_separated:
|
|
self.add_column("mtg_cards", col_name, col_type)
|
|
|
|
# ========================
|
|
# STRATEGY 4: Related Tables (One-to-many relationships)
|
|
# ========================
|
|
print("\n📚 Strategy 4: Related Tables (One-to-many relationships)")
|
|
|
|
# Card faces table
|
|
self.create_table("""
|
|
CREATE TABLE IF NOT EXISTS mtg_card_faces (
|
|
id SERIAL PRIMARY KEY,
|
|
card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
|
|
face_number INTEGER,
|
|
name VARCHAR(255),
|
|
mana_cost VARCHAR(255),
|
|
type_line VARCHAR(255),
|
|
oracle_text TEXT,
|
|
power VARCHAR(50),
|
|
toughness VARCHAR(50),
|
|
loyalty VARCHAR(50),
|
|
flavor_text TEXT,
|
|
artist VARCHAR(255),
|
|
illustration_id VARCHAR(100),
|
|
image_uri TEXT,
|
|
image_png TEXT,
|
|
image_art_crop TEXT,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
""")
|
|
|
|
# Foreign names table
|
|
self.create_table("""
|
|
CREATE TABLE IF NOT EXISTS mtg_card_foreign_names (
|
|
id SERIAL PRIMARY KEY,
|
|
card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
|
|
language VARCHAR(20),
|
|
name VARCHAR(255),
|
|
type_line VARCHAR(255),
|
|
oracle_text TEXT,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
""")
|
|
|
|
# Rulings table
|
|
self.create_table("""
|
|
CREATE TABLE IF NOT EXISTS mtg_card_rulings (
|
|
id SERIAL PRIMARY KEY,
|
|
card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
|
|
published_date DATE,
|
|
text TEXT,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
""")
|
|
|
|
# Related cards table
|
|
self.create_table("""
|
|
CREATE TABLE IF NOT EXISTS mtg_card_related (
|
|
id SERIAL PRIMARY KEY,
|
|
card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
|
|
related_type VARCHAR(50),
|
|
related_id INTEGER,
|
|
related_name VARCHAR(255),
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
""")
|
|
|
|
# Card types table (for normalized type search)
|
|
self.create_table("""
|
|
CREATE TABLE IF NOT EXISTS mtg_card_types (
|
|
id SERIAL PRIMARY KEY,
|
|
card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
|
|
type_category VARCHAR(50),
|
|
type_name VARCHAR(100),
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
UNIQUE(card_id, type_category, type_name)
|
|
)
|
|
""")
|
|
|
|
# Color identity table (for multi-card color identity)
|
|
self.create_table("""
|
|
CREATE TABLE IF NOT EXISTS mtg_card_color_identity (
|
|
id SERIAL PRIMARY KEY,
|
|
card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
|
|
color CHAR(1),
|
|
identity_type VARCHAR(20) DEFAULT 'color',
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
UNIQUE(card_id, color, identity_type)
|
|
)
|
|
""")
|
|
|
|
# ========================
|
|
# CREATE INDEXES
|
|
# ========================
|
|
print("\n🔍 Creating indexes...")
|
|
|
|
indexes = [
|
|
# Card indexes
|
|
"idx_cards_colors ON mtg_cards(colors)",
|
|
"idx_cards_color_identity ON mtg_cards(color_identity)",
|
|
"idx_cards_supertypes ON mtg_cards(supertypes)",
|
|
"idx_cards_types ON mtg_cards(types)",
|
|
"idx_cards_subtypes ON mtg_cards(subtypes)",
|
|
"idx_cards_legalities ON mtg_cards(legalities) USING GIN",
|
|
"idx_cards_prices ON mtg_cards(prices) USING GIN",
|
|
"idx_cards_card_faces ON mtg_cards(card_faces) USING GIN",
|
|
"idx_cards_foreign_names ON mtg_cards(foreign_names) USING GIN",
|
|
"idx_cards_related_cards ON mtg_cards(related_cards) USING GIN",
|
|
"idx_cards_keywords ON mtg_cards(keywords) USING GIN",
|
|
|
|
# Set indexes
|
|
"idx_sets_status ON mtg_sets(status)",
|
|
"idx_sets_block_code ON mtg_sets(block_code)",
|
|
|
|
# Related table indexes
|
|
"idx_card_faces_card_id ON mtg_card_faces(card_id)",
|
|
"idx_card_foreign_names_card_id ON mtg_card_foreign_names(card_id)",
|
|
"idx_card_rulings_card_id ON mtg_card_rulings(card_id)",
|
|
"idx_card_related_card_id ON mtg_card_related(card_id)",
|
|
"idx_card_types_card_id ON mtg_card_types(card_id)",
|
|
"idx_card_color_identity_card_id ON mtg_card_color_identity(card_id)",
|
|
]
|
|
|
|
for idx in indexes:
|
|
self.create_index(f"idx_{idx}")
|
|
|
|
print("\n✅ Card table migration complete!")
|
|
|
|
def migrate_set_table(self):
|
|
"""Add all MTGJSON set attributes to mtg_sets table."""
|
|
print("\n📊 Migrating mtg_sets table...")
|
|
|
|
# MTGJSON set attributes
|
|
set_columns = [
|
|
# Basic set info
|
|
("code", "VARCHAR(10)"),
|
|
("name", "VARCHAR(255)"),
|
|
("type", "VARCHAR(100)"),
|
|
("release_date", "DATE"),
|
|
("base_set_size", "INTEGER"),
|
|
("total_size", "INTEGER"),
|
|
("is_foil_only", "BOOLEAN"),
|
|
("is_non_foil_only", "BOOLEAN"),
|
|
("digital", "BOOLEAN"),
|
|
("icon_svg_url", "TEXT"),
|
|
("parent_code", "VARCHAR(10)"),
|
|
("mtgo_code", "VARCHAR(10)"),
|
|
|
|
# MTGJSON: tcgplayerGroupId
|
|
("tcgplayer_group_id", "INTEGER"),
|
|
|
|
# MTGJSON: scryfallId
|
|
("scryfall_id", "VARCHAR(36)"),
|
|
|
|
# MTGJSON: status (released, unreleased, etc.)
|
|
("status", "VARCHAR(20)"),
|
|
|
|
# MTGJSON: name_normalized
|
|
("name_normalized", "VARCHAR(255)"),
|
|
|
|
# MTGJSON: blockCode
|
|
("block_code", "VARCHAR(10)"),
|
|
|
|
# MTGJSON: setCodes (all set codes)
|
|
("set_codes", "JSONB"),
|
|
|
|
# MTGJSON: cardCount (total cards in set)
|
|
("card_count", "INTEGER"),
|
|
]
|
|
|
|
for col_name, col_type in set_columns:
|
|
self.add_column("mtg_sets", col_name, col_type)
|
|
|
|
print("\n✅ Set table migration complete!")
|
|
|
|
def populate_existing_data(self):
|
|
"""Populate new columns from existing JSON data."""
|
|
print("\n🔄 Populating existing data from JSON columns...")
|
|
|
|
# Extract data from identifiers JSON
|
|
self.conn.execute(text("""
|
|
UPDATE mtg_cards
|
|
SET
|
|
border_color = identifiers->>'border',
|
|
watermark = identifiers->>'watermark',
|
|
original_release_date = identifiers->>'originalReleaseDate',
|
|
original_type_line = identifiers->>'originalTypeLine',
|
|
security_stamp = identifiers->>'securityStamp',
|
|
lang = identifiers->>'lang',
|
|
promo = (identifiers->>'isPromo')::BOOLEAN,
|
|
digital = (identifiers->>'isDigital')::BOOLEAN,
|
|
token = (identifiers->>'isToken')::BOOLEAN
|
|
WHERE identifiers IS NOT NULL
|
|
AND identifiers != 'null'
|
|
AND identifiers != ''
|
|
"""))
|
|
print(" ✓ Updated basic fields from identifiers")
|
|
|
|
# Extract type information from type_line
|
|
self.conn.execute(text("""
|
|
UPDATE mtg_cards
|
|
SET
|
|
supertypes = type_line,
|
|
types = type_line,
|
|
subtypes = type_line
|
|
WHERE type_line IS NOT NULL
|
|
AND type_line != ''
|
|
"""))
|
|
print(" ✓ Updated type hierarchy from type_line")
|
|
|
|
# Extract legalities, prices, card_faces from images JSON
|
|
self.conn.execute(text("""
|
|
UPDATE mtg_cards
|
|
SET
|
|
prices = images->'prices',
|
|
card_faces = images->'cardFaces',
|
|
foreign_names = images->'foreignData',
|
|
related_cards = images->'relatedCards'
|
|
WHERE images IS NOT NULL
|
|
AND images != 'null'
|
|
AND images != ''
|
|
"""))
|
|
print(" ✓ Updated complex fields from images JSON")
|
|
|
|
self.conn.commit()
|
|
print("\n✅ Data population complete!")
|
|
|
|
def run_migration(self):
|
|
"""Run the full migration."""
|
|
print("=" * 60)
|
|
print("🚀 Starting MTGJSON Database Migration")
|
|
print("=" * 60)
|
|
|
|
self.connect()
|
|
|
|
# Migrate card table
|
|
self.migrate_card_table()
|
|
|
|
# Migrate set table
|
|
self.migrate_set_table()
|
|
|
|
# Populate existing data
|
|
self.populate_existing_data()
|
|
|
|
self.disconnect()
|
|
|
|
print("\n" + "=" * 60)
|
|
print("✅ Migration completed successfully!")
|
|
print("=" * 60)
|
|
|
|
|
|
def main():
|
|
"""Main entry point."""
|
|
migration = MTGJSONMigration()
|
|
migration.run_migration()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|