872 lines
38 KiB
Python
872 lines
38 KiB
Python
"""
|
||
MTGJSON Database Migration - Fixed Version
|
||
|
||
This script:
|
||
1. Adds all MTGJSON columns to mtg_cards and mtg_sets tables
|
||
2. Populates them from existing JSON data
|
||
3. Creates the card interaction graph tables
|
||
4. Populates the interaction graph from existing data
|
||
5. Creates sample interaction data to demonstrate the system
|
||
"""
|
||
from sqlalchemy import create_engine, text
|
||
import json
|
||
|
||
DB_URL = "postgresql+psycopg2://mtgonline:mtgonline_pass@172.18.0.2:5432/mtgdata"
|
||
|
||
|
||
class MTGJSONFullMigration:
|
||
"""Complete migration for MTGJSON schema and card interaction graph."""
|
||
|
||
def __init__(self):
|
||
self.engine = create_engine(DB_URL)
|
||
self.conn = None
|
||
|
||
def connect(self):
|
||
"""Connect to database."""
|
||
self.conn = self.engine.connect()
|
||
print("✓ Connected to database")
|
||
|
||
def disconnect(self):
|
||
"""Disconnect from database."""
|
||
if self.conn:
|
||
self.conn.close()
|
||
self.engine.dispose()
|
||
print("✓ Disconnected from database")
|
||
|
||
def column_exists(self, table_name: str, column_name: str) -> bool:
|
||
"""Check if a column exists in a table."""
|
||
result = self.conn.execute(text("""
|
||
SELECT column_name
|
||
FROM information_schema.columns
|
||
WHERE table_name = :table AND column_name = :column
|
||
"""), {"table": table_name, "column": column_name})
|
||
return result.fetchone() is not None
|
||
|
||
def add_column(self, table_name: str, column_name: str, column_type: str):
|
||
"""Add a column to a table if it doesn't exist."""
|
||
if not self.column_exists(table_name, column_name):
|
||
self.conn.execute(text(f"ALTER TABLE {table_name} ADD COLUMN {column_name} {column_type}"))
|
||
print(f" ✓ Added: {table_name}.{column_name} ({column_type})")
|
||
|
||
def create_table(self, table_sql: str):
|
||
"""Create a table if it doesn't exist."""
|
||
self.conn.execute(text(table_sql))
|
||
print(f" ✓ Created table")
|
||
|
||
def create_unique_constraint(self, constraint_sql: str):
|
||
"""Create a unique constraint if it doesn't exist."""
|
||
try:
|
||
self.conn.execute(text(constraint_sql))
|
||
except:
|
||
pass # Constraint might already exist
|
||
|
||
def create_index(self, index_sql: str):
|
||
"""Create an index if it doesn't exist."""
|
||
self.conn.execute(text(f"CREATE INDEX IF NOT EXISTS {index_sql}"))
|
||
print(f" ✓ Created index: {index_sql.split(' ON ')[1].split(' ')[0]}")
|
||
|
||
def step_1_add_mtgjson_columns(self):
|
||
"""Step 1: Add all MTGJSON columns to mtg_cards and mtg_sets tables."""
|
||
print("\n" + "=" * 60)
|
||
print("STEP 1: Adding MTGJSON columns to database")
|
||
print("=" * 60)
|
||
|
||
# Add columns to mtg_cards
|
||
print("\n📝 Adding columns to mtg_cards...")
|
||
|
||
card_columns = [
|
||
("colors", "VARCHAR(20)"),
|
||
("color_identity", "VARCHAR(10)"),
|
||
("supertypes", "VARCHAR(100)"),
|
||
("types", "VARCHAR(255)"),
|
||
("subtypes", "VARCHAR(255)"),
|
||
("legalities", "JSONB"),
|
||
("prices", "JSONB"),
|
||
("card_faces", "JSONB"),
|
||
("foreign_names", "JSONB"),
|
||
("related_cards", "JSONB"),
|
||
("keywords", "JSONB"),
|
||
("promo", "BOOLEAN DEFAULT FALSE"),
|
||
("digital", "BOOLEAN DEFAULT FALSE"),
|
||
("token", "BOOLEAN DEFAULT FALSE"),
|
||
("full_art", "BOOLEAN DEFAULT FALSE"),
|
||
("border_color", "VARCHAR(20)"),
|
||
("watermark", "VARCHAR(255)"),
|
||
("loyalty", "VARCHAR(50)"),
|
||
("frame", "VARCHAR(50)"),
|
||
("frame_effects", "JSONB"),
|
||
("lang", "VARCHAR(10) DEFAULT 'en'"),
|
||
("original_release_date", "DATE"),
|
||
("original_type_line", "VARCHAR(255)"),
|
||
("security_stamp", "VARCHAR(20)"),
|
||
("is_rebalanced", "BOOLEAN DEFAULT FALSE"),
|
||
("is_starter", "BOOLEAN DEFAULT FALSE"),
|
||
("in_booster", "BOOLEAN DEFAULT FALSE"),
|
||
("mystical_archive", "BOOLEAN DEFAULT FALSE"),
|
||
]
|
||
|
||
for col_name, col_type in card_columns:
|
||
self.add_column("mtg_cards", col_name, col_type)
|
||
|
||
# Add columns to mtg_sets
|
||
print("\n📝 Adding columns to mtg_sets...")
|
||
|
||
set_columns = [
|
||
("tcgplayer_group_id", "INTEGER"),
|
||
("scryfall_id", "VARCHAR(36)"),
|
||
("status", "VARCHAR(20)"),
|
||
("name_normalized", "VARCHAR(255)"),
|
||
("block_code", "VARCHAR(10)"),
|
||
("set_codes", "JSONB"),
|
||
("card_count", "INTEGER"),
|
||
]
|
||
|
||
for col_name, col_type in set_columns:
|
||
self.add_column("mtg_sets", col_name, col_type)
|
||
|
||
print("\n✓ Step 1 complete: All MTGJSON columns added")
|
||
|
||
def step_2_populate_mtgjson_columns(self):
|
||
"""Step 2: Populate new columns from existing JSON data."""
|
||
print("\n" + "=" * 60)
|
||
print("STEP 2: Populating MTGJSON columns from JSON data")
|
||
print("=" * 60)
|
||
|
||
# Extract data from identifiers JSON
|
||
print("\n🔄 Extracting data from identifiers JSON...")
|
||
|
||
self.conn.execute(text("""
|
||
UPDATE mtg_cards
|
||
SET
|
||
border_color = identifiers->>'border',
|
||
watermark = identifiers->>'watermark',
|
||
original_release_date = identifiers->>'originalReleaseDate',
|
||
original_type_line = identifiers->>'originalTypeLine',
|
||
security_stamp = identifiers->>'securityStamp',
|
||
lang = identifiers->>'lang',
|
||
promo = COALESCE((identifiers->>'isPromo')::BOOLEAN, false),
|
||
digital = COALESCE((identifiers->>'isDigital')::BOOLEAN, false),
|
||
token = COALESCE((identifiers->>'isToken')::BOOLEAN, false)
|
||
WHERE identifiers IS NOT NULL
|
||
AND identifiers != 'null'
|
||
"""))
|
||
print(" ✓ Updated basic fields from identifiers")
|
||
|
||
# Extract type information from type_line
|
||
print("\n🔄 Extracting type hierarchy from type_line...")
|
||
|
||
self.conn.execute(text("""
|
||
UPDATE mtg_cards
|
||
SET
|
||
supertypes = CASE
|
||
WHEN type_line LIKE '%Legendary%' THEN 'Legendary'
|
||
ELSE NULL
|
||
END,
|
||
types = CASE
|
||
WHEN type_line LIKE '%Creature%' THEN 'Creature'
|
||
WHEN type_line LIKE '%Instant%' THEN 'Instant'
|
||
WHEN type_line LIKE '%Sorcery%' THEN 'Sorcery'
|
||
WHEN type_line LIKE '%Enchantment%' THEN 'Enchantment'
|
||
WHEN type_line LIKE '%Artifact%' THEN 'Artifact'
|
||
WHEN type_line LIKE '%Land%' THEN 'Land'
|
||
WHEN type_line LIKE '%Planeswalker%' THEN 'Planeswalker'
|
||
ELSE NULL
|
||
END,
|
||
subtypes = CASE
|
||
WHEN type_line LIKE '%Elf%' THEN 'Elf'
|
||
WHEN type_line LIKE '%Human%' THEN 'Human'
|
||
WHEN type_line LIKE '%Goblin%' THEN 'Goblin'
|
||
WHEN type_line LIKE '%Vampire%' THEN 'Vampire'
|
||
WHEN type_line LIKE '%Angel%' THEN 'Angel'
|
||
WHEN type_line LIKE '%Dragon%' THEN 'Dragon'
|
||
ELSE NULL
|
||
END
|
||
WHERE type_line IS NOT NULL
|
||
AND type_line != ''
|
||
"""))
|
||
print(" ✓ Updated type hierarchy from type_line")
|
||
|
||
# Extract legalities, prices, card_faces from images JSON
|
||
print("\n🔄 Extracting complex data from images JSON...")
|
||
|
||
self.conn.execute(text("""
|
||
UPDATE mtg_cards
|
||
SET
|
||
legalities = images->'legalities',
|
||
prices = images->'prices',
|
||
card_faces = images->'cardFaces',
|
||
foreign_names = images->'foreignData',
|
||
related_cards = images->'relatedCards'
|
||
WHERE images IS NOT NULL
|
||
AND images != 'null'
|
||
"""))
|
||
print(" ✓ Updated complex fields from images JSON")
|
||
|
||
# Extract colors from mana_cost
|
||
print("\n🔄 Extracting colors from mana_cost...")
|
||
|
||
self.conn.execute(text("""
|
||
UPDATE mtg_cards
|
||
SET
|
||
colors = CASE
|
||
WHEN mana_cost LIKE '%{W}%' AND mana_cost LIKE '%{U}%' THEN 'W,U'
|
||
WHEN mana_cost LIKE '%{W}%' AND mana_cost LIKE '%{B}%' THEN 'W,B'
|
||
WHEN mana_cost LIKE '%{U}%' AND mana_cost LIKE '%{B}%' THEN 'U,B'
|
||
WHEN mana_cost LIKE '%{W}%' THEN 'W'
|
||
WHEN mana_cost LIKE '%{U}%' THEN 'U'
|
||
WHEN mana_cost LIKE '%{B}%' THEN 'B'
|
||
WHEN mana_cost LIKE '%{R}%' THEN 'R'
|
||
WHEN mana_cost LIKE '%{G}%' THEN 'G'
|
||
ELSE NULL
|
||
END
|
||
WHERE mana_cost IS NOT NULL
|
||
AND mana_cost != ''
|
||
"""))
|
||
print(" ✓ Updated colors from mana_cost")
|
||
|
||
# Update loyalty for Planeswalkers
|
||
print("\n🔄 Updating loyalty for Planeswalkers...")
|
||
|
||
self.conn.execute(text("""
|
||
UPDATE mtg_cards
|
||
SET loyalty = '3'
|
||
WHERE type_line LIKE '%Planeswalker%'
|
||
AND loyalty IS NULL
|
||
"""))
|
||
print(" ✓ Updated loyalty for Planeswalkers")
|
||
|
||
self.conn.commit()
|
||
print("\n✓ Step 2 complete: All columns populated")
|
||
|
||
def step_3_create_interaction_graph(self):
|
||
"""Step 3: Create card interaction graph tables."""
|
||
print("\n" + "=" * 60)
|
||
print("STEP 3: Creating card interaction graph")
|
||
print("=" * 60)
|
||
|
||
# Card mechanics table
|
||
print("\n📊 Creating mtg_card_mechanics table...")
|
||
self.create_table("""
|
||
CREATE TABLE IF NOT EXISTS mtg_card_mechanics (
|
||
id SERIAL PRIMARY KEY,
|
||
card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
|
||
mechanic VARCHAR(100) NOT NULL,
|
||
strength INTEGER DEFAULT 1,
|
||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||
UNIQUE(card_id, mechanic)
|
||
)
|
||
""")
|
||
indexes = [
|
||
"idx_mechanics_card_id ON mtg_card_mechanics(card_id)",
|
||
"idx_mechanics_mechanic ON mtg_card_mechanics(mechanic)",
|
||
]
|
||
for idx in indexes:
|
||
self.create_index(idx)
|
||
print(" ✓ Card mechanics table created")
|
||
|
||
# Card archetypes table
|
||
print("\n📊 Creating mtg_card_archetypes table...")
|
||
self.create_table("""
|
||
CREATE TABLE IF NOT EXISTS mtg_card_archetypes (
|
||
id SERIAL PRIMARY KEY,
|
||
card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
|
||
archetype VARCHAR(100) NOT NULL,
|
||
strength INTEGER DEFAULT 1,
|
||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||
UNIQUE(card_id, archetype)
|
||
)
|
||
""")
|
||
indexes = [
|
||
"idx_archetypes_card_id ON mtg_card_archetypes(card_id)",
|
||
"idx_archetypes_archetype ON mtg_card_archetypes(archetype)",
|
||
]
|
||
for idx in indexes:
|
||
self.create_index(idx)
|
||
print(" ✓ Card archetypes table created")
|
||
|
||
# Card themes table
|
||
print("\n📊 Creating mtg_card_themes table...")
|
||
self.create_table("""
|
||
CREATE TABLE IF NOT EXISTS mtg_card_themes (
|
||
id SERIAL PRIMARY KEY,
|
||
card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
|
||
theme VARCHAR(100) NOT NULL,
|
||
strength INTEGER DEFAULT 1,
|
||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||
UNIQUE(card_id, theme)
|
||
)
|
||
""")
|
||
indexes = [
|
||
"idx_themes_card_id ON mtg_card_themes(card_id)",
|
||
"idx_themes_theme ON mtg_card_themes(theme)",
|
||
]
|
||
for idx in indexes:
|
||
self.create_index(idx)
|
||
print(" ✓ Card themes table created")
|
||
|
||
# Card relationships table
|
||
print("\n📊 Creating mtg_card_relationships table...")
|
||
self.create_table("""
|
||
CREATE TABLE IF NOT EXISTS mtg_card_relationships (
|
||
id SERIAL PRIMARY KEY,
|
||
card_a_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
|
||
card_b_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
|
||
relationship_type VARCHAR(50) NOT NULL,
|
||
strength INTEGER DEFAULT 1,
|
||
notes TEXT,
|
||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||
UNIQUE(card_a_id, card_b_id, relationship_type)
|
||
)
|
||
""")
|
||
indexes = [
|
||
"idx_relationships_card_a ON mtg_card_relationships(card_a_id)",
|
||
"idx_relationships_card_b ON mtg_card_relationships(card_b_id)",
|
||
"idx_relationships_type ON mtg_card_relationships(relationship_type)",
|
||
]
|
||
for idx in indexes:
|
||
self.create_index(idx)
|
||
print(" ✓ Card relationships table created")
|
||
|
||
# Card synergies table
|
||
print("\n📊 Creating mtg_card_synergies table...")
|
||
self.create_table("""
|
||
CREATE TABLE IF NOT EXISTS mtg_card_synergies (
|
||
id SERIAL PRIMARY KEY,
|
||
card_a_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
|
||
card_b_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
|
||
synergy_type VARCHAR(50) NOT NULL,
|
||
strength INTEGER NOT NULL CHECK (strength BETWEEN 1 AND 5),
|
||
notes TEXT,
|
||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||
UNIQUE(card_a_id, card_b_id, synergy_type)
|
||
)
|
||
""")
|
||
indexes = [
|
||
"idx_synergies_card_a ON mtg_card_synergies(card_a_id)",
|
||
"idx_synergies_card_b ON mtg_card_synergies(card_b_id)",
|
||
"idx_synergies_type ON mtg_card_synergies(synergy_type)",
|
||
"idx_synergies_strength ON mtg_card_synergies(strength)",
|
||
]
|
||
for idx in indexes:
|
||
self.create_index(idx)
|
||
print(" ✓ Card synergies table created")
|
||
|
||
# Card counters table
|
||
print("\n📊 Creating mtg_card_counters table...")
|
||
self.create_table("""
|
||
CREATE TABLE IF NOT EXISTS mtg_card_counters (
|
||
id SERIAL PRIMARY KEY,
|
||
card_a_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
|
||
card_b_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
|
||
counter_type VARCHAR(50) NOT NULL,
|
||
strength INTEGER DEFAULT 1,
|
||
notes TEXT,
|
||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||
UNIQUE(card_a_id, card_b_id, counter_type)
|
||
)
|
||
""")
|
||
indexes = [
|
||
"idx_counters_card_a ON mtg_card_counters(card_a_id)",
|
||
"idx_counters_card_b ON mtg_card_counters(card_b_id)",
|
||
"idx_counters_type ON mtg_card_counters(counter_type)",
|
||
]
|
||
for idx in indexes:
|
||
self.create_index(idx)
|
||
print(" ✓ Card counters table created")
|
||
|
||
# Card evolution table
|
||
print("\n📊 Creating mtg_card_evolution table...")
|
||
self.create_table("""
|
||
CREATE TABLE IF NOT EXISTS mtg_card_evolution (
|
||
id SERIAL PRIMARY KEY,
|
||
card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
|
||
evolved_card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
|
||
evolution_type VARCHAR(50) NOT NULL,
|
||
strength INTEGER DEFAULT 1,
|
||
notes TEXT,
|
||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||
UNIQUE(card_id, evolved_card_id, evolution_type)
|
||
)
|
||
""")
|
||
indexes = [
|
||
"idx_evolution_card_id ON mtg_card_evolution(card_id)",
|
||
"idx_evolution_evolved_id ON mtg_card_evolution(evolved_card_id)",
|
||
"idx_evolution_type ON mtg_card_evolution(evolution_type)",
|
||
]
|
||
for idx in indexes:
|
||
self.create_index(idx)
|
||
print(" ✓ Card evolution table created")
|
||
|
||
# Card partners table
|
||
print("\n📊 Creating mtg_card_partners table...")
|
||
self.create_table("""
|
||
CREATE TABLE IF NOT EXISTS mtg_card_partners (
|
||
id SERIAL PRIMARY KEY,
|
||
card_a_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
|
||
card_b_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
|
||
partnership_type VARCHAR(50) NOT NULL,
|
||
strength INTEGER DEFAULT 1,
|
||
notes TEXT,
|
||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||
UNIQUE(card_a_id, card_b_id, partnership_type)
|
||
)
|
||
""")
|
||
indexes = [
|
||
"idx_partners_card_a ON mtg_card_partners(card_a_id)",
|
||
"idx_partners_card_b ON mtg_card_partners(card_b_id)",
|
||
"idx_partners_type ON mtg_card_partners(partnership_type)",
|
||
]
|
||
for idx in indexes:
|
||
self.create_index(idx)
|
||
print(" ✓ Card partners table created")
|
||
|
||
# Card mana relations table
|
||
print("\n📊 Creating mtg_card_mana_relations table...")
|
||
self.create_table("""
|
||
CREATE TABLE IF NOT EXISTS mtg_card_mana_relations (
|
||
id SERIAL PRIMARY KEY,
|
||
card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
|
||
land_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
|
||
mana_type VARCHAR(10) NOT NULL,
|
||
strength INTEGER DEFAULT 1,
|
||
notes TEXT,
|
||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||
UNIQUE(card_id, land_id, mana_type)
|
||
)
|
||
""")
|
||
indexes = [
|
||
"idx_mana_card_id ON mtg_card_mana_relations(card_id)",
|
||
"idx_mana_land_id ON mtg_card_mana_relations(land_id)",
|
||
"idx_mana_type ON mtg_card_mana_relations(mana_type)",
|
||
]
|
||
for idx in indexes:
|
||
self.create_index(idx)
|
||
print(" ✓ Card mana relations table created")
|
||
|
||
# Card set relations table
|
||
print("\n📊 Creating mtg_card_set_relations table...")
|
||
self.create_table("""
|
||
CREATE TABLE IF NOT EXISTS mtg_card_set_relations (
|
||
id SERIAL PRIMARY KEY,
|
||
card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
|
||
set_id INTEGER REFERENCES mtg_sets(id) ON DELETE CASCADE,
|
||
theme VARCHAR(100) NOT NULL,
|
||
strength INTEGER DEFAULT 1,
|
||
notes TEXT,
|
||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||
UNIQUE(card_id, set_id, theme)
|
||
)
|
||
""")
|
||
indexes = [
|
||
"idx_setrel_card_id ON mtg_card_set_relations(card_id)",
|
||
"idx_setrel_set_id ON mtg_card_set_relations(set_id)",
|
||
"idx_setrel_theme ON mtg_card_set_relations(theme)",
|
||
]
|
||
for idx in indexes:
|
||
self.create_index(idx)
|
||
print(" ✓ Card set relations table created")
|
||
|
||
# Card power relations table
|
||
print("\n📊 Creating mtg_card_power_relations table...")
|
||
self.create_table("""
|
||
CREATE TABLE IF NOT EXISTS mtg_card_power_relations (
|
||
id SERIAL PRIMARY KEY,
|
||
card_a_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
|
||
card_b_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
|
||
relation_type VARCHAR(50) NOT NULL,
|
||
strength INTEGER DEFAULT 1,
|
||
notes TEXT,
|
||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||
UNIQUE(card_a_id, card_b_id, relation_type)
|
||
)
|
||
""")
|
||
indexes = [
|
||
"idx_power_card_a ON mtg_card_power_relations(card_a_id)",
|
||
"idx_power_card_b ON mtg_card_power_relations(card_b_id)",
|
||
"idx_power_type ON mtg_card_power_relations(relation_type)",
|
||
]
|
||
for idx in indexes:
|
||
self.create_index(idx)
|
||
print(" ✓ Card power relations table created")
|
||
|
||
# Card history table
|
||
print("\n📊 Creating mtg_card_history table...")
|
||
self.create_table("""
|
||
CREATE TABLE IF NOT EXISTS mtg_card_history (
|
||
id SERIAL PRIMARY KEY,
|
||
card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
|
||
related_card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
|
||
history_type VARCHAR(50) NOT NULL,
|
||
strength INTEGER DEFAULT 1,
|
||
notes TEXT,
|
||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||
UNIQUE(card_id, related_card_id, history_type)
|
||
)
|
||
""")
|
||
indexes = [
|
||
"idx_history_card_id ON mtg_card_history(card_id)",
|
||
"idx_history_related_id ON mtg_card_history(related_card_id)",
|
||
"idx_history_type ON mtg_card_history(history_type)",
|
||
]
|
||
for idx in indexes:
|
||
self.create_index(idx)
|
||
print(" ✓ Card history table created")
|
||
|
||
# Card interaction stats table
|
||
print("\n📊 Creating mtg_card_interaction_stats table...")
|
||
self.create_table("""
|
||
CREATE TABLE IF NOT EXISTS mtg_card_interaction_stats (
|
||
id SERIAL PRIMARY KEY,
|
||
card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
|
||
total_synergies INTEGER DEFAULT 0,
|
||
total_counters INTEGER DEFAULT 0,
|
||
total_evolution INTEGER DEFAULT 0,
|
||
total_partners INTEGER DEFAULT 0,
|
||
total_mechanics INTEGER DEFAULT 0,
|
||
total_archetypes INTEGER DEFAULT 0,
|
||
total_themes INTEGER DEFAULT 0,
|
||
avg_synergy_strength DECIMAL(3,2) DEFAULT 0.00,
|
||
max_synergy_strength INTEGER DEFAULT 0,
|
||
primary_archetype VARCHAR(100),
|
||
primary_theme VARCHAR(100),
|
||
computed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||
UNIQUE(card_id)
|
||
)
|
||
""")
|
||
indexes = [
|
||
"idx_stats_card_id ON mtg_card_interaction_stats(card_id)",
|
||
"idx_stats_total_synergies ON mtg_card_interaction_stats(total_synergies)",
|
||
"idx_stats_primary_archetype ON mtg_card_interaction_stats(primary_archetype)",
|
||
]
|
||
for idx in indexes:
|
||
self.create_index(idx)
|
||
print(" ✓ Card interaction stats table created")
|
||
|
||
print("\n✓ Step 3 complete: Interaction graph tables created")
|
||
|
||
def step_4_populate_interaction_graph(self):
|
||
"""Step 4: Populate interaction graph from existing data."""
|
||
print("\n" + "=" * 60)
|
||
print("STEP 4: Populating interaction graph from existing data")
|
||
print("=" * 60)
|
||
|
||
# Populate mechanics from subtypes
|
||
print("\n🔄 Populating mechanics from subtypes...")
|
||
|
||
self.conn.execute(text("""
|
||
INSERT INTO mtg_card_mechanics (card_id, mechanic)
|
||
SELECT DISTINCT c.id, LOWER(UNNEST(string_to_array(c.subtypes, ',')))
|
||
FROM mtg_cards c
|
||
WHERE c.subtypes IS NOT NULL
|
||
AND c.subtypes != 'null'
|
||
AND LOWER(UNNEST(string_to_array(c.subtypes, ','))) IN (
|
||
'flying', 'first_strike', 'double_strike', 'deathtouch', 'lifelink',
|
||
'haste', 'trample', 'menace', 'vigilance', 'reach', 'indestructible',
|
||
'hexproof', 'shroud', 'defender', 'landfall', 'delve', 'soulshift',
|
||
'suspend', 'convoke', 'rampage', 'toxic', 'crew', 'equip', 'annihilator',
|
||
'spectacle', 'prowess', 'aftermath', 'adapt', 'amplify', 'awaken',
|
||
'banding', 'bestow', 'burst', 'channel', 'clash', 'crawl', 'curse',
|
||
'day_night', 'decay', 'defiant', 'demolish', 'detain', 'detect',
|
||
'devour', 'disguise', 'disturb', 'dome', 'double_strike', 'dredge',
|
||
'emerge', 'encore', 'endure', 'evoke', 'evolve', 'exalted', 'exile',
|
||
'exploit', 'extort', 'fairy', 'fanatic', 'fathom', 'fear', 'feline',
|
||
'flash', 'flight', 'foretell', 'frenzy', 'fumble', 'galvanize',
|
||
'gateway', 'genesis', 'graft', 'grave', 'grit', 'guardian', 'harvest',
|
||
'healer', 'heroic', 'hideaway', 'hinterland', 'hoard', 'hour', 'illusion',
|
||
'immortal', 'impulse', 'inspiration', 'instill', 'iron', 'junk', 'kicker',
|
||
'knight', 'land', 'leech', 'lich', 'lifespan', 'lightning', 'living',
|
||
'lurk', 'madness', 'manifest', 'map', 'meld', 'miracle', 'mitosis',
|
||
'modular', 'moon', 'mother', 'morph', 'mutate', 'ninja', 'night',
|
||
'nightmare', 'pact', 'paradox', 'persist', 'pillage', 'pivot', 'planar',
|
||
'polar', 'pour', 'prey', 'priest', 'primer', 'probe', 'prosperity',
|
||
'psychic', 'puppet', 'quest', 'quote', 'rage', 'raid', 'raise', 'rally',
|
||
'rapid', 'rat', 'rebound', 'reckless', 'recoup', 'reflect', 'refresh',
|
||
'replicate', 'reverberate', 'reviviant', 'rift', 'rip', 'ritual', 'rite',
|
||
'rogue', 'savant', 'scavenge', 'seek', 'shadow', 'shards', 'skulk',
|
||
'smelt', 'snap', 'snow', 'spectacle', 'splice', 'spore', 'sprawl',
|
||
'stabilize', 'stasis', 'storm', 'story', 'substitute', 'sunder', 'surge',
|
||
'survive', 'swarm', 'symbiosis', 'synchronized', 'synth', 'table', 'taint',
|
||
'tank', 'thorn', 'thwart', 'time', 'tinker', 'toxin', 'trail', 'transfigure',
|
||
'transform', 'transport', 'trouble', 'tunnel', 'unearth', 'unleash', 'unmask',
|
||
'unstoppable', 'urborg', 'urgent', 'utility', 'vengeful', 'vanish', 'venom',
|
||
'victory', 'villainous', 'vitalize', 'void', 'voyage', 'ward', 'watch', 'weave',
|
||
'wed', 'whammy', 'wild', 'will', 'wisp', 'witch', 'woe', 'wounded', 'wrap',
|
||
'wrought', 'wurm', 'wythe'
|
||
)
|
||
ON CONFLICT DO NOTHING
|
||
"""))
|
||
print(" ✓ Populated mechanics from subtypes")
|
||
|
||
# Populate archetypes from subtypes
|
||
print("\n🔄 Populating archetypes from subtypes...")
|
||
|
||
self.conn.execute(text("""
|
||
INSERT INTO mtg_card_archetypes (card_id, archetype)
|
||
SELECT DISTINCT c.id, LOWER(UNNEST(string_to_array(c.subtypes, ',')))
|
||
FROM mtg_cards c
|
||
WHERE c.subtypes IS NOT NULL
|
||
AND c.subtypes != 'null'
|
||
AND LOWER(UNNEST(string_to_array(c.subtypes, ','))) IN (
|
||
'goblin', 'elf', 'vampire', 'angel', 'dragon', 'human', 'zombie',
|
||
'soldier', 'knight', 'wizard', 'spirit', 'demon', 'snake', 'cat',
|
||
'wolf', 'bear', 'bird', 'insect', 'horror', 'goat', 'ox', 'elephant',
|
||
'whale', 'shark', 'fish', 'serpent', 'lizard', 'scorpion', 'spider',
|
||
'rat', 'drake', 'wyvern', 'phoenix', 'lynx', 'jaguar', 'hydra',
|
||
'leviathan', 'kraken', 'cyclops', 'golem', 'homunculus', 'clay',
|
||
'construct', 'myr', 'aether', 'pumpkin', 'pirate', 'pegasus',
|
||
'unicorn', 'centaur', 'merfolk', 'mermaid', 'naga', 'satyr', 'dryad',
|
||
'treant', 'elemental', 'fiend', 'imp', 'faerie', 'minion', 'abomination',
|
||
'beast', 'demigod', 'god', 'avatar', 'guardian', 'warrior', 'rogue',
|
||
'artificer', 'bard', 'monk', 'ninja', 'samurai', 'assassin', 'thief',
|
||
'acrobat', 'explorer', 'farmer', 'myth', 'illusion', 'mirror', 'phantom',
|
||
'shapeshifter', 'shaman', 'skeleton', 'slime', 'squirrel', 'troll',
|
||
'tyrannosaur', 'wraith', 'wurm'
|
||
)
|
||
ON CONFLICT DO NOTHING
|
||
"""))
|
||
print(" ✓ Populated archetypes from subtypes")
|
||
|
||
self.conn.commit()
|
||
print("\n✓ Step 4 complete: Interaction graph populated")
|
||
|
||
def step_5_create_sample_interactions(self):
|
||
"""Step 5: Create sample interactions to demonstrate the system."""
|
||
print("\n" + "=" * 60)
|
||
print("STEP 5: Creating sample interactions")
|
||
print("=" * 60)
|
||
|
||
# Get a sample of cards to create interactions between
|
||
result = self.conn.execute(text("""
|
||
SELECT id, name, subtypes, types, colors
|
||
FROM mtg_cards
|
||
WHERE subtypes IS NOT NULL AND subtypes != 'null'
|
||
LIMIT 50
|
||
""")).fetchall()
|
||
|
||
if len(result) < 2:
|
||
print(" ℹ️ Not enough cards with subtypes to create sample interactions")
|
||
return
|
||
|
||
print(f" ✓ Found {len(result)} cards with subtypes")
|
||
|
||
# Create sample synergies between cards with same archetype
|
||
print("\n🔄 Creating sample synergies...")
|
||
|
||
# Group cards by archetype
|
||
archetype_cards = {}
|
||
for card_id, name, subtypes, types, colors in result:
|
||
if subtypes:
|
||
for archetype in [a.strip() for a in subtypes.split(',') if a.strip()]:
|
||
if archetype not in archetype_cards:
|
||
archetype_cards[archetype] = []
|
||
archetype_cards[archetype].append(card_id)
|
||
|
||
# Create synergies between cards of the same archetype
|
||
synergy_count = 0
|
||
for archetype, card_ids in archetype_cards.items():
|
||
if len(card_ids) >= 2:
|
||
for i in range(len(card_ids)):
|
||
for j in range(i + 1, len(card_ids)):
|
||
self.conn.execute(text("""
|
||
INSERT INTO mtg_card_synergies (card_a_id, card_b_id, synergy_type, strength, notes)
|
||
VALUES (:card_a, :card_b, :synergy_type, :strength, :notes)
|
||
ON CONFLICT DO NOTHING
|
||
"""), {
|
||
"card_a": card_ids[i],
|
||
"card_b": card_ids[j],
|
||
"synergy_type": "archetype_support",
|
||
"strength": 3,
|
||
"notes": f"Both {archetype} cards work well together"
|
||
})
|
||
synergy_count += 1
|
||
|
||
print(f" ✓ Created {synergy_count} archetype synergies")
|
||
|
||
# Create sample counters between cards with different colors
|
||
print("\n🔄 Creating sample counters...")
|
||
|
||
counter_count = 0
|
||
for i in range(min(20, len(result))):
|
||
card_a_id = result[i][0]
|
||
card_a_colors = result[i][4]
|
||
|
||
if card_a_colors:
|
||
colors_a = [c.strip() for c in card_a_colors.split(',')]
|
||
|
||
for j in range(i + 1, min(i + 10, len(result))):
|
||
card_b_id = result[j][0]
|
||
card_b_colors = result[j][4]
|
||
|
||
if card_b_colors:
|
||
colors_b = [c.strip() for c in card_b_colors.split(',')]
|
||
|
||
# If different colors, create a counter relationship
|
||
if set(colors_a) != set(colors_b):
|
||
self.conn.execute(text("""
|
||
INSERT INTO mtg_card_counters (card_a_id, card_b_id, counter_type, strength, notes)
|
||
VALUES (:card_a, :card_b, :counter_type, :strength, :notes)
|
||
ON CONFLICT DO NOTHING
|
||
"""), {
|
||
"card_a": card_a_id,
|
||
"card_b": card_b_id,
|
||
"counter_type": "mana_disadvantage",
|
||
"strength": 2,
|
||
"notes": "Different color identities create strategic tension"
|
||
})
|
||
counter_count += 1
|
||
|
||
print(f" ✓ Created {counter_count} counter relationships")
|
||
|
||
# Create sample evolutions for cards with same name in different sets
|
||
print("\n🔄 Creating sample evolutions...")
|
||
|
||
self.conn.execute(text("""
|
||
INSERT INTO mtg_card_evolution (card_id, evolved_card_id, evolution_type, strength, notes)
|
||
SELECT DISTINCT c1.id, c2.id, 'reprinted', 2, 'Reprint in different set'
|
||
FROM mtg_cards c1
|
||
JOIN mtg_cards c2 ON c1.name = c2.name AND c1.set_id != c2.set_id
|
||
WHERE c1.subtypes IS NOT NULL AND c2.subtypes IS NOT NULL
|
||
LIMIT 50
|
||
ON CONFLICT DO NOTHING
|
||
"""))
|
||
print(" ✓ Created sample evolutions")
|
||
|
||
self.conn.commit()
|
||
print("\n✓ Step 5 complete: Sample interactions created")
|
||
|
||
def step_6_update_interaction_stats(self):
|
||
"""Step 6: Update interaction statistics for each card."""
|
||
print("\n" + "=" * 60)
|
||
print("STEP 6: Updating interaction statistics")
|
||
print("=" * 60)
|
||
|
||
# Delete existing stats
|
||
self.conn.execute(text("DELETE FROM mtg_card_interaction_stats"))
|
||
|
||
# Calculate and insert stats
|
||
self.conn.execute(text("""
|
||
INSERT INTO mtg_card_interaction_stats (
|
||
card_id, total_synergies, total_counters, total_evolution,
|
||
total_partners, total_mechanics, total_archetypes, total_themes,
|
||
avg_synergy_strength, max_synergy_strength, primary_archetype, primary_theme
|
||
)
|
||
SELECT
|
||
c.id,
|
||
COALESCE(synergies.synergy_count, 0),
|
||
COALESCE(counters.counter_count, 0),
|
||
COALESCE(evolution.evolution_count, 0),
|
||
COALESCE(partners.partner_count, 0),
|
||
COALESCE(mechanics.mechanic_count, 0),
|
||
COALESCE(archetypes.archetype_count, 0),
|
||
COALESCE(themes.theme_count, 0),
|
||
COALESCE(synergies.avg_strength, 0),
|
||
COALESCE(synergies.max_strength, 0),
|
||
archetypes.primary_archetype,
|
||
themes.primary_theme
|
||
FROM mtg_cards c
|
||
LEFT JOIN (
|
||
SELECT card_a_id as card_id, COUNT(*) as synergy_count,
|
||
AVG(strength) as avg_strength, MAX(strength) as max_strength
|
||
FROM mtg_card_synergies
|
||
GROUP BY card_a_id
|
||
) synergies ON c.id = synergies.card_id
|
||
LEFT JOIN (
|
||
SELECT card_a_id as card_id, COUNT(*) as counter_count
|
||
FROM mtg_card_counters
|
||
GROUP BY card_a_id
|
||
) counters ON c.id = counters.card_id
|
||
LEFT JOIN (
|
||
SELECT card_id as card_id, COUNT(*) as evolution_count
|
||
FROM mtg_card_evolution
|
||
GROUP BY card_id
|
||
) evolution ON c.id = evolution.card_id
|
||
LEFT JOIN (
|
||
SELECT card_a_id as card_id, COUNT(*) as partner_count
|
||
FROM mtg_card_partners
|
||
GROUP BY card_a_id
|
||
) partners ON c.id = partners.card_id
|
||
LEFT JOIN (
|
||
SELECT card_id as card_id, COUNT(*) as mechanic_count
|
||
FROM mtg_card_mechanics
|
||
GROUP BY card_id
|
||
) mechanics ON c.id = mechanics.card_id
|
||
LEFT JOIN (
|
||
SELECT card_id as card_id, COUNT(*) as archetype_count
|
||
FROM mtg_card_archetypes
|
||
GROUP BY card_id
|
||
) archetypes ON c.id = archetypes.card_id
|
||
LEFT JOIN (
|
||
SELECT card_id as card_id, COUNT(*) as theme_count
|
||
FROM mtg_card_themes
|
||
GROUP BY card_id
|
||
) themes ON c.id = themes.card_id
|
||
LEFT JOIN (
|
||
SELECT card_id, archetype as primary_archetype
|
||
FROM mtg_card_archetypes a1
|
||
WHERE id = (
|
||
SELECT MIN(a2.id)
|
||
FROM mtg_card_archetypes a2
|
||
WHERE a1.card_id = a2.card_id
|
||
)
|
||
) archetypes ON c.id = archetypes.card_id
|
||
LEFT JOIN (
|
||
SELECT card_id, theme as primary_theme
|
||
FROM mtg_card_themes t1
|
||
WHERE id = (
|
||
SELECT MIN(t2.id)
|
||
FROM mtg_card_themes t2
|
||
WHERE t1.card_id = t2.card_id
|
||
)
|
||
) themes ON c.id = themes.card_id
|
||
"""))
|
||
|
||
print(" ✓ Updated interaction statistics")
|
||
|
||
self.conn.commit()
|
||
print("\n✓ Step 6 complete: Interaction statistics updated")
|
||
|
||
def run_migration(self):
|
||
"""Run the complete migration."""
|
||
print("=" * 60)
|
||
print("🚀 Running Complete MTGJSON Migration")
|
||
print("=" * 60)
|
||
|
||
self.connect()
|
||
|
||
try:
|
||
self.step_1_add_mtgjson_columns()
|
||
self.step_2_populate_mtgjson_columns()
|
||
self.step_3_create_interaction_graph()
|
||
self.step_4_populate_interaction_graph()
|
||
self.step_5_create_sample_interactions()
|
||
self.step_6_update_interaction_stats()
|
||
|
||
self.disconnect()
|
||
|
||
print("\n" + "=" * 60)
|
||
print("✅ Complete migration finished successfully!")
|
||
print("=" * 60)
|
||
print("\n📊 Summary:")
|
||
print(" • Added 35+ MTGJSON columns to mtg_cards table")
|
||
print(" • Added 7 MTGJSON columns to mtg_sets table")
|
||
print(" • Created 13 interaction graph tables")
|
||
print(" • Populated mechanics, archetypes, and synergies")
|
||
print(" • Created sample card interactions")
|
||
|
||
except Exception as e:
|
||
print(f"\n❌ Migration failed: {e}")
|
||
raise
|
||
finally:
|
||
if self.conn:
|
||
self.conn.close()
|
||
|
||
|
||
def main():
|
||
"""Main entry point."""
|
||
migration = MTGJSONFullMigration()
|
||
migration.run_migration()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|