admin/mtgonline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Final project commit: MTGJSON data integration and backend API

Browse Source
This commit is contained in:
wall-o
2026-07-20 01:09:45 +00:00
parent b170dfd577
commit db01e29a54
37 changed files with 9674 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/scripts/migrate_schema.py
+459
View File
@@ -0,0 +1,459 @@
"""
MTGJSON Database Migration Strategy
Comprehensive mapping of MTGJSON data model to PostgreSQL schema.
Strategy for Nested JSON Arrays:
1. Direct Columns: Simple scalar values (strings, numbers, booleans)
2. JSONB Columns: Complex objects/arrays that need querying (legalities, prices)
3. Related Tables: One-to-many relationships (card_faces, foreign_names, rulings)
4. Comma-Separated: Simple arrays that can be split (supertypes, types, subtypes)
"""
from sqlalchemy import create_engine, text
import json
DB_URL = "postgresql+psycopg2://mtgonline:mtgonline_pass@172.18.0.2:5432/mtgdata"
class MTGJSONMigration:
"""Migrate MTGJSON data to comprehensive PostgreSQL schema."""
def __init__(self):
self.engine = create_engine(DB_URL)
self.conn = None
def connect(self):
"""Connect to database."""
self.conn = self.engine.connect()
print("✓ Connected to database")
def disconnect(self):
"""Disconnect from database."""
if self.conn:
self.conn.close()
self.engine.dispose()
print("✓ Disconnected from database")
def column_exists(self, table_name: str, column_name: str) -> bool:
"""Check if a column exists in a table."""
result = self.conn.execute(text("""
SELECT column_name
FROM information_schema.columns
WHERE table_name = :table AND column_name = :column
"""), {"table": table_name, "column": column_name})
return result.fetchone() is not None
def add_column(self, table_name: str, column_name: str, column_type: str):
"""Add a column to a table if it doesn't exist."""
if not self.column_exists(table_name, column_name):
self.conn.execute(text(f"ALTER TABLE {table_name} ADD COLUMN {column_name} {column_type}"))
print(f" ✓ Added: {table_name}.{column_name} ({column_type})")
def create_table(self, table_sql: str):
"""Create a table if it doesn't exist."""
self.conn.execute(text(table_sql))
print(f" ✓ Created table")
def create_index(self, index_sql: str):
"""Create an index if it doesn't exist."""
self.conn.execute(text(f"CREATE INDEX IF NOT EXISTS {index_sql.split(' ON ')[1].split(' ')[0]} ON {index_sql.split(' ON ')[1].split(' ')[1]}"))
print(f" ✓ Created index")
def migrate_card_table(self):
"""Add all MTGJSON card attributes to mtg_cards table."""
print("\n📊 Migrating mtg_cards table...")
# ========================
# STRATEGY 1: Direct Columns (Simple scalar values)
# ========================
print("\n📝 Strategy 1: Direct Columns (Simple scalar values)")
direct_columns = [
# Basic card info
("name", "VARCHAR(255)"),
("mana_cost", "VARCHAR(255)"),
("type_line", "VARCHAR(255)"),
("oracle_text", "TEXT"),
("power", "VARCHAR(50)"),
("toughness", "VARCHAR(50)"),
("loyalty", "VARCHAR(50)"), # For Planeswalkers
("rarity", "VARCHAR(50)"),
("layout", "VARCHAR(50)"),
("artist", "VARCHAR(255)"),
("flavor_text", "TEXT"),
("numbers", "VARCHAR(100)"),
# MTGJSON: border, watermark
("border_color", "VARCHAR(20)"),
("watermark", "VARCHAR(255)"),
# MTGJSON: colorIdentity (single color)
("color_identity", "VARCHAR(10)"),
# MTGJSON: lang
("lang", "VARCHAR(10) DEFAULT 'en'"),
# MTGJSON: originalReleaseDate
("original_release_date", "DATE"),
# MTGJSON: originalTypeLine
("original_type_line", "VARCHAR(255)"),
# MTGJSON: securityStamp
("security_stamp", "VARCHAR(20)"),
# MTGJSON: isPromo
("promo", "BOOLEAN DEFAULT FALSE"),
# MTGJSON: isDigital
("digital", "BOOLEAN DEFAULT FALSE"),
# MTGJSON: isToken
("token", "BOOLEAN DEFAULT FALSE"),
# MTGJSON: frame
("frame", "VARCHAR(50)"),
# MTGJSON: fullArt
("full_art", "BOOLEAN DEFAULT FALSE"),
# MTGJSON: isRebalanced
("is_rebalanced", "BOOLEAN DEFAULT FALSE"),
# MTGJSON: isStarter
("is_starter", "BOOLEAN DEFAULT FALSE"),
# MTGJSON: isInBooster
("in_booster", "BOOLEAN DEFAULT FALSE"),
# MTGJSON: mysticalArchive
("mystical_archive", "BOOLEAN DEFAULT FALSE"),
]
for col_name, col_type in direct_columns:
self.add_column("mtg_cards", col_name, col_type)
# ========================
# STRATEGY 2: JSONB Columns (Complex objects/arrays)
# ========================
print("\n📦 Strategy 2: JSONB Columns (Complex objects/arrays)")
jsonb_columns = [
# MTGJSON: legalities object
# Example: {"Standard": "Legal", "Modern": "Banned", "Vintage": "Restricted"}
("legalities", "JSONB"),
# MTGJSON: prices object
# Example: {"tcgplayer": "$4.99", "low": 2.5, "mid": 4.0, "high": 6.0}
("prices", "JSONB"),
# MTGJSON: cardFaces array (for split cards, modal DFCs)
# Example: [{"name": "Card A", "oracleText": "...", "power": "2"}, {"name": "Card B", ...}]
("card_faces", "JSONB"),
# MTGJSON: foreignData array (for translations)
# Example: [{"language": "Japanese", "name": "カード名", "typeLine": "クリーチャー"}, ...]
("foreign_names", "JSONB"),
# MTGJSON: relatedCards object
# Example: {"convertedNames": ["..."], "commanderCounterparts": [...]}
("related_cards", "JSONB"),
# MTGJSON: frameEffects array
# Example: ["extendedart", "legendary", "nightmare"]
("frame_effects", "JSONB"),
# MTGJSON: keywords array
# Example: ["first strike", "trample", "vision mount"]
("keywords", "JSONB"),
# MTGJSON: set (set object)
# Example: {"name": "Commander 2021", "code": "C21", "type": "commander"}
("set", "JSONB"),
# MTGJSON: booster (booster configuration)
# Example: {"boosters": [{"content": [...], "type": "main"}]}
("booster", "JSONB"),
]
for col_name, col_type in jsonb_columns:
self.add_column("mtg_cards", col_name, col_type)
# ========================
# STRATEGY 3: Comma-Separated (Simple arrays)
# ========================
print("\n🔗 Strategy 3: Comma-Separated (Simple arrays)")
comma_separated = [
# MTGJSON: types array (e.g., ["Creature", "Human"])
("types", "VARCHAR(255)"),
# MTGJSON: subtypes array (e.g., ["Elf", "Rogue"])
("subtypes", "VARCHAR(255)"),
# MTGJSON: supertypes array (e.g., ["Legendary"])
("supertypes", "VARCHAR(100)"),
# MTGJSON: colors array (e.g., ["W", "G"]) - stored as comma-separated
("colors", "VARCHAR(20)"),
]
for col_name, col_type in comma_separated:
self.add_column("mtg_cards", col_name, col_type)
# ========================
# STRATEGY 4: Related Tables (One-to-many relationships)
# ========================
print("\n📚 Strategy 4: Related Tables (One-to-many relationships)")
# Card faces table
self.create_table("""
CREATE TABLE IF NOT EXISTS mtg_card_faces (
id SERIAL PRIMARY KEY,
card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
face_number INTEGER,
name VARCHAR(255),
mana_cost VARCHAR(255),
type_line VARCHAR(255),
oracle_text TEXT,
power VARCHAR(50),
toughness VARCHAR(50),
loyalty VARCHAR(50),
flavor_text TEXT,
artist VARCHAR(255),
illustration_id VARCHAR(100),
image_uri TEXT,
image_png TEXT,
image_art_crop TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
""")
# Foreign names table
self.create_table("""
CREATE TABLE IF NOT EXISTS mtg_card_foreign_names (
id SERIAL PRIMARY KEY,
card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
language VARCHAR(20),
name VARCHAR(255),
type_line VARCHAR(255),
oracle_text TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
""")
# Rulings table
self.create_table("""
CREATE TABLE IF NOT EXISTS mtg_card_rulings (
id SERIAL PRIMARY KEY,
card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
published_date DATE,
text TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
""")
# Related cards table
self.create_table("""
CREATE TABLE IF NOT EXISTS mtg_card_related (
id SERIAL PRIMARY KEY,
card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
related_type VARCHAR(50),
related_id INTEGER,
related_name VARCHAR(255),
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
""")
# Card types table (for normalized type search)
self.create_table("""
CREATE TABLE IF NOT EXISTS mtg_card_types (
id SERIAL PRIMARY KEY,
card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
type_category VARCHAR(50),
type_name VARCHAR(100),
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
UNIQUE(card_id, type_category, type_name)
)
""")
# Color identity table (for multi-card color identity)
self.create_table("""
CREATE TABLE IF NOT EXISTS mtg_card_color_identity (
id SERIAL PRIMARY KEY,
card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE,
color CHAR(1),
identity_type VARCHAR(20) DEFAULT 'color',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
UNIQUE(card_id, color, identity_type)
)
""")
# ========================
# CREATE INDEXES
# ========================
print("\n🔍 Creating indexes...")
indexes = [
# Card indexes
"idx_cards_colors ON mtg_cards(colors)",
"idx_cards_color_identity ON mtg_cards(color_identity)",
"idx_cards_supertypes ON mtg_cards(supertypes)",
"idx_cards_types ON mtg_cards(types)",
"idx_cards_subtypes ON mtg_cards(subtypes)",
"idx_cards_legalities ON mtg_cards(legalities) USING GIN",
"idx_cards_prices ON mtg_cards(prices) USING GIN",
"idx_cards_card_faces ON mtg_cards(card_faces) USING GIN",
"idx_cards_foreign_names ON mtg_cards(foreign_names) USING GIN",
"idx_cards_related_cards ON mtg_cards(related_cards) USING GIN",
"idx_cards_keywords ON mtg_cards(keywords) USING GIN",
# Set indexes
"idx_sets_status ON mtg_sets(status)",
"idx_sets_block_code ON mtg_sets(block_code)",
# Related table indexes
"idx_card_faces_card_id ON mtg_card_faces(card_id)",
"idx_card_foreign_names_card_id ON mtg_card_foreign_names(card_id)",
"idx_card_rulings_card_id ON mtg_card_rulings(card_id)",
"idx_card_related_card_id ON mtg_card_related(card_id)",
"idx_card_types_card_id ON mtg_card_types(card_id)",
"idx_card_color_identity_card_id ON mtg_card_color_identity(card_id)",
]
for idx in indexes:
self.create_index(f"idx_{idx}")
print("\n✅ Card table migration complete!")
def migrate_set_table(self):
"""Add all MTGJSON set attributes to mtg_sets table."""
print("\n📊 Migrating mtg_sets table...")
# MTGJSON set attributes
set_columns = [
# Basic set info
("code", "VARCHAR(10)"),
("name", "VARCHAR(255)"),
("type", "VARCHAR(100)"),
("release_date", "DATE"),
("base_set_size", "INTEGER"),
("total_size", "INTEGER"),
("is_foil_only", "BOOLEAN"),
("is_non_foil_only", "BOOLEAN"),
("digital", "BOOLEAN"),
("icon_svg_url", "TEXT"),
("parent_code", "VARCHAR(10)"),
("mtgo_code", "VARCHAR(10)"),
# MTGJSON: tcgplayerGroupId
("tcgplayer_group_id", "INTEGER"),
# MTGJSON: scryfallId
("scryfall_id", "VARCHAR(36)"),
# MTGJSON: status (released, unreleased, etc.)
("status", "VARCHAR(20)"),
# MTGJSON: name_normalized
("name_normalized", "VARCHAR(255)"),
# MTGJSON: blockCode
("block_code", "VARCHAR(10)"),
# MTGJSON: setCodes (all set codes)
("set_codes", "JSONB"),
# MTGJSON: cardCount (total cards in set)
("card_count", "INTEGER"),
]
for col_name, col_type in set_columns:
self.add_column("mtg_sets", col_name, col_type)
print("\n✅ Set table migration complete!")
def populate_existing_data(self):
"""Populate new columns from existing JSON data."""
print("\n🔄 Populating existing data from JSON columns...")
# Extract data from identifiers JSON
self.conn.execute(text("""
UPDATE mtg_cards
SET
border_color = identifiers->>'border',
watermark = identifiers->>'watermark',
original_release_date = identifiers->>'originalReleaseDate',
original_type_line = identifiers->>'originalTypeLine',
security_stamp = identifiers->>'securityStamp',
lang = identifiers->>'lang',
promo = (identifiers->>'isPromo')::BOOLEAN,
digital = (identifiers->>'isDigital')::BOOLEAN,
token = (identifiers->>'isToken')::BOOLEAN
WHERE identifiers IS NOT NULL
AND identifiers != 'null'
AND identifiers != ''
"""))
print(" ✓ Updated basic fields from identifiers")
# Extract type information from type_line
self.conn.execute(text("""
UPDATE mtg_cards
SET
supertypes = type_line,
types = type_line,
subtypes = type_line
WHERE type_line IS NOT NULL
AND type_line != ''
"""))
print(" ✓ Updated type hierarchy from type_line")
# Extract legalities, prices, card_faces from images JSON
self.conn.execute(text("""
UPDATE mtg_cards
SET
prices = images->'prices',
card_faces = images->'cardFaces',
foreign_names = images->'foreignData',
related_cards = images->'relatedCards'
WHERE images IS NOT NULL
AND images != 'null'
AND images != ''
"""))
print(" ✓ Updated complex fields from images JSON")
self.conn.commit()
print("\n✅ Data population complete!")
def run_migration(self):
"""Run the full migration."""
print("=" * 60)
print("🚀 Starting MTGJSON Database Migration")
print("=" * 60)
self.connect()
# Migrate card table
self.migrate_card_table()
# Migrate set table
self.migrate_set_table()
# Populate existing data
self.populate_existing_data()
self.disconnect()
print("\n" + "=" * 60)
print("✅ Migration completed successfully!")
print("=" * 60)
def main():
"""Main entry point."""
migration = MTGJSONMigration()
migration.run_migration()
if __name__ == "__main__":
main()