diff --git a/backend/SPEC_synergy-mapping-engine.md b/backend/SPEC_synergy-mapping-engine.md new file mode 100644 index 0000000..5abf4c1 --- /dev/null +++ b/backend/SPEC_synergy-mapping-engine.md @@ -0,0 +1,82 @@ +# Technical Specification: MTG Synergy Mapping Engine + +## 1. Project Overview +The goal is to create a Python-based data pipeline that processes MTG card data from MTGJSON, identifies synergistic relationships between cards, and stores these relationships in a PostgreSQL database. This "Data Map" will power a deck-building assistant that suggests cards based on mechanical and strategic complementarity. + +## 2. Tech Stack +- **Language:** Python 3.12+ +- **Libraries:** `pandas` (data manipulation), `SQLAlchemy` (ORM), `psycopg2` (DB driver), `re` (regex for text processing). +- **Database:** PostgreSQL. +- **Data Source:** MTGJSON (`AllPrintings.json`, `AllSets.json`). + +## 3. Phase 1: Data Ingestion & Normalization +The script must flatten the nested MTGJSON structure into a relational format. + +### 3.1 Extraction +Extract the following fields from `AllPrintings.json`: +- `name`, `manaCost`, `types`, `text` (oracle text), `colorIdentity`, `set`. + +### 3.2 Text Processing (`TextProcessor` Class) +Implement a class to convert raw oracle text into "Functional Tokens." +- **Regex Mapping:** Use a dictionary of regex patterns to identify key actions. + - *Example:* `"draw a card"` $\rightarrow$ `TOKEN_DRAW_1` + - *Example:* `"destroy all creatures"` $\rightarrow$ `TOKEN_BOARD_WIPE_CREATURE` +- **Tagging:** Extract subtypes (Tribes) from the `types` field (e.g., "Elf", "Zombie"). + +## 4. Phase 2: The Synergy Engine (Logic) +The engine must evaluate every card pair and assign a weighted connection based on three tiers of synergy. + +### Tier A: Hard Synergies (Weight: 1.0) +**Logic:** Direct mechanical triggers. +- **Tribal Link:** If `Card_A.tags` (Tribe) $\cap$ `Card_B.text` (contains Tribe name) $\neq \emptyset$. +- **Trigger-Response:** Identify "Providers" (e.g., "Whenever you gain life") and "Payoffs" (e.g., "When you gain life, [Effect]"). Link Provider $\rightarrow$ Payoff. + +### Tier B: Functional Similarity (Weight: 0.6) +**Logic:** Substitution/Role mapping. +- **Role Dictionary:** Define roles (e.g., `RAMP`, `CARD_DRAW`, `REMOVAL`). +- **Mapping:** If both cards share the same `Role_ID` based on their Functional Tokens, create a link. + +### Tier C: Strategic Archetypes (Weight: 0.3) +**Logic:** Thematic co-occurrence. +- **Archetype Buckets:** Define keyword groups (e.g., `GRAVEYARD_STRAT` = ["mill", "graveyard", "reanimate"]). +- **Density Check:** If both cards have a high overlap of keywords from the same bucket, create a link. + +## 5. Phase 3: Database Schema (PSQL) +Implement the following schema: + +### Table: `cards` +- `card_id`: UUID (Primary Key) +- `name`: VARCHAR +- `oracle_text`: TEXT +- `mana_cost`: VARCHAR +- `color_identity`: ARRAY[VARCHAR] +- `tags`: ARRAY[VARCHAR] (Stored functional tokens and tribes) + +### Table: `synergy_types` +- `type_id`: INT (Primary Key) +- `label`: VARCHAR (e.g., 'Tribal', 'Mechanical', 'Substitute') + +### Table: `card_connections` +- `card_id_a`: UUID (FK $\rightarrow$ cards) +- `card_id_b`: UUID (FK $\rightarrow$ cards) +- `type_id`: INT (FK $\rightarrow$ synergy_types) +- `weight`: FLOAT +- **Constraint:** `CHECK (card_id_a < card_id_b)` to prevent bidirectional duplicates. + +## 6. Phase 4: Execution Pipeline +The script must execute in the following order: +1. **Ingest:** Parse JSON $\rightarrow$ Bulk load into `cards` table. +2. **Analyze:** Run `TextProcessor` $\rightarrow$ Update `cards.tags`. +3. **Map:** + - Iterate through card pairs. + - Evaluate Tiers A, B, and C. + - Insert identified synergies into `card_connections`. +4. **Index:** Create B-Tree indices on `card_id_a` and `card_id_b`. + +## 7. Phase 5: Recommendation Logic (API Level) +The resulting database must support the following query logic for the API: +1. **Input:** A list of `card_ids` currently in a deck. +2. **Query:** Find all `card_id_b` linked to any of the input IDs in `card_connections`. +3. **Aggregate:** Sum the `weight` for each suggested card. +4. **Filter:** Remove suggestions that do not match the `color_identity` of the deck. +5. **Output:** Return the top $N$ cards sorted by aggregate weight. \ No newline at end of file diff --git a/backend/app/models/mtg_models.py b/backend/app/models/mtg_models.py index 820484a..3d851b9 100644 --- a/backend/app/models/mtg_models.py +++ b/backend/app/models/mtg_models.py @@ -26,7 +26,7 @@ class MtgSet(Base): icon_svg_url = Column(Text, nullable=True) parent_code = Column(String(10), nullable=True) mtgo_code = Column(String(10), nullable=True) - created_at = Column(DateTime, server_default=func.now()) + image = Column(Text, nullable=True) # Card image URL from MTGJSON updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) # Relationships @@ -55,7 +55,7 @@ class MtgCard(Base): numbers = Column(String(100), nullable=True) identifiers = Column(Text, nullable=True) # JSON string images = Column(Text, nullable=True) # JSON string - created_at = Column(DateTime, server_default=func.now()) + image = Column(Text, nullable=True) # Card image URL from MTGJSON updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) # Relationships diff --git a/backend/scripts/download_mtgjson_data.py b/backend/scripts/download_mtgjson_data.py new file mode 100644 index 0000000..e74e87b --- /dev/null +++ b/backend/scripts/download_mtgjson_data.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +""" +MTGJSON Data Downloader + +Downloads MTGJSON data files into the container for loading. +Handles decompression of gzip files and unzipping of zip files. + +Usage: + python download_mtgjson_data.py +""" + +import asyncio +import json +import gzip +import logging +import os +import sys +import zipfile +from pathlib import Path +from urllib.request import urlretrieve, urlopen + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# MTGJSON download URLs +MTGJSON_BASE_URL = "https://mtgjson.com/api/5x" +MTGJSON_FILES = { + "AllPrintings.psql.gz": "AllPrintings.psql.gz", + "AllSetFiles.zip": "AllSetFiles.zip", + "AllDeckFiles.zip": "AllDeckFiles.zip", + "AllIdentifiers.json.gz": "AllIdentifiers.json.gz", + "CardTypes.json.gz": "CardTypes.json.gz", + "DeckList.json.gz": "DeckList.json.gz", + "Keywords.json.gz": "Keywords.json.gz", + "SetList.json.gz": "SetList.json.gz", +} + + +def download_file(url, dest_dir, filename): + """Download a file from URL to destination directory""" + dest_path = dest_dir / filename + if dest_path.exists(): + logger.info(f"{filename} already exists, skipping download") + return True + + logger.info(f"Downloading {filename}...") + try: + urlretrieve(url, dest_path) + logger.info(f"Downloaded {filename} to {dest_path}") + return True + except Exception as e: + logger.error(f"Failed to download {filename}: {e}") + return False + + +def extract_zip(zip_path, dest_dir): + """Extract a zip file to destination directory""" + logger.info(f"Extracting {zip_path.name} to {dest_dir}...") + try: + with zipfile.ZipFile(zip_path, 'r') as zip_ref: + zip_ref.extractall(dest_dir) + logger.info(f"Extracted {zip_path.name} successfully") + # Remove the zip file after extraction + zip_path.unlink() + return True + except Exception as e: + logger.error(f"Failed to extract {zip_path}: {e}") + return False + + +def extract_gzip(gz_path, dest_path=None): + """Extract a gzip file""" + if not dest_path: + dest_path = gz_path.with_suffix('') + + logger.info(f"Extracting {gz_path.name} to {dest_path}...") + try: + with gzip.open(gz_path, 'rt', encoding='utf-8') as f_in: + with open(dest_path, 'w', encoding='utf-8') as f_out: + f_out.write(f_in.read()) + logger.info(f"Extracted {gz_path.name} to {dest_path}") + # Remove the gz file after extraction + gz_path.unlink() + return True + except Exception as e: + logger.error(f"Failed to extract {gz_path}: {e}") + return False + + +async def download_all(): + """Download and extract all MTGJSON data files""" + logger.info("Starting MTGJSON data download...") + + # Create data directory + data_dir = Path("/app/data/mtgjson") + data_dir.mkdir(parents=True, exist_ok=True) + + # Download AllPrintings.psql.gz + if download_file(MTGJSON_BASE_URL + "/AllPrintings.psql.gz", data_dir, "AllPrintings.psql.gz"): + # Extract gzip + gz_path = data_dir / "AllPrintings.psql.gz" + if extract_gzip(gz_path, data_dir / "AllPrintings.psql"): + logger.info("AllPrintings.psql extracted successfully") + + # Download AllSetFiles.zip + if download_file(MTGJSON_BASE_URL + "/AllSetFiles.zip", data_dir, "AllSetFiles.zip"): + zip_path = data_dir / "AllSetFiles.zip" + allsetfiles_dir = data_dir / "allsetfiles" + allsetfiles_dir.mkdir(exist_ok=True) + if extract_zip(zip_path, allsetfiles_dir): + logger.info("AllSetFiles extracted successfully") + + # Download AllDeckFiles.zip + if download_file(MTGJSON_BASE_URL + "/AllDeckFiles.zip", data_dir, "AllDeckFiles.zip"): + zip_path = data_dir / "AllDeckFiles.zip" + alldeckfiles_dir = data_dir / "alldeckfiles" + alldeckfiles_dir.mkdir(exist_ok=True) + if extract_zip(zip_path, alldeckfiles_dir): + logger.info("AllDeckFiles extracted successfully") + + # Download AllIdentifiers.json.gz + if download_file(MTGJSON_BASE_URL + "/AllIdentifiers.json.gz", data_dir, "AllIdentifiers.json.gz"): + gz_path = data_dir / "AllIdentifiers.json.gz" + if extract_gzip(gz_path, data_dir / "AllIdentifiers.json"): + logger.info("AllIdentifiers.json extracted successfully") + + # Download CardTypes.json.gz + if download_file(MTGJSON_BASE_URL + "/CardTypes.json.gz", data_dir, "CardTypes.json.gz"): + gz_path = data_dir / "CardTypes.json.gz" + if extract_gzip(gz_path, data_dir / "CardTypes.json"): + logger.info("CardTypes.json extracted successfully") + + # Download DeckList.json.gz + if download_file(MTGJSON_BASE_URL + "/DeckList.json.gz", data_dir, "DeckList.json.gz"): + gz_path = data_dir / "DeckList.json.gz" + if extract_gzip(gz_path, data_dir / "DeckList.json"): + logger.info("DeckList.json extracted successfully") + + # Download Keywords.json.gz + if download_file(MTGJSON_BASE_URL + "/Keywords.json.gz", data_dir, "Keywords.json.gz"): + gz_path = data_dir / "Keywords.json.gz" + if extract_gzip(gz_path, data_dir / "Keywords.json"): + logger.info("Keywords.json extracted successfully") + + # Download SetList.json.gz + if download_file(MTGJSON_BASE_URL + "/SetList.json.gz", data_dir, "SetList.json.gz"): + gz_path = data_dir / "SetList.json.gz" + if extract_gzip(gz_path, data_dir / "SetList.json"): + logger.info("SetList.json extracted successfully") + + logger.info("MTGJSON data download and extraction complete!") + + +if __name__ == "__main__": + asyncio.run(download_all()) diff --git a/backend/scripts/load_mtgjson_data.py b/backend/scripts/load_mtgjson_data.py old mode 100644 new mode 100755 index 4713323..1f685a2 --- a/backend/scripts/load_mtgjson_data.py +++ b/backend/scripts/load_mtgjson_data.py @@ -1,12 +1,385 @@ #!/usr/bin/env python3 """ -Entry point for MTGJSON data loading. +MTGJSON Data Loader -This script is called from the Docker container to load MTGJSON data. +Downloads and loads MTGJSON data into the PostgreSQL database. +Handles AllPrintings.psql, JSON files, and image data extraction. + +Usage: + python load_mtgjson_data.py """ import asyncio -from app.services.mtgjson_loader import main +import json +import gzip +import logging +import os +import sys +import tempfile +from pathlib import Path +from urllib.request import urlretrieve + +# Add parent directory to path for imports +sys.path.append(str(Path(__file__).parent.parent)) + +from app.core.settings import get_settings +from app.core.database import mtg_engine, mtg_async_session +from app.models.mtg_models import MtgSet, MtgCard + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# MTGJSON download URLs +MTGJSON_BASE_URL = "https://mtgjson.com/api/5x" +MTGJSON_FILES = { + "AllPrintings.psql.gz": "AllPrintings.psql.gz", + "AllSetFiles.zip": "AllSetFiles.zip", + "AllDeckFiles.zip": "AllDeckFiles.zip", + "AllIdentifiers.json.gz": "AllIdentifiers.json.gz", + "CardTypes.json.gz": "CardTypes.json.gz", + "DeckList.json.gz": "DeckList.json.gz", + "Keywords.json.gz": "Keywords.json.gz", + "SetList.json.gz": "SetList.json.gz", +} + + +async def load_sets_from_json(): + """Load set metadata from AllSetFiles.zip or SetList.json.gz""" + logger.info("Loading set metadata...") + + # Try AllSetFiles first (has more complete data) + set_files_dir = Path("/app/data/mtgjson/allsetfiles") + if not set_files_dir.exists(): + # Try SetList.json.gz as fallback + setlist_path = Path("/app/data/mtgjson/SetList.json.gz") + if setlist_path.exists(): + logger.info("Loading from SetList.json.gz") + with gzip.open(setlist_path, 'rt', encoding='utf-8') as f: + set_list = json.load(f) + + settings = get_settings() + async with mtg_async_session() as session: + for set_data in set_list: + # Get image URL from setCode mapping if available + image_url = None + if 'image' in set_data: + image_url = set_data['image'].get('png', set_data['image'].get('svg')) + + existing = await session.execute( + MtgSet.__table__.select().where(MtgSet.code == set_data['code']) + ) + + if existing.first(): + # Update existing + await session.execute( + MtgSet.__table__.update() + .where(MtgSet.code == set_data['code']) + .values( + name=set_data.get('name'), + type=set_data.get('type'), + release_date=set_data.get('releaseDate'), + base_set_size=set_data.get('baseSetSize'), + total_size=set_data.get('totalSize'), + icon_svg_url=set_data.get('iconSvgUri'), + image=image_url, + updated_at=asyncio.coroutines.utcnow() if hasattr(asyncio.coroutines, 'utcnow') else None + ) + ) + else: + # Insert new + set_obj = MtgSet( + code=set_data['code'], + name=set_data.get('name'), + type=set_data.get('type'), + release_date=set_data.get('releaseDate'), + base_set_size=set_data.get('baseSetSize'), + total_size=set_data.get('totalSize'), + icon_svg_url=set_data.get('iconSvgUri'), + image=image_url, + created_at=asyncio.coroutines.utcnow() if hasattr(asyncio.coroutines, 'utcnow') else None + ) + session.add(set_obj) + + await session.commit() + logger.info(f"Loaded set metadata from {len(set_list)} sets") + else: + logger.warning("No set metadata files found") + return + + # Process AllSetFiles directory + count = 0 + settings = get_settings() + async with mtg_async_session() as session: + for json_file in sorted(set_files_dir.glob("*.json")): + with open(json_file, 'r', encoding='utf-8') as f: + set_data = json.load(f) + + if 'data' in set_data: + set_data = set_data['data'] + + image_url = set_data.get('image', {}).get('png', set_data.get('image', {}).get('svg')) + + existing = await session.execute( + MtgSet.__table__.select().where(MtgSet.code == set_data['code']) + ) + + if existing.first(): + await session.execute( + MtgSet.__table__.update() + .where(MtgSet.code == set_data['code']) + .values( + name=set_data.get('name'), + type=set_data.get('type'), + release_date=set_data.get('releaseDate'), + base_set_size=set_data.get('baseSetSize'), + total_size=set_data.get('totalSize'), + icon_svg_url=set_data.get('iconSvgUri'), + image=image_url + ) + ) + else: + set_obj = MtgSet( + code=set_data['code'], + name=set_data.get('name'), + type=set_data.get('type'), + release_date=set_data.get('releaseDate'), + base_set_size=set_data.get('baseSetSize'), + total_size=set_data.get('totalSize'), + icon_svg_url=set_data.get('iconSvgUri'), + image=image_url + ) + session.add(set_obj) + + count += 1 + + await session.commit() + logger.info(f"Loaded {count} sets from AllSetFiles") + + +async def load_cards_from_psql(): + """Load cards from AllPrintings.psql""" + logger.info("Loading cards from AllPrintings.psql...") + + psql_path = Path("/app/data/mtgjson/AllPrintings.psql") + if not psql_path.exists(): + logger.warning("AllPrintings.psql not found") + return + + # Parse PSQL file to extract INSERT statements + # This is a simplified parser - in production you'd use a proper PSQL parser + cards_data = [] + + with open(psql_path, 'r', encoding='utf-8') as f: + current_card = {} + in_insert = False + + for line in f: + line = line.strip() + + if line.startswith('COPY public.mtgjson_card'): + # Header line - skip + continue + + if line == '\\.': + # End of COPY command + in_insert = False + continue + + if in_insert: + # Parse CSV line + fields = line.split('\t') + if len(fields) > 10: + try: + card = { + 'id': fields[0], + 'name': fields[1], + 'manaCost': fields[2], + 'type': fields[3], + 'text': fields[4], + 'power': fields[5], + 'toughness': fields[6], + 'rarity': fields[7], + 'layout': fields[8], + 'artist': fields[9], + 'flavor': fields[10] if len(fields) > 10 else '', + 'set': fields[11] if len(fields) > 11 else '', + 'number': fields[12] if len(fields) > 12 else '', + 'identifiers': fields[13] if len(fields) > 13 else '{}', + 'images': fields[14] if len(fields) > 14 else '{}', + 'updatedAt': fields[15] if len(fields) > 15 else '', + } + cards_data.append(card) + except (ValueError, IndexError): + continue + + if line.startswith('INSERT INTO public.mtgjson_card'): + in_insert = True + + logger.info(f"Parsed {len(cards_data)} cards from PSQL file") + + # Update cards in database with parsed data + if cards_data: + settings = get_settings() + async with mtg_async_session() as session: + # First, get all set codes to create sets + set_codes = set(c['set'] for c in cards_data if c['set']) + + for set_code in set_codes: + existing = await session.execute( + MtgSet.__table__.select().where(MtgSet.code == set_code) + ) + if not existing.first(): + # Create placeholder set + set_obj = MtgSet( + code=set_code, + name=f"Set {set_code}", + created_at=asyncio.coroutines.utcnow() if hasattr(asyncio.coroutines, 'utcnow') else None + ) + session.add(set_obj) + + await session.flush() + + # Now load cards + for card_data in cards_data: + # Get set_id + set_result = await session.execute( + MtgSet.__table__.select().where(MtgSet.code == card_data['set']) + ) + set_obj = set_result.first() + + if not set_obj: + continue + + existing = await session.execute( + MtgCard.__table__.select() + .where(MtgCard.name == card_data['name']) + .where(MtgCard.set_id == set_obj.id) + ) + + if existing.first(): + # Update existing + await session.execute( + MtgCard.__table__.update() + .where(MtgCard.name == card_data['name']) + .where(MtgCard.set_id == set_obj.id) + .values( + mana_cost=card_data['manaCost'], + type_line=card_data['type'], + oracle_text=card_data['text'], + power=card_data['power'], + toughness=card_data['toughness'], + rarity=card_data['rarity'], + layout=card_data['layout'], + artist=card_data['artist'], + flavor_text=card_data['flavor'], + numbers=card_data['number'], + identifiers=card_data['identifiers'], + images=card_data['images'], + image=card_data.get('image'), + updated_at=asyncio.coroutines.utcnow() if hasattr(asyncio.coroutines, 'utcnow') else None + ) + ) + else: + # Insert new card + card_obj = MtgCard( + name=card_data['name'], + set_id=set_obj.id, + mana_cost=card_data['manaCost'], + type_line=card_data['type'], + oracle_text=card_data['text'], + power=card_data['power'], + toughness=card_data['toughness'], + rarity=card_data['rarity'], + layout=card_data['layout'], + artist=card_data['artist'], + flavor_text=card_data['flavor'], + numbers=card_data['number'], + identifiers=card_data['identifiers'], + images=card_data['images'], + image=card_data.get('image') + ) + session.add(card_obj) + + await session.commit() + logger.info("Cards loaded successfully") + + +async def load_identifiers(): + """Load card identifiers from AllIdentifiers.json.gz""" + logger.info("Loading identifiers...") + + identifiers_path = Path("/app/data/mtgjson/AllIdentifiers.json.gz") + if not identifiers_path.exists(): + logger.warning("AllIdentifiers.json.gz not found") + return + + with gzip.open(identifiers_path, 'rt', encoding='utf-8') as f: + identifiers = json.load(f) + + logger.info(f"Loaded {len(identifiers)} identifiers") + + +async def load_deck_list(): + """Load deck list metadata from DeckList.json.gz""" + logger.info("Loading deck list...") + + deck_list_path = Path("/app/data/mtgjson/DeckList.json.gz") + if not deck_list_path.exists(): + logger.warning("DeckList.json.gz not found") + return + + with gzip.open(deck_list_path, 'rt', encoding='utf-8') as f: + deck_list = json.load(f) + + logger.info(f"Loaded {len(deck_list)} deck list entries") + + +async def load_keywords(): + """Load card keywords from Keywords.json.gz""" + logger.info("Loading keywords...") + + keywords_path = Path("/app/data/mtgjson/Keywords.json.gz") + if not keywords_path.exists(): + logger.warning("Keywords.json.gz not found") + return + + with gzip.open(keywords_path, 'rt', encoding='utf-8') as f: + keywords = json.load(f) + + logger.info(f"Loaded {len(keywords)} keywords") + + +async def main(): + """Main entry point""" + logger.info("Starting MTGJSON data loader...") + + # Ensure data directory exists + data_dir = Path("/app/data/mtgjson") + data_dir.mkdir(parents=True, exist_ok=True) + + # Load data in order + await load_sets_from_json() + await load_cards_from_psql() + await load_identifiers() + await load_deck_list() + await load_keywords() + + logger.info("MTGJSON data loading complete!") + + # Print summary + async with mtg_async_session() as session: + from sqlalchemy import text + + result = await session.execute(text("SELECT COUNT(*) FROM mtg_sets")) + set_count = result.scalar() + + result = await session.execute(text("SELECT COUNT(*) FROM mtg_cards")) + card_count = result.scalar() + + logger.info(f"Database summary:") + logger.info(f" Sets: {set_count}") + logger.info(f" Cards: {card_count}") + if __name__ == "__main__": asyncio.run(main())