Complete architecture simplification: 54+ files → 37 files
- Removed 11 duplicate router files (auth.py, users.py, decks.py, card_router.py, interactions.py, user_data.py, rooms.py, games/router.py, refresh.py, card_import.py, ws.py) - Removed duplicate core files (logging_config.py, settings.py) - Removed unused services (card_mirror_service.py, mtgjson_loader.py, mtgjson_uploader.py) - Removed unused schemas (proto_messages.py, protocol_constants.py) - Created run.py startup script for simple execution - Created app/core/config.py for unified configuration management - Created app/core/logging.py for logging configuration - Consolidated routers into 6 focused files (auth_router.py, users_router.py, decks_router.py, cards_router.py, user_data_router.py, games_router.py) - Updated all imports to use consolidated model paths - Added backward compatibility aliases in database.py - Created SIMPLIFICATION_SUMMARY.md documentation Architecture improvements: - Single unified database engine with auto-detection (SQLite/PostgreSQL) - Consolidated models into 2 files (models.py, mtg_models.py) - Clean router structure with one file per domain - Simplified configuration with sensible defaults - SQLite by default for development, PostgreSQL for production Testing: - All imports verified working - App loads successfully - Ready for development and testing
This commit is contained in:
@@ -1,11 +1,8 @@
|
||||
"""Services package initialization."""
|
||||
from app.services.deck_parser import DeckParser
|
||||
from app.services.card_database import search_cards, get_card_by_name, get_cards_by_set, get_card_types, get_card_rarities, get_sets, get_set_by_code, get_card_statistics
|
||||
from app.services.card_mirror_service import upsert_card_mirror, get_card_mirror_by_name, search_card_mirrors, add_card_to_deck, remove_card_from_deck, get_deck_cards, get_user_deck_summaries, sync_mirrors_from_mtg_cards, get_card_statistics
|
||||
from app.services.mtgjson_manager import MTGJSONManager, get_manager
|
||||
from app.services.mtgjson_downloader import download_all_files, verify_downloads, get_file_list
|
||||
from app.services.mtgjson_loader import create_tables, download_file, extract_zip, get_all_printings_psql_file, parse_psql_file, import_cards, import_all_printings_psql, import_all_set_files, import_all_identifiers, import_all_deck_files, import_simple_json, create_indexes, show_summary, main
|
||||
from app.services.mtgjson_uploader import create_tables, import_all_printings_psql, import_all_set_files, import_all_identifiers, import_all_deck_files, import_json_files, download_file, extract_zip, main
|
||||
from app.services.file_parser import FileParser
|
||||
from app.services.fuzzy_card_matcher import FuzzyCardMatcher
|
||||
from app.services.import_batch_processor import ImportBatchProcessor
|
||||
@@ -23,34 +20,11 @@ __all__ = [
|
||||
"get_sets",
|
||||
"get_set_by_code",
|
||||
"get_card_statistics",
|
||||
"upsert_card_mirror",
|
||||
"get_card_mirror_by_name",
|
||||
"search_card_mirrors",
|
||||
"add_card_to_deck",
|
||||
"remove_card_from_deck",
|
||||
"get_deck_cards",
|
||||
"get_user_deck_summaries",
|
||||
"sync_mirrors_from_mtg_cards",
|
||||
"MTGJSONManager",
|
||||
"get_manager",
|
||||
"download_all_files",
|
||||
"verify_downloads",
|
||||
"get_file_list",
|
||||
"create_tables",
|
||||
"download_file",
|
||||
"extract_zip",
|
||||
"get_all_printings_psql_file",
|
||||
"parse_psql_file",
|
||||
"import_cards",
|
||||
"import_all_printings_psql",
|
||||
"import_all_set_files",
|
||||
"import_all_identifiers",
|
||||
"import_all_deck_files",
|
||||
"import_simple_json",
|
||||
"import_json_files",
|
||||
"create_indexes",
|
||||
"show_summary",
|
||||
"main",
|
||||
"FileParser",
|
||||
"FuzzyCardMatcher",
|
||||
"ImportBatchProcessor",
|
||||
|
||||
@@ -1,359 +0,0 @@
|
||||
"""
|
||||
Card Mirror Service
|
||||
|
||||
Manages mirrored card data in mtgo_platform for fast deckbuilding queries.
|
||||
Syncs with mtg_cards (mtg_data) when the card database is refreshed.
|
||||
"""
|
||||
from typing import List, Dict, Any, Optional, Tuple
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select, func, update, delete
|
||||
from sqlalchemy.orm import selectinload
|
||||
|
||||
from app.models.mtg_models import MtgCard, MtgSet
|
||||
from app.models.mirror_models import MtgCardMirror, DeckCardLink
|
||||
from app.core.database import mirror_get_db
|
||||
|
||||
|
||||
async def upsert_card_mirror(
|
||||
db: AsyncSession,
|
||||
card_data: Dict[str, Any],
|
||||
source_id: Optional[int] = None
|
||||
) -> MtgCardMirror:
|
||||
"""
|
||||
Upsert a card into the mirror table.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
card_data: Card data dictionary
|
||||
source_id: Optional reference to mtg_cards.id
|
||||
|
||||
Returns:
|
||||
The upserted MtgCardMirror instance
|
||||
"""
|
||||
# Check if card already exists by name
|
||||
existing = await db.execute(
|
||||
select(MtgCardMirror).where(MtgCardMirror.name == card_data.get("name"))
|
||||
)
|
||||
existing_card = existing.scalar_one_or_none()
|
||||
|
||||
if existing_card:
|
||||
# Update existing mirror
|
||||
for key, value in card_data.items():
|
||||
if hasattr(existing_card, key):
|
||||
setattr(existing_card, key, value)
|
||||
if source_id:
|
||||
existing_card.source_id = source_id
|
||||
else:
|
||||
# Create new mirror
|
||||
mirror_data = {
|
||||
"name": card_data.get("name"),
|
||||
"mana_cost": card_data.get("mana_cost"),
|
||||
"type_line": card_data.get("type_line"),
|
||||
"oracle_text": card_data.get("oracle_text"),
|
||||
"power": card_data.get("power"),
|
||||
"toughness": card_data.get("toughness"),
|
||||
"rarity": card_data.get("rarity"),
|
||||
"layout": card_data.get("layout"),
|
||||
"artist": card_data.get("artist"),
|
||||
"flavor_text": card_data.get("flavor_text"),
|
||||
"numbers": card_data.get("numbers"),
|
||||
"identifiers": card_data.get("identifiers"),
|
||||
"images": card_data.get("images"),
|
||||
"image": card_data.get("image"),
|
||||
"card_parts": card_data.get("card_parts"),
|
||||
"keywords": card_data.get("keywords"),
|
||||
"legalities": card_data.get("legalities"),
|
||||
"set_code": card_data.get("set_code"),
|
||||
"set_name": card_data.get("set_name"),
|
||||
"source_id": source_id,
|
||||
}
|
||||
mirror_card = MtgCardMirror(**mirror_data)
|
||||
db.add(mirror_card)
|
||||
await db.flush()
|
||||
return mirror_card
|
||||
|
||||
return existing_card
|
||||
|
||||
|
||||
async def get_card_mirror_by_name(
|
||||
db: AsyncSession,
|
||||
name: str
|
||||
) -> Optional[MtgCardMirror]:
|
||||
"""
|
||||
Get a mirrored card by name.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
name: Card name
|
||||
|
||||
Returns:
|
||||
MtgCardMirror instance or None
|
||||
"""
|
||||
result = await db.execute(
|
||||
select(MtgCardMirror).where(MtgCardMirror.name == name)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def search_card_mirrors(
|
||||
db: AsyncSession,
|
||||
query: str,
|
||||
limit: int = 100,
|
||||
offset: int = 0
|
||||
) -> Tuple[List[MtgCardMirror], int]:
|
||||
"""
|
||||
Search mirrored cards by name.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
query: Search query
|
||||
limit: Maximum results
|
||||
offset: Pagination offset
|
||||
|
||||
Returns:
|
||||
Tuple of (list of mirrors, total count)
|
||||
"""
|
||||
search_term = f"%{query.lower()}%"
|
||||
|
||||
# Search query
|
||||
stmt = (
|
||||
select(MtgCardMirror)
|
||||
.where(MtgCardMirror.name.ilike(search_term))
|
||||
.offset(offset)
|
||||
.limit(limit)
|
||||
)
|
||||
|
||||
result = await db.execute(stmt)
|
||||
mirrors = result.scalars().all()
|
||||
|
||||
# Total count
|
||||
count_stmt = select(func.count()).select_from(MtgCardMirror).where(
|
||||
MtgCardMirror.name.ilike(search_term)
|
||||
)
|
||||
count_result = await db.execute(count_stmt)
|
||||
total = count_result.scalar()
|
||||
|
||||
return mirrors, total
|
||||
|
||||
|
||||
async def add_card_to_deck(
|
||||
db: AsyncSession,
|
||||
deck_id: int,
|
||||
card_id: int,
|
||||
quantity: int = 1,
|
||||
zone: str = "main"
|
||||
) -> DeckCardLink:
|
||||
"""
|
||||
Add a card to a deck.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
deck_id: Deck ID
|
||||
card_id: Mirrored card ID
|
||||
quantity: Number of copies
|
||||
zone: 'main' or 'sideboard'
|
||||
|
||||
Returns:
|
||||
DeckCardLink instance
|
||||
"""
|
||||
# Check if link already exists
|
||||
existing = await db.execute(
|
||||
select(DeckCardLink).where(
|
||||
DeckCardLink.deck_id == deck_id,
|
||||
DeckCardLink.card_id == card_id,
|
||||
DeckCardLink.zone == zone
|
||||
)
|
||||
)
|
||||
existing_link = existing.scalar_one_or_none()
|
||||
|
||||
if existing_link:
|
||||
# Update quantity
|
||||
existing_link.quantity = quantity
|
||||
await db.flush()
|
||||
return existing_link
|
||||
else:
|
||||
# Create new link
|
||||
link = DeckCardLink(
|
||||
deck_id=deck_id,
|
||||
card_id=card_id,
|
||||
quantity=quantity,
|
||||
zone=zone
|
||||
)
|
||||
db.add(link)
|
||||
await db.flush()
|
||||
return link
|
||||
|
||||
|
||||
async def remove_card_from_deck(
|
||||
db: AsyncSession,
|
||||
deck_id: int,
|
||||
card_id: int,
|
||||
zone: str = "main"
|
||||
) -> bool:
|
||||
"""
|
||||
Remove a card from a deck.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
deck_id: Deck ID
|
||||
card_id: Mirrored card ID
|
||||
zone: 'main' or 'sideboard'
|
||||
|
||||
Returns:
|
||||
True if removed, False if not found
|
||||
"""
|
||||
result = await db.execute(
|
||||
delete(DeckCardLink).where(
|
||||
DeckCardLink.deck_id == deck_id,
|
||||
DeckCardLink.card_id == card_id,
|
||||
DeckCardLink.zone == zone
|
||||
)
|
||||
)
|
||||
return result.rowcount > 0
|
||||
|
||||
|
||||
async def get_deck_cards(
|
||||
db: AsyncSession,
|
||||
deck_id: int
|
||||
) -> List[DeckCardLink]:
|
||||
"""
|
||||
Get all cards in a deck with their mirrored data.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
deck_id: Deck ID
|
||||
|
||||
Returns:
|
||||
List of DeckCardLink instances with joined card data
|
||||
"""
|
||||
stmt = (
|
||||
select(DeckCardLink, MtgCardMirror)
|
||||
.join(MtgCardMirror, DeckCardLink.card_id == MtgCardMirror.id)
|
||||
.where(DeckCardLink.deck_id == deck_id)
|
||||
.order_by(DeckCardLink.id)
|
||||
)
|
||||
|
||||
result = await db.execute(stmt)
|
||||
rows = result.all()
|
||||
|
||||
links = []
|
||||
for link, card in rows:
|
||||
link.card = card
|
||||
links.append(link)
|
||||
|
||||
return links
|
||||
|
||||
|
||||
async def get_user_deck_summaries(
|
||||
db: AsyncSession,
|
||||
user_id: int
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Get all decks for a user with card counts.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
user_id: User ID
|
||||
|
||||
Returns:
|
||||
List of deck summaries with card counts
|
||||
"""
|
||||
from app.models.models import DecklistFile
|
||||
|
||||
stmt = (
|
||||
select(DecklistFile)
|
||||
.where(DecklistFile.owner_id == user_id)
|
||||
.order_by(DecklistFile.creation_date.desc())
|
||||
)
|
||||
|
||||
result = await db.execute(stmt)
|
||||
decks = result.scalars().all()
|
||||
|
||||
summaries = []
|
||||
for deck in decks:
|
||||
# Get card count
|
||||
count_stmt = (
|
||||
select(func.count())
|
||||
.select_from(DeckCardLink)
|
||||
.where(DeckCardLink.deck_id == deck.id)
|
||||
)
|
||||
count_result = await db.execute(count_stmt)
|
||||
card_count = count_result.scalar()
|
||||
|
||||
summaries.append({
|
||||
"id": deck.id,
|
||||
"name": deck.name,
|
||||
"format": deck.format,
|
||||
"status": deck.status,
|
||||
"folder_id": deck.folder_id,
|
||||
"owner_id": deck.owner_id,
|
||||
"creation_date": deck.creation_date,
|
||||
"card_count": card_count,
|
||||
})
|
||||
|
||||
return summaries
|
||||
|
||||
|
||||
async def sync_mirrors_from_mtg_cards(
|
||||
db: AsyncSession,
|
||||
mtg_db: Optional[AsyncSession] = None
|
||||
) -> int:
|
||||
"""
|
||||
Sync all mirrored cards from the mtg_cards table.
|
||||
|
||||
This is called when the card database is refreshed.
|
||||
|
||||
Args:
|
||||
db: Mirror database session
|
||||
mtg_db: Optional MTG database session for cross-DB queries
|
||||
|
||||
Returns:
|
||||
Number of cards synced
|
||||
"""
|
||||
if not mtg_db:
|
||||
# Use the same session if no MTG session provided
|
||||
# Note: In production, you'd need a cross-DB connection
|
||||
# For now, we'll just refresh the mirror from existing data
|
||||
pass
|
||||
|
||||
# For now, this is a no-op. In a full implementation,
|
||||
# you'd query mtg_cards and upsert into mtg_cards_mirror.
|
||||
# This requires cross-database connections which SQLAlchemy
|
||||
# can handle with proper configuration.
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
async def get_card_statistics(db: AsyncSession) -> Dict[str, Any]:
|
||||
"""
|
||||
Get statistics about mirrored cards.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
Dictionary with statistics
|
||||
"""
|
||||
# Total mirrored cards
|
||||
total_stmt = select(func.count()).select_from(MtgCardMirror)
|
||||
total = (await db.execute(total_stmt)).scalar()
|
||||
|
||||
# Cards by rarity
|
||||
rarity_stmt = select(MtgCardMirror.rarity, func.count()).group_by(
|
||||
MtgCardMirror.rarity
|
||||
)
|
||||
rarity_result = await db.execute(rarity_stmt)
|
||||
rarities = {row[0]: row[1] for row in rarity_result if row[0]}
|
||||
|
||||
# Cards by type
|
||||
type_stmt = select(MtgCardMirror.type_line, func.count()).group_by(
|
||||
MtgCardMirror.type_line
|
||||
)
|
||||
type_result = await db.execute(type_stmt)
|
||||
types = {row[0]: row[1] for row in type_result if row[0]}
|
||||
|
||||
return {
|
||||
"total_mirrored_cards": total,
|
||||
"cards_by_rarity": rarities,
|
||||
"cards_by_type": types,
|
||||
}
|
||||
@@ -4,8 +4,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select, or_, and_
|
||||
from sqlalchemy.orm import selectinload
|
||||
|
||||
from app.models.mtg_models import MtgCard, MtgSet
|
||||
from app.models.mirror_models import MtgCardMirror
|
||||
from app.models.mtg_models import MtgCard, MtgSet, MtgCardMirror
|
||||
|
||||
|
||||
class CardSearchService:
|
||||
|
||||
@@ -4,8 +4,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select, update, delete
|
||||
from sqlalchemy.orm import selectinload
|
||||
|
||||
from app.models.user_deck import UserDeck, UserDeckCard, DeckPrecedent, DeckPrecedentCard
|
||||
from app.models.models import MtgonlineCard
|
||||
from app.models.models import UserDeck, UserDeckCard, DeckPrecedent, DeckPrecedentCard, MtgonlineCard
|
||||
|
||||
|
||||
class DeckManager:
|
||||
|
||||
@@ -4,9 +4,8 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select, or_, and_, func, case
|
||||
from sqlalchemy.orm import selectinload
|
||||
|
||||
from app.models.user_deck import UserDeck, UserDeckCard, CardSuggestion
|
||||
from app.models.mtg_models import MtgCard
|
||||
from app.models.mirror_models import MtgCardMirror
|
||||
from app.models.models import UserDeck, UserDeckCard, CardSuggestion
|
||||
from app.models.mtg_models import MtgCard, MtgCardMirror
|
||||
from app.services.fuzzy_card_matcher import FuzzyCardMatcher
|
||||
|
||||
|
||||
|
||||
@@ -6,10 +6,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select, update, insert, delete
|
||||
from sqlalchemy.sql import func
|
||||
|
||||
from app.models.card_import_batch import CardImportBatch
|
||||
from app.models.user_card_import_record import UserCardImportRecord
|
||||
from app.models.user_data import UserCardCollection
|
||||
from app.models.models import MtgonlineCard
|
||||
from app.models.models import CardImportBatch, UserCardImportRecord, UserCardCollection, MtgonlineCard
|
||||
from app.services.fuzzy_card_matcher import FuzzyCardMatcher
|
||||
|
||||
|
||||
|
||||
@@ -1,838 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
MTGJSON Data Loader
|
||||
|
||||
Downloads and loads all MTGJSON data into the PostgreSQL database.
|
||||
Designed to run inside the Docker container.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import gzip
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from sqlalchemy import text, insert, update, select, Table, Column, String, Text, Integer, Float, Boolean, DateTime, MetaData, UniqueConstraint
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
|
||||
from app.core.settings import get_settings
|
||||
|
||||
# MTGJSON API v5 base URL
|
||||
MTGJSON_API_V5 = "https://mtgjson.com/api/v5"
|
||||
|
||||
|
||||
async def create_tables(engine):
|
||||
"""Create all required database tables."""
|
||||
async with engine.begin() as conn:
|
||||
await conn.execute(text("""
|
||||
CREATE TABLE IF NOT EXISTS cards (
|
||||
id SERIAL PRIMARY KEY,
|
||||
artist TEXT,
|
||||
asciiName TEXT,
|
||||
attractionLights TEXT,
|
||||
availability TEXT,
|
||||
boosterTypes TEXT,
|
||||
borderColor TEXT,
|
||||
cardParts TEXT,
|
||||
colorIdentity TEXT,
|
||||
colorIndicator TEXT,
|
||||
colors TEXT,
|
||||
defense TEXT,
|
||||
duelDeck TEXT,
|
||||
edhrecRank INTEGER,
|
||||
edhrecSaltiness FLOAT,
|
||||
faceConvertedManaCost FLOAT,
|
||||
faceFlavorName TEXT,
|
||||
faceManaValue FLOAT,
|
||||
faceName TEXT,
|
||||
facePrintedName TEXT,
|
||||
finishes TEXT,
|
||||
flavorName TEXT,
|
||||
flavorText TEXT,
|
||||
frameEffects TEXT,
|
||||
frameVersion TEXT,
|
||||
hand TEXT,
|
||||
hasAlternativeDeckLimit BOOLEAN,
|
||||
hasContentWarning BOOLEAN,
|
||||
isAlternative BOOLEAN,
|
||||
isFullArt BOOLEAN,
|
||||
isFunny BOOLEAN,
|
||||
isGameChanger BOOLEAN,
|
||||
isOnlineOnly BOOLEAN,
|
||||
isOversized BOOLEAN,
|
||||
isPromo BOOLEAN,
|
||||
isRebalanced BOOLEAN,
|
||||
isReprint BOOLEAN,
|
||||
isReserved BOOLEAN,
|
||||
isStorySpotlight BOOLEAN,
|
||||
isTextless BOOLEAN,
|
||||
isTimeshifted BOOLEAN,
|
||||
keywords TEXT,
|
||||
language TEXT,
|
||||
layout TEXT,
|
||||
leadershipSkills TEXT,
|
||||
life TEXT,
|
||||
loyalty TEXT,
|
||||
manaCost TEXT,
|
||||
manaValue FLOAT,
|
||||
name TEXT,
|
||||
number TEXT,
|
||||
originalPrintings TEXT,
|
||||
originalReleaseDate TEXT,
|
||||
originalText TEXT,
|
||||
otherFaceIds TEXT,
|
||||
power TEXT,
|
||||
printedName TEXT,
|
||||
printedText TEXT,
|
||||
printedType TEXT,
|
||||
printings TEXT,
|
||||
producedMana TEXT,
|
||||
promoTypes TEXT,
|
||||
rarity TEXT,
|
||||
rebalancedPrintings TEXT,
|
||||
relatedCards TEXT,
|
||||
securityStamp TEXT,
|
||||
setCode TEXT,
|
||||
side TEXT,
|
||||
signature TEXT,
|
||||
skuIds TEXT,
|
||||
sourceProducts TEXT,
|
||||
subsets TEXT,
|
||||
subtypes TEXT,
|
||||
supertypes TEXT,
|
||||
text TEXT,
|
||||
toughness TEXT,
|
||||
type TEXT,
|
||||
types TEXT,
|
||||
uuid TEXT,
|
||||
variations TEXT,
|
||||
watermark TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
"""))
|
||||
|
||||
await conn.execute(text("""
|
||||
CREATE TABLE IF NOT EXISTS mtg_sets (
|
||||
id SERIAL PRIMARY KEY,
|
||||
code VARCHAR(10) UNIQUE NOT NULL,
|
||||
name VARCHAR(255),
|
||||
type VARCHAR(100),
|
||||
release_date DATE,
|
||||
base_set_size INTEGER,
|
||||
total_size INTEGER,
|
||||
is_foil_only BOOLEAN,
|
||||
is_non_foil_only BOOLEAN,
|
||||
digital BOOLEAN,
|
||||
icon_svg_url TEXT,
|
||||
parent_code VARCHAR(10),
|
||||
mtgo_code VARCHAR(10),
|
||||
card_count INTEGER,
|
||||
image_url TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
"""))
|
||||
|
||||
await conn.execute(text("""
|
||||
CREATE TABLE IF NOT EXISTS mtg_cards (
|
||||
id SERIAL PRIMARY KEY,
|
||||
set_id INTEGER REFERENCES mtg_sets(id) ON DELETE CASCADE,
|
||||
name VARCHAR(255),
|
||||
mana_cost VARCHAR(255),
|
||||
type_line VARCHAR(255),
|
||||
oracle_text TEXT,
|
||||
power VARCHAR(50),
|
||||
toughness VARCHAR(50),
|
||||
rarity VARCHAR(50),
|
||||
layout VARCHAR(50),
|
||||
artist VARCHAR(255),
|
||||
flavor_text TEXT,
|
||||
numbers VARCHAR(100),
|
||||
identifiers TEXT,
|
||||
images TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
"""))
|
||||
|
||||
await conn.execute(text("""
|
||||
CREATE TABLE IF NOT EXISTS card_identifiers (
|
||||
id SERIAL PRIMARY KEY,
|
||||
uuid TEXT UNIQUE NOT NULL,
|
||||
name VARCHAR(255),
|
||||
mana_cost VARCHAR(255),
|
||||
type_line VARCHAR(255),
|
||||
oracle_text TEXT,
|
||||
power VARCHAR(50),
|
||||
toughness VARCHAR(50),
|
||||
rarity VARCHAR(50),
|
||||
layout VARCHAR(50),
|
||||
artist VARCHAR(255),
|
||||
flavor_text TEXT,
|
||||
set_code VARCHAR(10),
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
"""))
|
||||
|
||||
await conn.execute(text("""
|
||||
CREATE TABLE IF NOT EXISTS decks (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
description TEXT,
|
||||
format VARCHAR(50),
|
||||
command TEXT,
|
||||
commander TEXT,
|
||||
creation_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
"""))
|
||||
|
||||
await conn.execute(text("""
|
||||
CREATE TABLE IF NOT EXISTS card_types (
|
||||
id SERIAL PRIMARY KEY,
|
||||
type VARCHAR(100) UNIQUE NOT NULL,
|
||||
description TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
"""))
|
||||
|
||||
await conn.execute(text("""
|
||||
CREATE TABLE IF NOT EXISTS deck_list (
|
||||
id SERIAL PRIMARY KEY,
|
||||
deck_id VARCHAR(100) UNIQUE NOT NULL,
|
||||
name VARCHAR(255),
|
||||
description TEXT,
|
||||
format VARCHAR(50),
|
||||
command TEXT,
|
||||
commander TEXT,
|
||||
total_cards INTEGER,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
"""))
|
||||
|
||||
await conn.execute(text("""
|
||||
CREATE TABLE IF NOT EXISTS card_keywords (
|
||||
id SERIAL PRIMARY KEY,
|
||||
keyword VARCHAR(100) UNIQUE NOT NULL,
|
||||
description TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
"""))
|
||||
|
||||
await conn.execute(text("""
|
||||
CREATE TABLE IF NOT EXISTS set_list (
|
||||
id SERIAL PRIMARY KEY,
|
||||
set_code VARCHAR(10) UNIQUE NOT NULL,
|
||||
set_name VARCHAR(255),
|
||||
set_type VARCHAR(100),
|
||||
release_date DATE,
|
||||
base_set_size INTEGER,
|
||||
total_size INTEGER,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
"""))
|
||||
|
||||
await conn.commit()
|
||||
print("✓ Database tables created")
|
||||
|
||||
|
||||
def download_file(url: str, destination: Path) -> bool:
|
||||
"""Download a file from URL to destination."""
|
||||
from urllib.request import urlretrieve
|
||||
|
||||
try:
|
||||
print(f"Downloading {url}...")
|
||||
urlretrieve(url, destination)
|
||||
size_mb = destination.stat().st_size / (1024 * 1024)
|
||||
print(f" ✓ Downloaded ({size_mb:.1f} MB)")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f" ✗ Failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def extract_zip(zip_path: Path, extract_dir: Path) -> None:
|
||||
"""Extract a zip file."""
|
||||
if zip_path.exists():
|
||||
if extract_dir.exists():
|
||||
shutil.rmtree(extract_dir)
|
||||
|
||||
print(f"Extracting {zip_path.name}...")
|
||||
with zipfile.ZipFile(zip_path, 'r') as zip_ref:
|
||||
zip_ref.extractall(extract_dir)
|
||||
|
||||
file_count = len(list(extract_dir.glob('*.json')))
|
||||
print(f" ✓ Extracted {file_count} files")
|
||||
|
||||
|
||||
def get_all_printings_psql_file(downloads_dir: Path) -> Path:
|
||||
"""Get or download AllPrintings.psql.gz."""
|
||||
psql_file = downloads_dir / "AllPrintings.psql.gz"
|
||||
|
||||
if not psql_file.exists():
|
||||
download_file(f"{MTGJSON_API_V5}/AllPrintings.psql.gz", psql_file)
|
||||
|
||||
return psql_file
|
||||
|
||||
|
||||
def parse_psql_file(psql_file: Path) -> list[dict]:
|
||||
"""Parse a PSQL file and extract card data.
|
||||
|
||||
The PSQL file contains a COPY statement with all card data.
|
||||
We need to parse it and extract the data rows.
|
||||
"""
|
||||
cards = []
|
||||
|
||||
# Read the file and find the COPY section
|
||||
with open(psql_file, 'r', encoding='utf-8', errors='replace') as f:
|
||||
content = f.read()
|
||||
|
||||
# Find the COPY section
|
||||
copy_start = content.find("COPY \"cards\"")
|
||||
if copy_start == -1:
|
||||
print("✗ Could not find COPY statement in PSQL file")
|
||||
return []
|
||||
|
||||
# Find the FROM \. section
|
||||
from_section = content.find("FROM \\.", copy_start)
|
||||
if from_section == -1:
|
||||
print("✗ Could not find FROM section in PSQL file")
|
||||
return []
|
||||
|
||||
# Extract everything after FROM \.
|
||||
data_section = content[from_section + len("FROM \\."):]
|
||||
|
||||
# Split into lines and filter out empty lines and comments
|
||||
lines = data_section.split('\n')
|
||||
data_lines = [line for line in lines if line.strip() and not line.startswith('--')]
|
||||
|
||||
# Parse each line (tab-separated values with \t representing tabs)
|
||||
for line in data_lines:
|
||||
if line == '\\.':
|
||||
break
|
||||
|
||||
# Split by tab character (represented as \t in the file)
|
||||
values = line.split('\t')
|
||||
|
||||
if len(values) >= 94: # We have at least 94 columns
|
||||
# Map values to card fields
|
||||
card = {
|
||||
'artist': values[0],
|
||||
'asciiName': values[1],
|
||||
'attractionLights': values[2],
|
||||
'availability': values[3],
|
||||
'boosterTypes': values[4],
|
||||
'borderColor': values[5],
|
||||
'cardParts': values[6],
|
||||
'colorIdentity': values[7],
|
||||
'colorIndicator': values[8],
|
||||
'colors': values[9],
|
||||
'defense': values[10],
|
||||
'duelDeck': values[11],
|
||||
'edhrecRank': int(values[12]) if values[12] != '\\N' else None,
|
||||
'edhrecSaltiness': float(values[13]) if values[13] != '\\N' else None,
|
||||
'faceConvertedManaCost': float(values[14]) if values[14] != '\\N' else None,
|
||||
'faceFlavorName': values[15],
|
||||
'faceManaValue': float(values[16]) if values[16] != '\\N' else None,
|
||||
'faceName': values[17],
|
||||
'facePrintedName': values[18],
|
||||
'finishes': values[19],
|
||||
'flavorName': values[20],
|
||||
'flavorText': values[21],
|
||||
'frameEffects': values[22],
|
||||
'frameVersion': values[23],
|
||||
'hand': values[24],
|
||||
'hasAlternativeDeckLimit': values[25] == 't',
|
||||
'hasContentWarning': values[26] == 't',
|
||||
'isAlternative': values[27] == 't',
|
||||
'isFullArt': values[28] == 't',
|
||||
'isFunny': values[29] == 't',
|
||||
'isGameChanger': values[30] == 't',
|
||||
'isOnlineOnly': values[31] == 't',
|
||||
'isOversized': values[32] == 't',
|
||||
'isPromo': values[33] == 't',
|
||||
'isRebalanced': values[34] == 't',
|
||||
'isReprint': values[35] == 't',
|
||||
'isReserved': values[36] == 't',
|
||||
'isStorySpotlight': values[37] == 't',
|
||||
'isTextless': values[38] == 't',
|
||||
'isTimeshifted': values[39] == 't',
|
||||
'keywords': values[40],
|
||||
'language': values[41],
|
||||
'layout': values[42],
|
||||
'leadershipSkills': values[43],
|
||||
'life': values[44],
|
||||
'loyalty': values[45],
|
||||
'manaCost': values[46],
|
||||
'manaValue': float(values[47]) if values[47] != '\\N' else None,
|
||||
'name': values[48],
|
||||
'number': values[49],
|
||||
'originalPrintings': values[50],
|
||||
'originalReleaseDate': values[51],
|
||||
'originalText': values[52],
|
||||
'otherFaceIds': values[53],
|
||||
'power': values[54],
|
||||
'printedName': values[55],
|
||||
'printedText': values[56],
|
||||
'printedType': values[57],
|
||||
'printings': values[58],
|
||||
'producedMana': values[59],
|
||||
'promoTypes': values[60],
|
||||
'rarity': values[61],
|
||||
'rebalancedPrintings': values[62],
|
||||
'relatedCards': values[63],
|
||||
'securityStamp': values[64],
|
||||
'setCode': values[65],
|
||||
'side': values[66],
|
||||
'signature': values[67],
|
||||
'skuIds': values[68],
|
||||
'sourceProducts': values[69],
|
||||
'subsets': values[70],
|
||||
'subtypes': values[71],
|
||||
'supertypes': values[72],
|
||||
'text': values[73],
|
||||
'toughness': values[74],
|
||||
'type': values[75],
|
||||
'types': values[76],
|
||||
'uuid': values[77],
|
||||
'variations': values[78],
|
||||
'watermark': values[79],
|
||||
}
|
||||
|
||||
cards.append(card)
|
||||
|
||||
return cards
|
||||
|
||||
|
||||
async def import_cards(engine, cards: list[dict]) -> int:
|
||||
"""Import cards data."""
|
||||
count = 0
|
||||
|
||||
async with engine.begin() as conn:
|
||||
for card in cards:
|
||||
await conn.execute(
|
||||
pg_insert(text('cards')).values(card).on_conflict_do_nothing(),
|
||||
execution_options={"autocommit": True}
|
||||
)
|
||||
count += 1
|
||||
|
||||
print(f"✓ Imported {count:,} cards")
|
||||
return count
|
||||
|
||||
|
||||
async def import_all_printings_psql(engine, psql_file: Path) -> int:
|
||||
"""Import AllPrintings.psql.gz."""
|
||||
if not psql_file.exists():
|
||||
print("✗ AllPrintings.psql.gz not found")
|
||||
return 0
|
||||
|
||||
print("Importing AllPrintings...")
|
||||
|
||||
# Parse the PSQL file
|
||||
cards = parse_psql_file(psql_file)
|
||||
|
||||
if not cards:
|
||||
print("✗ No cards found in PSQL file")
|
||||
return 0
|
||||
|
||||
# Import cards
|
||||
return await import_cards(engine, cards)
|
||||
|
||||
|
||||
async def import_all_set_files(engine, set_files_dir: Path) -> tuple[int, int]:
|
||||
"""Import AllSetFiles - sets and cards."""
|
||||
if not set_files_dir.exists():
|
||||
print("✗ AllSetFiles directory not found")
|
||||
return 0, 0
|
||||
|
||||
set_count = 0
|
||||
card_count = 0
|
||||
|
||||
async with engine.begin() as conn:
|
||||
# Import sets
|
||||
for set_file in sorted(set_files_dir.glob('*.json')):
|
||||
data = json.loads(set_file.read_text())
|
||||
|
||||
if 'code' in data:
|
||||
image_url = None
|
||||
if 'image' in data and data['image']:
|
||||
image_url = data['image'].get('normal')
|
||||
|
||||
# Upsert set
|
||||
result = await conn.execute(
|
||||
pg_insert(text('mtg_sets')).values(
|
||||
code=data.get('code'),
|
||||
name=data.get('name'),
|
||||
type=data.get('type'),
|
||||
release_date=data.get('releaseDate'),
|
||||
base_set_size=data.get('baseSetSize'),
|
||||
total_size=data.get('totalSetSize'),
|
||||
is_foil_only=data.get('isFoilOnly'),
|
||||
is_non_foil_only=data.get('isNonFoilOnly'),
|
||||
digital=data.get('digital'),
|
||||
icon_svg_url=data.get('iconSvgUrl'),
|
||||
parent_code=data.get('parentCode'),
|
||||
mtgo_code=data.get('mtgoCode'),
|
||||
card_count=data.get('cardCount'),
|
||||
image_url=image_url,
|
||||
).on_conflict_do_update(
|
||||
index_elements=['code'],
|
||||
set_={
|
||||
'name': data.get('name'),
|
||||
'type': data.get('type'),
|
||||
'release_date': data.get('releaseDate'),
|
||||
'base_set_size': data.get('baseSetSize'),
|
||||
'total_size': data.get('totalSetSize'),
|
||||
'is_foil_only': data.get('isFoilOnly'),
|
||||
'is_non_foil_only': data.get('isNonFoilOnly'),
|
||||
'digital': data.get('digital'),
|
||||
'icon_svg_url': data.get('iconSvgUrl'),
|
||||
'parent_code': data.get('parentCode'),
|
||||
'mtgo_code': data.get('mtgoCode'),
|
||||
'card_count': data.get('cardCount'),
|
||||
'image_url': image_url,
|
||||
'updated_at': datetime.utcnow(),
|
||||
}
|
||||
).returning(text('mtg_sets.id')),
|
||||
execution_options={"autocommit": True}
|
||||
)
|
||||
set_id = result.scalar()
|
||||
|
||||
if 'cards' in data:
|
||||
for card_data in data['cards']:
|
||||
identifiers = {
|
||||
'multiId': card_data.get('multiverseIds'),
|
||||
'tcgplayerProductId': card_data.get('tcgplayerProductId'),
|
||||
'cardmarketId': card_data.get('cardmarketId'),
|
||||
}
|
||||
|
||||
images = {}
|
||||
if 'image_uris' in card_data:
|
||||
images = {
|
||||
'small': card_data['image_uris'].get('small'),
|
||||
'normal': card_data['image_uris'].get('normal'),
|
||||
'large': card_data['image_uris'].get('large'),
|
||||
'png': card_data['image_uris'].get('png'),
|
||||
'art_crop': card_data['image_uris'].get('art_crop'),
|
||||
}
|
||||
|
||||
# Upsert card
|
||||
await conn.execute(
|
||||
pg_insert(text('mtg_cards')).values(
|
||||
set_id=set_id,
|
||||
name=card_data.get('name'),
|
||||
mana_cost=card_data.get('manaCost'),
|
||||
type_line=card_data.get('typeLine'),
|
||||
oracle_text=card_data.get('oracleText'),
|
||||
power=card_data.get('power'),
|
||||
toughness=card_data.get('toughness'),
|
||||
rarity=card_data.get('rarity'),
|
||||
layout=card_data.get('layout'),
|
||||
artist=card_data.get('artist'),
|
||||
flavor_text=card_data.get('flavorText'),
|
||||
numbers=str(card_data.get('number')),
|
||||
identifiers=json.dumps(identifiers),
|
||||
images=json.dumps(images),
|
||||
).on_conflict_do_nothing(),
|
||||
execution_options={"autocommit": True}
|
||||
)
|
||||
card_count += 1
|
||||
|
||||
set_count += 1
|
||||
|
||||
print(f"✓ Imported {set_count} sets and {card_count:,} cards from sets")
|
||||
return set_count, card_count
|
||||
|
||||
|
||||
async def import_all_identifiers(engine, file_path: Path) -> int:
|
||||
"""Import AllIdentifiers.json.gz."""
|
||||
if not file_path.exists():
|
||||
print("✗ AllIdentifiers.json.gz not found")
|
||||
return 0
|
||||
|
||||
data = json.loads(gzip.decompress(file_path.read_bytes()))
|
||||
|
||||
count = 0
|
||||
async with engine.begin() as conn:
|
||||
for uuid, card_data in data.items():
|
||||
await conn.execute(
|
||||
pg_insert(text('card_identifiers')).values(
|
||||
uuid=uuid,
|
||||
name=card_data.get('name'),
|
||||
mana_cost=card_data.get('manaCost'),
|
||||
type_line=card_data.get('typeLine'),
|
||||
oracle_text=card_data.get('oracleText'),
|
||||
power=card_data.get('power'),
|
||||
toughness=card_data.get('toughness'),
|
||||
rarity=card_data.get('rarity'),
|
||||
layout=card_data.get('layout'),
|
||||
artist=card_data.get('artist'),
|
||||
flavor_text=card_data.get('flavorText'),
|
||||
set_code=card_data.get('setCode'),
|
||||
).on_conflict_do_update(
|
||||
index_elements=['uuid'],
|
||||
set_={
|
||||
'name': card_data.get('name'),
|
||||
'mana_cost': card_data.get('manaCost'),
|
||||
'type_line': card_data.get('typeLine'),
|
||||
'oracle_text': card_data.get('oracleText'),
|
||||
'power': card_data.get('power'),
|
||||
'toughness': card_data.get('toughness'),
|
||||
'rarity': card_data.get('rarity'),
|
||||
'layout': card_data.get('layout'),
|
||||
'artist': card_data.get('artist'),
|
||||
'flavor_text': card_data.get('flavorText'),
|
||||
'set_code': card_data.get('setCode'),
|
||||
'updated_at': datetime.utcnow(),
|
||||
}
|
||||
),
|
||||
execution_options={"autocommit": True}
|
||||
)
|
||||
count += 1
|
||||
|
||||
print(f"✓ Imported {count:,} identifiers")
|
||||
return count
|
||||
|
||||
|
||||
async def import_all_deck_files(engine, deck_files_dir: Path) -> int:
|
||||
"""Import AllDeckFiles.zip."""
|
||||
if not deck_files_dir.exists():
|
||||
print("✗ AllDeckFiles directory not found")
|
||||
return 0
|
||||
|
||||
count = 0
|
||||
async with engine.begin() as conn:
|
||||
for deck_file in sorted(deck_files_dir.glob('*.json')):
|
||||
data = json.loads(deck_file.read_text())
|
||||
|
||||
if 'name' in data and 'cards' in data:
|
||||
await conn.execute(
|
||||
pg_insert(text('decks')).values(
|
||||
name=data.get('name'),
|
||||
description=data.get('description'),
|
||||
format=data.get('format'),
|
||||
command=data.get('command'),
|
||||
commander=data.get('commander'),
|
||||
).on_conflict_do_update(
|
||||
index_elements=['name'],
|
||||
set_={
|
||||
'description': data.get('description'),
|
||||
'format': data.get('format'),
|
||||
'command': data.get('command'),
|
||||
'commander': data.get('commander'),
|
||||
'updated_at': datetime.utcnow(),
|
||||
}
|
||||
),
|
||||
execution_options={"autocommit": True}
|
||||
)
|
||||
count += 1
|
||||
|
||||
print(f"✓ Imported {count} decks")
|
||||
return count
|
||||
|
||||
|
||||
async def import_simple_json(engine, file_path: Path, table_name: str,
|
||||
name_field: str, id_field: str) -> int:
|
||||
"""Import a simple JSON.gz file."""
|
||||
if not file_path.exists():
|
||||
return 0
|
||||
|
||||
data = json.loads(gzip.decompress(file_path.read_bytes()))
|
||||
|
||||
count = 0
|
||||
async with engine.begin() as conn:
|
||||
for item in data:
|
||||
# Build the insert statement based on table name
|
||||
if table_name == 'card_types':
|
||||
insert_stmt = pg_insert(text('card_types')).values(
|
||||
type=item.get('type'),
|
||||
description=item.get('description')
|
||||
).on_conflict_do_nothing()
|
||||
elif table_name == 'deck_list':
|
||||
insert_stmt = pg_insert(text('deck_list')).values(
|
||||
deck_id=item.get('id'),
|
||||
name=item.get('name'),
|
||||
description=item.get('description'),
|
||||
format=item.get('format'),
|
||||
command=item.get('command'),
|
||||
commander=item.get('commander'),
|
||||
total_cards=item.get('totalCards')
|
||||
).on_conflict_do_nothing()
|
||||
elif table_name == 'card_keywords':
|
||||
insert_stmt = pg_insert(text('card_keywords')).values(
|
||||
keyword=item.get('keyword'),
|
||||
description=item.get('description')
|
||||
).on_conflict_do_nothing()
|
||||
elif table_name == 'set_list':
|
||||
insert_stmt = pg_insert(text('set_list')).values(
|
||||
set_code=item.get('code'),
|
||||
set_name=item.get('name'),
|
||||
set_type=item.get('type'),
|
||||
release_date=item.get('releaseDate'),
|
||||
base_set_size=item.get('baseSetSize'),
|
||||
total_size=item.get('totalSetSize')
|
||||
).on_conflict_do_nothing()
|
||||
else:
|
||||
continue
|
||||
|
||||
await conn.execute(insert_stmt, execution_options={"autocommit": True})
|
||||
count += 1
|
||||
|
||||
print(f"✓ Imported {count} items into {table_name}")
|
||||
return count
|
||||
|
||||
|
||||
async def create_indexes(engine):
|
||||
"""Create indexes for better performance."""
|
||||
async with engine.begin() as conn:
|
||||
indexes = [
|
||||
"CREATE INDEX IF NOT EXISTS idx_cards_name ON cards(name)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_cards_mana_cost ON cards(manaCost)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_cards_type ON cards(type)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_cards_rarity ON cards(rarity)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_cards_set_code ON cards(setCode)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_cards_uuid ON cards(uuid)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_mtg_sets_code ON mtg_sets(code)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_mtg_cards_name ON mtg_cards(name)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_card_identifiers_uuid ON card_identifiers(uuid)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_card_identifiers_name ON card_identifiers(name)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_deck_list_deck_id ON deck_list(deck_id)",
|
||||
]
|
||||
|
||||
for idx in indexes:
|
||||
await conn.execute(text(idx))
|
||||
|
||||
await conn.commit()
|
||||
print("✓ Indexes created")
|
||||
|
||||
|
||||
async def show_summary(engine):
|
||||
"""Show import summary."""
|
||||
print("\n=== Import Summary ===")
|
||||
async with engine.connect() as conn:
|
||||
tables = [
|
||||
'cards', 'mtg_sets', 'mtg_cards', 'card_identifiers',
|
||||
'decks', 'card_types', 'deck_list', 'card_keywords', 'set_list'
|
||||
]
|
||||
|
||||
for table in tables:
|
||||
result = await conn.execute(text(f"SELECT COUNT(*) FROM {table}"))
|
||||
count = result.scalar()
|
||||
print(f" {table:20} {count:>10,} records")
|
||||
|
||||
|
||||
async def main():
|
||||
"""Main function to download and import all MTGJSON data."""
|
||||
settings = get_settings()
|
||||
|
||||
# Setup data directory
|
||||
data_dir = Path(settings.DATA_DIR) / "mtgjson"
|
||||
downloads_dir = data_dir / "downloads"
|
||||
downloads_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Create engine
|
||||
engine = create_async_engine(settings.MTG_DATABASE_URL)
|
||||
|
||||
# Create tables
|
||||
print("=== Creating Database Tables ===")
|
||||
await create_tables(engine)
|
||||
|
||||
# Download and import AllPrintings
|
||||
print("\n=== Importing AllPrintings ===")
|
||||
psql_file = get_all_printings_psql_file(downloads_dir)
|
||||
await import_all_printings_psql(engine, psql_file)
|
||||
|
||||
# Download and extract AllSetFiles
|
||||
print("\n=== Importing AllSetFiles ===")
|
||||
set_files_zip = downloads_dir / "AllSetFiles.zip"
|
||||
set_files_dir = downloads_dir / "AllSetFiles"
|
||||
|
||||
if not set_files_dir.exists():
|
||||
if not set_files_zip.exists():
|
||||
download_file(f"{MTGJSON_API_V5}/AllSetFiles.zip", set_files_zip)
|
||||
extract_zip(set_files_zip, set_files_dir)
|
||||
|
||||
await import_all_set_files(engine, set_files_dir)
|
||||
|
||||
# Download and extract AllDeckFiles
|
||||
print("\n=== Importing AllDeckFiles ===")
|
||||
deck_files_zip = downloads_dir / "AllDeckFiles.zip"
|
||||
deck_files_dir = downloads_dir / "AllDeckFiles"
|
||||
|
||||
if not deck_files_dir.exists():
|
||||
if not deck_files_zip.exists():
|
||||
download_file(f"{MTGJSON_API_V5}/AllDeckFiles.zip", deck_files_zip)
|
||||
extract_zip(deck_files_zip, deck_files_dir)
|
||||
|
||||
await import_all_deck_files(engine, deck_files_dir)
|
||||
|
||||
# Download and import AllIdentifiers
|
||||
print("\n=== Importing AllIdentifiers ===")
|
||||
identifiers_file = downloads_dir / "AllIdentifiers.json.gz"
|
||||
if not identifiers_file.exists():
|
||||
download_file(f"{MTGJSON_API_V5}/AllIdentifiers.json.gz", identifiers_file)
|
||||
|
||||
await import_all_identifiers(engine, identifiers_file)
|
||||
|
||||
# Download and import CardTypes
|
||||
print("\n=== Importing CardTypes ===")
|
||||
card_types_file = downloads_dir / "CardTypes.json.gz"
|
||||
if not card_types_file.exists():
|
||||
download_file(f"{MTGJSON_API_V5}/CardTypes.json.gz", card_types_file)
|
||||
|
||||
if card_types_file.exists():
|
||||
await import_simple_json(engine, card_types_file, 'card_types', 'type', 'id')
|
||||
|
||||
# Download and import DeckList
|
||||
print("\n=== Importing DeckList ===")
|
||||
deck_list_file = downloads_dir / "DeckList.json.gz"
|
||||
if not deck_list_file.exists():
|
||||
download_file(f"{MTGJSON_API_V5}/DeckList.json.gz", deck_list_file)
|
||||
|
||||
if deck_list_file.exists():
|
||||
await import_simple_json(engine, deck_list_file, 'deck_list', 'name', 'deck_id')
|
||||
|
||||
# Download and import Keywords
|
||||
print("\n=== Importing Keywords ===")
|
||||
keywords_file = downloads_dir / "Keywords.json.gz"
|
||||
if not keywords_file.exists():
|
||||
download_file(f"{MTGJSON_API_V5}/Keywords.json.gz", keywords_file)
|
||||
|
||||
if keywords_file.exists():
|
||||
await import_simple_json(engine, keywords_file, 'card_keywords', 'keyword', 'id')
|
||||
|
||||
# Download and import SetList
|
||||
print("\n=== Importing SetList ===")
|
||||
set_list_file = downloads_dir / "SetList.json.gz"
|
||||
if not set_list_file.exists():
|
||||
download_file(f"{MTGJSON_API_V5}/SetList.json.gz", set_list_file)
|
||||
|
||||
if set_list_file.exists():
|
||||
await import_simple_json(engine, set_list_file, 'set_list', 'set_name', 'set_code')
|
||||
|
||||
# Create indexes
|
||||
print("\n=== Creating Indexes ===")
|
||||
await create_indexes(engine)
|
||||
|
||||
# Show summary
|
||||
await show_summary(engine)
|
||||
|
||||
await engine.dispose()
|
||||
print("\n✓ All MTGJSON data imported successfully!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -1,883 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
MTGJSON Data Uploader
|
||||
|
||||
Downloads all MTGJSON data and upserts it into the PostgreSQL database.
|
||||
This script is designed to run inside the Docker container.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import gzip
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from urllib.request import urlretrieve
|
||||
|
||||
# Add parent directory to path for imports
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from sqlalchemy import text, insert, update, select, and_, Table, MetaData, Column, Integer, String, Text, Boolean, DateTime, Date, ForeignKey
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
|
||||
from app.core.settings import get_settings
|
||||
|
||||
# Define table metadata
|
||||
metadata = MetaData()
|
||||
|
||||
# Define table objects
|
||||
mtg_sets_table = Table('mtg_sets', metadata,
|
||||
Column('id', Integer, primary_key=True),
|
||||
Column('code', String(10), unique=True, nullable=False),
|
||||
Column('name', String(255)),
|
||||
Column('type', String(100)),
|
||||
Column('release_date', Date),
|
||||
Column('base_set_size', Integer),
|
||||
Column('total_size', Integer),
|
||||
Column('is_foil_only', Boolean),
|
||||
Column('is_non_foil_only', Boolean),
|
||||
Column('digital', Boolean),
|
||||
Column('icon_svg_url', Text),
|
||||
Column('parent_code', String(10)),
|
||||
Column('mtgo_code', String(10)),
|
||||
Column('card_count', Integer),
|
||||
Column('image_url', Text),
|
||||
Column('created_at', DateTime, default=datetime.utcnow),
|
||||
Column('updated_at', DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
)
|
||||
|
||||
mtg_cards_table = Table('mtg_cards', metadata,
|
||||
Column('id', Integer, primary_key=True),
|
||||
Column('set_id', Integer, ForeignKey('mtg_sets.id')),
|
||||
Column('name', String(255)),
|
||||
Column('mana_cost', String(255)),
|
||||
Column('type_line', String(255)),
|
||||
Column('oracle_text', Text),
|
||||
Column('power', String(50)),
|
||||
Column('toughness', String(50)),
|
||||
Column('rarity', String(50)),
|
||||
Column('layout', String(50)),
|
||||
Column('artist', String(255)),
|
||||
Column('flavor_text', Text),
|
||||
Column('numbers', String(100)),
|
||||
Column('identifiers', Text),
|
||||
Column('images', Text),
|
||||
Column('created_at', DateTime, default=datetime.utcnow),
|
||||
Column('updated_at', DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
)
|
||||
|
||||
card_identifiers_table = Table('card_identifiers', metadata,
|
||||
Column('id', Integer, primary_key=True),
|
||||
Column('uuid', String, unique=True, nullable=False),
|
||||
Column('name', String(255)),
|
||||
Column('mana_cost', String(255)),
|
||||
Column('type_line', String(255)),
|
||||
Column('oracle_text', Text),
|
||||
Column('power', String(50)),
|
||||
Column('toughness', String(50)),
|
||||
Column('rarity', String(50)),
|
||||
Column('layout', String(50)),
|
||||
Column('artist', String(255)),
|
||||
Column('flavor_text', Text),
|
||||
Column('set_code', String(10)),
|
||||
Column('created_at', DateTime, default=datetime.utcnow),
|
||||
Column('updated_at', DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
)
|
||||
|
||||
decks_table = Table('decks', metadata,
|
||||
Column('id', Integer, primary_key=True),
|
||||
Column('name', String(255), nullable=False),
|
||||
Column('description', Text),
|
||||
Column('format', String(50)),
|
||||
Column('command', Text),
|
||||
Column('commander', Text),
|
||||
Column('creation_date', DateTime, default=datetime.utcnow),
|
||||
Column('updated_at', DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
)
|
||||
|
||||
card_types_table = Table('card_types', metadata,
|
||||
Column('id', Integer, primary_key=True),
|
||||
Column('type', String(100), unique=True, nullable=False),
|
||||
Column('description', Text),
|
||||
Column('created_at', DateTime, default=datetime.utcnow),
|
||||
Column('updated_at', DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
)
|
||||
|
||||
deck_list_table = Table('deck_list', metadata,
|
||||
Column('id', Integer, primary_key=True),
|
||||
Column('deck_id', String(100), unique=True, nullable=False),
|
||||
Column('name', String(255)),
|
||||
Column('description', Text),
|
||||
Column('format', String(50)),
|
||||
Column('command', Text),
|
||||
Column('commander', Text),
|
||||
Column('total_cards', Integer),
|
||||
Column('created_at', DateTime, default=datetime.utcnow),
|
||||
Column('updated_at', DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
)
|
||||
|
||||
card_keywords_table = Table('card_keywords', metadata,
|
||||
Column('id', Integer, primary_key=True),
|
||||
Column('keyword', String(100), unique=True, nullable=False),
|
||||
Column('description', Text),
|
||||
Column('created_at', DateTime, default=datetime.utcnow),
|
||||
Column('updated_at', DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
)
|
||||
|
||||
set_list_table = Table('set_list', metadata,
|
||||
Column('id', Integer, primary_key=True),
|
||||
Column('set_code', String(10), unique=True, nullable=False),
|
||||
Column('set_name', String(255)),
|
||||
Column('set_type', String(100)),
|
||||
Column('release_date', Date),
|
||||
Column('base_set_size', Integer),
|
||||
Column('total_size', Integer),
|
||||
Column('created_at', DateTime, default=datetime.utcnow),
|
||||
Column('updated_at', DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
)
|
||||
|
||||
# MTGJSON API v5 base URL
|
||||
MTGJSON_API_V5 = "https://mtgjson.com/api/v5"
|
||||
|
||||
# Files to download
|
||||
MTGJSON_FILES = [
|
||||
"AllPrintings.psql.gz",
|
||||
"AllSetFiles.zip",
|
||||
"AllDeckFiles.zip",
|
||||
"AllIdentifiers.json.gz",
|
||||
"CardTypes.json.gz",
|
||||
"DeckList.json.gz",
|
||||
"Keywords.json.gz",
|
||||
"SetList.json.gz",
|
||||
]
|
||||
|
||||
|
||||
async def create_tables(engine: create_async_engine) -> None:
|
||||
"""Create all required tables."""
|
||||
async with engine.begin() as conn:
|
||||
# Cards table (from AllPrintings)
|
||||
await conn.execute(text("""
|
||||
CREATE TABLE IF NOT EXISTS cards (
|
||||
id SERIAL PRIMARY KEY,
|
||||
artist TEXT,
|
||||
asciiName TEXT,
|
||||
attractionLights TEXT,
|
||||
availability TEXT,
|
||||
boosterTypes TEXT,
|
||||
borderColor TEXT,
|
||||
cardParts TEXT,
|
||||
colorIdentity TEXT,
|
||||
colorIndicator TEXT,
|
||||
colors TEXT,
|
||||
defense TEXT,
|
||||
duelDeck TEXT,
|
||||
edhrecRank INTEGER,
|
||||
edhrecSaltiness FLOAT,
|
||||
faceConvertedManaCost FLOAT,
|
||||
faceFlavorName TEXT,
|
||||
faceManaValue FLOAT,
|
||||
faceName TEXT,
|
||||
facePrintedName TEXT,
|
||||
finishes TEXT,
|
||||
flavorName TEXT,
|
||||
flavorText TEXT,
|
||||
frameEffects TEXT,
|
||||
frameVersion TEXT,
|
||||
hand TEXT,
|
||||
hasAlternativeDeckLimit BOOLEAN,
|
||||
hasContentWarning BOOLEAN,
|
||||
isAlternative BOOLEAN,
|
||||
isFullArt BOOLEAN,
|
||||
isFunny BOOLEAN,
|
||||
isGameChanger BOOLEAN,
|
||||
isOnlineOnly BOOLEAN,
|
||||
isOversized BOOLEAN,
|
||||
isPromo BOOLEAN,
|
||||
isRebalanced BOOLEAN,
|
||||
isReprint BOOLEAN,
|
||||
isReserved BOOLEAN,
|
||||
isStorySpotlight BOOLEAN,
|
||||
isTextless BOOLEAN,
|
||||
isTimeshifted BOOLEAN,
|
||||
keywords TEXT,
|
||||
language TEXT,
|
||||
layout TEXT,
|
||||
leadershipSkills TEXT,
|
||||
life TEXT,
|
||||
loyalty TEXT,
|
||||
manaCost TEXT,
|
||||
manaValue FLOAT,
|
||||
name TEXT,
|
||||
number TEXT,
|
||||
originalPrintings TEXT,
|
||||
originalReleaseDate TEXT,
|
||||
originalText TEXT,
|
||||
otherFaceIds TEXT,
|
||||
power TEXT,
|
||||
printedName TEXT,
|
||||
printedText TEXT,
|
||||
printedType TEXT,
|
||||
printings TEXT,
|
||||
producedMana TEXT,
|
||||
promoTypes TEXT,
|
||||
rarity TEXT,
|
||||
rebalancedPrintings TEXT,
|
||||
relatedCards TEXT,
|
||||
securityStamp TEXT,
|
||||
setCode TEXT,
|
||||
side TEXT,
|
||||
signature TEXT,
|
||||
skuIds TEXT,
|
||||
sourceProducts TEXT,
|
||||
subsets TEXT,
|
||||
subtypes TEXT,
|
||||
supertypes TEXT,
|
||||
text TEXT,
|
||||
toughness TEXT,
|
||||
type TEXT,
|
||||
types TEXT,
|
||||
uuid TEXT,
|
||||
variations TEXT,
|
||||
watermark TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
"""))
|
||||
|
||||
# MTG Sets table
|
||||
await conn.execute(text("""
|
||||
CREATE TABLE IF NOT EXISTS mtg_sets (
|
||||
id SERIAL PRIMARY KEY,
|
||||
code VARCHAR(10) UNIQUE NOT NULL,
|
||||
name VARCHAR(255),
|
||||
type VARCHAR(100),
|
||||
release_date DATE,
|
||||
base_set_size INTEGER,
|
||||
total_size INTEGER,
|
||||
is_foil_only BOOLEAN,
|
||||
is_non_foil_only BOOLEAN,
|
||||
digital BOOLEAN,
|
||||
icon_svg_url TEXT,
|
||||
parent_code VARCHAR(10),
|
||||
mtgo_code VARCHAR(10),
|
||||
card_count INTEGER,
|
||||
image_url TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
"""))
|
||||
|
||||
# MTG Cards table (from set files)
|
||||
await conn.execute(text("""
|
||||
CREATE TABLE IF NOT EXISTS mtg_cards (
|
||||
id SERIAL PRIMARY KEY,
|
||||
set_id INTEGER REFERENCES mtg_sets(id) ON DELETE CASCADE,
|
||||
name VARCHAR(255),
|
||||
mana_cost VARCHAR(255),
|
||||
type_line VARCHAR(255),
|
||||
oracle_text TEXT,
|
||||
power VARCHAR(50),
|
||||
toughness VARCHAR(50),
|
||||
rarity VARCHAR(50),
|
||||
layout VARCHAR(50),
|
||||
artist VARCHAR(255),
|
||||
flavor_text TEXT,
|
||||
numbers VARCHAR(100),
|
||||
identifiers TEXT,
|
||||
images TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
"""))
|
||||
|
||||
# Card Identifiers table
|
||||
await conn.execute(text("""
|
||||
CREATE TABLE IF NOT EXISTS card_identifiers (
|
||||
id SERIAL PRIMARY KEY,
|
||||
uuid TEXT UNIQUE NOT NULL,
|
||||
name VARCHAR(255),
|
||||
mana_cost VARCHAR(255),
|
||||
type_line VARCHAR(255),
|
||||
oracle_text TEXT,
|
||||
power VARCHAR(50),
|
||||
toughness VARCHAR(50),
|
||||
rarity VARCHAR(50),
|
||||
layout VARCHAR(50),
|
||||
artist VARCHAR(255),
|
||||
flavor_text TEXT,
|
||||
set_code VARCHAR(10),
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
"""))
|
||||
|
||||
# Decks table
|
||||
await conn.execute(text("""
|
||||
CREATE TABLE IF NOT EXISTS decks (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
description TEXT,
|
||||
format VARCHAR(50),
|
||||
command TEXT,
|
||||
commander TEXT,
|
||||
creation_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
"""))
|
||||
|
||||
# Card Types table
|
||||
await conn.execute(text("""
|
||||
CREATE TABLE IF NOT EXISTS card_types (
|
||||
id SERIAL PRIMARY KEY,
|
||||
type VARCHAR(100) UNIQUE NOT NULL,
|
||||
description TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
"""))
|
||||
|
||||
# Deck List table
|
||||
await conn.execute(text("""
|
||||
CREATE TABLE IF NOT EXISTS deck_list (
|
||||
id SERIAL PRIMARY KEY,
|
||||
deck_id VARCHAR(100) UNIQUE NOT NULL,
|
||||
name VARCHAR(255),
|
||||
description TEXT,
|
||||
format VARCHAR(50),
|
||||
command TEXT,
|
||||
commander TEXT,
|
||||
total_cards INTEGER,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
"""))
|
||||
|
||||
# Card Keywords table
|
||||
await conn.execute(text("""
|
||||
CREATE TABLE IF NOT EXISTS card_keywords (
|
||||
id SERIAL PRIMARY KEY,
|
||||
keyword VARCHAR(100) UNIQUE NOT NULL,
|
||||
description TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
"""))
|
||||
|
||||
# Set List table
|
||||
await conn.execute(text("""
|
||||
CREATE TABLE IF NOT EXISTS set_list (
|
||||
id SERIAL PRIMARY KEY,
|
||||
set_code VARCHAR(10) UNIQUE NOT NULL,
|
||||
set_name VARCHAR(255),
|
||||
set_type VARCHAR(100),
|
||||
release_date DATE,
|
||||
base_set_size INTEGER,
|
||||
total_size INTEGER,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
"""))
|
||||
|
||||
await conn.commit()
|
||||
print("✓ Database tables created")
|
||||
|
||||
|
||||
async def import_all_printings_psql(engine: create_async_engine, psql_file: Path) -> int:
|
||||
"""Import AllPrintings.psql.gz file."""
|
||||
if not psql_file.exists():
|
||||
print("✗ AllPrintings.psql.gz not found")
|
||||
return 0
|
||||
|
||||
print(f"Importing {psql_file.name}...")
|
||||
|
||||
# Decompress
|
||||
psql_content = gzip.decompress(psql_file.read_bytes())
|
||||
psql_path = psql_file.with_suffix('.psql')
|
||||
psql_path.write_bytes(psql_content)
|
||||
|
||||
# Use psql command to import
|
||||
import subprocess
|
||||
|
||||
settings = get_settings()
|
||||
db_url = settings.MTG_DATABASE_URL
|
||||
|
||||
# Parse database URL to extract connection details
|
||||
# Format: postgresql+asyncpg://user:pass@host:port/database
|
||||
url_parts = db_url.replace('postgresql+asyncpg://', '').split('@')
|
||||
user_pass = url_parts[0].split('//')[1]
|
||||
host_db = url_parts[1]
|
||||
|
||||
user, password = user_pass.split(':')
|
||||
host, port_db = host_db.split(':')
|
||||
database = port_db.split('/')[1]
|
||||
|
||||
cmd = [
|
||||
'psql', '-h', host, '-p', port_db.split('/')[0],
|
||||
'-U', user, '-d', database,
|
||||
'-f', str(psql_path)
|
||||
]
|
||||
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
|
||||
if result.returncode != 0:
|
||||
print(f"✗ Import failed: {result.stderr[:500]}")
|
||||
psql_path.unlink()
|
||||
return 0
|
||||
|
||||
# Count records
|
||||
async with engine.connect() as conn:
|
||||
result = await conn.execute(text("SELECT COUNT(*) FROM cards"))
|
||||
count = result.scalar()
|
||||
print(f"✓ Imported {count:,} cards")
|
||||
|
||||
# Clean up
|
||||
psql_path.unlink()
|
||||
return count
|
||||
|
||||
|
||||
async def import_all_set_files(engine: create_async_engine, set_files_dir: Path) -> tuple[int, int]:
|
||||
"""Import AllSetFiles.zip - sets and cards."""
|
||||
if not set_files_dir.exists():
|
||||
print("✗ AllSetFiles directory not found")
|
||||
return 0, 0
|
||||
|
||||
set_count = 0
|
||||
card_count = 0
|
||||
|
||||
async with engine.begin() as conn:
|
||||
# Import sets
|
||||
for set_file in sorted(set_files_dir.glob('*.json')):
|
||||
data = json.loads(set_file.read_text())
|
||||
|
||||
if 'code' in data:
|
||||
image_url = None
|
||||
if 'image' in data and data['image']:
|
||||
image_url = data['image'].get('normal')
|
||||
|
||||
# Upsert set
|
||||
result = await conn.execute(
|
||||
pg_insert(mtg_sets_table).values(
|
||||
code=data.get('code'),
|
||||
name=data.get('name'),
|
||||
type=data.get('type'),
|
||||
release_date=data.get('releaseDate'),
|
||||
base_set_size=data.get('baseSetSize'),
|
||||
total_size=data.get('totalSetSize'),
|
||||
is_foil_only=data.get('isFoilOnly'),
|
||||
is_non_foil_only=data.get('isNonFoilOnly'),
|
||||
digital=data.get('digital'),
|
||||
icon_svg_url=data.get('iconSvgUrl'),
|
||||
parent_code=data.get('parentCode'),
|
||||
mtgo_code=data.get('mtgoCode'),
|
||||
card_count=data.get('cardCount'),
|
||||
image_url=image_url,
|
||||
).on_conflict_do_update(
|
||||
index_elements=['code'],
|
||||
set_={
|
||||
'name': data.get('name'),
|
||||
'type': data.get('type'),
|
||||
'release_date': data.get('releaseDate'),
|
||||
'base_set_size': data.get('baseSetSize'),
|
||||
'total_size': data.get('totalSetSize'),
|
||||
'is_foil_only': data.get('isFoilOnly'),
|
||||
'is_non_foil_only': data.get('isNonFoilOnly'),
|
||||
'digital': data.get('digital'),
|
||||
'icon_svg_url': data.get('iconSvgUrl'),
|
||||
'parent_code': data.get('parentCode'),
|
||||
'mtgo_code': data.get('mtgoCode'),
|
||||
'card_count': data.get('cardCount'),
|
||||
'image_url': image_url,
|
||||
'updated_at': datetime.utcnow(),
|
||||
}
|
||||
).returning(mtg_sets_table.id),
|
||||
execution_options={"autocommit": True}
|
||||
)
|
||||
set_id = result.scalar()
|
||||
|
||||
if 'cards' in data:
|
||||
for card_data in data['cards']:
|
||||
identifiers = {
|
||||
'multiId': card_data.get('multiverseIds'),
|
||||
'tcgplayerProductId': card_data.get('tcgplayerProductId'),
|
||||
'cardmarketId': card_data.get('cardmarketId'),
|
||||
}
|
||||
|
||||
images = {}
|
||||
if 'image_uris' in card_data:
|
||||
images = {
|
||||
'small': card_data['image_uris'].get('small'),
|
||||
'normal': card_data['image_uris'].get('normal'),
|
||||
'large': card_data['image_uris'].get('large'),
|
||||
'png': card_data['image_uris'].get('png'),
|
||||
'art_crop': card_data['image_uris'].get('art_crop'),
|
||||
}
|
||||
|
||||
# Upsert card
|
||||
await conn.execute(
|
||||
pg_insert(mtg_cards_table).values(
|
||||
set_id=set_id,
|
||||
name=card_data.get('name'),
|
||||
mana_cost=card_data.get('manaCost'),
|
||||
type_line=card_data.get('typeLine'),
|
||||
oracle_text=card_data.get('oracleText'),
|
||||
power=card_data.get('power'),
|
||||
toughness=card_data.get('toughness'),
|
||||
rarity=card_data.get('rarity'),
|
||||
layout=card_data.get('layout'),
|
||||
artist=card_data.get('artist'),
|
||||
flavor_text=card_data.get('flavorText'),
|
||||
numbers=str(card_data.get('number')),
|
||||
identifiers=json.dumps(identifiers),
|
||||
images=json.dumps(images),
|
||||
).on_conflict_do_nothing(),
|
||||
execution_options={"autocommit": True}
|
||||
)
|
||||
card_count += 1
|
||||
|
||||
set_count += 1
|
||||
|
||||
print(f"✓ Imported {set_count} sets and {card_count} cards from sets")
|
||||
return set_count, card_count
|
||||
|
||||
|
||||
async def import_all_identifiers(engine: create_async_engine, file_path: Path) -> int:
|
||||
"""Import AllIdentifiers.json.gz."""
|
||||
if not file_path.exists():
|
||||
print("✗ AllIdentifiers.json.gz not found")
|
||||
return 0
|
||||
|
||||
data = json.loads(gzip.decompress(file_path.read_bytes()))
|
||||
|
||||
count = 0
|
||||
async with engine.begin() as conn:
|
||||
for uuid, card_data in data.items():
|
||||
await conn.execute(
|
||||
pg_insert(card_identifiers_table).values(
|
||||
uuid=uuid,
|
||||
name=card_data.get('name'),
|
||||
mana_cost=card_data.get('manaCost'),
|
||||
type_line=card_data.get('typeLine'),
|
||||
oracle_text=card_data.get('oracleText'),
|
||||
power=card_data.get('power'),
|
||||
toughness=card_data.get('toughness'),
|
||||
rarity=card_data.get('rarity'),
|
||||
layout=card_data.get('layout'),
|
||||
artist=card_data.get('artist'),
|
||||
flavor_text=card_data.get('flavorText'),
|
||||
set_code=card_data.get('setCode'),
|
||||
).on_conflict_do_update(
|
||||
index_elements=['uuid'],
|
||||
set_={
|
||||
'name': card_data.get('name'),
|
||||
'mana_cost': card_data.get('manaCost'),
|
||||
'type_line': card_data.get('typeLine'),
|
||||
'oracle_text': card_data.get('oracleText'),
|
||||
'power': card_data.get('power'),
|
||||
'toughness': card_data.get('toughness'),
|
||||
'rarity': card_data.get('rarity'),
|
||||
'layout': card_data.get('layout'),
|
||||
'artist': card_data.get('artist'),
|
||||
'flavor_text': card_data.get('flavorText'),
|
||||
'set_code': card_data.get('setCode'),
|
||||
'updated_at': datetime.utcnow(),
|
||||
}
|
||||
),
|
||||
execution_options={"autocommit": True}
|
||||
)
|
||||
count += 1
|
||||
|
||||
print(f"✓ Imported {count} identifiers")
|
||||
return count
|
||||
|
||||
|
||||
async def import_all_deck_files(engine: create_async_engine, deck_files_dir: Path) -> int:
|
||||
"""Import AllDeckFiles.zip."""
|
||||
if not deck_files_dir.exists():
|
||||
print("✗ AllDeckFiles directory not found")
|
||||
return 0
|
||||
|
||||
count = 0
|
||||
async with engine.begin() as conn:
|
||||
for deck_file in sorted(deck_files_dir.glob('*.json')):
|
||||
data = json.loads(deck_file.read_text())
|
||||
|
||||
if 'name' in data and 'cards' in data:
|
||||
await conn.execute(
|
||||
pg_insert(decks_table).values(
|
||||
name=data.get('name'),
|
||||
description=data.get('description'),
|
||||
format=data.get('format'),
|
||||
command=data.get('command'),
|
||||
commander=data.get('commander'),
|
||||
).on_conflict_do_update(
|
||||
index_elements=['name'],
|
||||
set_={
|
||||
'description': data.get('description'),
|
||||
'format': data.get('format'),
|
||||
'command': data.get('command'),
|
||||
'commander': data.get('commander'),
|
||||
'updated_at': datetime.utcnow(),
|
||||
}
|
||||
),
|
||||
execution_options={"autocommit": True}
|
||||
)
|
||||
count += 1
|
||||
|
||||
print(f"✓ Imported {count} decks")
|
||||
return count
|
||||
|
||||
|
||||
async def import_json_files(engine: create_async_engine, file_path: Path,
|
||||
table_name: str, name_field: str, id_field: str) -> int:
|
||||
"""Import a JSON.gz file into a table."""
|
||||
if not file_path.exists():
|
||||
return 0
|
||||
|
||||
data = json.loads(gzip.decompress(file_path.read_bytes()))
|
||||
|
||||
count = 0
|
||||
async with engine.begin() as conn:
|
||||
for item in data:
|
||||
values = {k: v for k, v in item.items() if k != id_field}
|
||||
await conn.execute(
|
||||
pg_insert(text(f'{table_name}_table')).values(values),
|
||||
execution_options={"autocommit": True}
|
||||
)
|
||||
count += 1
|
||||
|
||||
print(f"✓ Imported {count} items into {table_name}")
|
||||
return count
|
||||
|
||||
|
||||
def download_file(url: str, destination: Path) -> bool:
|
||||
"""Download a file from URL to destination."""
|
||||
try:
|
||||
print(f"Downloading {url}...")
|
||||
urlretrieve(url, destination)
|
||||
size_mb = destination.stat().st_size / (1024 * 1024)
|
||||
print(f" ✓ Downloaded to {destination} ({size_mb:.1f} MB)")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f" ✗ Failed to download {url}: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def extract_zip(zip_file: Path, extract_dir: Path) -> None:
|
||||
"""Extract a zip file."""
|
||||
if not zip_file.exists():
|
||||
print(f"✗ {zip_file.name} not found")
|
||||
return
|
||||
|
||||
if extract_dir.exists():
|
||||
import shutil
|
||||
shutil.rmtree(extract_dir)
|
||||
|
||||
print(f"Extracting {zip_file.name}...")
|
||||
with zipfile.ZipFile(zip_file, 'r') as zip_ref:
|
||||
zip_ref.extractall(extract_dir)
|
||||
|
||||
file_count = len(list(extract_dir.glob('*.json')))
|
||||
print(f"✓ Extracted to {extract_dir} ({file_count} files)")
|
||||
|
||||
|
||||
async def main():
|
||||
"""Main function to download and import all MTGJSON data."""
|
||||
settings = get_settings()
|
||||
|
||||
# Setup data directory
|
||||
data_dir = Path(settings.DATA_DIR) / "mtgjson"
|
||||
downloads_dir = data_dir / "downloads"
|
||||
downloads_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Create engine
|
||||
engine = create_async_engine(settings.MTG_DATABASE_URL)
|
||||
|
||||
# Create tables
|
||||
print("=== Creating Database Tables ===")
|
||||
await create_tables(engine)
|
||||
|
||||
# Download AllPrintings.psql.gz
|
||||
print("\n=== Downloading AllPrintings ===")
|
||||
psql_file = downloads_dir / "AllPrintings.psql.gz"
|
||||
if not psql_file.exists():
|
||||
download_file(f"{MTGJSON_API_V5}/AllPrintings.psql.gz", psql_file)
|
||||
|
||||
# Import AllPrintings
|
||||
await import_all_printings_psql(engine, psql_file)
|
||||
|
||||
# Download and extract AllSetFiles
|
||||
print("\n=== Importing AllSetFiles ===")
|
||||
set_files_zip = downloads_dir / "AllSetFiles.zip"
|
||||
set_files_dir = downloads_dir / "AllSetFiles"
|
||||
|
||||
if not set_files_dir.exists():
|
||||
if not set_files_zip.exists():
|
||||
download_file(f"{MTGJSON_API_V5}/AllSetFiles.zip", set_files_zip)
|
||||
extract_zip(set_files_zip, set_files_dir)
|
||||
|
||||
await import_all_set_files(engine, set_files_dir)
|
||||
|
||||
# Download and extract AllDeckFiles
|
||||
print("\n=== Importing AllDeckFiles ===")
|
||||
deck_files_zip = downloads_dir / "AllDeckFiles.zip"
|
||||
deck_files_dir = downloads_dir / "AllDeckFiles"
|
||||
|
||||
if not deck_files_dir.exists():
|
||||
if not deck_files_zip.exists():
|
||||
download_file(f"{MTGJSON_API_V5}/AllDeckFiles.zip", deck_files_zip)
|
||||
extract_zip(deck_files_zip, deck_files_dir)
|
||||
|
||||
await import_all_deck_files(engine, deck_files_dir)
|
||||
|
||||
# Download and import AllIdentifiers
|
||||
print("\n=== Importing AllIdentifiers ===")
|
||||
identifiers_file = downloads_dir / "AllIdentifiers.json.gz"
|
||||
if not identifiers_file.exists():
|
||||
download_file(f"{MTGJSON_API_V5}/AllIdentifiers.json.gz", identifiers_file)
|
||||
|
||||
await import_all_identifiers(engine, identifiers_file)
|
||||
|
||||
# Download and import CardTypes
|
||||
print("\n=== Importing CardTypes ===")
|
||||
card_types_file = downloads_dir / "CardTypes.json.gz"
|
||||
if not card_types_file.exists():
|
||||
download_file(f"{MTGJSON_API_V5}/CardTypes.json.gz", card_types_file)
|
||||
|
||||
# Import CardTypes (simplified)
|
||||
if card_types_file.exists():
|
||||
data = json.loads(gzip.decompress(card_types_file.read_bytes()))
|
||||
async with engine.begin() as conn:
|
||||
count = 0
|
||||
for card_type in data:
|
||||
await conn.execute(
|
||||
pg_insert(card_types_table).values(
|
||||
type=card_type.get('type'),
|
||||
description=card_type.get('description'),
|
||||
).on_conflict_do_nothing(),
|
||||
execution_options={"autocommit": True}
|
||||
)
|
||||
count += 1
|
||||
print(f"✓ Imported {count} card types")
|
||||
|
||||
# Download and import DeckList
|
||||
print("\n=== Importing DeckList ===")
|
||||
deck_list_file = downloads_dir / "DeckList.json.gz"
|
||||
if not deck_list_file.exists():
|
||||
download_file(f"{MTGJSON_API_V5}/DeckList.json.gz", deck_list_file)
|
||||
|
||||
# Import DeckList (simplified)
|
||||
if deck_list_file.exists():
|
||||
data = json.loads(gzip.decompress(deck_list_file.read_bytes()))
|
||||
async with engine.begin() as conn:
|
||||
count = 0
|
||||
for deck in data:
|
||||
await conn.execute(
|
||||
pg_insert(deck_list_table).values(
|
||||
deck_id=deck.get('id'),
|
||||
name=deck.get('name'),
|
||||
description=deck.get('description'),
|
||||
format=deck.get('format'),
|
||||
command=deck.get('command'),
|
||||
commander=deck.get('commander'),
|
||||
total_cards=deck.get('totalCards'),
|
||||
).on_conflict_do_nothing(),
|
||||
execution_options={"autocommit": True}
|
||||
)
|
||||
count += 1
|
||||
print(f"✓ Imported {count} deck list entries")
|
||||
|
||||
# Download and import Keywords
|
||||
print("\n=== Importing Keywords ===")
|
||||
keywords_file = downloads_dir / "Keywords.json.gz"
|
||||
if not keywords_file.exists():
|
||||
download_file(f"{MTGJSON_API_V5}/Keywords.json.gz", keywords_file)
|
||||
|
||||
# Import Keywords (simplified)
|
||||
if keywords_file.exists():
|
||||
data = json.loads(gzip.decompress(keywords_file.read_bytes()))
|
||||
async with engine.begin() as conn:
|
||||
count = 0
|
||||
for keyword in data:
|
||||
await conn.execute(
|
||||
pg_insert(card_keywords_table).values(
|
||||
keyword=keyword.get('keyword'),
|
||||
description=keyword.get('description'),
|
||||
).on_conflict_do_nothing(),
|
||||
execution_options={"autocommit": True}
|
||||
)
|
||||
count += 1
|
||||
print(f"✓ Imported {count} keywords")
|
||||
|
||||
# Download and import SetList
|
||||
print("\n=== Importing SetList ===")
|
||||
set_list_file = downloads_dir / "SetList.json.gz"
|
||||
if not set_list_file.exists():
|
||||
download_file(f"{MTGJSON_API_V5}/SetList.json.gz", set_list_file)
|
||||
|
||||
# Import SetList (simplified)
|
||||
if set_list_file.exists():
|
||||
data = json.loads(gzip.decompress(set_list_file.read_bytes()))
|
||||
async with engine.begin() as conn:
|
||||
count = 0
|
||||
for set_data in data:
|
||||
await conn.execute(
|
||||
pg_insert(set_list_table).values(
|
||||
set_code=set_data.get('code'),
|
||||
set_name=set_data.get('name'),
|
||||
set_type=set_data.get('type'),
|
||||
release_date=set_data.get('releaseDate'),
|
||||
base_set_size=set_data.get('baseSetSize'),
|
||||
total_size=set_data.get('totalSetSize'),
|
||||
).on_conflict_do_nothing(),
|
||||
execution_options={"autocommit": True}
|
||||
)
|
||||
count += 1
|
||||
print(f"✓ Imported {count} set list entries")
|
||||
|
||||
# Create indexes
|
||||
print("\n=== Creating Indexes ===")
|
||||
async with engine.begin() as conn:
|
||||
indexes = [
|
||||
"CREATE INDEX IF NOT EXISTS idx_cards_name ON cards(name)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_cards_mana_cost ON cards(manaCost)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_cards_type ON cards(type)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_cards_rarity ON cards(rarity)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_cards_set_code ON cards(setCode)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_cards_uuid ON cards(uuid)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_mtg_sets_code ON mtg_sets(code)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_mtg_cards_name ON mtg_cards(name)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_card_identifiers_uuid ON card_identifiers(uuid)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_card_identifiers_name ON card_identifiers(name)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_deck_list_deck_id ON deck_list(deck_id)",
|
||||
]
|
||||
|
||||
for idx in indexes:
|
||||
await conn.execute(text(idx))
|
||||
|
||||
await conn.commit()
|
||||
print("✓ Indexes created")
|
||||
|
||||
# Show summary
|
||||
print("\n=== Import Summary ===")
|
||||
async with engine.connect() as conn:
|
||||
tables = [
|
||||
'cards', 'mtg_sets', 'mtg_cards', 'card_identifiers',
|
||||
'decks', 'card_types', 'deck_list', 'card_keywords', 'set_list'
|
||||
]
|
||||
|
||||
for table in tables:
|
||||
result = await conn.execute(text(f"SELECT COUNT(*) FROM {table}"))
|
||||
count = result.scalar()
|
||||
print(f" {table:20} {count:>10,} records")
|
||||
|
||||
await engine.dispose()
|
||||
print("\n✓ All MTGJSON data imported successfully!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user