admin/mtgonline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Final project commit: MTGJSON data integration and backend API

Browse Source
This commit is contained in:
wall-o
2026-07-20 01:09:45 +00:00
parent b170dfd577
commit db01e29a54
37 changed files with 9674 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/scripts/refresh_mtg.py
+87 -44
View File
@@ -44,30 +44,72 @@ async def download_mtgjson(session: aiohttp.ClientSession, output_path: Path) ->
return False
async def parse_mtgjson(filepath: Path) -> dict:
"""Parse the AllPrintings JSON file."""
async def parse_mtgjson(filepath: Path) -> tuple[dict, dict]:
"""Parse the AllPrintings JSON file.
MTGJSON v5 AllPrintings structure:
{
"meta": {...},
"data": {
"10E": {"baseSetSize": 383, "block": "Core Set", "cards": [...]},
"UNH": {...}
}
}
Returns:
(sets_dict, cards_dict) where sets_dict maps set_code -> set_data
and cards_dict maps set_code -> list of card dicts
"""
try:
with open(filepath, 'r', encoding='utf-8') as f:
data = json.load(f)
# Verify structure
if 'data' not in data or 'sets' not in data:
raise ValueError("Invalid MTGJSON structure")
# Verify structure - data key is required
if 'data' not in data:
raise ValueError("Invalid MTGJSON structure: missing 'data' key")
mtg_data = data['data']
# MTGJSON v5: data contains set codes directly as keys
# Each set code maps to {baseSetSize, block, cards: [...]}
sets_dict = {}
cards_dict = {}
for set_code, set_data in mtg_data.items():
# Skip if it looks like metadata, not a set
if isinstance(set_data, dict) and 'baseSetSize' in set_data:
# Convert release_date string to datetime object
release_date_str = set_data.get('releaseDate')
if release_date_str:
try:
set_data['releaseDate'] = datetime.fromisoformat(release_date_str.replace('Z', '+00:00'))
except (ValueError, AttributeError):
pass # Keep as string if parsing fails
sets_dict[set_code] = set_data
# Extract cards for this set
if 'cards' in set_data and isinstance(set_data['cards'], list):
cards_dict[set_code] = set_data['cards']
if not sets_dict:
raise ValueError("No sets found in MTGJSON data")
logger.info(f"Parsed {len(sets_dict)} sets, {sum(len(c) for c in cards_dict.values())} cards")
return sets_dict, cards_dict
return data['data']
except Exception as e:
logger.error(f"Parse error: {e}")
return {}
return {}, {}
async def update_database(session: AsyncSession, data: dict) -> tuple[int, int]:
async def update_database(session: AsyncSession, sets_dict: dict, cards_dict: dict) -> tuple[int, int]:
"""Update the database with parsed MTGJSON data."""
cards_updated = 0
sets_updated = 0
try:
# Process sets
for set_code, set_data in data.get('sets', {}).items():
for set_code, set_data in sets_dict.items():
stmt = text("""
INSERT INTO mtg_sets (code, name, type, release_date, base_set_size,
total_size, is_foil_only, is_non_foil_only,
@@ -96,37 +138,38 @@ async def update_database(session: AsyncSession, data: dict) -> tuple[int, int]:
})
sets_updated += 1
# Process cards
for card_data in data.get('cards', []):
stmt = text("""
INSERT INTO mtg_cards (set_id, name, mana_cost, type_line, oracle_text,
power, toughness, rarity, layout, artist,
flavor_text, numbers, identifiers, images)
SELECT s.id, :name, :mana_cost, :type_line, :oracle_text,
:power, :toughness, :rarity, :layout, :artist,
:flavor_text, :numbers, :identifiers, :images
FROM mtg_sets s
WHERE s.code = :set_code
ON CONFLICT DO NOTHING
""")
await session.execute(stmt, {
'set_code': card_data.get('set'),
'name': card_data.get('name'),
'mana_cost': card_data.get('manaCost'),
'type_line': card_data.get('type'),
'oracle_text': card_data.get('text'),
'power': card_data.get('power'),
'toughness': card_data.get('toughness'),
'rarity': card_data.get('rarity'),
'layout': card_data.get('layout'),
'artist': card_data.get('artist'),
'flavor_text': card_data.get('flavorText'),
'numbers': str(card_data.get('numbers', '')),
'identifiers': json.dumps(card_data.get('identifiers', {})),
'images': json.dumps(card_data.get('images', {})),
})
cards_updated += 1
# Process cards grouped by set
for set_code, cards in cards_dict.items():
for card_data in cards:
stmt = text("""
INSERT INTO mtg_cards (set_id, name, mana_cost, type_line, oracle_text,
power, toughness, rarity, layout, artist,
flavor_text, numbers, identifiers, images)
SELECT s.id, :name, :mana_cost, :type_line, :oracle_text,
:power, :toughness, :rarity, :layout, :artist,
:flavor_text, :numbers, :identifiers, :images
FROM mtg_sets s
WHERE s.code = :set_code
ON CONFLICT DO NOTHING
""")
await session.execute(stmt, {
'set_code': set_code,
'name': card_data.get('name'),
'mana_cost': card_data.get('manaCost'),
'type_line': card_data.get('type'),
'oracle_text': card_data.get('text'),
'power': card_data.get('power'),
'toughness': card_data.get('toughness'),
'rarity': card_data.get('rarity'),
'layout': card_data.get('layout'),
'artist': card_data.get('artist'),
'flavor_text': card_data.get('flavorText'),
'numbers': str(card_data.get('numbers', '')),
'identifiers': json.dumps(card_data.get('identifiers', {})),
'images': json.dumps(card_data.get('images', {})),
})
cards_updated += 1
await session.commit()
return cards_updated, sets_updated
@@ -199,15 +242,15 @@ async def main():
await log_refresh(engine, "FAILED", 0, 0, 0, "Download failed")
return
# Parse data
data = await parse_mtgjson(download_path)
if not data:
# Parse data (returns sets_dict, cards_dict)
sets_dict, cards_dict = await parse_mtgjson(download_path)
if not sets_dict:
await log_refresh(engine, "FAILED", 0, 0, 0, "Parse failed")
return
# Update database
async with AsyncSession(engine) as db_session:
cards_updated, sets_updated = await update_database(db_session, data)
cards_updated, sets_updated = await update_database(db_session, sets_dict, cards_dict)
# Log success
duration = int(time.time() - start_time)