admin/mtgonline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix MTGJSON download: use .json files, remove gzip unpacking

Browse Source 
- Changed REQUIRED_FILES to REQUIRED_JSON_FILES and REQUIRED_ZIP_FILES
- AllSetFiles.zip kept as zip (only available compressed on MTGJSON)
- All other files downloaded as .json (no compression)
- Removed gzip import and unpacking logic for .gz files
- Updated EXPECTED_MIN_SIZES to reflect actual .json file sizes
- Removed validation for non-existent files
This commit is contained in:
akadmin
2026-07-20 13:59:45 +00:00
parent 2369c2cc8c
commit 3fae593a22
3 changed files with 127 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/mtgjson_manager.py
+40 -77
View File
@@ -5,8 +5,8 @@ Handles downloading, unpacking, and upserting MTGJSON data into PostgreSQL.
Manages the complete data lifecycle from download to database upsert.
Features:
- Downloads required MTGJSON datasets from the API
- Unpacks gzip and zip files
- Downloads required MTGJSON datasets from the API (JSON files, except AllSetFiles as zip)
- Unpacks zip files (AllSetFiles.zip only)
- Converts JSON to PostgreSQL-compatible format
- Upserts data without overwriting existing entries
- Tracks refresh timestamps and status
@@ -17,11 +17,9 @@ Usage:
"""
import asyncio
import gzip
import json
import logging
import os
import re
import time
import zipfile
from datetime import datetime, timedelta
@@ -38,35 +36,33 @@ from app.core.settings import get_settings
logger = logging.getLogger(__name__)
# MTGJSON API URLs
# MTGJSON API URLs - .json files for most datasets, .zip for AllSetFiles
MTGJSON_BASE_URL = "https://mtgjson.com/api/v5"
REQUIRED_FILES = {
"AllPrintings.json.gz": MTGJSON_BASE_URL + "/AllPrintings.json.gz",
REQUIRED_JSON_FILES = {
"AllPrintings.json": MTGJSON_BASE_URL + "/AllPrintings.json",
"AllIdentifiers.json": MTGJSON_BASE_URL + "/AllIdentifiers.json",
"CardTypes.json": MTGJSON_BASE_URL + "/CardTypes.json",
"DeckList.json": MTGJSON_BASE_URL + "/DeckList.json",
"Keywords.json": MTGJSON_BASE_URL + "/Keywords.json",
"SetList.json": MTGJSON_BASE_URL + "/SetList.json",
}
REQUIRED_ZIP_FILES = {
"AllSetFiles.zip": MTGJSON_BASE_URL + "/AllSetFiles.zip",
"AllIdentifiers.json.gz": MTGJSON_BASE_URL + "/AllIdentifiers.json.gz",
"CardTypes.json.gz": MTGJSON_BASE_URL + "/CardTypes.json.gz",
"DeckList.json.gz": MTGJSON_BASE_URL + "/DeckList.json.gz",
"Keywords.json.gz": MTGJSON_BASE_URL + "/Keywords.json.gz",
"SetList.json.gz": MTGJSON_BASE_URL + "/SetList.json.gz",
}
DATA_DIR = Path("/app/data/mtgjson")
REFRESH_LOG_TABLE = "mtg_refresh_log"
# Expected minimum file sizes (in bytes) for MTGJSON v5 files
# Expected minimum file sizes (in bytes) for MTGJSON v5 JSON files
EXPECTED_MIN_SIZES = {
"AllPrintings.json.gz": 500 * 1024 * 1024, # 500 MB
"AllPrintings.json": 500 * 1024 * 1024, # 500 MB
"AllSetFiles.json": 10 * 1024 * 1024, # 10 MB
"AllIdentifiers.json.gz": 100 * 1024 * 1024, # 100 MB
"AllIdentifiers.json": 100 * 1024 * 1024, # 100 MB
"CardTypes.json.gz": 1 * 1024 * 1024, # 1 MB
"CardTypes.json": 1 * 1024 * 1024, # 1 MB
"Keywords.json.gz": 0.5 * 1024 * 1024, # 0.5 MB
"Keywords.json": 0.5 * 1024 * 1024, # 0.5 MB
"MagicSets.json": 50 * 1024 * 1024, # 50 MB
"MagicRoots.json": 1 * 1024 * 1024, # 1 MB
"SetTranslations.json": 5 * 1024 * 1024, # 5 MB
"AllPrintings.json": 500 * 1024 * 1024, # 500 MB (actual: ~620 MB)
"AllIdentifiers.json": 500 * 1024 * 1024, # 500 MB (actual: ~599 MB)
"CardTypes.json": 0.001 * 1024 * 1024, # 0.001 MB (actual: ~0.01 MB)
"DeckList.json": 0.5 * 1024 * 1024, # 0.5 MB (actual: ~0.59 MB)
"Keywords.json": 0.001 * 1024 * 1024, # 0.001 MB (actual: ~0.00 MB)
"SetList.json": 5 * 1024 * 1024, # 5 MB (actual: ~11 MB)
"AllSetFiles.zip": 50 * 1024 * 1024, # 50 MB (expected size)
}
MAX_DOWNLOAD_RETRIES = 3
@@ -130,13 +126,13 @@ class MTGJSONManager:
def _get_estimated_size(self, filename: str) -> int:
"""Get estimated file size in MB."""
estimates = {
"AllPrintings.json.gz": 500,
"AllPrintings.json": 620,
"AllSetFiles.zip": 10,
"AllIdentifiers.json.gz": 100,
"CardTypes.json.gz": 5,
"DeckList.json.gz": 5,
"Keywords.json.gz": 2,
"SetList.json.gz": 10,
"AllIdentifiers.json": 600,
"CardTypes.json": 0.01,
"DeckList.json": 0.6,
"Keywords.json": 0.01,
"SetList.json": 11,
}
return estimates.get(filename, 10) # Default 10MB
@@ -178,7 +174,7 @@ class MTGJSONManager:
continue
return False
# Unpack files
# Unpack zip files (AllSetFiles.zip only)
unpack_success = await self.unpack_files()
if not unpack_success:
@@ -210,12 +206,14 @@ class MTGJSONManager:
return False
async def download_files(self) -> bool:
"""Download all required MTGJSON files."""
"""Download all required MTGJSON files (JSON and ZIP)."""
logger.info(f"Starting MTGJSON data download to {self.data_dir}")
async with aiohttp.ClientSession() as session:
tasks = []
for filename, url in REQUIRED_FILES.items():
for filename, url in REQUIRED_JSON_FILES.items():
tasks.append(self._download_file(session, url, filename))
for filename, url in REQUIRED_ZIP_FILES.items():
tasks.append(self._download_file(session, url, filename))
results = await asyncio.gather(*tasks, return_exceptions=True)
@@ -288,56 +286,21 @@ class MTGJSONManager:
return False
async def unpack_files(self) -> bool:
"""Unpack all downloaded files."""
logger.info("Unpacking MTGJSON files")
"""Unpack all downloaded zip files."""
logger.info("Unpacking MTGJSON zip files")
unpacked_files = []
# Unpack gzip files
for gz_file in self.data_dir.glob("*.gz"):
if self._unpack_gzip(gz_file):
unpacked_files.append(gz_file.with_suffix(""))
# Unpack zip files
# Unpack zip files only
for zip_file in self.data_dir.glob("*.zip"):
if self._unpack_zip(zip_file):
unpacked_files.append(zip_file.with_suffix(""))
unpacked_files.append(zip_file)
if len(unpacked_files) > 0:
logger.info(f"Unpacked {len(unpacked_files)} files")
logger.info(f"Unpacked {len(unpacked_files)} zip files")
return True
else:
logger.warning("No files were unpacked")
return False
def _unpack_gzip(self, gz_file: Path) -> bool:
"""Unpack a gzip file or handle pre-uncompressed JSON."""
dest_file = gz_file.with_suffix("")
logger.info(f"Unpacking {gz_file.name}")
try:
# Check if file is actually gzipped by reading first bytes
with open(gz_file, 'rb') as f:
magic = f.read(2)
if magic == b'\x1f\x8b':
# File is gzipped - proceed normally
with gzip.open(gz_file, 'rt', encoding='utf-8') as f_in:
content = f_in.read()
dest_file.write_text(content, encoding='utf-8')
logger.info(f"Unpacked {gz_file.name} to {dest_file.name}")
return True
else:
# File is already plain JSON - just rename
logger.info(f"{gz_file.name} is pre-uncompressed JSON, renaming to {dest_file.name}")
dest_file.write_bytes(gz_file.read_bytes())
gz_file.unlink()
return True
except Exception as e:
logger.error(f"Failed to unpack {gz_file.name}: {e}")
logger.warning("No zip files were unpacked")
return False
def _unpack_zip(self, zip_file: Path) -> bool:
@@ -375,7 +338,7 @@ class MTGJSONManager:
}
try:
# Process AllSetFiles directory
# Process AllSetFiles directory (from zip)
await self._upsert_sets()
counts["sets"] = await self._count_sets()
@@ -827,4 +790,4 @@ if __name__ == "__main__":
else:
print("Usage: python -m app.services.mtgjson_manager [--refresh | --health]")
asyncio.run(main())
asyncio.run(main())