Fix MTGJSON download: use .json files, remove gzip unpacking
- Changed REQUIRED_FILES to REQUIRED_JSON_FILES and REQUIRED_ZIP_FILES - AllSetFiles.zip kept as zip (only available compressed on MTGJSON) - All other files downloaded as .json (no compression) - Removed gzip import and unpacking logic for .gz files - Updated EXPECTED_MIN_SIZES to reflect actual .json file sizes - Removed validation for non-existent files
This commit is contained in:
@@ -5,8 +5,8 @@ Handles downloading, unpacking, and upserting MTGJSON data into PostgreSQL.
|
||||
Manages the complete data lifecycle from download to database upsert.
|
||||
|
||||
Features:
|
||||
- Downloads required MTGJSON datasets from the API
|
||||
- Unpacks gzip and zip files
|
||||
- Downloads required MTGJSON datasets from the API (JSON files, except AllSetFiles as zip)
|
||||
- Unpacks zip files (AllSetFiles.zip only)
|
||||
- Converts JSON to PostgreSQL-compatible format
|
||||
- Upserts data without overwriting existing entries
|
||||
- Tracks refresh timestamps and status
|
||||
@@ -17,11 +17,9 @@ Usage:
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import gzip
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import zipfile
|
||||
from datetime import datetime, timedelta
|
||||
@@ -38,35 +36,33 @@ from app.core.settings import get_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# MTGJSON API URLs
|
||||
# MTGJSON API URLs - .json files for most datasets, .zip for AllSetFiles
|
||||
MTGJSON_BASE_URL = "https://mtgjson.com/api/v5"
|
||||
REQUIRED_FILES = {
|
||||
"AllPrintings.json.gz": MTGJSON_BASE_URL + "/AllPrintings.json.gz",
|
||||
REQUIRED_JSON_FILES = {
|
||||
"AllPrintings.json": MTGJSON_BASE_URL + "/AllPrintings.json",
|
||||
"AllIdentifiers.json": MTGJSON_BASE_URL + "/AllIdentifiers.json",
|
||||
"CardTypes.json": MTGJSON_BASE_URL + "/CardTypes.json",
|
||||
"DeckList.json": MTGJSON_BASE_URL + "/DeckList.json",
|
||||
"Keywords.json": MTGJSON_BASE_URL + "/Keywords.json",
|
||||
"SetList.json": MTGJSON_BASE_URL + "/SetList.json",
|
||||
}
|
||||
|
||||
REQUIRED_ZIP_FILES = {
|
||||
"AllSetFiles.zip": MTGJSON_BASE_URL + "/AllSetFiles.zip",
|
||||
"AllIdentifiers.json.gz": MTGJSON_BASE_URL + "/AllIdentifiers.json.gz",
|
||||
"CardTypes.json.gz": MTGJSON_BASE_URL + "/CardTypes.json.gz",
|
||||
"DeckList.json.gz": MTGJSON_BASE_URL + "/DeckList.json.gz",
|
||||
"Keywords.json.gz": MTGJSON_BASE_URL + "/Keywords.json.gz",
|
||||
"SetList.json.gz": MTGJSON_BASE_URL + "/SetList.json.gz",
|
||||
}
|
||||
|
||||
DATA_DIR = Path("/app/data/mtgjson")
|
||||
REFRESH_LOG_TABLE = "mtg_refresh_log"
|
||||
|
||||
# Expected minimum file sizes (in bytes) for MTGJSON v5 files
|
||||
# Expected minimum file sizes (in bytes) for MTGJSON v5 JSON files
|
||||
EXPECTED_MIN_SIZES = {
|
||||
"AllPrintings.json.gz": 500 * 1024 * 1024, # 500 MB
|
||||
"AllPrintings.json": 500 * 1024 * 1024, # 500 MB
|
||||
"AllSetFiles.json": 10 * 1024 * 1024, # 10 MB
|
||||
"AllIdentifiers.json.gz": 100 * 1024 * 1024, # 100 MB
|
||||
"AllIdentifiers.json": 100 * 1024 * 1024, # 100 MB
|
||||
"CardTypes.json.gz": 1 * 1024 * 1024, # 1 MB
|
||||
"CardTypes.json": 1 * 1024 * 1024, # 1 MB
|
||||
"Keywords.json.gz": 0.5 * 1024 * 1024, # 0.5 MB
|
||||
"Keywords.json": 0.5 * 1024 * 1024, # 0.5 MB
|
||||
"MagicSets.json": 50 * 1024 * 1024, # 50 MB
|
||||
"MagicRoots.json": 1 * 1024 * 1024, # 1 MB
|
||||
"SetTranslations.json": 5 * 1024 * 1024, # 5 MB
|
||||
"AllPrintings.json": 500 * 1024 * 1024, # 500 MB (actual: ~620 MB)
|
||||
"AllIdentifiers.json": 500 * 1024 * 1024, # 500 MB (actual: ~599 MB)
|
||||
"CardTypes.json": 0.001 * 1024 * 1024, # 0.001 MB (actual: ~0.01 MB)
|
||||
"DeckList.json": 0.5 * 1024 * 1024, # 0.5 MB (actual: ~0.59 MB)
|
||||
"Keywords.json": 0.001 * 1024 * 1024, # 0.001 MB (actual: ~0.00 MB)
|
||||
"SetList.json": 5 * 1024 * 1024, # 5 MB (actual: ~11 MB)
|
||||
"AllSetFiles.zip": 50 * 1024 * 1024, # 50 MB (expected size)
|
||||
}
|
||||
|
||||
MAX_DOWNLOAD_RETRIES = 3
|
||||
@@ -130,13 +126,13 @@ class MTGJSONManager:
|
||||
def _get_estimated_size(self, filename: str) -> int:
|
||||
"""Get estimated file size in MB."""
|
||||
estimates = {
|
||||
"AllPrintings.json.gz": 500,
|
||||
"AllPrintings.json": 620,
|
||||
"AllSetFiles.zip": 10,
|
||||
"AllIdentifiers.json.gz": 100,
|
||||
"CardTypes.json.gz": 5,
|
||||
"DeckList.json.gz": 5,
|
||||
"Keywords.json.gz": 2,
|
||||
"SetList.json.gz": 10,
|
||||
"AllIdentifiers.json": 600,
|
||||
"CardTypes.json": 0.01,
|
||||
"DeckList.json": 0.6,
|
||||
"Keywords.json": 0.01,
|
||||
"SetList.json": 11,
|
||||
}
|
||||
return estimates.get(filename, 10) # Default 10MB
|
||||
|
||||
@@ -178,7 +174,7 @@ class MTGJSONManager:
|
||||
continue
|
||||
return False
|
||||
|
||||
# Unpack files
|
||||
# Unpack zip files (AllSetFiles.zip only)
|
||||
unpack_success = await self.unpack_files()
|
||||
|
||||
if not unpack_success:
|
||||
@@ -210,12 +206,14 @@ class MTGJSONManager:
|
||||
return False
|
||||
|
||||
async def download_files(self) -> bool:
|
||||
"""Download all required MTGJSON files."""
|
||||
"""Download all required MTGJSON files (JSON and ZIP)."""
|
||||
logger.info(f"Starting MTGJSON data download to {self.data_dir}")
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
tasks = []
|
||||
for filename, url in REQUIRED_FILES.items():
|
||||
for filename, url in REQUIRED_JSON_FILES.items():
|
||||
tasks.append(self._download_file(session, url, filename))
|
||||
for filename, url in REQUIRED_ZIP_FILES.items():
|
||||
tasks.append(self._download_file(session, url, filename))
|
||||
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
@@ -288,56 +286,21 @@ class MTGJSONManager:
|
||||
return False
|
||||
|
||||
async def unpack_files(self) -> bool:
|
||||
"""Unpack all downloaded files."""
|
||||
logger.info("Unpacking MTGJSON files")
|
||||
"""Unpack all downloaded zip files."""
|
||||
logger.info("Unpacking MTGJSON zip files")
|
||||
|
||||
unpacked_files = []
|
||||
|
||||
# Unpack gzip files
|
||||
for gz_file in self.data_dir.glob("*.gz"):
|
||||
if self._unpack_gzip(gz_file):
|
||||
unpacked_files.append(gz_file.with_suffix(""))
|
||||
|
||||
# Unpack zip files
|
||||
# Unpack zip files only
|
||||
for zip_file in self.data_dir.glob("*.zip"):
|
||||
if self._unpack_zip(zip_file):
|
||||
unpacked_files.append(zip_file.with_suffix(""))
|
||||
unpacked_files.append(zip_file)
|
||||
|
||||
if len(unpacked_files) > 0:
|
||||
logger.info(f"Unpacked {len(unpacked_files)} files")
|
||||
logger.info(f"Unpacked {len(unpacked_files)} zip files")
|
||||
return True
|
||||
else:
|
||||
logger.warning("No files were unpacked")
|
||||
return False
|
||||
|
||||
def _unpack_gzip(self, gz_file: Path) -> bool:
|
||||
"""Unpack a gzip file or handle pre-uncompressed JSON."""
|
||||
dest_file = gz_file.with_suffix("")
|
||||
|
||||
logger.info(f"Unpacking {gz_file.name}")
|
||||
|
||||
try:
|
||||
# Check if file is actually gzipped by reading first bytes
|
||||
with open(gz_file, 'rb') as f:
|
||||
magic = f.read(2)
|
||||
|
||||
if magic == b'\x1f\x8b':
|
||||
# File is gzipped - proceed normally
|
||||
with gzip.open(gz_file, 'rt', encoding='utf-8') as f_in:
|
||||
content = f_in.read()
|
||||
|
||||
dest_file.write_text(content, encoding='utf-8')
|
||||
logger.info(f"Unpacked {gz_file.name} to {dest_file.name}")
|
||||
return True
|
||||
else:
|
||||
# File is already plain JSON - just rename
|
||||
logger.info(f"{gz_file.name} is pre-uncompressed JSON, renaming to {dest_file.name}")
|
||||
dest_file.write_bytes(gz_file.read_bytes())
|
||||
gz_file.unlink()
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to unpack {gz_file.name}: {e}")
|
||||
logger.warning("No zip files were unpacked")
|
||||
return False
|
||||
|
||||
def _unpack_zip(self, zip_file: Path) -> bool:
|
||||
@@ -375,7 +338,7 @@ class MTGJSONManager:
|
||||
}
|
||||
|
||||
try:
|
||||
# Process AllSetFiles directory
|
||||
# Process AllSetFiles directory (from zip)
|
||||
await self._upsert_sets()
|
||||
counts["sets"] = await self._count_sets()
|
||||
|
||||
@@ -827,4 +790,4 @@ if __name__ == "__main__":
|
||||
else:
|
||||
print("Usage: python -m app.services.mtgjson_manager [--refresh | --health]")
|
||||
|
||||
asyncio.run(main())
|
||||
asyncio.run(main())
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Test MTGJSON API endpoints and download JSON files."""
|
||||
import asyncio
|
||||
import aiohttp
|
||||
from pathlib import Path
|
||||
|
||||
DATA_DIR = Path("/app/data/mtgjson")
|
||||
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# MTGJSON API endpoints for .json files
|
||||
MTGJSON_BASE_URL = "https://mtgjson.com/api/v5"
|
||||
JSON_FILES = {
|
||||
"AllPrintings.json": MTGJSON_BASE_URL + "/AllPrintings.json",
|
||||
"AllSetFiles.json": MTGJSON_BASE_URL + "/AllSetFiles.json",
|
||||
"AllIdentifiers.json": MTGJSON_BASE_URL + "/AllIdentifiers.json",
|
||||
"CardTypes.json": MTGJSON_BASE_URL + "/CardTypes.json",
|
||||
"DeckList.json": MTGJSON_BASE_URL + "/DeckList.json",
|
||||
"Keywords.json": MTGJSON_BASE_URL + "/Keywords.json",
|
||||
"SetList.json": MTGJSON_BASE_URL + "/SetList.json",
|
||||
}
|
||||
|
||||
async def test_download_file(session, url, filename):
|
||||
"""Test downloading a single JSON file."""
|
||||
dest_path = DATA_DIR / filename
|
||||
print(f"Testing {filename}...")
|
||||
|
||||
try:
|
||||
async with session.get(url) as response:
|
||||
if response.status == 200:
|
||||
content = await response.read()
|
||||
print(f" ✓ Status: {response.status}")
|
||||
print(f" ✓ Size: {len(content) / (1024*1024):.2f} MB")
|
||||
print(f" ✓ Content-Type: {response.content_type}")
|
||||
|
||||
# Write to file
|
||||
with open(dest_path, 'wb') as f:
|
||||
f.write(content)
|
||||
print(f" ✓ Saved to {dest_path}")
|
||||
|
||||
# Test JSON parsing
|
||||
import json
|
||||
with open(dest_path, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
print(f" ✓ Valid JSON: {type(data).__name__}")
|
||||
|
||||
return True
|
||||
else:
|
||||
print(f" ✗ Status: {response.status}")
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f" ✗ Error: {e}")
|
||||
return False
|
||||
|
||||
async def main():
|
||||
"""Test all MTGJSON JSON endpoints."""
|
||||
print("Testing MTGJSON JSON API endpoints...\n")
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
results = []
|
||||
for filename, url in JSON_FILES.items():
|
||||
success = await test_download_file(session, url, filename)
|
||||
results.append((filename, success))
|
||||
print()
|
||||
|
||||
print("\n=== Results ===")
|
||||
for filename, success in results:
|
||||
status = "✓" if success else "✗"
|
||||
print(f"{status} {filename}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
+16
-32
@@ -1,43 +1,27 @@
|
||||
{
|
||||
"task_description": "MTG Online Backend - MTGJSON Data Integration & Docker Deployment",
|
||||
"current_step": "Testing full container lifecycle",
|
||||
"task_description": "Update MTGJSON data manager to download .json files instead of .json.gz files, remove unnecessary unpacking logic",
|
||||
"current_step": "Docker image rebuilt with updated code",
|
||||
"files_created": [
|
||||
"backend/app/services/mtgjson_manager.py",
|
||||
"backend/scripts/sanity_check_mtgjson.py",
|
||||
"backend/scripts/verify_mtgjson_data.py",
|
||||
"backend/scripts/maintain_mtgjson.sh"
|
||||
"backend/app/services/mtgjson_manager.py"
|
||||
],
|
||||
"files_modified": [
|
||||
"backend/app/services/mtgjson_manager.py",
|
||||
"backend/app/scripts/refresh_mtg.py",
|
||||
"backend/app/main.py",
|
||||
"docker-compose.yml",
|
||||
"docker-compose.dev.yml",
|
||||
"backend/Dockerfile"
|
||||
"backend/app/services/mtgjson_manager.py"
|
||||
],
|
||||
"decisions": [
|
||||
"Created comprehensive MTGJSONManager service in app/services/",
|
||||
"Manager handles download, unpack, upsert with ON CONFLICT DO UPDATE",
|
||||
"Startup triggers initial download on first container init",
|
||||
"Health check verifies MTG data exists in database",
|
||||
"Refresh interval set to 7 days (weekly)",
|
||||
"Docker compose start_period set to 600s for download time",
|
||||
"Added file size validation against expected minimums",
|
||||
"Implemented retry logic with exponential backoff (up to 3 attempts)",
|
||||
"Cleanup deletes corrupted data before retry",
|
||||
"Container marked unhealthy if validation fails after all retries",
|
||||
"Download timeout increased to 60 minutes (3600s) for large files"
|
||||
"Changed REQUIRED_FILES to REQUIRED_JSON_FILES and REQUIRED_ZIP_FILES",
|
||||
"AllSetFiles.zip kept as zip (only available compressed on MTGJSON API)",
|
||||
"All other files downloaded as .json (no compression)",
|
||||
"Removed gzip import and unpacking logic for .gz files",
|
||||
"Updated EXPECTED_MIN_SIZES to reflect actual .json file sizes (500MB+ for large files)",
|
||||
"Removed validation for non-existent files (MagicSets.json, MagicRoots.json, SetTranslations.json)"
|
||||
],
|
||||
"next_steps": [
|
||||
"Save state to state.json",
|
||||
"Commit and push to Gitea",
|
||||
"Stop and destroy all Docker containers",
|
||||
"Build backend Docker container",
|
||||
"Deploy stack and monitor",
|
||||
"Check container health status",
|
||||
"Verify logs show successful initialization"
|
||||
"Restart Docker container with new image",
|
||||
"Run data refresh to verify download and upsert work correctly",
|
||||
"Monitor logs for any issues",
|
||||
"Verify database has correct data"
|
||||
],
|
||||
"blockers": [],
|
||||
"commit_hash": "0643544",
|
||||
"timestamp": 1784598400
|
||||
"commit_hash": "",
|
||||
"timestamp": "2026-07-20T13:52:00Z"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user