158 lines
4.3 KiB
Python
158 lines
4.3 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
MTGJSON v5 Data Downloader
|
|
|
|
Downloads MTGJSON v5 data files and loads them into the database.
|
|
Updated to use current MTGJSON API endpoints.
|
|
|
|
Usage:
|
|
python download_mtgjson_v5.py
|
|
"""
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
# Add the app directory to Python path
|
|
app_dir = Path(__file__).parent.parent
|
|
sys.path.insert(0, str(app_dir))
|
|
|
|
import asyncio
|
|
import json
|
|
import gzip
|
|
import logging
|
|
from sqlalchemy import create_engine, text
|
|
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
|
import aiohttp
|
|
|
|
from app.services.mtgjson_manager import MTGJSONManager
|
|
from app.config import get_settings
|
|
|
|
logging.basicConfig(level=logging.INFO)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# MTGJSON v5 API base URL
|
|
MTGJSON_API_BASE = "https://mtgjson.com/api/v5"
|
|
|
|
|
|
async def download_file(session, url, dest_path):
|
|
"""Download a file from URL."""
|
|
logger.info(f"Downloading: {url}")
|
|
|
|
try:
|
|
async with session.get(url, timeout=aiohttp.ClientTimeout(total=600)) as response:
|
|
if response.status == 200:
|
|
dest_path.parent.mkdir(parents=True, exist_ok=True)
|
|
with open(dest_path, 'wb') as f:
|
|
async for chunk in response.content.iter_chunked(8192):
|
|
f.write(chunk)
|
|
logger.info(f"✓ Downloaded: {dest_path.name}")
|
|
return True
|
|
else:
|
|
logger.error(f"✗ Failed to download {url}: HTTP {response.status}")
|
|
return False
|
|
except Exception as e:
|
|
logger.error(f"✗ Error downloading {url}: {e}")
|
|
return False
|
|
|
|
|
|
def extract_gzip(gz_path, dest_path=None):
|
|
"""Extract a gzip file."""
|
|
if not dest_path:
|
|
dest_path = gz_path.with_suffix('')
|
|
|
|
logger.info(f"Extracting: {gz_path.name}")
|
|
|
|
try:
|
|
with gzip.open(gz_path, 'rt', encoding='utf-8') as f_in:
|
|
with open(dest_path, 'w', encoding='utf-8') as f_out:
|
|
f_out.write(f_in.read())
|
|
logger.info(f"✓ Extracted: {gz_path.name}")
|
|
# Remove gz file after extraction
|
|
gz_path.unlink()
|
|
return True
|
|
except Exception as e:
|
|
logger.error(f"✗ Failed to extract {gz_path}: {e}")
|
|
return False
|
|
|
|
|
|
async def download_all_files(data_dir):
|
|
"""Download all MTGJSON v5 files."""
|
|
logger.info("=" * 60)
|
|
logger.info("MTGJSON v5 Data Download")
|
|
logger.info("=" * 60)
|
|
|
|
files = [
|
|
"AllPrintings.json.gz",
|
|
"AllSetFiles.json.gz",
|
|
"AllIdentifiers.json.gz",
|
|
"CardTypes.json.gz",
|
|
"Keywords.json.gz",
|
|
"MagicRoots.json.gz",
|
|
"MagicSets.json.gz",
|
|
"SetTranslations.json.gz"
|
|
]
|
|
|
|
async with aiohttp.ClientSession() as session:
|
|
for filename in files:
|
|
url = f"{MTGJSON_API_BASE}/{filename}"
|
|
dest_path = data_dir / filename
|
|
await download_file(session, url, dest_path)
|
|
await asyncio.sleep(1) # Be nice to the API
|
|
|
|
|
|
async def load_data_to_database(data_dir):
|
|
"""Load downloaded MTGJSON data into PostgreSQL."""
|
|
logger.info("=" * 60)
|
|
logger.info("Loading MTGJSON data into database")
|
|
logger.info("=" * 60)
|
|
|
|
manager = MTGJSONManager(data_dir)
|
|
|
|
# Download files
|
|
success = await manager.download_files()
|
|
|
|
if not success:
|
|
logger.error("✗ Download failed")
|
|
return False
|
|
|
|
# Unpack files
|
|
success = manager.unpack_files()
|
|
|
|
if not success:
|
|
logger.error("✗ Unpack failed")
|
|
return False
|
|
|
|
# Upsert to database
|
|
success = manager.upsert_to_database()
|
|
|
|
if success:
|
|
logger.info("✓ Data loaded successfully")
|
|
else:
|
|
logger.error("✗ Database upsert failed")
|
|
|
|
return success
|
|
|
|
|
|
async def main():
|
|
"""Main entry point."""
|
|
settings = get_settings()
|
|
data_dir = Path(settings.DATA_DIR)
|
|
|
|
logger.info(f"Data directory: {data_dir}")
|
|
|
|
# Clear corrupted data
|
|
if data_dir.exists():
|
|
logger.info("Clearing corrupted data...")
|
|
for f in data_dir.glob("*.json.gz"):
|
|
f.unlink()
|
|
logger.info(f" Removed: {f.name}")
|
|
|
|
data_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Download and load data
|
|
await load_data_to_database(data_dir)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|