wall-o
b170dfd577
- Added MTGJSON data downloader (downloads all MTGJSON API v5 files)
- Added MTGJSON data loader (imports data into PostgreSQL)
- Added MTGJSON data uploader (alternative upsert logic)
- Fixed route ordering in card_router.py (/sets before /{card_name})
- Added load_mtgjson_data.py entry point script
MTGJSON data sources:
- AllPrintings.psql.gz (main cards database)
- AllSetFiles.zip (set and card data)
- AllDeckFiles.zip (deck data)
- AllIdentifiers.json.gz (card identifiers)
- CardTypes.json.gz (card types)
- DeckList.json.gz (deck list metadata)
- Keywords.json.gz (card keywords)
- SetList.json.gz (set list metadata)
Note: MTGJSON set.json does NOT contain image URLs. Only cards have image_uris.
Sets have iconSvgUrl (SVG icons) but no raster image URLs.
138 lines
3.8 KiB
Python
138 lines
3.8 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
MTGJSON Data Downloader
|
|
|
|
Downloads all MTGJSON data files and stores them for import.
|
|
"""
|
|
|
|
import gzip
|
|
import json
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
from urllib.request import urlretrieve
|
|
from urllib.parse import urljoin
|
|
|
|
# MTGJSON API v5 base URL
|
|
MTGJSON_API_V5 = "https://mtgjson.com/api/v5"
|
|
|
|
# Files to download with their types
|
|
MTGJSON_FILES = {
|
|
"AllPrintings.psql.gz": {
|
|
"name": "AllPrintings",
|
|
"description": "Main cards database (PSQL format)",
|
|
"type": "psql",
|
|
},
|
|
"AllSetFiles.zip": {
|
|
"name": "AllSetFiles",
|
|
"description": "Set and card data",
|
|
"type": "zip",
|
|
},
|
|
"AllDeckFiles.zip": {
|
|
"name": "AllDeckFiles",
|
|
"description": "Deck data",
|
|
"type": "zip",
|
|
},
|
|
"AllIdentifiers.json.gz": {
|
|
"name": "AllIdentifiers",
|
|
"description": "Card identifiers",
|
|
"type": "json",
|
|
},
|
|
"CardTypes.json.gz": {
|
|
"name": "CardTypes",
|
|
"description": "Card types",
|
|
"type": "json",
|
|
},
|
|
"DeckList.json.gz": {
|
|
"name": "DeckList",
|
|
"description": "Deck list metadata",
|
|
"type": "json",
|
|
},
|
|
"Keywords.json.gz": {
|
|
"name": "Keywords",
|
|
"description": "Card keywords",
|
|
"type": "json",
|
|
},
|
|
"SetList.json.gz": {
|
|
"name": "SetList",
|
|
"description": "Set list metadata",
|
|
"type": "json",
|
|
},
|
|
}
|
|
|
|
|
|
def get_downloads_dir() -> Path:
|
|
"""Get the downloads directory path."""
|
|
data_dir = Path(os.environ.get("MTGDATA_DIR", "/app/data"))
|
|
downloads_dir = data_dir / "mtgjson" / "downloads"
|
|
downloads_dir.mkdir(parents=True, exist_ok=True)
|
|
return downloads_dir
|
|
|
|
|
|
def download_file(url: str, destination: Path) -> bool:
|
|
"""Download a file from URL to destination."""
|
|
try:
|
|
print(f"Downloading {url}...")
|
|
urlretrieve(url, destination)
|
|
size_mb = destination.stat().st_size / (1024 * 1024)
|
|
print(f" ✓ Downloaded to {destination} ({size_mb:.1f} MB)")
|
|
return True
|
|
except Exception as e:
|
|
print(f" ✗ Failed to download {url}: {e}")
|
|
return False
|
|
|
|
|
|
def download_all_files() -> list[Path]:
|
|
"""Download all MTGJSON files."""
|
|
downloads_dir = get_downloads_dir()
|
|
downloaded_files = []
|
|
|
|
print("=== MTGJSON Data Download ===\n")
|
|
|
|
for filename, file_info in MTGJSON_FILES.items():
|
|
url = urljoin(MTGJSON_API_V5, filename)
|
|
destination = downloads_dir / filename
|
|
|
|
if download_file(url, destination):
|
|
downloaded_files.append(destination)
|
|
else:
|
|
print(f" ⚠ Continuing with downloaded files only")
|
|
|
|
print(f"\n=== Download Complete ===")
|
|
print(f"Downloaded {len(downloaded_files)} files to {downloads_dir}")
|
|
return downloaded_files
|
|
|
|
|
|
def verify_downloads(downloaded_files: list[Path]) -> bool:
|
|
"""Verify all expected files are downloaded."""
|
|
print("\n=== Verifying Downloads ===\n")
|
|
|
|
all_ok = True
|
|
for filename, file_info in MTGJSON_FILES.items():
|
|
filepath = Path(get_downloads_dir() / filename)
|
|
if filepath.exists():
|
|
size_mb = filepath.stat().st_size / (1024 * 1024)
|
|
print(f"✓ {filename:30} ({size_mb:.1f} MB)")
|
|
else:
|
|
print(f"✗ {filename:30} (MISSING)")
|
|
all_ok = False
|
|
|
|
if all_ok:
|
|
print("\n✓ All files downloaded successfully")
|
|
else:
|
|
print("\n⚠ Some files are missing")
|
|
|
|
return all_ok
|
|
|
|
|
|
def get_file_list() -> list[Path]:
|
|
"""Get list of all downloaded files."""
|
|
downloads_dir = get_downloads_dir()
|
|
files = [downloads_dir / filename for filename in MTGJSON_FILES.keys()]
|
|
return [f for f in files if f.exists()]
|
|
|
|
|
|
if __name__ == "__main__":
|
|
downloaded = download_all_files()
|
|
verify_downloads(downloaded)
|