feat: Add MTGJSON data integration
- Added MTGJSON data downloader (downloads all MTGJSON API v5 files)
- Added MTGJSON data loader (imports data into PostgreSQL)
- Added MTGJSON data uploader (alternative upsert logic)
- Fixed route ordering in card_router.py (/sets before /{card_name})
- Added load_mtgjson_data.py entry point script
MTGJSON data sources:
- AllPrintings.psql.gz (main cards database)
- AllSetFiles.zip (set and card data)
- AllDeckFiles.zip (deck data)
- AllIdentifiers.json.gz (card identifiers)
- CardTypes.json.gz (card types)
- DeckList.json.gz (deck list metadata)
- Keywords.json.gz (card keywords)
- SetList.json.gz (set list metadata)
Note: MTGJSON set.json does NOT contain image URLs. Only cards have image_uris.
Sets have iconSvgUrl (SVG icons) but no raster image URLs.
This commit is contained in:
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
MTGJSON Data Downloader
|
||||
|
||||
Downloads all MTGJSON data files and stores them for import.
|
||||
"""
|
||||
|
||||
import gzip
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from urllib.request import urlretrieve
|
||||
from urllib.parse import urljoin
|
||||
|
||||
# MTGJSON API v5 base URL
|
||||
MTGJSON_API_V5 = "https://mtgjson.com/api/v5"
|
||||
|
||||
# Files to download with their types
|
||||
MTGJSON_FILES = {
|
||||
"AllPrintings.psql.gz": {
|
||||
"name": "AllPrintings",
|
||||
"description": "Main cards database (PSQL format)",
|
||||
"type": "psql",
|
||||
},
|
||||
"AllSetFiles.zip": {
|
||||
"name": "AllSetFiles",
|
||||
"description": "Set and card data",
|
||||
"type": "zip",
|
||||
},
|
||||
"AllDeckFiles.zip": {
|
||||
"name": "AllDeckFiles",
|
||||
"description": "Deck data",
|
||||
"type": "zip",
|
||||
},
|
||||
"AllIdentifiers.json.gz": {
|
||||
"name": "AllIdentifiers",
|
||||
"description": "Card identifiers",
|
||||
"type": "json",
|
||||
},
|
||||
"CardTypes.json.gz": {
|
||||
"name": "CardTypes",
|
||||
"description": "Card types",
|
||||
"type": "json",
|
||||
},
|
||||
"DeckList.json.gz": {
|
||||
"name": "DeckList",
|
||||
"description": "Deck list metadata",
|
||||
"type": "json",
|
||||
},
|
||||
"Keywords.json.gz": {
|
||||
"name": "Keywords",
|
||||
"description": "Card keywords",
|
||||
"type": "json",
|
||||
},
|
||||
"SetList.json.gz": {
|
||||
"name": "SetList",
|
||||
"description": "Set list metadata",
|
||||
"type": "json",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def get_downloads_dir() -> Path:
|
||||
"""Get the downloads directory path."""
|
||||
data_dir = Path(os.environ.get("MTGDATA_DIR", "/app/data"))
|
||||
downloads_dir = data_dir / "mtgjson" / "downloads"
|
||||
downloads_dir.mkdir(parents=True, exist_ok=True)
|
||||
return downloads_dir
|
||||
|
||||
|
||||
def download_file(url: str, destination: Path) -> bool:
|
||||
"""Download a file from URL to destination."""
|
||||
try:
|
||||
print(f"Downloading {url}...")
|
||||
urlretrieve(url, destination)
|
||||
size_mb = destination.stat().st_size / (1024 * 1024)
|
||||
print(f" ✓ Downloaded to {destination} ({size_mb:.1f} MB)")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f" ✗ Failed to download {url}: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def download_all_files() -> list[Path]:
|
||||
"""Download all MTGJSON files."""
|
||||
downloads_dir = get_downloads_dir()
|
||||
downloaded_files = []
|
||||
|
||||
print("=== MTGJSON Data Download ===\n")
|
||||
|
||||
for filename, file_info in MTGJSON_FILES.items():
|
||||
url = urljoin(MTGJSON_API_V5, filename)
|
||||
destination = downloads_dir / filename
|
||||
|
||||
if download_file(url, destination):
|
||||
downloaded_files.append(destination)
|
||||
else:
|
||||
print(f" ⚠ Continuing with downloaded files only")
|
||||
|
||||
print(f"\n=== Download Complete ===")
|
||||
print(f"Downloaded {len(downloaded_files)} files to {downloads_dir}")
|
||||
return downloaded_files
|
||||
|
||||
|
||||
def verify_downloads(downloaded_files: list[Path]) -> bool:
|
||||
"""Verify all expected files are downloaded."""
|
||||
print("\n=== Verifying Downloads ===\n")
|
||||
|
||||
all_ok = True
|
||||
for filename, file_info in MTGJSON_FILES.items():
|
||||
filepath = Path(get_downloads_dir() / filename)
|
||||
if filepath.exists():
|
||||
size_mb = filepath.stat().st_size / (1024 * 1024)
|
||||
print(f"✓ {filename:30} ({size_mb:.1f} MB)")
|
||||
else:
|
||||
print(f"✗ {filename:30} (MISSING)")
|
||||
all_ok = False
|
||||
|
||||
if all_ok:
|
||||
print("\n✓ All files downloaded successfully")
|
||||
else:
|
||||
print("\n⚠ Some files are missing")
|
||||
|
||||
return all_ok
|
||||
|
||||
|
||||
def get_file_list() -> list[Path]:
|
||||
"""Get list of all downloaded files."""
|
||||
downloads_dir = get_downloads_dir()
|
||||
files = [downloads_dir / filename for filename in MTGJSON_FILES.keys()]
|
||||
return [f for f in files if f.exists()]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
downloaded = download_all_files()
|
||||
verify_downloads(downloaded)
|
||||
Reference in New Issue
Block a user