diff --git a/backend/app/services/mtgjson_manager.py b/backend/app/services/mtgjson_manager.py index 52f64cb..bb550da 100644 --- a/backend/app/services/mtgjson_manager.py +++ b/backend/app/services/mtgjson_manager.py @@ -5,8 +5,8 @@ Handles downloading, unpacking, and upserting MTGJSON data into PostgreSQL. Manages the complete data lifecycle from download to database upsert. Features: -- Downloads required MTGJSON datasets from the API -- Unpacks gzip and zip files +- Downloads required MTGJSON datasets from the API (JSON files, except AllSetFiles as zip) +- Unpacks zip files (AllSetFiles.zip only) - Converts JSON to PostgreSQL-compatible format - Upserts data without overwriting existing entries - Tracks refresh timestamps and status @@ -17,11 +17,9 @@ Usage: """ import asyncio -import gzip import json import logging import os -import re import time import zipfile from datetime import datetime, timedelta @@ -38,35 +36,33 @@ from app.core.settings import get_settings logger = logging.getLogger(__name__) -# MTGJSON API URLs +# MTGJSON API URLs - .json files for most datasets, .zip for AllSetFiles MTGJSON_BASE_URL = "https://mtgjson.com/api/v5" -REQUIRED_FILES = { - "AllPrintings.json.gz": MTGJSON_BASE_URL + "/AllPrintings.json.gz", +REQUIRED_JSON_FILES = { + "AllPrintings.json": MTGJSON_BASE_URL + "/AllPrintings.json", + "AllIdentifiers.json": MTGJSON_BASE_URL + "/AllIdentifiers.json", + "CardTypes.json": MTGJSON_BASE_URL + "/CardTypes.json", + "DeckList.json": MTGJSON_BASE_URL + "/DeckList.json", + "Keywords.json": MTGJSON_BASE_URL + "/Keywords.json", + "SetList.json": MTGJSON_BASE_URL + "/SetList.json", +} + +REQUIRED_ZIP_FILES = { "AllSetFiles.zip": MTGJSON_BASE_URL + "/AllSetFiles.zip", - "AllIdentifiers.json.gz": MTGJSON_BASE_URL + "/AllIdentifiers.json.gz", - "CardTypes.json.gz": MTGJSON_BASE_URL + "/CardTypes.json.gz", - "DeckList.json.gz": MTGJSON_BASE_URL + "/DeckList.json.gz", - "Keywords.json.gz": MTGJSON_BASE_URL + "/Keywords.json.gz", - "SetList.json.gz": MTGJSON_BASE_URL + "/SetList.json.gz", } DATA_DIR = Path("/app/data/mtgjson") REFRESH_LOG_TABLE = "mtg_refresh_log" -# Expected minimum file sizes (in bytes) for MTGJSON v5 files +# Expected minimum file sizes (in bytes) for MTGJSON v5 JSON files EXPECTED_MIN_SIZES = { - "AllPrintings.json.gz": 500 * 1024 * 1024, # 500 MB - "AllPrintings.json": 500 * 1024 * 1024, # 500 MB - "AllSetFiles.json": 10 * 1024 * 1024, # 10 MB - "AllIdentifiers.json.gz": 100 * 1024 * 1024, # 100 MB - "AllIdentifiers.json": 100 * 1024 * 1024, # 100 MB - "CardTypes.json.gz": 1 * 1024 * 1024, # 1 MB - "CardTypes.json": 1 * 1024 * 1024, # 1 MB - "Keywords.json.gz": 0.5 * 1024 * 1024, # 0.5 MB - "Keywords.json": 0.5 * 1024 * 1024, # 0.5 MB - "MagicSets.json": 50 * 1024 * 1024, # 50 MB - "MagicRoots.json": 1 * 1024 * 1024, # 1 MB - "SetTranslations.json": 5 * 1024 * 1024, # 5 MB + "AllPrintings.json": 500 * 1024 * 1024, # 500 MB (actual: ~620 MB) + "AllIdentifiers.json": 500 * 1024 * 1024, # 500 MB (actual: ~599 MB) + "CardTypes.json": 0.001 * 1024 * 1024, # 0.001 MB (actual: ~0.01 MB) + "DeckList.json": 0.5 * 1024 * 1024, # 0.5 MB (actual: ~0.59 MB) + "Keywords.json": 0.001 * 1024 * 1024, # 0.001 MB (actual: ~0.00 MB) + "SetList.json": 5 * 1024 * 1024, # 5 MB (actual: ~11 MB) + "AllSetFiles.zip": 50 * 1024 * 1024, # 50 MB (expected size) } MAX_DOWNLOAD_RETRIES = 3 @@ -130,13 +126,13 @@ class MTGJSONManager: def _get_estimated_size(self, filename: str) -> int: """Get estimated file size in MB.""" estimates = { - "AllPrintings.json.gz": 500, + "AllPrintings.json": 620, "AllSetFiles.zip": 10, - "AllIdentifiers.json.gz": 100, - "CardTypes.json.gz": 5, - "DeckList.json.gz": 5, - "Keywords.json.gz": 2, - "SetList.json.gz": 10, + "AllIdentifiers.json": 600, + "CardTypes.json": 0.01, + "DeckList.json": 0.6, + "Keywords.json": 0.01, + "SetList.json": 11, } return estimates.get(filename, 10) # Default 10MB @@ -178,7 +174,7 @@ class MTGJSONManager: continue return False - # Unpack files + # Unpack zip files (AllSetFiles.zip only) unpack_success = await self.unpack_files() if not unpack_success: @@ -210,12 +206,14 @@ class MTGJSONManager: return False async def download_files(self) -> bool: - """Download all required MTGJSON files.""" + """Download all required MTGJSON files (JSON and ZIP).""" logger.info(f"Starting MTGJSON data download to {self.data_dir}") async with aiohttp.ClientSession() as session: tasks = [] - for filename, url in REQUIRED_FILES.items(): + for filename, url in REQUIRED_JSON_FILES.items(): + tasks.append(self._download_file(session, url, filename)) + for filename, url in REQUIRED_ZIP_FILES.items(): tasks.append(self._download_file(session, url, filename)) results = await asyncio.gather(*tasks, return_exceptions=True) @@ -288,56 +286,21 @@ class MTGJSONManager: return False async def unpack_files(self) -> bool: - """Unpack all downloaded files.""" - logger.info("Unpacking MTGJSON files") + """Unpack all downloaded zip files.""" + logger.info("Unpacking MTGJSON zip files") unpacked_files = [] - # Unpack gzip files - for gz_file in self.data_dir.glob("*.gz"): - if self._unpack_gzip(gz_file): - unpacked_files.append(gz_file.with_suffix("")) - - # Unpack zip files + # Unpack zip files only for zip_file in self.data_dir.glob("*.zip"): if self._unpack_zip(zip_file): - unpacked_files.append(zip_file.with_suffix("")) + unpacked_files.append(zip_file) if len(unpacked_files) > 0: - logger.info(f"Unpacked {len(unpacked_files)} files") + logger.info(f"Unpacked {len(unpacked_files)} zip files") return True else: - logger.warning("No files were unpacked") - return False - - def _unpack_gzip(self, gz_file: Path) -> bool: - """Unpack a gzip file or handle pre-uncompressed JSON.""" - dest_file = gz_file.with_suffix("") - - logger.info(f"Unpacking {gz_file.name}") - - try: - # Check if file is actually gzipped by reading first bytes - with open(gz_file, 'rb') as f: - magic = f.read(2) - - if magic == b'\x1f\x8b': - # File is gzipped - proceed normally - with gzip.open(gz_file, 'rt', encoding='utf-8') as f_in: - content = f_in.read() - - dest_file.write_text(content, encoding='utf-8') - logger.info(f"Unpacked {gz_file.name} to {dest_file.name}") - return True - else: - # File is already plain JSON - just rename - logger.info(f"{gz_file.name} is pre-uncompressed JSON, renaming to {dest_file.name}") - dest_file.write_bytes(gz_file.read_bytes()) - gz_file.unlink() - return True - - except Exception as e: - logger.error(f"Failed to unpack {gz_file.name}: {e}") + logger.warning("No zip files were unpacked") return False def _unpack_zip(self, zip_file: Path) -> bool: @@ -375,7 +338,7 @@ class MTGJSONManager: } try: - # Process AllSetFiles directory + # Process AllSetFiles directory (from zip) await self._upsert_sets() counts["sets"] = await self._count_sets() @@ -827,4 +790,4 @@ if __name__ == "__main__": else: print("Usage: python -m app.services.mtgjson_manager [--refresh | --health]") - asyncio.run(main()) \ No newline at end of file + asyncio.run(main()) diff --git a/backend/test_download.py b/backend/test_download.py new file mode 100644 index 0000000..746390b --- /dev/null +++ b/backend/test_download.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Test MTGJSON API endpoints and download JSON files.""" +import asyncio +import aiohttp +from pathlib import Path + +DATA_DIR = Path("/app/data/mtgjson") +DATA_DIR.mkdir(parents=True, exist_ok=True) + +# MTGJSON API endpoints for .json files +MTGJSON_BASE_URL = "https://mtgjson.com/api/v5" +JSON_FILES = { + "AllPrintings.json": MTGJSON_BASE_URL + "/AllPrintings.json", + "AllSetFiles.json": MTGJSON_BASE_URL + "/AllSetFiles.json", + "AllIdentifiers.json": MTGJSON_BASE_URL + "/AllIdentifiers.json", + "CardTypes.json": MTGJSON_BASE_URL + "/CardTypes.json", + "DeckList.json": MTGJSON_BASE_URL + "/DeckList.json", + "Keywords.json": MTGJSON_BASE_URL + "/Keywords.json", + "SetList.json": MTGJSON_BASE_URL + "/SetList.json", +} + +async def test_download_file(session, url, filename): + """Test downloading a single JSON file.""" + dest_path = DATA_DIR / filename + print(f"Testing {filename}...") + + try: + async with session.get(url) as response: + if response.status == 200: + content = await response.read() + print(f" ✓ Status: {response.status}") + print(f" ✓ Size: {len(content) / (1024*1024):.2f} MB") + print(f" ✓ Content-Type: {response.content_type}") + + # Write to file + with open(dest_path, 'wb') as f: + f.write(content) + print(f" ✓ Saved to {dest_path}") + + # Test JSON parsing + import json + with open(dest_path, 'r', encoding='utf-8') as f: + data = json.load(f) + print(f" ✓ Valid JSON: {type(data).__name__}") + + return True + else: + print(f" ✗ Status: {response.status}") + return False + except Exception as e: + print(f" ✗ Error: {e}") + return False + +async def main(): + """Test all MTGJSON JSON endpoints.""" + print("Testing MTGJSON JSON API endpoints...\n") + + async with aiohttp.ClientSession() as session: + results = [] + for filename, url in JSON_FILES.items(): + success = await test_download_file(session, url, filename) + results.append((filename, success)) + print() + + print("\n=== Results ===") + for filename, success in results: + status = "✓" if success else "✗" + print(f"{status} {filename}") + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/state.json b/state.json index 186ab2d..15aed3f 100644 --- a/state.json +++ b/state.json @@ -1,43 +1,27 @@ { - "task_description": "MTG Online Backend - MTGJSON Data Integration & Docker Deployment", - "current_step": "Testing full container lifecycle", + "task_description": "Update MTGJSON data manager to download .json files instead of .json.gz files, remove unnecessary unpacking logic", + "current_step": "Docker image rebuilt with updated code", "files_created": [ - "backend/app/services/mtgjson_manager.py", - "backend/scripts/sanity_check_mtgjson.py", - "backend/scripts/verify_mtgjson_data.py", - "backend/scripts/maintain_mtgjson.sh" + "backend/app/services/mtgjson_manager.py" ], "files_modified": [ - "backend/app/services/mtgjson_manager.py", - "backend/app/scripts/refresh_mtg.py", - "backend/app/main.py", - "docker-compose.yml", - "docker-compose.dev.yml", - "backend/Dockerfile" + "backend/app/services/mtgjson_manager.py" ], "decisions": [ - "Created comprehensive MTGJSONManager service in app/services/", - "Manager handles download, unpack, upsert with ON CONFLICT DO UPDATE", - "Startup triggers initial download on first container init", - "Health check verifies MTG data exists in database", - "Refresh interval set to 7 days (weekly)", - "Docker compose start_period set to 600s for download time", - "Added file size validation against expected minimums", - "Implemented retry logic with exponential backoff (up to 3 attempts)", - "Cleanup deletes corrupted data before retry", - "Container marked unhealthy if validation fails after all retries", - "Download timeout increased to 60 minutes (3600s) for large files" + "Changed REQUIRED_FILES to REQUIRED_JSON_FILES and REQUIRED_ZIP_FILES", + "AllSetFiles.zip kept as zip (only available compressed on MTGJSON API)", + "All other files downloaded as .json (no compression)", + "Removed gzip import and unpacking logic for .gz files", + "Updated EXPECTED_MIN_SIZES to reflect actual .json file sizes (500MB+ for large files)", + "Removed validation for non-existent files (MagicSets.json, MagicRoots.json, SetTranslations.json)" ], "next_steps": [ - "Save state to state.json", - "Commit and push to Gitea", - "Stop and destroy all Docker containers", - "Build backend Docker container", - "Deploy stack and monitor", - "Check container health status", - "Verify logs show successful initialization" + "Restart Docker container with new image", + "Run data refresh to verify download and upsert work correctly", + "Monitor logs for any issues", + "Verify database has correct data" ], "blockers": [], - "commit_hash": "0643544", - "timestamp": 1784598400 + "commit_hash": "", + "timestamp": "2026-07-20T13:52:00Z" }