From ad2742c1cf9d9bb6e2378ec1824ef50b9ac2df8e Mon Sep 17 00:00:00 2001 From: admin Date: Mon, 20 Jul 2026 04:46:15 +0000 Subject: [PATCH] Fix MTGJSON download: use .gz URLs and handle pre-uncompressed files --- backend/app/services/mtgjson_manager.py | 44 +++++++++++++++++-------- state.json | 17 +++++----- 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/backend/app/services/mtgjson_manager.py b/backend/app/services/mtgjson_manager.py index f694521..52f64cb 100644 --- a/backend/app/services/mtgjson_manager.py +++ b/backend/app/services/mtgjson_manager.py @@ -41,13 +41,13 @@ logger = logging.getLogger(__name__) # MTGJSON API URLs MTGJSON_BASE_URL = "https://mtgjson.com/api/v5" REQUIRED_FILES = { - "AllPrintings.json.gz": MTGJSON_BASE_URL + "/AllPrintings.json", + "AllPrintings.json.gz": MTGJSON_BASE_URL + "/AllPrintings.json.gz", "AllSetFiles.zip": MTGJSON_BASE_URL + "/AllSetFiles.zip", - "AllIdentifiers.json.gz": MTGJSON_BASE_URL + "/AllIdentifiers.json", - "CardTypes.json.gz": MTGJSON_BASE_URL + "/CardTypes.json", - "DeckList.json.gz": MTGJSON_BASE_URL + "/DeckList.json", - "Keywords.json.gz": MTGJSON_BASE_URL + "/Keywords.json", - "SetList.json.gz": MTGJSON_BASE_URL + "/SetList.json", + "AllIdentifiers.json.gz": MTGJSON_BASE_URL + "/AllIdentifiers.json.gz", + "CardTypes.json.gz": MTGJSON_BASE_URL + "/CardTypes.json.gz", + "DeckList.json.gz": MTGJSON_BASE_URL + "/DeckList.json.gz", + "Keywords.json.gz": MTGJSON_BASE_URL + "/Keywords.json.gz", + "SetList.json.gz": MTGJSON_BASE_URL + "/SetList.json.gz", } DATA_DIR = Path("/app/data/mtgjson") @@ -55,10 +55,14 @@ REFRESH_LOG_TABLE = "mtg_refresh_log" # Expected minimum file sizes (in bytes) for MTGJSON v5 files EXPECTED_MIN_SIZES = { + "AllPrintings.json.gz": 500 * 1024 * 1024, # 500 MB "AllPrintings.json": 500 * 1024 * 1024, # 500 MB "AllSetFiles.json": 10 * 1024 * 1024, # 10 MB + "AllIdentifiers.json.gz": 100 * 1024 * 1024, # 100 MB "AllIdentifiers.json": 100 * 1024 * 1024, # 100 MB + "CardTypes.json.gz": 1 * 1024 * 1024, # 1 MB "CardTypes.json": 1 * 1024 * 1024, # 1 MB + "Keywords.json.gz": 0.5 * 1024 * 1024, # 0.5 MB "Keywords.json": 0.5 * 1024 * 1024, # 0.5 MB "MagicSets.json": 50 * 1024 * 1024, # 50 MB "MagicRoots.json": 1 * 1024 * 1024, # 1 MB @@ -307,19 +311,31 @@ class MTGJSONManager: return False def _unpack_gzip(self, gz_file: Path) -> bool: - """Unpack a gzip file.""" + """Unpack a gzip file or handle pre-uncompressed JSON.""" dest_file = gz_file.with_suffix("") logger.info(f"Unpacking {gz_file.name}") try: - with gzip.open(gz_file, 'rt', encoding='utf-8') as f_in: - content = f_in.read() - - dest_file.write_text(content, encoding='utf-8') - logger.info(f"Unpacked {gz_file.name} to {dest_file.name}") - return True - + # Check if file is actually gzipped by reading first bytes + with open(gz_file, 'rb') as f: + magic = f.read(2) + + if magic == b'\x1f\x8b': + # File is gzipped - proceed normally + with gzip.open(gz_file, 'rt', encoding='utf-8') as f_in: + content = f_in.read() + + dest_file.write_text(content, encoding='utf-8') + logger.info(f"Unpacked {gz_file.name} to {dest_file.name}") + return True + else: + # File is already plain JSON - just rename + logger.info(f"{gz_file.name} is pre-uncompressed JSON, renaming to {dest_file.name}") + dest_file.write_bytes(gz_file.read_bytes()) + gz_file.unlink() + return True + except Exception as e: logger.error(f"Failed to unpack {gz_file.name}: {e}") return False diff --git a/state.json b/state.json index f77a952..186ab2d 100644 --- a/state.json +++ b/state.json @@ -1,6 +1,6 @@ { "task_description": "MTG Online Backend - MTGJSON Data Integration & Docker Deployment", - "current_step": "Testing: destroy all containers, rebuild backend, deploy stack", + "current_step": "Testing full container lifecycle", "files_created": [ "backend/app/services/mtgjson_manager.py", "backend/scripts/sanity_check_mtgjson.py", @@ -26,17 +26,18 @@ "Implemented retry logic with exponential backoff (up to 3 attempts)", "Cleanup deletes corrupted data before retry", "Container marked unhealthy if validation fails after all retries", - "Doubled all timeout values to prevent timeouts during download/upsert", "Download timeout increased to 60 minutes (3600s) for large files" ], "next_steps": [ - "Destroy all containers", - "Build backend image", - "Deploy stack", - "Monitor until healthy", - "Check logs for success" + "Save state to state.json", + "Commit and push to Gitea", + "Stop and destroy all Docker containers", + "Build backend Docker container", + "Deploy stack and monitor", + "Check container health status", + "Verify logs show successful initialization" ], "blockers": [], - "commit_hash": "20fb572", + "commit_hash": "0643544", "timestamp": 1784598400 }