Fix MTGJSON download: use .gz URLs and handle pre-uncompressed files

backend/app/services/mtgjson_manager.py

@@ -41,13 +41,13 @@ logger = logging.getLogger(__name__)
 # MTGJSON API URLs
 MTGJSON_BASE_URL = "https://mtgjson.com/api/v5"
 REQUIRED_FILES = {
-    "AllPrintings.json.gz": MTGJSON_BASE_URL + "/AllPrintings.json",
+    "AllPrintings.json.gz": MTGJSON_BASE_URL + "/AllPrintings.json.gz",
     "AllSetFiles.zip": MTGJSON_BASE_URL + "/AllSetFiles.zip",
-    "AllIdentifiers.json.gz": MTGJSON_BASE_URL + "/AllIdentifiers.json",
+    "AllIdentifiers.json.gz": MTGJSON_BASE_URL + "/AllIdentifiers.json.gz",
-    "CardTypes.json.gz": MTGJSON_BASE_URL + "/CardTypes.json",
+    "CardTypes.json.gz": MTGJSON_BASE_URL + "/CardTypes.json.gz",
-    "DeckList.json.gz": MTGJSON_BASE_URL + "/DeckList.json",
+    "DeckList.json.gz": MTGJSON_BASE_URL + "/DeckList.json.gz",
-    "Keywords.json.gz": MTGJSON_BASE_URL + "/Keywords.json",
+    "Keywords.json.gz": MTGJSON_BASE_URL + "/Keywords.json.gz",
-    "SetList.json.gz": MTGJSON_BASE_URL + "/SetList.json",
+    "SetList.json.gz": MTGJSON_BASE_URL + "/SetList.json.gz",
 }
 
 DATA_DIR = Path("/app/data/mtgjson")
@@ -55,10 +55,14 @@ REFRESH_LOG_TABLE = "mtg_refresh_log"
 
 # Expected minimum file sizes (in bytes) for MTGJSON v5 files
 EXPECTED_MIN_SIZES = {
+    "AllPrintings.json.gz": 500 * 1024 * 1024,  # 500 MB
     "AllPrintings.json": 500 * 1024 * 1024,  # 500 MB
     "AllSetFiles.json": 10 * 1024 * 1024,     # 10 MB
+    "AllIdentifiers.json.gz": 100 * 1024 * 1024, # 100 MB
     "AllIdentifiers.json": 100 * 1024 * 1024, # 100 MB
+    "CardTypes.json.gz": 1 * 1024 * 1024,        # 1 MB
     "CardTypes.json": 1 * 1024 * 1024,        # 1 MB
+    "Keywords.json.gz": 0.5 * 1024 * 1024,       # 0.5 MB
     "Keywords.json": 0.5 * 1024 * 1024,       # 0.5 MB
     "MagicSets.json": 50 * 1024 * 1024,       # 50 MB
     "MagicRoots.json": 1 * 1024 * 1024,       # 1 MB
@@ -307,19 +311,31 @@ class MTGJSONManager:
             return False
 
     def _unpack_gzip(self, gz_file: Path) -> bool:
-        """Unpack a gzip file."""
+        """Unpack a gzip file or handle pre-uncompressed JSON."""
         dest_file = gz_file.with_suffix("")
         
         logger.info(f"Unpacking {gz_file.name}")
         
         try:
-            with gzip.open(gz_file, 'rt', encoding='utf-8') as f_in:
+            # Check if file is actually gzipped by reading first bytes
-                content = f_in.read()
+            with open(gz_file, 'rb') as f:
-            
+                magic = f.read(2)
-            dest_file.write_text(content, encoding='utf-8')
+                
-            logger.info(f"Unpacked {gz_file.name} to {dest_file.name}")
+            if magic == b'\x1f\x8b':
-            return True
+                # File is gzipped - proceed normally
-            
+                with gzip.open(gz_file, 'rt', encoding='utf-8') as f_in:
+                    content = f_in.read()
+                
+                dest_file.write_text(content, encoding='utf-8')
+                logger.info(f"Unpacked {gz_file.name} to {dest_file.name}")
+                return True
+            else:
+                # File is already plain JSON - just rename
+                logger.info(f"{gz_file.name} is pre-uncompressed JSON, renaming to {dest_file.name}")
+                dest_file.write_bytes(gz_file.read_bytes())
+                gz_file.unlink()
+                return True
+                
         except Exception as e:
             logger.error(f"Failed to unpack {gz_file.name}: {e}")
             return False

state.json

@@ -1,6 +1,6 @@
 {
   "task_description": "MTG Online Backend - MTGJSON Data Integration & Docker Deployment",
-  "current_step": "Testing: destroy all containers, rebuild backend, deploy stack",
+  "current_step": "Testing full container lifecycle",
   "files_created": [
     "backend/app/services/mtgjson_manager.py",
     "backend/scripts/sanity_check_mtgjson.py",
@@ -26,17 +26,18 @@
     "Implemented retry logic with exponential backoff (up to 3 attempts)",
     "Cleanup deletes corrupted data before retry",
     "Container marked unhealthy if validation fails after all retries",
-    "Doubled all timeout values to prevent timeouts during download/upsert",
     "Download timeout increased to 60 minutes (3600s) for large files"
   ],
   "next_steps": [
-    "Destroy all containers",
+    "Save state to state.json",
-    "Build backend image",
+    "Commit and push to Gitea",
-    "Deploy stack",
+    "Stop and destroy all Docker containers",
-    "Monitor until healthy",
+    "Build backend Docker container",
-    "Check logs for success"
+    "Deploy stack and monitor",
+    "Check container health status",
+    "Verify logs show successful initialization"
   ],
   "blockers": [],
-  "commit_hash": "20fb572",
+  "commit_hash": "0643544",
   "timestamp": 1784598400
 }