Fix MTGJSON download: use .json files, remove gzip unpacking

- Changed REQUIRED_FILES to REQUIRED_JSON_FILES and REQUIRED_ZIP_FILES
- AllSetFiles.zip kept as zip (only available compressed on MTGJSON)
- All other files downloaded as .json (no compression)
- Removed gzip import and unpacking logic for .gz files
- Updated EXPECTED_MIN_SIZES to reflect actual .json file sizes
- Removed validation for non-existent files

backend/app/services/mtgjson_manager.py

@@ -5,8 +5,8 @@ Handles downloading, unpacking, and upserting MTGJSON data into PostgreSQL.
 Manages the complete data lifecycle from download to database upsert.
 
 Features:
-- Downloads required MTGJSON datasets from the API
+- Downloads required MTGJSON datasets from the API (JSON files, except AllSetFiles as zip)
-- Unpacks gzip and zip files
+- Unpacks zip files (AllSetFiles.zip only)
 - Converts JSON to PostgreSQL-compatible format
 - Upserts data without overwriting existing entries
 - Tracks refresh timestamps and status
@@ -17,11 +17,9 @@ Usage:
 """
 
 import asyncio
-import gzip
 import json
 import logging
 import os
-import re
 import time
 import zipfile
 from datetime import datetime, timedelta
@@ -38,35 +36,33 @@ from app.core.settings import get_settings
 
 logger = logging.getLogger(__name__)
 
-# MTGJSON API URLs
+# MTGJSON API URLs - .json files for most datasets, .zip for AllSetFiles
 MTGJSON_BASE_URL = "https://mtgjson.com/api/v5"
-REQUIRED_FILES = {
+REQUIRED_JSON_FILES = {
-    "AllPrintings.json.gz": MTGJSON_BASE_URL + "/AllPrintings.json.gz",
+    "AllPrintings.json": MTGJSON_BASE_URL + "/AllPrintings.json",
+    "AllIdentifiers.json": MTGJSON_BASE_URL + "/AllIdentifiers.json",
+    "CardTypes.json": MTGJSON_BASE_URL + "/CardTypes.json",
+    "DeckList.json": MTGJSON_BASE_URL + "/DeckList.json",
+    "Keywords.json": MTGJSON_BASE_URL + "/Keywords.json",
+    "SetList.json": MTGJSON_BASE_URL + "/SetList.json",
+}
+
+REQUIRED_ZIP_FILES = {
     "AllSetFiles.zip": MTGJSON_BASE_URL + "/AllSetFiles.zip",
-    "AllIdentifiers.json.gz": MTGJSON_BASE_URL + "/AllIdentifiers.json.gz",
-    "CardTypes.json.gz": MTGJSON_BASE_URL + "/CardTypes.json.gz",
-    "DeckList.json.gz": MTGJSON_BASE_URL + "/DeckList.json.gz",
-    "Keywords.json.gz": MTGJSON_BASE_URL + "/Keywords.json.gz",
-    "SetList.json.gz": MTGJSON_BASE_URL + "/SetList.json.gz",
 }
 
 DATA_DIR = Path("/app/data/mtgjson")
 REFRESH_LOG_TABLE = "mtg_refresh_log"
 
-# Expected minimum file sizes (in bytes) for MTGJSON v5 files
+# Expected minimum file sizes (in bytes) for MTGJSON v5 JSON files
 EXPECTED_MIN_SIZES = {
-    "AllPrintings.json.gz": 500 * 1024 * 1024,  # 500 MB
+    "AllPrintings.json": 500 * 1024 * 1024,      # 500 MB (actual: ~620 MB)
-    "AllPrintings.json": 500 * 1024 * 1024,  # 500 MB
+    "AllIdentifiers.json": 500 * 1024 * 1024,     # 500 MB (actual: ~599 MB)
-    "AllSetFiles.json": 10 * 1024 * 1024,     # 10 MB
+    "CardTypes.json": 0.001 * 1024 * 1024,        # 0.001 MB (actual: ~0.01 MB)
-    "AllIdentifiers.json.gz": 100 * 1024 * 1024, # 100 MB
+    "DeckList.json": 0.5 * 1024 * 1024,           # 0.5 MB (actual: ~0.59 MB)
-    "AllIdentifiers.json": 100 * 1024 * 1024, # 100 MB
+    "Keywords.json": 0.001 * 1024 * 1024,         # 0.001 MB (actual: ~0.00 MB)
-    "CardTypes.json.gz": 1 * 1024 * 1024,        # 1 MB
+    "SetList.json": 5 * 1024 * 1024,              # 5 MB (actual: ~11 MB)
-    "CardTypes.json": 1 * 1024 * 1024,        # 1 MB
+    "AllSetFiles.zip": 50 * 1024 * 1024,          # 50 MB (expected size)
-    "Keywords.json.gz": 0.5 * 1024 * 1024,       # 0.5 MB
-    "Keywords.json": 0.5 * 1024 * 1024,       # 0.5 MB
-    "MagicSets.json": 50 * 1024 * 1024,       # 50 MB
-    "MagicRoots.json": 1 * 1024 * 1024,       # 1 MB
-    "SetTranslations.json": 5 * 1024 * 1024,  # 5 MB
 }
 
 MAX_DOWNLOAD_RETRIES = 3
@@ -130,13 +126,13 @@ class MTGJSONManager:
     def _get_estimated_size(self, filename: str) -> int:
         """Get estimated file size in MB."""
         estimates = {
-            "AllPrintings.json.gz": 500,
+            "AllPrintings.json": 620,
             "AllSetFiles.zip": 10,
-            "AllIdentifiers.json.gz": 100,
+            "AllIdentifiers.json": 600,
-            "CardTypes.json.gz": 5,
+            "CardTypes.json": 0.01,
-            "DeckList.json.gz": 5,
+            "DeckList.json": 0.6,
-            "Keywords.json.gz": 2,
+            "Keywords.json": 0.01,
-            "SetList.json.gz": 10,
+            "SetList.json": 11,
         }
         return estimates.get(filename, 10)  # Default 10MB
     
@@ -178,7 +174,7 @@ class MTGJSONManager:
                     continue
                 return False
             
-            # Unpack files
+            # Unpack zip files (AllSetFiles.zip only)
             unpack_success = await self.unpack_files()
             
             if not unpack_success:
@@ -210,12 +206,14 @@ class MTGJSONManager:
         return False
 
     async def download_files(self) -> bool:
-        """Download all required MTGJSON files."""
+        """Download all required MTGJSON files (JSON and ZIP)."""
         logger.info(f"Starting MTGJSON data download to {self.data_dir}")
         
         async with aiohttp.ClientSession() as session:
             tasks = []
-            for filename, url in REQUIRED_FILES.items():
+            for filename, url in REQUIRED_JSON_FILES.items():
+                tasks.append(self._download_file(session, url, filename))
+            for filename, url in REQUIRED_ZIP_FILES.items():
                 tasks.append(self._download_file(session, url, filename))
             
             results = await asyncio.gather(*tasks, return_exceptions=True)
@@ -288,56 +286,21 @@ class MTGJSONManager:
             return False
 
     async def unpack_files(self) -> bool:
-        """Unpack all downloaded files."""
+        """Unpack all downloaded zip files."""
-        logger.info("Unpacking MTGJSON files")
+        logger.info("Unpacking MTGJSON zip files")
         
         unpacked_files = []
         
-        # Unpack gzip files
+        # Unpack zip files only
-        for gz_file in self.data_dir.glob("*.gz"):
-            if self._unpack_gzip(gz_file):
-                unpacked_files.append(gz_file.with_suffix(""))
-        
-        # Unpack zip files
         for zip_file in self.data_dir.glob("*.zip"):
             if self._unpack_zip(zip_file):
-                unpacked_files.append(zip_file.with_suffix(""))
+                unpacked_files.append(zip_file)
         
         if len(unpacked_files) > 0:
-            logger.info(f"Unpacked {len(unpacked_files)} files")
+            logger.info(f"Unpacked {len(unpacked_files)} zip files")
             return True
         else:
-            logger.warning("No files were unpacked")
+            logger.warning("No zip files were unpacked")
-            return False
-
-    def _unpack_gzip(self, gz_file: Path) -> bool:
-        """Unpack a gzip file or handle pre-uncompressed JSON."""
-        dest_file = gz_file.with_suffix("")
-        
-        logger.info(f"Unpacking {gz_file.name}")
-        
-        try:
-            # Check if file is actually gzipped by reading first bytes
-            with open(gz_file, 'rb') as f:
-                magic = f.read(2)
-                
-            if magic == b'\x1f\x8b':
-                # File is gzipped - proceed normally
-                with gzip.open(gz_file, 'rt', encoding='utf-8') as f_in:
-                    content = f_in.read()
-                
-                dest_file.write_text(content, encoding='utf-8')
-                logger.info(f"Unpacked {gz_file.name} to {dest_file.name}")
-                return True
-            else:
-                # File is already plain JSON - just rename
-                logger.info(f"{gz_file.name} is pre-uncompressed JSON, renaming to {dest_file.name}")
-                dest_file.write_bytes(gz_file.read_bytes())
-                gz_file.unlink()
-                return True
-                
-        except Exception as e:
-            logger.error(f"Failed to unpack {gz_file.name}: {e}")
             return False
 
     def _unpack_zip(self, zip_file: Path) -> bool:
@@ -375,7 +338,7 @@ class MTGJSONManager:
         }
         
         try:
-            # Process AllSetFiles directory
+            # Process AllSetFiles directory (from zip)
             await self._upsert_sets()
             counts["sets"] = await self._count_sets()
             
@@ -827,4 +790,4 @@ if __name__ == "__main__":
         else:
             print("Usage: python -m app.services.mtgjson_manager [--refresh | --health]")
     
-    asyncio.run(main())
+    asyncio.run(main())

backend/test_download.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+"""Test MTGJSON API endpoints and download JSON files."""
+import asyncio
+import aiohttp
+from pathlib import Path
+
+DATA_DIR = Path("/app/data/mtgjson")
+DATA_DIR.mkdir(parents=True, exist_ok=True)
+
+# MTGJSON API endpoints for .json files
+MTGJSON_BASE_URL = "https://mtgjson.com/api/v5"
+JSON_FILES = {
+    "AllPrintings.json": MTGJSON_BASE_URL + "/AllPrintings.json",
+    "AllSetFiles.json": MTGJSON_BASE_URL + "/AllSetFiles.json",
+    "AllIdentifiers.json": MTGJSON_BASE_URL + "/AllIdentifiers.json",
+    "CardTypes.json": MTGJSON_BASE_URL + "/CardTypes.json",
+    "DeckList.json": MTGJSON_BASE_URL + "/DeckList.json",
+    "Keywords.json": MTGJSON_BASE_URL + "/Keywords.json",
+    "SetList.json": MTGJSON_BASE_URL + "/SetList.json",
+}
+
+async def test_download_file(session, url, filename):
+    """Test downloading a single JSON file."""
+    dest_path = DATA_DIR / filename
+    print(f"Testing {filename}...")
+    
+    try:
+        async with session.get(url) as response:
+            if response.status == 200:
+                content = await response.read()
+                print(f"  ✓ Status: {response.status}")
+                print(f"  ✓ Size: {len(content) / (1024*1024):.2f} MB")
+                print(f"  ✓ Content-Type: {response.content_type}")
+                
+                # Write to file
+                with open(dest_path, 'wb') as f:
+                    f.write(content)
+                print(f"  ✓ Saved to {dest_path}")
+                
+                # Test JSON parsing
+                import json
+                with open(dest_path, 'r', encoding='utf-8') as f:
+                    data = json.load(f)
+                print(f"  ✓ Valid JSON: {type(data).__name__}")
+                
+                return True
+            else:
+                print(f"  ✗ Status: {response.status}")
+                return False
+    except Exception as e:
+        print(f"  ✗ Error: {e}")
+        return False
+
+async def main():
+    """Test all MTGJSON JSON endpoints."""
+    print("Testing MTGJSON JSON API endpoints...\n")
+    
+    async with aiohttp.ClientSession() as session:
+        results = []
+        for filename, url in JSON_FILES.items():
+            success = await test_download_file(session, url, filename)
+            results.append((filename, success))
+            print()
+    
+    print("\n=== Results ===")
+    for filename, success in results:
+        status = "✓" if success else "✗"
+        print(f"{status} {filename}")
+
+if __name__ == "__main__":
+    asyncio.run(main())

state.json

@@ -1,43 +1,27 @@
 {
-  "task_description": "MTG Online Backend - MTGJSON Data Integration & Docker Deployment",
+  "task_description": "Update MTGJSON data manager to download .json files instead of .json.gz files, remove unnecessary unpacking logic",
-  "current_step": "Testing full container lifecycle",
+  "current_step": "Docker image rebuilt with updated code",
   "files_created": [
-    "backend/app/services/mtgjson_manager.py",
+    "backend/app/services/mtgjson_manager.py"
-    "backend/scripts/sanity_check_mtgjson.py",
-    "backend/scripts/verify_mtgjson_data.py",
-    "backend/scripts/maintain_mtgjson.sh"
   ],
   "files_modified": [
-    "backend/app/services/mtgjson_manager.py",
+    "backend/app/services/mtgjson_manager.py"
-    "backend/app/scripts/refresh_mtg.py",
-    "backend/app/main.py",
-    "docker-compose.yml",
-    "docker-compose.dev.yml",
-    "backend/Dockerfile"
   ],
   "decisions": [
-    "Created comprehensive MTGJSONManager service in app/services/",
+    "Changed REQUIRED_FILES to REQUIRED_JSON_FILES and REQUIRED_ZIP_FILES",
-    "Manager handles download, unpack, upsert with ON CONFLICT DO UPDATE",
+    "AllSetFiles.zip kept as zip (only available compressed on MTGJSON API)",
-    "Startup triggers initial download on first container init",
+    "All other files downloaded as .json (no compression)",
-    "Health check verifies MTG data exists in database",
+    "Removed gzip import and unpacking logic for .gz files",
-    "Refresh interval set to 7 days (weekly)",
+    "Updated EXPECTED_MIN_SIZES to reflect actual .json file sizes (500MB+ for large files)",
-    "Docker compose start_period set to 600s for download time",
+    "Removed validation for non-existent files (MagicSets.json, MagicRoots.json, SetTranslations.json)"
-    "Added file size validation against expected minimums",
-    "Implemented retry logic with exponential backoff (up to 3 attempts)",
-    "Cleanup deletes corrupted data before retry",
-    "Container marked unhealthy if validation fails after all retries",
-    "Download timeout increased to 60 minutes (3600s) for large files"
   ],
   "next_steps": [
-    "Save state to state.json",
+    "Restart Docker container with new image",
-    "Commit and push to Gitea",
+    "Run data refresh to verify download and upsert work correctly",
-    "Stop and destroy all Docker containers",
+    "Monitor logs for any issues",
-    "Build backend Docker container",
+    "Verify database has correct data"
-    "Deploy stack and monitor",
-    "Check container health status",
-    "Verify logs show successful initialization"
   ],
   "blockers": [],
-  "commit_hash": "0643544",
+  "commit_hash": "",
-  "timestamp": 1784598400
+  "timestamp": "2026-07-20T13:52:00Z"
 }