Add MTGJSON data sanity check with file size validation and retry logic

backend/app/services/mtgjson_manager.py

@@ -53,15 +53,145 @@ REQUIRED_FILES = {
 DATA_DIR = Path("/app/data/mtgjson")
 REFRESH_LOG_TABLE = "mtg_refresh_log"
 
+# Expected minimum file sizes (in bytes) for MTGJSON v5 files
+EXPECTED_MIN_SIZES = {
+    "AllPrintings.json": 500 * 1024 * 1024,  # 500 MB
+    "AllSetFiles.json": 10 * 1024 * 1024,     # 10 MB
+    "AllIdentifiers.json": 100 * 1024 * 1024, # 100 MB
+    "CardTypes.json": 1 * 1024 * 1024,        # 1 MB
+    "Keywords.json": 0.5 * 1024 * 1024,       # 0.5 MB
+    "MagicSets.json": 50 * 1024 * 1024,       # 50 MB
+    "MagicRoots.json": 1 * 1024 * 1024,       # 1 MB
+    "SetTranslations.json": 5 * 1024 * 1024,  # 5 MB
+}
+
+MAX_DOWNLOAD_RETRIES = 3
+RETRY_DELAY_SECONDS = 60
+
 
 class MTGJSONManager:
     """Manages MTGJSON data download, unpacking, and database upsert."""
 
-    def __init__(self, data_dir: Path = DATA_DIR):
+    def __init__(self, data_dir: Path = None):
+        """Initialize MTGJSONManager.
+        
+        Args:
+            data_dir: Path to data directory. If None, uses settings.DATA_DIR
+        """
+        if data_dir is None:
+            settings = get_settings()
+            data_dir = settings.DATA_DIR
+        
+        # Convert string to Path if needed
+        if isinstance(data_dir, str):
+            data_dir = Path(data_dir)
+            
         self.data_dir = data_dir
         self.data_dir.mkdir(parents=True, exist_ok=True)
         self.settings = get_settings()
 
+    async def validate_data_integrity(self) -> tuple[bool, list[str]]:
+        """
+        Validate downloaded MTGJSON files for expected sizes.
+        
+        Returns:
+            Tuple of (is_valid, list_of_issues)
+        """
+        issues = []
+        
+        for filename, min_size in EXPECTED_MIN_SIZES.items():
+            filepath = self.data_dir / filename
+            
+            if not filepath.exists():
+                issues.append(f"Missing file: {filename}")
+                continue
+            
+            actual_size = filepath.stat().st_size
+            
+            if actual_size < min_size:
+                issues.append(
+                    f"{filename}: {actual_size / (1024*1024):.1f} MB "
+                    f"(minimum: {min_size / (1024*1024):.1f} MB)"
+                )
+        
+        if issues:
+            logger.error(f"Data integrity check failed with {len(issues)} issues:")
+            for issue in issues:
+                logger.error(f"  - {issue}")
+            return False, issues
+        
+        logger.info("Data integrity check passed - all files meet minimum size requirements")
+        return True, []
+    
+    async def cleanup_data_files(self) -> None:
+        """Delete all downloaded MTGJSON data files."""
+        logger.info(f"Cleaning up data files in {self.data_dir}")
+        
+        try:
+            for f in self.data_dir.glob("*"):
+                if f.is_file():
+                    f.unlink()
+                    logger.info(f"Deleted: {f.name}")
+            logger.info("Data cleanup complete")
+        except Exception as e:
+            logger.error(f"Failed to clean up data files: {e}")
+            raise
+    
+    async def download_with_sanity_check(self) -> bool:
+        """
+        Download MTGJSON files with sanity checking and retry logic.
+        
+        Returns:
+            True if download and validation succeed, False otherwise
+        """
+        logger.info("Starting MTGJSON download with sanity checks")
+        
+        for attempt in range(1, MAX_DOWNLOAD_RETRIES + 1):
+            logger.info(f"Download attempt {attempt}/{MAX_DOWNLOAD_RETRIES}")
+            
+            # Download files
+            download_success = await self.download_files()
+            
+            if not download_success:
+                logger.error(f"Download failed on attempt {attempt}")
+                if attempt < MAX_DOWNLOAD_RETRIES:
+                    logger.info(f"Waiting {RETRY_DELAY_SECONDS * attempt}s before retry...")
+                    await asyncio.sleep(RETRY_DELAY_SECONDS * attempt)
+                    await self.cleanup_data_files()
+                    continue
+                return False
+            
+            # Unpack files
+            unpack_success = await self.unpack_files()
+            
+            if not unpack_success:
+                logger.error(f"Unpack failed on attempt {attempt}")
+                if attempt < MAX_DOWNLOAD_RETRIES:
+                    logger.info(f"Waiting {RETRY_DELAY_SECONDS * attempt}s before retry...")
+                    await asyncio.sleep(RETRY_DELAY_SECONDS * attempt)
+                    await self.cleanup_data_files()
+                    continue
+                return False
+            
+            # Validate data integrity
+            is_valid, issues = await self.validate_data_integrity()
+            
+            if is_valid:
+                logger.info(f"✓ Download and validation successful on attempt {attempt}")
+                return True
+            
+            logger.warning(f"Data validation failed on attempt {attempt} with {len(issues)} issues")
+            
+            if attempt < MAX_DOWNLOAD_RETRIES:
+                logger.info(f"Cleaning up and retrying in {RETRY_DELAY_SECONDS * attempt}s...")
+                await self.cleanup_data_files()
+                await asyncio.sleep(RETRY_DELAY_SECONDS * attempt)
+            else:
+                logger.error(f"Data validation failed after {MAX_DOWNLOAD_RETRIES} attempts")
+                return False
+        
+        return False
+
     async def download_files(self) -> bool:
         """Download all required MTGJSON files."""
         logger.info(f"Starting MTGJSON data download to {self.data_dir}")
@@ -627,8 +757,9 @@ if __name__ == "__main__":
             start_time = time.time()
             try:
                 if args.force or not await manager.get_last_refresh():
-                    await manager.download_files()
-                    await manager.unpack_files()
+                    if not await manager.download_with_sanity_check():
+                        logger.error("Failed to download MTGJSON data after retries")
+                        return
                 
                 counts = await manager.upsert_data()
                 duration = int(time.time() - start_time)

state.json

@@ -0,0 +1,30 @@
+{
+  "task_description": "MTG Online Backend - MTGJSON Data Integration & Docker Deployment",
+  "current_step": "Docker deployment and health check verification",
+  "files_created": [
+    "backend/app/services/mtgjson_manager.py"
+  ],
+  "files_modified": [
+    "backend/app/scripts/refresh_mtg.py",
+    "backend/app/main.py",
+    "docker-compose.yml"
+  ],
+  "decisions": [
+    "Created comprehensive MTGJSONManager service in app/services/",
+    "Manager handles download, unpack, upsert with ON CONFLICT DO UPDATE",
+    "Startup triggers initial download on first container init",
+    "Health check verifies MTG data exists in database",
+    "Refresh interval set to 7 days (weekly)",
+    "Docker compose start_period set to 600s for download time"
+  ],
+  "next_steps": [
+    "Test MTGJSON data download and upsert",
+    "Stop and destroy all Docker containers",
+    "Build backend Docker container",
+    "Deploy stack and verify health",
+    "Check logs for successful initialization"
+  ],
+  "blockers": [],
+  "commit_hash": "99c7d08",
+  "timestamp": 1784598234
+}