Double all timeout values to prevent download/upsert timeouts

2026-07-20 04:37:31 +00:00
parent 6b3fa63501
commit d347e0e586
13 changed files with 1070 additions and 29 deletions
@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+"""
+MTGJSON Data Sanity Check
+
+Validates downloaded MTGJSON files for expected sizes before upserting to database.
+This prevents corrupted or incomplete data from being loaded into PostgreSQL.
+"""
+
+import asyncio
+import json
+import logging
+from pathlib import Path
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Expected minimum file sizes (in bytes) for MTGJSON v5 files
+# These are approximate minimums based on typical MTGJSON data sizes
+EXPECTED_MIN_SIZES = {
+    "AllPrintings.json": 500 * 1024 * 1024,  # 500 MB (should be 500-600 MB)
+    "AllSetFiles.json": 10 * 1024 * 1024,     # 10 MB
+    "AllIdentifiers.json": 100 * 1024 * 1024, # 100 MB
+    "CardTypes.json": 1 * 1024 * 1024,        # 1 MB
+    "Keywords.json": 0.5 * 1024 * 1024,       # 0.5 MB
+    "MagicSets.json": 50 * 1024 * 1024,       # 50 MB
+    "MagicRoots.json": 1 * 1024 * 1024,       # 1 MB
+    "SetTranslations.json": 5 * 1024 * 1024,  # 5 MB
+}
+
+
+def validate_file_sizes(data_dir: Path) -> dict:
+    """
+    Validate downloaded MTGJSON files for expected sizes.
+    
+    Args:
+        data_dir: Path to the MTGJSON data directory
+        
+    Returns:
+        Dict with validation results
+    """
+    results = {
+        "valid": True,
+        "files_checked": 0,
+        "files_valid": 0,
+        "files_invalid": 0,
+        "issues": []
+    }
+    
+    if not data_dir.exists():
+        results["valid"] = False
+        results["issues"].append(f"Data directory does not exist: {data_dir}")
+        return results
+    
+    # Check each expected file
+    for filename, min_size in EXPECTED_MIN_SIZES.items():
+        filepath = data_dir / filename
+        
+        if not filepath.exists():
+            results["issues"].append(f"Missing file: {filename}")
+            results["valid"] = False
+            results["files_checked"] += 1
+            results["files_invalid"] += 1
+            continue
+        
+        results["files_checked"] += 1
+        actual_size = filepath.stat().st_size
+        
+        if actual_size < min_size:
+            results["valid"] = False
+            results["files_invalid"] += 1
+            results["issues"].append(
+                f"{filename}: {actual_size / (1024*1024):.1f} MB (minimum: {min_size / (1024*1024):.1f} MB)"
+            )
+            logger.warning(
+                f"File {filename} is too small: {actual_size / (1024*1024):.1f} MB "
+                f"(expected minimum: {min_size / (1024*1024):.1f} MB)"
+            )
+        else:
+            results["files_valid"] += 1
+            logger.info(
+                f"✓ {filename}: {actual_size / (1024*1024):.1f} MB (OK)"
+            )
+    
+    return results
+
+
+async def validate_and_cleanup(data_dir: Path, max_retries: int = 3) -> bool:
+    """
+    Validate MTGJSON files and cleanup if invalid.
+    
+    Args:
+        data_dir: Path to the MTGJSON data directory
+        max_retries: Maximum number of retry attempts
+        
+    Returns:
+        True if validation passes, False otherwise
+    """
+    logger.info("=" * 60)
+    logger.info("MTGJSON Data Sanity Check")
+    logger.info("=" * 60)
+    
+    for attempt in range(1, max_retries + 1):
+        logger.info(f"\nAttempt {attempt}/{max_retries}")
+        
+        # Validate file sizes
+        results = validate_file_sizes(data_dir)
+        
+        if results["valid"]:
+            logger.info("\n✓ All files passed validation")
+            logger.info(f"   Checked: {results['files_checked']} files")
+            logger.info(f"   Valid: {results['files_valid']} files")
+            return True
+        
+        # Validation failed
+        logger.warning("\n✗ Validation failed:")
+        for issue in results["issues"]:
+            logger.warning(f"   - {issue}")
+        
+        if attempt < max_retries:
+            logger.warning(f"\nCleanup and retry in {60 * attempt} seconds...")
+            await asyncio.sleep(60 * attempt)
+            
+            # Delete all downloaded files
+            logger.warning("Deleting downloaded files...")
+            for f in data_dir.glob("*"):
+                if f.is_file():
+                    f.unlink()
+                    logger.warning(f"  Deleted: {f.name}")
+        
+    logger.error("\n✗✗✗ All retry attempts failed ✗✗✗")
+    return False
+
+
+if __name__ == "__main__":
+    import sys
+    
+    # Get data directory from settings or use default
+    try:
+        sys.path.insert(0, "/app")
+        from app.config import get_settings
+        settings = get_settings()
+        data_dir = Path(settings.DATA_DIR)
+    except Exception as e:
+        logger.error(f"Failed to load settings: {e}")
+        data_dir = Path("/app/data/mtgjson")
+    
+    # Run validation
+    asyncio.run(validate_and_cleanup(data_dir))