#!/usr/bin/env python3 """ MTGJSON Data Sanity Check Validates downloaded MTGJSON files for expected sizes before upserting to database. This prevents corrupted or incomplete data from being loaded into PostgreSQL. """ import asyncio import json import logging from pathlib import Path logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) # Expected minimum file sizes (in bytes) for MTGJSON v5 files # These are approximate minimums based on typical MTGJSON data sizes EXPECTED_MIN_SIZES = { "AllPrintings.json": 500 * 1024 * 1024, # 500 MB (should be 500-600 MB) "AllSetFiles.json": 10 * 1024 * 1024, # 10 MB "AllIdentifiers.json": 100 * 1024 * 1024, # 100 MB "CardTypes.json": 1 * 1024 * 1024, # 1 MB "Keywords.json": 0.5 * 1024 * 1024, # 0.5 MB "MagicSets.json": 50 * 1024 * 1024, # 50 MB "MagicRoots.json": 1 * 1024 * 1024, # 1 MB "SetTranslations.json": 5 * 1024 * 1024, # 5 MB } def validate_file_sizes(data_dir: Path) -> dict: """ Validate downloaded MTGJSON files for expected sizes. Args: data_dir: Path to the MTGJSON data directory Returns: Dict with validation results """ results = { "valid": True, "files_checked": 0, "files_valid": 0, "files_invalid": 0, "issues": [] } if not data_dir.exists(): results["valid"] = False results["issues"].append(f"Data directory does not exist: {data_dir}") return results # Check each expected file for filename, min_size in EXPECTED_MIN_SIZES.items(): filepath = data_dir / filename if not filepath.exists(): results["issues"].append(f"Missing file: {filename}") results["valid"] = False results["files_checked"] += 1 results["files_invalid"] += 1 continue results["files_checked"] += 1 actual_size = filepath.stat().st_size if actual_size < min_size: results["valid"] = False results["files_invalid"] += 1 results["issues"].append( f"{filename}: {actual_size / (1024*1024):.1f} MB (minimum: {min_size / (1024*1024):.1f} MB)" ) logger.warning( f"File {filename} is too small: {actual_size / (1024*1024):.1f} MB " f"(expected minimum: {min_size / (1024*1024):.1f} MB)" ) else: results["files_valid"] += 1 logger.info( f"✓ {filename}: {actual_size / (1024*1024):.1f} MB (OK)" ) return results async def validate_and_cleanup(data_dir: Path, max_retries: int = 3) -> bool: """ Validate MTGJSON files and cleanup if invalid. Args: data_dir: Path to the MTGJSON data directory max_retries: Maximum number of retry attempts Returns: True if validation passes, False otherwise """ logger.info("=" * 60) logger.info("MTGJSON Data Sanity Check") logger.info("=" * 60) for attempt in range(1, max_retries + 1): logger.info(f"\nAttempt {attempt}/{max_retries}") # Validate file sizes results = validate_file_sizes(data_dir) if results["valid"]: logger.info("\n✓ All files passed validation") logger.info(f" Checked: {results['files_checked']} files") logger.info(f" Valid: {results['files_valid']} files") return True # Validation failed logger.warning("\n✗ Validation failed:") for issue in results["issues"]: logger.warning(f" - {issue}") if attempt < max_retries: logger.warning(f"\nCleanup and retry in {60 * attempt} seconds...") await asyncio.sleep(60 * attempt) # Delete all downloaded files logger.warning("Deleting downloaded files...") for f in data_dir.glob("*"): if f.is_file(): f.unlink() logger.warning(f" Deleted: {f.name}") logger.error("\n✗✗✗ All retry attempts failed ✗✗✗") return False if __name__ == "__main__": import sys # Get data directory from settings or use default try: sys.path.insert(0, "/app") from app.config import get_settings settings = get_settings() data_dir = Path(settings.DATA_DIR) except Exception as e: logger.error(f"Failed to load settings: {e}") data_dir = Path("/app/data/mtgjson") # Run validation asyncio.run(validate_and_cleanup(data_dir))