149 lines
4.7 KiB
Python
149 lines
4.7 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
MTGJSON Data Sanity Check
|
|
|
|
Validates downloaded MTGJSON files for expected sizes before upserting to database.
|
|
This prevents corrupted or incomplete data from being loaded into PostgreSQL.
|
|
"""
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
from pathlib import Path
|
|
|
|
logging.basicConfig(level=logging.INFO)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Expected minimum file sizes (in bytes) for MTGJSON v5 files
|
|
# These are approximate minimums based on typical MTGJSON data sizes
|
|
EXPECTED_MIN_SIZES = {
|
|
"AllPrintings.json": 500 * 1024 * 1024, # 500 MB (should be 500-600 MB)
|
|
"AllSetFiles.json": 10 * 1024 * 1024, # 10 MB
|
|
"AllIdentifiers.json": 100 * 1024 * 1024, # 100 MB
|
|
"CardTypes.json": 1 * 1024 * 1024, # 1 MB
|
|
"Keywords.json": 0.5 * 1024 * 1024, # 0.5 MB
|
|
"MagicSets.json": 50 * 1024 * 1024, # 50 MB
|
|
"MagicRoots.json": 1 * 1024 * 1024, # 1 MB
|
|
"SetTranslations.json": 5 * 1024 * 1024, # 5 MB
|
|
}
|
|
|
|
|
|
def validate_file_sizes(data_dir: Path) -> dict:
|
|
"""
|
|
Validate downloaded MTGJSON files for expected sizes.
|
|
|
|
Args:
|
|
data_dir: Path to the MTGJSON data directory
|
|
|
|
Returns:
|
|
Dict with validation results
|
|
"""
|
|
results = {
|
|
"valid": True,
|
|
"files_checked": 0,
|
|
"files_valid": 0,
|
|
"files_invalid": 0,
|
|
"issues": []
|
|
}
|
|
|
|
if not data_dir.exists():
|
|
results["valid"] = False
|
|
results["issues"].append(f"Data directory does not exist: {data_dir}")
|
|
return results
|
|
|
|
# Check each expected file
|
|
for filename, min_size in EXPECTED_MIN_SIZES.items():
|
|
filepath = data_dir / filename
|
|
|
|
if not filepath.exists():
|
|
results["issues"].append(f"Missing file: {filename}")
|
|
results["valid"] = False
|
|
results["files_checked"] += 1
|
|
results["files_invalid"] += 1
|
|
continue
|
|
|
|
results["files_checked"] += 1
|
|
actual_size = filepath.stat().st_size
|
|
|
|
if actual_size < min_size:
|
|
results["valid"] = False
|
|
results["files_invalid"] += 1
|
|
results["issues"].append(
|
|
f"{filename}: {actual_size / (1024*1024):.1f} MB (minimum: {min_size / (1024*1024):.1f} MB)"
|
|
)
|
|
logger.warning(
|
|
f"File {filename} is too small: {actual_size / (1024*1024):.1f} MB "
|
|
f"(expected minimum: {min_size / (1024*1024):.1f} MB)"
|
|
)
|
|
else:
|
|
results["files_valid"] += 1
|
|
logger.info(
|
|
f"✓ {filename}: {actual_size / (1024*1024):.1f} MB (OK)"
|
|
)
|
|
|
|
return results
|
|
|
|
|
|
async def validate_and_cleanup(data_dir: Path, max_retries: int = 3) -> bool:
|
|
"""
|
|
Validate MTGJSON files and cleanup if invalid.
|
|
|
|
Args:
|
|
data_dir: Path to the MTGJSON data directory
|
|
max_retries: Maximum number of retry attempts
|
|
|
|
Returns:
|
|
True if validation passes, False otherwise
|
|
"""
|
|
logger.info("=" * 60)
|
|
logger.info("MTGJSON Data Sanity Check")
|
|
logger.info("=" * 60)
|
|
|
|
for attempt in range(1, max_retries + 1):
|
|
logger.info(f"\nAttempt {attempt}/{max_retries}")
|
|
|
|
# Validate file sizes
|
|
results = validate_file_sizes(data_dir)
|
|
|
|
if results["valid"]:
|
|
logger.info("\n✓ All files passed validation")
|
|
logger.info(f" Checked: {results['files_checked']} files")
|
|
logger.info(f" Valid: {results['files_valid']} files")
|
|
return True
|
|
|
|
# Validation failed
|
|
logger.warning("\n✗ Validation failed:")
|
|
for issue in results["issues"]:
|
|
logger.warning(f" - {issue}")
|
|
|
|
if attempt < max_retries:
|
|
logger.warning(f"\nCleanup and retry in {60 * attempt} seconds...")
|
|
await asyncio.sleep(60 * attempt)
|
|
|
|
# Delete all downloaded files
|
|
logger.warning("Deleting downloaded files...")
|
|
for f in data_dir.glob("*"):
|
|
if f.is_file():
|
|
f.unlink()
|
|
logger.warning(f" Deleted: {f.name}")
|
|
|
|
logger.error("\n✗✗✗ All retry attempts failed ✗✗✗")
|
|
return False
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import sys
|
|
|
|
# Get data directory from settings or use default
|
|
try:
|
|
sys.path.insert(0, "/app")
|
|
from app.config import get_settings
|
|
settings = get_settings()
|
|
data_dir = Path(settings.DATA_DIR)
|
|
except Exception as e:
|
|
logger.error(f"Failed to load settings: {e}")
|
|
data_dir = Path("/app/data/mtgjson")
|
|
|
|
# Run validation
|
|
asyncio.run(validate_and_cleanup(data_dir))
|