mtgonline/backend/scripts/verify_mtgjson_data.py

#!/usr/bin/env python3
"""
MTGJSON Data Verification Script

Checks:
1. Data file download status
2. Sanity check validation
3. Database upsert status
4. File size validation
"""

import asyncio
import json
from pathlib import Path
from sqlalchemy import create_engine, text
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
import sys

sys.path.insert(0, "/app")

from app.config import get_settings
from app.services.mtgjson_manager import MTGJSONManager

# Expected minimum file sizes (in bytes)
EXPECTED_MIN_SIZES = {
    "AllPrintings.json": 500 * 1024 * 1024,  # 500 MB
    "AllSetFiles.json": 10 * 1024 * 1024,     # 10 MB
    "AllIdentifiers.json": 100 * 1024 * 1024, # 100 MB
    "CardTypes.json": 1 * 1024 * 1024,        # 1 MB
    "Keywords.json": 0.5 * 1024 * 1024,       # 0.5 MB
    "MagicSets.json": 50 * 1024 * 1024,       # 50 MB
    "MagicRoots.json": 1 * 1024 * 1024,       # 1 MB
    "SetTranslations.json": 5 * 1024 * 1024,  # 5 MB
}


async def check_data_files(data_dir: Path) -> dict:
    """Check downloaded data files and validate sizes."""
    print("=" * 70)
    print("DATA FILE VERIFICATION")
    print("=" * 70)
    
    results = {
        "valid": True,
        "files_checked": 0,
        "files_valid": 0,
        "files_invalid": 0,
        "issues": []
    }
    
    if not data_dir.exists():
        results["valid"] = False
        results["issues"].append(f"Data directory does not exist: {data_dir}")
        return results
    
    # Check each expected file
    for filename, min_size in EXPECTED_MIN_SIZES.items():
        filepath = data_dir / filename
        
        if not filepath.exists():
            results["issues"].append(f"Missing file: {filename}")
            results["valid"] = False
            results["files_checked"] += 1
            results["files_invalid"] += 1
            continue
        
        results["files_checked"] += 1
        actual_size = filepath.stat().st_size
        
        if actual_size < min_size:
            results["valid"] = False
            results["files_invalid"] += 1
            results["issues"].append(
                f"{filename}: {actual_size / (1024*1024):.1f} MB (minimum: {min_size / (1024*1024):.1f} MB)"
            )
            print(f"  ✗ {filename:30} - {actual_size / (1024*1024):8.1f} MB (too small)")
        else:
            results["files_valid"] += 1
            print(f"  ✓ {filename:30} - {actual_size / (1024*1024):8.1f} MB")
    
    print(f"\nSummary: {results['files_valid']}/{results['files_checked']} files valid")
    
    if results["issues"]:
        print("\nIssues:")
        for issue in results["issues"]:
            print(f"  - {issue}")
    
    return results


async def check_database_status() -> dict:
    """Check database upsert status."""
    print("\n" + "=" * 70)
    print("DATABASE STATUS VERIFICATION")
    print("=" * 70)
    
    settings = get_settings()
    db_url = f"postgresql+asyncpg://{settings.POSTGRES_USER}:{settings.POSTGRES_PASSWORD}"
    db_url += f"@postgres-mtgdata:5432/{settings.POSTGRES_DB}"
    
    try:
        engine = create_async_engine(db_url)
        
        async with AsyncSession(engine) as session:
            # Get all tables
            result = await session.execute(text("""
                SELECT table_name 
                FROM information_schema.tables 
                WHERE table_schema = 'public'
                ORDER BY table_name;
            """))
            tables = [row[0] for row in result.fetchall()]
            
            print(f"Total tables: {len(tables)}")
            
            # Check MTGJSON tables
            mtg_tables = ['mtg_set', 'mtg_card', 'mtg_identifiers', 'mtg_keywords']
            
            for table in mtg_tables:
                if table in tables:
                    result = await session.execute(text(f"SELECT COUNT(*) FROM {table}"))
                    count = result.scalar()
                    print(f"  ✓ {table:30} - {count:8,} records")
                else:
                    print(f"  ✗ {table:30} - TABLE NOT FOUND")
            
            # Check refresh log
            if 'mtg_refresh_log' in tables:
                result = await session.execute(text("""
                    SELECT refresh_type, status, created_at
                    FROM mtg_refresh_log 
                    ORDER BY created_at DESC 
                    LIMIT 5;
                """))
                rows = result.fetchall()
                
                if rows:
                    print("\nRecent refresh operations:")
                    for row in rows:
                        status_icon = "✓" if row[1] == 'SUCCESS' else "✗"
                        print(f"  {status_icon} {row[0]:15} - {row[1]:8} - {row[2]}")
        
        return {"valid": True, "error": None}
        
    except Exception as e:
        print(f"\n✗ Database error: {e}")
        return {"valid": False, "error": str(e)}


async def main():
    """Main verification function."""
    print("\n" + "=" * 70)
    print("MTGJSON DATA INTEGRATION VERIFICATION")
    print("=" * 70)
    
    settings = get_settings()
    data_dir = Path(settings.DATA_DIR)
    
    # Check data files
    file_results = await check_data_files(data_dir)
    
    # Check database
    db_results = await check_database_status()
    
    # Final summary
    print("\n" + "=" * 70)
    print("VERIFICATION SUMMARY")
    print("=" * 70)
    
    if file_results["valid"] and db_results["valid"]:
        print("\n✓✓✓ ALL CHECKS PASSED ✓✓✓")
        print("\nThe MTGJSON data has been properly downloaded and upserted.")
        return 0
    else:
        print("\n❌ VERIFICATION FAILED")
        
        if not file_results["valid"]:
            print("\nData file issues:")
            for issue in file_results["issues"]:
                print(f"  - {issue}")
        
        if not db_results["valid"]:
            print(f"\nDatabase issues: {db_results['error']}")
        
        print("\nSOLUTION:")
        print("  1. Delete corrupted data files")
        print("  2. Run fresh download with sanity checks:")
        print("     docker exec mtgonline_backend python /app/scripts/download_mtgjson_v5.py")
        return 1


if __name__ == "__main__":
    exit_code = asyncio.run(main())
    sys.exit(exit_code)