#!/usr/bin/env python3 """ Comprehensive MTGJSON Data Verification Checks both: 1. MTGJSON data file downloads 2. PostgreSQL database upsert status """ import asyncio from pathlib import Path from sqlalchemy import create_engine, text from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession import sys sys.path.append("/app") from app.services.mtgjson_manager import MTGJSONManager from app.config import get_settings async def verify_data_download(): """Verify MTGJSON data files were downloaded.""" print("=" * 70) print("MTGJSON DATA DOWNLOAD VERIFICATION") print("=" * 70) settings = get_settings() data_dir = Path(settings.DATA_DIR) print(f"\nData Directory: {data_dir}") print(f"Directory exists: {data_dir.exists()}") if not data_dir.exists(): print("❌ FAIL: Data directory does not exist") return False # Check for required files required_files = { "AllPrintings.json.gz": "All sets data", "AllSetFiles.json.gz": "Set metadata", "AllIdentifiers.json.gz": "Card identifiers", "CardTypes.json.gz": "Card type definitions", "Keywords.json.gz": "Card keywords", "MagicRoots.json.gz": "Root data", "MagicSets.json.gz": "Set data", "SetTranslations.json.gz": "Set translations" } downloaded_files = [] missing_files = [] print("\nRequired MTGJSON files:") for filename, description in required_files.items(): filepath = data_dir / filename if filepath.exists(): size_mb = filepath.stat().st_size / (1024 * 1024) downloaded_files.append(filename) print(f" ✓ {filename:30} - {size_mb:8.1f} MB") else: missing_files.append(filename) print(f" ✗ {filename:30} - MISSING") print(f"\nDownload Status: {len(downloaded_files)}/{len(required_files)} files") if missing_files: print(f"\n❌ FAIL: Missing {len(missing_files)} required files: {', '.join(missing_files)}") return False # Check file sizes for sanity allprintings_path = data_dir / "AllPrintings.json.gz" if allprintings_path.exists(): size_mb = allprintings_path.stat().st_size / (1024 * 1024) if size_mb < 100: print(f"\n⚠️ WARNING: AllPrintings.json.gz is suspiciously small ({size_mb:.1f} MB). Expected ~500-600 MB") return False else: print(f"\n✓ AllPrintings.json.gz size looks good: {size_mb:.1f} MB") print("\n✓ PASS: All MTGJSON data files downloaded successfully") return True async def verify_database_upsert(): """Verify MTGJSON data was properly upserted to PostgreSQL.""" print("\n" + "=" * 70) print("POSTGRESQL DATABASE UPSERT VERIFICATION") print("=" * 70) settings = get_settings() db_url = f"postgresql+asyncpg://{settings.POSTGRES_USER}:{settings.POSTGRES_PASSWORD}" db_url += f"@postgres-mtgdata:5432/{settings.POSTGRES_DB}" print(f"\nDatabase: {settings.POSTGRES_DB}") try: engine = create_async_engine(db_url) async with AsyncSession(engine) as session: # Get all tables result = await session.execute(text(""" SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' ORDER BY table_name; """)) tables = [row[0] for row in result.fetchall()] print(f"\nTotal tables: {len(tables)}") # Check MTGJSON-specific tables mtg_tables = ['mtg_set', 'mtg_card', 'mtg_identifiers', 'mtg_keywords'] print("\nMTGJSON tables:") for table in mtg_tables: if table in tables: result = await session.execute(text(f"SELECT COUNT(*) FROM {table}")) count = result.scalar() print(f" ✓ {table:30} - {count:8,} records") else: print(f" ✗ {table:30} - TABLE NOT FOUND") # Check refresh log if 'mtg_refresh_log' in tables: result = await session.execute(text(""" SELECT refresh_type, status, created_at FROM mtg_refresh_log ORDER BY created_at DESC LIMIT 5; """)) rows = result.fetchall() if rows: print("\nRecent refresh operations:") for row in rows: status_icon = "✓" if row[1] == 'SUCCESS' else "✗" print(f" {status_icon} {row[0]:15} - {row[1]:8} - {row[2]}") # Verify data quality print("\nData quality checks:") # Check for sets if 'mtg_set' in tables: result = await session.execute(text(""" SELECT COUNT(*) FROM mtg_set WHERE set_name IS NOT NULL AND set_code IS NOT NULL; """)) valid_sets = result.scalar() result = await session.execute(text("SELECT COUNT(*) FROM mtg_set")) total_sets = result.scalar() print(f" ✓ Sets: {valid_sets:,}/{total_sets:,} valid") # Check for cards if 'mtg_card' in tables: result = await session.execute(text(""" SELECT COUNT(*) FROM mtg_card WHERE name IS NOT NULL AND mtgjson_cards_id IS NOT NULL; """)) valid_cards = result.scalar() result = await session.execute(text("SELECT COUNT(*) FROM mtg_card")) total_cards = result.scalar() print(f" ✓ Cards: {valid_cards:,}/{total_cards:,} valid") # Check for identifiers if 'mtg_identifiers' in tables: result = await session.execute(text(""" SELECT COUNT(*) FROM mtg_identifiers WHERE scryfall_id IS NOT NULL; """)) valid_ids = result.scalar() result = await session.execute(text("SELECT COUNT(*) FROM mtg_identifiers")) total_ids = result.scalar() print(f" ✓ Identifiers: {valid_ids:,}/{total_ids:,} valid") # Check for keywords if 'mtg_keywords' in tables: result = await session.execute(text("SELECT COUNT(*) FROM mtg_keywords")) keywords_count = result.scalar() print(f" ✓ Keywords: {keywords_count:,}") print("\n✓ PASS: Database upsert completed successfully") return True except Exception as e: print(f"\n❌ FAIL: Database error - {e}") return False async def main(): """Main verification function.""" print("\n" + "=" * 70) print("MTGJSON DATA INTEGRATION VERIFICATION") print("=" * 70) # Check data download download_ok = await verify_data_download() # Check database upsert db_ok = await verify_database_upsert() # Final summary print("\n" + "=" * 70) print("VERIFICATION SUMMARY") print("=" * 70) if download_ok and db_ok: print("\n✓✓✓ ALL CHECKS PASSED ✓✓✓") print("\nMTGJSON data has been successfully downloaded and upserted to PostgreSQL.") print("The backend is ready to use.") return 0 else: print("\n❌❌❌ VERIFICATION FAILED ❌❌❌") if not download_ok: print("\nDownload issues:") print(" - Some MTGJSON data files are missing or corrupted") print(" - Run: docker exec mtgonline_backend python /app/scripts/download_mtgjson_v5.py") if not db_ok: print("\nDatabase issues:") print(" - Data was not properly upserted to PostgreSQL") print(" - Check backend logs for errors") return 1 if __name__ == "__main__": exit_code = asyncio.run(main()) sys.exit(exit_code)