227 lines
8.1 KiB
Python
227 lines
8.1 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Comprehensive MTGJSON Data Verification
|
|
|
|
Checks both:
|
|
1. MTGJSON data file downloads
|
|
2. PostgreSQL database upsert status
|
|
"""
|
|
|
|
import asyncio
|
|
from pathlib import Path
|
|
from sqlalchemy import create_engine, text
|
|
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
|
import sys
|
|
|
|
sys.path.append("/app")
|
|
|
|
from app.services.mtgjson_manager import MTGJSONManager
|
|
from app.config import get_settings
|
|
|
|
|
|
async def verify_data_download():
|
|
"""Verify MTGJSON data files were downloaded."""
|
|
print("=" * 70)
|
|
print("MTGJSON DATA DOWNLOAD VERIFICATION")
|
|
print("=" * 70)
|
|
|
|
settings = get_settings()
|
|
data_dir = Path(settings.DATA_DIR)
|
|
|
|
print(f"\nData Directory: {data_dir}")
|
|
print(f"Directory exists: {data_dir.exists()}")
|
|
|
|
if not data_dir.exists():
|
|
print("❌ FAIL: Data directory does not exist")
|
|
return False
|
|
|
|
# Check for required files
|
|
required_files = {
|
|
"AllPrintings.json.gz": "All sets data",
|
|
"AllSetFiles.json.gz": "Set metadata",
|
|
"AllIdentifiers.json.gz": "Card identifiers",
|
|
"CardTypes.json.gz": "Card type definitions",
|
|
"Keywords.json.gz": "Card keywords",
|
|
"MagicRoots.json.gz": "Root data",
|
|
"MagicSets.json.gz": "Set data",
|
|
"SetTranslations.json.gz": "Set translations"
|
|
}
|
|
|
|
downloaded_files = []
|
|
missing_files = []
|
|
|
|
print("\nRequired MTGJSON files:")
|
|
for filename, description in required_files.items():
|
|
filepath = data_dir / filename
|
|
if filepath.exists():
|
|
size_mb = filepath.stat().st_size / (1024 * 1024)
|
|
downloaded_files.append(filename)
|
|
print(f" ✓ {filename:30} - {size_mb:8.1f} MB")
|
|
else:
|
|
missing_files.append(filename)
|
|
print(f" ✗ {filename:30} - MISSING")
|
|
|
|
print(f"\nDownload Status: {len(downloaded_files)}/{len(required_files)} files")
|
|
|
|
if missing_files:
|
|
print(f"\n❌ FAIL: Missing {len(missing_files)} required files: {', '.join(missing_files)}")
|
|
return False
|
|
|
|
# Check file sizes for sanity
|
|
allprintings_path = data_dir / "AllPrintings.json.gz"
|
|
if allprintings_path.exists():
|
|
size_mb = allprintings_path.stat().st_size / (1024 * 1024)
|
|
if size_mb < 100:
|
|
print(f"\n⚠️ WARNING: AllPrintings.json.gz is suspiciously small ({size_mb:.1f} MB). Expected ~500-600 MB")
|
|
return False
|
|
else:
|
|
print(f"\n✓ AllPrintings.json.gz size looks good: {size_mb:.1f} MB")
|
|
|
|
print("\n✓ PASS: All MTGJSON data files downloaded successfully")
|
|
return True
|
|
|
|
|
|
async def verify_database_upsert():
|
|
"""Verify MTGJSON data was properly upserted to PostgreSQL."""
|
|
print("\n" + "=" * 70)
|
|
print("POSTGRESQL DATABASE UPSERT VERIFICATION")
|
|
print("=" * 70)
|
|
|
|
settings = get_settings()
|
|
db_url = f"postgresql+asyncpg://{settings.POSTGRES_USER}:{settings.POSTGRES_PASSWORD}"
|
|
db_url += f"@postgres-mtgdata:5432/{settings.POSTGRES_DB}"
|
|
|
|
print(f"\nDatabase: {settings.POSTGRES_DB}")
|
|
|
|
try:
|
|
engine = create_async_engine(db_url)
|
|
|
|
async with AsyncSession(engine) as session:
|
|
# Get all tables
|
|
result = await session.execute(text("""
|
|
SELECT table_name
|
|
FROM information_schema.tables
|
|
WHERE table_schema = 'public'
|
|
ORDER BY table_name;
|
|
"""))
|
|
tables = [row[0] for row in result.fetchall()]
|
|
|
|
print(f"\nTotal tables: {len(tables)}")
|
|
|
|
# Check MTGJSON-specific tables
|
|
mtg_tables = ['mtg_set', 'mtg_card', 'mtg_identifiers', 'mtg_keywords']
|
|
|
|
print("\nMTGJSON tables:")
|
|
for table in mtg_tables:
|
|
if table in tables:
|
|
result = await session.execute(text(f"SELECT COUNT(*) FROM {table}"))
|
|
count = result.scalar()
|
|
print(f" ✓ {table:30} - {count:8,} records")
|
|
else:
|
|
print(f" ✗ {table:30} - TABLE NOT FOUND")
|
|
|
|
# Check refresh log
|
|
if 'mtg_refresh_log' in tables:
|
|
result = await session.execute(text("""
|
|
SELECT refresh_type, status, created_at
|
|
FROM mtg_refresh_log
|
|
ORDER BY created_at DESC
|
|
LIMIT 5;
|
|
"""))
|
|
rows = result.fetchall()
|
|
|
|
if rows:
|
|
print("\nRecent refresh operations:")
|
|
for row in rows:
|
|
status_icon = "✓" if row[1] == 'SUCCESS' else "✗"
|
|
print(f" {status_icon} {row[0]:15} - {row[1]:8} - {row[2]}")
|
|
|
|
# Verify data quality
|
|
print("\nData quality checks:")
|
|
|
|
# Check for sets
|
|
if 'mtg_set' in tables:
|
|
result = await session.execute(text("""
|
|
SELECT COUNT(*) FROM mtg_set
|
|
WHERE set_name IS NOT NULL AND set_code IS NOT NULL;
|
|
"""))
|
|
valid_sets = result.scalar()
|
|
result = await session.execute(text("SELECT COUNT(*) FROM mtg_set"))
|
|
total_sets = result.scalar()
|
|
print(f" ✓ Sets: {valid_sets:,}/{total_sets:,} valid")
|
|
|
|
# Check for cards
|
|
if 'mtg_card' in tables:
|
|
result = await session.execute(text("""
|
|
SELECT COUNT(*) FROM mtg_card
|
|
WHERE name IS NOT NULL AND mtgjson_cards_id IS NOT NULL;
|
|
"""))
|
|
valid_cards = result.scalar()
|
|
result = await session.execute(text("SELECT COUNT(*) FROM mtg_card"))
|
|
total_cards = result.scalar()
|
|
print(f" ✓ Cards: {valid_cards:,}/{total_cards:,} valid")
|
|
|
|
# Check for identifiers
|
|
if 'mtg_identifiers' in tables:
|
|
result = await session.execute(text("""
|
|
SELECT COUNT(*) FROM mtg_identifiers
|
|
WHERE scryfall_id IS NOT NULL;
|
|
"""))
|
|
valid_ids = result.scalar()
|
|
result = await session.execute(text("SELECT COUNT(*) FROM mtg_identifiers"))
|
|
total_ids = result.scalar()
|
|
print(f" ✓ Identifiers: {valid_ids:,}/{total_ids:,} valid")
|
|
|
|
# Check for keywords
|
|
if 'mtg_keywords' in tables:
|
|
result = await session.execute(text("SELECT COUNT(*) FROM mtg_keywords"))
|
|
keywords_count = result.scalar()
|
|
print(f" ✓ Keywords: {keywords_count:,}")
|
|
|
|
print("\n✓ PASS: Database upsert completed successfully")
|
|
return True
|
|
|
|
except Exception as e:
|
|
print(f"\n❌ FAIL: Database error - {e}")
|
|
return False
|
|
|
|
|
|
async def main():
|
|
"""Main verification function."""
|
|
print("\n" + "=" * 70)
|
|
print("MTGJSON DATA INTEGRATION VERIFICATION")
|
|
print("=" * 70)
|
|
|
|
# Check data download
|
|
download_ok = await verify_data_download()
|
|
|
|
# Check database upsert
|
|
db_ok = await verify_database_upsert()
|
|
|
|
# Final summary
|
|
print("\n" + "=" * 70)
|
|
print("VERIFICATION SUMMARY")
|
|
print("=" * 70)
|
|
|
|
if download_ok and db_ok:
|
|
print("\n✓✓✓ ALL CHECKS PASSED ✓✓✓")
|
|
print("\nMTGJSON data has been successfully downloaded and upserted to PostgreSQL.")
|
|
print("The backend is ready to use.")
|
|
return 0
|
|
else:
|
|
print("\n❌❌❌ VERIFICATION FAILED ❌❌❌")
|
|
if not download_ok:
|
|
print("\nDownload issues:")
|
|
print(" - Some MTGJSON data files are missing or corrupted")
|
|
print(" - Run: docker exec mtgonline_backend python /app/scripts/download_mtgjson_v5.py")
|
|
if not db_ok:
|
|
print("\nDatabase issues:")
|
|
print(" - Data was not properly upserted to PostgreSQL")
|
|
print(" - Check backend logs for errors")
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
exit_code = asyncio.run(main())
|
|
sys.exit(exit_code)
|