Double all timeout values to prevent download/upsert timeouts
This commit is contained in:
@@ -0,0 +1,226 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Comprehensive MTGJSON Data Verification
|
||||
|
||||
Checks both:
|
||||
1. MTGJSON data file downloads
|
||||
2. PostgreSQL database upsert status
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
||||
import sys
|
||||
|
||||
sys.path.append("/app")
|
||||
|
||||
from app.services.mtgjson_manager import MTGJSONManager
|
||||
from app.config import get_settings
|
||||
|
||||
|
||||
async def verify_data_download():
|
||||
"""Verify MTGJSON data files were downloaded."""
|
||||
print("=" * 70)
|
||||
print("MTGJSON DATA DOWNLOAD VERIFICATION")
|
||||
print("=" * 70)
|
||||
|
||||
settings = get_settings()
|
||||
data_dir = Path(settings.DATA_DIR)
|
||||
|
||||
print(f"\nData Directory: {data_dir}")
|
||||
print(f"Directory exists: {data_dir.exists()}")
|
||||
|
||||
if not data_dir.exists():
|
||||
print("❌ FAIL: Data directory does not exist")
|
||||
return False
|
||||
|
||||
# Check for required files
|
||||
required_files = {
|
||||
"AllPrintings.json.gz": "All sets data",
|
||||
"AllSetFiles.json.gz": "Set metadata",
|
||||
"AllIdentifiers.json.gz": "Card identifiers",
|
||||
"CardTypes.json.gz": "Card type definitions",
|
||||
"Keywords.json.gz": "Card keywords",
|
||||
"MagicRoots.json.gz": "Root data",
|
||||
"MagicSets.json.gz": "Set data",
|
||||
"SetTranslations.json.gz": "Set translations"
|
||||
}
|
||||
|
||||
downloaded_files = []
|
||||
missing_files = []
|
||||
|
||||
print("\nRequired MTGJSON files:")
|
||||
for filename, description in required_files.items():
|
||||
filepath = data_dir / filename
|
||||
if filepath.exists():
|
||||
size_mb = filepath.stat().st_size / (1024 * 1024)
|
||||
downloaded_files.append(filename)
|
||||
print(f" ✓ {filename:30} - {size_mb:8.1f} MB")
|
||||
else:
|
||||
missing_files.append(filename)
|
||||
print(f" ✗ {filename:30} - MISSING")
|
||||
|
||||
print(f"\nDownload Status: {len(downloaded_files)}/{len(required_files)} files")
|
||||
|
||||
if missing_files:
|
||||
print(f"\n❌ FAIL: Missing {len(missing_files)} required files: {', '.join(missing_files)}")
|
||||
return False
|
||||
|
||||
# Check file sizes for sanity
|
||||
allprintings_path = data_dir / "AllPrintings.json.gz"
|
||||
if allprintings_path.exists():
|
||||
size_mb = allprintings_path.stat().st_size / (1024 * 1024)
|
||||
if size_mb < 100:
|
||||
print(f"\n⚠️ WARNING: AllPrintings.json.gz is suspiciously small ({size_mb:.1f} MB). Expected ~500-600 MB")
|
||||
return False
|
||||
else:
|
||||
print(f"\n✓ AllPrintings.json.gz size looks good: {size_mb:.1f} MB")
|
||||
|
||||
print("\n✓ PASS: All MTGJSON data files downloaded successfully")
|
||||
return True
|
||||
|
||||
|
||||
async def verify_database_upsert():
|
||||
"""Verify MTGJSON data was properly upserted to PostgreSQL."""
|
||||
print("\n" + "=" * 70)
|
||||
print("POSTGRESQL DATABASE UPSERT VERIFICATION")
|
||||
print("=" * 70)
|
||||
|
||||
settings = get_settings()
|
||||
db_url = f"postgresql+asyncpg://{settings.POSTGRES_USER}:{settings.POSTGRES_PASSWORD}"
|
||||
db_url += f"@postgres-mtgdata:5432/{settings.POSTGRES_DB}"
|
||||
|
||||
print(f"\nDatabase: {settings.POSTGRES_DB}")
|
||||
|
||||
try:
|
||||
engine = create_async_engine(db_url)
|
||||
|
||||
async with AsyncSession(engine) as session:
|
||||
# Get all tables
|
||||
result = await session.execute(text("""
|
||||
SELECT table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'public'
|
||||
ORDER BY table_name;
|
||||
"""))
|
||||
tables = [row[0] for row in result.fetchall()]
|
||||
|
||||
print(f"\nTotal tables: {len(tables)}")
|
||||
|
||||
# Check MTGJSON-specific tables
|
||||
mtg_tables = ['mtg_set', 'mtg_card', 'mtg_identifiers', 'mtg_keywords']
|
||||
|
||||
print("\nMTGJSON tables:")
|
||||
for table in mtg_tables:
|
||||
if table in tables:
|
||||
result = await session.execute(text(f"SELECT COUNT(*) FROM {table}"))
|
||||
count = result.scalar()
|
||||
print(f" ✓ {table:30} - {count:8,} records")
|
||||
else:
|
||||
print(f" ✗ {table:30} - TABLE NOT FOUND")
|
||||
|
||||
# Check refresh log
|
||||
if 'mtg_refresh_log' in tables:
|
||||
result = await session.execute(text("""
|
||||
SELECT refresh_type, status, created_at
|
||||
FROM mtg_refresh_log
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 5;
|
||||
"""))
|
||||
rows = result.fetchall()
|
||||
|
||||
if rows:
|
||||
print("\nRecent refresh operations:")
|
||||
for row in rows:
|
||||
status_icon = "✓" if row[1] == 'SUCCESS' else "✗"
|
||||
print(f" {status_icon} {row[0]:15} - {row[1]:8} - {row[2]}")
|
||||
|
||||
# Verify data quality
|
||||
print("\nData quality checks:")
|
||||
|
||||
# Check for sets
|
||||
if 'mtg_set' in tables:
|
||||
result = await session.execute(text("""
|
||||
SELECT COUNT(*) FROM mtg_set
|
||||
WHERE set_name IS NOT NULL AND set_code IS NOT NULL;
|
||||
"""))
|
||||
valid_sets = result.scalar()
|
||||
result = await session.execute(text("SELECT COUNT(*) FROM mtg_set"))
|
||||
total_sets = result.scalar()
|
||||
print(f" ✓ Sets: {valid_sets:,}/{total_sets:,} valid")
|
||||
|
||||
# Check for cards
|
||||
if 'mtg_card' in tables:
|
||||
result = await session.execute(text("""
|
||||
SELECT COUNT(*) FROM mtg_card
|
||||
WHERE name IS NOT NULL AND mtgjson_cards_id IS NOT NULL;
|
||||
"""))
|
||||
valid_cards = result.scalar()
|
||||
result = await session.execute(text("SELECT COUNT(*) FROM mtg_card"))
|
||||
total_cards = result.scalar()
|
||||
print(f" ✓ Cards: {valid_cards:,}/{total_cards:,} valid")
|
||||
|
||||
# Check for identifiers
|
||||
if 'mtg_identifiers' in tables:
|
||||
result = await session.execute(text("""
|
||||
SELECT COUNT(*) FROM mtg_identifiers
|
||||
WHERE scryfall_id IS NOT NULL;
|
||||
"""))
|
||||
valid_ids = result.scalar()
|
||||
result = await session.execute(text("SELECT COUNT(*) FROM mtg_identifiers"))
|
||||
total_ids = result.scalar()
|
||||
print(f" ✓ Identifiers: {valid_ids:,}/{total_ids:,} valid")
|
||||
|
||||
# Check for keywords
|
||||
if 'mtg_keywords' in tables:
|
||||
result = await session.execute(text("SELECT COUNT(*) FROM mtg_keywords"))
|
||||
keywords_count = result.scalar()
|
||||
print(f" ✓ Keywords: {keywords_count:,}")
|
||||
|
||||
print("\n✓ PASS: Database upsert completed successfully")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ FAIL: Database error - {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def main():
|
||||
"""Main verification function."""
|
||||
print("\n" + "=" * 70)
|
||||
print("MTGJSON DATA INTEGRATION VERIFICATION")
|
||||
print("=" * 70)
|
||||
|
||||
# Check data download
|
||||
download_ok = await verify_data_download()
|
||||
|
||||
# Check database upsert
|
||||
db_ok = await verify_database_upsert()
|
||||
|
||||
# Final summary
|
||||
print("\n" + "=" * 70)
|
||||
print("VERIFICATION SUMMARY")
|
||||
print("=" * 70)
|
||||
|
||||
if download_ok and db_ok:
|
||||
print("\n✓✓✓ ALL CHECKS PASSED ✓✓✓")
|
||||
print("\nMTGJSON data has been successfully downloaded and upserted to PostgreSQL.")
|
||||
print("The backend is ready to use.")
|
||||
return 0
|
||||
else:
|
||||
print("\n❌❌❌ VERIFICATION FAILED ❌❌❌")
|
||||
if not download_ok:
|
||||
print("\nDownload issues:")
|
||||
print(" - Some MTGJSON data files are missing or corrupted")
|
||||
print(" - Run: docker exec mtgonline_backend python /app/scripts/download_mtgjson_v5.py")
|
||||
if not db_ok:
|
||||
print("\nDatabase issues:")
|
||||
print(" - Data was not properly upserted to PostgreSQL")
|
||||
print(" - Check backend logs for errors")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit_code = asyncio.run(main())
|
||||
sys.exit(exit_code)
|
||||
Reference in New Issue
Block a user