Double all timeout values to prevent download/upsert timeouts
This commit is contained in:
@@ -0,0 +1,194 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
MTGJSON Data Verification Script
|
||||
|
||||
Checks:
|
||||
1. Data file download status
|
||||
2. Sanity check validation
|
||||
3. Database upsert status
|
||||
4. File size validation
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, "/app")
|
||||
|
||||
from app.config import get_settings
|
||||
from app.services.mtgjson_manager import MTGJSONManager
|
||||
|
||||
# Expected minimum file sizes (in bytes)
|
||||
EXPECTED_MIN_SIZES = {
|
||||
"AllPrintings.json": 500 * 1024 * 1024, # 500 MB
|
||||
"AllSetFiles.json": 10 * 1024 * 1024, # 10 MB
|
||||
"AllIdentifiers.json": 100 * 1024 * 1024, # 100 MB
|
||||
"CardTypes.json": 1 * 1024 * 1024, # 1 MB
|
||||
"Keywords.json": 0.5 * 1024 * 1024, # 0.5 MB
|
||||
"MagicSets.json": 50 * 1024 * 1024, # 50 MB
|
||||
"MagicRoots.json": 1 * 1024 * 1024, # 1 MB
|
||||
"SetTranslations.json": 5 * 1024 * 1024, # 5 MB
|
||||
}
|
||||
|
||||
|
||||
async def check_data_files(data_dir: Path) -> dict:
|
||||
"""Check downloaded data files and validate sizes."""
|
||||
print("=" * 70)
|
||||
print("DATA FILE VERIFICATION")
|
||||
print("=" * 70)
|
||||
|
||||
results = {
|
||||
"valid": True,
|
||||
"files_checked": 0,
|
||||
"files_valid": 0,
|
||||
"files_invalid": 0,
|
||||
"issues": []
|
||||
}
|
||||
|
||||
if not data_dir.exists():
|
||||
results["valid"] = False
|
||||
results["issues"].append(f"Data directory does not exist: {data_dir}")
|
||||
return results
|
||||
|
||||
# Check each expected file
|
||||
for filename, min_size in EXPECTED_MIN_SIZES.items():
|
||||
filepath = data_dir / filename
|
||||
|
||||
if not filepath.exists():
|
||||
results["issues"].append(f"Missing file: {filename}")
|
||||
results["valid"] = False
|
||||
results["files_checked"] += 1
|
||||
results["files_invalid"] += 1
|
||||
continue
|
||||
|
||||
results["files_checked"] += 1
|
||||
actual_size = filepath.stat().st_size
|
||||
|
||||
if actual_size < min_size:
|
||||
results["valid"] = False
|
||||
results["files_invalid"] += 1
|
||||
results["issues"].append(
|
||||
f"{filename}: {actual_size / (1024*1024):.1f} MB (minimum: {min_size / (1024*1024):.1f} MB)"
|
||||
)
|
||||
print(f" ✗ {filename:30} - {actual_size / (1024*1024):8.1f} MB (too small)")
|
||||
else:
|
||||
results["files_valid"] += 1
|
||||
print(f" ✓ {filename:30} - {actual_size / (1024*1024):8.1f} MB")
|
||||
|
||||
print(f"\nSummary: {results['files_valid']}/{results['files_checked']} files valid")
|
||||
|
||||
if results["issues"]:
|
||||
print("\nIssues:")
|
||||
for issue in results["issues"]:
|
||||
print(f" - {issue}")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
async def check_database_status() -> dict:
|
||||
"""Check database upsert status."""
|
||||
print("\n" + "=" * 70)
|
||||
print("DATABASE STATUS VERIFICATION")
|
||||
print("=" * 70)
|
||||
|
||||
settings = get_settings()
|
||||
db_url = f"postgresql+asyncpg://{settings.POSTGRES_USER}:{settings.POSTGRES_PASSWORD}"
|
||||
db_url += f"@postgres-mtgdata:5432/{settings.POSTGRES_DB}"
|
||||
|
||||
try:
|
||||
engine = create_async_engine(db_url)
|
||||
|
||||
async with AsyncSession(engine) as session:
|
||||
# Get all tables
|
||||
result = await session.execute(text("""
|
||||
SELECT table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'public'
|
||||
ORDER BY table_name;
|
||||
"""))
|
||||
tables = [row[0] for row in result.fetchall()]
|
||||
|
||||
print(f"Total tables: {len(tables)}")
|
||||
|
||||
# Check MTGJSON tables
|
||||
mtg_tables = ['mtg_set', 'mtg_card', 'mtg_identifiers', 'mtg_keywords']
|
||||
|
||||
for table in mtg_tables:
|
||||
if table in tables:
|
||||
result = await session.execute(text(f"SELECT COUNT(*) FROM {table}"))
|
||||
count = result.scalar()
|
||||
print(f" ✓ {table:30} - {count:8,} records")
|
||||
else:
|
||||
print(f" ✗ {table:30} - TABLE NOT FOUND")
|
||||
|
||||
# Check refresh log
|
||||
if 'mtg_refresh_log' in tables:
|
||||
result = await session.execute(text("""
|
||||
SELECT refresh_type, status, created_at
|
||||
FROM mtg_refresh_log
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 5;
|
||||
"""))
|
||||
rows = result.fetchall()
|
||||
|
||||
if rows:
|
||||
print("\nRecent refresh operations:")
|
||||
for row in rows:
|
||||
status_icon = "✓" if row[1] == 'SUCCESS' else "✗"
|
||||
print(f" {status_icon} {row[0]:15} - {row[1]:8} - {row[2]}")
|
||||
|
||||
return {"valid": True, "error": None}
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n✗ Database error: {e}")
|
||||
return {"valid": False, "error": str(e)}
|
||||
|
||||
|
||||
async def main():
|
||||
"""Main verification function."""
|
||||
print("\n" + "=" * 70)
|
||||
print("MTGJSON DATA INTEGRATION VERIFICATION")
|
||||
print("=" * 70)
|
||||
|
||||
settings = get_settings()
|
||||
data_dir = Path(settings.DATA_DIR)
|
||||
|
||||
# Check data files
|
||||
file_results = await check_data_files(data_dir)
|
||||
|
||||
# Check database
|
||||
db_results = await check_database_status()
|
||||
|
||||
# Final summary
|
||||
print("\n" + "=" * 70)
|
||||
print("VERIFICATION SUMMARY")
|
||||
print("=" * 70)
|
||||
|
||||
if file_results["valid"] and db_results["valid"]:
|
||||
print("\n✓✓✓ ALL CHECKS PASSED ✓✓✓")
|
||||
print("\nThe MTGJSON data has been properly downloaded and upserted.")
|
||||
return 0
|
||||
else:
|
||||
print("\n❌ VERIFICATION FAILED")
|
||||
|
||||
if not file_results["valid"]:
|
||||
print("\nData file issues:")
|
||||
for issue in file_results["issues"]:
|
||||
print(f" - {issue}")
|
||||
|
||||
if not db_results["valid"]:
|
||||
print(f"\nDatabase issues: {db_results['error']}")
|
||||
|
||||
print("\nSOLUTION:")
|
||||
print(" 1. Delete corrupted data files")
|
||||
print(" 2. Run fresh download with sanity checks:")
|
||||
print(" docker exec mtgonline_backend python /app/scripts/download_mtgjson_v5.py")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit_code = asyncio.run(main())
|
||||
sys.exit(exit_code)
|
||||
Reference in New Issue
Block a user