Double all timeout values to prevent download/upsert timeouts

2026-07-20 04:37:31 +00:00
parent 6b3fa63501
commit d347e0e586
13 changed files with 1070 additions and 29 deletions
@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+"""Check MTGJSON data status in filesystem and database."""
+
+import asyncio
+from pathlib import Path
+from sqlalchemy import create_engine, text
+from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
+import sys
+
+sys.path.append("/app")
+
+from app.services.mtgjson_manager import MTGJSONManager
+from app.config import get_settings
+
+async def check_mtgjson_status():
+    """Comprehensive check of MTGJSON data status."""
+    
+    print("=" * 60)
+    print("MTGJSON DATA STATUS REPORT")
+    print("=" * 60)
+    
+    settings = get_settings()
+    
+    # 1. Check data directory
+    print("\n1. DATA DIRECTORY CHECK")
+    print("-" * 40)
+    data_dir = Path(settings.DATA_DIR)
+    print(f"Data Directory: {data_dir}")
+    print(f"Directory exists: {data_dir.exists()}")
+    
+    if data_dir.exists():
+        files = list(data_dir.glob("*.json.gz")) + list(data_dir.glob("*.json"))
+        print(f"MTGJSON files found: {len(files)}")
+        
+        # Check specific files
+        required_files = [
+            "AllPrintings.json.gz",
+            "AllSetFiles.json.gz",
+            "AllIdentifiers.json.gz",
+            "CardTypes.json.gz",
+            "Keywords.json.gz",
+            "MagicRoots.json.gz",
+            "MagicSets.json.gz",
+            "SetTranslations.json.gz"
+        ]
+        
+        missing_files = []
+        existing_files = []
+        
+        for f in required_files:
+            filepath = data_dir / f
+            if filepath.exists():
+                size_mb = filepath.stat().st_size / (1024 * 1024)
+                existing_files.append((f, size_mb))
+                print(f"  ✓ {f}: {size_mb:.1f} MB")
+            else:
+                missing_files.append(f)
+                print(f"  ✗ {f}: MISSING")
+        
+        print(f"\n  Summary: {len(existing_files)}/{len(required_files)} required files present")
+        if missing_files:
+            print(f"  Missing: {', '.join(missing_files)}")
+    else:
+        print("  ERROR: Data directory does not exist!")
+    
+    # 2. Check database status
+    print("\n2. DATABASE STATUS CHECK")
+    print("-" * 40)
+    
+    db_url = f"postgresql+asyncpg://{settings.POSTGRES_USER}:{settings.POSTGRES_PASSWORD}"
+    db_url += f"@postgres-mtgdata:5432/{settings.POSTGRES_DB}"
+    
+    try:
+        engine = create_async_engine(db_url)
+        
+        async with AsyncSession(engine) as session:
+            # Check tables
+            result = await session.execute(text("""
+                SELECT table_name 
+                FROM information_schema.tables 
+                WHERE table_schema = 'public'
+                ORDER BY table_name;
+            """))
+            
+            tables = [row[0] for row in result.fetchall()]
+            print(f"Tables found: {len(tables)}")
+            for table in tables:
+                print(f"  - {table}")
+            
+            # Check key tables
+            print("\nKey table statistics:")
+            key_tables = ['mtg_set', 'mtg_card', 'mtg_identifiers', 'mtg_keywords', 'mtg_refresh_log']
+            
+            for table in key_tables:
+                if table in tables:
+                    result = await session.execute(text(f"SELECT COUNT(*) FROM {table}"))
+                    count = result.scalar()
+                    print(f"  {table}: {count:,} records")
+                else:
+                    print(f"  {table}: TABLE NOT FOUND")
+            
+            # Check refresh log
+            if 'mtg_refresh_log' in tables:
+                result = await session.execute(text("""
+                    SELECT refresh_type, status, created_at, error_message
+                    FROM mtg_refresh_log 
+                    ORDER BY created_at DESC 
+                    LIMIT 5;
+                """))
+                
+                rows = result.fetchall()
+                if rows:
+                    print("\nRecent refresh operations:")
+                    for row in rows:
+                        status_icon = "✓" if row[1] == 'SUCCESS' else "✗"
+                        print(f"  {status_icon} {row[0]}: {row[1]} at {row[2]}")
+                        if row[3]:
+                            print(f"    Error: {row[3]}")
+                else:
+                    print("\nNo refresh operations logged")
+    
+    except Exception as e:
+        print(f"ERROR: Could not connect to database: {e}")
+        return
+    
+    # 3. Check MTGJSON manager status
+    print("\n3. MTGJSON MANAGER STATUS")
+    print("-" * 40)
+    
+    try:
+        manager = MTGJSONManager()
+        status = manager.get_status()
+        
+        print(f"Status: {status['status']}")
+        if status.get('data'):
+            print(f"  Sets count: {status['data'].get('sets_count', 0)}")
+            print(f"  Cards count: {status['data'].get('cards_count', 0)}")
+            print(f"  Last refresh: {status['data'].get('last_refresh')}")
+    except Exception as e:
+        print(f"ERROR: Could not create MTGJSON manager: {e}")
+    
+    print("\n" + "=" * 60)
+
+if __name__ == "__main__":
+    asyncio.run(check_mtgjson_status())