Double all timeout values to prevent download/upsert timeouts

backend/Dockerfile

@@ -45,7 +45,7 @@ ENV PYTHONPATH=/app
 ENV PYTHONUNBUFFERED=1
 ENV PYTHONDONTWRITEBYTECODE=1
 
-HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
+HEALTHCHECK --interval=30s --timeout=20s --start-period=5s --retries=3 \
     CMD curl -f http://localhost:8000/health || exit 1
 
 USER appuser

backend/app/core/redis_client.py

@@ -22,8 +22,8 @@ async def get_redis() -> aioredis.Redis:
             redis_client = aioredis.from_url(
                 settings.REDIS_URL,
                 decode_responses=True,
-                socket_connect_timeout=5,
+                socket_connect_timeout=10,
-                socket_timeout=5,
+                socket_timeout=10,
             )
             await redis_client.ping()
             logger.info("Connected to Redis")

backend/app/services/mtgjson_manager.py

@@ -66,7 +66,7 @@ EXPECTED_MIN_SIZES = {
 }
 
 MAX_DOWNLOAD_RETRIES = 3
-RETRY_DELAY_SECONDS = 60
+RETRY_DELAY_SECONDS = 120
 
 
 class MTGJSONManager:
@@ -123,6 +123,19 @@ class MTGJSONManager:
         logger.info("Data integrity check passed - all files meet minimum size requirements")
         return True, []
     
+    def _get_estimated_size(self, filename: str) -> int:
+        """Get estimated file size in MB."""
+        estimates = {
+            "AllPrintings.json.gz": 500,
+            "AllSetFiles.zip": 10,
+            "AllIdentifiers.json.gz": 100,
+            "CardTypes.json.gz": 5,
+            "DeckList.json.gz": 5,
+            "Keywords.json.gz": 2,
+            "SetList.json.gz": 10,
+        }
+        return estimates.get(filename, 10)  # Default 10MB
+    
     async def cleanup_data_files(self) -> None:
         """Delete all downloaded MTGJSON data files."""
         logger.info(f"Cleaning up data files in {self.data_dir}")
@@ -215,7 +228,7 @@ class MTGJSONManager:
                 return False
 
     async def _download_file(self, session: aiohttp.ClientSession, url: str, filename: str) -> bool:
-        """Download a single file."""
+        """Download a single file with adaptive timeout based on size."""
         dest_path = self.data_dir / filename
         
         # Skip if file already exists
@@ -223,26 +236,51 @@ class MTGJSONManager:
             logger.info(f"Skipping {filename} (already exists)")
             return True
         
-        logger.info(f"Downloading {filename} from {url}")
+        # Calculate timeout based on estimated file size
+        # Large files (>100MB) get up to 60 minutes
+        estimated_mb = self._get_estimated_size(filename)
+        timeout = max(1200, estimated_mb * 8)  # At least 20 min, 8x estimated MB
+        
+        logger.info(f"Downloading {filename} from {url} (timeout: {timeout}s, est: {estimated_mb}MB)")
         
         try:
-            async with session.get(url) as response:
+            # Set timeout for connection AND download
+            timeout_obj = aiohttp.ClientTimeout(total=timeout)
+            
+            async with session.get(url, timeout=timeout_obj) as response:
                 if response.status != 200:
                     logger.error(f"Failed to download {filename}: HTTP {response.status}")
                     return False
                 
-                # Download with progress logging
+                # Download with progress monitoring
-                content = b""
+                total_size = 0
-                async for chunk in response.content.iter_chunked(8192):
+                last_log_time = time.time()
-                    content += chunk
                 
-                # Write to file
+                with open(dest_path, 'wb') as f:
-                dest_path.write_bytes(content)
+                    async for chunk in response.content.iter_chunked(8192):
-                logger.info(f"Downloaded {filename} ({len(content)} bytes)")
+                        f.write(chunk)
+                        total_size += len(chunk)
+                        
+                        # Log progress every 60 seconds
+                        now = time.time()
+                        if now - last_log_time >= 60:
+                            mb_downloaded = total_size / (1024 * 1024)
+                            logger.info(f"  Downloaded {mb_downloaded:.1f} MB of {filename}")
+                            last_log_time = now
+                
+                logger.info(f"Downloaded {filename} ({total_size / (1024*1024):.1f} MB)")
                 return True
                 
+        except asyncio.TimeoutError:
+            logger.error(f"Download timeout for {filename} (timeout: {timeout}s)")
+            # Clean up partial download
+            if dest_path.exists():
+                dest_path.unlink()
+            return False
         except Exception as e:
             logger.error(f"Download error for {filename}: {e}")
+            if dest_path.exists():
+                dest_path.unlink()
             return False
 
     async def unpack_files(self) -> bool:

backend/scripts/check_mtgjson_full.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+"""Check MTGJSON data status in filesystem and database."""
+
+import asyncio
+from pathlib import Path
+from sqlalchemy import create_engine, text
+from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
+import sys
+
+sys.path.append("/app")
+
+from app.services.mtgjson_manager import MTGJSONManager
+from app.config import get_settings
+
+async def check_mtgjson_status():
+    """Comprehensive check of MTGJSON data status."""
+    
+    print("=" * 60)
+    print("MTGJSON DATA STATUS REPORT")
+    print("=" * 60)
+    
+    settings = get_settings()
+    
+    # 1. Check data directory
+    print("\n1. DATA DIRECTORY CHECK")
+    print("-" * 40)
+    data_dir = Path(settings.DATA_DIR)
+    print(f"Data Directory: {data_dir}")
+    print(f"Directory exists: {data_dir.exists()}")
+    
+    if data_dir.exists():
+        files = list(data_dir.glob("*.json.gz")) + list(data_dir.glob("*.json"))
+        print(f"MTGJSON files found: {len(files)}")
+        
+        # Check specific files
+        required_files = [
+            "AllPrintings.json.gz",
+            "AllSetFiles.json.gz",
+            "AllIdentifiers.json.gz",
+            "CardTypes.json.gz",
+            "Keywords.json.gz",
+            "MagicRoots.json.gz",
+            "MagicSets.json.gz",
+            "SetTranslations.json.gz"
+        ]
+        
+        missing_files = []
+        existing_files = []
+        
+        for f in required_files:
+            filepath = data_dir / f
+            if filepath.exists():
+                size_mb = filepath.stat().st_size / (1024 * 1024)
+                existing_files.append((f, size_mb))
+                print(f"  ✓ {f}: {size_mb:.1f} MB")
+            else:
+                missing_files.append(f)
+                print(f"  ✗ {f}: MISSING")
+        
+        print(f"\n  Summary: {len(existing_files)}/{len(required_files)} required files present")
+        if missing_files:
+            print(f"  Missing: {', '.join(missing_files)}")
+    else:
+        print("  ERROR: Data directory does not exist!")
+    
+    # 2. Check database status
+    print("\n2. DATABASE STATUS CHECK")
+    print("-" * 40)
+    
+    db_url = f"postgresql+asyncpg://{settings.POSTGRES_USER}:{settings.POSTGRES_PASSWORD}"
+    db_url += f"@postgres-mtgdata:5432/{settings.POSTGRES_DB}"
+    
+    try:
+        engine = create_async_engine(db_url)
+        
+        async with AsyncSession(engine) as session:
+            # Check tables
+            result = await session.execute(text("""
+                SELECT table_name 
+                FROM information_schema.tables 
+                WHERE table_schema = 'public'
+                ORDER BY table_name;
+            """))
+            
+            tables = [row[0] for row in result.fetchall()]
+            print(f"Tables found: {len(tables)}")
+            for table in tables:
+                print(f"  - {table}")
+            
+            # Check key tables
+            print("\nKey table statistics:")
+            key_tables = ['mtg_set', 'mtg_card', 'mtg_identifiers', 'mtg_keywords', 'mtg_refresh_log']
+            
+            for table in key_tables:
+                if table in tables:
+                    result = await session.execute(text(f"SELECT COUNT(*) FROM {table}"))
+                    count = result.scalar()
+                    print(f"  {table}: {count:,} records")
+                else:
+                    print(f"  {table}: TABLE NOT FOUND")
+            
+            # Check refresh log
+            if 'mtg_refresh_log' in tables:
+                result = await session.execute(text("""
+                    SELECT refresh_type, status, created_at, error_message
+                    FROM mtg_refresh_log 
+                    ORDER BY created_at DESC 
+                    LIMIT 5;
+                """))
+                
+                rows = result.fetchall()
+                if rows:
+                    print("\nRecent refresh operations:")
+                    for row in rows:
+                        status_icon = "✓" if row[1] == 'SUCCESS' else "✗"
+                        print(f"  {status_icon} {row[0]}: {row[1]} at {row[2]}")
+                        if row[3]:
+                            print(f"    Error: {row[3]}")
+                else:
+                    print("\nNo refresh operations logged")
+    
+    except Exception as e:
+        print(f"ERROR: Could not connect to database: {e}")
+        return
+    
+    # 3. Check MTGJSON manager status
+    print("\n3. MTGJSON MANAGER STATUS")
+    print("-" * 40)
+    
+    try:
+        manager = MTGJSONManager()
+        status = manager.get_status()
+        
+        print(f"Status: {status['status']}")
+        if status.get('data'):
+            print(f"  Sets count: {status['data'].get('sets_count', 0)}")
+            print(f"  Cards count: {status['data'].get('cards_count', 0)}")
+            print(f"  Last refresh: {status['data'].get('last_refresh')}")
+    except Exception as e:
+        print(f"ERROR: Could not create MTGJSON manager: {e}")
+    
+    print("\n" + "=" * 60)
+
+if __name__ == "__main__":
+    asyncio.run(check_mtgjson_status())

backend/scripts/check_mtgjson_status.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+"""Check MTGJSON data status in the database and filesystem."""
+
+import asyncio
+import os
+from pathlib import Path
+from sqlalchemy import create_engine, text
+from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
+import sys
+sys.path.append("/home/wall-o/projects/mtgonline/backend")
+
+from app.services.mtgjson_manager import MTGJSONManager
+from app.config import get_settings
+
+async def check_mtgjson_status():
+    """Check MTGJSON data download and database status."""
+    
+    print("=== MTGJSON Data Status Check ===\n")
+    
+    # Check data directory
+    settings = get_settings()
+    data_dir = Path(settings.DATA_DIR)
+    
+    print(f"Data Directory: {data_dir}")
+    print(f"Directory exists: {data_dir.exists()}")
+    
+    if data_dir.exists():
+        files = list(data_dir.glob("*.json.gz"))
+        files += list(data_dir.glob("*.json"))
+        print(f"Found {len(files)} MTGJSON files:")
+        for f in sorted(files)[:20]:  # Show first 20
+            size_mb = f.stat().st_size / (1024 * 1024)
+            print(f"  - {f.name} ({size_mb:.1f} MB)")
+        if len(files) > 20:
+            print(f"  ... and {len(files) - 20} more files")
+    else:
+        print("WARNING: Data directory does not exist!")
+    
+    # Check database status
+    print("\n=== Database Status ===")
+    
+    try:
+        # Try to connect to the MTGJSON database
+        engine = create_async_engine(
+            f"postgresql+asyncpg://{settings.POSTGRES_USER}:{settings.POSTGRES_PASSWORD}"
+            f"@postgres-mtgdata:5432/{settings.POSTGRES_DB}"
+        )
+        
+        async with AsyncSession(engine) as session:
+            # Check if tables exist
+            result = await session.execute(text("""
+                SELECT table_name 
+                FROM information_schema.tables 
+                WHERE table_schema = 'public'
+                ORDER BY table_name;
+            """))
+            
+            tables = [row[0] for row in result.fetchall()]
+            print(f"Found {len(tables)} tables in database:")
+            for table in tables:
+                print(f"  - {table}")
+            
+            # Check specific MTGJSON tables
+            mtg_tables = ['mtg_set', 'mtg_card', 'mtg_identifiers', 'mtg_keywords']
+            if 'mtg_refresh_log' in tables:
+                result = await session.execute(text("SELECT COUNT(*) FROM mtg_refresh_log"))
+                count = result.scalar()
+                print(f"\nRefresh log entries: {count}")
+            
+            # Check key tables
+            for table in ['mtg_set', 'mtg_card', 'mtg_identifiers']:
+                if table in tables:
+                    result = await session.execute(text(f"SELECT COUNT(*) FROM {table}"))
+                    count = result.scalar()
+                    print(f"{table}: {count:,} records")
+                
+    except Exception as e:
+        print(f"ERROR connecting to database: {e}")
+    
+    # Try to create MTGJSON manager and check status
+    print("\n=== MTGJSON Manager Status ===")
+    try:
+        manager = MTGJSONManager()
+        status = manager.get_status()
+        print(f"Status: {status}")
+    except Exception as e:
+        print(f"ERROR creating MTGJSON manager: {e}")
+
+if __name__ == "__main__":
+    asyncio.run(check_mtgjson_status())

backend/scripts/download_mtgjson_v5.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+"""
+MTGJSON v5 Data Downloader
+
+Downloads MTGJSON v5 data files and loads them into the database.
+Updated to use current MTGJSON API endpoints.
+
+Usage:
+    python download_mtgjson_v5.py
+"""
+
+import sys
+from pathlib import Path
+
+# Add the app directory to Python path
+app_dir = Path(__file__).parent.parent
+sys.path.insert(0, str(app_dir))
+
+import asyncio
+import json
+import gzip
+import logging
+from sqlalchemy import create_engine, text
+from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
+import aiohttp
+
+from app.services.mtgjson_manager import MTGJSONManager
+from app.config import get_settings
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# MTGJSON v5 API base URL
+MTGJSON_API_BASE = "https://mtgjson.com/api/v5"
+
+
+async def download_file(session, url, dest_path):
+    """Download a file from URL."""
+    logger.info(f"Downloading: {url}")
+    
+    try:
+        async with session.get(url, timeout=aiohttp.ClientTimeout(total=600)) as response:
+            if response.status == 200:
+                dest_path.parent.mkdir(parents=True, exist_ok=True)
+                with open(dest_path, 'wb') as f:
+                    async for chunk in response.content.iter_chunked(8192):
+                        f.write(chunk)
+                logger.info(f"✓ Downloaded: {dest_path.name}")
+                return True
+            else:
+                logger.error(f"✗ Failed to download {url}: HTTP {response.status}")
+                return False
+    except Exception as e:
+        logger.error(f"✗ Error downloading {url}: {e}")
+        return False
+
+
+def extract_gzip(gz_path, dest_path=None):
+    """Extract a gzip file."""
+    if not dest_path:
+        dest_path = gz_path.with_suffix('')
+    
+    logger.info(f"Extracting: {gz_path.name}")
+    
+    try:
+        with gzip.open(gz_path, 'rt', encoding='utf-8') as f_in:
+            with open(dest_path, 'w', encoding='utf-8') as f_out:
+                f_out.write(f_in.read())
+        logger.info(f"✓ Extracted: {gz_path.name}")
+        # Remove gz file after extraction
+        gz_path.unlink()
+        return True
+    except Exception as e:
+        logger.error(f"✗ Failed to extract {gz_path}: {e}")
+        return False
+
+
+async def download_all_files(data_dir):
+    """Download all MTGJSON v5 files."""
+    logger.info("=" * 60)
+    logger.info("MTGJSON v5 Data Download")
+    logger.info("=" * 60)
+    
+    files = [
+        "AllPrintings.json.gz",
+        "AllSetFiles.json.gz",
+        "AllIdentifiers.json.gz",
+        "CardTypes.json.gz",
+        "Keywords.json.gz",
+        "MagicRoots.json.gz",
+        "MagicSets.json.gz",
+        "SetTranslations.json.gz"
+    ]
+    
+    async with aiohttp.ClientSession() as session:
+        for filename in files:
+            url = f"{MTGJSON_API_BASE}/{filename}"
+            dest_path = data_dir / filename
+            await download_file(session, url, dest_path)
+            await asyncio.sleep(1)  # Be nice to the API
+
+
+async def load_data_to_database(data_dir):
+    """Load downloaded MTGJSON data into PostgreSQL."""
+    logger.info("=" * 60)
+    logger.info("Loading MTGJSON data into database")
+    logger.info("=" * 60)
+    
+    manager = MTGJSONManager(data_dir)
+    
+    # Download files
+    success = await manager.download_files()
+    
+    if not success:
+        logger.error("✗ Download failed")
+        return False
+    
+    # Unpack files
+    success = manager.unpack_files()
+    
+    if not success:
+        logger.error("✗ Unpack failed")
+        return False
+    
+    # Upsert to database
+    success = manager.upsert_to_database()
+    
+    if success:
+        logger.info("✓ Data loaded successfully")
+    else:
+        logger.error("✗ Database upsert failed")
+    
+    return success
+
+
+async def main():
+    """Main entry point."""
+    settings = get_settings()
+    data_dir = Path(settings.DATA_DIR)
+    
+    logger.info(f"Data directory: {data_dir}")
+    
+    # Clear corrupted data
+    if data_dir.exists():
+        logger.info("Clearing corrupted data...")
+        for f in data_dir.glob("*.json.gz"):
+            f.unlink()
+            logger.info(f"  Removed: {f.name}")
+    
+    data_dir.mkdir(parents=True, exist_ok=True)
+    
+    # Download and load data
+    await load_data_to_database(data_dir)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

backend/scripts/maintain_mtgjson.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+# MTGJSON Data Maintenance Script
+# This script handles data download, validation, and cleanup
+set -e
+
+echo "=== MTGJSON Data Maintenance ==="
+echo "Starting maintenance at: $(date)"
+
+# Run the download with sanity checks
+echo "Running download with sanity checks..."
+docker exec mtgonline_backend python -m app.services.mtgjson_manager --refresh --force
+
+# Verify the download
+echo ""
+echo "Verifying data integrity..."
+docker exec mtgonline_backend python -c "
+import sys
+sys.path.insert(0, '/app')
+from app.services.mtgjson_manager import MTGJSONManager
+from pathlib import Path
+import asyncio
+
+async def verify():
+    manager = MTGJSONManager(Path('/app/data/mtgjson'))
+    is_valid, issues = await manager.validate_data_integrity()
+    
+    if is_valid:
+        print('✓ Data integrity check passed')
+        return 0
+    else:
+        print('✗ Data integrity check failed')
+        for issue in issues:
+            print(f'  - {issue}')
+        return 1
+
+sys.exit(asyncio.run(verify()))
+"
+
+echo ""
+echo "=== Maintenance Complete ==="

backend/scripts/sanity_check_mtgjson.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+"""
+MTGJSON Data Sanity Check
+
+Validates downloaded MTGJSON files for expected sizes before upserting to database.
+This prevents corrupted or incomplete data from being loaded into PostgreSQL.
+"""
+
+import asyncio
+import json
+import logging
+from pathlib import Path
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Expected minimum file sizes (in bytes) for MTGJSON v5 files
+# These are approximate minimums based on typical MTGJSON data sizes
+EXPECTED_MIN_SIZES = {
+    "AllPrintings.json": 500 * 1024 * 1024,  # 500 MB (should be 500-600 MB)
+    "AllSetFiles.json": 10 * 1024 * 1024,     # 10 MB
+    "AllIdentifiers.json": 100 * 1024 * 1024, # 100 MB
+    "CardTypes.json": 1 * 1024 * 1024,        # 1 MB
+    "Keywords.json": 0.5 * 1024 * 1024,       # 0.5 MB
+    "MagicSets.json": 50 * 1024 * 1024,       # 50 MB
+    "MagicRoots.json": 1 * 1024 * 1024,       # 1 MB
+    "SetTranslations.json": 5 * 1024 * 1024,  # 5 MB
+}
+
+
+def validate_file_sizes(data_dir: Path) -> dict:
+    """
+    Validate downloaded MTGJSON files for expected sizes.
+    
+    Args:
+        data_dir: Path to the MTGJSON data directory
+        
+    Returns:
+        Dict with validation results
+    """
+    results = {
+        "valid": True,
+        "files_checked": 0,
+        "files_valid": 0,
+        "files_invalid": 0,
+        "issues": []
+    }
+    
+    if not data_dir.exists():
+        results["valid"] = False
+        results["issues"].append(f"Data directory does not exist: {data_dir}")
+        return results
+    
+    # Check each expected file
+    for filename, min_size in EXPECTED_MIN_SIZES.items():
+        filepath = data_dir / filename
+        
+        if not filepath.exists():
+            results["issues"].append(f"Missing file: {filename}")
+            results["valid"] = False
+            results["files_checked"] += 1
+            results["files_invalid"] += 1
+            continue
+        
+        results["files_checked"] += 1
+        actual_size = filepath.stat().st_size
+        
+        if actual_size < min_size:
+            results["valid"] = False
+            results["files_invalid"] += 1
+            results["issues"].append(
+                f"{filename}: {actual_size / (1024*1024):.1f} MB (minimum: {min_size / (1024*1024):.1f} MB)"
+            )
+            logger.warning(
+                f"File {filename} is too small: {actual_size / (1024*1024):.1f} MB "
+                f"(expected minimum: {min_size / (1024*1024):.1f} MB)"
+            )
+        else:
+            results["files_valid"] += 1
+            logger.info(
+                f"✓ {filename}: {actual_size / (1024*1024):.1f} MB (OK)"
+            )
+    
+    return results
+
+
+async def validate_and_cleanup(data_dir: Path, max_retries: int = 3) -> bool:
+    """
+    Validate MTGJSON files and cleanup if invalid.
+    
+    Args:
+        data_dir: Path to the MTGJSON data directory
+        max_retries: Maximum number of retry attempts
+        
+    Returns:
+        True if validation passes, False otherwise
+    """
+    logger.info("=" * 60)
+    logger.info("MTGJSON Data Sanity Check")
+    logger.info("=" * 60)
+    
+    for attempt in range(1, max_retries + 1):
+        logger.info(f"\nAttempt {attempt}/{max_retries}")
+        
+        # Validate file sizes
+        results = validate_file_sizes(data_dir)
+        
+        if results["valid"]:
+            logger.info("\n✓ All files passed validation")
+            logger.info(f"   Checked: {results['files_checked']} files")
+            logger.info(f"   Valid: {results['files_valid']} files")
+            return True
+        
+        # Validation failed
+        logger.warning("\n✗ Validation failed:")
+        for issue in results["issues"]:
+            logger.warning(f"   - {issue}")
+        
+        if attempt < max_retries:
+            logger.warning(f"\nCleanup and retry in {60 * attempt} seconds...")
+            await asyncio.sleep(60 * attempt)
+            
+            # Delete all downloaded files
+            logger.warning("Deleting downloaded files...")
+            for f in data_dir.glob("*"):
+                if f.is_file():
+                    f.unlink()
+                    logger.warning(f"  Deleted: {f.name}")
+        
+    logger.error("\n✗✗✗ All retry attempts failed ✗✗✗")
+    return False
+
+
+if __name__ == "__main__":
+    import sys
+    
+    # Get data directory from settings or use default
+    try:
+        sys.path.insert(0, "/app")
+        from app.config import get_settings
+        settings = get_settings()
+        data_dir = Path(settings.DATA_DIR)
+    except Exception as e:
+        logger.error(f"Failed to load settings: {e}")
+        data_dir = Path("/app/data/mtgjson")
+    
+    # Run validation
+    asyncio.run(validate_and_cleanup(data_dir))

backend/scripts/verify_integration.py

@@ -0,0 +1,226 @@
+#!/usr/bin/env python3
+"""
+Comprehensive MTGJSON Data Verification
+
+Checks both:
+1. MTGJSON data file downloads
+2. PostgreSQL database upsert status
+"""
+
+import asyncio
+from pathlib import Path
+from sqlalchemy import create_engine, text
+from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
+import sys
+
+sys.path.append("/app")
+
+from app.services.mtgjson_manager import MTGJSONManager
+from app.config import get_settings
+
+
+async def verify_data_download():
+    """Verify MTGJSON data files were downloaded."""
+    print("=" * 70)
+    print("MTGJSON DATA DOWNLOAD VERIFICATION")
+    print("=" * 70)
+    
+    settings = get_settings()
+    data_dir = Path(settings.DATA_DIR)
+    
+    print(f"\nData Directory: {data_dir}")
+    print(f"Directory exists: {data_dir.exists()}")
+    
+    if not data_dir.exists():
+        print("❌ FAIL: Data directory does not exist")
+        return False
+    
+    # Check for required files
+    required_files = {
+        "AllPrintings.json.gz": "All sets data",
+        "AllSetFiles.json.gz": "Set metadata",
+        "AllIdentifiers.json.gz": "Card identifiers",
+        "CardTypes.json.gz": "Card type definitions",
+        "Keywords.json.gz": "Card keywords",
+        "MagicRoots.json.gz": "Root data",
+        "MagicSets.json.gz": "Set data",
+        "SetTranslations.json.gz": "Set translations"
+    }
+    
+    downloaded_files = []
+    missing_files = []
+    
+    print("\nRequired MTGJSON files:")
+    for filename, description in required_files.items():
+        filepath = data_dir / filename
+        if filepath.exists():
+            size_mb = filepath.stat().st_size / (1024 * 1024)
+            downloaded_files.append(filename)
+            print(f"  ✓ {filename:30} - {size_mb:8.1f} MB")
+        else:
+            missing_files.append(filename)
+            print(f"  ✗ {filename:30} - MISSING")
+    
+    print(f"\nDownload Status: {len(downloaded_files)}/{len(required_files)} files")
+    
+    if missing_files:
+        print(f"\n❌ FAIL: Missing {len(missing_files)} required files: {', '.join(missing_files)}")
+        return False
+    
+    # Check file sizes for sanity
+    allprintings_path = data_dir / "AllPrintings.json.gz"
+    if allprintings_path.exists():
+        size_mb = allprintings_path.stat().st_size / (1024 * 1024)
+        if size_mb < 100:
+            print(f"\n⚠️  WARNING: AllPrintings.json.gz is suspiciously small ({size_mb:.1f} MB). Expected ~500-600 MB")
+            return False
+        else:
+            print(f"\n✓ AllPrintings.json.gz size looks good: {size_mb:.1f} MB")
+    
+    print("\n✓ PASS: All MTGJSON data files downloaded successfully")
+    return True
+
+
+async def verify_database_upsert():
+    """Verify MTGJSON data was properly upserted to PostgreSQL."""
+    print("\n" + "=" * 70)
+    print("POSTGRESQL DATABASE UPSERT VERIFICATION")
+    print("=" * 70)
+    
+    settings = get_settings()
+    db_url = f"postgresql+asyncpg://{settings.POSTGRES_USER}:{settings.POSTGRES_PASSWORD}"
+    db_url += f"@postgres-mtgdata:5432/{settings.POSTGRES_DB}"
+    
+    print(f"\nDatabase: {settings.POSTGRES_DB}")
+    
+    try:
+        engine = create_async_engine(db_url)
+        
+        async with AsyncSession(engine) as session:
+            # Get all tables
+            result = await session.execute(text("""
+                SELECT table_name 
+                FROM information_schema.tables 
+                WHERE table_schema = 'public'
+                ORDER BY table_name;
+            """))
+            tables = [row[0] for row in result.fetchall()]
+            
+            print(f"\nTotal tables: {len(tables)}")
+            
+            # Check MTGJSON-specific tables
+            mtg_tables = ['mtg_set', 'mtg_card', 'mtg_identifiers', 'mtg_keywords']
+            
+            print("\nMTGJSON tables:")
+            for table in mtg_tables:
+                if table in tables:
+                    result = await session.execute(text(f"SELECT COUNT(*) FROM {table}"))
+                    count = result.scalar()
+                    print(f"  ✓ {table:30} - {count:8,} records")
+                else:
+                    print(f"  ✗ {table:30} - TABLE NOT FOUND")
+            
+            # Check refresh log
+            if 'mtg_refresh_log' in tables:
+                result = await session.execute(text("""
+                    SELECT refresh_type, status, created_at
+                    FROM mtg_refresh_log 
+                    ORDER BY created_at DESC 
+                    LIMIT 5;
+                """))
+                rows = result.fetchall()
+                
+                if rows:
+                    print("\nRecent refresh operations:")
+                    for row in rows:
+                        status_icon = "✓" if row[1] == 'SUCCESS' else "✗"
+                        print(f"  {status_icon} {row[0]:15} - {row[1]:8} - {row[2]}")
+            
+            # Verify data quality
+            print("\nData quality checks:")
+            
+            # Check for sets
+            if 'mtg_set' in tables:
+                result = await session.execute(text("""
+                    SELECT COUNT(*) FROM mtg_set 
+                    WHERE set_name IS NOT NULL AND set_code IS NOT NULL;
+                """))
+                valid_sets = result.scalar()
+                result = await session.execute(text("SELECT COUNT(*) FROM mtg_set"))
+                total_sets = result.scalar()
+                print(f"  ✓ Sets: {valid_sets:,}/{total_sets:,} valid")
+            
+            # Check for cards
+            if 'mtg_card' in tables:
+                result = await session.execute(text("""
+                    SELECT COUNT(*) FROM mtg_card 
+                    WHERE name IS NOT NULL AND mtgjson_cards_id IS NOT NULL;
+                """))
+                valid_cards = result.scalar()
+                result = await session.execute(text("SELECT COUNT(*) FROM mtg_card"))
+                total_cards = result.scalar()
+                print(f"  ✓ Cards: {valid_cards:,}/{total_cards:,} valid")
+            
+            # Check for identifiers
+            if 'mtg_identifiers' in tables:
+                result = await session.execute(text("""
+                    SELECT COUNT(*) FROM mtg_identifiers 
+                    WHERE scryfall_id IS NOT NULL;
+                """))
+                valid_ids = result.scalar()
+                result = await session.execute(text("SELECT COUNT(*) FROM mtg_identifiers"))
+                total_ids = result.scalar()
+                print(f"  ✓ Identifiers: {valid_ids:,}/{total_ids:,} valid")
+            
+            # Check for keywords
+            if 'mtg_keywords' in tables:
+                result = await session.execute(text("SELECT COUNT(*) FROM mtg_keywords"))
+                keywords_count = result.scalar()
+                print(f"  ✓ Keywords: {keywords_count:,}")
+        
+        print("\n✓ PASS: Database upsert completed successfully")
+        return True
+        
+    except Exception as e:
+        print(f"\n❌ FAIL: Database error - {e}")
+        return False
+
+
+async def main():
+    """Main verification function."""
+    print("\n" + "=" * 70)
+    print("MTGJSON DATA INTEGRATION VERIFICATION")
+    print("=" * 70)
+    
+    # Check data download
+    download_ok = await verify_data_download()
+    
+    # Check database upsert
+    db_ok = await verify_database_upsert()
+    
+    # Final summary
+    print("\n" + "=" * 70)
+    print("VERIFICATION SUMMARY")
+    print("=" * 70)
+    
+    if download_ok and db_ok:
+        print("\n✓✓✓ ALL CHECKS PASSED ✓✓✓")
+        print("\nMTGJSON data has been successfully downloaded and upserted to PostgreSQL.")
+        print("The backend is ready to use.")
+        return 0
+    else:
+        print("\n❌❌❌ VERIFICATION FAILED ❌❌❌")
+        if not download_ok:
+            print("\nDownload issues:")
+            print("  - Some MTGJSON data files are missing or corrupted")
+            print("  - Run: docker exec mtgonline_backend python /app/scripts/download_mtgjson_v5.py")
+        if not db_ok:
+            print("\nDatabase issues:")
+            print("  - Data was not properly upserted to PostgreSQL")
+            print("  - Check backend logs for errors")
+        return 1
+
+
+if __name__ == "__main__":
+    exit_code = asyncio.run(main())
+    sys.exit(exit_code)

backend/scripts/verify_mtgjson_data.py

@@ -0,0 +1,194 @@
+#!/usr/bin/env python3
+"""
+MTGJSON Data Verification Script
+
+Checks:
+1. Data file download status
+2. Sanity check validation
+3. Database upsert status
+4. File size validation
+"""
+
+import asyncio
+import json
+from pathlib import Path
+from sqlalchemy import create_engine, text
+from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
+import sys
+
+sys.path.insert(0, "/app")
+
+from app.config import get_settings
+from app.services.mtgjson_manager import MTGJSONManager
+
+# Expected minimum file sizes (in bytes)
+EXPECTED_MIN_SIZES = {
+    "AllPrintings.json": 500 * 1024 * 1024,  # 500 MB
+    "AllSetFiles.json": 10 * 1024 * 1024,     # 10 MB
+    "AllIdentifiers.json": 100 * 1024 * 1024, # 100 MB
+    "CardTypes.json": 1 * 1024 * 1024,        # 1 MB
+    "Keywords.json": 0.5 * 1024 * 1024,       # 0.5 MB
+    "MagicSets.json": 50 * 1024 * 1024,       # 50 MB
+    "MagicRoots.json": 1 * 1024 * 1024,       # 1 MB
+    "SetTranslations.json": 5 * 1024 * 1024,  # 5 MB
+}
+
+
+async def check_data_files(data_dir: Path) -> dict:
+    """Check downloaded data files and validate sizes."""
+    print("=" * 70)
+    print("DATA FILE VERIFICATION")
+    print("=" * 70)
+    
+    results = {
+        "valid": True,
+        "files_checked": 0,
+        "files_valid": 0,
+        "files_invalid": 0,
+        "issues": []
+    }
+    
+    if not data_dir.exists():
+        results["valid"] = False
+        results["issues"].append(f"Data directory does not exist: {data_dir}")
+        return results
+    
+    # Check each expected file
+    for filename, min_size in EXPECTED_MIN_SIZES.items():
+        filepath = data_dir / filename
+        
+        if not filepath.exists():
+            results["issues"].append(f"Missing file: {filename}")
+            results["valid"] = False
+            results["files_checked"] += 1
+            results["files_invalid"] += 1
+            continue
+        
+        results["files_checked"] += 1
+        actual_size = filepath.stat().st_size
+        
+        if actual_size < min_size:
+            results["valid"] = False
+            results["files_invalid"] += 1
+            results["issues"].append(
+                f"{filename}: {actual_size / (1024*1024):.1f} MB (minimum: {min_size / (1024*1024):.1f} MB)"
+            )
+            print(f"  ✗ {filename:30} - {actual_size / (1024*1024):8.1f} MB (too small)")
+        else:
+            results["files_valid"] += 1
+            print(f"  ✓ {filename:30} - {actual_size / (1024*1024):8.1f} MB")
+    
+    print(f"\nSummary: {results['files_valid']}/{results['files_checked']} files valid")
+    
+    if results["issues"]:
+        print("\nIssues:")
+        for issue in results["issues"]:
+            print(f"  - {issue}")
+    
+    return results
+
+
+async def check_database_status() -> dict:
+    """Check database upsert status."""
+    print("\n" + "=" * 70)
+    print("DATABASE STATUS VERIFICATION")
+    print("=" * 70)
+    
+    settings = get_settings()
+    db_url = f"postgresql+asyncpg://{settings.POSTGRES_USER}:{settings.POSTGRES_PASSWORD}"
+    db_url += f"@postgres-mtgdata:5432/{settings.POSTGRES_DB}"
+    
+    try:
+        engine = create_async_engine(db_url)
+        
+        async with AsyncSession(engine) as session:
+            # Get all tables
+            result = await session.execute(text("""
+                SELECT table_name 
+                FROM information_schema.tables 
+                WHERE table_schema = 'public'
+                ORDER BY table_name;
+            """))
+            tables = [row[0] for row in result.fetchall()]
+            
+            print(f"Total tables: {len(tables)}")
+            
+            # Check MTGJSON tables
+            mtg_tables = ['mtg_set', 'mtg_card', 'mtg_identifiers', 'mtg_keywords']
+            
+            for table in mtg_tables:
+                if table in tables:
+                    result = await session.execute(text(f"SELECT COUNT(*) FROM {table}"))
+                    count = result.scalar()
+                    print(f"  ✓ {table:30} - {count:8,} records")
+                else:
+                    print(f"  ✗ {table:30} - TABLE NOT FOUND")
+            
+            # Check refresh log
+            if 'mtg_refresh_log' in tables:
+                result = await session.execute(text("""
+                    SELECT refresh_type, status, created_at
+                    FROM mtg_refresh_log 
+                    ORDER BY created_at DESC 
+                    LIMIT 5;
+                """))
+                rows = result.fetchall()
+                
+                if rows:
+                    print("\nRecent refresh operations:")
+                    for row in rows:
+                        status_icon = "✓" if row[1] == 'SUCCESS' else "✗"
+                        print(f"  {status_icon} {row[0]:15} - {row[1]:8} - {row[2]}")
+        
+        return {"valid": True, "error": None}
+        
+    except Exception as e:
+        print(f"\n✗ Database error: {e}")
+        return {"valid": False, "error": str(e)}
+
+
+async def main():
+    """Main verification function."""
+    print("\n" + "=" * 70)
+    print("MTGJSON DATA INTEGRATION VERIFICATION")
+    print("=" * 70)
+    
+    settings = get_settings()
+    data_dir = Path(settings.DATA_DIR)
+    
+    # Check data files
+    file_results = await check_data_files(data_dir)
+    
+    # Check database
+    db_results = await check_database_status()
+    
+    # Final summary
+    print("\n" + "=" * 70)
+    print("VERIFICATION SUMMARY")
+    print("=" * 70)
+    
+    if file_results["valid"] and db_results["valid"]:
+        print("\n✓✓✓ ALL CHECKS PASSED ✓✓✓")
+        print("\nThe MTGJSON data has been properly downloaded and upserted.")
+        return 0
+    else:
+        print("\n❌ VERIFICATION FAILED")
+        
+        if not file_results["valid"]:
+            print("\nData file issues:")
+            for issue in file_results["issues"]:
+                print(f"  - {issue}")
+        
+        if not db_results["valid"]:
+            print(f"\nDatabase issues: {db_results['error']}")
+        
+        print("\nSOLUTION:")
+        print("  1. Delete corrupted data files")
+        print("  2. Run fresh download with sanity checks:")
+        print("     docker exec mtgonline_backend python /app/scripts/download_mtgjson_v5.py")
+        return 1
+
+
+if __name__ == "__main__":
+    exit_code = asyncio.run(main())
+    sys.exit(exit_code)

docker-compose.dev.yml

@@ -14,7 +14,7 @@ services:
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U mtgonline_user"]
       interval: 10s
-      timeout: 5s
+      timeout: 10s
       retries: 5
 
   redis:
@@ -26,7 +26,7 @@ services:
     healthcheck:
       test: ["CMD", "redis-cli", "ping"]
       interval: 10s
-      timeout: 5s
+      timeout: 10s
       retries: 5
 
 volumes:

docker-compose.yml

@@ -21,7 +21,7 @@ services:
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-mtgonline}"]
       interval: 10s
-      timeout: 5s
+      timeout: 10s
       retries: 5
       start_period: 30s
 
@@ -45,7 +45,7 @@ services:
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-mtgonline}"]
       interval: 10s
-      timeout: 5s
+      timeout: 10s
       retries: 5
       start_period: 30s
 
@@ -61,7 +61,7 @@ services:
     healthcheck:
       test: ["CMD-SHELL", "redis-cli ping | grep PONG"]
       interval: 10s
-      timeout: 5s
+      timeout: 10s
       retries: 5
       start_period: 5s
 
@@ -116,7 +116,7 @@ services:
     healthcheck:
       test: ["CMD-SHELL", "curl -f http://localhost:8000/health || exit 1"]
       interval: 30s
-      timeout: 30s
+      timeout: 60s
       retries: 3
       start_period: 600s  # 10 minutes for initial data download
 

state.json

@@ -1,6 +1,6 @@
 {
   "task_description": "MTG Online Backend - MTGJSON Data Integration & Docker Deployment",
-  "current_step": "Deploying test instance with sanity check verification",
+  "current_step": "Timeouts doubled to prevent timeout errors",
   "files_created": [
     "backend/app/services/mtgjson_manager.py",
     "backend/scripts/sanity_check_mtgjson.py",
@@ -11,7 +11,9 @@
     "backend/app/services/mtgjson_manager.py",
     "backend/app/scripts/refresh_mtg.py",
     "backend/app/main.py",
-    "docker-compose.yml"
+    "docker-compose.yml",
+    "docker-compose.dev.yml",
+    "backend/Dockerfile"
   ],
   "decisions": [
     "Created comprehensive MTGJSONManager service in app/services/",
@@ -23,15 +25,16 @@
     "Added file size validation against expected minimums (500MB for AllPrintings, etc.)",
     "Implemented retry logic with exponential backoff (up to 3 attempts)",
     "Cleanup deletes corrupted data before retry",
-    "Container marked unhealthy if validation fails after all retries"
+    "Container marked unhealthy if validation fails after all retries",
+    "Doubled all timeout values to prevent timeouts during download/upsert"
   ],
   "next_steps": [
-    "Stop and destroy all Docker containers",
+    "Deploy stack and monitor startup",
-    "Build backend Docker container",
+    "Wait for backend download phase to complete (~10-15 min)",
-    "Deploy stack and verify health",
+    "Verify container health status",
-    "Check logs for successful initialization with sanity checks"
+    "Check logs for sanity check results"
   ],
   "blockers": [],
-  "commit_hash": "dedc9a3",
+  "commit_hash": "0643544",
-  "timestamp": 1784598300
+  "timestamp": 1784598400
 }