Double all timeout values to prevent download/upsert timeouts

backend/scripts/download_mtgjson_v5.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+"""
+MTGJSON v5 Data Downloader
+
+Downloads MTGJSON v5 data files and loads them into the database.
+Updated to use current MTGJSON API endpoints.
+
+Usage:
+    python download_mtgjson_v5.py
+"""
+
+import sys
+from pathlib import Path
+
+# Add the app directory to Python path
+app_dir = Path(__file__).parent.parent
+sys.path.insert(0, str(app_dir))
+
+import asyncio
+import json
+import gzip
+import logging
+from sqlalchemy import create_engine, text
+from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
+import aiohttp
+
+from app.services.mtgjson_manager import MTGJSONManager
+from app.config import get_settings
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# MTGJSON v5 API base URL
+MTGJSON_API_BASE = "https://mtgjson.com/api/v5"
+
+
+async def download_file(session, url, dest_path):
+    """Download a file from URL."""
+    logger.info(f"Downloading: {url}")
+    
+    try:
+        async with session.get(url, timeout=aiohttp.ClientTimeout(total=600)) as response:
+            if response.status == 200:
+                dest_path.parent.mkdir(parents=True, exist_ok=True)
+                with open(dest_path, 'wb') as f:
+                    async for chunk in response.content.iter_chunked(8192):
+                        f.write(chunk)
+                logger.info(f"✓ Downloaded: {dest_path.name}")
+                return True
+            else:
+                logger.error(f"✗ Failed to download {url}: HTTP {response.status}")
+                return False
+    except Exception as e:
+        logger.error(f"✗ Error downloading {url}: {e}")
+        return False
+
+
+def extract_gzip(gz_path, dest_path=None):
+    """Extract a gzip file."""
+    if not dest_path:
+        dest_path = gz_path.with_suffix('')
+    
+    logger.info(f"Extracting: {gz_path.name}")
+    
+    try:
+        with gzip.open(gz_path, 'rt', encoding='utf-8') as f_in:
+            with open(dest_path, 'w', encoding='utf-8') as f_out:
+                f_out.write(f_in.read())
+        logger.info(f"✓ Extracted: {gz_path.name}")
+        # Remove gz file after extraction
+        gz_path.unlink()
+        return True
+    except Exception as e:
+        logger.error(f"✗ Failed to extract {gz_path}: {e}")
+        return False
+
+
+async def download_all_files(data_dir):
+    """Download all MTGJSON v5 files."""
+    logger.info("=" * 60)
+    logger.info("MTGJSON v5 Data Download")
+    logger.info("=" * 60)
+    
+    files = [
+        "AllPrintings.json.gz",
+        "AllSetFiles.json.gz",
+        "AllIdentifiers.json.gz",
+        "CardTypes.json.gz",
+        "Keywords.json.gz",
+        "MagicRoots.json.gz",
+        "MagicSets.json.gz",
+        "SetTranslations.json.gz"
+    ]
+    
+    async with aiohttp.ClientSession() as session:
+        for filename in files:
+            url = f"{MTGJSON_API_BASE}/{filename}"
+            dest_path = data_dir / filename
+            await download_file(session, url, dest_path)
+            await asyncio.sleep(1)  # Be nice to the API
+
+
+async def load_data_to_database(data_dir):
+    """Load downloaded MTGJSON data into PostgreSQL."""
+    logger.info("=" * 60)
+    logger.info("Loading MTGJSON data into database")
+    logger.info("=" * 60)
+    
+    manager = MTGJSONManager(data_dir)
+    
+    # Download files
+    success = await manager.download_files()
+    
+    if not success:
+        logger.error("✗ Download failed")
+        return False
+    
+    # Unpack files
+    success = manager.unpack_files()
+    
+    if not success:
+        logger.error("✗ Unpack failed")
+        return False
+    
+    # Upsert to database
+    success = manager.upsert_to_database()
+    
+    if success:
+        logger.info("✓ Data loaded successfully")
+    else:
+        logger.error("✗ Database upsert failed")
+    
+    return success
+
+
+async def main():
+    """Main entry point."""
+    settings = get_settings()
+    data_dir = Path(settings.DATA_DIR)
+    
+    logger.info(f"Data directory: {data_dir}")
+    
+    # Clear corrupted data
+    if data_dir.exists():
+        logger.info("Clearing corrupted data...")
+        for f in data_dir.glob("*.json.gz"):
+            f.unlink()
+            logger.info(f"  Removed: {f.name}")
+    
+    data_dir.mkdir(parents=True, exist_ok=True)
+    
+    # Download and load data
+    await load_data_to_database(data_dir)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())