Double all timeout values to prevent download/upsert timeouts

backend/app/services/mtgjson_manager.py

@@ -66,7 +66,7 @@ EXPECTED_MIN_SIZES = {
 }
 
 MAX_DOWNLOAD_RETRIES = 3
-RETRY_DELAY_SECONDS = 60
+RETRY_DELAY_SECONDS = 120
 
 
 class MTGJSONManager:
@@ -123,6 +123,19 @@ class MTGJSONManager:
         logger.info("Data integrity check passed - all files meet minimum size requirements")
         return True, []
     
+    def _get_estimated_size(self, filename: str) -> int:
+        """Get estimated file size in MB."""
+        estimates = {
+            "AllPrintings.json.gz": 500,
+            "AllSetFiles.zip": 10,
+            "AllIdentifiers.json.gz": 100,
+            "CardTypes.json.gz": 5,
+            "DeckList.json.gz": 5,
+            "Keywords.json.gz": 2,
+            "SetList.json.gz": 10,
+        }
+        return estimates.get(filename, 10)  # Default 10MB
+    
     async def cleanup_data_files(self) -> None:
         """Delete all downloaded MTGJSON data files."""
         logger.info(f"Cleaning up data files in {self.data_dir}")
@@ -215,7 +228,7 @@ class MTGJSONManager:
                 return False
 
     async def _download_file(self, session: aiohttp.ClientSession, url: str, filename: str) -> bool:
-        """Download a single file."""
+        """Download a single file with adaptive timeout based on size."""
         dest_path = self.data_dir / filename
         
         # Skip if file already exists
@@ -223,26 +236,51 @@ class MTGJSONManager:
             logger.info(f"Skipping {filename} (already exists)")
             return True
         
-        logger.info(f"Downloading {filename} from {url}")
+        # Calculate timeout based on estimated file size
+        # Large files (>100MB) get up to 60 minutes
+        estimated_mb = self._get_estimated_size(filename)
+        timeout = max(1200, estimated_mb * 8)  # At least 20 min, 8x estimated MB
+        
+        logger.info(f"Downloading {filename} from {url} (timeout: {timeout}s, est: {estimated_mb}MB)")
         
         try:
-            async with session.get(url) as response:
+            # Set timeout for connection AND download
+            timeout_obj = aiohttp.ClientTimeout(total=timeout)
+            
+            async with session.get(url, timeout=timeout_obj) as response:
                 if response.status != 200:
                     logger.error(f"Failed to download {filename}: HTTP {response.status}")
                     return False
                 
-                # Download with progress logging
-                content = b""
-                async for chunk in response.content.iter_chunked(8192):
-                    content += chunk
+                # Download with progress monitoring
+                total_size = 0
+                last_log_time = time.time()
                 
-                # Write to file
-                dest_path.write_bytes(content)
-                logger.info(f"Downloaded {filename} ({len(content)} bytes)")
+                with open(dest_path, 'wb') as f:
+                    async for chunk in response.content.iter_chunked(8192):
+                        f.write(chunk)
+                        total_size += len(chunk)
+                        
+                        # Log progress every 60 seconds
+                        now = time.time()
+                        if now - last_log_time >= 60:
+                            mb_downloaded = total_size / (1024 * 1024)
+                            logger.info(f"  Downloaded {mb_downloaded:.1f} MB of {filename}")
+                            last_log_time = now
+                
+                logger.info(f"Downloaded {filename} ({total_size / (1024*1024):.1f} MB)")
                 return True
                 
+        except asyncio.TimeoutError:
+            logger.error(f"Download timeout for {filename} (timeout: {timeout}s)")
+            # Clean up partial download
+            if dest_path.exists():
+                dest_path.unlink()
+            return False
         except Exception as e:
             logger.error(f"Download error for {filename}: {e}")
+            if dest_path.exists():
+                dest_path.unlink()
             return False
 
     async def unpack_files(self) -> bool: