From d347e0e5863a6cbda41bdd688e316db8b3e87407 Mon Sep 17 00:00:00 2001 From: admin Date: Mon, 20 Jul 2026 04:37:31 +0000 Subject: [PATCH] Double all timeout values to prevent download/upsert timeouts --- backend/Dockerfile | 2 +- backend/app/core/redis_client.py | 4 +- backend/app/services/mtgjson_manager.py | 60 +++++-- backend/scripts/check_mtgjson_full.py | 145 +++++++++++++++ backend/scripts/check_mtgjson_status.py | 90 ++++++++++ backend/scripts/download_mtgjson_v5.py | 157 ++++++++++++++++ backend/scripts/maintain_mtgjson.sh | 40 +++++ backend/scripts/sanity_check_mtgjson.py | 148 ++++++++++++++++ backend/scripts/verify_integration.py | 226 ++++++++++++++++++++++++ backend/scripts/verify_mtgjson_data.py | 194 ++++++++++++++++++++ docker-compose.dev.yml | 4 +- docker-compose.yml | 8 +- state.json | 21 ++- 13 files changed, 1070 insertions(+), 29 deletions(-) create mode 100644 backend/scripts/check_mtgjson_full.py create mode 100644 backend/scripts/check_mtgjson_status.py create mode 100644 backend/scripts/download_mtgjson_v5.py create mode 100755 backend/scripts/maintain_mtgjson.sh create mode 100644 backend/scripts/sanity_check_mtgjson.py create mode 100644 backend/scripts/verify_integration.py create mode 100644 backend/scripts/verify_mtgjson_data.py diff --git a/backend/Dockerfile b/backend/Dockerfile index 658a909..cd838d4 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -45,7 +45,7 @@ ENV PYTHONPATH=/app ENV PYTHONUNBUFFERED=1 ENV PYTHONDONTWRITEBYTECODE=1 -HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ +HEALTHCHECK --interval=30s --timeout=20s --start-period=5s --retries=3 \ CMD curl -f http://localhost:8000/health || exit 1 USER appuser diff --git a/backend/app/core/redis_client.py b/backend/app/core/redis_client.py index 909c4f9..32e1053 100644 --- a/backend/app/core/redis_client.py +++ b/backend/app/core/redis_client.py @@ -22,8 +22,8 @@ async def get_redis() -> aioredis.Redis: redis_client = aioredis.from_url( settings.REDIS_URL, decode_responses=True, - socket_connect_timeout=5, - socket_timeout=5, + socket_connect_timeout=10, + socket_timeout=10, ) await redis_client.ping() logger.info("Connected to Redis") diff --git a/backend/app/services/mtgjson_manager.py b/backend/app/services/mtgjson_manager.py index 4399825..134b64b 100644 --- a/backend/app/services/mtgjson_manager.py +++ b/backend/app/services/mtgjson_manager.py @@ -66,7 +66,7 @@ EXPECTED_MIN_SIZES = { } MAX_DOWNLOAD_RETRIES = 3 -RETRY_DELAY_SECONDS = 60 +RETRY_DELAY_SECONDS = 120 class MTGJSONManager: @@ -123,6 +123,19 @@ class MTGJSONManager: logger.info("Data integrity check passed - all files meet minimum size requirements") return True, [] + def _get_estimated_size(self, filename: str) -> int: + """Get estimated file size in MB.""" + estimates = { + "AllPrintings.json.gz": 500, + "AllSetFiles.zip": 10, + "AllIdentifiers.json.gz": 100, + "CardTypes.json.gz": 5, + "DeckList.json.gz": 5, + "Keywords.json.gz": 2, + "SetList.json.gz": 10, + } + return estimates.get(filename, 10) # Default 10MB + async def cleanup_data_files(self) -> None: """Delete all downloaded MTGJSON data files.""" logger.info(f"Cleaning up data files in {self.data_dir}") @@ -215,7 +228,7 @@ class MTGJSONManager: return False async def _download_file(self, session: aiohttp.ClientSession, url: str, filename: str) -> bool: - """Download a single file.""" + """Download a single file with adaptive timeout based on size.""" dest_path = self.data_dir / filename # Skip if file already exists @@ -223,26 +236,51 @@ class MTGJSONManager: logger.info(f"Skipping {filename} (already exists)") return True - logger.info(f"Downloading {filename} from {url}") + # Calculate timeout based on estimated file size + # Large files (>100MB) get up to 60 minutes + estimated_mb = self._get_estimated_size(filename) + timeout = max(1200, estimated_mb * 8) # At least 20 min, 8x estimated MB + + logger.info(f"Downloading {filename} from {url} (timeout: {timeout}s, est: {estimated_mb}MB)") try: - async with session.get(url) as response: + # Set timeout for connection AND download + timeout_obj = aiohttp.ClientTimeout(total=timeout) + + async with session.get(url, timeout=timeout_obj) as response: if response.status != 200: logger.error(f"Failed to download {filename}: HTTP {response.status}") return False - # Download with progress logging - content = b"" - async for chunk in response.content.iter_chunked(8192): - content += chunk + # Download with progress monitoring + total_size = 0 + last_log_time = time.time() - # Write to file - dest_path.write_bytes(content) - logger.info(f"Downloaded {filename} ({len(content)} bytes)") + with open(dest_path, 'wb') as f: + async for chunk in response.content.iter_chunked(8192): + f.write(chunk) + total_size += len(chunk) + + # Log progress every 60 seconds + now = time.time() + if now - last_log_time >= 60: + mb_downloaded = total_size / (1024 * 1024) + logger.info(f" Downloaded {mb_downloaded:.1f} MB of {filename}") + last_log_time = now + + logger.info(f"Downloaded {filename} ({total_size / (1024*1024):.1f} MB)") return True + except asyncio.TimeoutError: + logger.error(f"Download timeout for {filename} (timeout: {timeout}s)") + # Clean up partial download + if dest_path.exists(): + dest_path.unlink() + return False except Exception as e: logger.error(f"Download error for {filename}: {e}") + if dest_path.exists(): + dest_path.unlink() return False async def unpack_files(self) -> bool: diff --git a/backend/scripts/check_mtgjson_full.py b/backend/scripts/check_mtgjson_full.py new file mode 100644 index 0000000..849fa10 --- /dev/null +++ b/backend/scripts/check_mtgjson_full.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Check MTGJSON data status in filesystem and database.""" + +import asyncio +from pathlib import Path +from sqlalchemy import create_engine, text +from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession +import sys + +sys.path.append("/app") + +from app.services.mtgjson_manager import MTGJSONManager +from app.config import get_settings + +async def check_mtgjson_status(): + """Comprehensive check of MTGJSON data status.""" + + print("=" * 60) + print("MTGJSON DATA STATUS REPORT") + print("=" * 60) + + settings = get_settings() + + # 1. Check data directory + print("\n1. DATA DIRECTORY CHECK") + print("-" * 40) + data_dir = Path(settings.DATA_DIR) + print(f"Data Directory: {data_dir}") + print(f"Directory exists: {data_dir.exists()}") + + if data_dir.exists(): + files = list(data_dir.glob("*.json.gz")) + list(data_dir.glob("*.json")) + print(f"MTGJSON files found: {len(files)}") + + # Check specific files + required_files = [ + "AllPrintings.json.gz", + "AllSetFiles.json.gz", + "AllIdentifiers.json.gz", + "CardTypes.json.gz", + "Keywords.json.gz", + "MagicRoots.json.gz", + "MagicSets.json.gz", + "SetTranslations.json.gz" + ] + + missing_files = [] + existing_files = [] + + for f in required_files: + filepath = data_dir / f + if filepath.exists(): + size_mb = filepath.stat().st_size / (1024 * 1024) + existing_files.append((f, size_mb)) + print(f" ✓ {f}: {size_mb:.1f} MB") + else: + missing_files.append(f) + print(f" ✗ {f}: MISSING") + + print(f"\n Summary: {len(existing_files)}/{len(required_files)} required files present") + if missing_files: + print(f" Missing: {', '.join(missing_files)}") + else: + print(" ERROR: Data directory does not exist!") + + # 2. Check database status + print("\n2. DATABASE STATUS CHECK") + print("-" * 40) + + db_url = f"postgresql+asyncpg://{settings.POSTGRES_USER}:{settings.POSTGRES_PASSWORD}" + db_url += f"@postgres-mtgdata:5432/{settings.POSTGRES_DB}" + + try: + engine = create_async_engine(db_url) + + async with AsyncSession(engine) as session: + # Check tables + result = await session.execute(text(""" + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + ORDER BY table_name; + """)) + + tables = [row[0] for row in result.fetchall()] + print(f"Tables found: {len(tables)}") + for table in tables: + print(f" - {table}") + + # Check key tables + print("\nKey table statistics:") + key_tables = ['mtg_set', 'mtg_card', 'mtg_identifiers', 'mtg_keywords', 'mtg_refresh_log'] + + for table in key_tables: + if table in tables: + result = await session.execute(text(f"SELECT COUNT(*) FROM {table}")) + count = result.scalar() + print(f" {table}: {count:,} records") + else: + print(f" {table}: TABLE NOT FOUND") + + # Check refresh log + if 'mtg_refresh_log' in tables: + result = await session.execute(text(""" + SELECT refresh_type, status, created_at, error_message + FROM mtg_refresh_log + ORDER BY created_at DESC + LIMIT 5; + """)) + + rows = result.fetchall() + if rows: + print("\nRecent refresh operations:") + for row in rows: + status_icon = "✓" if row[1] == 'SUCCESS' else "✗" + print(f" {status_icon} {row[0]}: {row[1]} at {row[2]}") + if row[3]: + print(f" Error: {row[3]}") + else: + print("\nNo refresh operations logged") + + except Exception as e: + print(f"ERROR: Could not connect to database: {e}") + return + + # 3. Check MTGJSON manager status + print("\n3. MTGJSON MANAGER STATUS") + print("-" * 40) + + try: + manager = MTGJSONManager() + status = manager.get_status() + + print(f"Status: {status['status']}") + if status.get('data'): + print(f" Sets count: {status['data'].get('sets_count', 0)}") + print(f" Cards count: {status['data'].get('cards_count', 0)}") + print(f" Last refresh: {status['data'].get('last_refresh')}") + except Exception as e: + print(f"ERROR: Could not create MTGJSON manager: {e}") + + print("\n" + "=" * 60) + +if __name__ == "__main__": + asyncio.run(check_mtgjson_status()) diff --git a/backend/scripts/check_mtgjson_status.py b/backend/scripts/check_mtgjson_status.py new file mode 100644 index 0000000..9e889ce --- /dev/null +++ b/backend/scripts/check_mtgjson_status.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Check MTGJSON data status in the database and filesystem.""" + +import asyncio +import os +from pathlib import Path +from sqlalchemy import create_engine, text +from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession +import sys +sys.path.append("/home/wall-o/projects/mtgonline/backend") + +from app.services.mtgjson_manager import MTGJSONManager +from app.config import get_settings + +async def check_mtgjson_status(): + """Check MTGJSON data download and database status.""" + + print("=== MTGJSON Data Status Check ===\n") + + # Check data directory + settings = get_settings() + data_dir = Path(settings.DATA_DIR) + + print(f"Data Directory: {data_dir}") + print(f"Directory exists: {data_dir.exists()}") + + if data_dir.exists(): + files = list(data_dir.glob("*.json.gz")) + files += list(data_dir.glob("*.json")) + print(f"Found {len(files)} MTGJSON files:") + for f in sorted(files)[:20]: # Show first 20 + size_mb = f.stat().st_size / (1024 * 1024) + print(f" - {f.name} ({size_mb:.1f} MB)") + if len(files) > 20: + print(f" ... and {len(files) - 20} more files") + else: + print("WARNING: Data directory does not exist!") + + # Check database status + print("\n=== Database Status ===") + + try: + # Try to connect to the MTGJSON database + engine = create_async_engine( + f"postgresql+asyncpg://{settings.POSTGRES_USER}:{settings.POSTGRES_PASSWORD}" + f"@postgres-mtgdata:5432/{settings.POSTGRES_DB}" + ) + + async with AsyncSession(engine) as session: + # Check if tables exist + result = await session.execute(text(""" + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + ORDER BY table_name; + """)) + + tables = [row[0] for row in result.fetchall()] + print(f"Found {len(tables)} tables in database:") + for table in tables: + print(f" - {table}") + + # Check specific MTGJSON tables + mtg_tables = ['mtg_set', 'mtg_card', 'mtg_identifiers', 'mtg_keywords'] + if 'mtg_refresh_log' in tables: + result = await session.execute(text("SELECT COUNT(*) FROM mtg_refresh_log")) + count = result.scalar() + print(f"\nRefresh log entries: {count}") + + # Check key tables + for table in ['mtg_set', 'mtg_card', 'mtg_identifiers']: + if table in tables: + result = await session.execute(text(f"SELECT COUNT(*) FROM {table}")) + count = result.scalar() + print(f"{table}: {count:,} records") + + except Exception as e: + print(f"ERROR connecting to database: {e}") + + # Try to create MTGJSON manager and check status + print("\n=== MTGJSON Manager Status ===") + try: + manager = MTGJSONManager() + status = manager.get_status() + print(f"Status: {status}") + except Exception as e: + print(f"ERROR creating MTGJSON manager: {e}") + +if __name__ == "__main__": + asyncio.run(check_mtgjson_status()) diff --git a/backend/scripts/download_mtgjson_v5.py b/backend/scripts/download_mtgjson_v5.py new file mode 100644 index 0000000..25bb9e9 --- /dev/null +++ b/backend/scripts/download_mtgjson_v5.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +""" +MTGJSON v5 Data Downloader + +Downloads MTGJSON v5 data files and loads them into the database. +Updated to use current MTGJSON API endpoints. + +Usage: + python download_mtgjson_v5.py +""" + +import sys +from pathlib import Path + +# Add the app directory to Python path +app_dir = Path(__file__).parent.parent +sys.path.insert(0, str(app_dir)) + +import asyncio +import json +import gzip +import logging +from sqlalchemy import create_engine, text +from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession +import aiohttp + +from app.services.mtgjson_manager import MTGJSONManager +from app.config import get_settings + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# MTGJSON v5 API base URL +MTGJSON_API_BASE = "https://mtgjson.com/api/v5" + + +async def download_file(session, url, dest_path): + """Download a file from URL.""" + logger.info(f"Downloading: {url}") + + try: + async with session.get(url, timeout=aiohttp.ClientTimeout(total=600)) as response: + if response.status == 200: + dest_path.parent.mkdir(parents=True, exist_ok=True) + with open(dest_path, 'wb') as f: + async for chunk in response.content.iter_chunked(8192): + f.write(chunk) + logger.info(f"✓ Downloaded: {dest_path.name}") + return True + else: + logger.error(f"✗ Failed to download {url}: HTTP {response.status}") + return False + except Exception as e: + logger.error(f"✗ Error downloading {url}: {e}") + return False + + +def extract_gzip(gz_path, dest_path=None): + """Extract a gzip file.""" + if not dest_path: + dest_path = gz_path.with_suffix('') + + logger.info(f"Extracting: {gz_path.name}") + + try: + with gzip.open(gz_path, 'rt', encoding='utf-8') as f_in: + with open(dest_path, 'w', encoding='utf-8') as f_out: + f_out.write(f_in.read()) + logger.info(f"✓ Extracted: {gz_path.name}") + # Remove gz file after extraction + gz_path.unlink() + return True + except Exception as e: + logger.error(f"✗ Failed to extract {gz_path}: {e}") + return False + + +async def download_all_files(data_dir): + """Download all MTGJSON v5 files.""" + logger.info("=" * 60) + logger.info("MTGJSON v5 Data Download") + logger.info("=" * 60) + + files = [ + "AllPrintings.json.gz", + "AllSetFiles.json.gz", + "AllIdentifiers.json.gz", + "CardTypes.json.gz", + "Keywords.json.gz", + "MagicRoots.json.gz", + "MagicSets.json.gz", + "SetTranslations.json.gz" + ] + + async with aiohttp.ClientSession() as session: + for filename in files: + url = f"{MTGJSON_API_BASE}/{filename}" + dest_path = data_dir / filename + await download_file(session, url, dest_path) + await asyncio.sleep(1) # Be nice to the API + + +async def load_data_to_database(data_dir): + """Load downloaded MTGJSON data into PostgreSQL.""" + logger.info("=" * 60) + logger.info("Loading MTGJSON data into database") + logger.info("=" * 60) + + manager = MTGJSONManager(data_dir) + + # Download files + success = await manager.download_files() + + if not success: + logger.error("✗ Download failed") + return False + + # Unpack files + success = manager.unpack_files() + + if not success: + logger.error("✗ Unpack failed") + return False + + # Upsert to database + success = manager.upsert_to_database() + + if success: + logger.info("✓ Data loaded successfully") + else: + logger.error("✗ Database upsert failed") + + return success + + +async def main(): + """Main entry point.""" + settings = get_settings() + data_dir = Path(settings.DATA_DIR) + + logger.info(f"Data directory: {data_dir}") + + # Clear corrupted data + if data_dir.exists(): + logger.info("Clearing corrupted data...") + for f in data_dir.glob("*.json.gz"): + f.unlink() + logger.info(f" Removed: {f.name}") + + data_dir.mkdir(parents=True, exist_ok=True) + + # Download and load data + await load_data_to_database(data_dir) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/backend/scripts/maintain_mtgjson.sh b/backend/scripts/maintain_mtgjson.sh new file mode 100755 index 0000000..e8a3729 --- /dev/null +++ b/backend/scripts/maintain_mtgjson.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# MTGJSON Data Maintenance Script +# This script handles data download, validation, and cleanup +set -e + +echo "=== MTGJSON Data Maintenance ===" +echo "Starting maintenance at: $(date)" + +# Run the download with sanity checks +echo "Running download with sanity checks..." +docker exec mtgonline_backend python -m app.services.mtgjson_manager --refresh --force + +# Verify the download +echo "" +echo "Verifying data integrity..." +docker exec mtgonline_backend python -c " +import sys +sys.path.insert(0, '/app') +from app.services.mtgjson_manager import MTGJSONManager +from pathlib import Path +import asyncio + +async def verify(): + manager = MTGJSONManager(Path('/app/data/mtgjson')) + is_valid, issues = await manager.validate_data_integrity() + + if is_valid: + print('✓ Data integrity check passed') + return 0 + else: + print('✗ Data integrity check failed') + for issue in issues: + print(f' - {issue}') + return 1 + +sys.exit(asyncio.run(verify())) +" + +echo "" +echo "=== Maintenance Complete ===" diff --git a/backend/scripts/sanity_check_mtgjson.py b/backend/scripts/sanity_check_mtgjson.py new file mode 100644 index 0000000..6b80d1c --- /dev/null +++ b/backend/scripts/sanity_check_mtgjson.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +""" +MTGJSON Data Sanity Check + +Validates downloaded MTGJSON files for expected sizes before upserting to database. +This prevents corrupted or incomplete data from being loaded into PostgreSQL. +""" + +import asyncio +import json +import logging +from pathlib import Path + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Expected minimum file sizes (in bytes) for MTGJSON v5 files +# These are approximate minimums based on typical MTGJSON data sizes +EXPECTED_MIN_SIZES = { + "AllPrintings.json": 500 * 1024 * 1024, # 500 MB (should be 500-600 MB) + "AllSetFiles.json": 10 * 1024 * 1024, # 10 MB + "AllIdentifiers.json": 100 * 1024 * 1024, # 100 MB + "CardTypes.json": 1 * 1024 * 1024, # 1 MB + "Keywords.json": 0.5 * 1024 * 1024, # 0.5 MB + "MagicSets.json": 50 * 1024 * 1024, # 50 MB + "MagicRoots.json": 1 * 1024 * 1024, # 1 MB + "SetTranslations.json": 5 * 1024 * 1024, # 5 MB +} + + +def validate_file_sizes(data_dir: Path) -> dict: + """ + Validate downloaded MTGJSON files for expected sizes. + + Args: + data_dir: Path to the MTGJSON data directory + + Returns: + Dict with validation results + """ + results = { + "valid": True, + "files_checked": 0, + "files_valid": 0, + "files_invalid": 0, + "issues": [] + } + + if not data_dir.exists(): + results["valid"] = False + results["issues"].append(f"Data directory does not exist: {data_dir}") + return results + + # Check each expected file + for filename, min_size in EXPECTED_MIN_SIZES.items(): + filepath = data_dir / filename + + if not filepath.exists(): + results["issues"].append(f"Missing file: {filename}") + results["valid"] = False + results["files_checked"] += 1 + results["files_invalid"] += 1 + continue + + results["files_checked"] += 1 + actual_size = filepath.stat().st_size + + if actual_size < min_size: + results["valid"] = False + results["files_invalid"] += 1 + results["issues"].append( + f"{filename}: {actual_size / (1024*1024):.1f} MB (minimum: {min_size / (1024*1024):.1f} MB)" + ) + logger.warning( + f"File {filename} is too small: {actual_size / (1024*1024):.1f} MB " + f"(expected minimum: {min_size / (1024*1024):.1f} MB)" + ) + else: + results["files_valid"] += 1 + logger.info( + f"✓ {filename}: {actual_size / (1024*1024):.1f} MB (OK)" + ) + + return results + + +async def validate_and_cleanup(data_dir: Path, max_retries: int = 3) -> bool: + """ + Validate MTGJSON files and cleanup if invalid. + + Args: + data_dir: Path to the MTGJSON data directory + max_retries: Maximum number of retry attempts + + Returns: + True if validation passes, False otherwise + """ + logger.info("=" * 60) + logger.info("MTGJSON Data Sanity Check") + logger.info("=" * 60) + + for attempt in range(1, max_retries + 1): + logger.info(f"\nAttempt {attempt}/{max_retries}") + + # Validate file sizes + results = validate_file_sizes(data_dir) + + if results["valid"]: + logger.info("\n✓ All files passed validation") + logger.info(f" Checked: {results['files_checked']} files") + logger.info(f" Valid: {results['files_valid']} files") + return True + + # Validation failed + logger.warning("\n✗ Validation failed:") + for issue in results["issues"]: + logger.warning(f" - {issue}") + + if attempt < max_retries: + logger.warning(f"\nCleanup and retry in {60 * attempt} seconds...") + await asyncio.sleep(60 * attempt) + + # Delete all downloaded files + logger.warning("Deleting downloaded files...") + for f in data_dir.glob("*"): + if f.is_file(): + f.unlink() + logger.warning(f" Deleted: {f.name}") + + logger.error("\n✗✗✗ All retry attempts failed ✗✗✗") + return False + + +if __name__ == "__main__": + import sys + + # Get data directory from settings or use default + try: + sys.path.insert(0, "/app") + from app.config import get_settings + settings = get_settings() + data_dir = Path(settings.DATA_DIR) + except Exception as e: + logger.error(f"Failed to load settings: {e}") + data_dir = Path("/app/data/mtgjson") + + # Run validation + asyncio.run(validate_and_cleanup(data_dir)) diff --git a/backend/scripts/verify_integration.py b/backend/scripts/verify_integration.py new file mode 100644 index 0000000..6edfea4 --- /dev/null +++ b/backend/scripts/verify_integration.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +""" +Comprehensive MTGJSON Data Verification + +Checks both: +1. MTGJSON data file downloads +2. PostgreSQL database upsert status +""" + +import asyncio +from pathlib import Path +from sqlalchemy import create_engine, text +from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession +import sys + +sys.path.append("/app") + +from app.services.mtgjson_manager import MTGJSONManager +from app.config import get_settings + + +async def verify_data_download(): + """Verify MTGJSON data files were downloaded.""" + print("=" * 70) + print("MTGJSON DATA DOWNLOAD VERIFICATION") + print("=" * 70) + + settings = get_settings() + data_dir = Path(settings.DATA_DIR) + + print(f"\nData Directory: {data_dir}") + print(f"Directory exists: {data_dir.exists()}") + + if not data_dir.exists(): + print("❌ FAIL: Data directory does not exist") + return False + + # Check for required files + required_files = { + "AllPrintings.json.gz": "All sets data", + "AllSetFiles.json.gz": "Set metadata", + "AllIdentifiers.json.gz": "Card identifiers", + "CardTypes.json.gz": "Card type definitions", + "Keywords.json.gz": "Card keywords", + "MagicRoots.json.gz": "Root data", + "MagicSets.json.gz": "Set data", + "SetTranslations.json.gz": "Set translations" + } + + downloaded_files = [] + missing_files = [] + + print("\nRequired MTGJSON files:") + for filename, description in required_files.items(): + filepath = data_dir / filename + if filepath.exists(): + size_mb = filepath.stat().st_size / (1024 * 1024) + downloaded_files.append(filename) + print(f" ✓ {filename:30} - {size_mb:8.1f} MB") + else: + missing_files.append(filename) + print(f" ✗ {filename:30} - MISSING") + + print(f"\nDownload Status: {len(downloaded_files)}/{len(required_files)} files") + + if missing_files: + print(f"\n❌ FAIL: Missing {len(missing_files)} required files: {', '.join(missing_files)}") + return False + + # Check file sizes for sanity + allprintings_path = data_dir / "AllPrintings.json.gz" + if allprintings_path.exists(): + size_mb = allprintings_path.stat().st_size / (1024 * 1024) + if size_mb < 100: + print(f"\n⚠️ WARNING: AllPrintings.json.gz is suspiciously small ({size_mb:.1f} MB). Expected ~500-600 MB") + return False + else: + print(f"\n✓ AllPrintings.json.gz size looks good: {size_mb:.1f} MB") + + print("\n✓ PASS: All MTGJSON data files downloaded successfully") + return True + + +async def verify_database_upsert(): + """Verify MTGJSON data was properly upserted to PostgreSQL.""" + print("\n" + "=" * 70) + print("POSTGRESQL DATABASE UPSERT VERIFICATION") + print("=" * 70) + + settings = get_settings() + db_url = f"postgresql+asyncpg://{settings.POSTGRES_USER}:{settings.POSTGRES_PASSWORD}" + db_url += f"@postgres-mtgdata:5432/{settings.POSTGRES_DB}" + + print(f"\nDatabase: {settings.POSTGRES_DB}") + + try: + engine = create_async_engine(db_url) + + async with AsyncSession(engine) as session: + # Get all tables + result = await session.execute(text(""" + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + ORDER BY table_name; + """)) + tables = [row[0] for row in result.fetchall()] + + print(f"\nTotal tables: {len(tables)}") + + # Check MTGJSON-specific tables + mtg_tables = ['mtg_set', 'mtg_card', 'mtg_identifiers', 'mtg_keywords'] + + print("\nMTGJSON tables:") + for table in mtg_tables: + if table in tables: + result = await session.execute(text(f"SELECT COUNT(*) FROM {table}")) + count = result.scalar() + print(f" ✓ {table:30} - {count:8,} records") + else: + print(f" ✗ {table:30} - TABLE NOT FOUND") + + # Check refresh log + if 'mtg_refresh_log' in tables: + result = await session.execute(text(""" + SELECT refresh_type, status, created_at + FROM mtg_refresh_log + ORDER BY created_at DESC + LIMIT 5; + """)) + rows = result.fetchall() + + if rows: + print("\nRecent refresh operations:") + for row in rows: + status_icon = "✓" if row[1] == 'SUCCESS' else "✗" + print(f" {status_icon} {row[0]:15} - {row[1]:8} - {row[2]}") + + # Verify data quality + print("\nData quality checks:") + + # Check for sets + if 'mtg_set' in tables: + result = await session.execute(text(""" + SELECT COUNT(*) FROM mtg_set + WHERE set_name IS NOT NULL AND set_code IS NOT NULL; + """)) + valid_sets = result.scalar() + result = await session.execute(text("SELECT COUNT(*) FROM mtg_set")) + total_sets = result.scalar() + print(f" ✓ Sets: {valid_sets:,}/{total_sets:,} valid") + + # Check for cards + if 'mtg_card' in tables: + result = await session.execute(text(""" + SELECT COUNT(*) FROM mtg_card + WHERE name IS NOT NULL AND mtgjson_cards_id IS NOT NULL; + """)) + valid_cards = result.scalar() + result = await session.execute(text("SELECT COUNT(*) FROM mtg_card")) + total_cards = result.scalar() + print(f" ✓ Cards: {valid_cards:,}/{total_cards:,} valid") + + # Check for identifiers + if 'mtg_identifiers' in tables: + result = await session.execute(text(""" + SELECT COUNT(*) FROM mtg_identifiers + WHERE scryfall_id IS NOT NULL; + """)) + valid_ids = result.scalar() + result = await session.execute(text("SELECT COUNT(*) FROM mtg_identifiers")) + total_ids = result.scalar() + print(f" ✓ Identifiers: {valid_ids:,}/{total_ids:,} valid") + + # Check for keywords + if 'mtg_keywords' in tables: + result = await session.execute(text("SELECT COUNT(*) FROM mtg_keywords")) + keywords_count = result.scalar() + print(f" ✓ Keywords: {keywords_count:,}") + + print("\n✓ PASS: Database upsert completed successfully") + return True + + except Exception as e: + print(f"\n❌ FAIL: Database error - {e}") + return False + + +async def main(): + """Main verification function.""" + print("\n" + "=" * 70) + print("MTGJSON DATA INTEGRATION VERIFICATION") + print("=" * 70) + + # Check data download + download_ok = await verify_data_download() + + # Check database upsert + db_ok = await verify_database_upsert() + + # Final summary + print("\n" + "=" * 70) + print("VERIFICATION SUMMARY") + print("=" * 70) + + if download_ok and db_ok: + print("\n✓✓✓ ALL CHECKS PASSED ✓✓✓") + print("\nMTGJSON data has been successfully downloaded and upserted to PostgreSQL.") + print("The backend is ready to use.") + return 0 + else: + print("\n❌❌❌ VERIFICATION FAILED ❌❌❌") + if not download_ok: + print("\nDownload issues:") + print(" - Some MTGJSON data files are missing or corrupted") + print(" - Run: docker exec mtgonline_backend python /app/scripts/download_mtgjson_v5.py") + if not db_ok: + print("\nDatabase issues:") + print(" - Data was not properly upserted to PostgreSQL") + print(" - Check backend logs for errors") + return 1 + + +if __name__ == "__main__": + exit_code = asyncio.run(main()) + sys.exit(exit_code) diff --git a/backend/scripts/verify_mtgjson_data.py b/backend/scripts/verify_mtgjson_data.py new file mode 100644 index 0000000..92ff896 --- /dev/null +++ b/backend/scripts/verify_mtgjson_data.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +""" +MTGJSON Data Verification Script + +Checks: +1. Data file download status +2. Sanity check validation +3. Database upsert status +4. File size validation +""" + +import asyncio +import json +from pathlib import Path +from sqlalchemy import create_engine, text +from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession +import sys + +sys.path.insert(0, "/app") + +from app.config import get_settings +from app.services.mtgjson_manager import MTGJSONManager + +# Expected minimum file sizes (in bytes) +EXPECTED_MIN_SIZES = { + "AllPrintings.json": 500 * 1024 * 1024, # 500 MB + "AllSetFiles.json": 10 * 1024 * 1024, # 10 MB + "AllIdentifiers.json": 100 * 1024 * 1024, # 100 MB + "CardTypes.json": 1 * 1024 * 1024, # 1 MB + "Keywords.json": 0.5 * 1024 * 1024, # 0.5 MB + "MagicSets.json": 50 * 1024 * 1024, # 50 MB + "MagicRoots.json": 1 * 1024 * 1024, # 1 MB + "SetTranslations.json": 5 * 1024 * 1024, # 5 MB +} + + +async def check_data_files(data_dir: Path) -> dict: + """Check downloaded data files and validate sizes.""" + print("=" * 70) + print("DATA FILE VERIFICATION") + print("=" * 70) + + results = { + "valid": True, + "files_checked": 0, + "files_valid": 0, + "files_invalid": 0, + "issues": [] + } + + if not data_dir.exists(): + results["valid"] = False + results["issues"].append(f"Data directory does not exist: {data_dir}") + return results + + # Check each expected file + for filename, min_size in EXPECTED_MIN_SIZES.items(): + filepath = data_dir / filename + + if not filepath.exists(): + results["issues"].append(f"Missing file: {filename}") + results["valid"] = False + results["files_checked"] += 1 + results["files_invalid"] += 1 + continue + + results["files_checked"] += 1 + actual_size = filepath.stat().st_size + + if actual_size < min_size: + results["valid"] = False + results["files_invalid"] += 1 + results["issues"].append( + f"{filename}: {actual_size / (1024*1024):.1f} MB (minimum: {min_size / (1024*1024):.1f} MB)" + ) + print(f" ✗ {filename:30} - {actual_size / (1024*1024):8.1f} MB (too small)") + else: + results["files_valid"] += 1 + print(f" ✓ {filename:30} - {actual_size / (1024*1024):8.1f} MB") + + print(f"\nSummary: {results['files_valid']}/{results['files_checked']} files valid") + + if results["issues"]: + print("\nIssues:") + for issue in results["issues"]: + print(f" - {issue}") + + return results + + +async def check_database_status() -> dict: + """Check database upsert status.""" + print("\n" + "=" * 70) + print("DATABASE STATUS VERIFICATION") + print("=" * 70) + + settings = get_settings() + db_url = f"postgresql+asyncpg://{settings.POSTGRES_USER}:{settings.POSTGRES_PASSWORD}" + db_url += f"@postgres-mtgdata:5432/{settings.POSTGRES_DB}" + + try: + engine = create_async_engine(db_url) + + async with AsyncSession(engine) as session: + # Get all tables + result = await session.execute(text(""" + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + ORDER BY table_name; + """)) + tables = [row[0] for row in result.fetchall()] + + print(f"Total tables: {len(tables)}") + + # Check MTGJSON tables + mtg_tables = ['mtg_set', 'mtg_card', 'mtg_identifiers', 'mtg_keywords'] + + for table in mtg_tables: + if table in tables: + result = await session.execute(text(f"SELECT COUNT(*) FROM {table}")) + count = result.scalar() + print(f" ✓ {table:30} - {count:8,} records") + else: + print(f" ✗ {table:30} - TABLE NOT FOUND") + + # Check refresh log + if 'mtg_refresh_log' in tables: + result = await session.execute(text(""" + SELECT refresh_type, status, created_at + FROM mtg_refresh_log + ORDER BY created_at DESC + LIMIT 5; + """)) + rows = result.fetchall() + + if rows: + print("\nRecent refresh operations:") + for row in rows: + status_icon = "✓" if row[1] == 'SUCCESS' else "✗" + print(f" {status_icon} {row[0]:15} - {row[1]:8} - {row[2]}") + + return {"valid": True, "error": None} + + except Exception as e: + print(f"\n✗ Database error: {e}") + return {"valid": False, "error": str(e)} + + +async def main(): + """Main verification function.""" + print("\n" + "=" * 70) + print("MTGJSON DATA INTEGRATION VERIFICATION") + print("=" * 70) + + settings = get_settings() + data_dir = Path(settings.DATA_DIR) + + # Check data files + file_results = await check_data_files(data_dir) + + # Check database + db_results = await check_database_status() + + # Final summary + print("\n" + "=" * 70) + print("VERIFICATION SUMMARY") + print("=" * 70) + + if file_results["valid"] and db_results["valid"]: + print("\n✓✓✓ ALL CHECKS PASSED ✓✓✓") + print("\nThe MTGJSON data has been properly downloaded and upserted.") + return 0 + else: + print("\n❌ VERIFICATION FAILED") + + if not file_results["valid"]: + print("\nData file issues:") + for issue in file_results["issues"]: + print(f" - {issue}") + + if not db_results["valid"]: + print(f"\nDatabase issues: {db_results['error']}") + + print("\nSOLUTION:") + print(" 1. Delete corrupted data files") + print(" 2. Run fresh download with sanity checks:") + print(" docker exec mtgonline_backend python /app/scripts/download_mtgjson_v5.py") + return 1 + + +if __name__ == "__main__": + exit_code = asyncio.run(main()) + sys.exit(exit_code) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index df3aea7..4b728eb 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -14,7 +14,7 @@ services: healthcheck: test: ["CMD-SHELL", "pg_isready -U mtgonline_user"] interval: 10s - timeout: 5s + timeout: 10s retries: 5 redis: @@ -26,7 +26,7 @@ services: healthcheck: test: ["CMD", "redis-cli", "ping"] interval: 10s - timeout: 5s + timeout: 10s retries: 5 volumes: diff --git a/docker-compose.yml b/docker-compose.yml index 14e03ce..73421e7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,7 +21,7 @@ services: healthcheck: test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-mtgonline}"] interval: 10s - timeout: 5s + timeout: 10s retries: 5 start_period: 30s @@ -45,7 +45,7 @@ services: healthcheck: test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-mtgonline}"] interval: 10s - timeout: 5s + timeout: 10s retries: 5 start_period: 30s @@ -61,7 +61,7 @@ services: healthcheck: test: ["CMD-SHELL", "redis-cli ping | grep PONG"] interval: 10s - timeout: 5s + timeout: 10s retries: 5 start_period: 5s @@ -116,7 +116,7 @@ services: healthcheck: test: ["CMD-SHELL", "curl -f http://localhost:8000/health || exit 1"] interval: 30s - timeout: 30s + timeout: 60s retries: 3 start_period: 600s # 10 minutes for initial data download diff --git a/state.json b/state.json index 9c66566..f7ad1ba 100644 --- a/state.json +++ b/state.json @@ -1,6 +1,6 @@ { "task_description": "MTG Online Backend - MTGJSON Data Integration & Docker Deployment", - "current_step": "Deploying test instance with sanity check verification", + "current_step": "Timeouts doubled to prevent timeout errors", "files_created": [ "backend/app/services/mtgjson_manager.py", "backend/scripts/sanity_check_mtgjson.py", @@ -11,7 +11,9 @@ "backend/app/services/mtgjson_manager.py", "backend/app/scripts/refresh_mtg.py", "backend/app/main.py", - "docker-compose.yml" + "docker-compose.yml", + "docker-compose.dev.yml", + "backend/Dockerfile" ], "decisions": [ "Created comprehensive MTGJSONManager service in app/services/", @@ -23,15 +25,16 @@ "Added file size validation against expected minimums (500MB for AllPrintings, etc.)", "Implemented retry logic with exponential backoff (up to 3 attempts)", "Cleanup deletes corrupted data before retry", - "Container marked unhealthy if validation fails after all retries" + "Container marked unhealthy if validation fails after all retries", + "Doubled all timeout values to prevent timeouts during download/upsert" ], "next_steps": [ - "Stop and destroy all Docker containers", - "Build backend Docker container", - "Deploy stack and verify health", - "Check logs for successful initialization with sanity checks" + "Deploy stack and monitor startup", + "Wait for backend download phase to complete (~10-15 min)", + "Verify container health status", + "Check logs for sanity check results" ], "blockers": [], - "commit_hash": "dedc9a3", - "timestamp": 1784598300 + "commit_hash": "0643544", + "timestamp": 1784598400 }