Double all timeout values to prevent download/upsert timeouts
This commit is contained in:
+1
-1
@@ -45,7 +45,7 @@ ENV PYTHONPATH=/app
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
ENV PYTHONDONTWRITEBYTECODE=1
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
||||
HEALTHCHECK --interval=30s --timeout=20s --start-period=5s --retries=3 \
|
||||
CMD curl -f http://localhost:8000/health || exit 1
|
||||
|
||||
USER appuser
|
||||
|
||||
@@ -22,8 +22,8 @@ async def get_redis() -> aioredis.Redis:
|
||||
redis_client = aioredis.from_url(
|
||||
settings.REDIS_URL,
|
||||
decode_responses=True,
|
||||
socket_connect_timeout=5,
|
||||
socket_timeout=5,
|
||||
socket_connect_timeout=10,
|
||||
socket_timeout=10,
|
||||
)
|
||||
await redis_client.ping()
|
||||
logger.info("Connected to Redis")
|
||||
|
||||
@@ -66,7 +66,7 @@ EXPECTED_MIN_SIZES = {
|
||||
}
|
||||
|
||||
MAX_DOWNLOAD_RETRIES = 3
|
||||
RETRY_DELAY_SECONDS = 60
|
||||
RETRY_DELAY_SECONDS = 120
|
||||
|
||||
|
||||
class MTGJSONManager:
|
||||
@@ -123,6 +123,19 @@ class MTGJSONManager:
|
||||
logger.info("Data integrity check passed - all files meet minimum size requirements")
|
||||
return True, []
|
||||
|
||||
def _get_estimated_size(self, filename: str) -> int:
|
||||
"""Get estimated file size in MB."""
|
||||
estimates = {
|
||||
"AllPrintings.json.gz": 500,
|
||||
"AllSetFiles.zip": 10,
|
||||
"AllIdentifiers.json.gz": 100,
|
||||
"CardTypes.json.gz": 5,
|
||||
"DeckList.json.gz": 5,
|
||||
"Keywords.json.gz": 2,
|
||||
"SetList.json.gz": 10,
|
||||
}
|
||||
return estimates.get(filename, 10) # Default 10MB
|
||||
|
||||
async def cleanup_data_files(self) -> None:
|
||||
"""Delete all downloaded MTGJSON data files."""
|
||||
logger.info(f"Cleaning up data files in {self.data_dir}")
|
||||
@@ -215,7 +228,7 @@ class MTGJSONManager:
|
||||
return False
|
||||
|
||||
async def _download_file(self, session: aiohttp.ClientSession, url: str, filename: str) -> bool:
|
||||
"""Download a single file."""
|
||||
"""Download a single file with adaptive timeout based on size."""
|
||||
dest_path = self.data_dir / filename
|
||||
|
||||
# Skip if file already exists
|
||||
@@ -223,26 +236,51 @@ class MTGJSONManager:
|
||||
logger.info(f"Skipping {filename} (already exists)")
|
||||
return True
|
||||
|
||||
logger.info(f"Downloading {filename} from {url}")
|
||||
# Calculate timeout based on estimated file size
|
||||
# Large files (>100MB) get up to 60 minutes
|
||||
estimated_mb = self._get_estimated_size(filename)
|
||||
timeout = max(1200, estimated_mb * 8) # At least 20 min, 8x estimated MB
|
||||
|
||||
logger.info(f"Downloading {filename} from {url} (timeout: {timeout}s, est: {estimated_mb}MB)")
|
||||
|
||||
try:
|
||||
async with session.get(url) as response:
|
||||
# Set timeout for connection AND download
|
||||
timeout_obj = aiohttp.ClientTimeout(total=timeout)
|
||||
|
||||
async with session.get(url, timeout=timeout_obj) as response:
|
||||
if response.status != 200:
|
||||
logger.error(f"Failed to download {filename}: HTTP {response.status}")
|
||||
return False
|
||||
|
||||
# Download with progress logging
|
||||
content = b""
|
||||
async for chunk in response.content.iter_chunked(8192):
|
||||
content += chunk
|
||||
# Download with progress monitoring
|
||||
total_size = 0
|
||||
last_log_time = time.time()
|
||||
|
||||
# Write to file
|
||||
dest_path.write_bytes(content)
|
||||
logger.info(f"Downloaded {filename} ({len(content)} bytes)")
|
||||
with open(dest_path, 'wb') as f:
|
||||
async for chunk in response.content.iter_chunked(8192):
|
||||
f.write(chunk)
|
||||
total_size += len(chunk)
|
||||
|
||||
# Log progress every 60 seconds
|
||||
now = time.time()
|
||||
if now - last_log_time >= 60:
|
||||
mb_downloaded = total_size / (1024 * 1024)
|
||||
logger.info(f" Downloaded {mb_downloaded:.1f} MB of {filename}")
|
||||
last_log_time = now
|
||||
|
||||
logger.info(f"Downloaded {filename} ({total_size / (1024*1024):.1f} MB)")
|
||||
return True
|
||||
|
||||
except asyncio.TimeoutError:
|
||||
logger.error(f"Download timeout for {filename} (timeout: {timeout}s)")
|
||||
# Clean up partial download
|
||||
if dest_path.exists():
|
||||
dest_path.unlink()
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Download error for {filename}: {e}")
|
||||
if dest_path.exists():
|
||||
dest_path.unlink()
|
||||
return False
|
||||
|
||||
async def unpack_files(self) -> bool:
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check MTGJSON data status in filesystem and database."""
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
||||
import sys
|
||||
|
||||
sys.path.append("/app")
|
||||
|
||||
from app.services.mtgjson_manager import MTGJSONManager
|
||||
from app.config import get_settings
|
||||
|
||||
async def check_mtgjson_status():
|
||||
"""Comprehensive check of MTGJSON data status."""
|
||||
|
||||
print("=" * 60)
|
||||
print("MTGJSON DATA STATUS REPORT")
|
||||
print("=" * 60)
|
||||
|
||||
settings = get_settings()
|
||||
|
||||
# 1. Check data directory
|
||||
print("\n1. DATA DIRECTORY CHECK")
|
||||
print("-" * 40)
|
||||
data_dir = Path(settings.DATA_DIR)
|
||||
print(f"Data Directory: {data_dir}")
|
||||
print(f"Directory exists: {data_dir.exists()}")
|
||||
|
||||
if data_dir.exists():
|
||||
files = list(data_dir.glob("*.json.gz")) + list(data_dir.glob("*.json"))
|
||||
print(f"MTGJSON files found: {len(files)}")
|
||||
|
||||
# Check specific files
|
||||
required_files = [
|
||||
"AllPrintings.json.gz",
|
||||
"AllSetFiles.json.gz",
|
||||
"AllIdentifiers.json.gz",
|
||||
"CardTypes.json.gz",
|
||||
"Keywords.json.gz",
|
||||
"MagicRoots.json.gz",
|
||||
"MagicSets.json.gz",
|
||||
"SetTranslations.json.gz"
|
||||
]
|
||||
|
||||
missing_files = []
|
||||
existing_files = []
|
||||
|
||||
for f in required_files:
|
||||
filepath = data_dir / f
|
||||
if filepath.exists():
|
||||
size_mb = filepath.stat().st_size / (1024 * 1024)
|
||||
existing_files.append((f, size_mb))
|
||||
print(f" ✓ {f}: {size_mb:.1f} MB")
|
||||
else:
|
||||
missing_files.append(f)
|
||||
print(f" ✗ {f}: MISSING")
|
||||
|
||||
print(f"\n Summary: {len(existing_files)}/{len(required_files)} required files present")
|
||||
if missing_files:
|
||||
print(f" Missing: {', '.join(missing_files)}")
|
||||
else:
|
||||
print(" ERROR: Data directory does not exist!")
|
||||
|
||||
# 2. Check database status
|
||||
print("\n2. DATABASE STATUS CHECK")
|
||||
print("-" * 40)
|
||||
|
||||
db_url = f"postgresql+asyncpg://{settings.POSTGRES_USER}:{settings.POSTGRES_PASSWORD}"
|
||||
db_url += f"@postgres-mtgdata:5432/{settings.POSTGRES_DB}"
|
||||
|
||||
try:
|
||||
engine = create_async_engine(db_url)
|
||||
|
||||
async with AsyncSession(engine) as session:
|
||||
# Check tables
|
||||
result = await session.execute(text("""
|
||||
SELECT table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'public'
|
||||
ORDER BY table_name;
|
||||
"""))
|
||||
|
||||
tables = [row[0] for row in result.fetchall()]
|
||||
print(f"Tables found: {len(tables)}")
|
||||
for table in tables:
|
||||
print(f" - {table}")
|
||||
|
||||
# Check key tables
|
||||
print("\nKey table statistics:")
|
||||
key_tables = ['mtg_set', 'mtg_card', 'mtg_identifiers', 'mtg_keywords', 'mtg_refresh_log']
|
||||
|
||||
for table in key_tables:
|
||||
if table in tables:
|
||||
result = await session.execute(text(f"SELECT COUNT(*) FROM {table}"))
|
||||
count = result.scalar()
|
||||
print(f" {table}: {count:,} records")
|
||||
else:
|
||||
print(f" {table}: TABLE NOT FOUND")
|
||||
|
||||
# Check refresh log
|
||||
if 'mtg_refresh_log' in tables:
|
||||
result = await session.execute(text("""
|
||||
SELECT refresh_type, status, created_at, error_message
|
||||
FROM mtg_refresh_log
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 5;
|
||||
"""))
|
||||
|
||||
rows = result.fetchall()
|
||||
if rows:
|
||||
print("\nRecent refresh operations:")
|
||||
for row in rows:
|
||||
status_icon = "✓" if row[1] == 'SUCCESS' else "✗"
|
||||
print(f" {status_icon} {row[0]}: {row[1]} at {row[2]}")
|
||||
if row[3]:
|
||||
print(f" Error: {row[3]}")
|
||||
else:
|
||||
print("\nNo refresh operations logged")
|
||||
|
||||
except Exception as e:
|
||||
print(f"ERROR: Could not connect to database: {e}")
|
||||
return
|
||||
|
||||
# 3. Check MTGJSON manager status
|
||||
print("\n3. MTGJSON MANAGER STATUS")
|
||||
print("-" * 40)
|
||||
|
||||
try:
|
||||
manager = MTGJSONManager()
|
||||
status = manager.get_status()
|
||||
|
||||
print(f"Status: {status['status']}")
|
||||
if status.get('data'):
|
||||
print(f" Sets count: {status['data'].get('sets_count', 0)}")
|
||||
print(f" Cards count: {status['data'].get('cards_count', 0)}")
|
||||
print(f" Last refresh: {status['data'].get('last_refresh')}")
|
||||
except Exception as e:
|
||||
print(f"ERROR: Could not create MTGJSON manager: {e}")
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(check_mtgjson_status())
|
||||
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check MTGJSON data status in the database and filesystem."""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from pathlib import Path
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
||||
import sys
|
||||
sys.path.append("/home/wall-o/projects/mtgonline/backend")
|
||||
|
||||
from app.services.mtgjson_manager import MTGJSONManager
|
||||
from app.config import get_settings
|
||||
|
||||
async def check_mtgjson_status():
|
||||
"""Check MTGJSON data download and database status."""
|
||||
|
||||
print("=== MTGJSON Data Status Check ===\n")
|
||||
|
||||
# Check data directory
|
||||
settings = get_settings()
|
||||
data_dir = Path(settings.DATA_DIR)
|
||||
|
||||
print(f"Data Directory: {data_dir}")
|
||||
print(f"Directory exists: {data_dir.exists()}")
|
||||
|
||||
if data_dir.exists():
|
||||
files = list(data_dir.glob("*.json.gz"))
|
||||
files += list(data_dir.glob("*.json"))
|
||||
print(f"Found {len(files)} MTGJSON files:")
|
||||
for f in sorted(files)[:20]: # Show first 20
|
||||
size_mb = f.stat().st_size / (1024 * 1024)
|
||||
print(f" - {f.name} ({size_mb:.1f} MB)")
|
||||
if len(files) > 20:
|
||||
print(f" ... and {len(files) - 20} more files")
|
||||
else:
|
||||
print("WARNING: Data directory does not exist!")
|
||||
|
||||
# Check database status
|
||||
print("\n=== Database Status ===")
|
||||
|
||||
try:
|
||||
# Try to connect to the MTGJSON database
|
||||
engine = create_async_engine(
|
||||
f"postgresql+asyncpg://{settings.POSTGRES_USER}:{settings.POSTGRES_PASSWORD}"
|
||||
f"@postgres-mtgdata:5432/{settings.POSTGRES_DB}"
|
||||
)
|
||||
|
||||
async with AsyncSession(engine) as session:
|
||||
# Check if tables exist
|
||||
result = await session.execute(text("""
|
||||
SELECT table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'public'
|
||||
ORDER BY table_name;
|
||||
"""))
|
||||
|
||||
tables = [row[0] for row in result.fetchall()]
|
||||
print(f"Found {len(tables)} tables in database:")
|
||||
for table in tables:
|
||||
print(f" - {table}")
|
||||
|
||||
# Check specific MTGJSON tables
|
||||
mtg_tables = ['mtg_set', 'mtg_card', 'mtg_identifiers', 'mtg_keywords']
|
||||
if 'mtg_refresh_log' in tables:
|
||||
result = await session.execute(text("SELECT COUNT(*) FROM mtg_refresh_log"))
|
||||
count = result.scalar()
|
||||
print(f"\nRefresh log entries: {count}")
|
||||
|
||||
# Check key tables
|
||||
for table in ['mtg_set', 'mtg_card', 'mtg_identifiers']:
|
||||
if table in tables:
|
||||
result = await session.execute(text(f"SELECT COUNT(*) FROM {table}"))
|
||||
count = result.scalar()
|
||||
print(f"{table}: {count:,} records")
|
||||
|
||||
except Exception as e:
|
||||
print(f"ERROR connecting to database: {e}")
|
||||
|
||||
# Try to create MTGJSON manager and check status
|
||||
print("\n=== MTGJSON Manager Status ===")
|
||||
try:
|
||||
manager = MTGJSONManager()
|
||||
status = manager.get_status()
|
||||
print(f"Status: {status}")
|
||||
except Exception as e:
|
||||
print(f"ERROR creating MTGJSON manager: {e}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(check_mtgjson_status())
|
||||
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
MTGJSON v5 Data Downloader
|
||||
|
||||
Downloads MTGJSON v5 data files and loads them into the database.
|
||||
Updated to use current MTGJSON API endpoints.
|
||||
|
||||
Usage:
|
||||
python download_mtgjson_v5.py
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add the app directory to Python path
|
||||
app_dir = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(app_dir))
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import gzip
|
||||
import logging
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
||||
import aiohttp
|
||||
|
||||
from app.services.mtgjson_manager import MTGJSONManager
|
||||
from app.config import get_settings
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# MTGJSON v5 API base URL
|
||||
MTGJSON_API_BASE = "https://mtgjson.com/api/v5"
|
||||
|
||||
|
||||
async def download_file(session, url, dest_path):
|
||||
"""Download a file from URL."""
|
||||
logger.info(f"Downloading: {url}")
|
||||
|
||||
try:
|
||||
async with session.get(url, timeout=aiohttp.ClientTimeout(total=600)) as response:
|
||||
if response.status == 200:
|
||||
dest_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(dest_path, 'wb') as f:
|
||||
async for chunk in response.content.iter_chunked(8192):
|
||||
f.write(chunk)
|
||||
logger.info(f"✓ Downloaded: {dest_path.name}")
|
||||
return True
|
||||
else:
|
||||
logger.error(f"✗ Failed to download {url}: HTTP {response.status}")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"✗ Error downloading {url}: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def extract_gzip(gz_path, dest_path=None):
|
||||
"""Extract a gzip file."""
|
||||
if not dest_path:
|
||||
dest_path = gz_path.with_suffix('')
|
||||
|
||||
logger.info(f"Extracting: {gz_path.name}")
|
||||
|
||||
try:
|
||||
with gzip.open(gz_path, 'rt', encoding='utf-8') as f_in:
|
||||
with open(dest_path, 'w', encoding='utf-8') as f_out:
|
||||
f_out.write(f_in.read())
|
||||
logger.info(f"✓ Extracted: {gz_path.name}")
|
||||
# Remove gz file after extraction
|
||||
gz_path.unlink()
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"✗ Failed to extract {gz_path}: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def download_all_files(data_dir):
|
||||
"""Download all MTGJSON v5 files."""
|
||||
logger.info("=" * 60)
|
||||
logger.info("MTGJSON v5 Data Download")
|
||||
logger.info("=" * 60)
|
||||
|
||||
files = [
|
||||
"AllPrintings.json.gz",
|
||||
"AllSetFiles.json.gz",
|
||||
"AllIdentifiers.json.gz",
|
||||
"CardTypes.json.gz",
|
||||
"Keywords.json.gz",
|
||||
"MagicRoots.json.gz",
|
||||
"MagicSets.json.gz",
|
||||
"SetTranslations.json.gz"
|
||||
]
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
for filename in files:
|
||||
url = f"{MTGJSON_API_BASE}/{filename}"
|
||||
dest_path = data_dir / filename
|
||||
await download_file(session, url, dest_path)
|
||||
await asyncio.sleep(1) # Be nice to the API
|
||||
|
||||
|
||||
async def load_data_to_database(data_dir):
|
||||
"""Load downloaded MTGJSON data into PostgreSQL."""
|
||||
logger.info("=" * 60)
|
||||
logger.info("Loading MTGJSON data into database")
|
||||
logger.info("=" * 60)
|
||||
|
||||
manager = MTGJSONManager(data_dir)
|
||||
|
||||
# Download files
|
||||
success = await manager.download_files()
|
||||
|
||||
if not success:
|
||||
logger.error("✗ Download failed")
|
||||
return False
|
||||
|
||||
# Unpack files
|
||||
success = manager.unpack_files()
|
||||
|
||||
if not success:
|
||||
logger.error("✗ Unpack failed")
|
||||
return False
|
||||
|
||||
# Upsert to database
|
||||
success = manager.upsert_to_database()
|
||||
|
||||
if success:
|
||||
logger.info("✓ Data loaded successfully")
|
||||
else:
|
||||
logger.error("✗ Database upsert failed")
|
||||
|
||||
return success
|
||||
|
||||
|
||||
async def main():
|
||||
"""Main entry point."""
|
||||
settings = get_settings()
|
||||
data_dir = Path(settings.DATA_DIR)
|
||||
|
||||
logger.info(f"Data directory: {data_dir}")
|
||||
|
||||
# Clear corrupted data
|
||||
if data_dir.exists():
|
||||
logger.info("Clearing corrupted data...")
|
||||
for f in data_dir.glob("*.json.gz"):
|
||||
f.unlink()
|
||||
logger.info(f" Removed: {f.name}")
|
||||
|
||||
data_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Download and load data
|
||||
await load_data_to_database(data_dir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Executable
+40
@@ -0,0 +1,40 @@
|
||||
#!/bin/bash
|
||||
# MTGJSON Data Maintenance Script
|
||||
# This script handles data download, validation, and cleanup
|
||||
set -e
|
||||
|
||||
echo "=== MTGJSON Data Maintenance ==="
|
||||
echo "Starting maintenance at: $(date)"
|
||||
|
||||
# Run the download with sanity checks
|
||||
echo "Running download with sanity checks..."
|
||||
docker exec mtgonline_backend python -m app.services.mtgjson_manager --refresh --force
|
||||
|
||||
# Verify the download
|
||||
echo ""
|
||||
echo "Verifying data integrity..."
|
||||
docker exec mtgonline_backend python -c "
|
||||
import sys
|
||||
sys.path.insert(0, '/app')
|
||||
from app.services.mtgjson_manager import MTGJSONManager
|
||||
from pathlib import Path
|
||||
import asyncio
|
||||
|
||||
async def verify():
|
||||
manager = MTGJSONManager(Path('/app/data/mtgjson'))
|
||||
is_valid, issues = await manager.validate_data_integrity()
|
||||
|
||||
if is_valid:
|
||||
print('✓ Data integrity check passed')
|
||||
return 0
|
||||
else:
|
||||
print('✗ Data integrity check failed')
|
||||
for issue in issues:
|
||||
print(f' - {issue}')
|
||||
return 1
|
||||
|
||||
sys.exit(asyncio.run(verify()))
|
||||
"
|
||||
|
||||
echo ""
|
||||
echo "=== Maintenance Complete ==="
|
||||
@@ -0,0 +1,148 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
MTGJSON Data Sanity Check
|
||||
|
||||
Validates downloaded MTGJSON files for expected sizes before upserting to database.
|
||||
This prevents corrupted or incomplete data from being loaded into PostgreSQL.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Expected minimum file sizes (in bytes) for MTGJSON v5 files
|
||||
# These are approximate minimums based on typical MTGJSON data sizes
|
||||
EXPECTED_MIN_SIZES = {
|
||||
"AllPrintings.json": 500 * 1024 * 1024, # 500 MB (should be 500-600 MB)
|
||||
"AllSetFiles.json": 10 * 1024 * 1024, # 10 MB
|
||||
"AllIdentifiers.json": 100 * 1024 * 1024, # 100 MB
|
||||
"CardTypes.json": 1 * 1024 * 1024, # 1 MB
|
||||
"Keywords.json": 0.5 * 1024 * 1024, # 0.5 MB
|
||||
"MagicSets.json": 50 * 1024 * 1024, # 50 MB
|
||||
"MagicRoots.json": 1 * 1024 * 1024, # 1 MB
|
||||
"SetTranslations.json": 5 * 1024 * 1024, # 5 MB
|
||||
}
|
||||
|
||||
|
||||
def validate_file_sizes(data_dir: Path) -> dict:
|
||||
"""
|
||||
Validate downloaded MTGJSON files for expected sizes.
|
||||
|
||||
Args:
|
||||
data_dir: Path to the MTGJSON data directory
|
||||
|
||||
Returns:
|
||||
Dict with validation results
|
||||
"""
|
||||
results = {
|
||||
"valid": True,
|
||||
"files_checked": 0,
|
||||
"files_valid": 0,
|
||||
"files_invalid": 0,
|
||||
"issues": []
|
||||
}
|
||||
|
||||
if not data_dir.exists():
|
||||
results["valid"] = False
|
||||
results["issues"].append(f"Data directory does not exist: {data_dir}")
|
||||
return results
|
||||
|
||||
# Check each expected file
|
||||
for filename, min_size in EXPECTED_MIN_SIZES.items():
|
||||
filepath = data_dir / filename
|
||||
|
||||
if not filepath.exists():
|
||||
results["issues"].append(f"Missing file: {filename}")
|
||||
results["valid"] = False
|
||||
results["files_checked"] += 1
|
||||
results["files_invalid"] += 1
|
||||
continue
|
||||
|
||||
results["files_checked"] += 1
|
||||
actual_size = filepath.stat().st_size
|
||||
|
||||
if actual_size < min_size:
|
||||
results["valid"] = False
|
||||
results["files_invalid"] += 1
|
||||
results["issues"].append(
|
||||
f"{filename}: {actual_size / (1024*1024):.1f} MB (minimum: {min_size / (1024*1024):.1f} MB)"
|
||||
)
|
||||
logger.warning(
|
||||
f"File {filename} is too small: {actual_size / (1024*1024):.1f} MB "
|
||||
f"(expected minimum: {min_size / (1024*1024):.1f} MB)"
|
||||
)
|
||||
else:
|
||||
results["files_valid"] += 1
|
||||
logger.info(
|
||||
f"✓ {filename}: {actual_size / (1024*1024):.1f} MB (OK)"
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
async def validate_and_cleanup(data_dir: Path, max_retries: int = 3) -> bool:
|
||||
"""
|
||||
Validate MTGJSON files and cleanup if invalid.
|
||||
|
||||
Args:
|
||||
data_dir: Path to the MTGJSON data directory
|
||||
max_retries: Maximum number of retry attempts
|
||||
|
||||
Returns:
|
||||
True if validation passes, False otherwise
|
||||
"""
|
||||
logger.info("=" * 60)
|
||||
logger.info("MTGJSON Data Sanity Check")
|
||||
logger.info("=" * 60)
|
||||
|
||||
for attempt in range(1, max_retries + 1):
|
||||
logger.info(f"\nAttempt {attempt}/{max_retries}")
|
||||
|
||||
# Validate file sizes
|
||||
results = validate_file_sizes(data_dir)
|
||||
|
||||
if results["valid"]:
|
||||
logger.info("\n✓ All files passed validation")
|
||||
logger.info(f" Checked: {results['files_checked']} files")
|
||||
logger.info(f" Valid: {results['files_valid']} files")
|
||||
return True
|
||||
|
||||
# Validation failed
|
||||
logger.warning("\n✗ Validation failed:")
|
||||
for issue in results["issues"]:
|
||||
logger.warning(f" - {issue}")
|
||||
|
||||
if attempt < max_retries:
|
||||
logger.warning(f"\nCleanup and retry in {60 * attempt} seconds...")
|
||||
await asyncio.sleep(60 * attempt)
|
||||
|
||||
# Delete all downloaded files
|
||||
logger.warning("Deleting downloaded files...")
|
||||
for f in data_dir.glob("*"):
|
||||
if f.is_file():
|
||||
f.unlink()
|
||||
logger.warning(f" Deleted: {f.name}")
|
||||
|
||||
logger.error("\n✗✗✗ All retry attempts failed ✗✗✗")
|
||||
return False
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
# Get data directory from settings or use default
|
||||
try:
|
||||
sys.path.insert(0, "/app")
|
||||
from app.config import get_settings
|
||||
settings = get_settings()
|
||||
data_dir = Path(settings.DATA_DIR)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load settings: {e}")
|
||||
data_dir = Path("/app/data/mtgjson")
|
||||
|
||||
# Run validation
|
||||
asyncio.run(validate_and_cleanup(data_dir))
|
||||
@@ -0,0 +1,226 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Comprehensive MTGJSON Data Verification
|
||||
|
||||
Checks both:
|
||||
1. MTGJSON data file downloads
|
||||
2. PostgreSQL database upsert status
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
||||
import sys
|
||||
|
||||
sys.path.append("/app")
|
||||
|
||||
from app.services.mtgjson_manager import MTGJSONManager
|
||||
from app.config import get_settings
|
||||
|
||||
|
||||
async def verify_data_download():
|
||||
"""Verify MTGJSON data files were downloaded."""
|
||||
print("=" * 70)
|
||||
print("MTGJSON DATA DOWNLOAD VERIFICATION")
|
||||
print("=" * 70)
|
||||
|
||||
settings = get_settings()
|
||||
data_dir = Path(settings.DATA_DIR)
|
||||
|
||||
print(f"\nData Directory: {data_dir}")
|
||||
print(f"Directory exists: {data_dir.exists()}")
|
||||
|
||||
if not data_dir.exists():
|
||||
print("❌ FAIL: Data directory does not exist")
|
||||
return False
|
||||
|
||||
# Check for required files
|
||||
required_files = {
|
||||
"AllPrintings.json.gz": "All sets data",
|
||||
"AllSetFiles.json.gz": "Set metadata",
|
||||
"AllIdentifiers.json.gz": "Card identifiers",
|
||||
"CardTypes.json.gz": "Card type definitions",
|
||||
"Keywords.json.gz": "Card keywords",
|
||||
"MagicRoots.json.gz": "Root data",
|
||||
"MagicSets.json.gz": "Set data",
|
||||
"SetTranslations.json.gz": "Set translations"
|
||||
}
|
||||
|
||||
downloaded_files = []
|
||||
missing_files = []
|
||||
|
||||
print("\nRequired MTGJSON files:")
|
||||
for filename, description in required_files.items():
|
||||
filepath = data_dir / filename
|
||||
if filepath.exists():
|
||||
size_mb = filepath.stat().st_size / (1024 * 1024)
|
||||
downloaded_files.append(filename)
|
||||
print(f" ✓ {filename:30} - {size_mb:8.1f} MB")
|
||||
else:
|
||||
missing_files.append(filename)
|
||||
print(f" ✗ {filename:30} - MISSING")
|
||||
|
||||
print(f"\nDownload Status: {len(downloaded_files)}/{len(required_files)} files")
|
||||
|
||||
if missing_files:
|
||||
print(f"\n❌ FAIL: Missing {len(missing_files)} required files: {', '.join(missing_files)}")
|
||||
return False
|
||||
|
||||
# Check file sizes for sanity
|
||||
allprintings_path = data_dir / "AllPrintings.json.gz"
|
||||
if allprintings_path.exists():
|
||||
size_mb = allprintings_path.stat().st_size / (1024 * 1024)
|
||||
if size_mb < 100:
|
||||
print(f"\n⚠️ WARNING: AllPrintings.json.gz is suspiciously small ({size_mb:.1f} MB). Expected ~500-600 MB")
|
||||
return False
|
||||
else:
|
||||
print(f"\n✓ AllPrintings.json.gz size looks good: {size_mb:.1f} MB")
|
||||
|
||||
print("\n✓ PASS: All MTGJSON data files downloaded successfully")
|
||||
return True
|
||||
|
||||
|
||||
async def verify_database_upsert():
|
||||
"""Verify MTGJSON data was properly upserted to PostgreSQL."""
|
||||
print("\n" + "=" * 70)
|
||||
print("POSTGRESQL DATABASE UPSERT VERIFICATION")
|
||||
print("=" * 70)
|
||||
|
||||
settings = get_settings()
|
||||
db_url = f"postgresql+asyncpg://{settings.POSTGRES_USER}:{settings.POSTGRES_PASSWORD}"
|
||||
db_url += f"@postgres-mtgdata:5432/{settings.POSTGRES_DB}"
|
||||
|
||||
print(f"\nDatabase: {settings.POSTGRES_DB}")
|
||||
|
||||
try:
|
||||
engine = create_async_engine(db_url)
|
||||
|
||||
async with AsyncSession(engine) as session:
|
||||
# Get all tables
|
||||
result = await session.execute(text("""
|
||||
SELECT table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'public'
|
||||
ORDER BY table_name;
|
||||
"""))
|
||||
tables = [row[0] for row in result.fetchall()]
|
||||
|
||||
print(f"\nTotal tables: {len(tables)}")
|
||||
|
||||
# Check MTGJSON-specific tables
|
||||
mtg_tables = ['mtg_set', 'mtg_card', 'mtg_identifiers', 'mtg_keywords']
|
||||
|
||||
print("\nMTGJSON tables:")
|
||||
for table in mtg_tables:
|
||||
if table in tables:
|
||||
result = await session.execute(text(f"SELECT COUNT(*) FROM {table}"))
|
||||
count = result.scalar()
|
||||
print(f" ✓ {table:30} - {count:8,} records")
|
||||
else:
|
||||
print(f" ✗ {table:30} - TABLE NOT FOUND")
|
||||
|
||||
# Check refresh log
|
||||
if 'mtg_refresh_log' in tables:
|
||||
result = await session.execute(text("""
|
||||
SELECT refresh_type, status, created_at
|
||||
FROM mtg_refresh_log
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 5;
|
||||
"""))
|
||||
rows = result.fetchall()
|
||||
|
||||
if rows:
|
||||
print("\nRecent refresh operations:")
|
||||
for row in rows:
|
||||
status_icon = "✓" if row[1] == 'SUCCESS' else "✗"
|
||||
print(f" {status_icon} {row[0]:15} - {row[1]:8} - {row[2]}")
|
||||
|
||||
# Verify data quality
|
||||
print("\nData quality checks:")
|
||||
|
||||
# Check for sets
|
||||
if 'mtg_set' in tables:
|
||||
result = await session.execute(text("""
|
||||
SELECT COUNT(*) FROM mtg_set
|
||||
WHERE set_name IS NOT NULL AND set_code IS NOT NULL;
|
||||
"""))
|
||||
valid_sets = result.scalar()
|
||||
result = await session.execute(text("SELECT COUNT(*) FROM mtg_set"))
|
||||
total_sets = result.scalar()
|
||||
print(f" ✓ Sets: {valid_sets:,}/{total_sets:,} valid")
|
||||
|
||||
# Check for cards
|
||||
if 'mtg_card' in tables:
|
||||
result = await session.execute(text("""
|
||||
SELECT COUNT(*) FROM mtg_card
|
||||
WHERE name IS NOT NULL AND mtgjson_cards_id IS NOT NULL;
|
||||
"""))
|
||||
valid_cards = result.scalar()
|
||||
result = await session.execute(text("SELECT COUNT(*) FROM mtg_card"))
|
||||
total_cards = result.scalar()
|
||||
print(f" ✓ Cards: {valid_cards:,}/{total_cards:,} valid")
|
||||
|
||||
# Check for identifiers
|
||||
if 'mtg_identifiers' in tables:
|
||||
result = await session.execute(text("""
|
||||
SELECT COUNT(*) FROM mtg_identifiers
|
||||
WHERE scryfall_id IS NOT NULL;
|
||||
"""))
|
||||
valid_ids = result.scalar()
|
||||
result = await session.execute(text("SELECT COUNT(*) FROM mtg_identifiers"))
|
||||
total_ids = result.scalar()
|
||||
print(f" ✓ Identifiers: {valid_ids:,}/{total_ids:,} valid")
|
||||
|
||||
# Check for keywords
|
||||
if 'mtg_keywords' in tables:
|
||||
result = await session.execute(text("SELECT COUNT(*) FROM mtg_keywords"))
|
||||
keywords_count = result.scalar()
|
||||
print(f" ✓ Keywords: {keywords_count:,}")
|
||||
|
||||
print("\n✓ PASS: Database upsert completed successfully")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ FAIL: Database error - {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def main():
|
||||
"""Main verification function."""
|
||||
print("\n" + "=" * 70)
|
||||
print("MTGJSON DATA INTEGRATION VERIFICATION")
|
||||
print("=" * 70)
|
||||
|
||||
# Check data download
|
||||
download_ok = await verify_data_download()
|
||||
|
||||
# Check database upsert
|
||||
db_ok = await verify_database_upsert()
|
||||
|
||||
# Final summary
|
||||
print("\n" + "=" * 70)
|
||||
print("VERIFICATION SUMMARY")
|
||||
print("=" * 70)
|
||||
|
||||
if download_ok and db_ok:
|
||||
print("\n✓✓✓ ALL CHECKS PASSED ✓✓✓")
|
||||
print("\nMTGJSON data has been successfully downloaded and upserted to PostgreSQL.")
|
||||
print("The backend is ready to use.")
|
||||
return 0
|
||||
else:
|
||||
print("\n❌❌❌ VERIFICATION FAILED ❌❌❌")
|
||||
if not download_ok:
|
||||
print("\nDownload issues:")
|
||||
print(" - Some MTGJSON data files are missing or corrupted")
|
||||
print(" - Run: docker exec mtgonline_backend python /app/scripts/download_mtgjson_v5.py")
|
||||
if not db_ok:
|
||||
print("\nDatabase issues:")
|
||||
print(" - Data was not properly upserted to PostgreSQL")
|
||||
print(" - Check backend logs for errors")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit_code = asyncio.run(main())
|
||||
sys.exit(exit_code)
|
||||
@@ -0,0 +1,194 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
MTGJSON Data Verification Script
|
||||
|
||||
Checks:
|
||||
1. Data file download status
|
||||
2. Sanity check validation
|
||||
3. Database upsert status
|
||||
4. File size validation
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, "/app")
|
||||
|
||||
from app.config import get_settings
|
||||
from app.services.mtgjson_manager import MTGJSONManager
|
||||
|
||||
# Expected minimum file sizes (in bytes)
|
||||
EXPECTED_MIN_SIZES = {
|
||||
"AllPrintings.json": 500 * 1024 * 1024, # 500 MB
|
||||
"AllSetFiles.json": 10 * 1024 * 1024, # 10 MB
|
||||
"AllIdentifiers.json": 100 * 1024 * 1024, # 100 MB
|
||||
"CardTypes.json": 1 * 1024 * 1024, # 1 MB
|
||||
"Keywords.json": 0.5 * 1024 * 1024, # 0.5 MB
|
||||
"MagicSets.json": 50 * 1024 * 1024, # 50 MB
|
||||
"MagicRoots.json": 1 * 1024 * 1024, # 1 MB
|
||||
"SetTranslations.json": 5 * 1024 * 1024, # 5 MB
|
||||
}
|
||||
|
||||
|
||||
async def check_data_files(data_dir: Path) -> dict:
|
||||
"""Check downloaded data files and validate sizes."""
|
||||
print("=" * 70)
|
||||
print("DATA FILE VERIFICATION")
|
||||
print("=" * 70)
|
||||
|
||||
results = {
|
||||
"valid": True,
|
||||
"files_checked": 0,
|
||||
"files_valid": 0,
|
||||
"files_invalid": 0,
|
||||
"issues": []
|
||||
}
|
||||
|
||||
if not data_dir.exists():
|
||||
results["valid"] = False
|
||||
results["issues"].append(f"Data directory does not exist: {data_dir}")
|
||||
return results
|
||||
|
||||
# Check each expected file
|
||||
for filename, min_size in EXPECTED_MIN_SIZES.items():
|
||||
filepath = data_dir / filename
|
||||
|
||||
if not filepath.exists():
|
||||
results["issues"].append(f"Missing file: {filename}")
|
||||
results["valid"] = False
|
||||
results["files_checked"] += 1
|
||||
results["files_invalid"] += 1
|
||||
continue
|
||||
|
||||
results["files_checked"] += 1
|
||||
actual_size = filepath.stat().st_size
|
||||
|
||||
if actual_size < min_size:
|
||||
results["valid"] = False
|
||||
results["files_invalid"] += 1
|
||||
results["issues"].append(
|
||||
f"{filename}: {actual_size / (1024*1024):.1f} MB (minimum: {min_size / (1024*1024):.1f} MB)"
|
||||
)
|
||||
print(f" ✗ {filename:30} - {actual_size / (1024*1024):8.1f} MB (too small)")
|
||||
else:
|
||||
results["files_valid"] += 1
|
||||
print(f" ✓ {filename:30} - {actual_size / (1024*1024):8.1f} MB")
|
||||
|
||||
print(f"\nSummary: {results['files_valid']}/{results['files_checked']} files valid")
|
||||
|
||||
if results["issues"]:
|
||||
print("\nIssues:")
|
||||
for issue in results["issues"]:
|
||||
print(f" - {issue}")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
async def check_database_status() -> dict:
|
||||
"""Check database upsert status."""
|
||||
print("\n" + "=" * 70)
|
||||
print("DATABASE STATUS VERIFICATION")
|
||||
print("=" * 70)
|
||||
|
||||
settings = get_settings()
|
||||
db_url = f"postgresql+asyncpg://{settings.POSTGRES_USER}:{settings.POSTGRES_PASSWORD}"
|
||||
db_url += f"@postgres-mtgdata:5432/{settings.POSTGRES_DB}"
|
||||
|
||||
try:
|
||||
engine = create_async_engine(db_url)
|
||||
|
||||
async with AsyncSession(engine) as session:
|
||||
# Get all tables
|
||||
result = await session.execute(text("""
|
||||
SELECT table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'public'
|
||||
ORDER BY table_name;
|
||||
"""))
|
||||
tables = [row[0] for row in result.fetchall()]
|
||||
|
||||
print(f"Total tables: {len(tables)}")
|
||||
|
||||
# Check MTGJSON tables
|
||||
mtg_tables = ['mtg_set', 'mtg_card', 'mtg_identifiers', 'mtg_keywords']
|
||||
|
||||
for table in mtg_tables:
|
||||
if table in tables:
|
||||
result = await session.execute(text(f"SELECT COUNT(*) FROM {table}"))
|
||||
count = result.scalar()
|
||||
print(f" ✓ {table:30} - {count:8,} records")
|
||||
else:
|
||||
print(f" ✗ {table:30} - TABLE NOT FOUND")
|
||||
|
||||
# Check refresh log
|
||||
if 'mtg_refresh_log' in tables:
|
||||
result = await session.execute(text("""
|
||||
SELECT refresh_type, status, created_at
|
||||
FROM mtg_refresh_log
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 5;
|
||||
"""))
|
||||
rows = result.fetchall()
|
||||
|
||||
if rows:
|
||||
print("\nRecent refresh operations:")
|
||||
for row in rows:
|
||||
status_icon = "✓" if row[1] == 'SUCCESS' else "✗"
|
||||
print(f" {status_icon} {row[0]:15} - {row[1]:8} - {row[2]}")
|
||||
|
||||
return {"valid": True, "error": None}
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n✗ Database error: {e}")
|
||||
return {"valid": False, "error": str(e)}
|
||||
|
||||
|
||||
async def main():
|
||||
"""Main verification function."""
|
||||
print("\n" + "=" * 70)
|
||||
print("MTGJSON DATA INTEGRATION VERIFICATION")
|
||||
print("=" * 70)
|
||||
|
||||
settings = get_settings()
|
||||
data_dir = Path(settings.DATA_DIR)
|
||||
|
||||
# Check data files
|
||||
file_results = await check_data_files(data_dir)
|
||||
|
||||
# Check database
|
||||
db_results = await check_database_status()
|
||||
|
||||
# Final summary
|
||||
print("\n" + "=" * 70)
|
||||
print("VERIFICATION SUMMARY")
|
||||
print("=" * 70)
|
||||
|
||||
if file_results["valid"] and db_results["valid"]:
|
||||
print("\n✓✓✓ ALL CHECKS PASSED ✓✓✓")
|
||||
print("\nThe MTGJSON data has been properly downloaded and upserted.")
|
||||
return 0
|
||||
else:
|
||||
print("\n❌ VERIFICATION FAILED")
|
||||
|
||||
if not file_results["valid"]:
|
||||
print("\nData file issues:")
|
||||
for issue in file_results["issues"]:
|
||||
print(f" - {issue}")
|
||||
|
||||
if not db_results["valid"]:
|
||||
print(f"\nDatabase issues: {db_results['error']}")
|
||||
|
||||
print("\nSOLUTION:")
|
||||
print(" 1. Delete corrupted data files")
|
||||
print(" 2. Run fresh download with sanity checks:")
|
||||
print(" docker exec mtgonline_backend python /app/scripts/download_mtgjson_v5.py")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit_code = asyncio.run(main())
|
||||
sys.exit(exit_code)
|
||||
@@ -14,7 +14,7 @@ services:
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U mtgonline_user"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
|
||||
redis:
|
||||
@@ -26,7 +26,7 @@ services:
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
|
||||
volumes:
|
||||
|
||||
+4
-4
@@ -21,7 +21,7 @@ services:
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-mtgonline}"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
start_period: 30s
|
||||
|
||||
@@ -45,7 +45,7 @@ services:
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-mtgonline}"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
start_period: 30s
|
||||
|
||||
@@ -61,7 +61,7 @@ services:
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "redis-cli ping | grep PONG"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
start_period: 5s
|
||||
|
||||
@@ -116,7 +116,7 @@ services:
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -f http://localhost:8000/health || exit 1"]
|
||||
interval: 30s
|
||||
timeout: 30s
|
||||
timeout: 60s
|
||||
retries: 3
|
||||
start_period: 600s # 10 minutes for initial data download
|
||||
|
||||
|
||||
+12
-9
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"task_description": "MTG Online Backend - MTGJSON Data Integration & Docker Deployment",
|
||||
"current_step": "Deploying test instance with sanity check verification",
|
||||
"current_step": "Timeouts doubled to prevent timeout errors",
|
||||
"files_created": [
|
||||
"backend/app/services/mtgjson_manager.py",
|
||||
"backend/scripts/sanity_check_mtgjson.py",
|
||||
@@ -11,7 +11,9 @@
|
||||
"backend/app/services/mtgjson_manager.py",
|
||||
"backend/app/scripts/refresh_mtg.py",
|
||||
"backend/app/main.py",
|
||||
"docker-compose.yml"
|
||||
"docker-compose.yml",
|
||||
"docker-compose.dev.yml",
|
||||
"backend/Dockerfile"
|
||||
],
|
||||
"decisions": [
|
||||
"Created comprehensive MTGJSONManager service in app/services/",
|
||||
@@ -23,15 +25,16 @@
|
||||
"Added file size validation against expected minimums (500MB for AllPrintings, etc.)",
|
||||
"Implemented retry logic with exponential backoff (up to 3 attempts)",
|
||||
"Cleanup deletes corrupted data before retry",
|
||||
"Container marked unhealthy if validation fails after all retries"
|
||||
"Container marked unhealthy if validation fails after all retries",
|
||||
"Doubled all timeout values to prevent timeouts during download/upsert"
|
||||
],
|
||||
"next_steps": [
|
||||
"Stop and destroy all Docker containers",
|
||||
"Build backend Docker container",
|
||||
"Deploy stack and verify health",
|
||||
"Check logs for successful initialization with sanity checks"
|
||||
"Deploy stack and monitor startup",
|
||||
"Wait for backend download phase to complete (~10-15 min)",
|
||||
"Verify container health status",
|
||||
"Check logs for sanity check results"
|
||||
],
|
||||
"blockers": [],
|
||||
"commit_hash": "dedc9a3",
|
||||
"timestamp": 1784598300
|
||||
"commit_hash": "0643544",
|
||||
"timestamp": 1784598400
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user