akadmin
99c7d08bb1
- Created MTGJSONManager service for complete data lifecycle - Handles download, unpack (gzip/zip), and PostgreSQL upsert - ON CONFLICT DO UPDATE preserves existing data - Startup triggers initial download on first container init - Health check verifies MTG data exists in database - Weekly refresh via MTG_REFRESH_INTERVAL_DAYS setting - Updated docker-compose start_period to 600s for download time
86 lines
2.6 KiB
Python
86 lines
2.6 KiB
Python
"""
|
|
MTGJSON Database Refresh Script
|
|
|
|
Downloads and updates the MTGJSON datasets weekly.
|
|
Uses the MTGJSONManager service for all data operations.
|
|
|
|
Usage:
|
|
python -m app.scripts.refresh_mtg
|
|
python -m app.scripts.refresh_mtg --force
|
|
"""
|
|
|
|
import asyncio
|
|
import argparse
|
|
import logging
|
|
import time
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
|
|
from app.core.settings import get_settings
|
|
from app.services.mtgjson_manager import MTGJSONManager
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
async def run_refresh(force: bool = False):
|
|
"""Run the refresh cycle."""
|
|
settings = get_settings()
|
|
manager = MTGJSONManager(settings.DATA_DIR)
|
|
|
|
start_time = time.time()
|
|
|
|
try:
|
|
# Check if refresh is needed
|
|
last_refresh = await manager.get_last_refresh()
|
|
|
|
if not force and not manager.is_refresh_needed(last_refresh, settings.MTG_REFRESH_INTERVAL_DAYS):
|
|
logger.info("Refresh not needed. Last refresh was within interval.")
|
|
return
|
|
|
|
logger.info("Starting MTGJSON refresh cycle...")
|
|
|
|
# Download files
|
|
if force or last_refresh is None:
|
|
logger.info("Downloading MTGJSON files...")
|
|
success = await manager.download_files()
|
|
if not success:
|
|
logger.error("Failed to download files")
|
|
await manager.log_refresh("FAILED_DOWNLOAD", {}, 0, "Download failed")
|
|
return
|
|
|
|
# Unpack files
|
|
logger.info("Unpacking MTGJSON files...")
|
|
await manager.unpack_files()
|
|
|
|
# Upsert data
|
|
logger.info("Upserting data into database...")
|
|
counts = await manager.upsert_data()
|
|
|
|
# Log success
|
|
duration = int(time.time() - start_time)
|
|
await manager.log_refresh("SUCCESS", counts, duration)
|
|
logger.info(f"Refresh completed successfully in {duration}s")
|
|
logger.info(f" Sets: {counts.get('sets', 0)}")
|
|
logger.info(f" Cards: {counts.get('cards', 0)}")
|
|
|
|
except Exception as e:
|
|
duration = int(time.time() - start_time)
|
|
await manager.log_refresh("FAILED", {}, duration, str(e))
|
|
logger.error(f"Refresh failed: {e}")
|
|
raise
|
|
|
|
|
|
async def main():
|
|
"""Main entry point."""
|
|
logging.basicConfig(level=logging.INFO)
|
|
|
|
parser = argparse.ArgumentParser(description="MTGJSON Refresh Script")
|
|
parser.add_argument("--force", action="store_true", help="Force refresh even if not needed")
|
|
|
|
args = parser.parse_args()
|
|
|
|
await run_refresh(force=args.force)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main()) |