diff --git a/backend/app/services/mtgjson_manager.py b/backend/app/services/mtgjson_manager.py index d3270bb..4399825 100644 --- a/backend/app/services/mtgjson_manager.py +++ b/backend/app/services/mtgjson_manager.py @@ -53,15 +53,145 @@ REQUIRED_FILES = { DATA_DIR = Path("/app/data/mtgjson") REFRESH_LOG_TABLE = "mtg_refresh_log" +# Expected minimum file sizes (in bytes) for MTGJSON v5 files +EXPECTED_MIN_SIZES = { + "AllPrintings.json": 500 * 1024 * 1024, # 500 MB + "AllSetFiles.json": 10 * 1024 * 1024, # 10 MB + "AllIdentifiers.json": 100 * 1024 * 1024, # 100 MB + "CardTypes.json": 1 * 1024 * 1024, # 1 MB + "Keywords.json": 0.5 * 1024 * 1024, # 0.5 MB + "MagicSets.json": 50 * 1024 * 1024, # 50 MB + "MagicRoots.json": 1 * 1024 * 1024, # 1 MB + "SetTranslations.json": 5 * 1024 * 1024, # 5 MB +} + +MAX_DOWNLOAD_RETRIES = 3 +RETRY_DELAY_SECONDS = 60 + class MTGJSONManager: """Manages MTGJSON data download, unpacking, and database upsert.""" - def __init__(self, data_dir: Path = DATA_DIR): + def __init__(self, data_dir: Path = None): + """Initialize MTGJSONManager. + + Args: + data_dir: Path to data directory. If None, uses settings.DATA_DIR + """ + if data_dir is None: + settings = get_settings() + data_dir = settings.DATA_DIR + + # Convert string to Path if needed + if isinstance(data_dir, str): + data_dir = Path(data_dir) + self.data_dir = data_dir self.data_dir.mkdir(parents=True, exist_ok=True) self.settings = get_settings() + async def validate_data_integrity(self) -> tuple[bool, list[str]]: + """ + Validate downloaded MTGJSON files for expected sizes. + + Returns: + Tuple of (is_valid, list_of_issues) + """ + issues = [] + + for filename, min_size in EXPECTED_MIN_SIZES.items(): + filepath = self.data_dir / filename + + if not filepath.exists(): + issues.append(f"Missing file: {filename}") + continue + + actual_size = filepath.stat().st_size + + if actual_size < min_size: + issues.append( + f"{filename}: {actual_size / (1024*1024):.1f} MB " + f"(minimum: {min_size / (1024*1024):.1f} MB)" + ) + + if issues: + logger.error(f"Data integrity check failed with {len(issues)} issues:") + for issue in issues: + logger.error(f" - {issue}") + return False, issues + + logger.info("Data integrity check passed - all files meet minimum size requirements") + return True, [] + + async def cleanup_data_files(self) -> None: + """Delete all downloaded MTGJSON data files.""" + logger.info(f"Cleaning up data files in {self.data_dir}") + + try: + for f in self.data_dir.glob("*"): + if f.is_file(): + f.unlink() + logger.info(f"Deleted: {f.name}") + logger.info("Data cleanup complete") + except Exception as e: + logger.error(f"Failed to clean up data files: {e}") + raise + + async def download_with_sanity_check(self) -> bool: + """ + Download MTGJSON files with sanity checking and retry logic. + + Returns: + True if download and validation succeed, False otherwise + """ + logger.info("Starting MTGJSON download with sanity checks") + + for attempt in range(1, MAX_DOWNLOAD_RETRIES + 1): + logger.info(f"Download attempt {attempt}/{MAX_DOWNLOAD_RETRIES}") + + # Download files + download_success = await self.download_files() + + if not download_success: + logger.error(f"Download failed on attempt {attempt}") + if attempt < MAX_DOWNLOAD_RETRIES: + logger.info(f"Waiting {RETRY_DELAY_SECONDS * attempt}s before retry...") + await asyncio.sleep(RETRY_DELAY_SECONDS * attempt) + await self.cleanup_data_files() + continue + return False + + # Unpack files + unpack_success = await self.unpack_files() + + if not unpack_success: + logger.error(f"Unpack failed on attempt {attempt}") + if attempt < MAX_DOWNLOAD_RETRIES: + logger.info(f"Waiting {RETRY_DELAY_SECONDS * attempt}s before retry...") + await asyncio.sleep(RETRY_DELAY_SECONDS * attempt) + await self.cleanup_data_files() + continue + return False + + # Validate data integrity + is_valid, issues = await self.validate_data_integrity() + + if is_valid: + logger.info(f"✓ Download and validation successful on attempt {attempt}") + return True + + logger.warning(f"Data validation failed on attempt {attempt} with {len(issues)} issues") + + if attempt < MAX_DOWNLOAD_RETRIES: + logger.info(f"Cleaning up and retrying in {RETRY_DELAY_SECONDS * attempt}s...") + await self.cleanup_data_files() + await asyncio.sleep(RETRY_DELAY_SECONDS * attempt) + else: + logger.error(f"Data validation failed after {MAX_DOWNLOAD_RETRIES} attempts") + return False + + return False + async def download_files(self) -> bool: """Download all required MTGJSON files.""" logger.info(f"Starting MTGJSON data download to {self.data_dir}") @@ -627,8 +757,9 @@ if __name__ == "__main__": start_time = time.time() try: if args.force or not await manager.get_last_refresh(): - await manager.download_files() - await manager.unpack_files() + if not await manager.download_with_sanity_check(): + logger.error("Failed to download MTGJSON data after retries") + return counts = await manager.upsert_data() duration = int(time.time() - start_time)