Add MTGJSON data sanity check with file size validation and retry logic
This commit is contained in:
@@ -53,15 +53,145 @@ REQUIRED_FILES = {
|
||||
DATA_DIR = Path("/app/data/mtgjson")
|
||||
REFRESH_LOG_TABLE = "mtg_refresh_log"
|
||||
|
||||
# Expected minimum file sizes (in bytes) for MTGJSON v5 files
|
||||
EXPECTED_MIN_SIZES = {
|
||||
"AllPrintings.json": 500 * 1024 * 1024, # 500 MB
|
||||
"AllSetFiles.json": 10 * 1024 * 1024, # 10 MB
|
||||
"AllIdentifiers.json": 100 * 1024 * 1024, # 100 MB
|
||||
"CardTypes.json": 1 * 1024 * 1024, # 1 MB
|
||||
"Keywords.json": 0.5 * 1024 * 1024, # 0.5 MB
|
||||
"MagicSets.json": 50 * 1024 * 1024, # 50 MB
|
||||
"MagicRoots.json": 1 * 1024 * 1024, # 1 MB
|
||||
"SetTranslations.json": 5 * 1024 * 1024, # 5 MB
|
||||
}
|
||||
|
||||
MAX_DOWNLOAD_RETRIES = 3
|
||||
RETRY_DELAY_SECONDS = 60
|
||||
|
||||
|
||||
class MTGJSONManager:
|
||||
"""Manages MTGJSON data download, unpacking, and database upsert."""
|
||||
|
||||
def __init__(self, data_dir: Path = DATA_DIR):
|
||||
def __init__(self, data_dir: Path = None):
|
||||
"""Initialize MTGJSONManager.
|
||||
|
||||
Args:
|
||||
data_dir: Path to data directory. If None, uses settings.DATA_DIR
|
||||
"""
|
||||
if data_dir is None:
|
||||
settings = get_settings()
|
||||
data_dir = settings.DATA_DIR
|
||||
|
||||
# Convert string to Path if needed
|
||||
if isinstance(data_dir, str):
|
||||
data_dir = Path(data_dir)
|
||||
|
||||
self.data_dir = data_dir
|
||||
self.data_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.settings = get_settings()
|
||||
|
||||
async def validate_data_integrity(self) -> tuple[bool, list[str]]:
|
||||
"""
|
||||
Validate downloaded MTGJSON files for expected sizes.
|
||||
|
||||
Returns:
|
||||
Tuple of (is_valid, list_of_issues)
|
||||
"""
|
||||
issues = []
|
||||
|
||||
for filename, min_size in EXPECTED_MIN_SIZES.items():
|
||||
filepath = self.data_dir / filename
|
||||
|
||||
if not filepath.exists():
|
||||
issues.append(f"Missing file: {filename}")
|
||||
continue
|
||||
|
||||
actual_size = filepath.stat().st_size
|
||||
|
||||
if actual_size < min_size:
|
||||
issues.append(
|
||||
f"{filename}: {actual_size / (1024*1024):.1f} MB "
|
||||
f"(minimum: {min_size / (1024*1024):.1f} MB)"
|
||||
)
|
||||
|
||||
if issues:
|
||||
logger.error(f"Data integrity check failed with {len(issues)} issues:")
|
||||
for issue in issues:
|
||||
logger.error(f" - {issue}")
|
||||
return False, issues
|
||||
|
||||
logger.info("Data integrity check passed - all files meet minimum size requirements")
|
||||
return True, []
|
||||
|
||||
async def cleanup_data_files(self) -> None:
|
||||
"""Delete all downloaded MTGJSON data files."""
|
||||
logger.info(f"Cleaning up data files in {self.data_dir}")
|
||||
|
||||
try:
|
||||
for f in self.data_dir.glob("*"):
|
||||
if f.is_file():
|
||||
f.unlink()
|
||||
logger.info(f"Deleted: {f.name}")
|
||||
logger.info("Data cleanup complete")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to clean up data files: {e}")
|
||||
raise
|
||||
|
||||
async def download_with_sanity_check(self) -> bool:
|
||||
"""
|
||||
Download MTGJSON files with sanity checking and retry logic.
|
||||
|
||||
Returns:
|
||||
True if download and validation succeed, False otherwise
|
||||
"""
|
||||
logger.info("Starting MTGJSON download with sanity checks")
|
||||
|
||||
for attempt in range(1, MAX_DOWNLOAD_RETRIES + 1):
|
||||
logger.info(f"Download attempt {attempt}/{MAX_DOWNLOAD_RETRIES}")
|
||||
|
||||
# Download files
|
||||
download_success = await self.download_files()
|
||||
|
||||
if not download_success:
|
||||
logger.error(f"Download failed on attempt {attempt}")
|
||||
if attempt < MAX_DOWNLOAD_RETRIES:
|
||||
logger.info(f"Waiting {RETRY_DELAY_SECONDS * attempt}s before retry...")
|
||||
await asyncio.sleep(RETRY_DELAY_SECONDS * attempt)
|
||||
await self.cleanup_data_files()
|
||||
continue
|
||||
return False
|
||||
|
||||
# Unpack files
|
||||
unpack_success = await self.unpack_files()
|
||||
|
||||
if not unpack_success:
|
||||
logger.error(f"Unpack failed on attempt {attempt}")
|
||||
if attempt < MAX_DOWNLOAD_RETRIES:
|
||||
logger.info(f"Waiting {RETRY_DELAY_SECONDS * attempt}s before retry...")
|
||||
await asyncio.sleep(RETRY_DELAY_SECONDS * attempt)
|
||||
await self.cleanup_data_files()
|
||||
continue
|
||||
return False
|
||||
|
||||
# Validate data integrity
|
||||
is_valid, issues = await self.validate_data_integrity()
|
||||
|
||||
if is_valid:
|
||||
logger.info(f"✓ Download and validation successful on attempt {attempt}")
|
||||
return True
|
||||
|
||||
logger.warning(f"Data validation failed on attempt {attempt} with {len(issues)} issues")
|
||||
|
||||
if attempt < MAX_DOWNLOAD_RETRIES:
|
||||
logger.info(f"Cleaning up and retrying in {RETRY_DELAY_SECONDS * attempt}s...")
|
||||
await self.cleanup_data_files()
|
||||
await asyncio.sleep(RETRY_DELAY_SECONDS * attempt)
|
||||
else:
|
||||
logger.error(f"Data validation failed after {MAX_DOWNLOAD_RETRIES} attempts")
|
||||
return False
|
||||
|
||||
return False
|
||||
|
||||
async def download_files(self) -> bool:
|
||||
"""Download all required MTGJSON files."""
|
||||
logger.info(f"Starting MTGJSON data download to {self.data_dir}")
|
||||
@@ -627,8 +757,9 @@ if __name__ == "__main__":
|
||||
start_time = time.time()
|
||||
try:
|
||||
if args.force or not await manager.get_last_refresh():
|
||||
await manager.download_files()
|
||||
await manager.unpack_files()
|
||||
if not await manager.download_with_sanity_check():
|
||||
logger.error("Failed to download MTGJSON data after retries")
|
||||
return
|
||||
|
||||
counts = await manager.upsert_data()
|
||||
duration = int(time.time() - start_time)
|
||||
|
||||
Reference in New Issue
Block a user