fix(review): _enum_fields validation, gate classify_from_text flags, OCR warnings
Address PR #858 review round 3: - document_tier1_engine / document_ocr_provider now validate + normalize via Settings.__post_init__ _enum_fields (the repo's canonical opt-in-enum pattern; case-insensitive) instead of dynaconf Validators. A typo now raises ValueError at load and "Gateway" normalizes to "gateway". - classify_from_text gates no_text_layer/bad_text_layer on ocr_frac >= OCR_PAGE_FRACTION, matching classify_pdf -- a "fast"-routed doc with a few junk pages no longer emits a misleading flag (keeps the shadow vs hot-path classification metrics consistent). - build_ocr_backend warns when an EXPLICIT provider is misconfigured (gateway without EMBEDDING_GATEWAY_URL, mistral without MISTRAL_API_KEY) instead of silently returning None. - Pypdfium2FastProcessor.health_check probes the import; documented why OcrProcessor.health_check is unconditionally True (lazy per-tenant backends). - Removed the leftover per-boundary / per-chunk debug logging loops. Tests: enum normalization + rejection for the two new settings. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
f1272dfe84
commit
bc957b5bc7
@@ -213,8 +213,12 @@ def classify_from_text(
|
||||
# recorded classification metric accurate rather than a misleading "ocr").
|
||||
ocr_frac = (sum(p.needs_ocr for p in pages) / sampled) if sampled else 0.0
|
||||
|
||||
# Flags gated on ocr_frac >= OCR_PAGE_FRACTION (matching classify_pdf): a
|
||||
# doc that routes "fast" must not carry a junk-layer flag just because a few
|
||||
# isolated pages are bad -- otherwise the classification metric diverges
|
||||
# between this hot path and the standalone classify_pdf.
|
||||
flags: set[str] = set()
|
||||
if sampled:
|
||||
if sampled and ocr_frac >= OCR_PAGE_FRACTION:
|
||||
if total_chars == 0:
|
||||
flags.add("no_text_layer")
|
||||
elif mean_quality < MIN_TEXT_QUALITY:
|
||||
|
||||
@@ -171,6 +171,19 @@ def build_ocr_backend(settings: Settings) -> _OcrBackend | None:
|
||||
settings.mistral_base_url,
|
||||
)
|
||||
|
||||
# An EXPLICIT provider that's missing its config is an operator error -- warn
|
||||
# loudly (once, since the backend is resolved+cached) rather than silently
|
||||
# disabling OCR. "auto"/"none" fall through to None quietly by design.
|
||||
if provider == "gateway":
|
||||
logger.warning(
|
||||
"DOCUMENT_OCR_PROVIDER=gateway but EMBEDDING_GATEWAY_URL is unset; "
|
||||
"OCR is disabled"
|
||||
)
|
||||
elif provider == "mistral":
|
||||
logger.warning(
|
||||
"DOCUMENT_OCR_PROVIDER=mistral but MISTRAL_API_KEY is unset; "
|
||||
"OCR is disabled"
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@@ -258,4 +271,7 @@ class OcrProcessor(DocumentProcessor):
|
||||
)
|
||||
|
||||
async def health_check(self) -> bool:
|
||||
# Backends are resolved lazily (and configured per tenant), so there is
|
||||
# nothing to probe here without making a billable upstream call -- the
|
||||
# processor reports healthy and surfaces a real failure per-document.
|
||||
return True
|
||||
|
||||
@@ -120,4 +120,9 @@ class Pypdfium2FastProcessor(DocumentProcessor):
|
||||
return ProcessingResult(text=full_text, metadata=metadata, processor=self.name)
|
||||
|
||||
async def health_check(self) -> bool:
|
||||
return True
|
||||
try:
|
||||
import pypdfium2 # noqa: F401, PLC0415 -- availability probe
|
||||
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
Reference in New Issue
Block a user