feat: quality + scan OCR escalation trigger (junk-text-layer scans)
The hot-path classifier escalated to OCR purely on character count, so a scanned/handwritten PDF with a low-quality embedded text layer (>16 chars/page but garbled) routed `fast` and indexed the junk -- e.g. Student 147.pdf's "Little Acoms Primary"/"0110912020", which pollutes the vector and demotes the doc in search (Deck #207). - classifier: recalibrate `_text_quality` with a long-token-fraction term that detects word-merging (dropped inter-word spaces) -- the dominant junk-layer failure the old whitespace/overlong(>20) terms missed. Measured: the Student 147 scan ~0.42 (60% pages junk) vs >=0.94 for clean digital docs. - classify_from_text now routes on quality + scan: a page is OCR-worthy if near-empty OR low text-quality OR (when OCR + scan detection are enabled) it's mostly a raster image. New `image_coverage_per_page` re-opens the PDF for the scan signal, so that cost is paid only by OCR-opted-in tenants. Thresholds are passed in from per-tenant settings (keyword-only). - config: 4 per-tenant settings -- DOCUMENT_OCR_MIN_TEXT_QUALITY (0.5), DOCUMENT_OCR_PAGE_FRACTION (0.5), DOCUMENT_OCR_MIN_PAGE_CHARS (16), DOCUMENT_OCR_DETECT_SCANNED (true) -- with range validators. - metrics: new astrolabe_document_ocr_page_fraction histogram (the value the page-fraction threshold acts on) alongside document_text_quality, so operators can tune the OCR escalation per tenant (quality vs cost). Escalation gate, OCR backends, and off-by-default behavior unchanged (#858). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
d421bf6953
commit
b1f347b8fc
@@ -203,3 +203,68 @@ def test_classify_from_text_junk_layer_flags_bad_text_layer():
|
||||
assert c.total_chars > 0
|
||||
assert "bad_text_layer" in c.flags
|
||||
assert "no_text_layer" not in c.flags
|
||||
|
||||
|
||||
# --- quality + scan escalation triggers (Deck #207) --------------------------
|
||||
|
||||
_JUNK = (
|
||||
"ST. TRINIAN'SSCHOOLSTUDENT RECORDFILE struggledsignificantlywith "
|
||||
"learningdifficulties demonstrateda positiveattitude academictasks"
|
||||
)
|
||||
_CLEAN = "the quick brown fox jumps over the lazy dog and then runs away home"
|
||||
|
||||
|
||||
def _two_page(text_a: str, text_b: str):
|
||||
na = len(text_a)
|
||||
return text_a + text_b, [
|
||||
{"page": 1, "start_offset": 0, "end_offset": na},
|
||||
{"page": 2, "start_offset": na, "end_offset": na + len(text_b)},
|
||||
]
|
||||
|
||||
|
||||
def test_classify_from_text_low_quality_routes_ocr():
|
||||
full, bounds = _two_page(_JUNK, _JUNK)
|
||||
c = clf.classify_from_text(full, bounds)
|
||||
assert c.recommended_tier == "ocr"
|
||||
assert "bad_text_layer" in c.flags
|
||||
|
||||
|
||||
def test_quality_floor_override_disables_trigger():
|
||||
# min_text_quality=0.0 => quality never trips; text present + not scanned => fast
|
||||
full, bounds = _two_page(_JUNK, _JUNK)
|
||||
c = clf.classify_from_text(full, bounds, min_text_quality=0.0)
|
||||
assert c.recommended_tier == "fast"
|
||||
|
||||
|
||||
def test_scan_signal_routes_ocr_even_with_clean_text():
|
||||
# clean text but every page is a raster scan -> OCR (the Student-147 case)
|
||||
full, bounds = _two_page(_CLEAN, _CLEAN)
|
||||
c = clf.classify_from_text(full, bounds, image_coverage=[1.0, 1.0])
|
||||
assert c.recommended_tier == "ocr"
|
||||
assert "image_heavy" in c.flags
|
||||
|
||||
|
||||
def test_scan_signal_ignored_when_coverage_low():
|
||||
full, bounds = _two_page(_CLEAN, _CLEAN)
|
||||
c = clf.classify_from_text(full, bounds, image_coverage=[0.1, 0.0])
|
||||
assert c.recommended_tier == "fast"
|
||||
|
||||
|
||||
def test_page_fraction_override():
|
||||
# exactly one of two pages is junk -> ocr_frac 0.5
|
||||
full, bounds = _two_page(_CLEAN, _JUNK)
|
||||
assert (
|
||||
clf.classify_from_text(full, bounds, page_fraction=0.5).recommended_tier
|
||||
== "ocr"
|
||||
)
|
||||
assert (
|
||||
clf.classify_from_text(full, bounds, page_fraction=0.6).recommended_tier
|
||||
== "fast"
|
||||
)
|
||||
|
||||
|
||||
def test_image_coverage_per_page():
|
||||
scan = clf.image_coverage_per_page(_full_page_image_pdf(pages=2))
|
||||
assert len(scan) == 2 and all(c >= 0.8 for c in scan)
|
||||
digital = clf.image_coverage_per_page(_digital_pdf(pages=2))
|
||||
assert len(digital) == 2 and all(c < 0.1 for c in digital)
|
||||
|
||||
Reference in New Issue
Block a user