refactor(classifier): rename IMAGE_COVERAGE_SCANNED → IMAGE_HEAVY_THRESHOLD
Round-1 review nits. The constant now only gates the diagnostic `image_heavy` flag (not routing), so the old name was misleading. Rename + reword its comment to state the diagnostic-only intent. Also add a classify_pdf symmetry test (`test_classify_pdf_image_heavy_clean_text_stays_fast`) pinning that a full-page raster image with a clean text layer routes fast on the classify_pdf path too. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
906a5805ef
commit
d75b7e1bd1
@@ -43,8 +43,10 @@ logger = logging.getLogger(__name__)
|
||||
# so the pass stays bounded regardless of page count.
|
||||
MAX_SAMPLED_PAGES = 24
|
||||
|
||||
# A page counts as "scanned-like" when a raster image covers most of it.
|
||||
IMAGE_COVERAGE_SCANNED = 0.80
|
||||
# Raster-image coverage above which a page raises the DIAGNOSTIC ``image_heavy``
|
||||
# flag. This is observability only -- it does NOT route to OCR (see module
|
||||
# docstring); routing is on the text signals alone.
|
||||
IMAGE_HEAVY_THRESHOLD = 0.80
|
||||
# Text-quality score below which the layer is treated as junk (mashed tokens).
|
||||
# Kept in sync with the DOCUMENT_OCR_MIN_TEXT_QUALITY setting default so the
|
||||
# module/diagnostic default matches production (the registry always passes the
|
||||
@@ -185,7 +187,7 @@ def classify_pdf(content: bytes) -> DocClassification:
|
||||
# one full-page photo is flagged image_heavy yet still routes "fast" -- the
|
||||
# flag_total{image_heavy} count is expected to exceed classified{ocr}.
|
||||
flags: set[str] = set()
|
||||
if any(p.image_coverage >= IMAGE_COVERAGE_SCANNED for p in pages):
|
||||
if any(p.image_coverage >= IMAGE_HEAVY_THRESHOLD for p in pages):
|
||||
flags.add("image_heavy")
|
||||
if (
|
||||
ocr_frac >= OCR_PAGE_FRACTION
|
||||
@@ -316,7 +318,7 @@ def classify_from_text(
|
||||
flags.add("scanned")
|
||||
elif mean_quality < min_text_quality:
|
||||
flags.add("bad_text_layer")
|
||||
if any(p.image_coverage >= IMAGE_COVERAGE_SCANNED for p in pages):
|
||||
if any(p.image_coverage >= IMAGE_HEAVY_THRESHOLD for p in pages):
|
||||
flags.add("image_heavy")
|
||||
|
||||
recommended = "ocr" if ocr_frac >= page_fraction else "fast"
|
||||
|
||||
@@ -118,6 +118,23 @@ def _image_with_mashed_text_pdf(pages: int = 2) -> bytes:
|
||||
return data
|
||||
|
||||
|
||||
def _image_with_clean_text_pdf(pages: int = 2) -> bytes:
|
||||
# Full-page image with a CLEAN embedded text layer -- a scan carrying a good
|
||||
# OCR layer, or a figure-heavy digital page. Image-heavy but usable text.
|
||||
doc = pymupdf.open()
|
||||
pix = pymupdf.Pixmap(pymupdf.csRGB, pymupdf.IRect(0, 0, 600, 850))
|
||||
pix.clear_with(255)
|
||||
img = pix.tobytes("png")
|
||||
del pix # Pixmap holds native memory; release it before the loop
|
||||
for _ in range(pages):
|
||||
page = doc.new_page(width=595, height=842)
|
||||
page.insert_image(page.rect, stream=img)
|
||||
page.insert_text((50, 60), "Hello world this is clean text. " * 8)
|
||||
data: bytes = doc.tobytes()
|
||||
doc.close()
|
||||
return data
|
||||
|
||||
|
||||
def test_scanned_flag_when_no_text_layer():
|
||||
c = clf.classify_pdf(_full_page_image_pdf())
|
||||
assert c.total_chars == 0
|
||||
@@ -250,6 +267,16 @@ def test_image_heavy_clean_text_stays_fast():
|
||||
assert all(p.needs_ocr is False for p in c.pages)
|
||||
|
||||
|
||||
def test_classify_pdf_image_heavy_clean_text_stays_fast():
|
||||
# classify_pdf symmetry with test_image_heavy_clean_text_stays_fast: full-page
|
||||
# raster images WITH a clean embedded text layer are image_heavy but route
|
||||
# fast -- coverage is diagnostic, not routing, on the classify_pdf path too.
|
||||
c = clf.classify_pdf(_image_with_clean_text_pdf())
|
||||
assert c.recommended_tier == "fast"
|
||||
assert "image_heavy" in c.flags
|
||||
assert c.mean_text_quality >= clf.MIN_TEXT_QUALITY
|
||||
|
||||
|
||||
def test_scan_signal_ignored_when_coverage_low():
|
||||
full, bounds = _two_page(_CLEAN, _CLEAN)
|
||||
c = clf.classify_from_text(full, bounds, image_coverage=[0.1, 0.0])
|
||||
|
||||
Reference in New Issue
Block a user