Chris CoutinhoandClaude Opus 4.8
b1f347b8fc
The hot-path classifier escalated to OCR purely on character count, so a scanned/handwritten PDF with a low-quality embedded text layer (>16 chars/page but garbled) routed `fast` and indexed the junk -- e.g. Student 147.pdf's "Little Acoms Primary"/"0110912020", which pollutes the vector and demotes the doc in search (Deck #207). - classifier: recalibrate `_text_quality` with a long-token-fraction term that detects word-merging (dropped inter-word spaces) -- the dominant junk-layer failure the old whitespace/overlong(>20) terms missed. Measured: the Student 147 scan ~0.42 (60% pages junk) vs >=0.94 for clean digital docs. - classify_from_text now routes on quality + scan: a page is OCR-worthy if near-empty OR low text-quality OR (when OCR + scan detection are enabled) it's mostly a raster image. New `image_coverage_per_page` re-opens the PDF for the scan signal, so that cost is paid only by OCR-opted-in tenants. Thresholds are passed in from per-tenant settings (keyword-only). - config: 4 per-tenant settings -- DOCUMENT_OCR_MIN_TEXT_QUALITY (0.5), DOCUMENT_OCR_PAGE_FRACTION (0.5), DOCUMENT_OCR_MIN_PAGE_CHARS (16), DOCUMENT_OCR_DETECT_SCANNED (true) -- with range validators. - metrics: new astrolabe_document_ocr_page_fraction histogram (the value the page-fraction threshold acts on) alongside document_text_quality, so operators can tune the OCR escalation per tenant (quality vs cost). Escalation gate, OCR backends, and off-by-default behavior unchanged (#858). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
271 lines
9.2 KiB
Python
271 lines
9.2 KiB
Python
"""Unit tests for the tier-0 document classifier.
|
|
|
|
Pins the routing decisions and the text-quality heuristic that drive which
|
|
extraction tier a PDF starts in:
|
|
* a clean born-digital PDF (text, no full-page images) -> ``fast`` (tier 1);
|
|
* a full-page-image scan -> ``ocr`` (tier 3), since handwriting/stamps aren't
|
|
in any text layer;
|
|
* the text-quality score distinguishes clean prose from mashed/space-less junk.
|
|
"""
|
|
|
|
import pymupdf
|
|
import pytest
|
|
|
|
from nextcloud_mcp_server.document_processors import classifier as clf
|
|
|
|
pytestmark = pytest.mark.unit
|
|
|
|
|
|
def _digital_pdf(
|
|
pages: int = 3, body: str = "Hello world this is clean text. "
|
|
) -> bytes:
|
|
doc = pymupdf.open()
|
|
for _ in range(pages):
|
|
page = doc.new_page(width=595, height=842)
|
|
page.insert_text((50, 60), body * 8)
|
|
data: bytes = doc.tobytes()
|
|
doc.close()
|
|
return data
|
|
|
|
|
|
def _full_page_image_pdf(pages: int = 2) -> bytes:
|
|
# A page whose entire area is a raster image -> looks scanned.
|
|
doc = pymupdf.open()
|
|
pix = pymupdf.Pixmap(pymupdf.csRGB, pymupdf.IRect(0, 0, 600, 850))
|
|
pix.clear_with(255)
|
|
img = pix.tobytes("png")
|
|
del pix # Pixmap holds native memory; release it before the loop
|
|
for _ in range(pages):
|
|
page = doc.new_page(width=595, height=842)
|
|
page.insert_image(page.rect, stream=img)
|
|
data: bytes = doc.tobytes()
|
|
doc.close()
|
|
return data
|
|
|
|
|
|
# --- text-quality heuristic --------------------------------------------------
|
|
|
|
|
|
def test_text_quality_clean_prose_scores_high():
|
|
assert clf._text_quality("the quick brown fox jumps over the lazy dog") > 0.8
|
|
|
|
|
|
def test_text_quality_mashed_tokens_scores_low():
|
|
# space-less / mashed layer (the "Student 147" failure mode)
|
|
mashed = "01322234567mobileoutstandingresilienceacademicachievementhurdles"
|
|
assert clf._text_quality(mashed) < clf.MIN_TEXT_QUALITY
|
|
|
|
|
|
def test_text_quality_empty_is_zero():
|
|
assert clf._text_quality("") == pytest.approx(0.0)
|
|
|
|
|
|
# --- routing -----------------------------------------------------------------
|
|
|
|
|
|
def test_digital_pdf_routes_fast():
|
|
c = clf.classify_pdf(_digital_pdf())
|
|
assert c.recommended_tier == "fast"
|
|
assert c.ocr_page_fraction == pytest.approx(0.0)
|
|
assert "image_heavy" not in c.flags
|
|
assert c.mean_text_quality > 0.8
|
|
|
|
|
|
def test_full_page_image_routes_ocr():
|
|
c = clf.classify_pdf(_full_page_image_pdf())
|
|
assert c.recommended_tier == "ocr"
|
|
assert c.ocr_page_fraction == pytest.approx(1.0)
|
|
assert "image_heavy" in c.flags
|
|
assert "scanned" in c.flags # no text layer at all
|
|
|
|
|
|
# --- sampling bounds large docs ----------------------------------------------
|
|
|
|
|
|
def test_large_doc_is_sampled():
|
|
c = clf.classify_pdf(_digital_pdf(pages=120))
|
|
assert c.page_count == 120
|
|
assert c.sampled_pages <= clf.MAX_SAMPLED_PAGES
|
|
|
|
|
|
def test_sample_indices_includes_first_and_last_page():
|
|
idx = clf._sample_indices(100)
|
|
assert idx[0] == 0
|
|
assert idx[-1] == 99 # last page must be sampled (scanned-tail case)
|
|
assert len(idx) <= clf.MAX_SAMPLED_PAGES
|
|
|
|
|
|
# --- flag paths --------------------------------------------------------------
|
|
|
|
|
|
def _image_with_mashed_text_pdf(pages: int = 2) -> bytes:
|
|
# Full-page image with a junk (mashed/space-less) text layer over it -- a
|
|
# scan whose OCR'd text layer is unusable.
|
|
doc = pymupdf.open()
|
|
pix = pymupdf.Pixmap(pymupdf.csRGB, pymupdf.IRect(0, 0, 600, 850))
|
|
pix.clear_with(255)
|
|
img = pix.tobytes("png")
|
|
del pix # Pixmap holds native memory; release it before the loop
|
|
mashed = "01322234567mobileoutstandingresilienceacademicachievement " * 3
|
|
for _ in range(pages):
|
|
page = doc.new_page(width=595, height=842)
|
|
page.insert_image(page.rect, stream=img)
|
|
page.insert_text((50, 60), mashed)
|
|
data: bytes = doc.tobytes()
|
|
doc.close()
|
|
return data
|
|
|
|
|
|
def test_scanned_flag_when_no_text_layer():
|
|
c = clf.classify_pdf(_full_page_image_pdf())
|
|
assert c.total_chars == 0
|
|
assert "scanned" in c.flags
|
|
assert c.recommended_tier == "ocr"
|
|
|
|
|
|
def test_bad_text_layer_flag_on_image_with_junk_text():
|
|
c = clf.classify_pdf(_image_with_mashed_text_pdf())
|
|
assert c.total_chars > 0
|
|
assert c.mean_text_quality < clf.MIN_TEXT_QUALITY
|
|
assert "bad_text_layer" in c.flags
|
|
assert c.recommended_tier == "ocr"
|
|
|
|
|
|
def _mostly_text_one_image_pdf() -> bytes:
|
|
# 3 digital text pages + 1 full-page-image page: one image-heavy page, but
|
|
# ocr_frac = 1/4 < OCR_PAGE_FRACTION, so the doc routes fast.
|
|
doc = pymupdf.open()
|
|
pix = pymupdf.Pixmap(pymupdf.csRGB, pymupdf.IRect(0, 0, 600, 850))
|
|
pix.clear_with(255)
|
|
img = pix.tobytes("png")
|
|
del pix # Pixmap holds native memory; release it before the loop
|
|
for _ in range(3):
|
|
page = doc.new_page(width=595, height=842)
|
|
page.insert_text((50, 60), "Hello world this is clean text. " * 8)
|
|
page = doc.new_page(width=595, height=842)
|
|
page.insert_image(page.rect, stream=img)
|
|
data: bytes = doc.tobytes()
|
|
doc.close()
|
|
return data
|
|
|
|
|
|
def test_image_heavy_flag_without_ocr_routing():
|
|
# The documented asymmetry operators rely on: a mostly-digital doc with one
|
|
# full-page image carries the image_heavy flag yet still routes fast.
|
|
c = clf.classify_pdf(_mostly_text_one_image_pdf())
|
|
assert "image_heavy" in c.flags
|
|
assert c.recommended_tier == "fast"
|
|
assert c.ocr_page_fraction < clf.OCR_PAGE_FRACTION
|
|
|
|
|
|
# --- classify_from_text (hot-path, derived from tier-1 extraction) -----------
|
|
|
|
|
|
def test_classify_from_text_clean_routes_fast():
|
|
txt = "the quick brown fox jumps over the lazy dog " * 3
|
|
c = clf.classify_from_text(
|
|
txt, [{"page": 1, "start_offset": 0, "end_offset": len(txt)}]
|
|
)
|
|
assert c.recommended_tier == "fast"
|
|
assert c.mean_text_quality > 0.8
|
|
assert c.flags == set()
|
|
|
|
|
|
def test_classify_from_text_empty_routes_ocr():
|
|
c = clf.classify_from_text("", [{"page": 1, "start_offset": 0, "end_offset": 0}])
|
|
assert c.recommended_tier == "ocr"
|
|
assert "no_text_layer" in c.flags
|
|
assert c.total_chars == 0
|
|
|
|
|
|
def test_classify_from_text_no_pages_routes_fast():
|
|
# An empty/corrupt PDF (no page boundaries) is not OCR evidence -> "fast",
|
|
# so the recorded classification metric isn't a misleading "ocr".
|
|
c = clf.classify_from_text("", [])
|
|
assert c.recommended_tier == "fast"
|
|
assert c.ocr_page_fraction == pytest.approx(0.0)
|
|
assert c.flags == set()
|
|
|
|
|
|
def test_classify_from_text_junk_layer_flags_bad_text_layer():
|
|
# Each short segment (<MIN_PAGE_CHARS) sets needs_ocr -> high ocr_frac, and
|
|
# total_chars>0 with mean_quality<MIN_TEXT_QUALITY (no-whitespace junk scores
|
|
# 0.0) -> bad_text_layer (gated on ocr_frac, matching classify_pdf).
|
|
text = "x1y2zx1y2z"
|
|
c = clf.classify_from_text(
|
|
text,
|
|
[
|
|
{"page": 1, "start_offset": 0, "end_offset": 5},
|
|
{"page": 2, "start_offset": 5, "end_offset": 10},
|
|
],
|
|
)
|
|
assert c.recommended_tier == "ocr"
|
|
assert c.total_chars > 0
|
|
assert "bad_text_layer" in c.flags
|
|
assert "no_text_layer" not in c.flags
|
|
|
|
|
|
# --- quality + scan escalation triggers (Deck #207) --------------------------
|
|
|
|
_JUNK = (
|
|
"ST. TRINIAN'SSCHOOLSTUDENT RECORDFILE struggledsignificantlywith "
|
|
"learningdifficulties demonstrateda positiveattitude academictasks"
|
|
)
|
|
_CLEAN = "the quick brown fox jumps over the lazy dog and then runs away home"
|
|
|
|
|
|
def _two_page(text_a: str, text_b: str):
|
|
na = len(text_a)
|
|
return text_a + text_b, [
|
|
{"page": 1, "start_offset": 0, "end_offset": na},
|
|
{"page": 2, "start_offset": na, "end_offset": na + len(text_b)},
|
|
]
|
|
|
|
|
|
def test_classify_from_text_low_quality_routes_ocr():
|
|
full, bounds = _two_page(_JUNK, _JUNK)
|
|
c = clf.classify_from_text(full, bounds)
|
|
assert c.recommended_tier == "ocr"
|
|
assert "bad_text_layer" in c.flags
|
|
|
|
|
|
def test_quality_floor_override_disables_trigger():
|
|
# min_text_quality=0.0 => quality never trips; text present + not scanned => fast
|
|
full, bounds = _two_page(_JUNK, _JUNK)
|
|
c = clf.classify_from_text(full, bounds, min_text_quality=0.0)
|
|
assert c.recommended_tier == "fast"
|
|
|
|
|
|
def test_scan_signal_routes_ocr_even_with_clean_text():
|
|
# clean text but every page is a raster scan -> OCR (the Student-147 case)
|
|
full, bounds = _two_page(_CLEAN, _CLEAN)
|
|
c = clf.classify_from_text(full, bounds, image_coverage=[1.0, 1.0])
|
|
assert c.recommended_tier == "ocr"
|
|
assert "image_heavy" in c.flags
|
|
|
|
|
|
def test_scan_signal_ignored_when_coverage_low():
|
|
full, bounds = _two_page(_CLEAN, _CLEAN)
|
|
c = clf.classify_from_text(full, bounds, image_coverage=[0.1, 0.0])
|
|
assert c.recommended_tier == "fast"
|
|
|
|
|
|
def test_page_fraction_override():
|
|
# exactly one of two pages is junk -> ocr_frac 0.5
|
|
full, bounds = _two_page(_CLEAN, _JUNK)
|
|
assert (
|
|
clf.classify_from_text(full, bounds, page_fraction=0.5).recommended_tier
|
|
== "ocr"
|
|
)
|
|
assert (
|
|
clf.classify_from_text(full, bounds, page_fraction=0.6).recommended_tier
|
|
== "fast"
|
|
)
|
|
|
|
|
|
def test_image_coverage_per_page():
|
|
scan = clf.image_coverage_per_page(_full_page_image_pdf(pages=2))
|
|
assert len(scan) == 2 and all(c >= 0.8 for c in scan)
|
|
digital = clf.image_coverage_per_page(_digital_pdf(pages=2))
|
|
assert len(digital) == 2 and all(c < 0.1 for c in digital)
|