docs(document-processors): round-2 review nits + classify_pdf glyph test

Address round-2 review on PR #914:
- Add corrupt_glyphs to the document_classifier_flag_total label comment (it is
  a live flag value emitted by record_document_classification).
- Mirror the full_text-vs-sampled control-ratio NOTE into classify_pdf so the
  diagnostic path's under-detection trade-off is documented in place.
- Add test_classify_pdf_glyph_corrupt_routes_structured for routing symmetry on
  the standalone classify_pdf path.

(SonarCloud quality gate is green — the prior S1244 finding was fixed last round.)

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
2026-06-16 20:24:33 +02:00
co-authored by Claude Opus 4.8
parent d5286e39d6
commit 425eb839bf
3 changed files with 27 additions and 1 deletions
@@ -272,6 +272,10 @@ def classify_pdf(content: bytes) -> DocClassification:
ocr_frac = (sum(p.needs_ocr for p in pages) / sampled) if sampled else 0.0
# Char-weighted doc-level control-char ratio (p.control_ratio * char_count is
# the per-page bad-char count). The glyph-leak signal -- see _control_char_ratio.
# NOTE: this is over the <=MAX_SAMPLED_PAGES sample, so unlike classify_from_text
# (which scans the whole full_text) this diagnostic path can under-detect
# corruption concentrated outside the sampled pages. Acceptable here: the hot
# path is classify_from_text; this standalone pass is for diagnostics.
control_ratio = (
sum(p.control_ratio * p.char_count for p in pages) / total_chars
if total_chars
@@ -334,7 +334,7 @@ document_classifier_flag_total = Counter(
# so flag{image_heavy} is expected to exceed classified{recommended_tier=ocr}.
"astrolabe_document_classifier_flag_total",
"Tier-0 classifier flags raised on documents",
["flag"], # image_heavy | scanned | bad_text_layer
["flag"], # image_heavy | scanned | bad_text_layer | corrupt_glyphs
)
document_text_quality = Histogram(
+22
View File
@@ -31,6 +31,18 @@ def _digital_pdf(
return data
def _glyph_corrupt_pdf(pages: int = 2) -> bytes:
# A born-digital PDF whose text layer carries the glyph-leak control chars,
# for the classify_pdf (diagnostic) path. pymupdf round-trips the C0 controls.
doc = pymupdf.open()
for _ in range(pages):
page = doc.new_page(width=595, height=842)
page.insert_text((50, 60), GLYPH_CORRUPT_TEXT)
data: bytes = doc.tobytes()
doc.close()
return data
def _full_page_image_pdf(pages: int = 2) -> bytes:
# A page whose entire area is a raster image -> looks scanned.
doc = pymupdf.open()
@@ -381,3 +393,13 @@ def test_empty_doc_routes_ocr_not_structured():
c = clf.classify_from_text("", [{"page": 1, "start_offset": 0, "end_offset": 0}])
assert c.recommended_tier == "ocr"
assert "corrupt_glyphs" not in c.flags
def test_classify_pdf_glyph_corrupt_routes_structured():
# Symmetry with the classify_from_text routing on the standalone/diagnostic
# classify_pdf path (which re-opens the PDF and samples pages).
c = clf.classify_pdf(_glyph_corrupt_pdf())
assert c.recommended_tier == "structured"
assert "corrupt_glyphs" in c.flags
assert c.mean_control_ratio > clf.GLYPH_CORRUPTION_RATIO
assert c.mean_text_quality >= clf.MIN_TEXT_QUALITY # control signal, not quality