diff --git a/nextcloud_mcp_server/document_processors/classifier.py b/nextcloud_mcp_server/document_processors/classifier.py index 7ce28fd4..7647769f 100644 --- a/nextcloud_mcp_server/document_processors/classifier.py +++ b/nextcloud_mcp_server/document_processors/classifier.py @@ -272,6 +272,10 @@ def classify_pdf(content: bytes) -> DocClassification: ocr_frac = (sum(p.needs_ocr for p in pages) / sampled) if sampled else 0.0 # Char-weighted doc-level control-char ratio (p.control_ratio * char_count is # the per-page bad-char count). The glyph-leak signal -- see _control_char_ratio. + # NOTE: this is over the <=MAX_SAMPLED_PAGES sample, so unlike classify_from_text + # (which scans the whole full_text) this diagnostic path can under-detect + # corruption concentrated outside the sampled pages. Acceptable here: the hot + # path is classify_from_text; this standalone pass is for diagnostics. control_ratio = ( sum(p.control_ratio * p.char_count for p in pages) / total_chars if total_chars diff --git a/nextcloud_mcp_server/observability/metrics.py b/nextcloud_mcp_server/observability/metrics.py index f2ca1060..26e20421 100644 --- a/nextcloud_mcp_server/observability/metrics.py +++ b/nextcloud_mcp_server/observability/metrics.py @@ -334,7 +334,7 @@ document_classifier_flag_total = Counter( # so flag{image_heavy} is expected to exceed classified{recommended_tier=ocr}. "astrolabe_document_classifier_flag_total", "Tier-0 classifier flags raised on documents", - ["flag"], # image_heavy | scanned | bad_text_layer + ["flag"], # image_heavy | scanned | bad_text_layer | corrupt_glyphs ) document_text_quality = Histogram( diff --git a/tests/unit/test_doc_classifier.py b/tests/unit/test_doc_classifier.py index 2a7d9ee3..e9dc9423 100644 --- a/tests/unit/test_doc_classifier.py +++ b/tests/unit/test_doc_classifier.py @@ -31,6 +31,18 @@ def _digital_pdf( return data +def _glyph_corrupt_pdf(pages: int = 2) -> bytes: + # A born-digital PDF whose text layer carries the glyph-leak control chars, + # for the classify_pdf (diagnostic) path. pymupdf round-trips the C0 controls. + doc = pymupdf.open() + for _ in range(pages): + page = doc.new_page(width=595, height=842) + page.insert_text((50, 60), GLYPH_CORRUPT_TEXT) + data: bytes = doc.tobytes() + doc.close() + return data + + def _full_page_image_pdf(pages: int = 2) -> bytes: # A page whose entire area is a raster image -> looks scanned. doc = pymupdf.open() @@ -381,3 +393,13 @@ def test_empty_doc_routes_ocr_not_structured(): c = clf.classify_from_text("", [{"page": 1, "start_offset": 0, "end_offset": 0}]) assert c.recommended_tier == "ocr" assert "corrupt_glyphs" not in c.flags + + +def test_classify_pdf_glyph_corrupt_routes_structured(): + # Symmetry with the classify_from_text routing on the standalone/diagnostic + # classify_pdf path (which re-opens the PDF and samples pages). + c = clf.classify_pdf(_glyph_corrupt_pdf()) + assert c.recommended_tier == "structured" + assert "corrupt_glyphs" in c.flags + assert c.mean_control_ratio > clf.GLYPH_CORRUPTION_RATIO + assert c.mean_text_quality >= clf.MIN_TEXT_QUALITY # control signal, not quality