feat: quality + scan OCR escalation trigger (junk-text-layer scans)

The hot-path classifier escalated to OCR purely on character count, so a
scanned/handwritten PDF with a low-quality embedded text layer (>16 chars/page
but garbled) routed `fast` and indexed the junk -- e.g. Student 147.pdf's
"Little Acoms Primary"/"0110912020", which pollutes the vector and demotes the
doc in search (Deck #207).

- classifier: recalibrate `_text_quality` with a long-token-fraction term that
  detects word-merging (dropped inter-word spaces) -- the dominant junk-layer
  failure the old whitespace/overlong(>20) terms missed. Measured: the Student
  147 scan ~0.42 (60% pages junk) vs >=0.94 for clean digital docs.
- classify_from_text now routes on quality + scan: a page is OCR-worthy if
  near-empty OR low text-quality OR (when OCR + scan detection are enabled) it's
  mostly a raster image. New `image_coverage_per_page` re-opens the PDF for the
  scan signal, so that cost is paid only by OCR-opted-in tenants. Thresholds are
  passed in from per-tenant settings (keyword-only).
- config: 4 per-tenant settings -- DOCUMENT_OCR_MIN_TEXT_QUALITY (0.5),
  DOCUMENT_OCR_PAGE_FRACTION (0.5), DOCUMENT_OCR_MIN_PAGE_CHARS (16),
  DOCUMENT_OCR_DETECT_SCANNED (true) -- with range validators.
- metrics: new astrolabe_document_ocr_page_fraction histogram (the value the
  page-fraction threshold acts on) alongside document_text_quality, so operators
  can tune the OCR escalation per tenant (quality vs cost).

Escalation gate, OCR backends, and off-by-default behavior unchanged (#858).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

tests/unit/test_config.py

@@ -381,6 +381,22 @@ class TestDynaconfValidators:
         with pytest.raises(ValidationError, match="LOG_FORMAT"):
             _reload_config()
 
+    @patch.dict(os.environ, {"DOCUMENT_OCR_MIN_TEXT_QUALITY": "1.5"}, clear=True)
+    def test_ocr_min_text_quality_out_of_range(self):
+        """DOCUMENT_OCR_MIN_TEXT_QUALITY must be in [0, 1]."""
+        from dynaconf import ValidationError
+
+        with pytest.raises(ValidationError, match="DOCUMENT_OCR_MIN_TEXT_QUALITY"):
+            _reload_config()
+
+    @patch.dict(os.environ, {"DOCUMENT_OCR_PAGE_FRACTION": "2"}, clear=True)
+    def test_ocr_page_fraction_out_of_range(self):
+        """DOCUMENT_OCR_PAGE_FRACTION must be in [0, 1]."""
+        from dynaconf import ValidationError
+
+        with pytest.raises(ValidationError, match="DOCUMENT_OCR_PAGE_FRACTION"):
+            _reload_config()
+
     @patch.dict(os.environ, {"LOG_LEVEL": "VERBOSE"}, clear=True)
     def test_invalid_log_level(self):
         """Test invalid LOG_LEVEL raises ValidationError."""

tests/unit/test_doc_classifier.py

@@ -203,3 +203,68 @@ def test_classify_from_text_junk_layer_flags_bad_text_layer():
     assert c.total_chars > 0
     assert "bad_text_layer" in c.flags
     assert "no_text_layer" not in c.flags
+
+
+# --- quality + scan escalation triggers (Deck #207) --------------------------
+
+_JUNK = (
+    "ST. TRINIAN'SSCHOOLSTUDENT RECORDFILE struggledsignificantlywith "
+    "learningdifficulties demonstrateda positiveattitude academictasks"
+)
+_CLEAN = "the quick brown fox jumps over the lazy dog and then runs away home"
+
+
+def _two_page(text_a: str, text_b: str):
+    na = len(text_a)
+    return text_a + text_b, [
+        {"page": 1, "start_offset": 0, "end_offset": na},
+        {"page": 2, "start_offset": na, "end_offset": na + len(text_b)},
+    ]
+
+
+def test_classify_from_text_low_quality_routes_ocr():
+    full, bounds = _two_page(_JUNK, _JUNK)
+    c = clf.classify_from_text(full, bounds)
+    assert c.recommended_tier == "ocr"
+    assert "bad_text_layer" in c.flags
+
+
+def test_quality_floor_override_disables_trigger():
+    # min_text_quality=0.0 => quality never trips; text present + not scanned => fast
+    full, bounds = _two_page(_JUNK, _JUNK)
+    c = clf.classify_from_text(full, bounds, min_text_quality=0.0)
+    assert c.recommended_tier == "fast"
+
+
+def test_scan_signal_routes_ocr_even_with_clean_text():
+    # clean text but every page is a raster scan -> OCR (the Student-147 case)
+    full, bounds = _two_page(_CLEAN, _CLEAN)
+    c = clf.classify_from_text(full, bounds, image_coverage=[1.0, 1.0])
+    assert c.recommended_tier == "ocr"
+    assert "image_heavy" in c.flags
+
+
+def test_scan_signal_ignored_when_coverage_low():
+    full, bounds = _two_page(_CLEAN, _CLEAN)
+    c = clf.classify_from_text(full, bounds, image_coverage=[0.1, 0.0])
+    assert c.recommended_tier == "fast"
+
+
+def test_page_fraction_override():
+    # exactly one of two pages is junk -> ocr_frac 0.5
+    full, bounds = _two_page(_CLEAN, _JUNK)
+    assert (
+        clf.classify_from_text(full, bounds, page_fraction=0.5).recommended_tier
+        == "ocr"
+    )
+    assert (
+        clf.classify_from_text(full, bounds, page_fraction=0.6).recommended_tier
+        == "fast"
+    )
+
+
+def test_image_coverage_per_page():
+    scan = clf.image_coverage_per_page(_full_page_image_pdf(pages=2))
+    assert len(scan) == 2 and all(c >= 0.8 for c in scan)
+    digital = clf.image_coverage_per_page(_digital_pdf(pages=2))
+    assert len(digital) == 2 and all(c < 0.1 for c in digital)

tests/unit/test_registry_tiering.py

@@ -65,10 +65,23 @@ class _Fake(DocumentProcessor):
 
 
 class _Settings:
-    def __init__(self, engine="pypdfium2", classify=True, ocr=False):
+    def __init__(
+        self,
+        engine="pypdfium2",
+        classify=True,
+        ocr=False,
+        min_text_quality=0.5,
+        page_fraction=0.5,
+        min_page_chars=16,
+        detect_scanned=False,
+    ):
         self.document_tier1_engine = engine
         self.document_classify_enabled = classify
         self.document_ocr_enabled = ocr
+        self.document_ocr_min_text_quality = min_text_quality
+        self.document_ocr_page_fraction = page_fraction
+        self.document_ocr_min_page_chars = min_page_chars
+        self.document_ocr_detect_scanned = detect_scanned
 
 
 def _registry(*procs: tuple[DocumentProcessor, int]) -> ProcessorRegistry:
@@ -113,6 +126,11 @@ async def test_records_classification(monkeypatch):
     r = _registry((_Fake("fast", "fast"), 20))
     await r.process(b"%PDF-1.7", "application/pdf")
     rec.assert_called_once()
+    # recommended_tier, flags, mean_text_quality, ocr_page_fraction all threaded
+    # through (the last two feed the per-tenant tuning histograms).
+    args = rec.call_args.args
+    assert len(args) == 4
+    assert isinstance(args[0], str) and isinstance(args[3], float)
 
 
 async def test_classify_disabled_skips_recording(monkeypatch):