feat(ingest): record suppressed OCR escalations (what-if-OCR signal)
OCR is the paid, opt-in tier (DOCUMENT_OCR_ENABLED, default off). The per-tier
escalation gate already declines to hop to OCR when it's disabled (the pre-OCR
tier is terminal — no surprise cost), but that left operators blind to how much
OCR demand exists.
evaluate_escalation now returns a structured EscalationDecision:
- "hop" — a higher tier can run; the caller raises EscalateError (queue-hop).
- "suppressed" — the ideal next tier (e.g. ocr) exists but is DISABLED; the caller
indexes the current tier's output as terminal and records the
would-be hop on the new astrolabe_document_escalation_suppressed_total
{from_tier,to_tier,reason} counter instead of hopping.
- None — index as-is (good text, or no such tier at all).
So with OCR off, escalation_suppressed_total{to_tier="ocr"} is the latent OCR
demand an operator weighs before enabling OCR; enabling it converts these into
real document_escalation_total{to_tier="ocr"} hops. next_available_tier gains an
ignore_enabled flag to compute the *ideal* (enabled-gate-ignored) target.
Tests: registry suppressed vs hop vs terminal (incl. structured-hop-not-suppressed
when OCR off but structured available); _parse_pdf_tier records suppressed +
indexes without raising.
Deck #324 (parent #323).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
6db48830af
commit
a27ddb2d5a
@@ -13,6 +13,7 @@ from nextcloud_mcp_server.document_processors.base import (
|
||||
DocumentProcessor,
|
||||
ProcessingResult,
|
||||
)
|
||||
from nextcloud_mcp_server.document_processors.escalation import EscalationDecision
|
||||
from nextcloud_mcp_server.document_processors.registry import ProcessorRegistry
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
@@ -334,7 +335,7 @@ def test_evaluate_escalation_empty_jumps_to_ocr(monkeypatch):
|
||||
processor="fast",
|
||||
)
|
||||
decision = r.evaluate_escalation(res, b"%PDF", "fast", _Settings(ocr=True))
|
||||
assert decision == ("ocr", "empty_text")
|
||||
assert decision == EscalationDecision("hop", "ocr", "empty_text")
|
||||
|
||||
|
||||
def test_evaluate_escalation_lowconf_goes_to_structured(monkeypatch):
|
||||
@@ -357,7 +358,7 @@ def test_evaluate_escalation_lowconf_goes_to_structured(monkeypatch):
|
||||
processor="fast",
|
||||
)
|
||||
decision = r.evaluate_escalation(res, b"%PDF", "fast", _Settings(ocr=True))
|
||||
assert decision == ("structured", "low_confidence")
|
||||
assert decision == EscalationDecision("hop", "structured", "low_confidence")
|
||||
|
||||
|
||||
def test_evaluate_escalation_failure_not_escalated(monkeypatch):
|
||||
@@ -419,4 +420,65 @@ def test_evaluate_escalation_lowconf_to_ocr_when_no_structured(monkeypatch):
|
||||
processor="fast",
|
||||
)
|
||||
decision = r.evaluate_escalation(res, b"%PDF", "fast", _Settings(ocr=True))
|
||||
assert decision == ("ocr", "low_confidence")
|
||||
assert decision == EscalationDecision("hop", "ocr", "low_confidence")
|
||||
|
||||
|
||||
def test_evaluate_escalation_suppressed_when_ocr_disabled(monkeypatch):
|
||||
"""OCR off: a scanned doc does NOT hop to ocr; it returns a 'suppressed'
|
||||
decision (the what-if-OCR signal) so the caller indexes at the current tier."""
|
||||
monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
|
||||
r = _registry((_Fake("fast", "fast"), 20), (_Fake("ocr", "ocr"), 5))
|
||||
res = ProcessingResult(
|
||||
text="",
|
||||
metadata={
|
||||
"page_count": 1,
|
||||
"page_boundaries": [{"page": 1, "start_offset": 0, "end_offset": 0}],
|
||||
},
|
||||
processor="fast",
|
||||
)
|
||||
decision = r.evaluate_escalation(res, b"%PDF", "fast", _Settings(ocr=False))
|
||||
assert decision == EscalationDecision("suppressed", "ocr", "empty_text")
|
||||
|
||||
|
||||
def test_evaluate_escalation_lowconf_suppressed_when_only_ocr_disabled(monkeypatch):
|
||||
"""fast+ocr only, OCR off, junk text: the next rung is the disabled ocr, so
|
||||
the would-be hop is suppressed (not a structured hop, which isn't registered)."""
|
||||
monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
|
||||
junk = "q" * 40
|
||||
r = _registry((_Fake("fast", "fast"), 20), (_Fake("ocr", "ocr"), 5))
|
||||
res = ProcessingResult(
|
||||
text=junk,
|
||||
metadata={
|
||||
"page_count": 1,
|
||||
"page_boundaries": [
|
||||
{"page": 1, "start_offset": 0, "end_offset": len(junk)}
|
||||
],
|
||||
},
|
||||
processor="fast",
|
||||
)
|
||||
decision = r.evaluate_escalation(res, b"%PDF", "fast", _Settings(ocr=False))
|
||||
assert decision == EscalationDecision("suppressed", "ocr", "low_confidence")
|
||||
|
||||
|
||||
def test_evaluate_escalation_structured_hop_not_suppressed_when_ocr_off(monkeypatch):
|
||||
"""OCR off but structured available + junk text → real hop to structured
|
||||
(not suppressed): the in-cluster rung can still run."""
|
||||
monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
|
||||
junk = "w" * 40
|
||||
r = _registry(
|
||||
(_Fake("fast", "fast"), 20),
|
||||
(_Fake("structured", "structured"), 10),
|
||||
(_Fake("ocr", "ocr"), 5),
|
||||
)
|
||||
res = ProcessingResult(
|
||||
text=junk,
|
||||
metadata={
|
||||
"page_count": 1,
|
||||
"page_boundaries": [
|
||||
{"page": 1, "start_offset": 0, "end_offset": len(junk)}
|
||||
],
|
||||
},
|
||||
processor="fast",
|
||||
)
|
||||
decision = r.evaluate_escalation(res, b"%PDF", "fast", _Settings(ocr=False))
|
||||
assert decision == EscalationDecision("hop", "structured", "low_confidence")
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
"""Unit tests for the per-tier PDF parse + escalation gate (Deck #323).
|
||||
"""Unit tests for the per-tier PDF parse + escalation gate (Deck #323/#324).
|
||||
|
||||
``processor._parse_pdf_tier`` runs one tier and either returns the result to
|
||||
index or raises ``EscalateError`` (a queue-hop). These exercise the decision
|
||||
without standing up the full ingest pipeline.
|
||||
``processor._parse_pdf_tier`` runs one tier and either indexes the result,
|
||||
raises ``EscalateError`` (a real queue-hop), or — when the ideal next tier is
|
||||
disabled (OCR off) — records a suppressed escalation and indexes as terminal.
|
||||
These exercise the decision without standing up the full ingest pipeline.
|
||||
"""
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
@@ -10,7 +11,10 @@ from unittest.mock import AsyncMock, MagicMock
|
||||
import pytest
|
||||
|
||||
from nextcloud_mcp_server.document_processors.base import ProcessingResult
|
||||
from nextcloud_mcp_server.document_processors.escalation import EscalateError
|
||||
from nextcloud_mcp_server.document_processors.escalation import (
|
||||
EscalateError,
|
||||
EscalationDecision,
|
||||
)
|
||||
from nextcloud_mcp_server.vector import processor
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
@@ -25,7 +29,9 @@ def _registry(result: ProcessingResult, decision):
|
||||
|
||||
async def test_good_parse_returns_result(monkeypatch):
|
||||
rec = MagicMock()
|
||||
sup = MagicMock()
|
||||
monkeypatch.setattr(processor, "record_document_escalation", rec)
|
||||
monkeypatch.setattr(processor, "record_document_escalation_suppressed", sup)
|
||||
result = ProcessingResult(text="clean", metadata={}, processor="fast")
|
||||
reg = _registry(result, decision=None)
|
||||
out = await processor._parse_pdf_tier(
|
||||
@@ -33,13 +39,14 @@ async def test_good_parse_returns_result(monkeypatch):
|
||||
)
|
||||
assert out is result
|
||||
rec.assert_not_called()
|
||||
sup.assert_not_called()
|
||||
|
||||
|
||||
async def test_low_quality_parse_raises_escalate(monkeypatch):
|
||||
rec = MagicMock()
|
||||
monkeypatch.setattr(processor, "record_document_escalation", rec)
|
||||
result = ProcessingResult(text="", metadata={}, processor="fast")
|
||||
reg = _registry(result, decision=("ocr", "empty_text"))
|
||||
reg = _registry(result, decision=EscalationDecision("hop", "ocr", "empty_text"))
|
||||
with pytest.raises(EscalateError) as ei:
|
||||
await processor._parse_pdf_tier(
|
||||
reg, b"%PDF", "application/pdf", "f.pdf", "fast", settings=object()
|
||||
@@ -51,16 +58,37 @@ async def test_low_quality_parse_raises_escalate(monkeypatch):
|
||||
rec.assert_called_once_with("fast", "ocr", "empty_text")
|
||||
|
||||
|
||||
async def test_suppressed_decision_indexes_without_hop(monkeypatch):
|
||||
"""OCR-off (suppressed): index this tier's result, record the would-be hop,
|
||||
do NOT raise EscalateError."""
|
||||
rec = MagicMock()
|
||||
sup = MagicMock()
|
||||
monkeypatch.setattr(processor, "record_document_escalation", rec)
|
||||
monkeypatch.setattr(processor, "record_document_escalation_suppressed", sup)
|
||||
result = ProcessingResult(text="junk", metadata={}, processor="fast")
|
||||
reg = _registry(
|
||||
result, decision=EscalationDecision("suppressed", "ocr", "empty_text")
|
||||
)
|
||||
out = await processor._parse_pdf_tier(
|
||||
reg, b"%PDF", "application/pdf", "f.pdf", "fast", settings=object()
|
||||
)
|
||||
assert out is result # indexed as terminal, no hop
|
||||
sup.assert_called_once_with("fast", "ocr", "empty_text")
|
||||
rec.assert_not_called()
|
||||
|
||||
|
||||
async def test_hard_failure_returns_result_without_escalating(monkeypatch):
|
||||
rec = MagicMock()
|
||||
sup = MagicMock()
|
||||
monkeypatch.setattr(processor, "record_document_escalation", rec)
|
||||
monkeypatch.setattr(processor, "record_document_escalation_suppressed", sup)
|
||||
result = ProcessingResult(
|
||||
text="",
|
||||
metadata={"parse_failed_reason": "oversize"},
|
||||
processor="size_guard",
|
||||
success=False,
|
||||
)
|
||||
reg = _registry(result, decision=("ocr", "empty_text"))
|
||||
reg = _registry(result, decision=EscalationDecision("hop", "ocr", "empty_text"))
|
||||
out = await processor._parse_pdf_tier(
|
||||
reg, b"%PDF", "application/pdf", "big.pdf", "fast", settings=object()
|
||||
)
|
||||
@@ -68,3 +96,4 @@ async def test_hard_failure_returns_result_without_escalating(monkeypatch):
|
||||
assert out is result
|
||||
reg.evaluate_escalation.assert_not_called()
|
||||
rec.assert_not_called()
|
||||
sup.assert_not_called()
|
||||
|
||||
Reference in New Issue
Block a user