admin/mcp-nextcloud
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ingest): split OCR into tier2 in-cluster (GPU, gateway-only) + tier3 upstream

Browse Source 
Insert a configurable in-cluster OCR rung into the escalation ladder (Deck #353):
a tier2-eligible doc is OCR'd on the on-demand burst GPU before falling through to
paid upstream OCR. The in-cluster backend is reached ONLY via the embedding gateway
(model prefix routes to the GPU over the tailnet) and is a config value (default
surya/surya-ocr-2, swappable to e.g. lightonocr) — never hard-coded.

Ladder: fast -> structured -> ocr-incluster -> ocr-upstream
(queues ingest-ocr-incluster / ingest-ocr-upstream).

- escalation.py: 4-tier ladder; in-cluster flag folded into the dead-letter signature.
- ocr.py: OcrProcessor(name, tier, model_setting, gateway_only); build_ocr_backend(
  ..., model=, gateway_only=) — gateway_only forces the gateway backend (never the
  direct Mistral fallback), disabling the tier with a warning if no gateway URL.
- registry.py: per-rung enable map; scanned docs target minimum="ocr-incluster";
  inline path runs the cheapest available OCR rung.
- procrastinate.py: two OCR queues; legacy ingest-ocr kept as a drain target.
- config.py: DOCUMENT_OCR_INCLUSTER_ENABLED (off) + DOCUMENT_OCR_INCLUSTER_MODEL.
- __init__.py: register the two OCR instances; vector/processor.py: pages_ocr
  metered for the upstream (paid) rung only; cli.py: new --tier choices + legacy drain.
- metrics.py: zero the legacy ingest-ocr queue gauge during rollout.
- tests: migrated to the split ladder + new tests (gateway-only forcing, per-tier
  model incl. lightonocr override, no-hard-coded-surya guard). 1792 pass; ruff + ty green.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
2026-06-17 23:28:29 +02:00
co-authored by Claude Opus 4.8
parent 060084029f
commit c21804fbbc
16 changed files with 384 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/test_registry_tiering.py
+70 -45
View File
@@ -74,6 +74,7 @@ class _Settings:
engine="pypdfium2",
classify=True,
ocr=False,
ocr_incluster=False,
min_text_quality=0.5,
page_fraction=0.5,
min_page_chars=16,
@@ -85,7 +86,13 @@ class _Settings:
):
self.document_tier1_engine = engine
self.document_classify_enabled = classify
# ``ocr`` enables the UPSTREAM rung (document_ocr_enabled); ``ocr_incluster``
# the in-cluster GPU rung (tried first). The model attrs let
# build_ocr_backend resolve a per-tier model id.
self.document_ocr_enabled = ocr
self.document_ocr_incluster_enabled = ocr_incluster
self.document_ocr_model = "mistral/mistral-ocr-latest"
self.document_ocr_incluster_model = "surya/surya-ocr-2"
self.document_ocr_min_text_quality = min_text_quality
self.document_ocr_page_fraction = page_fraction
self.document_ocr_min_page_chars = min_page_chars
@@ -195,10 +202,10 @@ async def test_ocr_escalation_on_empty_text(monkeypatch):
monkeypatch.setattr(reg_mod, "record_document_escalation", esc)
r = _registry(
(_Fake("fast", "fast", text=""), 20),
(_Fake("ocr", "ocr", text="ocr text"), 5),
(_Fake("ocr-upstream", "ocr-upstream", text="ocr text"), 5),
)
res = await r.process(b"%PDF-1.7", "application/pdf")
assert res.processor == "ocr"
assert res.processor == "ocr-upstream"
esc.assert_called_once()
@@ -210,7 +217,7 @@ async def test_zero_page_pdf_does_not_escalate(monkeypatch):
monkeypatch.setattr(reg_mod, "record_document_escalation", esc)
r = _registry(
(_Fake("fast", "fast", text="", pages=0), 20),
(_Fake("ocr", "ocr"), 5),
(_Fake("ocr-upstream", "ocr-upstream"), 5),
)
res = await r.process(b"%PDF-1.7", "application/pdf")
assert res.processor == "fast"
@@ -231,7 +238,7 @@ async def test_ocr_failure_falls_back_to_fast(monkeypatch):
monkeypatch.setattr(reg_mod, "record_document_escalation", MagicMock())
r = _registry(
(_Fake("fast", "fast", text=""), 20),
(_Fake("ocr", "ocr", text="", success=False), 5),
(_Fake("ocr-upstream", "ocr-upstream", text="", success=False), 5),
)
res = await r.process(b"%PDF-1.7", "application/pdf")
assert res.processor == "fast"
@@ -242,7 +249,7 @@ async def test_no_ocr_escalation_when_disabled(monkeypatch):
monkeypatch.setattr(reg_mod, "get_settings", lambda: _Settings(ocr=False))
r = _registry(
(_Fake("fast", "fast", text=""), 20),
(_Fake("ocr", "ocr"), 5),
(_Fake("ocr-upstream", "ocr-upstream"), 5),
)
res = await r.process(b"%PDF-1.7", "application/pdf")
# Fast tier is terminal when OCR is disabled.
@@ -289,11 +296,11 @@ async def test_glyph_corrupt_no_structured_falls_through_to_ocr(monkeypatch):
monkeypatch.setattr(reg_mod, "record_document_escalation", esc)
r = _registry(
(_Fake("fast", "fast", text=_GLYPH), 20),
(_Fake("ocr", "ocr", text="ocr recovered text"), 5),
(_Fake("ocr-upstream", "ocr-upstream", text="ocr recovered text"), 5),
) # no structured registered
res = await r.process(b"%PDF-1.7", "application/pdf")
assert res.processor == "ocr"
esc.assert_called_once_with("fast", "ocr", "corrupt_glyphs")
assert res.processor == "ocr-upstream"
esc.assert_called_once_with("fast", "ocr-upstream", "corrupt_glyphs")
async def test_inline_lowconf_tries_structured_before_ocr(monkeypatch):
@@ -305,7 +312,7 @@ async def test_inline_lowconf_tries_structured_before_ocr(monkeypatch):
r = _registry(
(_Fake("fast", "fast", text="x" * 40), 20), # one long token -> quality ~0
(_Fake("structured", "structured", text="clean recovered prose text here"), 10),
(_Fake("ocr", "ocr", text="ocr text"), 5),
(_Fake("ocr-upstream", "ocr-upstream", text="ocr text"), 5),
)
res = await r.process(b"%PDF-1.7", "application/pdf")
assert res.processor == "structured"
@@ -321,11 +328,11 @@ async def test_inline_empty_skips_structured_straight_to_ocr(monkeypatch):
r = _registry(
(_Fake("fast", "fast", text=""), 20),
(_Fake("structured", "structured", text="should not run"), 10),
(_Fake("ocr", "ocr", text="ocr text"), 5),
(_Fake("ocr-upstream", "ocr-upstream", text="ocr text"), 5),
)
res = await r.process(b"%PDF-1.7", "application/pdf")
assert res.processor == "ocr"
esc.assert_called_once_with("fast", "ocr", "empty_text")
assert res.processor == "ocr-upstream"
esc.assert_called_once_with("fast", "ocr-upstream", "empty_text")
async def test_inline_fast_structured_ocr_cascade(monkeypatch):
@@ -339,13 +346,13 @@ async def test_inline_fast_structured_ocr_cascade(monkeypatch):
r = _registry(
(_Fake("fast", "fast", text="x" * 40), 20), # quality ~0, non-empty
(_Fake("structured", "structured", text=""), 10), # re-extract empty
(_Fake("ocr", "ocr", text="ocr recovered text"), 5),
(_Fake("ocr-upstream", "ocr-upstream", text="ocr recovered text"), 5),
)
res = await r.process(b"%PDF-1.7", "application/pdf")
assert res.processor == "ocr"
assert res.processor == "ocr-upstream"
assert esc.call_args_list == [
call("fast", "structured", "low_confidence"),
call("structured", "ocr", "empty_text"),
call("structured", "ocr-upstream", "empty_text"),
]
@@ -359,13 +366,13 @@ async def test_inline_structured_still_corrupt_escalates_to_ocr(monkeypatch):
r = _registry(
(_Fake("fast", "fast", text=_GLYPH), 20),
(_Fake("structured", "structured", text=_GLYPH), 10), # still corrupt
(_Fake("ocr", "ocr", text="ocr recovered text"), 5),
(_Fake("ocr-upstream", "ocr-upstream", text="ocr recovered text"), 5),
)
res = await r.process(b"%PDF-1.7", "application/pdf")
assert res.processor == "ocr"
assert res.processor == "ocr-upstream"
assert esc.call_args_list == [
call("fast", "structured", "corrupt_glyphs"),
call("structured", "ocr", "corrupt_glyphs"),
call("structured", "ocr-upstream", "corrupt_glyphs"),
]
@@ -375,7 +382,7 @@ def test_evaluate_escalation_glyph_corrupt_goes_structured(monkeypatch):
r = _registry(
(_Fake("fast", "fast"), 20),
(_Fake("structured", "structured"), 10),
(_Fake("ocr", "ocr"), 5),
(_Fake("ocr-upstream", "ocr-upstream"), 5),
)
res = ProcessingResult(
text=_GLYPH,
@@ -399,7 +406,7 @@ def test_evaluate_escalation_glyph_corrupt_no_structured_falls_through_to_ocr(
monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
r = _registry(
(_Fake("fast", "fast"), 20),
(_Fake("ocr", "ocr"), 5),
(_Fake("ocr-upstream", "ocr-upstream"), 5),
) # no structured registered
res = ProcessingResult(
text=_GLYPH,
@@ -412,7 +419,7 @@ def test_evaluate_escalation_glyph_corrupt_no_structured_falls_through_to_ocr(
processor="fast",
)
decision = r.evaluate_escalation(res, b"%PDF", "fast", _Settings(ocr=True))
assert decision == EscalationDecision("hop", "ocr", "corrupt_glyphs")
assert decision == EscalationDecision("hop", "ocr-upstream", "corrupt_glyphs")
def test_evaluate_escalation_glyph_corrupt_no_structured_ocr_disabled_suppressed(
@@ -424,7 +431,7 @@ def test_evaluate_escalation_glyph_corrupt_no_structured_ocr_disabled_suppressed
monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
r = _registry(
(_Fake("fast", "fast"), 20),
(_Fake("ocr", "ocr"), 5),
(_Fake("ocr-upstream", "ocr-upstream"), 5),
) # structured not registered; ocr registered but disabled below
res = ProcessingResult(
text=_GLYPH,
@@ -437,7 +444,9 @@ def test_evaluate_escalation_glyph_corrupt_no_structured_ocr_disabled_suppressed
processor="fast",
)
decision = r.evaluate_escalation(res, b"%PDF", "fast", _Settings(ocr=False))
assert decision == EscalationDecision("suppressed", "ocr", "corrupt_glyphs")
assert decision == EscalationDecision(
"suppressed", "ocr-upstream", "corrupt_glyphs"
)
# --- Per-tier external path (Deck #323) -------------------------------------
@@ -449,7 +458,7 @@ async def test_process_tier_runs_named_tier(monkeypatch):
r = _registry(
(_Fake("fast", "fast"), 20),
(_Fake("structured", "structured"), 10),
(_Fake("ocr", "ocr"), 5),
(_Fake("ocr-upstream", "ocr-upstream"), 5),
)
res = await r.process_tier(b"%PDF-1.7", "application/pdf", "f.pdf", "structured")
assert res.processor == "structured"
@@ -469,7 +478,7 @@ async def test_process_tier_oversize_fails_fast(monkeypatch):
monkeypatch.setattr(
reg_mod, "get_settings", lambda: _Settings(max_pdf_size_mb=0.001)
)
r = _registry((_Fake("ocr", "ocr"), 5))
r = _registry((_Fake("ocr-upstream", "ocr-upstream"), 5))
res = await r.process_tier(b"x" * 4096, "application/pdf", "big.pdf", "ocr")
assert res.success is False
assert res.metadata["parse_failed_reason"] == "oversize"
@@ -479,7 +488,7 @@ def test_next_available_tier_walks_ladder():
r = _registry(
(_Fake("fast", "fast"), 20),
(_Fake("structured", "structured"), 10),
(_Fake("ocr", "ocr"), 5),
(_Fake("ocr-upstream", "ocr-upstream"), 5),
)
# ocr disabled -> structured is the only target above fast.
s = _Settings(ocr=False)
@@ -487,21 +496,25 @@ def test_next_available_tier_walks_ladder():
assert r.next_available_tier("structured", s) is None # ocr gated off
# ocr enabled -> reachable; minimum skips the structured rung.
s_ocr = _Settings(ocr=True)
assert r.next_available_tier("structured", s_ocr) == "ocr"
assert r.next_available_tier("fast", s_ocr, minimum="ocr") == "ocr"
assert r.next_available_tier("structured", s_ocr) == "ocr-upstream"
assert (
r.next_available_tier("fast", s_ocr, minimum="ocr-upstream") == "ocr-upstream"
)
def test_next_available_tier_skips_unregistered():
# No structured processor -> fast escalates straight to ocr.
r = _registry((_Fake("fast", "fast"), 20), (_Fake("ocr", "ocr"), 5))
assert r.next_available_tier("fast", _Settings(ocr=True)) == "ocr"
r = _registry(
(_Fake("fast", "fast"), 20), (_Fake("ocr-upstream", "ocr-upstream"), 5)
)
assert r.next_available_tier("fast", _Settings(ocr=True)) == "ocr-upstream"
def test_evaluate_escalation_good_text_indexes(monkeypatch):
monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
r = _registry(
(_Fake("fast", "fast", text="This is clean readable prose text."), 20),
(_Fake("ocr", "ocr"), 5),
(_Fake("ocr-upstream", "ocr-upstream"), 5),
)
res = ProcessingResult(
text="This is clean readable prose text.",
@@ -520,7 +533,7 @@ def test_evaluate_escalation_empty_jumps_to_ocr(monkeypatch):
r = _registry(
(_Fake("fast", "fast"), 20),
(_Fake("structured", "structured"), 10),
(_Fake("ocr", "ocr"), 5),
(_Fake("ocr-upstream", "ocr-upstream"), 5),
)
res = ProcessingResult(
text="",
@@ -531,7 +544,7 @@ def test_evaluate_escalation_empty_jumps_to_ocr(monkeypatch):
processor="fast",
)
decision = r.evaluate_escalation(res, b"%PDF", "fast", _Settings(ocr=True))
assert decision == EscalationDecision("hop", "ocr", "empty_text")
assert decision == EscalationDecision("hop", "ocr-upstream", "empty_text")
def test_evaluate_escalation_lowconf_goes_to_structured(monkeypatch):
@@ -541,7 +554,7 @@ def test_evaluate_escalation_lowconf_goes_to_structured(monkeypatch):
r = _registry(
(_Fake("fast", "fast"), 20),
(_Fake("structured", "structured"), 10),
(_Fake("ocr", "ocr"), 5),
(_Fake("ocr-upstream", "ocr-upstream"), 5),
)
res = ProcessingResult(
text=junk,
@@ -559,7 +572,9 @@ def test_evaluate_escalation_lowconf_goes_to_structured(monkeypatch):
def test_evaluate_escalation_failure_not_escalated(monkeypatch):
monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
r = _registry((_Fake("fast", "fast"), 20), (_Fake("ocr", "ocr"), 5))
r = _registry(
(_Fake("fast", "fast"), 20), (_Fake("ocr-upstream", "ocr-upstream"), 5)
)
res = ProcessingResult(
text="",
metadata={"parse_failed_reason": "error"},
@@ -590,7 +605,9 @@ def test_evaluate_escalation_terminal_when_no_higher_tier(monkeypatch):
def test_evaluate_escalation_zero_page_does_not_escalate(monkeypatch):
"""A zero-page (empty/corrupt) PDF never escalates on the external path."""
monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
r = _registry((_Fake("fast", "fast"), 20), (_Fake("ocr", "ocr"), 5))
r = _registry(
(_Fake("fast", "fast"), 20), (_Fake("ocr-upstream", "ocr-upstream"), 5)
)
res = ProcessingResult(
text="",
metadata={"page_count": 0, "page_boundaries": []},
@@ -604,7 +621,9 @@ def test_evaluate_escalation_lowconf_to_ocr_when_no_structured(monkeypatch):
unregistered structured rung), not to None."""
monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
junk = "z" * 40 # non-empty but junk -> recommended ocr, total_chars > 0
r = _registry((_Fake("fast", "fast"), 20), (_Fake("ocr", "ocr"), 5))
r = _registry(
(_Fake("fast", "fast"), 20), (_Fake("ocr-upstream", "ocr-upstream"), 5)
)
res = ProcessingResult(
text=junk,
metadata={
@@ -616,14 +635,16 @@ def test_evaluate_escalation_lowconf_to_ocr_when_no_structured(monkeypatch):
processor="fast",
)
decision = r.evaluate_escalation(res, b"%PDF", "fast", _Settings(ocr=True))
assert decision == EscalationDecision("hop", "ocr", "low_confidence")
assert decision == EscalationDecision("hop", "ocr-upstream", "low_confidence")
def test_evaluate_escalation_suppressed_when_ocr_disabled(monkeypatch):
"""OCR off: a scanned doc does NOT hop to ocr; it returns a 'suppressed'
decision (the what-if-OCR signal) so the caller indexes at the current tier."""
monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
r = _registry((_Fake("fast", "fast"), 20), (_Fake("ocr", "ocr"), 5))
r = _registry(
(_Fake("fast", "fast"), 20), (_Fake("ocr-upstream", "ocr-upstream"), 5)
)
res = ProcessingResult(
text="",
metadata={
@@ -633,7 +654,7 @@ def test_evaluate_escalation_suppressed_when_ocr_disabled(monkeypatch):
processor="fast",
)
decision = r.evaluate_escalation(res, b"%PDF", "fast", _Settings(ocr=False))
assert decision == EscalationDecision("suppressed", "ocr", "empty_text")
assert decision == EscalationDecision("suppressed", "ocr-upstream", "empty_text")
def test_evaluate_escalation_lowconf_suppressed_when_only_ocr_disabled(monkeypatch):
@@ -641,7 +662,9 @@ def test_evaluate_escalation_lowconf_suppressed_when_only_ocr_disabled(monkeypat
the would-be hop is suppressed (not a structured hop, which isn't registered)."""
monkeypatch.setattr(reg_mod, "record_document_classification", MagicMock())
junk = "q" * 40
r = _registry((_Fake("fast", "fast"), 20), (_Fake("ocr", "ocr"), 5))
r = _registry(
(_Fake("fast", "fast"), 20), (_Fake("ocr-upstream", "ocr-upstream"), 5)
)
res = ProcessingResult(
text=junk,
metadata={
@@ -653,7 +676,9 @@ def test_evaluate_escalation_lowconf_suppressed_when_only_ocr_disabled(monkeypat
processor="fast",
)
decision = r.evaluate_escalation(res, b"%PDF", "fast", _Settings(ocr=False))
assert decision == EscalationDecision("suppressed", "ocr", "low_confidence")
assert decision == EscalationDecision(
"suppressed", "ocr-upstream", "low_confidence"
)
def test_evaluate_escalation_structured_hop_not_suppressed_when_ocr_off(monkeypatch):
@@ -664,7 +689,7 @@ def test_evaluate_escalation_structured_hop_not_suppressed_when_ocr_off(monkeypa
r = _registry(
(_Fake("fast", "fast"), 20),
(_Fake("structured", "structured"), 10),
(_Fake("ocr", "ocr"), 5),
(_Fake("ocr-upstream", "ocr-upstream"), 5),
)
res = ProcessingResult(
text=junk,
@@ -706,7 +731,7 @@ def test_evaluate_escalation_empty_suppressed_even_when_structured_registered(
r = _registry(
(_Fake("fast", "fast"), 20),
(_Fake("structured", "structured"), 10), # registered but skipped for empty
(_Fake("ocr", "ocr"), 5),
(_Fake("ocr-upstream", "ocr-upstream"), 5),
)
res = ProcessingResult(
text="",
@@ -717,4 +742,4 @@ def test_evaluate_escalation_empty_suppressed_even_when_structured_registered(
processor="fast",
)
decision = r.evaluate_escalation(res, b"%PDF", "fast", _Settings(ocr=False))
assert decision == EscalationDecision("suppressed", "ocr", "empty_text")
assert decision == EscalationDecision("suppressed", "ocr-upstream", "empty_text")