fix(ocr): wire batch settings into _field_map + review nits
Round 1 review (PR #910): - BLOCKING: add document_ocr_mode / _batch_poll_seconds / _batch_max_wait_seconds to config._field_map — without it dynaconf silently ignored the env vars and DOCUMENT_OCR_MODE=batch could never be enabled in production. Add a regression test asserting the three round-trip from env. - migration 008: give batch_ocr_jobs a composite PRIMARY KEY on (user_id, doc_id, doc_type, etag) instead of a bare UniqueConstraint (N1). - OcrProcessor: use a dedicated _batch_client_lock instead of sharing the sync backend lock (N3). - tests: use https:// gateway URLs in the new fixtures to clear SonarCloud's "insecure http" security hotspots (all 14 were test-only http://gw literals). 1653 unit tests pass; ruff + ty green. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
3b7e8d779b
commit
2b7dfc8535
@@ -36,7 +36,11 @@ def upgrade() -> None:
|
||||
# Document identity (the same keys the OCR tier receives via the
|
||||
# processor ``options``). ``etag`` is the content-version key: a changed
|
||||
# document (new etag) is a new job, so a stale row never serves results
|
||||
# for the wrong content.
|
||||
# for the wrong content. The four-column natural key IS the primary key:
|
||||
# one in-flight job per (document, content version), and the PK doubles as
|
||||
# the unique index ``insert_pending``'s ON CONFLICT target relies on. A
|
||||
# resubmit for a new etag inserts a new row; the superseded row is swept on
|
||||
# resubmit (delete_stale_for_doc).
|
||||
sa.Column("user_id", sa.Text(), nullable=False),
|
||||
sa.Column("doc_id", sa.Text(), nullable=False),
|
||||
sa.Column("doc_type", sa.Text(), nullable=False),
|
||||
@@ -51,14 +55,8 @@ def upgrade() -> None:
|
||||
# (DOCUMENT_OCR_BATCH_MAX_WAIT_SECONDS).
|
||||
sa.Column("submitted_at", sa.BigInteger(), nullable=False),
|
||||
sa.Column("updated_at", sa.BigInteger(), nullable=False),
|
||||
# One in-flight job per (document, content version). A resubmit for a new
|
||||
# etag inserts a new row; the superseded row is swept on resubmit.
|
||||
sa.UniqueConstraint(
|
||||
"user_id",
|
||||
"doc_id",
|
||||
"doc_type",
|
||||
"etag",
|
||||
name="uq_batch_ocr_jobs_doc",
|
||||
sa.PrimaryKeyConstraint(
|
||||
"user_id", "doc_id", "doc_type", "etag", name="pk_batch_ocr_jobs"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -1521,6 +1521,9 @@ def get_settings() -> Settings:
|
||||
"document_ocr_provider": "DOCUMENT_OCR_PROVIDER",
|
||||
"document_ocr_model": "DOCUMENT_OCR_MODEL",
|
||||
"document_ocr_timeout_seconds": "DOCUMENT_OCR_TIMEOUT_SECONDS",
|
||||
"document_ocr_mode": "DOCUMENT_OCR_MODE",
|
||||
"document_ocr_batch_poll_seconds": "DOCUMENT_OCR_BATCH_POLL_SECONDS",
|
||||
"document_ocr_batch_max_wait_seconds": "DOCUMENT_OCR_BATCH_MAX_WAIT_SECONDS",
|
||||
"document_ocr_min_text_quality": "DOCUMENT_OCR_MIN_TEXT_QUALITY",
|
||||
"document_ocr_page_fraction": "DOCUMENT_OCR_PAGE_FRACTION",
|
||||
"document_ocr_min_page_chars": "DOCUMENT_OCR_MIN_PAGE_CHARS",
|
||||
|
||||
@@ -272,6 +272,7 @@ class OcrProcessor(DocumentProcessor):
|
||||
# using sync" warning to once per pod.
|
||||
self._batch_client_resolved = False
|
||||
self._batch_client: Any = None
|
||||
self._batch_client_lock: anyio.Lock | None = None
|
||||
self._batch_fallback_warned = False
|
||||
|
||||
@property
|
||||
@@ -377,9 +378,9 @@ class OcrProcessor(DocumentProcessor):
|
||||
provider=mistral / no gateway). Resolved once under the backend lock so the
|
||||
token provider's M2M cache survives across documents."""
|
||||
if not self._batch_client_resolved:
|
||||
if self._backend_lock is None:
|
||||
self._backend_lock = anyio.Lock()
|
||||
async with self._backend_lock:
|
||||
if self._batch_client_lock is None:
|
||||
self._batch_client_lock = anyio.Lock()
|
||||
async with self._batch_client_lock:
|
||||
if not self._batch_client_resolved: # double-checked
|
||||
self._batch_client = build_gateway_batch_client(get_settings())
|
||||
self._batch_client_resolved = True
|
||||
|
||||
Reference in New Issue
Block a user