feat: add IngestTransport port for local/distributed ingest backends

Finish the hexagonal ports-&-adapters split started in #183. The producer side
already had a TaskProducer port + adapters, but the consumer side was
unabstracted and the INGEST_QUEUE selection leaked into a duplicated
`if use_postgres:` branch across both app.py lifespan paths.

Introduce an IngestTransport ABC (vector/queue/transport.py) that bundles the
producer with running (or not running) the in-process consumer pool, built by a
single build_transport() factory:

- LocalTransport (INGEST_QUEUE=memory): in-process anyio stream drained by an
  N-worker pool that run_consumers starts.
- DistributedTransport (INGEST_QUEUE=postgres): wraps ProcrastinateTaskProducer;
  run_consumers is a no-op because the consumer is the external `worker` role.

Both lifespan paths now call build_transport + _wire_vector_sync_state (new
helper that centralizes the app.state / module-singleton / browser-app writes) +
transport.run_consumers + transport.aclose(), with no INGEST_QUEUE branching and
no getattr drain probe. Adding a future backend (Redis/NATS/SQS) is one new
adapter + one build_transport arm, with no app.py or scanner change.

Preserves the single-tenant parallelism invariant (one shared multiplexed queue
+ N-worker pool, per-document not per-user dispatch) and documents it in
ADR-028. The worker CLI is unchanged (it is the external consumer).

Refs: Deck #196 (Deck #197 tracks the explicit parallelism regression test)

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
2026-06-04 19:43:31 +02:00
co-authored by Claude Opus 4.8
parent 55ea8dd358
commit e7bcdb1950
6 changed files with 593 additions and 142 deletions
+114
View File
@@ -0,0 +1,114 @@
"""Unit tests for the ingest transport port (ADR-028; Deck #196).
Covers the factory's backend selection and each adapter's contract. The
single-tenant parallelism invariant (cross-user overlap) has its own follow-up
(Deck #197); here we only assert that ``LocalTransport.run_consumers`` starts the
requested number of workers off the shared stream.
"""
from types import SimpleNamespace
from typing import cast
from unittest.mock import AsyncMock
import anyio
import pytest
from anyio.abc import TaskGroup
import nextcloud_mcp_server.vector.queue.transport as transport_mod
from nextcloud_mcp_server.config import Settings
from nextcloud_mcp_server.vector.queue import (
DistributedTransport,
LocalTransport,
MemoryTaskProducer,
SpawnWorker,
build_transport,
)
pytestmark = pytest.mark.unit
def _settings(**kwargs) -> Settings:
"""A duck-typed Settings carrying only the fields build_transport reads.
cast keeps ``ty`` honest about the real signature while avoiding the cost of
constructing a full Settings (dynaconf + validators) for a two-field read.
"""
return cast(Settings, SimpleNamespace(**kwargs))
class TestBuildTransport:
async def test_memory_returns_local_transport(self):
settings = _settings(ingest_queue="memory", vector_sync_queue_max_size=7)
transport = await build_transport(settings)
assert isinstance(transport, LocalTransport)
assert isinstance(transport.producer, MemoryTaskProducer)
# Memory backend exposes both raw stream ends.
assert transport.send_stream is not None
assert transport.receive_stream is not None
async def test_postgres_returns_distributed_transport(self, monkeypatch):
producer = AsyncMock()
async def fake_build_producer(settings):
return producer
monkeypatch.setattr(transport_mod, "build_producer", fake_build_producer)
settings = _settings(ingest_queue="postgres")
transport = await build_transport(settings)
assert isinstance(transport, DistributedTransport)
assert transport.producer is producer
# Schema applied once on the open pool before any defer.
producer.ensure_schema.assert_awaited_once()
# No in-process stream for the distributed backend.
assert transport.send_stream is None
assert transport.receive_stream is None
class TestLocalTransport:
async def test_run_consumers_starts_count_workers_off_shared_stream(self):
transport = LocalTransport(max_buffer_size=5)
started: list[int] = []
received_streams: list[object] = []
async def fake_worker(
worker_id, receive_stream, *, task_status=anyio.TASK_STATUS_IGNORED
):
started.append(worker_id)
received_streams.append(receive_stream)
# Must signal readiness or tg.start blocks forever.
task_status.started()
async with anyio.create_task_group() as tg:
await transport.run_consumers(tg, fake_worker, 3)
assert sorted(started) == [0, 1, 2]
# Each worker gets its own (cloned) receive handle, none None.
assert len(received_streams) == 3
assert all(s is not None for s in received_streams)
class TestDistributedTransport:
async def test_run_consumers_is_noop(self):
producer = AsyncMock()
transport = DistributedTransport(producer)
# Passing sentinel task group / spawn callback proves the no-op never
# touches them (the external worker is the consumer). The casts satisfy
# the signature; the values are deliberately unusable to catch any
# accidental use.
sentinel_tg = cast(TaskGroup, object())
sentinel_spawn = cast(SpawnWorker, None)
await transport.run_consumers(sentinel_tg, sentinel_spawn, count=3)
producer.assert_not_awaited()
async def test_aclose_drains_producer(self):
producer = AsyncMock()
transport = DistributedTransport(producer)
await transport.aclose()
producer.drain.assert_awaited_once()