feat: add IngestTransport port for local/distributed ingest backends
Finish the hexagonal ports-&-adapters split started in #183. The producer side already had a TaskProducer port + adapters, but the consumer side was unabstracted and the INGEST_QUEUE selection leaked into a duplicated `if use_postgres:` branch across both app.py lifespan paths. Introduce an IngestTransport ABC (vector/queue/transport.py) that bundles the producer with running (or not running) the in-process consumer pool, built by a single build_transport() factory: - LocalTransport (INGEST_QUEUE=memory): in-process anyio stream drained by an N-worker pool that run_consumers starts. - DistributedTransport (INGEST_QUEUE=postgres): wraps ProcrastinateTaskProducer; run_consumers is a no-op because the consumer is the external `worker` role. Both lifespan paths now call build_transport + _wire_vector_sync_state (new helper that centralizes the app.state / module-singleton / browser-app writes) + transport.run_consumers + transport.aclose(), with no INGEST_QUEUE branching and no getattr drain probe. Adding a future backend (Redis/NATS/SQS) is one new adapter + one build_transport arm, with no app.py or scanner change. Preserves the single-tenant parallelism invariant (one shared multiplexed queue + N-worker pool, per-document not per-user dispatch) and documents it in ADR-028. The worker CLI is unchanged (it is the external consumer). Refs: Deck #196 (Deck #197 tracks the explicit parallelism regression test) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
55ea8dd358
commit
e7bcdb1950
@@ -0,0 +1,114 @@
|
||||
"""Unit tests for the ingest transport port (ADR-028; Deck #196).
|
||||
|
||||
Covers the factory's backend selection and each adapter's contract. The
|
||||
single-tenant parallelism invariant (cross-user overlap) has its own follow-up
|
||||
(Deck #197); here we only assert that ``LocalTransport.run_consumers`` starts the
|
||||
requested number of workers off the shared stream.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import cast
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import anyio
|
||||
import pytest
|
||||
from anyio.abc import TaskGroup
|
||||
|
||||
import nextcloud_mcp_server.vector.queue.transport as transport_mod
|
||||
from nextcloud_mcp_server.config import Settings
|
||||
from nextcloud_mcp_server.vector.queue import (
|
||||
DistributedTransport,
|
||||
LocalTransport,
|
||||
MemoryTaskProducer,
|
||||
SpawnWorker,
|
||||
build_transport,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def _settings(**kwargs) -> Settings:
|
||||
"""A duck-typed Settings carrying only the fields build_transport reads.
|
||||
|
||||
cast keeps ``ty`` honest about the real signature while avoiding the cost of
|
||||
constructing a full Settings (dynaconf + validators) for a two-field read.
|
||||
"""
|
||||
return cast(Settings, SimpleNamespace(**kwargs))
|
||||
|
||||
|
||||
class TestBuildTransport:
|
||||
async def test_memory_returns_local_transport(self):
|
||||
settings = _settings(ingest_queue="memory", vector_sync_queue_max_size=7)
|
||||
transport = await build_transport(settings)
|
||||
|
||||
assert isinstance(transport, LocalTransport)
|
||||
assert isinstance(transport.producer, MemoryTaskProducer)
|
||||
# Memory backend exposes both raw stream ends.
|
||||
assert transport.send_stream is not None
|
||||
assert transport.receive_stream is not None
|
||||
|
||||
async def test_postgres_returns_distributed_transport(self, monkeypatch):
|
||||
producer = AsyncMock()
|
||||
|
||||
async def fake_build_producer(settings):
|
||||
return producer
|
||||
|
||||
monkeypatch.setattr(transport_mod, "build_producer", fake_build_producer)
|
||||
|
||||
settings = _settings(ingest_queue="postgres")
|
||||
transport = await build_transport(settings)
|
||||
|
||||
assert isinstance(transport, DistributedTransport)
|
||||
assert transport.producer is producer
|
||||
# Schema applied once on the open pool before any defer.
|
||||
producer.ensure_schema.assert_awaited_once()
|
||||
# No in-process stream for the distributed backend.
|
||||
assert transport.send_stream is None
|
||||
assert transport.receive_stream is None
|
||||
|
||||
|
||||
class TestLocalTransport:
|
||||
async def test_run_consumers_starts_count_workers_off_shared_stream(self):
|
||||
transport = LocalTransport(max_buffer_size=5)
|
||||
started: list[int] = []
|
||||
received_streams: list[object] = []
|
||||
|
||||
async def fake_worker(
|
||||
worker_id, receive_stream, *, task_status=anyio.TASK_STATUS_IGNORED
|
||||
):
|
||||
started.append(worker_id)
|
||||
received_streams.append(receive_stream)
|
||||
# Must signal readiness or tg.start blocks forever.
|
||||
task_status.started()
|
||||
|
||||
async with anyio.create_task_group() as tg:
|
||||
await transport.run_consumers(tg, fake_worker, 3)
|
||||
|
||||
assert sorted(started) == [0, 1, 2]
|
||||
# Each worker gets its own (cloned) receive handle, none None.
|
||||
assert len(received_streams) == 3
|
||||
assert all(s is not None for s in received_streams)
|
||||
|
||||
|
||||
class TestDistributedTransport:
|
||||
async def test_run_consumers_is_noop(self):
|
||||
producer = AsyncMock()
|
||||
transport = DistributedTransport(producer)
|
||||
|
||||
# Passing sentinel task group / spawn callback proves the no-op never
|
||||
# touches them (the external worker is the consumer). The casts satisfy
|
||||
# the signature; the values are deliberately unusable to catch any
|
||||
# accidental use.
|
||||
sentinel_tg = cast(TaskGroup, object())
|
||||
sentinel_spawn = cast(SpawnWorker, None)
|
||||
await transport.run_consumers(sentinel_tg, sentinel_spawn, count=3)
|
||||
|
||||
producer.assert_not_awaited()
|
||||
|
||||
async def test_aclose_drains_producer(self):
|
||||
producer = AsyncMock()
|
||||
transport = DistributedTransport(producer)
|
||||
|
||||
await transport.aclose()
|
||||
|
||||
producer.drain.assert_awaited_once()
|
||||
Reference in New Issue
Block a user