refactor: address PR #851 review round 1 (ingest transport)

- Add IngestTransport.active_consumer_count (0 by default; LocalTransport stores
  the started count) so app.py logs the worker count without re-checking
  INGEST_QUEUE — the last backend-knowledge leak in the lifespan is gone.
- Document that DistributedTransport is postgres/procrastinate-specific by design
  (aclose() calls ProcrastinateTaskProducer.drain()); other distributed backends
  would be separate IngestTransport subclasses.
- Clarify the _wire_vector_sync_state log line (writes app.state + singleton, not
  only the singleton).
- Strengthen the LocalTransport test: assert active_consumer_count transitions
  0→N and that each worker receives a distinct cloned receive stream.

Refs: Deck #196

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
2026-06-04 20:07:19 +02:00
co-authored by Claude Opus 4.8
parent e7bcdb1950
commit 655d608fb7
3 changed files with 36 additions and 8 deletions
+10 -1
View File
@@ -70,6 +70,8 @@ class TestBuildTransport:
class TestLocalTransport:
async def test_run_consumers_starts_count_workers_off_shared_stream(self):
transport = LocalTransport(max_buffer_size=5)
# Not yet started → no active consumers.
assert transport.active_consumer_count == 0
started: list[int] = []
received_streams: list[object] = []
@@ -85,9 +87,12 @@ class TestLocalTransport:
await transport.run_consumers(tg, fake_worker, 3)
assert sorted(started) == [0, 1, 2]
# Each worker gets its own (cloned) receive handle, none None.
assert transport.active_consumer_count == 3
# Each worker gets its own distinct cloned receive handle (so each
# observes end-of-stream when the senders all close), none None.
assert len(received_streams) == 3
assert all(s is not None for s in received_streams)
assert len({id(s) for s in received_streams}) == 3
class TestDistributedTransport:
@@ -95,6 +100,9 @@ class TestDistributedTransport:
producer = AsyncMock()
transport = DistributedTransport(producer)
# No in-process consumers for the distributed backend.
assert transport.active_consumer_count == 0
# Passing sentinel task group / spawn callback proves the no-op never
# touches them (the external worker is the consumer). The casts satisfy
# the signature; the values are deliberately unusable to catch any
@@ -104,6 +112,7 @@ class TestDistributedTransport:
await transport.run_consumers(sentinel_tg, sentinel_spawn, count=3)
producer.assert_not_awaited()
assert transport.active_consumer_count == 0
async def test_aclose_drains_producer(self):
producer = AsyncMock()