fix(vector): make NATS status subscriber resilient at startup

Follow-up to PR #814 review.

NatsStatusSubscriber.run() called task_status.started() *after* the fallible
pull_subscribe, so a NATS broker that wasn't ready when the MCP server started
would crash the lifespan instead of retrying. Bus status is a non-critical
observability path, so:

- signal started() before the first subscribe (semantics: "loop is running",
  not "subscription succeeded");
- retry a failed subscribe with backoff instead of propagating;
- on a real fetch error (not an idle timeout) drop the subscription and
  re-subscribe rather than fetching against a possibly-dead handle.

Also anchor the _content_hash etag-threading TODO to the PR #814 review thread
so it is discoverable outside git blame.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
2026-05-31 21:56:12 +02:00
co-authored by Claude Opus 4.8
parent f65ebfc65b
commit cd2145df09
3 changed files with 127 additions and 10 deletions
+4 -4
View File
@@ -54,10 +54,10 @@ def _content_hash(task: DocumentTask) -> str:
"""etag is the change-detection token; fall back to modified_at when it is
absent (e.g. deletes, or sources whose etag we don't thread through).
TODO(follow-up): thread etags for file / deck_card / news_item scans too
(only note scans pass etag today). Until then their JetStream Nats-Msg-Id
dedup keys off modified_at, which misses content changes that leave
modified_at unchanged (e.g. a file move/rename).
TODO(follow-up, PR #814 review): thread etags for file / deck_card /
news_item scans too (only note scans pass etag today). Until then their
JetStream Nats-Msg-Id dedup keys off modified_at, which misses content
changes that leave modified_at unchanged (e.g. a file move/rename).
"""
return task.etag or str(task.modified_at)
+26 -6
View File
@@ -144,12 +144,29 @@ class NatsStatusSubscriber:
import nats.errors # noqa: PLC0415
subject = f"mcp.document.*.{self.tenant_id}"
sub = await self._js.pull_subscribe(
subject, durable=f"mcp-status-{self.tenant_id}"
)
# Signal "task running" *before* the first (fallible) subscribe: bus
# status is a non-critical observability path, so a broker that isn't
# ready at startup should retry below rather than crash the lifespan.
# ``started()`` therefore means "the subscriber loop is running", not
# "the subscription succeeded".
if task_status is not None:
task_status.started()
sub = None
while not shutdown_event.is_set():
if sub is None:
try:
sub = await self._js.pull_subscribe(
subject, durable=f"mcp-status-{self.tenant_id}"
)
except Exception:
# Broker not ready / transient connect error: back off and
# retry the subscribe instead of giving up.
logger.warning(
"NATS status subscribe failed; retrying", exc_info=True
)
await anyio.sleep(5)
continue
try:
msgs = await sub.fetch(batch=16, timeout=5)
except nats.errors.TimeoutError:
@@ -157,11 +174,14 @@ class NatsStatusSubscriber:
# straight back to re-check shutdown — no log, no extra sleep.
continue
except Exception:
# Real broker error (disconnect, auth failure, stream deleted).
# Log it and back off so we don't hot-spin against a dead broker.
# Real broker error (disconnect, auth failure, stream deleted):
# drop the (possibly dead) subscription, back off, and
# re-subscribe on the next iteration rather than hot-spinning.
logger.warning(
"NATS status subscriber fetch failed; retrying", exc_info=True
"NATS status subscriber fetch failed; re-subscribing",
exc_info=True,
)
sub = None
await anyio.sleep(5)
continue
for msg in msgs: