fix(vector): make NATS status subscriber resilient at startup
Follow-up to PR #814 review. NatsStatusSubscriber.run() called task_status.started() *after* the fallible pull_subscribe, so a NATS broker that wasn't ready when the MCP server started would crash the lifespan instead of retrying. Bus status is a non-critical observability path, so: - signal started() before the first subscribe (semantics: "loop is running", not "subscription succeeded"); - retry a failed subscribe with backoff instead of propagating; - on a real fetch error (not an idle timeout) drop the subscription and re-subscribe rather than fetching against a possibly-dead handle. Also anchor the _content_hash etag-threading TODO to the PR #814 review thread so it is discoverable outside git blame. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
f65ebfc65b
commit
cd2145df09
@@ -54,10 +54,10 @@ def _content_hash(task: DocumentTask) -> str:
|
||||
"""etag is the change-detection token; fall back to modified_at when it is
|
||||
absent (e.g. deletes, or sources whose etag we don't thread through).
|
||||
|
||||
TODO(follow-up): thread etags for file / deck_card / news_item scans too
|
||||
(only note scans pass etag today). Until then their JetStream Nats-Msg-Id
|
||||
dedup keys off modified_at, which misses content changes that leave
|
||||
modified_at unchanged (e.g. a file move/rename).
|
||||
TODO(follow-up, PR #814 review): thread etags for file / deck_card /
|
||||
news_item scans too (only note scans pass etag today). Until then their
|
||||
JetStream Nats-Msg-Id dedup keys off modified_at, which misses content
|
||||
changes that leave modified_at unchanged (e.g. a file move/rename).
|
||||
"""
|
||||
return task.etag or str(task.modified_at)
|
||||
|
||||
|
||||
@@ -144,12 +144,29 @@ class NatsStatusSubscriber:
|
||||
import nats.errors # noqa: PLC0415
|
||||
|
||||
subject = f"mcp.document.*.{self.tenant_id}"
|
||||
sub = await self._js.pull_subscribe(
|
||||
subject, durable=f"mcp-status-{self.tenant_id}"
|
||||
)
|
||||
# Signal "task running" *before* the first (fallible) subscribe: bus
|
||||
# status is a non-critical observability path, so a broker that isn't
|
||||
# ready at startup should retry below rather than crash the lifespan.
|
||||
# ``started()`` therefore means "the subscriber loop is running", not
|
||||
# "the subscription succeeded".
|
||||
if task_status is not None:
|
||||
task_status.started()
|
||||
|
||||
sub = None
|
||||
while not shutdown_event.is_set():
|
||||
if sub is None:
|
||||
try:
|
||||
sub = await self._js.pull_subscribe(
|
||||
subject, durable=f"mcp-status-{self.tenant_id}"
|
||||
)
|
||||
except Exception:
|
||||
# Broker not ready / transient connect error: back off and
|
||||
# retry the subscribe instead of giving up.
|
||||
logger.warning(
|
||||
"NATS status subscribe failed; retrying", exc_info=True
|
||||
)
|
||||
await anyio.sleep(5)
|
||||
continue
|
||||
try:
|
||||
msgs = await sub.fetch(batch=16, timeout=5)
|
||||
except nats.errors.TimeoutError:
|
||||
@@ -157,11 +174,14 @@ class NatsStatusSubscriber:
|
||||
# straight back to re-check shutdown — no log, no extra sleep.
|
||||
continue
|
||||
except Exception:
|
||||
# Real broker error (disconnect, auth failure, stream deleted).
|
||||
# Log it and back off so we don't hot-spin against a dead broker.
|
||||
# Real broker error (disconnect, auth failure, stream deleted):
|
||||
# drop the (possibly dead) subscription, back off, and
|
||||
# re-subscribe on the next iteration rather than hot-spinning.
|
||||
logger.warning(
|
||||
"NATS status subscriber fetch failed; retrying", exc_info=True
|
||||
"NATS status subscriber fetch failed; re-subscribing",
|
||||
exc_info=True,
|
||||
)
|
||||
sub = None
|
||||
await anyio.sleep(5)
|
||||
continue
|
||||
for msg in msgs:
|
||||
|
||||
Reference in New Issue
Block a user