feat: add opt-in MCP decomposition hook points (design §10)

Adds the seven §10.2 hook-point modules + five env vars so Astrolabe Cloud can
offload document processing to the external document-processor / embedding
gateway. Purely additive: with every setting unset the server behaves exactly
as today, so self-hosters are unaffected (Deck #92).

Hook points (all default to current monolith behavior):
- config: EMBEDDING_PROVIDER, INGEST_MODE, STATUS_BACKEND,
  COLLECTION_METADATA_SOURCE, FACT_EVENT_EMITTER (+ supporting settings),
  validated in Settings.__post_init__ (fail-fast STATUS_BACKEND=local with
  INGEST_MODE=external); shared canonical.py.
- vector/payload_keys.py + acl_hash.py: cross-impl NAMESPACE/point_id (§2.2)
  and BLAKE2b-128 ACL hash (§11), pinned by fixtures shared with the
  document-processor repo.
- embedding/gateway_client.py: OpenAI-compatible GatewayProvider authenticating
  via M2M OIDC client-credentials (separate realm); manual-only registry entry.
- vector/collection_metadata.py: sentinel-point / API metadata source with env
  fallback.
- vector/queue/: hexagonal ingest producer ports + memory/NATS adapters
  (Postgres seam); INGEST_MODE=external publishes mcp.ingest.requested.{tenant}
  instead of the in-memory stream and skips the in-process processor pool. The
  lifespan becomes a composition root across both deployment branches.
- vector/queue/status.py: STATUS_BACKEND=bus subscriber feeding a StatusStore
  the vector-sync status endpoint reads.
- admin/payload_backfill.py: POST /api/v1/admin/payload-backfill (admin scope);
  processor writes the new payload keys; query-side ACL pre-filter gated behind
  ACL_PREFILTER_ENABLED (default off).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

tests/unit/vector/test_collection_metadata.py

@@ -0,0 +1,91 @@
+"""Collection metadata source + sentinel (design §10.1)."""
+
+from types import SimpleNamespace
+
+import httpx
+
+from nextcloud_mcp_server.config import Settings
+from nextcloud_mcp_server.vector import collection_metadata as cm
+from nextcloud_mcp_server.vector.payload_keys import EMBEDDING_IDENTITY
+
+
+async def test_qdrant_read_hit(mocker):
+    client = mocker.AsyncMock()
+    client.retrieve.return_value = [
+        SimpleNamespace(
+            payload={
+                EMBEDDING_IDENTITY: "mistral-embed",
+                cm.CHUNKING_CONFIG: {"chunk_size": 1024, "chunk_overlap": 100},
+                cm.IS_SENTINEL: True,
+            }
+        )
+    ]
+    meta = await cm.read_collection_metadata(client, "col", Settings())
+    assert meta["embedding_identity"] == "mistral-embed"
+    assert meta["chunking_config"]["chunk_size"] == 1024
+
+
+async def test_qdrant_miss_falls_back_to_env(mocker):
+    client = mocker.AsyncMock()
+    client.retrieve.return_value = []  # no sentinel
+    settings = Settings(document_chunk_size=2048, document_chunk_overlap=200)
+    meta = await cm.read_collection_metadata(client, "col", settings)
+    assert meta == cm.env_default_metadata(settings)
+    assert meta["chunking_config"]["chunk_size"] == 2048
+
+
+async def test_qdrant_error_falls_back_to_env(mocker):
+    client = mocker.AsyncMock()
+    client.retrieve.side_effect = RuntimeError("qdrant down")
+    settings = Settings()
+    meta = await cm.read_collection_metadata(client, "col", settings)
+    assert meta == cm.env_default_metadata(settings)
+
+
+async def test_api_source(mocker):
+    settings = Settings(
+        collection_metadata_source="api",
+        collection_metadata_api_url="http://cp",
+    )
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        assert request.url.path == "/v1/qdrant-collections/col/metadata"
+        return httpx.Response(
+            200,
+            json={
+                "embedding_identity": "amazon.titan-embed-text-v2:0",
+                "chunking_config": {"chunk_size": 512, "chunk_overlap": 50},
+            },
+        )
+
+    transport = httpx.MockTransport(handler)
+    orig = httpx.AsyncClient
+    mocker.patch.object(
+        httpx,
+        "AsyncClient",
+        lambda *a, **k: orig(*a, **{**k, "transport": transport}),
+    )
+
+    meta = await cm.read_collection_metadata(mocker.AsyncMock(), "col", settings)
+    assert meta["embedding_identity"] == "amazon.titan-embed-text-v2:0"
+
+
+async def test_upsert_sentinel_builds_point(mocker):
+    client = mocker.AsyncMock()
+    await cm.upsert_sentinel(
+        client,
+        "col",
+        embedding_identity="mistral-embed",
+        chunking_config={"chunk_size": 2048, "chunk_overlap": 200},
+        dimension=4,
+    )
+    client.upsert.assert_awaited_once()
+    kwargs = client.upsert.await_args.kwargs
+    assert kwargs["collection_name"] == "col"
+    point = kwargs["points"][0]
+    assert str(point.id) == cm.SENTINEL_POINT_ID
+    # Non-zero dense (cosine-safe), empty sparse.
+    assert point.vector["dense"][0] != 0.0
+    assert len(point.vector["dense"]) == 4
+    assert point.payload[EMBEDDING_IDENTITY] == "mistral-embed"
+    assert point.payload[cm.IS_SENTINEL] is True

tests/unit/vector/test_nats_producer.py

@@ -0,0 +1,136 @@
+"""NATS ingest producer: DocumentTask → IngestMessage + dedup header (§3.4)."""
+
+import hashlib
+import json
+from pathlib import Path
+
+import pytest
+
+from nextcloud_mcp_server.canonical import canonical_json
+from nextcloud_mcp_server.vector.queue.factory import _transport_for
+from nextcloud_mcp_server.vector.queue.nats import (
+    NatsTaskProducer,
+    _modified_at_rfc3339,
+    msg_id,
+)
+from nextcloud_mcp_server.vector.queue.postgres import PostgresTaskProducer
+from nextcloud_mcp_server.vector.scanner import DocumentTask
+
+FIXTURE = Path(__file__).parents[2] / "fixtures" / "ingest_message_example.json"
+TENANT = "00000000-0000-0000-0000-000000000001"
+
+
+def _producer(mocker, tenant_id=TENANT):
+    return NatsTaskProducer(
+        nc=mocker.MagicMock(), js=mocker.AsyncMock(), tenant_id=tenant_id
+    )
+
+
+def test_ingest_message_translation(mocker):
+    p = _producer(mocker)
+    task = DocumentTask(
+        user_id="alice",
+        doc_id="12345",
+        doc_type="file",
+        operation="index",
+        modified_at=1700000000,
+        file_path="/Documents/report.pdf",
+        etag="etag-abc123",
+    )
+    msg = p.ingest_message(task)
+    assert msg["tenant_id"] == TENANT  # from settings, not the task
+    assert msg["content_hash"] == "etag-abc123"  # etag wins
+    assert msg["user_id"] == "alice"
+    assert msg["doc_type"] == "file"
+    assert msg["operation"] == "index"
+    assert msg["file_path"] == "/Documents/report.pdf"
+
+
+def test_content_hash_falls_back_to_modified_at(mocker):
+    p = _producer(mocker)
+    task = DocumentTask(
+        user_id="u", doc_id="d", doc_type="note", operation="delete", modified_at=0
+    )
+    assert p.ingest_message(task)["content_hash"] == "0"
+
+
+async def test_send_publishes_with_dedup_header(mocker):
+    p = _producer(mocker)
+    task = DocumentTask(
+        user_id="alice",
+        doc_id="12345",
+        doc_type="file",
+        operation="index",
+        modified_at=1700000000,
+        etag="e",
+    )
+    await p.send(task)
+    p._js.publish.assert_awaited_once()
+    args = p._js.publish.await_args.args
+    kwargs = p._js.publish.await_args.kwargs
+    assert args[0] == f"mcp.ingest.requested.{TENANT}"
+    expected_mid = msg_id(TENANT, "12345", _modified_at_rfc3339(1700000000))
+    assert kwargs["headers"]["Nats-Msg-Id"] == expected_mid
+    assert json.loads(args[1])["doc_id"] == "12345"
+
+
+def test_msg_id_known_vector():
+    mid = msg_id("t", "d", "2026-01-01T00:00:00+00:00")
+    expected = hashlib.sha256(
+        canonical_json(
+            {
+                "tenant_id": "t",
+                "doc_id": "d",
+                "modified_at": "2026-01-01T00:00:00+00:00",
+            }
+        )
+    ).hexdigest()
+    assert mid == expected
+
+
+def test_publisher_matches_shared_fixture(mocker):
+    # The same fixture is validated as an IngestMessage in the processor repo.
+    # Here we assert the publisher emits exactly the fixture's key set + stable
+    # field values (modified_at format is allowed to differ — epoch→ISO).
+    fixture = json.loads(FIXTURE.read_text(encoding="utf-8"))
+    p = _producer(mocker, tenant_id=fixture["tenant_id"])
+    task = DocumentTask(
+        user_id=fixture["user_id"],
+        doc_id=fixture["doc_id"],
+        doc_type=fixture["doc_type"],
+        operation=fixture["operation"],
+        modified_at=1764201600,
+        file_path=fixture["file_path"],
+        etag=fixture["content_hash"],
+    )
+    msg = p.ingest_message(task)
+    assert set(msg.keys()) == set(fixture.keys())
+    for key in (
+        "tenant_id",
+        "doc_id",
+        "content_hash",
+        "doc_type",
+        "operation",
+        "user_id",
+        "file_path",
+    ):
+        assert msg[key] == fixture[key]
+    assert msg["modified_at"]  # non-empty ISO timestamp
+
+
+@pytest.mark.parametrize(
+    "url,expected",
+    [
+        ("nats://nats:4222", "nats"),
+        ("postgres://h/db", "postgres"),
+        ("postgresql://h/db", "postgres"),
+        ("http://elsewhere", "nats"),
+    ],
+)
+def test_transport_for(url, expected):
+    assert _transport_for(url) == expected
+
+
+async def test_postgres_producer_is_a_seam():
+    with pytest.raises(NotImplementedError, match="documented seam"):
+        await PostgresTaskProducer.connect(object())

tests/unit/vector/test_payload_keys.py

@@ -0,0 +1,44 @@
+"""Drift guard for the cross-implementation point-ID namespace (design §2.2).
+
+The MCP server and the external document-processor must compute identical chunk
+point IDs. This test pins the NAMESPACE against a fixture checked into both
+repos; the processor repo runs a mirror of this test against the same fixture.
+"""
+
+import uuid
+from pathlib import Path
+
+from nextcloud_mcp_server.vector import payload_keys
+
+FIXTURE = Path(__file__).parents[2] / "fixtures" / "namespace_uuid.txt"
+
+
+def test_namespace_matches_fixture():
+    assert str(payload_keys.NAMESPACE) == FIXTURE.read_text().strip()
+
+
+def test_point_id_deterministic():
+    a = payload_keys.point_id("t", "d", 0)
+    assert a == payload_keys.point_id("t", "d", 0)
+    assert a != payload_keys.point_id("t", "d", 1)
+    assert a != payload_keys.point_id("t2", "d", 0)
+
+
+def test_point_id_exact_value():
+    # Pins the canonical-JSON name + uuid5 formula so a refactor that changes
+    # key ordering or encoding is caught (would silently break idempotency).
+    expected = str(
+        uuid.uuid5(
+            payload_keys.NAMESPACE,
+            '{"chunk_index":0,"doc_id":"d","tenant_id":"t"}',
+        )
+    )
+    assert payload_keys.point_id("t", "d", 0) == expected
+
+
+def test_payload_key_constants():
+    assert payload_keys.EMBEDDING_IDENTITY == "embedding_identity"
+    assert payload_keys.ACL_HASH == "acl_hash"
+    assert payload_keys.PROCESSOR_VERSION == "processor_version"
+    assert payload_keys.PARSED_AT == "parsed_at"
+    assert payload_keys.PIPELINE_TIER == "pipeline_tier"

tests/unit/vector/test_status_store.py

@@ -0,0 +1,70 @@
+"""StatusStore + NATS status message handling (design §10.1, STATUS_BACKEND=bus)."""
+
+import json
+
+from nextcloud_mcp_server.vector.queue.status import (
+    NatsStatusSubscriber,
+    StatusStore,
+    state_from_subject,
+)
+
+
+def test_store_records_and_counts():
+    store = StatusStore()
+    store.record("d1", "ready", content_hash="h1")
+    store.record("d2", "failed")
+    store.record("d1", "ready", content_hash="h1")  # idempotent overwrite
+    assert len(store) == 2
+    assert store.counts() == {"ready": 1, "failed": 1}
+    assert store.get("d1")["content_hash"] == "h1"
+
+
+def test_store_is_bounded_lru():
+    store = StatusStore(max_size=2)
+    store.record("d1", "ready")
+    store.record("d2", "ready")
+    store.record("d3", "ready")  # evicts d1
+    assert len(store) == 2
+    assert store.get("d1") is None
+    assert store.get("d3") is not None
+
+
+def test_state_from_subject():
+    assert state_from_subject("mcp.document.ready.tenant-1") == "ready"
+    assert state_from_subject("mcp.document.failed.tenant-1") == "failed"
+    assert state_from_subject("mcp.document.reparsed.tenant-1") == "reparsed"
+    assert state_from_subject("mcp.document.bogus.tenant-1") is None
+    assert state_from_subject("mcp.ingest.requested.tenant-1") is None
+
+
+def test_handle_message_records_state():
+    store = StatusStore()
+    events = []
+    sub = NatsStatusSubscriber(
+        nc=None,
+        js=None,
+        tenant_id="t1",
+        store=store,
+        on_event=lambda d, s: events.append((d, s)),
+    )
+    payload = json.dumps(
+        {
+            "tenant_id": "t1",
+            "doc_id": "doc-9",
+            "content_hash": "abc",
+            "transitioned_at": "2026-05-27T00:00:00Z",
+        }
+    ).encode()
+    sub.handle_message("mcp.document.ready.t1", payload)
+    entry = store.get("doc-9")
+    assert entry["state"] == "ready"
+    assert entry["content_hash"] == "abc"
+    assert events == [("doc-9", "ready")]
+
+
+def test_handle_message_ignores_bad_payload_and_subject():
+    store = StatusStore()
+    sub = NatsStatusSubscriber(nc=None, js=None, tenant_id="t1", store=store)
+    sub.handle_message("mcp.document.ready.t1", b"not json")
+    sub.handle_message("mcp.ingest.requested.t1", b'{"doc_id":"x"}')
+    assert len(store) == 0