feat: add opt-in MCP decomposition hook points (design §10)
Adds the seven §10.2 hook-point modules + five env vars so Astrolabe Cloud can offload document processing to the external document-processor / embedding gateway. Purely additive: with every setting unset the server behaves exactly as today, so self-hosters are unaffected (Deck #92). Hook points (all default to current monolith behavior): - config: EMBEDDING_PROVIDER, INGEST_MODE, STATUS_BACKEND, COLLECTION_METADATA_SOURCE, FACT_EVENT_EMITTER (+ supporting settings), validated in Settings.__post_init__ (fail-fast STATUS_BACKEND=local with INGEST_MODE=external); shared canonical.py. - vector/payload_keys.py + acl_hash.py: cross-impl NAMESPACE/point_id (§2.2) and BLAKE2b-128 ACL hash (§11), pinned by fixtures shared with the document-processor repo. - embedding/gateway_client.py: OpenAI-compatible GatewayProvider authenticating via M2M OIDC client-credentials (separate realm); manual-only registry entry. - vector/collection_metadata.py: sentinel-point / API metadata source with env fallback. - vector/queue/: hexagonal ingest producer ports + memory/NATS adapters (Postgres seam); INGEST_MODE=external publishes mcp.ingest.requested.{tenant} instead of the in-memory stream and skips the in-process processor pool. The lifespan becomes a composition root across both deployment branches. - vector/queue/status.py: STATUS_BACKEND=bus subscriber feeding a StatusStore the vector-sync status endpoint reads. - admin/payload_backfill.py: POST /api/v1/admin/payload-backfill (admin scope); processor writes the new payload keys; query-side ACL pre-filter gated behind ACL_PREFILTER_ENABLED (default off). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
c7da612f20
commit
d883052fb8
Vendored
+285
@@ -0,0 +1,285 @@
|
||||
{
|
||||
"_comment": "Cross-impl ACL hash corpus (design §11.5). Pinned BLAKE2b-128 values; both repos must reproduce. Do not edit by hand.",
|
||||
"cases": [
|
||||
{
|
||||
"name": "single_user_ascii",
|
||||
"share_set": [
|
||||
[
|
||||
"user",
|
||||
"alice"
|
||||
]
|
||||
],
|
||||
"expected": [
|
||||
"c63a5fa2d6df5c20ee3700dd85e39e1d"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "single_group",
|
||||
"share_set": [
|
||||
[
|
||||
"group",
|
||||
"admins"
|
||||
]
|
||||
],
|
||||
"expected": [
|
||||
"fe57bf2fd0682f713ff967073808b2c1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "public_only",
|
||||
"share_set": [
|
||||
[
|
||||
"public",
|
||||
"public"
|
||||
]
|
||||
],
|
||||
"expected": [
|
||||
"97990046391027e6ec24575a3ebdd136"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "user_group_public",
|
||||
"share_set": [
|
||||
[
|
||||
"user",
|
||||
"alice"
|
||||
],
|
||||
[
|
||||
"group",
|
||||
"admins"
|
||||
],
|
||||
[
|
||||
"public",
|
||||
"public"
|
||||
]
|
||||
],
|
||||
"expected": [
|
||||
"c63a5fa2d6df5c20ee3700dd85e39e1d",
|
||||
"fe57bf2fd0682f713ff967073808b2c1",
|
||||
"97990046391027e6ec24575a3ebdd136"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "user_uppercase_distinct",
|
||||
"share_set": [
|
||||
[
|
||||
"user",
|
||||
"Alice"
|
||||
]
|
||||
],
|
||||
"expected": [
|
||||
"245e3e55068fbb623b397a8ca21b0126"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "unicode_precomposed",
|
||||
"share_set": [
|
||||
[
|
||||
"user",
|
||||
"café"
|
||||
]
|
||||
],
|
||||
"expected": [
|
||||
"053451c238fa2c6118b95b2ab2d7964a"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "unicode_decomposed_same_as_precomposed",
|
||||
"share_set": [
|
||||
[
|
||||
"user",
|
||||
"café"
|
||||
]
|
||||
],
|
||||
"expected": [
|
||||
"053451c238fa2c6118b95b2ab2d7964a"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "pipe_in_username",
|
||||
"share_set": [
|
||||
[
|
||||
"user",
|
||||
"a|b"
|
||||
]
|
||||
],
|
||||
"expected": [
|
||||
"a5f7e378a36ed9dd4e8e02007bca3624"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "colon_in_username",
|
||||
"share_set": [
|
||||
[
|
||||
"user",
|
||||
"a:b"
|
||||
]
|
||||
],
|
||||
"expected": [
|
||||
"7049ec93c0bf28a5f02a8527434dd0a0"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "newline_in_username",
|
||||
"share_set": [
|
||||
[
|
||||
"user",
|
||||
"a\nb"
|
||||
]
|
||||
],
|
||||
"expected": [
|
||||
"8e62aee620abaf9852e471d603cbe88b"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "backslash_in_username",
|
||||
"share_set": [
|
||||
[
|
||||
"user",
|
||||
"a\\b"
|
||||
]
|
||||
],
|
||||
"expected": [
|
||||
"c16a30d33ef2f834ebb642496206467e"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "quote_in_username",
|
||||
"share_set": [
|
||||
[
|
||||
"user",
|
||||
"a\"b"
|
||||
]
|
||||
],
|
||||
"expected": [
|
||||
"61405bcb3db7afd0fbf13693d2157bbb"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "space_in_group",
|
||||
"share_set": [
|
||||
[
|
||||
"group",
|
||||
"Dept 7"
|
||||
]
|
||||
],
|
||||
"expected": [
|
||||
"e80d92f6227d1ae64a8c8231491335d8"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "emoji_username",
|
||||
"share_set": [
|
||||
[
|
||||
"user",
|
||||
"🦏fox"
|
||||
]
|
||||
],
|
||||
"expected": [
|
||||
"52156572976858eec6361800b2f5ad03"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "cjk_group",
|
||||
"share_set": [
|
||||
[
|
||||
"group",
|
||||
"管理员"
|
||||
]
|
||||
],
|
||||
"expected": [
|
||||
"a3e5a6e8b405d5c749a031595309491c"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "email_style_user",
|
||||
"share_set": [
|
||||
[
|
||||
"user",
|
||||
"jane.doe@example.com"
|
||||
]
|
||||
],
|
||||
"expected": [
|
||||
"3216fecd6203036e6c23ff9c055031d9"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "numeric_group",
|
||||
"share_set": [
|
||||
[
|
||||
"group",
|
||||
"12345"
|
||||
]
|
||||
],
|
||||
"expected": [
|
||||
"1ae023f071c8d70bacdb388230fc2cfb"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "multi_group_large",
|
||||
"share_set": [
|
||||
[
|
||||
"user",
|
||||
"bob"
|
||||
],
|
||||
[
|
||||
"group",
|
||||
"g1"
|
||||
],
|
||||
[
|
||||
"group",
|
||||
"g2"
|
||||
],
|
||||
[
|
||||
"group",
|
||||
"g3"
|
||||
],
|
||||
[
|
||||
"group",
|
||||
"g4"
|
||||
],
|
||||
[
|
||||
"group",
|
||||
"g5"
|
||||
]
|
||||
],
|
||||
"expected": [
|
||||
"830bb4f705bc0aa13929061db587b696",
|
||||
"4eac98d1b59fec00e15e4fe28a32a3f9",
|
||||
"2c76357c3f95284911f6be1a30d4850e",
|
||||
"898271cc3d99563bf8dd396ef6c0f0b2",
|
||||
"4704c80af47f56c566cf5b83f506e704",
|
||||
"80bcc464007a51518f87876758c6ccf5"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "group_and_public",
|
||||
"share_set": [
|
||||
[
|
||||
"group",
|
||||
"staff"
|
||||
],
|
||||
[
|
||||
"public",
|
||||
"public"
|
||||
]
|
||||
],
|
||||
"expected": [
|
||||
"b2d6c7e2a6bc94f2b0310492ac6316fc",
|
||||
"97990046391027e6ec24575a3ebdd136"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "trailing_space_user_distinct",
|
||||
"share_set": [
|
||||
[
|
||||
"user",
|
||||
"alice "
|
||||
]
|
||||
],
|
||||
"expected": [
|
||||
"d7f07f4624e3e06b7f0b77f483e9941e"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"tenant_id": "00000000-0000-0000-0000-000000000001",
|
||||
"doc_id": "12345",
|
||||
"content_hash": "etag-abc123",
|
||||
"modified_at": "2026-05-27T00:00:00Z",
|
||||
"doc_type": "file",
|
||||
"operation": "index",
|
||||
"user_id": "alice",
|
||||
"file_path": "/Documents/report.pdf"
|
||||
}
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
b050b8ac-c6aa-5566-9584-506b39c1c096
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Gateway provider registration + M2M OIDC auth (design §10.2).
|
||||
|
||||
The gateway is manual-only: selected by EMBEDDING_PROVIDER=gateway and never by
|
||||
the autodetect chain. Auth is the gateway's own M2M OIDC realm (parallel to the
|
||||
tenant realm); creds are all-or-nothing.
|
||||
"""
|
||||
|
||||
import time
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from nextcloud_mcp_server.config import Settings
|
||||
from nextcloud_mcp_server.embedding.gateway_client import (
|
||||
GatewayProvider,
|
||||
GatewayTokenProvider,
|
||||
)
|
||||
from nextcloud_mcp_server.providers.registry import ProviderRegistry, reset_provider
|
||||
from nextcloud_mcp_server.providers.simple import SimpleProvider
|
||||
|
||||
|
||||
def _patch_settings(monkeypatch, settings):
|
||||
monkeypatch.setattr(
|
||||
"nextcloud_mcp_server.providers.registry.get_settings", lambda: settings
|
||||
)
|
||||
reset_provider()
|
||||
|
||||
|
||||
def test_gateway_selected_unauthenticated(monkeypatch):
|
||||
settings = Settings(
|
||||
embedding_provider="gateway",
|
||||
embedding_gateway_url="http://gateway:8083",
|
||||
embedding_gateway_model="mistral-embed",
|
||||
)
|
||||
_patch_settings(monkeypatch, settings)
|
||||
provider = ProviderRegistry.create_provider()
|
||||
assert isinstance(provider, GatewayProvider)
|
||||
assert provider.embedding_model == "mistral-embed"
|
||||
assert provider.supports_embeddings is True
|
||||
assert provider.supports_generation is False
|
||||
assert provider._token_provider is None # unauthenticated
|
||||
|
||||
|
||||
def test_gateway_selected_with_m2m_oidc(monkeypatch):
|
||||
settings = Settings(
|
||||
embedding_provider="gateway",
|
||||
embedding_gateway_url="http://gateway:8083",
|
||||
embedding_gateway_token_url="https://idp.example/oauth2/token",
|
||||
embedding_gateway_client_id="mcp-server",
|
||||
embedding_gateway_client_secret="shh",
|
||||
embedding_gateway_scope="astrolabe-embedding-gateway/embed",
|
||||
)
|
||||
_patch_settings(monkeypatch, settings)
|
||||
provider = ProviderRegistry.create_provider()
|
||||
assert isinstance(provider, GatewayProvider)
|
||||
assert isinstance(provider._token_provider, GatewayTokenProvider)
|
||||
|
||||
|
||||
def test_partial_m2m_creds_rejected():
|
||||
with pytest.raises(ValueError, match="must be set together"):
|
||||
Settings(
|
||||
embedding_provider="gateway",
|
||||
embedding_gateway_url="http://gateway:8083",
|
||||
embedding_gateway_client_id="mcp-server", # missing token_url/secret
|
||||
)
|
||||
|
||||
|
||||
def test_autodetect_default_does_not_pick_gateway(monkeypatch):
|
||||
settings = Settings()
|
||||
_patch_settings(monkeypatch, settings)
|
||||
assert isinstance(ProviderRegistry.create_provider(), SimpleProvider)
|
||||
|
||||
|
||||
def test_openai_creds_do_not_trigger_gateway(monkeypatch):
|
||||
settings = Settings(openai_api_key="sk-test")
|
||||
_patch_settings(monkeypatch, settings)
|
||||
assert not isinstance(ProviderRegistry.create_provider(), GatewayProvider)
|
||||
|
||||
|
||||
async def test_token_provider_caches_and_refreshes(monkeypatch):
|
||||
calls = {"n": 0}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
calls["n"] += 1
|
||||
assert request.headers["Authorization"].startswith("Basic ")
|
||||
body = dict(httpx.QueryParams(request.content.decode()))
|
||||
assert body["grant_type"] == "client_credentials"
|
||||
assert body["scope"] == "embed"
|
||||
return httpx.Response(
|
||||
200, json={"access_token": f"tok{calls['n']}", "expires_in": 3600}
|
||||
)
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
orig_async_client = httpx.AsyncClient
|
||||
|
||||
def _client(*args, **kwargs):
|
||||
kwargs["transport"] = transport
|
||||
return orig_async_client(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(httpx, "AsyncClient", _client)
|
||||
|
||||
tp = GatewayTokenProvider(
|
||||
token_url="https://idp.example/oauth2/token",
|
||||
client_id="cid",
|
||||
client_secret="sec",
|
||||
scope="embed",
|
||||
)
|
||||
t1 = await tp.get_token()
|
||||
t2 = await tp.get_token() # cached → no new HTTP call
|
||||
assert t1 == t2 == "tok1"
|
||||
assert calls["n"] == 1
|
||||
|
||||
# Expire the cache → next call refreshes.
|
||||
tp._cache = (tp._cache[0], time.time() - 1)
|
||||
t3 = await tp.get_token()
|
||||
assert t3 == "tok2"
|
||||
assert calls["n"] == 2
|
||||
@@ -0,0 +1,71 @@
|
||||
"""ACL hash spec tests (design §11) + cross-impl corpus.
|
||||
|
||||
The corpus fixture is checked into both repos; the processor runs a mirror of
|
||||
``test_corpus`` against the same file. Divergence is caught in CI.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from nextcloud_mcp_server.acl_hash import (
|
||||
PUBLIC_PRINCIPAL,
|
||||
accessible_hash_set,
|
||||
compute_acl_hash,
|
||||
compute_principal_hash,
|
||||
)
|
||||
|
||||
CORPUS = json.loads(
|
||||
(Path(__file__).parent.parent / "fixtures" / "acl_hash_corpus.json").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case", CORPUS["cases"], ids=[c["name"] for c in CORPUS["cases"]]
|
||||
)
|
||||
def test_corpus(case):
|
||||
share_set = [tuple(p) for p in case["share_set"]]
|
||||
assert compute_acl_hash(share_set) == case["expected"]
|
||||
|
||||
|
||||
def test_blake2b_128_bit_length():
|
||||
assert len(compute_principal_hash("user", "alice")) == 32
|
||||
|
||||
|
||||
def test_case_is_preserved_not_folded():
|
||||
assert compute_principal_hash("user", "Alice") != compute_principal_hash(
|
||||
"user", "alice"
|
||||
)
|
||||
|
||||
|
||||
def test_nfc_normalization():
|
||||
# decomposed 'café' (e + U+0301) hashes the same as precomposed 'café'.
|
||||
assert compute_principal_hash("user", "café") == compute_principal_hash(
|
||||
"user", "café"
|
||||
)
|
||||
|
||||
|
||||
def test_invalid_principal_type_rejected():
|
||||
with pytest.raises(ValueError, match="principal_type must be one of"):
|
||||
compute_principal_hash("link", "sometoken")
|
||||
|
||||
|
||||
def test_accessible_set_always_includes_public():
|
||||
s = accessible_hash_set("alice", [])
|
||||
assert compute_principal_hash(*PUBLIC_PRINCIPAL) in s
|
||||
|
||||
|
||||
def test_accessible_set_membership_matches_share():
|
||||
# A document shared with a group the requester holds is admitted.
|
||||
doc = compute_acl_hash([("group", "engineering")])
|
||||
accessible = accessible_hash_set("bob", ["engineering", "all-staff"])
|
||||
assert any(h in accessible for h in doc)
|
||||
|
||||
|
||||
def test_accessible_set_excludes_unheld_group():
|
||||
doc = compute_acl_hash([("group", "finance")])
|
||||
accessible = accessible_hash_set("bob", ["engineering"])
|
||||
assert not any(h in accessible for h in doc)
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Tests for the MCP decomposition hook-point settings (design §10).
|
||||
|
||||
Every default must reproduce the monolith; the opt-in settings are validated
|
||||
in ``Settings.__post_init__``.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from nextcloud_mcp_server.canonical import canonical_json
|
||||
from nextcloud_mcp_server.config import Settings
|
||||
|
||||
|
||||
class TestDecompositionDefaults:
|
||||
"""With nothing set, behavior matches today's monolith."""
|
||||
|
||||
def test_defaults_are_monolith(self):
|
||||
s = Settings()
|
||||
assert s.embedding_provider == "autodetect"
|
||||
assert s.ingest_mode == "local"
|
||||
assert s.status_backend == "local"
|
||||
assert s.collection_metadata_source == "qdrant"
|
||||
assert s.fact_event_emitter == "none"
|
||||
assert s.ingest_bus_url is None
|
||||
assert s.embedding_gateway_url is None
|
||||
assert s.tenant_id is None
|
||||
assert s.ingest_bus_num_replicas == 1
|
||||
|
||||
def test_enum_values_normalized(self):
|
||||
# Mixed case / surrounding whitespace is normalized before validation.
|
||||
s = Settings(
|
||||
collection_metadata_source=" QDRANT ",
|
||||
fact_event_emitter="NONE",
|
||||
)
|
||||
assert s.collection_metadata_source == "qdrant"
|
||||
assert s.fact_event_emitter == "none"
|
||||
|
||||
|
||||
class TestEnumValidation:
|
||||
@pytest.mark.parametrize(
|
||||
"field,value",
|
||||
[
|
||||
("embedding_provider", "openai"),
|
||||
("ingest_mode", "remote"),
|
||||
("status_backend", "redis"),
|
||||
("collection_metadata_source", "postgres"),
|
||||
("fact_event_emitter", "kafka"),
|
||||
],
|
||||
)
|
||||
def test_invalid_enum_rejected(self, field, value):
|
||||
with pytest.raises(ValueError, match=field.upper()):
|
||||
Settings(**{field: value})
|
||||
|
||||
|
||||
class TestFailFast:
|
||||
def test_external_with_local_status_crashes(self):
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="STATUS_BACKEND=local is incompatible with INGEST_MODE=external",
|
||||
):
|
||||
Settings(
|
||||
ingest_mode="external",
|
||||
status_backend="local",
|
||||
ingest_bus_url="nats://nats:4222",
|
||||
tenant_id="tenant-uuid",
|
||||
)
|
||||
|
||||
|
||||
class TestConditionalRequired:
|
||||
def test_external_requires_bus_url(self):
|
||||
with pytest.raises(ValueError, match="INGEST_BUS_URL is required"):
|
||||
Settings(ingest_mode="external", status_backend="bus", tenant_id="t1")
|
||||
|
||||
def test_external_requires_tenant_id(self):
|
||||
with pytest.raises(ValueError, match="TENANT_ID is required"):
|
||||
Settings(
|
||||
ingest_mode="external",
|
||||
status_backend="bus",
|
||||
ingest_bus_url="nats://nats:4222",
|
||||
)
|
||||
|
||||
def test_gateway_requires_gateway_url(self):
|
||||
with pytest.raises(ValueError, match="EMBEDDING_GATEWAY_URL is required"):
|
||||
Settings(embedding_provider="gateway")
|
||||
|
||||
def test_external_happy_path(self):
|
||||
s = Settings(
|
||||
ingest_mode="external",
|
||||
status_backend="bus",
|
||||
ingest_bus_url="nats://nats:4222",
|
||||
tenant_id="0a1b2c3d-0000-0000-0000-000000000000",
|
||||
)
|
||||
assert s.ingest_mode == "external"
|
||||
assert s.status_backend == "bus"
|
||||
|
||||
def test_gateway_happy_path(self):
|
||||
s = Settings(
|
||||
embedding_provider="gateway",
|
||||
embedding_gateway_url="http://gateway:8083",
|
||||
)
|
||||
assert s.embedding_provider == "gateway"
|
||||
|
||||
|
||||
class TestTenantIdSubjectToken:
|
||||
@pytest.mark.parametrize(
|
||||
"tenant_id",
|
||||
["a.b", "a*b", "a>b", "a b", "a\tb"],
|
||||
)
|
||||
def test_illegal_subject_chars_rejected(self, tenant_id):
|
||||
with pytest.raises(ValueError, match="TENANT_ID must not contain"):
|
||||
Settings(tenant_id=tenant_id)
|
||||
|
||||
def test_uuid_form_accepted(self):
|
||||
s = Settings(tenant_id="0a1b2c3d-0000-0000-0000-000000000000")
|
||||
assert s.tenant_id == "0a1b2c3d-0000-0000-0000-000000000000"
|
||||
|
||||
|
||||
class TestReplicas:
|
||||
def test_zero_replicas_rejected(self):
|
||||
with pytest.raises(ValueError, match="INGEST_BUS_NUM_REPLICAS must be >= 1"):
|
||||
Settings(ingest_bus_num_replicas=0)
|
||||
|
||||
|
||||
class TestCanonicalJson:
|
||||
def test_sorted_keys_no_whitespace(self):
|
||||
assert canonical_json({"b": 1, "a": 2}) == b'{"a":2,"b":1}'
|
||||
|
||||
def test_non_ascii_preserved(self):
|
||||
# ensure_ascii=False keeps the literal UTF-8 bytes.
|
||||
assert canonical_json({"k": "café"}) == '{"k":"café"}'.encode("utf-8")
|
||||
|
||||
def test_stable_across_calls(self):
|
||||
obj = {"tenant_id": "t", "doc_id": "d", "modified_at": "2026-01-01T00:00:00Z"}
|
||||
assert canonical_json(obj) == canonical_json(dict(reversed(list(obj.items()))))
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Admin payload-backfill endpoint (design §10.2)."""
|
||||
|
||||
import json
|
||||
|
||||
from nextcloud_mcp_server.api.management import AdminScopeRequired
|
||||
from nextcloud_mcp_server.config import Settings
|
||||
|
||||
|
||||
def _request(mocker):
|
||||
return mocker.MagicMock()
|
||||
|
||||
|
||||
async def test_backfill_requires_admin_scope(mocker):
|
||||
from nextcloud_mcp_server.admin import payload_backfill as mod
|
||||
|
||||
mocker.patch.object(
|
||||
mod, "require_admin_scope", side_effect=AdminScopeRequired("nope")
|
||||
)
|
||||
resp = await mod.handle_payload_backfill(_request(mocker))
|
||||
assert resp.status_code == 403
|
||||
|
||||
|
||||
async def test_backfill_unauthorized_on_auth_error(mocker):
|
||||
from nextcloud_mcp_server.admin import payload_backfill as mod
|
||||
|
||||
mocker.patch.object(mod, "require_admin_scope", side_effect=ValueError("no token"))
|
||||
resp = await mod.handle_payload_backfill(_request(mocker))
|
||||
assert resp.status_code == 401
|
||||
|
||||
|
||||
async def test_backfill_happy_path_sets_keys_and_sentinel(mocker):
|
||||
from nextcloud_mcp_server.admin import payload_backfill as mod
|
||||
|
||||
mocker.patch.object(mod, "require_admin_scope", return_value="admin")
|
||||
mocker.patch.object(
|
||||
mod, "get_settings", return_value=Settings(vector_sync_enabled=True)
|
||||
)
|
||||
qdrant = mocker.AsyncMock()
|
||||
mocker.patch.object(mod, "get_qdrant_client", return_value=qdrant)
|
||||
embed = mocker.MagicMock()
|
||||
embed.get_dimension.return_value = 4
|
||||
mocker.patch(
|
||||
"nextcloud_mcp_server.embedding.get_embedding_service", return_value=embed
|
||||
)
|
||||
# upsert_sentinel uses the same qdrant client; it is an AsyncMock so .upsert
|
||||
# is awaitable. Patch upsert_sentinel to assert it was invoked.
|
||||
sentinel = mocker.patch.object(mod, "upsert_sentinel", new=mocker.AsyncMock())
|
||||
|
||||
resp = await mod.handle_payload_backfill(_request(mocker))
|
||||
assert resp.status_code == 200
|
||||
body = json.loads(resp.body)
|
||||
assert body["status"] == "ok"
|
||||
# processor_version, pipeline_tier, embedding_identity → 3 set_payload calls.
|
||||
assert qdrant.set_payload.await_count == 3
|
||||
sentinel.assert_awaited_once()
|
||||
assert body["sentinel_upserted"] is True
|
||||
@@ -42,7 +42,10 @@ def _make_app(send_stream=None) -> Starlette:
|
||||
Route("/webhooks/nextcloud", handle_nextcloud_webhook, methods=["POST"])
|
||||
]
|
||||
)
|
||||
# The webhook reads app.state.task_producer; a raw MemoryObjectSendStream
|
||||
# satisfies the TaskProducer.send contract directly.
|
||||
app.state.document_send_stream = send_stream
|
||||
app.state.task_producer = send_stream
|
||||
return app
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Collection metadata source + sentinel (design §10.1)."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import httpx
|
||||
|
||||
from nextcloud_mcp_server.config import Settings
|
||||
from nextcloud_mcp_server.vector import collection_metadata as cm
|
||||
from nextcloud_mcp_server.vector.payload_keys import EMBEDDING_IDENTITY
|
||||
|
||||
|
||||
async def test_qdrant_read_hit(mocker):
|
||||
client = mocker.AsyncMock()
|
||||
client.retrieve.return_value = [
|
||||
SimpleNamespace(
|
||||
payload={
|
||||
EMBEDDING_IDENTITY: "mistral-embed",
|
||||
cm.CHUNKING_CONFIG: {"chunk_size": 1024, "chunk_overlap": 100},
|
||||
cm.IS_SENTINEL: True,
|
||||
}
|
||||
)
|
||||
]
|
||||
meta = await cm.read_collection_metadata(client, "col", Settings())
|
||||
assert meta["embedding_identity"] == "mistral-embed"
|
||||
assert meta["chunking_config"]["chunk_size"] == 1024
|
||||
|
||||
|
||||
async def test_qdrant_miss_falls_back_to_env(mocker):
|
||||
client = mocker.AsyncMock()
|
||||
client.retrieve.return_value = [] # no sentinel
|
||||
settings = Settings(document_chunk_size=2048, document_chunk_overlap=200)
|
||||
meta = await cm.read_collection_metadata(client, "col", settings)
|
||||
assert meta == cm.env_default_metadata(settings)
|
||||
assert meta["chunking_config"]["chunk_size"] == 2048
|
||||
|
||||
|
||||
async def test_qdrant_error_falls_back_to_env(mocker):
|
||||
client = mocker.AsyncMock()
|
||||
client.retrieve.side_effect = RuntimeError("qdrant down")
|
||||
settings = Settings()
|
||||
meta = await cm.read_collection_metadata(client, "col", settings)
|
||||
assert meta == cm.env_default_metadata(settings)
|
||||
|
||||
|
||||
async def test_api_source(mocker):
|
||||
settings = Settings(
|
||||
collection_metadata_source="api",
|
||||
collection_metadata_api_url="http://cp",
|
||||
)
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
assert request.url.path == "/v1/qdrant-collections/col/metadata"
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"embedding_identity": "amazon.titan-embed-text-v2:0",
|
||||
"chunking_config": {"chunk_size": 512, "chunk_overlap": 50},
|
||||
},
|
||||
)
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
orig = httpx.AsyncClient
|
||||
mocker.patch.object(
|
||||
httpx,
|
||||
"AsyncClient",
|
||||
lambda *a, **k: orig(*a, **{**k, "transport": transport}),
|
||||
)
|
||||
|
||||
meta = await cm.read_collection_metadata(mocker.AsyncMock(), "col", settings)
|
||||
assert meta["embedding_identity"] == "amazon.titan-embed-text-v2:0"
|
||||
|
||||
|
||||
async def test_upsert_sentinel_builds_point(mocker):
|
||||
client = mocker.AsyncMock()
|
||||
await cm.upsert_sentinel(
|
||||
client,
|
||||
"col",
|
||||
embedding_identity="mistral-embed",
|
||||
chunking_config={"chunk_size": 2048, "chunk_overlap": 200},
|
||||
dimension=4,
|
||||
)
|
||||
client.upsert.assert_awaited_once()
|
||||
kwargs = client.upsert.await_args.kwargs
|
||||
assert kwargs["collection_name"] == "col"
|
||||
point = kwargs["points"][0]
|
||||
assert str(point.id) == cm.SENTINEL_POINT_ID
|
||||
# Non-zero dense (cosine-safe), empty sparse.
|
||||
assert point.vector["dense"][0] != 0.0
|
||||
assert len(point.vector["dense"]) == 4
|
||||
assert point.payload[EMBEDDING_IDENTITY] == "mistral-embed"
|
||||
assert point.payload[cm.IS_SENTINEL] is True
|
||||
@@ -0,0 +1,136 @@
|
||||
"""NATS ingest producer: DocumentTask → IngestMessage + dedup header (§3.4)."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from nextcloud_mcp_server.canonical import canonical_json
|
||||
from nextcloud_mcp_server.vector.queue.factory import _transport_for
|
||||
from nextcloud_mcp_server.vector.queue.nats import (
|
||||
NatsTaskProducer,
|
||||
_modified_at_rfc3339,
|
||||
msg_id,
|
||||
)
|
||||
from nextcloud_mcp_server.vector.queue.postgres import PostgresTaskProducer
|
||||
from nextcloud_mcp_server.vector.scanner import DocumentTask
|
||||
|
||||
FIXTURE = Path(__file__).parents[2] / "fixtures" / "ingest_message_example.json"
|
||||
TENANT = "00000000-0000-0000-0000-000000000001"
|
||||
|
||||
|
||||
def _producer(mocker, tenant_id=TENANT):
|
||||
return NatsTaskProducer(
|
||||
nc=mocker.MagicMock(), js=mocker.AsyncMock(), tenant_id=tenant_id
|
||||
)
|
||||
|
||||
|
||||
def test_ingest_message_translation(mocker):
|
||||
p = _producer(mocker)
|
||||
task = DocumentTask(
|
||||
user_id="alice",
|
||||
doc_id="12345",
|
||||
doc_type="file",
|
||||
operation="index",
|
||||
modified_at=1700000000,
|
||||
file_path="/Documents/report.pdf",
|
||||
etag="etag-abc123",
|
||||
)
|
||||
msg = p.ingest_message(task)
|
||||
assert msg["tenant_id"] == TENANT # from settings, not the task
|
||||
assert msg["content_hash"] == "etag-abc123" # etag wins
|
||||
assert msg["user_id"] == "alice"
|
||||
assert msg["doc_type"] == "file"
|
||||
assert msg["operation"] == "index"
|
||||
assert msg["file_path"] == "/Documents/report.pdf"
|
||||
|
||||
|
||||
def test_content_hash_falls_back_to_modified_at(mocker):
|
||||
p = _producer(mocker)
|
||||
task = DocumentTask(
|
||||
user_id="u", doc_id="d", doc_type="note", operation="delete", modified_at=0
|
||||
)
|
||||
assert p.ingest_message(task)["content_hash"] == "0"
|
||||
|
||||
|
||||
async def test_send_publishes_with_dedup_header(mocker):
|
||||
p = _producer(mocker)
|
||||
task = DocumentTask(
|
||||
user_id="alice",
|
||||
doc_id="12345",
|
||||
doc_type="file",
|
||||
operation="index",
|
||||
modified_at=1700000000,
|
||||
etag="e",
|
||||
)
|
||||
await p.send(task)
|
||||
p._js.publish.assert_awaited_once()
|
||||
args = p._js.publish.await_args.args
|
||||
kwargs = p._js.publish.await_args.kwargs
|
||||
assert args[0] == f"mcp.ingest.requested.{TENANT}"
|
||||
expected_mid = msg_id(TENANT, "12345", _modified_at_rfc3339(1700000000))
|
||||
assert kwargs["headers"]["Nats-Msg-Id"] == expected_mid
|
||||
assert json.loads(args[1])["doc_id"] == "12345"
|
||||
|
||||
|
||||
def test_msg_id_known_vector():
|
||||
mid = msg_id("t", "d", "2026-01-01T00:00:00+00:00")
|
||||
expected = hashlib.sha256(
|
||||
canonical_json(
|
||||
{
|
||||
"tenant_id": "t",
|
||||
"doc_id": "d",
|
||||
"modified_at": "2026-01-01T00:00:00+00:00",
|
||||
}
|
||||
)
|
||||
).hexdigest()
|
||||
assert mid == expected
|
||||
|
||||
|
||||
def test_publisher_matches_shared_fixture(mocker):
|
||||
# The same fixture is validated as an IngestMessage in the processor repo.
|
||||
# Here we assert the publisher emits exactly the fixture's key set + stable
|
||||
# field values (modified_at format is allowed to differ — epoch→ISO).
|
||||
fixture = json.loads(FIXTURE.read_text(encoding="utf-8"))
|
||||
p = _producer(mocker, tenant_id=fixture["tenant_id"])
|
||||
task = DocumentTask(
|
||||
user_id=fixture["user_id"],
|
||||
doc_id=fixture["doc_id"],
|
||||
doc_type=fixture["doc_type"],
|
||||
operation=fixture["operation"],
|
||||
modified_at=1764201600,
|
||||
file_path=fixture["file_path"],
|
||||
etag=fixture["content_hash"],
|
||||
)
|
||||
msg = p.ingest_message(task)
|
||||
assert set(msg.keys()) == set(fixture.keys())
|
||||
for key in (
|
||||
"tenant_id",
|
||||
"doc_id",
|
||||
"content_hash",
|
||||
"doc_type",
|
||||
"operation",
|
||||
"user_id",
|
||||
"file_path",
|
||||
):
|
||||
assert msg[key] == fixture[key]
|
||||
assert msg["modified_at"] # non-empty ISO timestamp
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"url,expected",
|
||||
[
|
||||
("nats://nats:4222", "nats"),
|
||||
("postgres://h/db", "postgres"),
|
||||
("postgresql://h/db", "postgres"),
|
||||
("http://elsewhere", "nats"),
|
||||
],
|
||||
)
|
||||
def test_transport_for(url, expected):
|
||||
assert _transport_for(url) == expected
|
||||
|
||||
|
||||
async def test_postgres_producer_is_a_seam():
|
||||
with pytest.raises(NotImplementedError, match="documented seam"):
|
||||
await PostgresTaskProducer.connect(object())
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Drift guard for the cross-implementation point-ID namespace (design §2.2).
|
||||
|
||||
The MCP server and the external document-processor must compute identical chunk
|
||||
point IDs. This test pins the NAMESPACE against a fixture checked into both
|
||||
repos; the processor repo runs a mirror of this test against the same fixture.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from nextcloud_mcp_server.vector import payload_keys
|
||||
|
||||
FIXTURE = Path(__file__).parents[2] / "fixtures" / "namespace_uuid.txt"
|
||||
|
||||
|
||||
def test_namespace_matches_fixture():
|
||||
assert str(payload_keys.NAMESPACE) == FIXTURE.read_text().strip()
|
||||
|
||||
|
||||
def test_point_id_deterministic():
|
||||
a = payload_keys.point_id("t", "d", 0)
|
||||
assert a == payload_keys.point_id("t", "d", 0)
|
||||
assert a != payload_keys.point_id("t", "d", 1)
|
||||
assert a != payload_keys.point_id("t2", "d", 0)
|
||||
|
||||
|
||||
def test_point_id_exact_value():
|
||||
# Pins the canonical-JSON name + uuid5 formula so a refactor that changes
|
||||
# key ordering or encoding is caught (would silently break idempotency).
|
||||
expected = str(
|
||||
uuid.uuid5(
|
||||
payload_keys.NAMESPACE,
|
||||
'{"chunk_index":0,"doc_id":"d","tenant_id":"t"}',
|
||||
)
|
||||
)
|
||||
assert payload_keys.point_id("t", "d", 0) == expected
|
||||
|
||||
|
||||
def test_payload_key_constants():
|
||||
assert payload_keys.EMBEDDING_IDENTITY == "embedding_identity"
|
||||
assert payload_keys.ACL_HASH == "acl_hash"
|
||||
assert payload_keys.PROCESSOR_VERSION == "processor_version"
|
||||
assert payload_keys.PARSED_AT == "parsed_at"
|
||||
assert payload_keys.PIPELINE_TIER == "pipeline_tier"
|
||||
@@ -0,0 +1,70 @@
|
||||
"""StatusStore + NATS status message handling (design §10.1, STATUS_BACKEND=bus)."""
|
||||
|
||||
import json
|
||||
|
||||
from nextcloud_mcp_server.vector.queue.status import (
|
||||
NatsStatusSubscriber,
|
||||
StatusStore,
|
||||
state_from_subject,
|
||||
)
|
||||
|
||||
|
||||
def test_store_records_and_counts():
|
||||
store = StatusStore()
|
||||
store.record("d1", "ready", content_hash="h1")
|
||||
store.record("d2", "failed")
|
||||
store.record("d1", "ready", content_hash="h1") # idempotent overwrite
|
||||
assert len(store) == 2
|
||||
assert store.counts() == {"ready": 1, "failed": 1}
|
||||
assert store.get("d1")["content_hash"] == "h1"
|
||||
|
||||
|
||||
def test_store_is_bounded_lru():
|
||||
store = StatusStore(max_size=2)
|
||||
store.record("d1", "ready")
|
||||
store.record("d2", "ready")
|
||||
store.record("d3", "ready") # evicts d1
|
||||
assert len(store) == 2
|
||||
assert store.get("d1") is None
|
||||
assert store.get("d3") is not None
|
||||
|
||||
|
||||
def test_state_from_subject():
|
||||
assert state_from_subject("mcp.document.ready.tenant-1") == "ready"
|
||||
assert state_from_subject("mcp.document.failed.tenant-1") == "failed"
|
||||
assert state_from_subject("mcp.document.reparsed.tenant-1") == "reparsed"
|
||||
assert state_from_subject("mcp.document.bogus.tenant-1") is None
|
||||
assert state_from_subject("mcp.ingest.requested.tenant-1") is None
|
||||
|
||||
|
||||
def test_handle_message_records_state():
|
||||
store = StatusStore()
|
||||
events = []
|
||||
sub = NatsStatusSubscriber(
|
||||
nc=None,
|
||||
js=None,
|
||||
tenant_id="t1",
|
||||
store=store,
|
||||
on_event=lambda d, s: events.append((d, s)),
|
||||
)
|
||||
payload = json.dumps(
|
||||
{
|
||||
"tenant_id": "t1",
|
||||
"doc_id": "doc-9",
|
||||
"content_hash": "abc",
|
||||
"transitioned_at": "2026-05-27T00:00:00Z",
|
||||
}
|
||||
).encode()
|
||||
sub.handle_message("mcp.document.ready.t1", payload)
|
||||
entry = store.get("doc-9")
|
||||
assert entry["state"] == "ready"
|
||||
assert entry["content_hash"] == "abc"
|
||||
assert events == [("doc-9", "ready")]
|
||||
|
||||
|
||||
def test_handle_message_ignores_bad_payload_and_subject():
|
||||
store = StatusStore()
|
||||
sub = NatsStatusSubscriber(nc=None, js=None, tenant_id="t1", store=store)
|
||||
sub.handle_message("mcp.document.ready.t1", b"not json")
|
||||
sub.handle_message("mcp.ingest.requested.t1", b'{"doc_id":"x"}')
|
||||
assert len(store) == 0
|
||||
Reference in New Issue
Block a user