feat: add opt-in MCP decomposition hook points (design §10)
Adds the seven §10.2 hook-point modules + five env vars so Astrolabe Cloud can offload document processing to the external document-processor / embedding gateway. Purely additive: with every setting unset the server behaves exactly as today, so self-hosters are unaffected (Deck #92). Hook points (all default to current monolith behavior): - config: EMBEDDING_PROVIDER, INGEST_MODE, STATUS_BACKEND, COLLECTION_METADATA_SOURCE, FACT_EVENT_EMITTER (+ supporting settings), validated in Settings.__post_init__ (fail-fast STATUS_BACKEND=local with INGEST_MODE=external); shared canonical.py. - vector/payload_keys.py + acl_hash.py: cross-impl NAMESPACE/point_id (§2.2) and BLAKE2b-128 ACL hash (§11), pinned by fixtures shared with the document-processor repo. - embedding/gateway_client.py: OpenAI-compatible GatewayProvider authenticating via M2M OIDC client-credentials (separate realm); manual-only registry entry. - vector/collection_metadata.py: sentinel-point / API metadata source with env fallback. - vector/queue/: hexagonal ingest producer ports + memory/NATS adapters (Postgres seam); INGEST_MODE=external publishes mcp.ingest.requested.{tenant} instead of the in-memory stream and skips the in-process processor pool. The lifespan becomes a composition root across both deployment branches. - vector/queue/status.py: STATUS_BACKEND=bus subscriber feeding a StatusStore the vector-sync status endpoint reads. - admin/payload_backfill.py: POST /api/v1/admin/payload-backfill (admin scope); processor writes the new payload keys; query-side ACL pre-filter gated behind ACL_PREFILTER_ENABLED (default off). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
c7da612f20
commit
d883052fb8
@@ -0,0 +1 @@
|
||||
"""Admin-only REST endpoints (``/api/v1/admin/*``)."""
|
||||
@@ -0,0 +1,104 @@
|
||||
"""One-shot payload backfill admin endpoint (design §10.2).
|
||||
|
||||
``POST /api/v1/admin/payload-backfill`` walks the collection and adds default
|
||||
values for any *missing* decomposition payload keys (so existing corpora gain
|
||||
them without a re-index), then upserts the collection-metadata sentinel.
|
||||
|
||||
Scope: this backfills the cheap, deployment-level scalar keys
|
||||
(``processor_version``, ``parsed_at``, ``pipeline_tier``, ``embedding_identity``)
|
||||
only. It deliberately does NOT synthesize ``acl_hash`` — a correct value needs
|
||||
per-document share enumeration (a separate job), and writing a placeholder
|
||||
``acl_hash`` would be unsafe to pre-filter on. The query-side ACL pre-filter
|
||||
therefore stays disabled until a real ACL backfill runs (see
|
||||
``ACL_PREFILTER_ENABLED``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from qdrant_client import models
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import JSONResponse
|
||||
|
||||
from nextcloud_mcp_server.api.management import (
|
||||
AdminScopeRequired,
|
||||
require_admin_scope,
|
||||
)
|
||||
from nextcloud_mcp_server.config import get_settings
|
||||
from nextcloud_mcp_server.embedding import get_embedding_service
|
||||
from nextcloud_mcp_server.vector import payload_keys
|
||||
from nextcloud_mcp_server.vector.collection_metadata import (
|
||||
env_default_metadata,
|
||||
upsert_sentinel,
|
||||
)
|
||||
from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def handle_payload_backfill(request: Request) -> JSONResponse:
|
||||
try:
|
||||
await require_admin_scope(request)
|
||||
except AdminScopeRequired:
|
||||
return JSONResponse({"error": "admin scope required"}, status_code=403)
|
||||
except Exception:
|
||||
return JSONResponse({"error": "unauthorized"}, status_code=401)
|
||||
|
||||
settings = get_settings()
|
||||
if not settings.vector_sync_enabled:
|
||||
return JSONResponse({"error": "vector sync disabled"}, status_code=404)
|
||||
|
||||
client = await get_qdrant_client()
|
||||
collection = settings.get_collection_name()
|
||||
meta = env_default_metadata(settings)
|
||||
|
||||
# Deployment-level scalar defaults (safe to set only where missing).
|
||||
defaults = {
|
||||
payload_keys.PROCESSOR_VERSION: "backfill",
|
||||
payload_keys.PIPELINE_TIER: "fast",
|
||||
payload_keys.EMBEDDING_IDENTITY: meta["embedding_identity"],
|
||||
}
|
||||
applied: dict[str, str] = {}
|
||||
for key, value in defaults.items():
|
||||
try:
|
||||
await client.set_payload(
|
||||
collection_name=collection,
|
||||
payload={key: value},
|
||||
# Only points missing this key (don't clobber existing values).
|
||||
points=models.Filter(
|
||||
must=[
|
||||
models.IsEmptyCondition(is_empty=models.PayloadField(key=key))
|
||||
]
|
||||
),
|
||||
wait=True,
|
||||
)
|
||||
applied[key] = "set-where-missing"
|
||||
except Exception as e:
|
||||
logger.warning("payload backfill failed for key %s: %s", key, e)
|
||||
applied[key] = f"error: {e}"
|
||||
|
||||
# Upsert the collection-metadata sentinel so query-path metadata reads work
|
||||
# for this collection even without a control plane.
|
||||
sentinel_ok = True
|
||||
try:
|
||||
dimension = get_embedding_service().get_dimension()
|
||||
await upsert_sentinel(
|
||||
client,
|
||||
collection,
|
||||
embedding_identity=meta["embedding_identity"],
|
||||
chunking_config=meta["chunking_config"],
|
||||
dimension=dimension,
|
||||
)
|
||||
except Exception as e:
|
||||
sentinel_ok = False
|
||||
logger.warning("sentinel upsert failed during backfill: %s", e)
|
||||
|
||||
return JSONResponse(
|
||||
{
|
||||
"status": "ok",
|
||||
"collection": collection,
|
||||
"keys_applied": applied,
|
||||
"sentinel_upserted": sentinel_ok,
|
||||
}
|
||||
)
|
||||
Reference in New Issue
Block a user