feat: add opt-in MCP decomposition hook points (design §10)
Adds the seven §10.2 hook-point modules + five env vars so Astrolabe Cloud can offload document processing to the external document-processor / embedding gateway. Purely additive: with every setting unset the server behaves exactly as today, so self-hosters are unaffected (Deck #92). Hook points (all default to current monolith behavior): - config: EMBEDDING_PROVIDER, INGEST_MODE, STATUS_BACKEND, COLLECTION_METADATA_SOURCE, FACT_EVENT_EMITTER (+ supporting settings), validated in Settings.__post_init__ (fail-fast STATUS_BACKEND=local with INGEST_MODE=external); shared canonical.py. - vector/payload_keys.py + acl_hash.py: cross-impl NAMESPACE/point_id (§2.2) and BLAKE2b-128 ACL hash (§11), pinned by fixtures shared with the document-processor repo. - embedding/gateway_client.py: OpenAI-compatible GatewayProvider authenticating via M2M OIDC client-credentials (separate realm); manual-only registry entry. - vector/collection_metadata.py: sentinel-point / API metadata source with env fallback. - vector/queue/: hexagonal ingest producer ports + memory/NATS adapters (Postgres seam); INGEST_MODE=external publishes mcp.ingest.requested.{tenant} instead of the in-memory stream and skips the in-process processor pool. The lifespan becomes a composition root across both deployment branches. - vector/queue/status.py: STATUS_BACKEND=bus subscriber feeding a StatusStore the vector-sync status endpoint reads. - admin/payload_backfill.py: POST /api/v1/admin/payload-backfill (admin scope); processor writes the new payload keys; query-side ACL pre-filter gated behind ACL_PREFILTER_ENABLED (default off). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
c7da612f20
commit
d883052fb8
@@ -13,7 +13,6 @@ from typing import cast
|
||||
|
||||
import anyio
|
||||
from anyio.abc import TaskStatus
|
||||
from anyio.streams.memory import MemoryObjectSendStream
|
||||
from qdrant_client import AsyncQdrantClient
|
||||
from qdrant_client.models import FieldCondition, Filter, MatchValue, Record
|
||||
|
||||
@@ -31,6 +30,7 @@ from nextcloud_mcp_server.vector.placeholder import (
|
||||
write_placeholder_point,
|
||||
)
|
||||
from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client
|
||||
from nextcloud_mcp_server.vector.queue.ports import TaskProducer
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -108,6 +108,11 @@ class DocumentTask:
|
||||
metadata: dict[str, int | str] | None = (
|
||||
None # Additional metadata (e.g., board_id/stack_id for deck_card)
|
||||
)
|
||||
# Change-detection token (Nextcloud etag). Used as the external ingest
|
||||
# content_hash when present; the bus producer falls back to modified_at
|
||||
# when it is None (deletes, or sources whose etag isn't threaded). Harmless
|
||||
# in local mode — the in-process processor reads its own etag.
|
||||
etag: str | None = None
|
||||
|
||||
|
||||
# Track documents potentially deleted (grace period before actual deletion)
|
||||
@@ -179,7 +184,7 @@ async def get_last_indexed_timestamp(user_id: str) -> int | None:
|
||||
|
||||
|
||||
async def scanner_task(
|
||||
send_stream: MemoryObjectSendStream[DocumentTask],
|
||||
send_stream: TaskProducer,
|
||||
shutdown_event: anyio.Event,
|
||||
wake_event: anyio.Event,
|
||||
nc_client: NextcloudClient,
|
||||
@@ -233,7 +238,7 @@ async def scanner_task(
|
||||
|
||||
async def scan_user_documents(
|
||||
user_id: str,
|
||||
send_stream: MemoryObjectSendStream[DocumentTask],
|
||||
send_stream: TaskProducer,
|
||||
nc_client: NextcloudClient,
|
||||
initial_sync: bool = False,
|
||||
):
|
||||
@@ -339,6 +344,7 @@ async def scan_user_documents(
|
||||
doc_type="note",
|
||||
operation="index",
|
||||
modified_at=modified_at,
|
||||
etag=note.get("etag", ""),
|
||||
)
|
||||
)
|
||||
queued += 1
|
||||
@@ -405,6 +411,7 @@ async def scan_user_documents(
|
||||
doc_type="note",
|
||||
operation="index",
|
||||
modified_at=modified_at,
|
||||
etag=note.get("etag", ""),
|
||||
)
|
||||
)
|
||||
queued += 1
|
||||
@@ -740,7 +747,7 @@ async def scan_user_documents(
|
||||
|
||||
async def scan_news_items(
|
||||
user_id: str,
|
||||
send_stream: MemoryObjectSendStream[DocumentTask],
|
||||
send_stream: TaskProducer,
|
||||
nc_client: NextcloudClient,
|
||||
initial_sync: bool,
|
||||
scan_id: int,
|
||||
@@ -928,7 +935,7 @@ async def scan_news_items(
|
||||
|
||||
async def scan_deck_cards(
|
||||
user_id: str,
|
||||
send_stream: MemoryObjectSendStream[DocumentTask],
|
||||
send_stream: TaskProducer,
|
||||
nc_client: NextcloudClient,
|
||||
initial_sync: bool,
|
||||
scan_id: int,
|
||||
|
||||
Reference in New Issue
Block a user