feat(vector): index files in real time on vector-index tag changes
Tagging an existing file/folder emits only OCP\SystemTag\MapperEvent — never a Node*Event — so tagged PDFs were previously only picked up by the hourly scanner. Subscribe to the tag event and reconcile membership so adding/removing the `vector-index` tag (re)indexes in near-real time. - webhook_presets: add OCP\SystemTag\MapperEvent to the files_sync preset (NC 32+, where MapperEvent gained getWebhookSerializable(); harmless on older servers — it just never fires). - webhook_parser: parse MapperEvent (objectType=files) into a path-less file "reconcile" task. The payload carries only a fileid + tagIds (no name/path), so assign and unassign both collapse to a reconcile. - processor._reconcile_tag_event: resolve the fileid against the user's current vector-index PDFs (find_files_by_tag). Present -> index with the resolved path/etag; absent -> flip to delete. Naturally handles "an unrelated tag changed" and a tagged folder's own fileid (no-op; the scanner still expands folders to descendants). - Unit tests for the parser branch and the reconcile. The matching admin-UI preset change ships separately in the astrolabe app repo. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
78f3f284a6
commit
1322e5aba0
@@ -165,6 +165,58 @@ async def processor_task(
|
||||
logger.info("Processor %s stopped", worker_id)
|
||||
|
||||
|
||||
async def _reconcile_tag_event(
|
||||
doc_task: DocumentTask, nc_client: NextcloudClient
|
||||
) -> None:
|
||||
"""Resolve a tag-webhook file task into a concrete index or delete.
|
||||
|
||||
A SystemTag ``MapperEvent`` only tells us a fileid's tags changed — not the
|
||||
path, nor whether our ``vector-index`` tag is (still) on it. Look up the
|
||||
user's current ``vector-index`` PDFs (the same call the scanner uses, which
|
||||
also expands tagged folders into their PDF descendants) and reconcile the
|
||||
task in place:
|
||||
|
||||
- fileid present -> index it; fill path/etag/mtime from the tag listing.
|
||||
- fileid absent -> it isn't a tagged PDF (anymore); flip ``operation`` to
|
||||
``delete`` so any existing points are released for this user.
|
||||
|
||||
A tagged *folder*'s own fileid won't appear in the file-level listing, so it
|
||||
resolves to a harmless no-op delete here; the hourly scanner still expands
|
||||
tagged folders into their descendants.
|
||||
"""
|
||||
tag_name = get_settings().vector_sync_pdf_tag
|
||||
tagged = await nc_client.find_files_by_tag(
|
||||
tag_name, mime_type_filter="application/pdf"
|
||||
)
|
||||
match = next(
|
||||
(f for f in tagged if str(f.get("id")) == str(doc_task.doc_id)),
|
||||
None,
|
||||
)
|
||||
|
||||
if match is None:
|
||||
doc_task.operation = "delete"
|
||||
logger.info(
|
||||
"Tag reconcile: file %s is not a %r PDF; releasing for %s",
|
||||
doc_task.doc_id,
|
||||
tag_name,
|
||||
doc_task.user_id,
|
||||
)
|
||||
return
|
||||
|
||||
doc_task.file_path = match["path"]
|
||||
if not doc_task.etag:
|
||||
doc_task.etag = match.get("etag")
|
||||
last_modified = match.get("last_modified_timestamp")
|
||||
if last_modified:
|
||||
doc_task.modified_at = int(last_modified)
|
||||
logger.info(
|
||||
"Tag reconcile: indexing %s (file %s) for %s",
|
||||
doc_task.file_path,
|
||||
doc_task.doc_id,
|
||||
doc_task.user_id,
|
||||
)
|
||||
|
||||
|
||||
async def process_document(
|
||||
doc_task: DocumentTask, nc_client: NextcloudClient, *, max_retries: int = 3
|
||||
):
|
||||
@@ -204,6 +256,18 @@ async def process_document(
|
||||
try:
|
||||
qdrant_client = await get_qdrant_client()
|
||||
|
||||
# Tag-webhook reconcile: a SystemTag MapperEvent enqueues a file task
|
||||
# carrying only a fileid (file_path is None — see
|
||||
# webhook_parser._parse_tag_event). Resolve the file's current
|
||||
# vector-index membership into a concrete index (path/etag filled) or
|
||||
# a delete before dispatching below.
|
||||
if (
|
||||
doc_task.doc_type == "file"
|
||||
and doc_task.operation == "index"
|
||||
and doc_task.file_path is None
|
||||
):
|
||||
await _reconcile_tag_event(doc_task, nc_client)
|
||||
|
||||
# Handle deletion
|
||||
if doc_task.operation == "delete":
|
||||
# Release this user rather than blind-delete: a file shared across
|
||||
|
||||
@@ -4,9 +4,10 @@ Maps Nextcloud webhook events to vector-sync DocumentTasks. The handler at
|
||||
``/webhooks/nextcloud`` calls :func:`extract_document_task` and forwards any
|
||||
non-None result to the same processor send-stream the scanner uses.
|
||||
|
||||
Currently scoped to file (note) events and Deck card events. Calendar /
|
||||
Tables events fall through to ``None`` for now; those parsers can be added
|
||||
in follow-up changes.
|
||||
Currently scoped to file (note) events, Deck card events, and SystemTag
|
||||
assign/unassign events (which drive ``vector-index`` (re)indexing of files).
|
||||
Calendar / Tables events fall through to ``None`` for now; those parsers can
|
||||
be added in follow-up changes.
|
||||
|
||||
See ADR-010 for the design and ``webhook-testing-findings.md`` for real
|
||||
captured payloads.
|
||||
@@ -28,6 +29,11 @@ _DECK_EVENT_CARD_UPDATED = "OCA\\Deck\\Event\\CardUpdatedEvent"
|
||||
_DECK_EVENT_CARD_DELETED = "OCA\\Deck\\Event\\CardDeletedEvent"
|
||||
_DECK_EVENT_BOARD_UPDATED = "OCA\\Deck\\Event\\BoardUpdatedEvent"
|
||||
|
||||
# System-tag assign/unassign. A single event class covers both directions; the
|
||||
# payload's ``eventType`` distinguishes them. Webhook-deliverable on NC 32+
|
||||
# (MapperEvent gained ``getWebhookSerializable()`` in 32.0.0).
|
||||
_SYSTEMTAG_EVENT_MAPPER = "OCP\\SystemTag\\MapperEvent"
|
||||
|
||||
_DECK_CARD_EVENTS = frozenset(
|
||||
{
|
||||
_DECK_EVENT_CARD_CREATED,
|
||||
@@ -67,6 +73,9 @@ def extract_document_task(payload: dict) -> DocumentTask | None:
|
||||
if event_class in _DECK_CARD_EVENTS or event_class == _DECK_EVENT_BOARD_UPDATED:
|
||||
return _parse_deck_event(event_class, event, user_id, time)
|
||||
|
||||
if event_class == _SYSTEMTAG_EVENT_MAPPER:
|
||||
return _parse_tag_event(event, user_id, time)
|
||||
|
||||
logger.debug("Ignoring webhook for unsupported event: %s", event_class)
|
||||
return None
|
||||
|
||||
@@ -104,6 +113,38 @@ def _parse_file_event(
|
||||
)
|
||||
|
||||
|
||||
def _parse_tag_event(event: dict, user_id: str, time: int) -> DocumentTask | None:
|
||||
"""Convert a SystemTag ``MapperEvent`` (assign/unassign) into a reconcile task.
|
||||
|
||||
The NC 32+ payload (``getWebhookSerializable``) carries only ``objectType``,
|
||||
``objectId`` (a fileid) and ``tagIds`` — not the tag *name*, the file path,
|
||||
or whether our ``vector-index`` tag specifically changed. So we can't decide
|
||||
index-vs-delete here. Emit a file task with ``file_path=None``; the processor
|
||||
resolves the file's *current* ``vector-index`` membership and indexes or
|
||||
deletes accordingly (see ``processor._reconcile_tag_event``). Both assign and
|
||||
unassign collapse to the same reconcile, which also makes "an unrelated tag
|
||||
changed" a cheap no-op.
|
||||
"""
|
||||
object_type = event.get("objectType")
|
||||
object_id = event.get("objectId")
|
||||
|
||||
# Only file tags drive vector sync; NC tags other object types too
|
||||
# (comments, etc.). ``objectId`` of 0/"" is malformed — skip.
|
||||
if object_type != "files" or not object_id:
|
||||
return None
|
||||
|
||||
return DocumentTask(
|
||||
user_id=user_id,
|
||||
doc_id=str(object_id),
|
||||
doc_type="file",
|
||||
# Reconciled in the processor; may be flipped to "delete" if the file is
|
||||
# no longer a tagged PDF. file_path=None is the "needs reconcile" signal.
|
||||
operation="index",
|
||||
modified_at=time,
|
||||
file_path=None,
|
||||
)
|
||||
|
||||
|
||||
def _parse_deck_event(
|
||||
event_class: str, event: dict, user_id: str, time: int
|
||||
) -> DocumentTask | None:
|
||||
|
||||
Reference in New Issue
Block a user