feat(vector): index files in real time on vector-index tag changes
Tagging an existing file/folder emits only OCP\SystemTag\MapperEvent — never a Node*Event — so tagged PDFs were previously only picked up by the hourly scanner. Subscribe to the tag event and reconcile membership so adding/removing the `vector-index` tag (re)indexes in near-real time. - webhook_presets: add OCP\SystemTag\MapperEvent to the files_sync preset (NC 32+, where MapperEvent gained getWebhookSerializable(); harmless on older servers — it just never fires). - webhook_parser: parse MapperEvent (objectType=files) into a path-less file "reconcile" task. The payload carries only a fileid + tagIds (no name/path), so assign and unassign both collapse to a reconcile. - processor._reconcile_tag_event: resolve the fileid against the user's current vector-index PDFs (find_files_by_tag). Present -> index with the resolved path/etag; absent -> flip to delete. Naturally handles "an unrelated tag changed" and a tagged folder's own fileid (no-op; the scanner still expands folders to descendants). - Unit tests for the parser branch and the reconcile. The matching admin-UI preset change ships separately in the astrolabe app repo. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
78f3f284a6
commit
1322e5aba0
@@ -216,3 +216,94 @@ def test_missing_time_field_defaults_to_zero():
|
||||
task = extract_document_task(payload)
|
||||
assert task is not None
|
||||
assert task.modified_at == 0
|
||||
|
||||
|
||||
# --- SystemTag MapperEvent (assign/unassign) -------------------------------
|
||||
#
|
||||
# NC 32+ serializes the event as {eventType, objectType, objectId, tagIds}.
|
||||
# Both directions collapse to a path-less file "reconcile" task; the processor
|
||||
# resolves current vector-index membership to decide index-vs-delete.
|
||||
|
||||
_TAG_MAPPER = "OCP\\SystemTag\\MapperEvent"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_tag_assign_event_returns_reconcile_task():
|
||||
payload = {
|
||||
"user": {"uid": "alice"},
|
||||
"time": 1762850245,
|
||||
"event": {
|
||||
"class": _TAG_MAPPER,
|
||||
"eventType": "OCP\\SystemTag\\ISystemTagObjectMapper::assignTags",
|
||||
"objectType": "files",
|
||||
"objectId": "478087",
|
||||
"tagIds": [7],
|
||||
},
|
||||
}
|
||||
|
||||
task = extract_document_task(payload)
|
||||
assert task is not None
|
||||
assert task.user_id == "alice"
|
||||
assert task.doc_type == "file"
|
||||
assert task.doc_id == "478087"
|
||||
assert task.operation == "index"
|
||||
# No path in the payload -> the processor reconciles membership.
|
||||
assert task.file_path is None
|
||||
assert task.modified_at == 1762850245
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_tag_unassign_event_also_returns_reconcile_task():
|
||||
"""Unassign collapses to the same reconcile (processor flips to delete)."""
|
||||
payload = {
|
||||
"user": {"uid": "alice"},
|
||||
"time": 1762850245,
|
||||
"event": {
|
||||
"class": _TAG_MAPPER,
|
||||
"eventType": "OCP\\SystemTag\\ISystemTagObjectMapper::unassignTags",
|
||||
"objectType": "files",
|
||||
"objectId": "478087",
|
||||
"tagIds": [7],
|
||||
},
|
||||
}
|
||||
|
||||
task = extract_document_task(payload)
|
||||
assert task is not None
|
||||
assert task.doc_type == "file"
|
||||
assert task.operation == "index"
|
||||
assert task.file_path is None
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_tag_event_non_files_object_type_returns_none():
|
||||
"""Tags on non-file objects (e.g. comments) don't drive vector sync."""
|
||||
payload = {
|
||||
"user": {"uid": "alice"},
|
||||
"time": 1762850245,
|
||||
"event": {
|
||||
"class": _TAG_MAPPER,
|
||||
"eventType": "OCP\\SystemTag\\ISystemTagObjectMapper::assignTags",
|
||||
"objectType": "comments",
|
||||
"objectId": "12",
|
||||
"tagIds": [7],
|
||||
},
|
||||
}
|
||||
|
||||
assert extract_document_task(payload) is None
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_tag_event_missing_object_id_returns_none():
|
||||
payload = {
|
||||
"user": {"uid": "alice"},
|
||||
"time": 1762850245,
|
||||
"event": {
|
||||
"class": _TAG_MAPPER,
|
||||
"eventType": "OCP\\SystemTag\\ISystemTagObjectMapper::assignTags",
|
||||
"objectType": "files",
|
||||
"objectId": "",
|
||||
"tagIds": [7],
|
||||
},
|
||||
}
|
||||
|
||||
assert extract_document_task(payload) is None
|
||||
|
||||
@@ -89,6 +89,18 @@ def test_get_deck_preset():
|
||||
assert "OCA\\Deck\\Event\\BoardUpdatedEvent" in event_classes
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_files_sync_includes_systemtag_event():
|
||||
"""files_sync registers the SystemTag MapperEvent so tag changes (NC 32+)
|
||||
drive vector-index (re)indexing, alongside the node create/write/delete
|
||||
events."""
|
||||
preset = get_preset("files_sync")
|
||||
assert preset is not None
|
||||
event_classes = [e["event"] for e in preset["events"]]
|
||||
assert "OCP\\SystemTag\\MapperEvent" in event_classes
|
||||
assert "OCP\\Files\\Events\\Node\\NodeCreatedEvent" in event_classes
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_filter_presets_subset_installed():
|
||||
"""Test filtering when only some apps are installed."""
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Unit tests for processor._reconcile_tag_event.
|
||||
|
||||
A SystemTag MapperEvent enqueues a file task with only a fileid (file_path is
|
||||
None). The reconcile resolves the file's *current* ``vector-index`` membership
|
||||
and mutates the task into a concrete index (path/etag filled) or a delete.
|
||||
"""
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from nextcloud_mcp_server.vector.processor import _reconcile_tag_event
|
||||
from nextcloud_mcp_server.vector.scanner import DocumentTask
|
||||
|
||||
|
||||
def _tag_task(doc_id: str = "478087") -> DocumentTask:
|
||||
return DocumentTask(
|
||||
user_id="alice",
|
||||
doc_id=doc_id,
|
||||
doc_type="file",
|
||||
operation="index",
|
||||
modified_at=0,
|
||||
file_path=None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_reconcile_tagged_file_becomes_index():
|
||||
"""A fileid still carrying the tag is resolved to a concrete index task."""
|
||||
nc_client = MagicMock()
|
||||
nc_client.find_files_by_tag = AsyncMock(
|
||||
return_value=[
|
||||
{
|
||||
"id": "478087",
|
||||
"path": "/alice/files/Docs/report.pdf",
|
||||
"etag": "abc123",
|
||||
"last_modified_timestamp": 1762850245,
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
task = _tag_task("478087")
|
||||
await _reconcile_tag_event(task, nc_client)
|
||||
|
||||
assert task.operation == "index"
|
||||
assert task.file_path == "/alice/files/Docs/report.pdf"
|
||||
assert task.etag == "abc123"
|
||||
assert task.modified_at == 1762850245
|
||||
nc_client.find_files_by_tag.assert_awaited_once_with(
|
||||
"vector-index", mime_type_filter="application/pdf"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_reconcile_untagged_file_becomes_delete():
|
||||
"""A fileid absent from the tagged set flips the task to a delete."""
|
||||
nc_client = MagicMock()
|
||||
nc_client.find_files_by_tag = AsyncMock(
|
||||
return_value=[
|
||||
{"id": "999", "path": "/alice/files/other.pdf", "etag": "z"},
|
||||
]
|
||||
)
|
||||
|
||||
task = _tag_task("478087")
|
||||
await _reconcile_tag_event(task, nc_client)
|
||||
|
||||
assert task.operation == "delete"
|
||||
# Path stays None — the delete path addresses points by doc_id only.
|
||||
assert task.file_path is None
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_reconcile_preserves_existing_etag():
|
||||
"""An etag already on the task is not overwritten by the tag listing."""
|
||||
nc_client = MagicMock()
|
||||
nc_client.find_files_by_tag = AsyncMock(
|
||||
return_value=[
|
||||
{"id": "478087", "path": "/alice/files/r.pdf", "etag": "from-listing"}
|
||||
]
|
||||
)
|
||||
|
||||
task = _tag_task("478087")
|
||||
task.etag = "preset"
|
||||
await _reconcile_tag_event(task, nc_client)
|
||||
|
||||
assert task.etag == "preset"
|
||||
assert task.file_path == "/alice/files/r.pdf"
|
||||
Reference in New Issue
Block a user