feat(vector): expand tagged directories for include + apply EXCLUDED_TAGS in scanner
NextcloudClient.find_files_by_tag now mirrors the directory semantics already used by the exclusion path (issue #710): when a tagged item is a folder, walk its descendants via WebDAV SEARCH (Depth: infinity) and include any files matching the MIME filter. Without this, tagging the root of a corpus with `vector-index` indexed nothing because the tag applies to the directory only, not to its children. The vector scanner additionally consults EXCLUDED_TAGS now, so a folder marked off-limits is skipped even if it (or an ancestor) carries the include tag — defense-in-depth, matching the "exclusion wins" contract already enforced by the MCP file tools. Also addressed a recurring memory-style nit: pre-existing f-string log lines in find_files_by_tag were converted to lazy %-style. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
55dc363a77
commit
43c6788555
@@ -1,5 +1,6 @@
|
|||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
from email.utils import parsedate_to_datetime
|
||||||
|
|
||||||
from httpx import (
|
from httpx import (
|
||||||
AsyncBaseTransport,
|
AsyncBaseTransport,
|
||||||
@@ -46,6 +47,44 @@ async def log_response(response: Response):
|
|||||||
logger.debug("Response [%s] %s", response.status_code, response.text)
|
logger.debug("Response [%s] %s", response.status_code, response.text)
|
||||||
|
|
||||||
|
|
||||||
|
def _normalise_search_result(item: dict) -> dict:
|
||||||
|
"""Normalise a webdav.search_files item to the get_files_by_tag shape.
|
||||||
|
|
||||||
|
``WebDAVClient.search_files`` and ``WebDAVClient.get_files_by_tag`` both
|
||||||
|
return per-file dicts but with subtly different keys (``file_id`` vs
|
||||||
|
``id``) and path conventions (no leading slash vs leading slash). This
|
||||||
|
helper makes a search result interchangeable with a tagged-file result
|
||||||
|
so callers (notably the vector scanner) can consume both via one shape.
|
||||||
|
"""
|
||||||
|
path = item.get("path", "")
|
||||||
|
if path and not path.startswith("/"):
|
||||||
|
path = "/" + path
|
||||||
|
|
||||||
|
last_modified_timestamp = item.get("last_modified_timestamp")
|
||||||
|
last_modified = item.get("last_modified")
|
||||||
|
if last_modified_timestamp is None and last_modified:
|
||||||
|
try:
|
||||||
|
last_modified_timestamp = int(
|
||||||
|
parsedate_to_datetime(last_modified).timestamp()
|
||||||
|
)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
last_modified_timestamp = None
|
||||||
|
|
||||||
|
file_id = item.get("file_id") if item.get("file_id") is not None else item.get("id")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"id": file_id,
|
||||||
|
"path": path,
|
||||||
|
"name": item.get("name") or (path.rsplit("/", 1)[-1] if path else ""),
|
||||||
|
"size": item.get("size", 0),
|
||||||
|
"content_type": item.get("content_type", ""),
|
||||||
|
"last_modified": last_modified,
|
||||||
|
"last_modified_timestamp": last_modified_timestamp,
|
||||||
|
"etag": item.get("etag"),
|
||||||
|
"is_directory": item.get("is_directory", False),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class AsyncDisableCookieTransport(AsyncBaseTransport):
|
class AsyncDisableCookieTransport(AsyncBaseTransport):
|
||||||
"""This Transport disable cookies from accumulating in the httpx AsyncClient
|
"""This Transport disable cookies from accumulating in the httpx AsyncClient
|
||||||
|
|
||||||
@@ -164,53 +203,123 @@ class NextcloudClient:
|
|||||||
|
|
||||||
This method coordinates tag lookup and file retrieval via WebDAV:
|
This method coordinates tag lookup and file retrieval via WebDAV:
|
||||||
1. Look up the tag ID by name
|
1. Look up the tag ID by name
|
||||||
2. Get all files with that tag (via REPORT with full metadata)
|
2. Get all entries (files and directories) with that tag via REPORT
|
||||||
3. Optionally filter by MIME type
|
3. For each tagged directory, walk descendants matching ``mime_type_filter``
|
||||||
|
via WebDAV SEARCH (``Depth: infinity``) so a tag on a folder applies
|
||||||
|
to every matching file beneath it. Mirrors the directory semantics
|
||||||
|
of :mod:`nextcloud_mcp_server.server.tag_exclusion` (issue #710).
|
||||||
|
4. Dedupe by file id — a file directly tagged AND living under a
|
||||||
|
tagged ancestor is returned once.
|
||||||
|
|
||||||
|
Directory expansion only runs when ``mime_type_filter`` is set:
|
||||||
|
without it, expanding a tagged folder would dump the user's entire
|
||||||
|
tree into the caller, which is almost never what the operator
|
||||||
|
wanted.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
tag_name: Name of the system tag to search for (e.g., "vector-index")
|
tag_name: Name of the system tag to search for (e.g., "vector-index")
|
||||||
mime_type_filter: Optional MIME type filter (e.g., "application/pdf")
|
mime_type_filter: Optional MIME type filter (e.g., "application/pdf").
|
||||||
|
When set, also enables directory expansion.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
List of file dictionaries with WebDAV properties (path, size, content_type, etc.)
|
List of file dictionaries with WebDAV properties (path, size, content_type, etc.)
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
RuntimeError: If tag lookup or file query fails
|
RuntimeError: If tag lookup or the initial file query fails. A
|
||||||
|
failure walking one tagged directory is logged and skipped — other
|
||||||
|
directly-tagged files are still returned.
|
||||||
|
|
||||||
Examples:
|
Examples:
|
||||||
# Find all files with "vector-index" tag
|
# Find all files with "vector-index" tag (no directory expansion)
|
||||||
files = await nc_client.find_files_by_tag("vector-index")
|
files = await nc_client.find_files_by_tag("vector-index")
|
||||||
|
|
||||||
# Find only PDFs with the tag
|
# Find only PDFs with the tag, including PDFs under any folder
|
||||||
|
# that carries the tag
|
||||||
pdfs = await nc_client.find_files_by_tag("vector-index", "application/pdf")
|
pdfs = await nc_client.find_files_by_tag("vector-index", "application/pdf")
|
||||||
"""
|
"""
|
||||||
# Look up tag by name using WebDAV
|
|
||||||
tag = await self.webdav.get_tag_by_name(tag_name)
|
tag = await self.webdav.get_tag_by_name(tag_name)
|
||||||
if not tag:
|
if not tag:
|
||||||
logger.debug(f"Tag '{tag_name}' not found, returning empty list")
|
logger.debug("Tag %r not found, returning empty list", tag_name)
|
||||||
return []
|
return []
|
||||||
|
|
||||||
# Get files with this tag (returns full file info from REPORT)
|
items = await self.webdav.get_files_by_tag(tag["id"])
|
||||||
files = await self.webdav.get_files_by_tag(tag["id"])
|
if not items:
|
||||||
if not files:
|
logger.debug("No items found with tag %r", tag_name)
|
||||||
logger.debug(f"No files found with tag '{tag_name}'")
|
|
||||||
return []
|
return []
|
||||||
|
|
||||||
logger.debug(f"Found {len(files)} files with tag '{tag_name}'")
|
logger.debug(
|
||||||
|
"Found %d directly-tagged item(s) with tag %r", len(items), tag_name
|
||||||
|
)
|
||||||
|
|
||||||
# Apply MIME type filter if specified
|
# Split into directly-tagged files vs tagged directories.
|
||||||
|
by_id: dict[int, dict] = {}
|
||||||
|
tagged_dirs: list[dict] = []
|
||||||
|
for item in items:
|
||||||
|
if item.get("is_directory"):
|
||||||
|
tagged_dirs.append(item)
|
||||||
|
continue
|
||||||
|
if mime_type_filter and not item.get("content_type", "").startswith(
|
||||||
|
mime_type_filter
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
file_id = item.get("id")
|
||||||
|
if file_id is None:
|
||||||
|
continue
|
||||||
|
by_id[file_id] = item
|
||||||
|
|
||||||
|
# Expand each tagged directory into its descendant files matching
|
||||||
|
# the MIME filter. Skip when no MIME filter is set — see docstring.
|
||||||
|
if mime_type_filter and tagged_dirs:
|
||||||
|
for dir_info in tagged_dirs:
|
||||||
|
dir_path = dir_info.get("path", "").strip("/")
|
||||||
|
try:
|
||||||
|
descendants = await self.webdav.find_by_type(
|
||||||
|
mime_type_filter, scope=dir_path
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(
|
||||||
|
"Tag-based directory walk failed for %r (tag %r): %s; "
|
||||||
|
"skipping descendants",
|
||||||
|
dir_path,
|
||||||
|
tag_name,
|
||||||
|
e,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
added = 0
|
||||||
|
for d in descendants:
|
||||||
|
if d.get("is_directory"):
|
||||||
|
continue
|
||||||
|
file_id = d.get("file_id") or d.get("id")
|
||||||
|
if file_id is None:
|
||||||
|
continue
|
||||||
|
if file_id in by_id:
|
||||||
|
# Directly-tagged entry already wins; keeps the
|
||||||
|
# canonical shape from get_files_by_tag.
|
||||||
|
continue
|
||||||
|
by_id[file_id] = _normalise_search_result(d)
|
||||||
|
added += 1
|
||||||
|
|
||||||
|
logger.debug(
|
||||||
|
"Tag %r: directory %r expanded to %d descendant %s file(s)",
|
||||||
|
tag_name,
|
||||||
|
dir_path,
|
||||||
|
added,
|
||||||
|
mime_type_filter,
|
||||||
|
)
|
||||||
|
|
||||||
|
files = list(by_id.values())
|
||||||
if mime_type_filter:
|
if mime_type_filter:
|
||||||
filtered_files = [
|
|
||||||
f
|
|
||||||
for f in files
|
|
||||||
if f.get("content_type", "").startswith(mime_type_filter)
|
|
||||||
]
|
|
||||||
logger.info(
|
logger.info(
|
||||||
f"Returning {len(filtered_files)} files with tag '{tag_name}' (filtered by {mime_type_filter})"
|
"Returning %d file(s) with tag %r (mime_type=%s, "
|
||||||
|
"%d directly-tagged folder(s) expanded)",
|
||||||
|
len(files),
|
||||||
|
tag_name,
|
||||||
|
mime_type_filter,
|
||||||
|
len(tagged_dirs),
|
||||||
)
|
)
|
||||||
return filtered_files
|
else:
|
||||||
|
logger.info("Returning %d file(s) with tag %r", len(files), tag_name)
|
||||||
logger.info(f"Returning {len(files)} files with tag '{tag_name}'")
|
|
||||||
return files
|
return files
|
||||||
|
|
||||||
def _get_webdav_base_path(self) -> str:
|
def _get_webdav_base_path(self) -> str:
|
||||||
|
|||||||
@@ -20,6 +20,10 @@ from nextcloud_mcp_server.client.news import NewsItemType
|
|||||||
from nextcloud_mcp_server.config import get_settings
|
from nextcloud_mcp_server.config import get_settings
|
||||||
from nextcloud_mcp_server.observability.metrics import record_vector_sync_scan
|
from nextcloud_mcp_server.observability.metrics import record_vector_sync_scan
|
||||||
from nextcloud_mcp_server.observability.tracing import trace_operation
|
from nextcloud_mcp_server.observability.tracing import trace_operation
|
||||||
|
from nextcloud_mcp_server.server.tag_exclusion import (
|
||||||
|
get_excluded_file_paths,
|
||||||
|
is_path_excluded,
|
||||||
|
)
|
||||||
from nextcloud_mcp_server.vector.placeholder import (
|
from nextcloud_mcp_server.vector.placeholder import (
|
||||||
query_document_metadata,
|
query_document_metadata,
|
||||||
write_placeholder_point,
|
write_placeholder_point,
|
||||||
@@ -410,15 +414,45 @@ async def scan_user_documents(
|
|||||||
nextcloud_file_ids = set()
|
nextcloud_file_ids = set()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Find files with vector-index tag using OCS Tags API
|
# Find files with vector-index tag using OCS Tags API.
|
||||||
|
# find_files_by_tag also expands tagged directories into their
|
||||||
|
# PDF descendants (Depth: infinity SEARCH), so a tag on a
|
||||||
|
# folder applies to every PDF beneath it.
|
||||||
settings = get_settings()
|
settings = get_settings()
|
||||||
tag_name = os.getenv("VECTOR_SYNC_PDF_TAG", "vector-index")
|
tag_name = os.getenv("VECTOR_SYNC_PDF_TAG", "vector-index")
|
||||||
# Use NextcloudClient.find_files_by_tag() which uses proper OCS API
|
|
||||||
# and filters by PDF MIME type
|
|
||||||
tagged_files = await nc_client.find_files_by_tag(
|
tagged_files = await nc_client.find_files_by_tag(
|
||||||
tag_name, mime_type_filter="application/pdf"
|
tag_name, mime_type_filter="application/pdf"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Apply EXCLUDED_TAGS as defense-in-depth: a folder marked
|
||||||
|
# off-limits via the exclusion tag must not be indexed even if
|
||||||
|
# it (or an ancestor) also carries the include tag. Mirrors the
|
||||||
|
# "exclusion wins" contract enforced by the MCP file tools.
|
||||||
|
try:
|
||||||
|
excluded_paths = await get_excluded_file_paths(nc_client.webdav)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(
|
||||||
|
"[SCAN-%s] EXCLUDED_TAGS lookup failed (%s); "
|
||||||
|
"proceeding without exclusion filter",
|
||||||
|
scan_id,
|
||||||
|
e,
|
||||||
|
)
|
||||||
|
excluded_paths = set()
|
||||||
|
if excluded_paths:
|
||||||
|
before = len(tagged_files)
|
||||||
|
tagged_files = [
|
||||||
|
f
|
||||||
|
for f in tagged_files
|
||||||
|
if not is_path_excluded(f.get("path", ""), excluded_paths)
|
||||||
|
]
|
||||||
|
skipped = before - len(tagged_files)
|
||||||
|
if skipped:
|
||||||
|
logger.info(
|
||||||
|
"[SCAN-%s] Skipped %d tagged file(s) under EXCLUDED_TAGS paths",
|
||||||
|
scan_id,
|
||||||
|
skipped,
|
||||||
|
)
|
||||||
|
|
||||||
for file_info in tagged_files:
|
for file_info in tagged_files:
|
||||||
# Files are already filtered by MIME type in find_files_by_tag()
|
# Files are already filtered by MIME type in find_files_by_tag()
|
||||||
file_count += 1
|
file_count += 1
|
||||||
|
|||||||
@@ -0,0 +1,172 @@
|
|||||||
|
"""End-to-end integration tests for tag-based file inclusion in
|
||||||
|
``NextcloudClient.find_files_by_tag``.
|
||||||
|
|
||||||
|
The vector scanner relies on this helper to enumerate files under the
|
||||||
|
``vector-index`` system tag (env: ``VECTOR_SYNC_PDF_TAG``). A user can
|
||||||
|
tag either an individual file *or* a folder; in the folder case the
|
||||||
|
tag should propagate to every matching descendant via a
|
||||||
|
``Depth: infinity`` WebDAV SEARCH.
|
||||||
|
|
||||||
|
Mirror of ``test_tag_exclusion.py`` but for the *inclusion* path.
|
||||||
|
Catches integration-level issues that the unit tests in
|
||||||
|
``tests/unit/client/test_nextcloud_client.py`` cannot, such as
|
||||||
|
PROPFIND/REPORT/SEARCH semantics, Nextcloud's actual MIME-type
|
||||||
|
reporting for the test fixtures, and the order in which directly-tagged
|
||||||
|
files vs descendants are returned.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from nextcloud_mcp_server.client import NextcloudClient
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
async def included_tag_environment(nc_client: NextcloudClient):
|
||||||
|
"""Provision a tag, a directly-tagged file, a tagged directory with
|
||||||
|
a child file, and an untagged sibling — all in a unique per-run
|
||||||
|
namespace.
|
||||||
|
|
||||||
|
Yields a dict with the layout. Cleanup runs in reverse order: untag,
|
||||||
|
delete files. The tag itself is left behind (no public delete-tag
|
||||||
|
API on the client today; tags are cheap and unique-per-run).
|
||||||
|
"""
|
||||||
|
suffix = uuid.uuid4().hex[:8]
|
||||||
|
tag_name = f"mcp-include-{suffix}"
|
||||||
|
test_dir = f"mcp_tag_incl_{suffix}"
|
||||||
|
tagged_file = f"{test_dir}/tagged.txt"
|
||||||
|
tagged_dir = f"{test_dir}/inside_dir"
|
||||||
|
tagged_dir_child = f"{tagged_dir}/child.txt"
|
||||||
|
nested_dir = f"{tagged_dir}/nested"
|
||||||
|
nested_dir_child = f"{nested_dir}/deep.txt"
|
||||||
|
untagged_file = f"{test_dir}/untagged.txt"
|
||||||
|
|
||||||
|
await nc_client.webdav.create_directory(test_dir)
|
||||||
|
await nc_client.webdav.create_directory(tagged_dir)
|
||||||
|
await nc_client.webdav.create_directory(nested_dir)
|
||||||
|
await nc_client.webdav.write_file(tagged_file, b"tagged file", "text/plain")
|
||||||
|
await nc_client.webdav.write_file(
|
||||||
|
tagged_dir_child, b"child of tagged dir", "text/plain"
|
||||||
|
)
|
||||||
|
await nc_client.webdav.write_file(
|
||||||
|
nested_dir_child, b"deep nested under tagged dir", "text/plain"
|
||||||
|
)
|
||||||
|
await nc_client.webdav.write_file(untagged_file, b"untagged sibling", "text/plain")
|
||||||
|
|
||||||
|
tag = await nc_client.webdav.get_or_create_tag(
|
||||||
|
name=tag_name, user_visible=True, user_assignable=True
|
||||||
|
)
|
||||||
|
assert tag["id"] is not None, "tag creation did not return an id"
|
||||||
|
|
||||||
|
tagged_file_info = await nc_client.webdav.get_file_info(tagged_file)
|
||||||
|
tagged_dir_info = await nc_client.webdav.get_file_info(tagged_dir)
|
||||||
|
assert tagged_file_info is not None and tagged_dir_info is not None
|
||||||
|
|
||||||
|
await nc_client.webdav.assign_tag_to_file(tagged_file_info["id"], tag["id"])
|
||||||
|
await nc_client.webdav.assign_tag_to_file(tagged_dir_info["id"], tag["id"])
|
||||||
|
|
||||||
|
yield {
|
||||||
|
"tag_name": tag_name,
|
||||||
|
"tag_id": tag["id"],
|
||||||
|
"test_dir": test_dir,
|
||||||
|
"tagged_file": tagged_file,
|
||||||
|
"tagged_file_id": tagged_file_info["id"],
|
||||||
|
"tagged_dir": tagged_dir,
|
||||||
|
"tagged_dir_id": tagged_dir_info["id"],
|
||||||
|
"tagged_dir_child": tagged_dir_child,
|
||||||
|
"nested_dir_child": nested_dir_child,
|
||||||
|
"untagged_file": untagged_file,
|
||||||
|
}
|
||||||
|
|
||||||
|
for file_id in (tagged_file_info["id"], tagged_dir_info["id"]):
|
||||||
|
try:
|
||||||
|
await nc_client.webdav.remove_tag_from_file(file_id, tag["id"])
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("failed to untag file %s: %s", file_id, e)
|
||||||
|
try:
|
||||||
|
await nc_client.webdav.delete_resource(test_dir)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("failed to delete %s: %s", test_dir, e)
|
||||||
|
|
||||||
|
|
||||||
|
def _basenames(files: list[dict]) -> set[str]:
|
||||||
|
"""Return the basename of each file path for assertion convenience."""
|
||||||
|
return {f["path"].rstrip("/").rsplit("/", 1)[-1] for f in files}
|
||||||
|
|
||||||
|
|
||||||
|
async def test_find_files_by_tag_includes_directly_tagged_file(
|
||||||
|
included_tag_environment, nc_client: NextcloudClient
|
||||||
|
):
|
||||||
|
"""A file with the tag directly applied is returned, regardless of the
|
||||||
|
folder-walk machinery."""
|
||||||
|
env = included_tag_environment
|
||||||
|
|
||||||
|
files = await nc_client.find_files_by_tag(
|
||||||
|
env["tag_name"], mime_type_filter="text/plain"
|
||||||
|
)
|
||||||
|
|
||||||
|
names = _basenames(files)
|
||||||
|
assert "tagged.txt" in names
|
||||||
|
# untagged sibling outside any tagged directory must not appear
|
||||||
|
assert "untagged.txt" not in names
|
||||||
|
|
||||||
|
|
||||||
|
async def test_find_files_by_tag_expands_tagged_directory(
|
||||||
|
included_tag_environment, nc_client: NextcloudClient
|
||||||
|
):
|
||||||
|
"""A tagged folder applies its tag to every matching descendant via
|
||||||
|
Depth: infinity SEARCH — including deeply nested files."""
|
||||||
|
env = included_tag_environment
|
||||||
|
|
||||||
|
files = await nc_client.find_files_by_tag(
|
||||||
|
env["tag_name"], mime_type_filter="text/plain"
|
||||||
|
)
|
||||||
|
|
||||||
|
names = _basenames(files)
|
||||||
|
# Direct child of the tagged folder
|
||||||
|
assert "child.txt" in names
|
||||||
|
# Grandchild — proves the walk is recursive, not single-level
|
||||||
|
assert "deep.txt" in names
|
||||||
|
|
||||||
|
|
||||||
|
async def test_find_files_by_tag_dedupes_directly_tagged_files_under_tagged_folder(
|
||||||
|
included_tag_environment, nc_client: NextcloudClient
|
||||||
|
):
|
||||||
|
"""When a file is *both* directly tagged and lives under a tagged
|
||||||
|
folder, it is returned exactly once. Verifies the dedup-by-id path."""
|
||||||
|
env = included_tag_environment
|
||||||
|
|
||||||
|
# Tag the deep child directly so it appears via two paths.
|
||||||
|
deep_info = await nc_client.webdav.get_file_info(env["nested_dir_child"])
|
||||||
|
assert deep_info is not None
|
||||||
|
await nc_client.webdav.assign_tag_to_file(deep_info["id"], env["tag_id"])
|
||||||
|
|
||||||
|
try:
|
||||||
|
files = await nc_client.find_files_by_tag(
|
||||||
|
env["tag_name"], mime_type_filter="text/plain"
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
await nc_client.webdav.remove_tag_from_file(deep_info["id"], env["tag_id"])
|
||||||
|
|
||||||
|
ids = [f["id"] for f in files]
|
||||||
|
assert ids.count(deep_info["id"]) == 1, f"deep child returned more than once: {ids}"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_find_files_by_tag_excludes_unrelated_paths(
|
||||||
|
included_tag_environment, nc_client: NextcloudClient
|
||||||
|
):
|
||||||
|
"""A file in the same parent directory as the tagged folder, but not
|
||||||
|
under it, is not returned. Guards against an over-broad SEARCH scope."""
|
||||||
|
env = included_tag_environment
|
||||||
|
|
||||||
|
files = await nc_client.find_files_by_tag(
|
||||||
|
env["tag_name"], mime_type_filter="text/plain"
|
||||||
|
)
|
||||||
|
|
||||||
|
paths = {f["path"].lstrip("/") for f in files}
|
||||||
|
assert env["untagged_file"] not in paths
|
||||||
@@ -0,0 +1,322 @@
|
|||||||
|
"""Unit tests for NextcloudClient orchestration logic.
|
||||||
|
|
||||||
|
Currently covers ``find_files_by_tag``: the wrapper that combines
|
||||||
|
``WebDAVClient.get_tag_by_name``, ``WebDAVClient.get_files_by_tag``, and
|
||||||
|
``WebDAVClient.find_by_type`` to resolve a system tag (and any tagged
|
||||||
|
folders) into a flat list of files.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from unittest.mock import AsyncMock
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from nextcloud_mcp_server.client import NextcloudClient, _normalise_search_result
|
||||||
|
|
||||||
|
|
||||||
|
def _make_client() -> NextcloudClient:
|
||||||
|
"""Build a NextcloudClient with mocked sub-clients.
|
||||||
|
|
||||||
|
The client constructor opens an httpx session; we don't need it, just
|
||||||
|
a stub instance whose ``webdav`` attribute we can replace.
|
||||||
|
"""
|
||||||
|
client = NextcloudClient.__new__(NextcloudClient)
|
||||||
|
client.username = "alice"
|
||||||
|
client.webdav = AsyncMock()
|
||||||
|
return client
|
||||||
|
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.unit
|
||||||
|
|
||||||
|
|
||||||
|
class TestNormaliseSearchResult:
|
||||||
|
def test_adds_leading_slash_to_path(self):
|
||||||
|
result = _normalise_search_result(
|
||||||
|
{"path": "Documents/foo.pdf", "file_id": 1, "is_directory": False}
|
||||||
|
)
|
||||||
|
assert result["path"] == "/Documents/foo.pdf"
|
||||||
|
|
||||||
|
def test_preserves_leading_slash_when_present(self):
|
||||||
|
result = _normalise_search_result(
|
||||||
|
{"path": "/Documents/foo.pdf", "file_id": 1, "is_directory": False}
|
||||||
|
)
|
||||||
|
assert result["path"] == "/Documents/foo.pdf"
|
||||||
|
|
||||||
|
def test_maps_file_id_to_id(self):
|
||||||
|
result = _normalise_search_result(
|
||||||
|
{"path": "/foo.pdf", "file_id": 99, "is_directory": False}
|
||||||
|
)
|
||||||
|
assert result["id"] == 99
|
||||||
|
|
||||||
|
def test_falls_back_to_id_when_file_id_missing(self):
|
||||||
|
result = _normalise_search_result(
|
||||||
|
{"path": "/foo.pdf", "id": 7, "is_directory": False}
|
||||||
|
)
|
||||||
|
assert result["id"] == 7
|
||||||
|
|
||||||
|
def test_computes_last_modified_timestamp(self):
|
||||||
|
result = _normalise_search_result(
|
||||||
|
{
|
||||||
|
"path": "/foo.pdf",
|
||||||
|
"file_id": 1,
|
||||||
|
"last_modified": "Wed, 01 Jan 2025 00:00:00 GMT",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
assert result["last_modified_timestamp"] == 1735689600
|
||||||
|
|
||||||
|
def test_preserves_existing_timestamp(self):
|
||||||
|
result = _normalise_search_result(
|
||||||
|
{
|
||||||
|
"path": "/foo.pdf",
|
||||||
|
"file_id": 1,
|
||||||
|
"last_modified_timestamp": 12345,
|
||||||
|
"last_modified": "Wed, 01 Jan 2025 00:00:00 GMT",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
assert result["last_modified_timestamp"] == 12345
|
||||||
|
|
||||||
|
def test_handles_unparseable_last_modified(self):
|
||||||
|
result = _normalise_search_result(
|
||||||
|
{"path": "/foo.pdf", "file_id": 1, "last_modified": "not-a-date"}
|
||||||
|
)
|
||||||
|
assert result["last_modified_timestamp"] is None
|
||||||
|
|
||||||
|
|
||||||
|
class TestFindFilesByTag:
|
||||||
|
async def test_returns_empty_when_tag_missing(self):
|
||||||
|
client = _make_client()
|
||||||
|
client.webdav.get_tag_by_name = AsyncMock(return_value=None)
|
||||||
|
|
||||||
|
result = await client.find_files_by_tag("does-not-exist")
|
||||||
|
|
||||||
|
assert result == []
|
||||||
|
client.webdav.get_files_by_tag.assert_not_called()
|
||||||
|
|
||||||
|
async def test_returns_empty_when_no_tagged_items(self):
|
||||||
|
client = _make_client()
|
||||||
|
client.webdav.get_tag_by_name = AsyncMock(return_value={"id": 5})
|
||||||
|
client.webdav.get_files_by_tag = AsyncMock(return_value=[])
|
||||||
|
|
||||||
|
result = await client.find_files_by_tag("vector-index")
|
||||||
|
|
||||||
|
assert result == []
|
||||||
|
client.webdav.find_by_type.assert_not_called()
|
||||||
|
|
||||||
|
async def test_directly_tagged_files_pass_through_with_mime_filter(self):
|
||||||
|
client = _make_client()
|
||||||
|
client.webdav.get_tag_by_name = AsyncMock(return_value={"id": 5})
|
||||||
|
client.webdav.get_files_by_tag = AsyncMock(
|
||||||
|
return_value=[
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"path": "/Documents/a.pdf",
|
||||||
|
"content_type": "application/pdf",
|
||||||
|
"is_directory": False,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 2,
|
||||||
|
"path": "/Documents/notes.md",
|
||||||
|
"content_type": "text/markdown",
|
||||||
|
"is_directory": False,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await client.find_files_by_tag(
|
||||||
|
"vector-index", mime_type_filter="application/pdf"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert {f["id"] for f in result} == {1}
|
||||||
|
# No tagged dirs → no SEARCH walk.
|
||||||
|
client.webdav.find_by_type.assert_not_called()
|
||||||
|
|
||||||
|
async def test_expands_tagged_directory_into_pdf_descendants(self):
|
||||||
|
client = _make_client()
|
||||||
|
client.webdav.get_tag_by_name = AsyncMock(return_value={"id": 5})
|
||||||
|
# One directly-tagged folder, no directly-tagged files.
|
||||||
|
client.webdav.get_files_by_tag = AsyncMock(
|
||||||
|
return_value=[
|
||||||
|
{
|
||||||
|
"id": 100,
|
||||||
|
"path": "/corpus",
|
||||||
|
"content_type": "httpd/unix-directory",
|
||||||
|
"is_directory": True,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
)
|
||||||
|
# Search inside the folder returns two PDFs.
|
||||||
|
client.webdav.find_by_type = AsyncMock(
|
||||||
|
return_value=[
|
||||||
|
{
|
||||||
|
"file_id": 11,
|
||||||
|
"path": "corpus/arxiv/a.pdf",
|
||||||
|
"content_type": "application/pdf",
|
||||||
|
"is_directory": False,
|
||||||
|
"last_modified": "Wed, 01 Jan 2025 00:00:00 GMT",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"file_id": 12,
|
||||||
|
"path": "corpus/arxiv/b.pdf",
|
||||||
|
"content_type": "application/pdf",
|
||||||
|
"is_directory": False,
|
||||||
|
"last_modified": "Wed, 01 Jan 2025 00:00:00 GMT",
|
||||||
|
},
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await client.find_files_by_tag(
|
||||||
|
"vector-index", mime_type_filter="application/pdf"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert {f["id"] for f in result} == {11, 12}
|
||||||
|
# Each result is normalised to the get_files_by_tag shape.
|
||||||
|
for f in result:
|
||||||
|
assert f["path"].startswith("/")
|
||||||
|
assert f["last_modified_timestamp"] is not None
|
||||||
|
# SEARCH was scoped to the tagged folder (no leading slash).
|
||||||
|
client.webdav.find_by_type.assert_awaited_once()
|
||||||
|
call_kwargs = client.webdav.find_by_type.await_args.kwargs
|
||||||
|
assert call_kwargs["scope"] == "corpus"
|
||||||
|
|
||||||
|
async def test_dedupes_when_file_directly_tagged_and_under_tagged_folder(self):
|
||||||
|
client = _make_client()
|
||||||
|
client.webdav.get_tag_by_name = AsyncMock(return_value={"id": 5})
|
||||||
|
client.webdav.get_files_by_tag = AsyncMock(
|
||||||
|
return_value=[
|
||||||
|
{
|
||||||
|
"id": 11,
|
||||||
|
"path": "/corpus/arxiv/a.pdf",
|
||||||
|
"content_type": "application/pdf",
|
||||||
|
"is_directory": False,
|
||||||
|
"name": "a.pdf",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 100,
|
||||||
|
"path": "/corpus",
|
||||||
|
"content_type": "httpd/unix-directory",
|
||||||
|
"is_directory": True,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
)
|
||||||
|
client.webdav.find_by_type = AsyncMock(
|
||||||
|
return_value=[
|
||||||
|
{
|
||||||
|
"file_id": 11,
|
||||||
|
"path": "corpus/arxiv/a.pdf",
|
||||||
|
"content_type": "application/pdf",
|
||||||
|
"is_directory": False,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"file_id": 12,
|
||||||
|
"path": "corpus/arxiv/b.pdf",
|
||||||
|
"content_type": "application/pdf",
|
||||||
|
"is_directory": False,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await client.find_files_by_tag(
|
||||||
|
"vector-index", mime_type_filter="application/pdf"
|
||||||
|
)
|
||||||
|
|
||||||
|
# File 11 is included exactly once and keeps the directly-tagged
|
||||||
|
# entry's metadata (name from get_files_by_tag, not search).
|
||||||
|
assert sorted(f["id"] for f in result) == [11, 12]
|
||||||
|
assert next(f for f in result if f["id"] == 11)["name"] == "a.pdf"
|
||||||
|
|
||||||
|
async def test_directory_walk_failure_skips_only_that_directory(self, caplog):
|
||||||
|
client = _make_client()
|
||||||
|
client.webdav.get_tag_by_name = AsyncMock(return_value={"id": 5})
|
||||||
|
client.webdav.get_files_by_tag = AsyncMock(
|
||||||
|
return_value=[
|
||||||
|
{
|
||||||
|
"id": 7,
|
||||||
|
"path": "/Documents/keep.pdf",
|
||||||
|
"content_type": "application/pdf",
|
||||||
|
"is_directory": False,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 100,
|
||||||
|
"path": "/broken",
|
||||||
|
"content_type": "httpd/unix-directory",
|
||||||
|
"is_directory": True,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
)
|
||||||
|
client.webdav.find_by_type = AsyncMock(side_effect=RuntimeError("REPORT 500"))
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
caplog.set_level(logging.WARNING, logger="nextcloud_mcp_server.client")
|
||||||
|
result = await client.find_files_by_tag(
|
||||||
|
"vector-index", mime_type_filter="application/pdf"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Directly-tagged file survives even though the dir walk blew up.
|
||||||
|
assert {f["id"] for f in result} == {7}
|
||||||
|
assert "Tag-based directory walk failed" in caplog.text
|
||||||
|
|
||||||
|
async def test_no_mime_filter_skips_directory_expansion(self):
|
||||||
|
client = _make_client()
|
||||||
|
client.webdav.get_tag_by_name = AsyncMock(return_value={"id": 5})
|
||||||
|
client.webdav.get_files_by_tag = AsyncMock(
|
||||||
|
return_value=[
|
||||||
|
{
|
||||||
|
"id": 7,
|
||||||
|
"path": "/Documents/keep.pdf",
|
||||||
|
"content_type": "application/pdf",
|
||||||
|
"is_directory": False,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 100,
|
||||||
|
"path": "/corpus",
|
||||||
|
"content_type": "httpd/unix-directory",
|
||||||
|
"is_directory": True,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await client.find_files_by_tag("vector-index")
|
||||||
|
|
||||||
|
# Without a MIME filter, directory expansion would fan out
|
||||||
|
# uncontrollably — the helper deliberately skips it.
|
||||||
|
assert {f["id"] for f in result} == {7}
|
||||||
|
client.webdav.find_by_type.assert_not_called()
|
||||||
|
|
||||||
|
async def test_skips_descendant_directories_in_search_results(self):
|
||||||
|
"""find_by_type can return collections too (e.g. when the SEARCH
|
||||||
|
backend treats a folder's mime type as matching). Those must not
|
||||||
|
slip through and clobber file IDs."""
|
||||||
|
client = _make_client()
|
||||||
|
client.webdav.get_tag_by_name = AsyncMock(return_value={"id": 5})
|
||||||
|
client.webdav.get_files_by_tag = AsyncMock(
|
||||||
|
return_value=[
|
||||||
|
{
|
||||||
|
"id": 100,
|
||||||
|
"path": "/corpus",
|
||||||
|
"content_type": "httpd/unix-directory",
|
||||||
|
"is_directory": True,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
)
|
||||||
|
client.webdav.find_by_type = AsyncMock(
|
||||||
|
return_value=[
|
||||||
|
{
|
||||||
|
"file_id": 50,
|
||||||
|
"path": "corpus/sub",
|
||||||
|
"content_type": "httpd/unix-directory",
|
||||||
|
"is_directory": True,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"file_id": 51,
|
||||||
|
"path": "corpus/sub/a.pdf",
|
||||||
|
"content_type": "application/pdf",
|
||||||
|
"is_directory": False,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await client.find_files_by_tag(
|
||||||
|
"vector-index", mime_type_filter="application/pdf"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert {f["id"] for f in result} == {51}
|
||||||
Reference in New Issue
Block a user