feat(search): support multiple folders in the semantic-search path filter
Extend the ADR-027 Phase 2 path filter from a single path_prefix to a list of folders. The new normalize_path_prefixes() helper is the single source of truth for trimming, dropping blanks, and de-duplicating, and folds the legacy single path_prefix into the list for backward compatibility. build_base_filter_conditions() adds one MatchText to the must clause for a single folder (unchanged shape) and OR-s multiple folders via a nested Filter(should=[...]) so a file under any selected folder matches while still AND-ing against the ACL/doc_type/date conditions. path_prefixes is threaded through every search surface: the nc_semantic_search MCP tool, the visualization API (JSON body), and the viz route (CSV query param). The Astrolabe frontend folder picker that produces these lists ships in a companion astrolabe PR. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
b91af923d2
commit
de6c4b360d
@@ -68,7 +68,7 @@ Filters can only be applied to fields that exist in the Qdrant payload (built in
|
|||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
| Modified-date range | `modified_at` | `int` (Unix ts) | ✅ **Ready** — value present on every point; needs a payload index (added in Phase 1, no content re-index) |
|
| Modified-date range | `modified_at` | `int` (Unix ts) | ✅ **Ready** — value present on every point; needs a payload index (added in Phase 1, no content re-index) |
|
||||||
| Document type | `doc_type` | keyword-indexed `str` | ✅ Implemented |
|
| Document type | `doc_type` | keyword-indexed `str` | ✅ Implemented |
|
||||||
| Directory / path | `file_path` (files only) | `str` | ✅ **Ready (Phase 2)** — value present on file points; TEXT payload index added (no content re-index), filtered with `MatchText` |
|
| Directory / path | `file_path` (files only) | `list[str]` (multi-folder) | ✅ **Implemented (Phase 2)** — TEXT payload index (no content re-index); one or more folders filtered with `MatchText`, multiple OR-ed via nested `Filter(should=...)`; picked from the native folder browser |
|
||||||
| Tags | — | — | ❌ **Not indexed** — no `tags` field is written during scanning |
|
| Tags | — | — | ❌ **Not indexed** — no `tags` field is written during scanning |
|
||||||
| Category (notes) | — | — | ❌ Not in payload — fetched from the Notes API at verify time only |
|
| Category (notes) | — | — | ❌ Not in payload — fetched from the Notes API at verify time only |
|
||||||
|
|
||||||
@@ -186,19 +186,30 @@ express:
|
|||||||
it on the answer path. When demand appears it can be threaded through using exactly the same
|
it on the answer path. When demand appears it can be threaded through using exactly the same
|
||||||
parameter + `parse_modified_timestamp` pattern; doing it now would add an unused parameter and
|
parameter + `parse_modified_timestamp` pattern; doing it now would add an unused parameter and
|
||||||
widen the change for no user-visible gain.
|
widen the change for no user-visible gain.
|
||||||
- **Phase 2 — directory / path (implemented).** Add a `path_prefix` parameter threaded through the
|
- **Phase 2 — directory / path (implemented).** Threaded through the same shared contract
|
||||||
same shared contract (`build_base_filter_conditions` → `FieldCondition(key="file_path",
|
(`build_base_filter_conditions`) and backed by a `file_path` **TEXT** payload index in
|
||||||
match=MatchText(text=path_prefix))`), and a `file_path` **TEXT** payload index in
|
|
||||||
`_PAYLOAD_INDEX_FIELDS` (no content re-index — `file_path` is already on every file point).
|
`_PAYLOAD_INDEX_FIELDS` (no content re-index — `file_path` is already on every file point).
|
||||||
`MatchText` semantics differ by backend: **server Qdrant** tokenizes (AND-of-tokens, so
|
`MatchText` semantics differ by backend: **server Qdrant** tokenizes (AND-of-tokens, so
|
||||||
`/Projects/Reports` matches files whose path contains both the `Projects` and `Reports` tokens),
|
`/Projects/Reports` matches files whose path contains both the `Projects` and `Reports` tokens),
|
||||||
while **local/embedded qdrant-client** matches by substring containment — both serve folder
|
while **local/embedded qdrant-client** matches by substring containment — both serve folder
|
||||||
scoping, neither is a strict left-anchored prefix (a future strict-prefix would need an indexed
|
scoping, neither is a strict left-anchored prefix (a future strict-prefix would need an indexed
|
||||||
ancestor-path array, i.e. a re-index, so it stays out of Phase 2). Because `file_path` is only
|
ancestor-path array, i.e. a re-index, so it stays out of Phase 2). Because `file_path` is only
|
||||||
written for `doc_type == "file"`, a non-empty `path_prefix` implicitly restricts results to
|
written for `doc_type == "file"`, a non-empty path filter implicitly restricts results to files.
|
||||||
files; the frontend uses a native path text input enabled only when the **Files** doc type is in
|
- **Multi-folder (list-valued).** The filter accepts **one or more** folders via a
|
||||||
scope (rather than `NcFilePicker`, to avoid a hard `@nextcloud/vue` component-version dependency —
|
`path_prefixes: list[str]` parameter (the original single `path_prefix` is retained for
|
||||||
same rationale as the date inputs). A blank value is treated as "no filter".
|
backward compatibility and folded into the list by `normalize_path_prefixes`, which trims,
|
||||||
|
drops blanks, and de-dupes). `build_base_filter_conditions` adds a single `MatchText` to the
|
||||||
|
`must` clause for one folder, and OR-s multiple folders via a nested `Filter(should=[...])` so a
|
||||||
|
file under **any** selected folder matches while still AND-ing against the ACL/doc_type/date
|
||||||
|
conditions. Every search surface parses the list: the MCP tool (`nc_semantic_search`), the
|
||||||
|
visualization API (JSON body), and the viz route (CSV query param).
|
||||||
|
- **Frontend uses the native folder picker.** Instead of a free-text path input, the Astrolabe
|
||||||
|
app opens Nextcloud's server-side folder browser via `getFilePickerBuilder()` from
|
||||||
|
`@nextcloud/dialogs` (already a dependency — no `@nextcloud/vue` component-version coupling),
|
||||||
|
configured directory-only + multi-select. Picked folders are real, validated server paths
|
||||||
|
(no typos), rendered as removable chips, and sent as a comma-separated `path_prefixes` list. The
|
||||||
|
Astrolabe PHP `ApiController`/`McpServerClient` forward the list to the MCP server. The control
|
||||||
|
is enabled only when the **Files** doc type is in scope; an empty selection means "no filter".
|
||||||
- **Phase 3 — tags (and optionally category).** Add a `tags: list[str]` payload field in
|
- **Phase 3 — tags (and optionally category).** Add a `tags: list[str]` payload field in
|
||||||
`processor.py`, propagate Nextcloud system tags during scanning, trigger a re-index, then wire
|
`processor.py`, propagate Nextcloud system tags during scanning, trigger a re-index, then wire
|
||||||
`NcSelectTags` (`MatchAny` over tags). Re-index cost lives here, isolated from the cheap wins.
|
`NcSelectTags` (`MatchAny` over tags). Re-index cost lives here, isolated from the cheap wins.
|
||||||
@@ -214,7 +225,9 @@ The Astrolabe app adds filter controls to the existing collapsible advanced pane
|
|||||||
specific `@nextcloud/vue` component version (`NcDateTimePicker` / `NcChip` remain a later option);
|
specific `@nextcloud/vue` component version (`NcDateTimePicker` / `NcChip` remain a later option);
|
||||||
this matches the component's existing native `<input type="range">` controls.
|
this matches the component's existing native `<input type="range">` controls.
|
||||||
- Doc types → existing checkbox grid, now also echoed as chips.
|
- Doc types → existing checkbox grid, now also echoed as chips.
|
||||||
- (Phase 2/3) path → `NcFilePicker`; tags → `NcSelectTags :fetch-tags`.
|
- (Phase 2) path → native folder picker (`getFilePickerBuilder` from `@nextcloud/dialogs`),
|
||||||
|
multi-select, rendered as one removable chip per folder. (Phase 3) tags → `NcSelectTags
|
||||||
|
:fetch-tags`.
|
||||||
|
|
||||||
Dates cross the wire as **RFC 3339 / ISO 8601 strings** (e.g. `"2026-01-01T00:00:00Z"`) — the format
|
Dates cross the wire as **RFC 3339 / ISO 8601 strings** (e.g. `"2026-01-01T00:00:00Z"`) — the format
|
||||||
Nextcloud's date pickers and Unified Search use. The MCP/HTTP boundary parses RFC 3339 (and bare
|
Nextcloud's date pickers and Unified Search use. The MCP/HTTP boundary parses RFC 3339 (and bare
|
||||||
|
|||||||
@@ -31,7 +31,10 @@ from nextcloud_mcp_server.search import (
|
|||||||
BM25HybridSearchAlgorithm,
|
BM25HybridSearchAlgorithm,
|
||||||
SemanticSearchAlgorithm,
|
SemanticSearchAlgorithm,
|
||||||
)
|
)
|
||||||
from nextcloud_mcp_server.search.access_filter import list_accessible_owners
|
from nextcloud_mcp_server.search.access_filter import (
|
||||||
|
list_accessible_owners,
|
||||||
|
normalize_path_prefixes,
|
||||||
|
)
|
||||||
from nextcloud_mcp_server.search.context import (
|
from nextcloud_mcp_server.search.context import (
|
||||||
get_chunk_bbox_and_page_from_qdrant,
|
get_chunk_bbox_and_page_from_qdrant,
|
||||||
get_chunk_with_context,
|
get_chunk_with_context,
|
||||||
@@ -225,8 +228,21 @@ async def unified_search(request: Request) -> JSONResponse:
|
|||||||
include_pca = body.get("include_pca", False)
|
include_pca = body.get("include_pca", False)
|
||||||
include_chunks = body.get("include_chunks", True)
|
include_chunks = body.get("include_chunks", True)
|
||||||
doc_types = body.get("doc_types") # Optional filter
|
doc_types = body.get("doc_types") # Optional filter
|
||||||
# ADR-027 Phase 2 path filter (files only); blank ⇒ no filter.
|
# ADR-027 Phase 2 path filter (files only); blank ⇒ no filter. Accept a
|
||||||
path_prefix = (body.get("path_prefix") or "").strip() or None
|
# path_prefixes list (multi-folder) alongside the legacy single
|
||||||
|
# path_prefix; normalize drops blanks and de-dupes.
|
||||||
|
_path_prefixes_raw = body.get("path_prefixes")
|
||||||
|
if isinstance(_path_prefixes_raw, list):
|
||||||
|
_path_prefixes_list = _path_prefixes_raw
|
||||||
|
elif isinstance(_path_prefixes_raw, str):
|
||||||
|
_path_prefixes_list = _path_prefixes_raw.split(",")
|
||||||
|
else:
|
||||||
|
# Ignore any other JSON shape (number, object, null) rather than
|
||||||
|
# blowing up on .split — the legacy path_prefix still applies.
|
||||||
|
_path_prefixes_list = []
|
||||||
|
path_prefixes = normalize_path_prefixes(
|
||||||
|
body.get("path_prefix"), _path_prefixes_list
|
||||||
|
)
|
||||||
|
|
||||||
if not query:
|
if not query:
|
||||||
return JSONResponse({"results": [], "total_found": 0})
|
return JSONResponse({"results": [], "total_found": 0})
|
||||||
@@ -268,7 +284,7 @@ async def unified_search(request: Request) -> JSONResponse:
|
|||||||
accessible_owners=owners,
|
accessible_owners=owners,
|
||||||
modified_after=modified_after,
|
modified_after=modified_after,
|
||||||
modified_before=modified_before,
|
modified_before=modified_before,
|
||||||
path_prefix=path_prefix,
|
path_prefixes=path_prefixes,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
# Sort, then cap to a fixed over-fetch budget before the result
|
# Sort, then cap to a fixed over-fetch budget before the result
|
||||||
@@ -288,7 +304,7 @@ async def unified_search(request: Request) -> JSONResponse:
|
|||||||
accessible_owners=owners,
|
accessible_owners=owners,
|
||||||
modified_after=modified_after,
|
modified_after=modified_after,
|
||||||
modified_before=modified_before,
|
modified_before=modified_before,
|
||||||
path_prefix=path_prefix,
|
path_prefixes=path_prefixes,
|
||||||
)
|
)
|
||||||
return results
|
return results
|
||||||
|
|
||||||
@@ -440,8 +456,21 @@ async def vector_search(request: Request) -> JSONResponse:
|
|||||||
limit = min(body.get("limit", 10), 50) # Enforce max limit
|
limit = min(body.get("limit", 10), 50) # Enforce max limit
|
||||||
include_pca = body.get("include_pca", True)
|
include_pca = body.get("include_pca", True)
|
||||||
doc_types = body.get("doc_types") # Optional list of document types
|
doc_types = body.get("doc_types") # Optional list of document types
|
||||||
# ADR-027 Phase 2 path filter (files only); blank ⇒ no filter.
|
# ADR-027 Phase 2 path filter (files only); blank ⇒ no filter. Accept a
|
||||||
path_prefix = (body.get("path_prefix") or "").strip() or None
|
# path_prefixes list (multi-folder) alongside the legacy single
|
||||||
|
# path_prefix; normalize drops blanks and de-dupes.
|
||||||
|
_path_prefixes_raw = body.get("path_prefixes")
|
||||||
|
if isinstance(_path_prefixes_raw, list):
|
||||||
|
_path_prefixes_list = _path_prefixes_raw
|
||||||
|
elif isinstance(_path_prefixes_raw, str):
|
||||||
|
_path_prefixes_list = _path_prefixes_raw.split(",")
|
||||||
|
else:
|
||||||
|
# Ignore any other JSON shape (number, object, null) rather than
|
||||||
|
# blowing up on .split — the legacy path_prefix still applies.
|
||||||
|
_path_prefixes_list = []
|
||||||
|
path_prefixes = normalize_path_prefixes(
|
||||||
|
body.get("path_prefix"), _path_prefixes_list
|
||||||
|
)
|
||||||
# ADR-027 modified-date range filter. Accepts RFC 3339 / ISO 8601
|
# ADR-027 modified-date range filter. Accepts RFC 3339 / ISO 8601
|
||||||
# datetimes or Unix seconds; normalized to int Unix seconds. None ⇒ open.
|
# datetimes or Unix seconds; normalized to int Unix seconds. None ⇒ open.
|
||||||
try:
|
try:
|
||||||
@@ -507,7 +536,7 @@ async def vector_search(request: Request) -> JSONResponse:
|
|||||||
accessible_owners=owners,
|
accessible_owners=owners,
|
||||||
modified_after=modified_after,
|
modified_after=modified_after,
|
||||||
modified_before=modified_before,
|
modified_before=modified_before,
|
||||||
path_prefix=path_prefix,
|
path_prefixes=path_prefixes,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
# Sort merged results by score and limit
|
# Sort merged results by score and limit
|
||||||
@@ -522,7 +551,7 @@ async def vector_search(request: Request) -> JSONResponse:
|
|||||||
accessible_owners=owners,
|
accessible_owners=owners,
|
||||||
modified_after=modified_after,
|
modified_after=modified_after,
|
||||||
modified_before=modified_before,
|
modified_before=modified_before,
|
||||||
path_prefix=path_prefix,
|
path_prefixes=path_prefixes,
|
||||||
)
|
)
|
||||||
return results
|
return results
|
||||||
|
|
||||||
|
|||||||
@@ -33,7 +33,10 @@ from nextcloud_mcp_server.search import (
|
|||||||
BM25HybridSearchAlgorithm,
|
BM25HybridSearchAlgorithm,
|
||||||
SemanticSearchAlgorithm,
|
SemanticSearchAlgorithm,
|
||||||
)
|
)
|
||||||
from nextcloud_mcp_server.search.access_filter import list_accessible_owners
|
from nextcloud_mcp_server.search.access_filter import (
|
||||||
|
list_accessible_owners,
|
||||||
|
normalize_path_prefixes,
|
||||||
|
)
|
||||||
from nextcloud_mcp_server.search.context import (
|
from nextcloud_mcp_server.search.context import (
|
||||||
get_chunk_bbox_and_page_from_qdrant,
|
get_chunk_bbox_and_page_from_qdrant,
|
||||||
get_chunk_with_context,
|
get_chunk_with_context,
|
||||||
@@ -144,8 +147,14 @@ async def vector_visualization_search(request: Request) -> JSONResponse:
|
|||||||
doc_types_param = request.query_params.get("doc_types", "")
|
doc_types_param = request.query_params.get("doc_types", "")
|
||||||
doc_types = doc_types_param.split(",") if doc_types_param else None
|
doc_types = doc_types_param.split(",") if doc_types_param else None
|
||||||
|
|
||||||
# ADR-027 Phase 2 path filter (files only); blank ⇒ no filter.
|
# ADR-027 Phase 2 path filter (files only); blank ⇒ no filter. Accept a
|
||||||
path_prefix = (request.query_params.get("path_prefix") or "").strip() or None
|
# comma-separated path_prefixes list (multi-folder) plus the legacy single
|
||||||
|
# path_prefix; normalize_path_prefixes drops blanks and de-dupes.
|
||||||
|
path_prefix = request.query_params.get("path_prefix")
|
||||||
|
path_prefixes = normalize_path_prefixes(
|
||||||
|
path_prefix,
|
||||||
|
(request.query_params.get("path_prefixes") or "").split(","),
|
||||||
|
)
|
||||||
|
|
||||||
# Parse ADR-027 modified-date range filter. Accepts RFC 3339 / ISO 8601
|
# Parse ADR-027 modified-date range filter. Accepts RFC 3339 / ISO 8601
|
||||||
# datetimes or Unix seconds; normalized to int Unix seconds. Absent ⇒
|
# datetimes or Unix seconds; normalized to int Unix seconds. Absent ⇒
|
||||||
@@ -235,7 +244,7 @@ async def vector_visualization_search(request: Request) -> JSONResponse:
|
|||||||
accessible_owners=accessible_owners,
|
accessible_owners=accessible_owners,
|
||||||
modified_after=modified_after,
|
modified_after=modified_after,
|
||||||
modified_before=modified_before,
|
modified_before=modified_before,
|
||||||
path_prefix=path_prefix,
|
path_prefixes=path_prefixes,
|
||||||
)
|
)
|
||||||
all_results.extend(unverified_results)
|
all_results.extend(unverified_results)
|
||||||
else:
|
else:
|
||||||
@@ -258,7 +267,7 @@ async def vector_visualization_search(request: Request) -> JSONResponse:
|
|||||||
accessible_owners=accessible_owners,
|
accessible_owners=accessible_owners,
|
||||||
modified_after=modified_after,
|
modified_after=modified_after,
|
||||||
modified_before=modified_before,
|
modified_before=modified_before,
|
||||||
path_prefix=path_prefix,
|
path_prefixes=path_prefixes,
|
||||||
)
|
)
|
||||||
all_results.extend(unverified_results)
|
all_results.extend(unverified_results)
|
||||||
# Sort by score, then cap to the same limit*2 over-fetch budget
|
# Sort by score, then cap to the same limit*2 over-fetch budget
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ from __future__ import annotations
|
|||||||
import logging
|
import logging
|
||||||
import time
|
import time
|
||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
|
from collections.abc import Iterable
|
||||||
from typing import Any, Protocol
|
from typing import Any, Protocol
|
||||||
|
|
||||||
from qdrant_client.models import (
|
from qdrant_client.models import (
|
||||||
@@ -189,6 +190,43 @@ def build_ownership_filter(
|
|||||||
return Filter(should=conditions)
|
return Filter(should=conditions)
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_path_prefixes(
|
||||||
|
path_prefix: str | None = None,
|
||||||
|
path_prefixes: Iterable[str] | None = None,
|
||||||
|
) -> list[str]:
|
||||||
|
"""Merge the legacy single ``path_prefix`` and list ``path_prefixes`` into
|
||||||
|
one clean, de-duplicated list of folder filters.
|
||||||
|
|
||||||
|
Blank/whitespace entries are dropped (an empty UI field must mean "no
|
||||||
|
filter", not "match everything"), surrounding whitespace is stripped, and
|
||||||
|
order is preserved while removing duplicates. Accepting both inputs keeps
|
||||||
|
the pre-ADR-027-Phase-2 single-value contract working while callers migrate
|
||||||
|
to the multi-folder list.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
path_prefix: Legacy single folder filter (deprecated; folded into the
|
||||||
|
returned list).
|
||||||
|
path_prefixes: Zero or more folder filters.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Ordered, de-duplicated list of non-empty folder filters (possibly empty).
|
||||||
|
"""
|
||||||
|
raw: list[str] = []
|
||||||
|
if path_prefix:
|
||||||
|
raw.append(path_prefix)
|
||||||
|
if path_prefixes:
|
||||||
|
raw.extend(path_prefixes)
|
||||||
|
|
||||||
|
seen: set[str] = set()
|
||||||
|
cleaned: list[str] = []
|
||||||
|
for value in raw:
|
||||||
|
stripped = value.strip()
|
||||||
|
if stripped and stripped not in seen:
|
||||||
|
seen.add(stripped)
|
||||||
|
cleaned.append(stripped)
|
||||||
|
return cleaned
|
||||||
|
|
||||||
|
|
||||||
def build_base_filter_conditions(
|
def build_base_filter_conditions(
|
||||||
user_id: str,
|
user_id: str,
|
||||||
accessible_owners: list[str] | None = None,
|
accessible_owners: list[str] | None = None,
|
||||||
@@ -196,6 +234,7 @@ def build_base_filter_conditions(
|
|||||||
modified_after: int | None = None,
|
modified_after: int | None = None,
|
||||||
modified_before: int | None = None,
|
modified_before: int | None = None,
|
||||||
path_prefix: str | None = None,
|
path_prefix: str | None = None,
|
||||||
|
path_prefixes: list[str] | None = None,
|
||||||
) -> list[Condition]:
|
) -> list[Condition]:
|
||||||
"""Build the common ``must`` conditions shared by every search algorithm.
|
"""Build the common ``must`` conditions shared by every search algorithm.
|
||||||
|
|
||||||
@@ -211,7 +250,10 @@ def build_base_filter_conditions(
|
|||||||
2. ``build_ownership_filter(...)`` — ACL-aware ``owner_id``/``user_id`` scope.
|
2. ``build_ownership_filter(...)`` — ACL-aware ``owner_id``/``user_id`` scope.
|
||||||
3. ``doc_type`` exact match — only when ``doc_type`` is truthy.
|
3. ``doc_type`` exact match — only when ``doc_type`` is truthy.
|
||||||
4. ``modified_at`` range — only when at least one bound is given.
|
4. ``modified_at`` range — only when at least one bound is given.
|
||||||
5. ``file_path`` text match — only when ``path_prefix`` is given.
|
5. ``file_path`` text match — only when a path filter is given. One folder
|
||||||
|
adds a single ``MatchText`` to ``must``; multiple folders are OR-ed via a
|
||||||
|
nested ``Filter(should=[...])`` so a file under *any* selected folder
|
||||||
|
matches.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
user_id: Querying user.
|
user_id: Querying user.
|
||||||
@@ -220,10 +262,13 @@ def build_base_filter_conditions(
|
|||||||
doc_type: Optional single document-type filter.
|
doc_type: Optional single document-type filter.
|
||||||
modified_after: Inclusive lower bound on ``modified_at`` (Unix seconds).
|
modified_after: Inclusive lower bound on ``modified_at`` (Unix seconds).
|
||||||
modified_before: Inclusive upper bound on ``modified_at`` (Unix seconds).
|
modified_before: Inclusive upper bound on ``modified_at`` (Unix seconds).
|
||||||
path_prefix: Optional folder/path filter on the ``file_path`` payload
|
path_prefix: Deprecated single folder/path filter; folded into
|
||||||
field (ADR-027 Phase 2). Implemented with ``MatchText`` against the
|
``path_prefixes``. Kept for backward compatibility.
|
||||||
text-indexed ``file_path``. ``file_path`` is only written for
|
path_prefixes: Optional folder/path filters on the ``file_path`` payload
|
||||||
``doc_type == "file"`` points, so a non-empty ``path_prefix``
|
field (ADR-027 Phase 2). Each is implemented with ``MatchText``
|
||||||
|
against the text-indexed ``file_path`` and multiple folders are
|
||||||
|
OR-ed together. ``file_path`` is only written for
|
||||||
|
``doc_type == "file"`` points, so any non-empty path filter
|
||||||
implicitly restricts results to files. NOTE the match semantics
|
implicitly restricts results to files. NOTE the match semantics
|
||||||
differ by backend: server Qdrant tokenizes (AND-of-tokens, so
|
differ by backend: server Qdrant tokenizes (AND-of-tokens, so
|
||||||
``"/Projects/Reports"`` matches files whose path contains both the
|
``"/Projects/Reports"`` matches files whose path contains both the
|
||||||
@@ -255,9 +300,23 @@ def build_base_filter_conditions(
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
if path_prefix:
|
# One folder ⇒ a single ``must`` condition (the original Phase 2 shape).
|
||||||
|
# Multiple folders ⇒ OR them in a nested ``Filter(should=...)`` so a file
|
||||||
|
# under any selected folder matches, while still AND-ing against the other
|
||||||
|
# ``must`` conditions (ACL, doc_type, date).
|
||||||
|
folders = normalize_path_prefixes(path_prefix, path_prefixes)
|
||||||
|
if len(folders) == 1:
|
||||||
conditions.append(
|
conditions.append(
|
||||||
FieldCondition(key="file_path", match=MatchText(text=path_prefix))
|
FieldCondition(key="file_path", match=MatchText(text=folders[0]))
|
||||||
|
)
|
||||||
|
elif len(folders) > 1:
|
||||||
|
conditions.append(
|
||||||
|
Filter(
|
||||||
|
should=[
|
||||||
|
FieldCondition(key="file_path", match=MatchText(text=folder))
|
||||||
|
for folder in folders
|
||||||
|
]
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
return conditions
|
return conditions
|
||||||
|
|||||||
@@ -300,6 +300,7 @@ class SearchAlgorithm(ABC):
|
|||||||
modified_after: int | None = None,
|
modified_after: int | None = None,
|
||||||
modified_before: int | None = None,
|
modified_before: int | None = None,
|
||||||
path_prefix: str | None = None,
|
path_prefix: str | None = None,
|
||||||
|
path_prefixes: list[str] | None = None,
|
||||||
**kwargs: Any,
|
**kwargs: Any,
|
||||||
) -> list[SearchResult]:
|
) -> list[SearchResult]:
|
||||||
"""Execute search with the given parameters.
|
"""Execute search with the given parameters.
|
||||||
@@ -321,10 +322,13 @@ class SearchAlgorithm(ABC):
|
|||||||
``accessible_owners`` (ADR-027). ``None`` ⇒ open-ended.
|
``accessible_owners`` (ADR-027). ``None`` ⇒ open-ended.
|
||||||
modified_before: Optional inclusive upper bound on ``modified_at``
|
modified_before: Optional inclusive upper bound on ``modified_at``
|
||||||
(Unix seconds, UTC). ``None`` ⇒ open-ended.
|
(Unix seconds, UTC). ``None`` ⇒ open-ended.
|
||||||
path_prefix: Optional folder/path filter on the ``file_path`` payload
|
path_prefix: Deprecated single folder/path filter; folded into
|
||||||
field (ADR-027 Phase 2). Only ``doc_type == "file"`` points carry
|
``path_prefixes``. Kept for backward compatibility.
|
||||||
``file_path``, so a non-empty value implicitly restricts results
|
path_prefixes: Optional folder/path filters on the ``file_path``
|
||||||
to files. ``None`` ⇒ no path filter.
|
payload field (ADR-027 Phase 2), OR-ed together. Only
|
||||||
|
``doc_type == "file"`` points carry ``file_path``, so any
|
||||||
|
non-empty value implicitly restricts results to files. ``None``
|
||||||
|
or empty ⇒ no path filter.
|
||||||
**kwargs: Algorithm-specific parameters
|
**kwargs: Algorithm-specific parameters
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
|
|||||||
@@ -75,6 +75,7 @@ class BM25HybridSearchAlgorithm(SearchAlgorithm):
|
|||||||
modified_after: int | None = None,
|
modified_after: int | None = None,
|
||||||
modified_before: int | None = None,
|
modified_before: int | None = None,
|
||||||
path_prefix: str | None = None,
|
path_prefix: str | None = None,
|
||||||
|
path_prefixes: list[str] | None = None,
|
||||||
**kwargs: Any,
|
**kwargs: Any,
|
||||||
) -> list[SearchResult]:
|
) -> list[SearchResult]:
|
||||||
"""
|
"""
|
||||||
@@ -100,8 +101,11 @@ class BM25HybridSearchAlgorithm(SearchAlgorithm):
|
|||||||
seconds, UTC); ``None`` ⇒ open-ended (ADR-027).
|
seconds, UTC); ``None`` ⇒ open-ended (ADR-027).
|
||||||
modified_before: Inclusive upper bound on ``modified_at`` (Unix
|
modified_before: Inclusive upper bound on ``modified_at`` (Unix
|
||||||
seconds, UTC); ``None`` ⇒ open-ended (ADR-027).
|
seconds, UTC); ``None`` ⇒ open-ended (ADR-027).
|
||||||
path_prefix: Folder/path filter on ``file_path`` (files only);
|
path_prefix: Deprecated single folder filter; folded into
|
||||||
``None`` ⇒ no path filter (ADR-027 Phase 2).
|
``path_prefixes`` (ADR-027 Phase 2).
|
||||||
|
path_prefixes: Folder/path filters on ``file_path`` (files only),
|
||||||
|
OR-ed together; ``None``/empty ⇒ no path filter (ADR-027
|
||||||
|
Phase 2).
|
||||||
**kwargs: Additional parameters (score_threshold override)
|
**kwargs: Additional parameters (score_threshold override)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
@@ -152,6 +156,7 @@ class BM25HybridSearchAlgorithm(SearchAlgorithm):
|
|||||||
modified_after=modified_after,
|
modified_after=modified_after,
|
||||||
modified_before=modified_before,
|
modified_before=modified_before,
|
||||||
path_prefix=path_prefix,
|
path_prefix=path_prefix,
|
||||||
|
path_prefixes=path_prefixes,
|
||||||
)
|
)
|
||||||
|
|
||||||
query_filter = Filter(must=filter_conditions)
|
query_filter = Filter(must=filter_conditions)
|
||||||
|
|||||||
@@ -55,6 +55,7 @@ class SemanticSearchAlgorithm(SearchAlgorithm):
|
|||||||
modified_after: int | None = None,
|
modified_after: int | None = None,
|
||||||
modified_before: int | None = None,
|
modified_before: int | None = None,
|
||||||
path_prefix: str | None = None,
|
path_prefix: str | None = None,
|
||||||
|
path_prefixes: list[str] | None = None,
|
||||||
**kwargs: Any,
|
**kwargs: Any,
|
||||||
) -> list[SearchResult]:
|
) -> list[SearchResult]:
|
||||||
"""Execute semantic search using vector similarity.
|
"""Execute semantic search using vector similarity.
|
||||||
@@ -79,8 +80,11 @@ class SemanticSearchAlgorithm(SearchAlgorithm):
|
|||||||
seconds, UTC); ``None`` ⇒ open-ended (ADR-027).
|
seconds, UTC); ``None`` ⇒ open-ended (ADR-027).
|
||||||
modified_before: Inclusive upper bound on ``modified_at`` (Unix
|
modified_before: Inclusive upper bound on ``modified_at`` (Unix
|
||||||
seconds, UTC); ``None`` ⇒ open-ended (ADR-027).
|
seconds, UTC); ``None`` ⇒ open-ended (ADR-027).
|
||||||
path_prefix: Folder/path filter on ``file_path`` (files only);
|
path_prefix: Deprecated single folder filter; folded into
|
||||||
``None`` ⇒ no path filter (ADR-027 Phase 2).
|
``path_prefixes`` (ADR-027 Phase 2).
|
||||||
|
path_prefixes: Folder/path filters on ``file_path`` (files only),
|
||||||
|
OR-ed together; ``None``/empty ⇒ no path filter (ADR-027
|
||||||
|
Phase 2).
|
||||||
**kwargs:
|
**kwargs:
|
||||||
- score_threshold (float): override the instance default
|
- score_threshold (float): override the instance default
|
||||||
|
|
||||||
@@ -122,6 +126,7 @@ class SemanticSearchAlgorithm(SearchAlgorithm):
|
|||||||
modified_after=modified_after,
|
modified_after=modified_after,
|
||||||
modified_before=modified_before,
|
modified_before=modified_before,
|
||||||
path_prefix=path_prefix,
|
path_prefix=path_prefix,
|
||||||
|
path_prefixes=path_prefixes,
|
||||||
)
|
)
|
||||||
|
|
||||||
# ACL pre-filter (design §11), opt-in via ACL_PREFILTER_ENABLED and OFF
|
# ACL pre-filter (design §11), opt-in via ACL_PREFILTER_ENABLED and OFF
|
||||||
|
|||||||
@@ -32,7 +32,10 @@ from nextcloud_mcp_server.models.semantic import (
|
|||||||
from nextcloud_mcp_server.observability.metrics import (
|
from nextcloud_mcp_server.observability.metrics import (
|
||||||
instrument_tool,
|
instrument_tool,
|
||||||
)
|
)
|
||||||
from nextcloud_mcp_server.search.access_filter import list_accessible_owners
|
from nextcloud_mcp_server.search.access_filter import (
|
||||||
|
list_accessible_owners,
|
||||||
|
normalize_path_prefixes,
|
||||||
|
)
|
||||||
from nextcloud_mcp_server.search.bm25_hybrid import BM25HybridSearchAlgorithm
|
from nextcloud_mcp_server.search.bm25_hybrid import BM25HybridSearchAlgorithm
|
||||||
from nextcloud_mcp_server.search.context import get_chunk_with_context
|
from nextcloud_mcp_server.search.context import get_chunk_with_context
|
||||||
from nextcloud_mcp_server.search.verification import verify_search_results
|
from nextcloud_mcp_server.search.verification import verify_search_results
|
||||||
@@ -88,6 +91,7 @@ def configure_semantic_tools(mcp: FastMCP):
|
|||||||
str | None,
|
str | None,
|
||||||
Field(
|
Field(
|
||||||
description=(
|
description=(
|
||||||
|
"Deprecated single-folder filter; prefer path_prefixes. "
|
||||||
"Restrict to files under this folder/path "
|
"Restrict to files under this folder/path "
|
||||||
"(e.g. '/Projects/Reports'). Matches the file_path of "
|
"(e.g. '/Projects/Reports'). Matches the file_path of "
|
||||||
"indexed files only, so setting it implicitly limits "
|
"indexed files only, so setting it implicitly limits "
|
||||||
@@ -95,6 +99,18 @@ def configure_semantic_tools(mcp: FastMCP):
|
|||||||
),
|
),
|
||||||
),
|
),
|
||||||
] = None,
|
] = None,
|
||||||
|
path_prefixes: Annotated[
|
||||||
|
list[str] | None,
|
||||||
|
Field(
|
||||||
|
description=(
|
||||||
|
"Restrict to files under any of these folders/paths "
|
||||||
|
"(e.g. ['/Projects/Reports', '/Shared/Specs']). Folders are "
|
||||||
|
"OR-ed together. Matches the file_path of indexed files "
|
||||||
|
"only, so setting it implicitly limits results to files. "
|
||||||
|
"None or empty = no path filter."
|
||||||
|
),
|
||||||
|
),
|
||||||
|
] = None,
|
||||||
) -> SemanticSearchResponse:
|
) -> SemanticSearchResponse:
|
||||||
"""
|
"""
|
||||||
Search Nextcloud content using BM25 hybrid search with cross-app support.
|
Search Nextcloud content using BM25 hybrid search with cross-app support.
|
||||||
@@ -127,9 +143,12 @@ def configure_semantic_tools(mcp: FastMCP):
|
|||||||
modified_before: Only return documents whose last-modified time is at or before this
|
modified_before: Only return documents whose last-modified time is at or before this
|
||||||
instant. Same formats as modified_after. None = no upper bound (default). Must be
|
instant. Same formats as modified_after. None = no upper bound (default). Must be
|
||||||
>= modified_after when both are supplied.
|
>= modified_after when both are supplied.
|
||||||
path_prefix: Restrict to files under this folder/path (e.g. "/Projects/Reports").
|
path_prefix: Deprecated single-folder filter; prefer path_prefixes. Restrict to files
|
||||||
Matches the file_path of indexed files only — setting it implicitly limits results
|
under this folder/path (e.g. "/Projects/Reports"). Folded into path_prefixes.
|
||||||
to files. None = no path filter (default).
|
path_prefixes: Restrict to files under any of these folders/paths (OR-ed), e.g.
|
||||||
|
["/Projects/Reports", "/Shared/Specs"]. Matches the file_path of indexed files
|
||||||
|
only — setting it implicitly limits results to files. None/empty = no path filter
|
||||||
|
(default).
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
SemanticSearchResponse with matching documents ranked by fusion scores.
|
SemanticSearchResponse with matching documents ranked by fusion scores.
|
||||||
@@ -200,11 +219,10 @@ def configure_semantic_tools(mcp: FastMCP):
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
# Treat a blank/whitespace path_prefix as "no filter" so an empty UI
|
# Merge the legacy single path_prefix and the path_prefixes list into one
|
||||||
# field doesn't filter out every result (ADR-027 Phase 2).
|
# cleaned list, dropping blank/whitespace entries so an empty UI field
|
||||||
path_prefix = path_prefix.strip() if path_prefix else None
|
# doesn't filter out every result (ADR-027 Phase 2).
|
||||||
if not path_prefix:
|
folder_prefixes = normalize_path_prefixes(path_prefix, path_prefixes)
|
||||||
path_prefix = None
|
|
||||||
|
|
||||||
# Expand the caller's identity to every owner whose content they
|
# Expand the caller's identity to every owner whose content they
|
||||||
# have read access to via Nextcloud shares. Lets a user find files
|
# have read access to via Nextcloud shares. Lets a user find files
|
||||||
@@ -252,7 +270,7 @@ def configure_semantic_tools(mcp: FastMCP):
|
|||||||
accessible_owners=accessible_owners,
|
accessible_owners=accessible_owners,
|
||||||
modified_after=modified_after_ts,
|
modified_after=modified_after_ts,
|
||||||
modified_before=modified_before_ts,
|
modified_before=modified_before_ts,
|
||||||
path_prefix=path_prefix,
|
path_prefixes=folder_prefixes,
|
||||||
)
|
)
|
||||||
all_results.extend(unverified_results)
|
all_results.extend(unverified_results)
|
||||||
else:
|
else:
|
||||||
@@ -280,7 +298,7 @@ def configure_semantic_tools(mcp: FastMCP):
|
|||||||
accessible_owners=accessible_owners,
|
accessible_owners=accessible_owners,
|
||||||
modified_after=modified_after_ts,
|
modified_after=modified_after_ts,
|
||||||
modified_before=modified_before_ts,
|
modified_before=modified_before_ts,
|
||||||
path_prefix=path_prefix,
|
path_prefixes=folder_prefixes,
|
||||||
)
|
)
|
||||||
all_results.extend(unverified_results)
|
all_results.extend(unverified_results)
|
||||||
|
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ from __future__ import annotations
|
|||||||
from unittest.mock import AsyncMock
|
from unittest.mock import AsyncMock
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from qdrant_client.models import FieldCondition, MatchText, Range
|
from qdrant_client.models import FieldCondition, Filter, MatchText, Range
|
||||||
|
|
||||||
from nextcloud_mcp_server.search import access_filter
|
from nextcloud_mcp_server.search import access_filter
|
||||||
from nextcloud_mcp_server.search.access_filter import (
|
from nextcloud_mcp_server.search.access_filter import (
|
||||||
@@ -13,6 +13,7 @@ from nextcloud_mcp_server.search.access_filter import (
|
|||||||
build_ownership_filter,
|
build_ownership_filter,
|
||||||
clear_accessible_owners_cache,
|
clear_accessible_owners_cache,
|
||||||
list_accessible_owners,
|
list_accessible_owners,
|
||||||
|
normalize_path_prefixes,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -270,6 +271,67 @@ class TestBuildBaseFilterConditions:
|
|||||||
isinstance(c, FieldCondition) and c.key == "file_path" for c in conditions
|
isinstance(c, FieldCondition) and c.key == "file_path" for c in conditions
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _path_should_texts(conditions) -> set[str] | None:
|
||||||
|
"""Return the file_path texts from the nested OR Filter, or None if no
|
||||||
|
such filter is present. Ignores the ownership Filter (which ORs
|
||||||
|
user_id/owner_id, not file_path)."""
|
||||||
|
for cond in conditions:
|
||||||
|
if not isinstance(cond, Filter) or not cond.should:
|
||||||
|
continue
|
||||||
|
if all(
|
||||||
|
isinstance(c, FieldCondition) and c.key == "file_path"
|
||||||
|
for c in cond.should
|
||||||
|
):
|
||||||
|
return {
|
||||||
|
c.match.text
|
||||||
|
for c in cond.should
|
||||||
|
if isinstance(c, FieldCondition) and isinstance(c.match, MatchText)
|
||||||
|
}
|
||||||
|
return None
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_multiple_path_prefixes_or_in_nested_should(self) -> None:
|
||||||
|
# Two+ folders must OR together: a single nested Filter(should=[...]) is
|
||||||
|
# appended (not two must conditions, which would AND and match nothing).
|
||||||
|
conditions = build_base_filter_conditions(
|
||||||
|
"alice", None, path_prefixes=["/Projects", "/Archive"]
|
||||||
|
)
|
||||||
|
assert self._path_should_texts(conditions) == {"/Projects", "/Archive"}
|
||||||
|
# No bare file_path FieldCondition in must for the multi-folder case.
|
||||||
|
assert not any(
|
||||||
|
isinstance(c, FieldCondition) and c.key == "file_path" for c in conditions
|
||||||
|
)
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_path_prefix_and_path_prefixes_merge_and_dedupe(self) -> None:
|
||||||
|
# Legacy single + list inputs merge; duplicates collapse so a folder
|
||||||
|
# passed both ways yields two distinct conditions, not three.
|
||||||
|
conditions = build_base_filter_conditions(
|
||||||
|
"alice",
|
||||||
|
None,
|
||||||
|
path_prefix="/Projects",
|
||||||
|
path_prefixes=["/Projects", "/Archive"],
|
||||||
|
)
|
||||||
|
assert self._path_should_texts(conditions) == {"/Projects", "/Archive"}
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_single_effective_prefix_uses_flat_must_condition(self) -> None:
|
||||||
|
# When dedupe/blank-stripping leaves exactly one folder, keep the
|
||||||
|
# original flat MatchText in must rather than a one-element should.
|
||||||
|
conditions = build_base_filter_conditions(
|
||||||
|
"alice", None, path_prefixes=["/Projects", " ", "/Projects"]
|
||||||
|
)
|
||||||
|
# The only nested Filter should be ownership, never a path OR.
|
||||||
|
assert self._path_should_texts(conditions) is None
|
||||||
|
path_conds = [
|
||||||
|
c
|
||||||
|
for c in conditions
|
||||||
|
if isinstance(c, FieldCondition) and c.key == "file_path"
|
||||||
|
]
|
||||||
|
assert len(path_conds) == 1
|
||||||
|
assert path_conds[0].match.text == "/Projects"
|
||||||
|
|
||||||
@pytest.mark.unit
|
@pytest.mark.unit
|
||||||
def test_all_filters_compose(self) -> None:
|
def test_all_filters_compose(self) -> None:
|
||||||
# placeholder + ownership + doc_type + modified_at range + file_path = 5.
|
# placeholder + ownership + doc_type + modified_at range + file_path = 5.
|
||||||
@@ -282,3 +344,18 @@ class TestBuildBaseFilterConditions:
|
|||||||
path_prefix="/Projects",
|
path_prefix="/Projects",
|
||||||
)
|
)
|
||||||
assert len(conditions) == 5
|
assert len(conditions) == 5
|
||||||
|
|
||||||
|
|
||||||
|
class TestNormalizePathPrefixes:
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_empty_inputs_return_empty_list(self) -> None:
|
||||||
|
assert normalize_path_prefixes(None, None) == []
|
||||||
|
assert normalize_path_prefixes("", []) == []
|
||||||
|
assert normalize_path_prefixes(" ", ["", " "]) == []
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_strips_dedupes_and_preserves_order(self) -> None:
|
||||||
|
result = normalize_path_prefixes(
|
||||||
|
" /Projects ", ["/Archive", "/Projects", " ", "/Specs"]
|
||||||
|
)
|
||||||
|
assert result == ["/Projects", "/Archive", "/Specs"]
|
||||||
|
|||||||
Reference in New Issue
Block a user