diff --git a/docs/ADR-027-rich-search-filters.md b/docs/ADR-027-rich-search-filters.md index d3721f37..d8acd28c 100644 --- a/docs/ADR-027-rich-search-filters.md +++ b/docs/ADR-027-rich-search-filters.md @@ -1,6 +1,6 @@ # ADR-027: Rich Search Filters for Semantic Search -**Status**: Accepted (Phase 1 implemented; Phases 2–3 deferred) +**Status**: Accepted (Phases 1–2 implemented; Phase 3 deferred) **Date**: 2026-06-02 **Depends On**: ADR-012 (Unified Multi-Algorithm Search), ADR-014 (BM25 Search), ADR-019 (Verify-on-Read for Semantic Search) **Tracking**: Astrolabe Cloud POC Deck card #177 @@ -68,7 +68,7 @@ Filters can only be applied to fields that exist in the Qdrant payload (built in |---|---|---|---| | Modified-date range | `modified_at` | `int` (Unix ts) | ✅ **Ready** — value present on every point; needs a payload index (added in Phase 1, no content re-index) | | Document type | `doc_type` | keyword-indexed `str` | ✅ Implemented | -| Directory / path | `file_path` (files only) | `str` | ⚠️ Stored but **not keyword-indexed** — prefix/match needs a payload index | +| Directory / path | `file_path` (files only) | `str` | ✅ **Ready (Phase 2)** — value present on file points; TEXT payload index added (no content re-index), filtered with `MatchText` | | Tags | — | — | ❌ **Not indexed** — no `tags` field is written during scanning | | Category (notes) | — | — | ❌ Not in payload — fetched from the Notes API at verify time only | @@ -186,11 +186,19 @@ express: it on the answer path. When demand appears it can be threaded through using exactly the same parameter + `parse_modified_timestamp` pattern; doing it now would add an unused parameter and widen the change for no user-visible gain. -- **Phase 2 — directory / path.** Create a Qdrant payload index on `file_path`, add a `path_prefix` - parameter, and add an `NcFilePicker` folder chooser. Scoped to `doc_type == "file"`: the frontend - must **hide/disable the path filter unless the file doc type is selected**, because `file_path` - is only written to the payload for `doc_type == "file"` (`processor.py`) — a path condition on - any other type matches nothing and silently returns zero results. +- **Phase 2 — directory / path (implemented).** Add a `path_prefix` parameter threaded through the + same shared contract (`build_base_filter_conditions` → `FieldCondition(key="file_path", + match=MatchText(text=path_prefix))`), and a `file_path` **TEXT** payload index in + `_PAYLOAD_INDEX_FIELDS` (no content re-index — `file_path` is already on every file point). + `MatchText` semantics differ by backend: **server Qdrant** tokenizes (AND-of-tokens, so + `/Projects/Reports` matches files whose path contains both the `Projects` and `Reports` tokens), + while **local/embedded qdrant-client** matches by substring containment — both serve folder + scoping, neither is a strict left-anchored prefix (a future strict-prefix would need an indexed + ancestor-path array, i.e. a re-index, so it stays out of Phase 2). Because `file_path` is only + written for `doc_type == "file"`, a non-empty `path_prefix` implicitly restricts results to + files; the frontend uses a native path text input enabled only when the **Files** doc type is in + scope (rather than `NcFilePicker`, to avoid a hard `@nextcloud/vue` component-version dependency — + same rationale as the date inputs). A blank value is treated as "no filter". - **Phase 3 — tags (and optionally category).** Add a `tags: list[str]` payload field in `processor.py`, propagate Nextcloud system tags during scanning, trigger a re-index, then wire `NcSelectTags` (`MatchAny` over tags). Re-index cost lives here, isolated from the cheap wins. diff --git a/nextcloud_mcp_server/api/visualization.py b/nextcloud_mcp_server/api/visualization.py index 17a37e5b..454984ea 100644 --- a/nextcloud_mcp_server/api/visualization.py +++ b/nextcloud_mcp_server/api/visualization.py @@ -225,6 +225,8 @@ async def unified_search(request: Request) -> JSONResponse: include_pca = body.get("include_pca", False) include_chunks = body.get("include_chunks", True) doc_types = body.get("doc_types") # Optional filter + # ADR-027 Phase 2 path filter (files only); blank ⇒ no filter. + path_prefix = (body.get("path_prefix") or "").strip() or None if not query: return JSONResponse({"results": [], "total_found": 0}) @@ -266,6 +268,7 @@ async def unified_search(request: Request) -> JSONResponse: accessible_owners=owners, modified_after=modified_after, modified_before=modified_before, + path_prefix=path_prefix, ) ) # Sort, then cap to a fixed over-fetch budget before the result @@ -285,6 +288,7 @@ async def unified_search(request: Request) -> JSONResponse: accessible_owners=owners, modified_after=modified_after, modified_before=modified_before, + path_prefix=path_prefix, ) return results @@ -436,6 +440,8 @@ async def vector_search(request: Request) -> JSONResponse: limit = min(body.get("limit", 10), 50) # Enforce max limit include_pca = body.get("include_pca", True) doc_types = body.get("doc_types") # Optional list of document types + # ADR-027 Phase 2 path filter (files only); blank ⇒ no filter. + path_prefix = (body.get("path_prefix") or "").strip() or None # ADR-027 modified-date range filter. Accepts RFC 3339 / ISO 8601 # datetimes or Unix seconds; normalized to int Unix seconds. None ⇒ open. try: @@ -501,6 +507,7 @@ async def vector_search(request: Request) -> JSONResponse: accessible_owners=owners, modified_after=modified_after, modified_before=modified_before, + path_prefix=path_prefix, ) ) # Sort merged results by score and limit @@ -515,6 +522,7 @@ async def vector_search(request: Request) -> JSONResponse: accessible_owners=owners, modified_after=modified_after, modified_before=modified_before, + path_prefix=path_prefix, ) return results diff --git a/nextcloud_mcp_server/auth/viz_routes.py b/nextcloud_mcp_server/auth/viz_routes.py index 57e9b6bb..82ed853d 100644 --- a/nextcloud_mcp_server/auth/viz_routes.py +++ b/nextcloud_mcp_server/auth/viz_routes.py @@ -144,6 +144,9 @@ async def vector_visualization_search(request: Request) -> JSONResponse: doc_types_param = request.query_params.get("doc_types", "") doc_types = doc_types_param.split(",") if doc_types_param else None + # ADR-027 Phase 2 path filter (files only); blank ⇒ no filter. + path_prefix = (request.query_params.get("path_prefix") or "").strip() or None + # Parse ADR-027 modified-date range filter. Accepts RFC 3339 / ISO 8601 # datetimes or Unix seconds; normalized to int Unix seconds. Absent ⇒ # open-ended. Unparseable input or an inverted range returns 400. @@ -232,6 +235,7 @@ async def vector_visualization_search(request: Request) -> JSONResponse: accessible_owners=accessible_owners, modified_after=modified_after, modified_before=modified_before, + path_prefix=path_prefix, ) all_results.extend(unverified_results) else: @@ -254,6 +258,7 @@ async def vector_visualization_search(request: Request) -> JSONResponse: accessible_owners=accessible_owners, modified_after=modified_after, modified_before=modified_before, + path_prefix=path_prefix, ) all_results.extend(unverified_results) # Sort by score, then cap to the same limit*2 over-fetch budget diff --git a/nextcloud_mcp_server/search/access_filter.py b/nextcloud_mcp_server/search/access_filter.py index 35d2866b..bb94f295 100644 --- a/nextcloud_mcp_server/search/access_filter.py +++ b/nextcloud_mcp_server/search/access_filter.py @@ -34,6 +34,7 @@ from qdrant_client.models import ( FieldCondition, Filter, MatchAny, + MatchText, MatchValue, Range, ) @@ -194,6 +195,7 @@ def build_base_filter_conditions( doc_type: str | None = None, modified_after: int | None = None, modified_before: int | None = None, + path_prefix: str | None = None, ) -> list[Condition]: """Build the common ``must`` conditions shared by every search algorithm. @@ -209,6 +211,7 @@ def build_base_filter_conditions( 2. ``build_ownership_filter(...)`` — ACL-aware ``owner_id``/``user_id`` scope. 3. ``doc_type`` exact match — only when ``doc_type`` is truthy. 4. ``modified_at`` range — only when at least one bound is given. + 5. ``file_path`` text match — only when ``path_prefix`` is given. Args: user_id: Querying user. @@ -217,6 +220,16 @@ def build_base_filter_conditions( doc_type: Optional single document-type filter. modified_after: Inclusive lower bound on ``modified_at`` (Unix seconds). modified_before: Inclusive upper bound on ``modified_at`` (Unix seconds). + path_prefix: Optional folder/path filter on the ``file_path`` payload + field (ADR-027 Phase 2). Implemented with ``MatchText`` against the + text-indexed ``file_path``. ``file_path`` is only written for + ``doc_type == "file"`` points, so a non-empty ``path_prefix`` + implicitly restricts results to files. NOTE the match semantics + differ by backend: server Qdrant tokenizes (AND-of-tokens, so + ``"/Projects/Reports"`` matches files whose path contains both the + ``Projects`` and ``Reports`` tokens), while the local/embedded + qdrant-client matches by substring containment. Both serve folder + scoping; neither is a strict left-anchored prefix. Returns: A list of Qdrant ``Condition`` objects for a parent ``must`` clause. @@ -242,4 +255,9 @@ def build_base_filter_conditions( ) ) + if path_prefix: + conditions.append( + FieldCondition(key="file_path", match=MatchText(text=path_prefix)) + ) + return conditions diff --git a/nextcloud_mcp_server/search/algorithms.py b/nextcloud_mcp_server/search/algorithms.py index ab168fe7..5d43572b 100644 --- a/nextcloud_mcp_server/search/algorithms.py +++ b/nextcloud_mcp_server/search/algorithms.py @@ -291,6 +291,7 @@ class SearchAlgorithm(ABC): accessible_owners: list[str] | None = None, modified_after: int | None = None, modified_before: int | None = None, + path_prefix: str | None = None, **kwargs: Any, ) -> list[SearchResult]: """Execute search with the given parameters. @@ -312,6 +313,10 @@ class SearchAlgorithm(ABC): ``accessible_owners`` (ADR-027). ``None`` ⇒ open-ended. modified_before: Optional inclusive upper bound on ``modified_at`` (Unix seconds, UTC). ``None`` ⇒ open-ended. + path_prefix: Optional folder/path filter on the ``file_path`` payload + field (ADR-027 Phase 2). Only ``doc_type == "file"`` points carry + ``file_path``, so a non-empty value implicitly restricts results + to files. ``None`` ⇒ no path filter. **kwargs: Algorithm-specific parameters Returns: diff --git a/nextcloud_mcp_server/search/bm25_hybrid.py b/nextcloud_mcp_server/search/bm25_hybrid.py index e313aa2d..f6d0e21e 100644 --- a/nextcloud_mcp_server/search/bm25_hybrid.py +++ b/nextcloud_mcp_server/search/bm25_hybrid.py @@ -74,6 +74,7 @@ class BM25HybridSearchAlgorithm(SearchAlgorithm): accessible_owners: list[str] | None = None, modified_after: int | None = None, modified_before: int | None = None, + path_prefix: str | None = None, **kwargs: Any, ) -> list[SearchResult]: """ @@ -99,6 +100,8 @@ class BM25HybridSearchAlgorithm(SearchAlgorithm): seconds, UTC); ``None`` ⇒ open-ended (ADR-027). modified_before: Inclusive upper bound on ``modified_at`` (Unix seconds, UTC); ``None`` ⇒ open-ended (ADR-027). + path_prefix: Folder/path filter on ``file_path`` (files only); + ``None`` ⇒ no path filter (ADR-027 Phase 2). **kwargs: Additional parameters (score_threshold override) Returns: @@ -148,6 +151,7 @@ class BM25HybridSearchAlgorithm(SearchAlgorithm): doc_type=doc_type, modified_after=modified_after, modified_before=modified_before, + path_prefix=path_prefix, ) query_filter = Filter(must=filter_conditions) diff --git a/nextcloud_mcp_server/search/semantic.py b/nextcloud_mcp_server/search/semantic.py index 46fdea49..d4ba755a 100644 --- a/nextcloud_mcp_server/search/semantic.py +++ b/nextcloud_mcp_server/search/semantic.py @@ -54,6 +54,7 @@ class SemanticSearchAlgorithm(SearchAlgorithm): accessible_owners: list[str] | None = None, modified_after: int | None = None, modified_before: int | None = None, + path_prefix: str | None = None, **kwargs: Any, ) -> list[SearchResult]: """Execute semantic search using vector similarity. @@ -78,6 +79,8 @@ class SemanticSearchAlgorithm(SearchAlgorithm): seconds, UTC); ``None`` ⇒ open-ended (ADR-027). modified_before: Inclusive upper bound on ``modified_at`` (Unix seconds, UTC); ``None`` ⇒ open-ended (ADR-027). + path_prefix: Folder/path filter on ``file_path`` (files only); + ``None`` ⇒ no path filter (ADR-027 Phase 2). **kwargs: - score_threshold (float): override the instance default @@ -118,6 +121,7 @@ class SemanticSearchAlgorithm(SearchAlgorithm): doc_type=doc_type, modified_after=modified_after, modified_before=modified_before, + path_prefix=path_prefix, ) # ACL pre-filter (design §11), opt-in via ACL_PREFILTER_ENABLED and OFF diff --git a/nextcloud_mcp_server/server/semantic.py b/nextcloud_mcp_server/server/semantic.py index abf37fe8..26e61967 100644 --- a/nextcloud_mcp_server/server/semantic.py +++ b/nextcloud_mcp_server/server/semantic.py @@ -84,6 +84,17 @@ def configure_semantic_tools(mcp: FastMCP): ), ), ] = None, + path_prefix: Annotated[ + str | None, + Field( + description=( + "Restrict to files under this folder/path " + "(e.g. '/Projects/Reports'). Matches the file_path of " + "indexed files only, so setting it implicitly limits " + "results to files. None = no path filter." + ), + ), + ] = None, ) -> SemanticSearchResponse: """ Search Nextcloud content using BM25 hybrid search with cross-app support. @@ -116,6 +127,9 @@ def configure_semantic_tools(mcp: FastMCP): modified_before: Only return documents whose last-modified time is at or before this instant. Same formats as modified_after. None = no upper bound (default). Must be >= modified_after when both are supplied. + path_prefix: Restrict to files under this folder/path (e.g. "/Projects/Reports"). + Matches the file_path of indexed files only — setting it implicitly limits results + to files. None = no path filter (default). Returns: SemanticSearchResponse with matching documents ranked by fusion scores. @@ -186,6 +200,12 @@ def configure_semantic_tools(mcp: FastMCP): ) ) + # Treat a blank/whitespace path_prefix as "no filter" so an empty UI + # field doesn't filter out every result (ADR-027 Phase 2). + path_prefix = path_prefix.strip() if path_prefix else None + if not path_prefix: + path_prefix = None + # Expand the caller's identity to every owner whose content they # have read access to via Nextcloud shares. Lets a user find files # owners have shared with them without having to re-index those @@ -232,6 +252,7 @@ def configure_semantic_tools(mcp: FastMCP): accessible_owners=accessible_owners, modified_after=modified_after_ts, modified_before=modified_before_ts, + path_prefix=path_prefix, ) all_results.extend(unverified_results) else: @@ -259,6 +280,7 @@ def configure_semantic_tools(mcp: FastMCP): accessible_owners=accessible_owners, modified_after=modified_after_ts, modified_before=modified_before_ts, + path_prefix=path_prefix, ) all_results.extend(unverified_results) diff --git a/nextcloud_mcp_server/vector/qdrant_client.py b/nextcloud_mcp_server/vector/qdrant_client.py index b2df4873..7417daa9 100644 --- a/nextcloud_mcp_server/vector/qdrant_client.py +++ b/nextcloud_mcp_server/vector/qdrant_client.py @@ -62,6 +62,14 @@ _PAYLOAD_INDEX_FIELDS: dict[str, PayloadSchemaType] = { # is idempotent, so existing collections gain this index at startup with no # content re-index and no operator action. "modified_at": PayloadSchemaType.INTEGER, + # file_path is the ADR-027 Phase 2 path filter field: searches apply + # MatchText(key="file_path", text=path_prefix) (see + # search/access_filter.build_base_filter_conditions). MatchText needs a TEXT + # index on server Qdrant; the value is already on every file point + # (processor.py, doc_type == "file" only), so this is a no-content-re-index + # migration like modified_at. Local/embedded qdrant-client matches by + # substring without an index, so dev stacks work without it too. + "file_path": PayloadSchemaType.TEXT, } # Sentinel point that records "this collection has been backfilled to str diff --git a/tests/unit/search/test_access_filter.py b/tests/unit/search/test_access_filter.py index b708aec3..2bf81582 100644 --- a/tests/unit/search/test_access_filter.py +++ b/tests/unit/search/test_access_filter.py @@ -5,7 +5,7 @@ from __future__ import annotations from unittest.mock import AsyncMock import pytest -from qdrant_client.models import FieldCondition, Range +from qdrant_client.models import FieldCondition, MatchText, Range from nextcloud_mcp_server.search import access_filter from nextcloud_mcp_server.search.access_filter import ( @@ -249,14 +249,36 @@ class TestBuildBaseFilterConditions: isinstance(c, FieldCondition) and c.key == "modified_at" for c in conditions ) + @pytest.mark.unit + @pytest.mark.parametrize("prefix", ["/Projects/Reports", "/Archive"]) + def test_path_prefix_appends_file_path_match_text(self, prefix) -> None: + conditions = build_base_filter_conditions("alice", None, path_prefix=prefix) + path_conds = [ + c + for c in conditions + if isinstance(c, FieldCondition) and c.key == "file_path" + ] + assert len(path_conds) == 1 + match = path_conds[0].match + assert isinstance(match, MatchText) + assert match.text == prefix + + @pytest.mark.unit + def test_no_path_condition_when_prefix_absent(self) -> None: + conditions = build_base_filter_conditions("alice", None, path_prefix=None) + assert not any( + isinstance(c, FieldCondition) and c.key == "file_path" for c in conditions + ) + @pytest.mark.unit def test_all_filters_compose(self) -> None: - # placeholder + ownership + doc_type + modified_at range = 4 conditions. + # placeholder + ownership + doc_type + modified_at range + file_path = 5. conditions = build_base_filter_conditions( "alice", ["alice", "bob"], doc_type="file", modified_after=100, modified_before=200, + path_prefix="/Projects", ) - assert len(conditions) == 4 + assert len(conditions) == 5 diff --git a/tests/unit/vector/test_qdrant_client.py b/tests/unit/vector/test_qdrant_client.py index f5ebdcbb..ba8f2edd 100644 --- a/tests/unit/vector/test_qdrant_client.py +++ b/tests/unit/vector/test_qdrant_client.py @@ -88,6 +88,13 @@ def test_modified_at_indexed_as_integer(): assert _PAYLOAD_INDEX_FIELDS.get("modified_at") == PayloadSchemaType.INTEGER +@pytest.mark.unit +def test_file_path_indexed_as_text(): + """ADR-027 Phase 2: the path filter uses MatchText, which needs a TEXT index + on file_path (server Qdrant); local qdrant-client matches by substring.""" + assert _PAYLOAD_INDEX_FIELDS.get("file_path") == PayloadSchemaType.TEXT + + # --------------------------------------------------------------------------- # _ensure_payload_indexes # ---------------------------------------------------------------------------