From e4d81d47d943dc48b62b38f165e61e99f885ae3f Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Thu, 4 Jun 2026 20:28:44 +0200 Subject: [PATCH] feat: harmonize MCP tool + userinfo page to documents/chunks model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend the documents-vs-chunks split to the remaining status surfaces so all three report consistently (Deck #195): - nc_get_vector_sync_status MCP tool + VectorSyncStatusResponse: add indexed_documents (distinct) and indexed_chunks; keep indexed_count as a deprecated alias of indexed_chunks. Reuses count_indexed. - userinfo HTML page (/app/vector-sync/status): show Indexed Documents AND Indexed Chunks rows; switch its count to count_indexed (which also excludes placeholder points — the old raw count included them). - /api/v1/vector-sync/status: restore indexed_count as a deprecated alias of indexed_chunks so existing consumers (integration tests, pre-#115 UI) keep working; the change is now purely additive for indexed_count. Tests: VectorSyncStatusResponse documents/chunks/alias + zeroed defaults. Co-Authored-By: Claude Opus 4.8 (1M context) --- nextcloud_mcp_server/api/management.py | 4 ++- nextcloud_mcp_server/auth/userinfo_routes.py | 38 ++++++++++++-------- nextcloud_mcp_server/models/semantic.py | 16 +++++++-- nextcloud_mcp_server/server/semantic.py | 28 +++++++-------- tests/unit/test_vector_sync_status_model.py | 36 +++++++++++++++++++ third_party/astrolabe | 2 +- 6 files changed, 91 insertions(+), 33 deletions(-) create mode 100644 tests/unit/test_vector_sync_status_model.py diff --git a/nextcloud_mcp_server/api/management.py b/nextcloud_mcp_server/api/management.py index c9fd698f..0853ad2d 100644 --- a/nextcloud_mcp_server/api/management.py +++ b/nextcloud_mcp_server/api/management.py @@ -327,9 +327,11 @@ async def get_vector_sync_status(request: Request) -> JSONResponse: "status": status, # indexed_documents is now the distinct-document count (was the chunk # count before — the two differ by the per-document chunk fan-out). - # indexed_chunks exposes the raw point count separately. + # indexed_chunks exposes the raw point count separately; indexed_count + # is kept as a deprecated alias of indexed_chunks for back-compat. "indexed_documents": indexed_documents, "indexed_chunks": indexed_chunks, + "indexed_count": indexed_chunks, "pending_documents": pending.pending, "ingest_queue": settings.ingest_queue, } diff --git a/nextcloud_mcp_server/auth/userinfo_routes.py b/nextcloud_mcp_server/auth/userinfo_routes.py index 85bc9771..0fcc5a66 100644 --- a/nextcloud_mcp_server/auth/userinfo_routes.py +++ b/nextcloud_mcp_server/auth/userinfo_routes.py @@ -104,7 +104,8 @@ async def _get_processing_status(request: Request) -> dict[str, Any] | None: Dictionary with processing status, or None if vector sync is disabled or components are unavailable: { - "indexed_count": int, # Number of documents in Qdrant + "indexed_documents": int, # Distinct documents in Qdrant + "indexed_chunks": int, # Total chunks/points in Qdrant "pending_count": int, # Number of documents in queue "status": str, # "syncing" or "idle" } @@ -130,30 +131,33 @@ async def _get_processing_status(request: Request) -> dict[str, Any] | None: ingest_queue=settings.ingest_queue, ) - # Get Qdrant client and query indexed count - indexed_count = 0 + # Corpus size: distinct documents AND total chunks (placeholders + # excluded — the prior count included them). + indexed_documents = 0 + indexed_chunks = 0 try: + from nextcloud_mcp_server.vector.metrics_publisher import ( # noqa: PLC0415 + count_indexed, + ) from nextcloud_mcp_server.vector.qdrant_client import ( # noqa: PLC0415 get_qdrant_client, ) qdrant_client = await get_qdrant_client() - - # Count documents in collection - count_result = await qdrant_client.count( - collection_name=settings.get_collection_name() + indexed_documents, indexed_chunks = await count_indexed( + qdrant_client, settings.get_collection_name() ) - indexed_count = count_result.count except Exception as e: - logger.warning("Failed to query Qdrant for indexed count: %s", e) - # Continue with indexed_count = 0 + logger.warning("Failed to query Qdrant for indexed counts: %s", e) + # Continue with zeroed counts # Determine status status = "syncing" if pending.pending > 0 else "idle" return { - "indexed_count": indexed_count, + "indexed_documents": indexed_documents, + "indexed_chunks": indexed_chunks, "pending_count": pending.pending, "status": status, } @@ -190,12 +194,14 @@ async def vector_sync_status_fragment(request: Request) -> HTMLResponse: """ ) - indexed_count = processing_status["indexed_count"] + indexed_documents = processing_status["indexed_documents"] + indexed_chunks = processing_status["indexed_chunks"] pending_count = processing_status["pending_count"] status = processing_status["status"] # Format numbers with commas for readability - indexed_count_str = f"{indexed_count:,}" + indexed_documents_str = f"{indexed_documents:,}" + indexed_chunks_str = f"{indexed_chunks:,}" pending_count_str = f"{pending_count:,}" # Status badge color and text @@ -212,7 +218,11 @@ async def vector_sync_status_fragment(request: Request) -> HTMLResponse: - + + + + + diff --git a/nextcloud_mcp_server/models/semantic.py b/nextcloud_mcp_server/models/semantic.py index ae586ee8..35f67ddf 100644 --- a/nextcloud_mcp_server/models/semantic.py +++ b/nextcloud_mcp_server/models/semantic.py @@ -153,14 +153,26 @@ class VectorSyncStatusResponse(BaseResponse): including how many documents are indexed and how many are pending. Attributes: - indexed_count: Number of documents in Qdrant vector database + indexed_documents: Distinct documents indexed in the vector database + indexed_chunks: Total indexed chunks (vector points); ~N per document + indexed_count: DEPRECATED alias of indexed_chunks pending_count: Number of documents in processing queue status: Current sync status ("idle" or "syncing") enabled: Whether vector sync is enabled """ + indexed_documents: int = Field( + default=0, description="Distinct documents indexed in the vector database" + ) + indexed_chunks: int = Field( + default=0, description="Total indexed chunks (vector points); ~N per document" + ) indexed_count: int = Field( - default=0, description="Number of documents indexed in vector database" + default=0, + description=( + "DEPRECATED alias of indexed_chunks (the chunk/point count). Use " + "indexed_documents for the distinct-document count." + ), ) pending_count: int = Field( default=0, description="Number of documents pending processing" diff --git a/nextcloud_mcp_server/server/semantic.py b/nextcloud_mcp_server/server/semantic.py index 8febb0ae..b2b8ec0c 100644 --- a/nextcloud_mcp_server/server/semantic.py +++ b/nextcloud_mcp_server/server/semantic.py @@ -18,7 +18,6 @@ from mcp.types import ( ToolAnnotations, ) from pydantic import Field -from qdrant_client.models import Filter from nextcloud_mcp_server.auth import require_scopes from nextcloud_mcp_server.config import get_settings @@ -41,7 +40,7 @@ from nextcloud_mcp_server.search.bm25_hybrid import BM25HybridSearchAlgorithm from nextcloud_mcp_server.search.context import get_chunk_with_context from nextcloud_mcp_server.search.verification import verify_search_results from nextcloud_mcp_server.utils.validation import parse_modified_timestamp -from nextcloud_mcp_server.vector.placeholder import get_placeholder_filter +from nextcloud_mcp_server.vector.metrics_publisher import count_indexed from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client logger = logging.getLogger(__name__) @@ -982,28 +981,27 @@ def configure_semantic_tools(mcp: FastMCP): ingest_queue=settings.ingest_queue, ) - # Get Qdrant client and query indexed count - indexed_count = 0 + # Corpus size: distinct documents AND total chunks (placeholders + # excluded). A single "indexed" figure is ambiguous because each + # document fans out to ~N chunks. + indexed_documents = 0 + indexed_chunks = 0 try: qdrant_client = await get_qdrant_client() - - # Count documents in collection, excluding placeholders - # Placeholders are zero-vector points used to track processing state - count_result = await qdrant_client.count( - collection_name=settings.get_collection_name(), - count_filter=Filter(must=[get_placeholder_filter()]), + indexed_documents, indexed_chunks = await count_indexed( + qdrant_client, settings.get_collection_name() ) - indexed_count = count_result.count - except Exception as e: - logger.warning("Failed to query Qdrant for indexed count: %s", e) - # Continue with indexed_count = 0 + logger.warning("Failed to query Qdrant for indexed counts: %s", e) + # Continue with zeroed counts # Determine status status = "syncing" if pending.pending > 0 else "idle" return VectorSyncStatusResponse( - indexed_count=indexed_count, + indexed_documents=indexed_documents, + indexed_chunks=indexed_chunks, + indexed_count=indexed_chunks, # deprecated alias pending_count=pending.pending, status=status, enabled=True, diff --git a/tests/unit/test_vector_sync_status_model.py b/tests/unit/test_vector_sync_status_model.py new file mode 100644 index 00000000..c82caff2 --- /dev/null +++ b/tests/unit/test_vector_sync_status_model.py @@ -0,0 +1,36 @@ +"""Unit tests for VectorSyncStatusResponse documents-vs-chunks fields.""" + +import pytest + +from nextcloud_mcp_server.models.semantic import VectorSyncStatusResponse + +pytestmark = pytest.mark.unit + + +def test_vector_sync_status_documents_and_chunks() -> None: + """Exposes documents AND chunks; indexed_count is a deprecated chunks alias.""" + response = VectorSyncStatusResponse( + indexed_documents=486, + indexed_chunks=16039, + indexed_count=16039, # deprecated alias + pending_count=2214, + status="syncing", + enabled=True, + ingest_queue="memory", + ) + + data = response.model_dump() + assert data["indexed_documents"] == 486 + assert data["indexed_chunks"] == 16039 + # Alias mirrors chunks (not documents) for back-compat. + assert data["indexed_count"] == data["indexed_chunks"] + assert data["pending_count"] == 2214 + + +def test_vector_sync_status_defaults_zeroed() -> None: + """New corpus fields default to 0 (disabled / pre-sync path).""" + response = VectorSyncStatusResponse(status="disabled", enabled=False) + data = response.model_dump() + assert data["indexed_documents"] == 0 + assert data["indexed_chunks"] == 0 + assert data["indexed_count"] == 0 diff --git a/third_party/astrolabe b/third_party/astrolabe index 54adb28f..3c0e2a58 160000 --- a/third_party/astrolabe +++ b/third_party/astrolabe @@ -1 +1 @@ -Subproject commit 54adb28f1d10ac5c143ca28e853d4289fdd0a69b +Subproject commit 3c0e2a58c24694259c61dcba166d4b0d326e081e
Indexed Documents{indexed_count_str}{indexed_documents_str}
Indexed Chunks{indexed_chunks_str}
Pending Documents