feat: harmonize MCP tool + userinfo page to documents/chunks model
Extend the documents-vs-chunks split to the remaining status surfaces so all three report consistently (Deck #195): - nc_get_vector_sync_status MCP tool + VectorSyncStatusResponse: add indexed_documents (distinct) and indexed_chunks; keep indexed_count as a deprecated alias of indexed_chunks. Reuses count_indexed. - userinfo HTML page (/app/vector-sync/status): show Indexed Documents AND Indexed Chunks rows; switch its count to count_indexed (which also excludes placeholder points — the old raw count included them). - /api/v1/vector-sync/status: restore indexed_count as a deprecated alias of indexed_chunks so existing consumers (integration tests, pre-#115 UI) keep working; the change is now purely additive for indexed_count. Tests: VectorSyncStatusResponse documents/chunks/alias + zeroed defaults. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
8c9f97d6c4
commit
e4d81d47d9
@@ -327,9 +327,11 @@ async def get_vector_sync_status(request: Request) -> JSONResponse:
|
|||||||
"status": status,
|
"status": status,
|
||||||
# indexed_documents is now the distinct-document count (was the chunk
|
# indexed_documents is now the distinct-document count (was the chunk
|
||||||
# count before — the two differ by the per-document chunk fan-out).
|
# count before — the two differ by the per-document chunk fan-out).
|
||||||
# indexed_chunks exposes the raw point count separately.
|
# indexed_chunks exposes the raw point count separately; indexed_count
|
||||||
|
# is kept as a deprecated alias of indexed_chunks for back-compat.
|
||||||
"indexed_documents": indexed_documents,
|
"indexed_documents": indexed_documents,
|
||||||
"indexed_chunks": indexed_chunks,
|
"indexed_chunks": indexed_chunks,
|
||||||
|
"indexed_count": indexed_chunks,
|
||||||
"pending_documents": pending.pending,
|
"pending_documents": pending.pending,
|
||||||
"ingest_queue": settings.ingest_queue,
|
"ingest_queue": settings.ingest_queue,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -104,7 +104,8 @@ async def _get_processing_status(request: Request) -> dict[str, Any] | None:
|
|||||||
Dictionary with processing status, or None if vector sync is disabled
|
Dictionary with processing status, or None if vector sync is disabled
|
||||||
or components are unavailable:
|
or components are unavailable:
|
||||||
{
|
{
|
||||||
"indexed_count": int, # Number of documents in Qdrant
|
"indexed_documents": int, # Distinct documents in Qdrant
|
||||||
|
"indexed_chunks": int, # Total chunks/points in Qdrant
|
||||||
"pending_count": int, # Number of documents in queue
|
"pending_count": int, # Number of documents in queue
|
||||||
"status": str, # "syncing" or "idle"
|
"status": str, # "syncing" or "idle"
|
||||||
}
|
}
|
||||||
@@ -130,30 +131,33 @@ async def _get_processing_status(request: Request) -> dict[str, Any] | None:
|
|||||||
ingest_queue=settings.ingest_queue,
|
ingest_queue=settings.ingest_queue,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Get Qdrant client and query indexed count
|
# Corpus size: distinct documents AND total chunks (placeholders
|
||||||
indexed_count = 0
|
# excluded — the prior count included them).
|
||||||
|
indexed_documents = 0
|
||||||
|
indexed_chunks = 0
|
||||||
try:
|
try:
|
||||||
|
from nextcloud_mcp_server.vector.metrics_publisher import ( # noqa: PLC0415
|
||||||
|
count_indexed,
|
||||||
|
)
|
||||||
from nextcloud_mcp_server.vector.qdrant_client import ( # noqa: PLC0415
|
from nextcloud_mcp_server.vector.qdrant_client import ( # noqa: PLC0415
|
||||||
get_qdrant_client,
|
get_qdrant_client,
|
||||||
)
|
)
|
||||||
|
|
||||||
qdrant_client = await get_qdrant_client()
|
qdrant_client = await get_qdrant_client()
|
||||||
|
indexed_documents, indexed_chunks = await count_indexed(
|
||||||
# Count documents in collection
|
qdrant_client, settings.get_collection_name()
|
||||||
count_result = await qdrant_client.count(
|
|
||||||
collection_name=settings.get_collection_name()
|
|
||||||
)
|
)
|
||||||
indexed_count = count_result.count
|
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning("Failed to query Qdrant for indexed count: %s", e)
|
logger.warning("Failed to query Qdrant for indexed counts: %s", e)
|
||||||
# Continue with indexed_count = 0
|
# Continue with zeroed counts
|
||||||
|
|
||||||
# Determine status
|
# Determine status
|
||||||
status = "syncing" if pending.pending > 0 else "idle"
|
status = "syncing" if pending.pending > 0 else "idle"
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"indexed_count": indexed_count,
|
"indexed_documents": indexed_documents,
|
||||||
|
"indexed_chunks": indexed_chunks,
|
||||||
"pending_count": pending.pending,
|
"pending_count": pending.pending,
|
||||||
"status": status,
|
"status": status,
|
||||||
}
|
}
|
||||||
@@ -190,12 +194,14 @@ async def vector_sync_status_fragment(request: Request) -> HTMLResponse:
|
|||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
|
|
||||||
indexed_count = processing_status["indexed_count"]
|
indexed_documents = processing_status["indexed_documents"]
|
||||||
|
indexed_chunks = processing_status["indexed_chunks"]
|
||||||
pending_count = processing_status["pending_count"]
|
pending_count = processing_status["pending_count"]
|
||||||
status = processing_status["status"]
|
status = processing_status["status"]
|
||||||
|
|
||||||
# Format numbers with commas for readability
|
# Format numbers with commas for readability
|
||||||
indexed_count_str = f"{indexed_count:,}"
|
indexed_documents_str = f"{indexed_documents:,}"
|
||||||
|
indexed_chunks_str = f"{indexed_chunks:,}"
|
||||||
pending_count_str = f"{pending_count:,}"
|
pending_count_str = f"{pending_count:,}"
|
||||||
|
|
||||||
# Status badge color and text
|
# Status badge color and text
|
||||||
@@ -212,7 +218,11 @@ async def vector_sync_status_fragment(request: Request) -> HTMLResponse:
|
|||||||
<table>
|
<table>
|
||||||
<tr>
|
<tr>
|
||||||
<td><strong>Indexed Documents</strong></td>
|
<td><strong>Indexed Documents</strong></td>
|
||||||
<td>{indexed_count_str}</td>
|
<td>{indexed_documents_str}</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><strong>Indexed Chunks</strong></td>
|
||||||
|
<td>{indexed_chunks_str}</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td><strong>Pending Documents</strong></td>
|
<td><strong>Pending Documents</strong></td>
|
||||||
|
|||||||
@@ -153,14 +153,26 @@ class VectorSyncStatusResponse(BaseResponse):
|
|||||||
including how many documents are indexed and how many are pending.
|
including how many documents are indexed and how many are pending.
|
||||||
|
|
||||||
Attributes:
|
Attributes:
|
||||||
indexed_count: Number of documents in Qdrant vector database
|
indexed_documents: Distinct documents indexed in the vector database
|
||||||
|
indexed_chunks: Total indexed chunks (vector points); ~N per document
|
||||||
|
indexed_count: DEPRECATED alias of indexed_chunks
|
||||||
pending_count: Number of documents in processing queue
|
pending_count: Number of documents in processing queue
|
||||||
status: Current sync status ("idle" or "syncing")
|
status: Current sync status ("idle" or "syncing")
|
||||||
enabled: Whether vector sync is enabled
|
enabled: Whether vector sync is enabled
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
indexed_documents: int = Field(
|
||||||
|
default=0, description="Distinct documents indexed in the vector database"
|
||||||
|
)
|
||||||
|
indexed_chunks: int = Field(
|
||||||
|
default=0, description="Total indexed chunks (vector points); ~N per document"
|
||||||
|
)
|
||||||
indexed_count: int = Field(
|
indexed_count: int = Field(
|
||||||
default=0, description="Number of documents indexed in vector database"
|
default=0,
|
||||||
|
description=(
|
||||||
|
"DEPRECATED alias of indexed_chunks (the chunk/point count). Use "
|
||||||
|
"indexed_documents for the distinct-document count."
|
||||||
|
),
|
||||||
)
|
)
|
||||||
pending_count: int = Field(
|
pending_count: int = Field(
|
||||||
default=0, description="Number of documents pending processing"
|
default=0, description="Number of documents pending processing"
|
||||||
|
|||||||
@@ -18,7 +18,6 @@ from mcp.types import (
|
|||||||
ToolAnnotations,
|
ToolAnnotations,
|
||||||
)
|
)
|
||||||
from pydantic import Field
|
from pydantic import Field
|
||||||
from qdrant_client.models import Filter
|
|
||||||
|
|
||||||
from nextcloud_mcp_server.auth import require_scopes
|
from nextcloud_mcp_server.auth import require_scopes
|
||||||
from nextcloud_mcp_server.config import get_settings
|
from nextcloud_mcp_server.config import get_settings
|
||||||
@@ -41,7 +40,7 @@ from nextcloud_mcp_server.search.bm25_hybrid import BM25HybridSearchAlgorithm
|
|||||||
from nextcloud_mcp_server.search.context import get_chunk_with_context
|
from nextcloud_mcp_server.search.context import get_chunk_with_context
|
||||||
from nextcloud_mcp_server.search.verification import verify_search_results
|
from nextcloud_mcp_server.search.verification import verify_search_results
|
||||||
from nextcloud_mcp_server.utils.validation import parse_modified_timestamp
|
from nextcloud_mcp_server.utils.validation import parse_modified_timestamp
|
||||||
from nextcloud_mcp_server.vector.placeholder import get_placeholder_filter
|
from nextcloud_mcp_server.vector.metrics_publisher import count_indexed
|
||||||
from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client
|
from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -982,28 +981,27 @@ def configure_semantic_tools(mcp: FastMCP):
|
|||||||
ingest_queue=settings.ingest_queue,
|
ingest_queue=settings.ingest_queue,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Get Qdrant client and query indexed count
|
# Corpus size: distinct documents AND total chunks (placeholders
|
||||||
indexed_count = 0
|
# excluded). A single "indexed" figure is ambiguous because each
|
||||||
|
# document fans out to ~N chunks.
|
||||||
|
indexed_documents = 0
|
||||||
|
indexed_chunks = 0
|
||||||
try:
|
try:
|
||||||
qdrant_client = await get_qdrant_client()
|
qdrant_client = await get_qdrant_client()
|
||||||
|
indexed_documents, indexed_chunks = await count_indexed(
|
||||||
# Count documents in collection, excluding placeholders
|
qdrant_client, settings.get_collection_name()
|
||||||
# Placeholders are zero-vector points used to track processing state
|
|
||||||
count_result = await qdrant_client.count(
|
|
||||||
collection_name=settings.get_collection_name(),
|
|
||||||
count_filter=Filter(must=[get_placeholder_filter()]),
|
|
||||||
)
|
)
|
||||||
indexed_count = count_result.count
|
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning("Failed to query Qdrant for indexed count: %s", e)
|
logger.warning("Failed to query Qdrant for indexed counts: %s", e)
|
||||||
# Continue with indexed_count = 0
|
# Continue with zeroed counts
|
||||||
|
|
||||||
# Determine status
|
# Determine status
|
||||||
status = "syncing" if pending.pending > 0 else "idle"
|
status = "syncing" if pending.pending > 0 else "idle"
|
||||||
|
|
||||||
return VectorSyncStatusResponse(
|
return VectorSyncStatusResponse(
|
||||||
indexed_count=indexed_count,
|
indexed_documents=indexed_documents,
|
||||||
|
indexed_chunks=indexed_chunks,
|
||||||
|
indexed_count=indexed_chunks, # deprecated alias
|
||||||
pending_count=pending.pending,
|
pending_count=pending.pending,
|
||||||
status=status,
|
status=status,
|
||||||
enabled=True,
|
enabled=True,
|
||||||
|
|||||||
@@ -0,0 +1,36 @@
|
|||||||
|
"""Unit tests for VectorSyncStatusResponse documents-vs-chunks fields."""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from nextcloud_mcp_server.models.semantic import VectorSyncStatusResponse
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.unit
|
||||||
|
|
||||||
|
|
||||||
|
def test_vector_sync_status_documents_and_chunks() -> None:
|
||||||
|
"""Exposes documents AND chunks; indexed_count is a deprecated chunks alias."""
|
||||||
|
response = VectorSyncStatusResponse(
|
||||||
|
indexed_documents=486,
|
||||||
|
indexed_chunks=16039,
|
||||||
|
indexed_count=16039, # deprecated alias
|
||||||
|
pending_count=2214,
|
||||||
|
status="syncing",
|
||||||
|
enabled=True,
|
||||||
|
ingest_queue="memory",
|
||||||
|
)
|
||||||
|
|
||||||
|
data = response.model_dump()
|
||||||
|
assert data["indexed_documents"] == 486
|
||||||
|
assert data["indexed_chunks"] == 16039
|
||||||
|
# Alias mirrors chunks (not documents) for back-compat.
|
||||||
|
assert data["indexed_count"] == data["indexed_chunks"]
|
||||||
|
assert data["pending_count"] == 2214
|
||||||
|
|
||||||
|
|
||||||
|
def test_vector_sync_status_defaults_zeroed() -> None:
|
||||||
|
"""New corpus fields default to 0 (disabled / pre-sync path)."""
|
||||||
|
response = VectorSyncStatusResponse(status="disabled", enabled=False)
|
||||||
|
data = response.model_dump()
|
||||||
|
assert data["indexed_documents"] == 0
|
||||||
|
assert data["indexed_chunks"] == 0
|
||||||
|
assert data["indexed_count"] == 0
|
||||||
Vendored
+1
-1
Submodule third_party/astrolabe updated: 54adb28f1d...3c0e2a58c2
Reference in New Issue
Block a user