fix(tests): Fix integration test failures in qdrant, sampling, and rag tests
- test_qdrant_collection_creation.py:
- Add get_vector_params() helper to handle named vectors format
- Collections use {"dense": VectorParams(...)} instead of direct VectorParams
- Fix otel_service_name setting in test_collection_name_generation
- test_sampling.py:
- Fix MCP response parsing: use json.loads(result.content[0].text)
instead of result.structuredContent (which is None)
- Add require_vector_sync_tools() helper for graceful skipping
- Add helper call to all 5 test functions
- test_rag.py:
- Add require_vector_sync_tools() helper for graceful skipping
- Fix MCP response parsing (same as sampling tests)
- Prevents 600s timeout when VECTOR_SYNC_ENABLED is not set
Tests now pass/skip cleanly when run independently. The anyio.WouldBlock
errors in full test suite runs are fixture isolation issues, not code bugs.
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.5
parent
894bf5f916
commit
779d474aaa
@@ -10,12 +10,21 @@ These tests validate that:
|
|||||||
from unittest.mock import Mock
|
from unittest.mock import Mock
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
from qdrant_client.models import VectorParams
|
||||||
|
|
||||||
from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client
|
from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client
|
||||||
|
|
||||||
pytestmark = pytest.mark.integration
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
|
|
||||||
|
def get_vector_params(collection_info) -> VectorParams:
|
||||||
|
"""Get vector params from collection info, handling named vectors format."""
|
||||||
|
vectors = collection_info.config.params.vectors
|
||||||
|
if isinstance(vectors, dict):
|
||||||
|
return vectors["dense"]
|
||||||
|
return vectors
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(autouse=True)
|
@pytest.fixture(autouse=True)
|
||||||
async def reset_singleton():
|
async def reset_singleton():
|
||||||
"""Reset the global Qdrant client singleton between tests."""
|
"""Reset the global Qdrant client singleton between tests."""
|
||||||
@@ -75,7 +84,7 @@ async def test_collection_auto_created_on_first_access(monkeypatch):
|
|||||||
|
|
||||||
# Verify collection has correct dimensions
|
# Verify collection has correct dimensions
|
||||||
collection_info = await client.get_collection(collection_name)
|
collection_info = await client.get_collection(collection_name)
|
||||||
assert collection_info.config.params.vectors.size == 384
|
assert get_vector_params(collection_info).size == 384
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.integration
|
@pytest.mark.integration
|
||||||
@@ -127,7 +136,7 @@ async def test_existing_collection_reused(monkeypatch):
|
|||||||
|
|
||||||
# Verify dimensions unchanged
|
# Verify dimensions unchanged
|
||||||
collection_info = await client2.get_collection(collection_name)
|
collection_info = await client2.get_collection(collection_name)
|
||||||
assert collection_info.config.params.vectors.size == 384
|
assert get_vector_params(collection_info).size == 384
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.integration
|
@pytest.mark.integration
|
||||||
@@ -164,7 +173,7 @@ async def test_dimension_mismatch_detected(monkeypatch, tmp_path):
|
|||||||
|
|
||||||
# Verify collection created
|
# Verify collection created
|
||||||
collection_info = await client1.get_collection(collection_name)
|
collection_info = await client1.get_collection(collection_name)
|
||||||
assert collection_info.config.params.vectors.size == 384
|
assert get_vector_params(collection_info).size == 384
|
||||||
|
|
||||||
# Close client1 to release file lock
|
# Close client1 to release file lock
|
||||||
await client1.close()
|
await client1.close()
|
||||||
@@ -248,12 +257,10 @@ async def test_collection_name_generation(monkeypatch):
|
|||||||
mock_settings = Settings(
|
mock_settings = Settings(
|
||||||
qdrant_location=":memory:",
|
qdrant_location=":memory:",
|
||||||
ollama_embedding_model="test-model",
|
ollama_embedding_model="test-model",
|
||||||
|
otel_service_name="test-deployment",
|
||||||
vector_sync_enabled=False,
|
vector_sync_enabled=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Mock deployment ID
|
|
||||||
monkeypatch.setenv("MCP_DEPLOYMENT_ID", "test-deployment")
|
|
||||||
|
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"nextcloud_mcp_server.vector.qdrant_client.get_settings", lambda: mock_settings
|
"nextcloud_mcp_server.vector.qdrant_client.get_settings", lambda: mock_settings
|
||||||
)
|
)
|
||||||
@@ -319,4 +326,4 @@ async def test_collection_uses_cosine_distance(monkeypatch):
|
|||||||
|
|
||||||
from qdrant_client.models import Distance
|
from qdrant_client.models import Distance
|
||||||
|
|
||||||
assert collection_info.config.params.vectors.distance == Distance.COSINE
|
assert get_vector_params(collection_info).distance == Distance.COSINE
|
||||||
|
|||||||
@@ -51,6 +51,14 @@ logger = logging.getLogger(__name__)
|
|||||||
DEFAULT_MANUAL_PATH = "Nextcloud Manual.pdf"
|
DEFAULT_MANUAL_PATH = "Nextcloud Manual.pdf"
|
||||||
|
|
||||||
|
|
||||||
|
async def require_vector_sync_tools(nc_mcp_client):
|
||||||
|
"""Skip test if vector sync tools are not available."""
|
||||||
|
tools = await nc_mcp_client.list_tools()
|
||||||
|
tool_names = [t.name for t in tools.tools]
|
||||||
|
if "nc_get_vector_sync_status" not in tool_names:
|
||||||
|
pytest.skip("Vector sync tools not available (VECTOR_SYNC_ENABLED not set)")
|
||||||
|
|
||||||
|
|
||||||
async def llm_judge(
|
async def llm_judge(
|
||||||
provider: Provider,
|
provider: Provider,
|
||||||
ground_truth: str,
|
ground_truth: str,
|
||||||
@@ -116,6 +124,8 @@ async def indexed_manual_pdf(nc_client, nc_mcp_client):
|
|||||||
Environment Variables:
|
Environment Variables:
|
||||||
RAG_MANUAL_PATH: Path to manual PDF in Nextcloud (default: Nextcloud Manual.pdf)
|
RAG_MANUAL_PATH: Path to manual PDF in Nextcloud (default: Nextcloud Manual.pdf)
|
||||||
"""
|
"""
|
||||||
|
await require_vector_sync_tools(nc_mcp_client)
|
||||||
|
|
||||||
manual_path = os.getenv("RAG_MANUAL_PATH", DEFAULT_MANUAL_PATH)
|
manual_path = os.getenv("RAG_MANUAL_PATH", DEFAULT_MANUAL_PATH)
|
||||||
|
|
||||||
logger.info(f"Setting up indexed manual PDF: {manual_path}")
|
logger.info(f"Setting up indexed manual PDF: {manual_path}")
|
||||||
@@ -152,7 +162,7 @@ async def indexed_manual_pdf(nc_client, nc_mcp_client):
|
|||||||
)
|
)
|
||||||
|
|
||||||
if not result.isError:
|
if not result.isError:
|
||||||
content = result.structuredContent or {}
|
content = json.loads(result.content[0].text) if result.content else {}
|
||||||
indexed = content.get("indexed_count", 0)
|
indexed = content.get("indexed_count", 0)
|
||||||
pending = content.get("pending_count", 1)
|
pending = content.get("pending_count", 1)
|
||||||
|
|
||||||
@@ -248,7 +258,7 @@ async def test_semantic_search_retrieval(
|
|||||||
)
|
)
|
||||||
|
|
||||||
assert result.isError is False, f"Tool call failed: {result}"
|
assert result.isError is False, f"Tool call failed: {result}"
|
||||||
data = result.structuredContent
|
data = json.loads(result.content[0].text)
|
||||||
|
|
||||||
# Verify we got results
|
# Verify we got results
|
||||||
assert data["success"] is True
|
assert data["success"] is True
|
||||||
@@ -295,7 +305,7 @@ async def test_semantic_search_answer_with_sampling(
|
|||||||
)
|
)
|
||||||
|
|
||||||
assert result.isError is False, f"Tool call failed: {result}"
|
assert result.isError is False, f"Tool call failed: {result}"
|
||||||
data = result.structuredContent
|
data = json.loads(result.content[0].text)
|
||||||
|
|
||||||
# Verify response structure
|
# Verify response structure
|
||||||
assert data["success"] is True
|
assert data["success"] is True
|
||||||
@@ -369,7 +379,7 @@ async def test_retrieval_quality_all_queries(
|
|||||||
)
|
)
|
||||||
|
|
||||||
assert result.isError is False
|
assert result.isError is False
|
||||||
data = result.structuredContent
|
data = json.loads(result.content[0].text)
|
||||||
|
|
||||||
assert data["total_found"] >= min_expected_results, (
|
assert data["total_found"] >= min_expected_results, (
|
||||||
f"Query '{query}' returned {data['total_found']} results, "
|
f"Query '{query}' returned {data['total_found']} results, "
|
||||||
@@ -393,7 +403,7 @@ async def test_no_results_for_unrelated_query(nc_mcp_client, indexed_manual_pdf)
|
|||||||
)
|
)
|
||||||
|
|
||||||
assert result.isError is False
|
assert result.isError is False
|
||||||
data = result.structuredContent
|
data = json.loads(result.content[0].text)
|
||||||
|
|
||||||
# Should have few or no high-scoring results
|
# Should have few or no high-scoring results
|
||||||
# Low score threshold means we might get some results, but they should be low quality
|
# Low score threshold means we might get some results, but they should be low quality
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ Note: These tests require VECTOR_SYNC_ENABLED=true and a configured
|
|||||||
vector database with indexed test data.
|
vector database with indexed test data.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
from unittest.mock import MagicMock
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@@ -21,6 +22,14 @@ from mcp.types import CreateMessageResult, TextContent
|
|||||||
pytestmark = pytest.mark.integration
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
|
|
||||||
|
async def require_vector_sync_tools(nc_mcp_client):
|
||||||
|
"""Skip test if vector sync tools are not available."""
|
||||||
|
tools = await nc_mcp_client.list_tools()
|
||||||
|
tool_names = [t.name for t in tools.tools]
|
||||||
|
if "nc_get_vector_sync_status" not in tool_names:
|
||||||
|
pytest.skip("Vector sync tools not available (VECTOR_SYNC_ENABLED not set)")
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def mock_sampling_result():
|
def mock_sampling_result():
|
||||||
"""Mock successful sampling result from MCP client."""
|
"""Mock successful sampling result from MCP client."""
|
||||||
@@ -55,13 +64,15 @@ async def test_semantic_search_answer_successful_sampling(
|
|||||||
4. Mock ctx.session.create_message to return answer
|
4. Mock ctx.session.create_message to return answer
|
||||||
5. Verify response contains generated answer and sources
|
5. Verify response contains generated answer and sources
|
||||||
"""
|
"""
|
||||||
|
await require_vector_sync_tools(nc_mcp_client)
|
||||||
|
|
||||||
# Get initial indexed count before creating note
|
# Get initial indexed count before creating note
|
||||||
import asyncio
|
import asyncio
|
||||||
|
|
||||||
initial_sync = await nc_mcp_client.call_tool(
|
initial_sync = await nc_mcp_client.call_tool(
|
||||||
"nc_get_vector_sync_status", arguments={}
|
"nc_get_vector_sync_status", arguments={}
|
||||||
)
|
)
|
||||||
initial_indexed_count = initial_sync.structuredContent["indexed_count"]
|
initial_indexed_count = json.loads(initial_sync.content[0].text)["indexed_count"]
|
||||||
print(f"Initial indexed count: {initial_indexed_count}")
|
print(f"Initial indexed count: {initial_indexed_count}")
|
||||||
|
|
||||||
# Create a note with content about Python async
|
# Create a note with content about Python async
|
||||||
@@ -90,7 +101,7 @@ Avoid blocking operations in async code.""",
|
|||||||
sync_status = await nc_mcp_client.call_tool(
|
sync_status = await nc_mcp_client.call_tool(
|
||||||
"nc_get_vector_sync_status", arguments={}
|
"nc_get_vector_sync_status", arguments={}
|
||||||
)
|
)
|
||||||
status_data = sync_status.structuredContent
|
status_data = json.loads(sync_status.content[0].text)
|
||||||
|
|
||||||
print(
|
print(
|
||||||
f"Sync status at {waited}s: indexed={status_data['indexed_count']}, pending={status_data['pending_count']}, status={status_data['status']}"
|
f"Sync status at {waited}s: indexed={status_data['indexed_count']}, pending={status_data['pending_count']}, status={status_data['status']}"
|
||||||
@@ -135,7 +146,7 @@ Avoid blocking operations in async code.""",
|
|||||||
assert call_result.isError is False, (
|
assert call_result.isError is False, (
|
||||||
f"Tool call failed: {call_result.content[0].text if call_result.isError else ''}"
|
f"Tool call failed: {call_result.content[0].text if call_result.isError else ''}"
|
||||||
)
|
)
|
||||||
result = call_result.structuredContent
|
result = json.loads(call_result.content[0].text)
|
||||||
|
|
||||||
# Verify response structure
|
# Verify response structure
|
||||||
assert result is not None
|
assert result is not None
|
||||||
@@ -179,6 +190,8 @@ async def test_semantic_search_answer_no_results(nc_mcp_client):
|
|||||||
2. Verify response indicates no documents found
|
2. Verify response indicates no documents found
|
||||||
3. Verify no sampling call was made (no sources to base answer on)
|
3. Verify no sampling call was made (no sources to base answer on)
|
||||||
"""
|
"""
|
||||||
|
await require_vector_sync_tools(nc_mcp_client)
|
||||||
|
|
||||||
call_result = await nc_mcp_client.call_tool(
|
call_result = await nc_mcp_client.call_tool(
|
||||||
"nc_semantic_search_answer",
|
"nc_semantic_search_answer",
|
||||||
arguments={
|
arguments={
|
||||||
@@ -192,7 +205,7 @@ async def test_semantic_search_answer_no_results(nc_mcp_client):
|
|||||||
assert call_result.isError is False, (
|
assert call_result.isError is False, (
|
||||||
f"Tool call failed: {call_result.content[0].text if call_result.isError else ''}"
|
f"Tool call failed: {call_result.content[0].text if call_result.isError else ''}"
|
||||||
)
|
)
|
||||||
result = call_result.structuredContent
|
result = json.loads(call_result.content[0].text)
|
||||||
|
|
||||||
# Should get "no documents found" message
|
# Should get "no documents found" message
|
||||||
assert result is not None
|
assert result is not None
|
||||||
@@ -214,6 +227,8 @@ async def test_semantic_search_answer_with_limit(nc_mcp_client, temporary_note_f
|
|||||||
3. Query with limit=2
|
3. Query with limit=2
|
||||||
4. Verify at most 2 sources in response
|
4. Verify at most 2 sources in response
|
||||||
"""
|
"""
|
||||||
|
await require_vector_sync_tools(nc_mcp_client)
|
||||||
|
|
||||||
# Create multiple related notes
|
# Create multiple related notes
|
||||||
_note1 = await temporary_note_factory(
|
_note1 = await temporary_note_factory(
|
||||||
title="Python Async Part 1",
|
title="Python Async Part 1",
|
||||||
@@ -242,7 +257,7 @@ async def test_semantic_search_answer_with_limit(nc_mcp_client, temporary_note_f
|
|||||||
sync_status = await nc_mcp_client.call_tool(
|
sync_status = await nc_mcp_client.call_tool(
|
||||||
"nc_get_vector_sync_status", arguments={}
|
"nc_get_vector_sync_status", arguments={}
|
||||||
)
|
)
|
||||||
status_data = sync_status.structuredContent
|
status_data = json.loads(sync_status.content[0].text)
|
||||||
|
|
||||||
if status_data["status"] == "idle" and status_data["pending_count"] == 0:
|
if status_data["status"] == "idle" and status_data["pending_count"] == 0:
|
||||||
break
|
break
|
||||||
@@ -265,7 +280,7 @@ async def test_semantic_search_answer_with_limit(nc_mcp_client, temporary_note_f
|
|||||||
assert call_result.isError is False, (
|
assert call_result.isError is False, (
|
||||||
f"Tool call failed: {call_result.content[0].text if call_result.isError else ''}"
|
f"Tool call failed: {call_result.content[0].text if call_result.isError else ''}"
|
||||||
)
|
)
|
||||||
result = call_result.structuredContent
|
result = json.loads(call_result.content[0].text)
|
||||||
|
|
||||||
# Should respect limit
|
# Should respect limit
|
||||||
assert len(result["sources"]) <= 2
|
assert len(result["sources"]) <= 2
|
||||||
@@ -282,6 +297,8 @@ async def test_semantic_search_answer_score_threshold(
|
|||||||
3. Query with high threshold (0.9)
|
3. Query with high threshold (0.9)
|
||||||
4. Verify only high-scoring results returned
|
4. Verify only high-scoring results returned
|
||||||
"""
|
"""
|
||||||
|
await require_vector_sync_tools(nc_mcp_client)
|
||||||
|
|
||||||
_note = await temporary_note_factory(
|
_note = await temporary_note_factory(
|
||||||
title="Exact Match Test",
|
title="Exact Match Test",
|
||||||
content="This is a very specific test document about widget manufacturing",
|
content="This is a very specific test document about widget manufacturing",
|
||||||
@@ -299,7 +316,7 @@ async def test_semantic_search_answer_score_threshold(
|
|||||||
sync_status = await nc_mcp_client.call_tool(
|
sync_status = await nc_mcp_client.call_tool(
|
||||||
"nc_get_vector_sync_status", arguments={}
|
"nc_get_vector_sync_status", arguments={}
|
||||||
)
|
)
|
||||||
status_data = sync_status.structuredContent
|
status_data = json.loads(sync_status.content[0].text)
|
||||||
|
|
||||||
if status_data["status"] == "idle" and status_data["pending_count"] == 0:
|
if status_data["status"] == "idle" and status_data["pending_count"] == 0:
|
||||||
break
|
break
|
||||||
@@ -323,7 +340,7 @@ async def test_semantic_search_answer_score_threshold(
|
|||||||
assert call_result.isError is False, (
|
assert call_result.isError is False, (
|
||||||
f"Tool call failed: {call_result.content[0].text if call_result.isError else ''}"
|
f"Tool call failed: {call_result.content[0].text if call_result.isError else ''}"
|
||||||
)
|
)
|
||||||
result = call_result.structuredContent
|
result = json.loads(call_result.content[0].text)
|
||||||
|
|
||||||
# Note: Semantic search scores depend on embedding model
|
# Note: Semantic search scores depend on embedding model
|
||||||
# We just verify the tool accepts the parameter
|
# We just verify the tool accepts the parameter
|
||||||
@@ -345,6 +362,8 @@ async def test_semantic_search_answer_max_tokens(nc_mcp_client, temporary_note_f
|
|||||||
Note: Token limiting is enforced by the MCP client's LLM, not the server.
|
Note: Token limiting is enforced by the MCP client's LLM, not the server.
|
||||||
This test just verifies the parameter is correctly passed.
|
This test just verifies the parameter is correctly passed.
|
||||||
"""
|
"""
|
||||||
|
await require_vector_sync_tools(nc_mcp_client)
|
||||||
|
|
||||||
_note = await temporary_note_factory(
|
_note = await temporary_note_factory(
|
||||||
title="Long Document",
|
title="Long Document",
|
||||||
content="This is a document with lots of content. " * 50,
|
content="This is a document with lots of content. " * 50,
|
||||||
@@ -362,7 +381,7 @@ async def test_semantic_search_answer_max_tokens(nc_mcp_client, temporary_note_f
|
|||||||
sync_status = await nc_mcp_client.call_tool(
|
sync_status = await nc_mcp_client.call_tool(
|
||||||
"nc_get_vector_sync_status", arguments={}
|
"nc_get_vector_sync_status", arguments={}
|
||||||
)
|
)
|
||||||
status_data = sync_status.structuredContent
|
status_data = json.loads(sync_status.content[0].text)
|
||||||
|
|
||||||
if status_data["status"] == "idle" and status_data["pending_count"] == 0:
|
if status_data["status"] == "idle" and status_data["pending_count"] == 0:
|
||||||
break
|
break
|
||||||
@@ -386,7 +405,7 @@ async def test_semantic_search_answer_max_tokens(nc_mcp_client, temporary_note_f
|
|||||||
assert call_result.isError is False, (
|
assert call_result.isError is False, (
|
||||||
f"Tool call failed: {call_result.content[0].text if call_result.isError else ''}"
|
f"Tool call failed: {call_result.content[0].text if call_result.isError else ''}"
|
||||||
)
|
)
|
||||||
result = call_result.structuredContent
|
result = json.loads(call_result.content[0].text)
|
||||||
|
|
||||||
# Should not error, even if sampling fails
|
# Should not error, even if sampling fails
|
||||||
assert result is not None
|
assert result is not None
|
||||||
|
|||||||
Reference in New Issue
Block a user