Merge remote-tracking branch 'origin/master' into feat/tier2-incluster-ocr
This commit is contained in:
+2
-2
@@ -1987,7 +1987,7 @@ async def playwright_oauth_token(
|
|||||||
# Wait for callback server to receive the auth code
|
# Wait for callback server to receive the auth code
|
||||||
# Browser will be redirected to localhost:8081 which will capture the code
|
# Browser will be redirected to localhost:8081 which will capture the code
|
||||||
logger.info("Waiting for callback server to receive auth code...")
|
logger.info("Waiting for callback server to receive auth code...")
|
||||||
timeout_seconds = 30
|
timeout_seconds = 60 # was 30; too tight for consent+redirect on loaded CI
|
||||||
start_time = time.time()
|
start_time = time.time()
|
||||||
while state not in auth_states:
|
while state not in auth_states:
|
||||||
if time.time() - start_time > timeout_seconds:
|
if time.time() - start_time > timeout_seconds:
|
||||||
@@ -2696,7 +2696,7 @@ async def _get_oauth_token_for_user(
|
|||||||
logger.info(
|
logger.info(
|
||||||
"Waiting for callback server to receive auth code for %s...", username
|
"Waiting for callback server to receive auth code for %s...", username
|
||||||
)
|
)
|
||||||
timeout_seconds = 30
|
timeout_seconds = 60 # was 30; too tight for consent+redirect on loaded CI
|
||||||
start_time = time.time()
|
start_time = time.time()
|
||||||
while state not in auth_states:
|
while state not in auth_states:
|
||||||
if time.time() - start_time > timeout_seconds:
|
if time.time() - start_time > timeout_seconds:
|
||||||
|
|||||||
@@ -0,0 +1,68 @@
|
|||||||
|
"""Shared helpers for asserting vector-sync visibility in integration tests.
|
||||||
|
|
||||||
|
Kept dependency-light (no Playwright) so both the multi-user-basic UI tests and
|
||||||
|
the single-user sampling tests can import it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
async def document_is_searchable(
|
||||||
|
mcp_client: Any, search_term: str, note_id: int | None = None
|
||||||
|
) -> bool:
|
||||||
|
"""Return True once a freshly-created document is retrievable.
|
||||||
|
|
||||||
|
Polls ``nc_semantic_search`` (hybrid: an exact unique term reliably matches
|
||||||
|
on the keyword side) and matches by ``note_id`` when provided, otherwise by
|
||||||
|
the term appearing in a result's title/excerpt. Transient errors return
|
||||||
|
False so callers can keep polling.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
search = await mcp_client.call_tool(
|
||||||
|
"nc_semantic_search",
|
||||||
|
# limit is generous: a fresh note can sit below seed data (e.g. deck
|
||||||
|
# cards) in a crowded corpus, and the query is cheap.
|
||||||
|
arguments={"query": search_term, "limit": 50, "score_threshold": 0.0},
|
||||||
|
)
|
||||||
|
except Exception as e: # transient transport/availability blip — keep polling
|
||||||
|
logger.debug("Semantic search poll failed: %s", e)
|
||||||
|
return False
|
||||||
|
if search.isError:
|
||||||
|
logger.debug("Semantic search poll error: %s", search)
|
||||||
|
return False
|
||||||
|
|
||||||
|
try:
|
||||||
|
results = json.loads(search.content[0].text).get("results", [])
|
||||||
|
except (IndexError, ValueError) as e: # empty content / malformed JSON
|
||||||
|
logger.debug("Semantic search parse failed: %s", e)
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Token match (not contiguous substring) so multi-word terms work in the
|
||||||
|
# note_id-less fallback path.
|
||||||
|
tokens = search_term.lower().split()
|
||||||
|
for r in results:
|
||||||
|
if note_id is not None:
|
||||||
|
# str-coerce both sides: nc_semantic_search returns int ids today,
|
||||||
|
# but the Astrolabe API serialises some ids as strings — match the
|
||||||
|
# defensive comparison in _poll_astrolabe_search_for_note so a future
|
||||||
|
# schema change can't silently break the match.
|
||||||
|
if str(r.get("id")) == str(note_id):
|
||||||
|
if r.get("doc_type") == "note":
|
||||||
|
return True
|
||||||
|
# id matched but not a note — surface possible schema drift at
|
||||||
|
# WARNING (CI runs --log-cli-level=WARN) instead of letting the
|
||||||
|
# caller time out with a generic message.
|
||||||
|
logger.warning(
|
||||||
|
"search hit id=%s has doc_type=%s (expected note)",
|
||||||
|
note_id,
|
||||||
|
r.get("doc_type"),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
haystack = f"{r.get('title', '')} {r.get('excerpt', '')}".lower()
|
||||||
|
if tokens and all(t in haystack for t in tokens):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
@@ -56,11 +56,11 @@ async def _poll_astrolabe_search_for_note(
|
|||||||
) -> dict:
|
) -> dict:
|
||||||
"""Poll Astrolabe's search endpoint until `note_id` shows up in results.
|
"""Poll Astrolabe's search endpoint until `note_id` shows up in results.
|
||||||
|
|
||||||
`wait_for_vector_sync` only waits for the total indexed count to grow —
|
`wait_for_vector_sync` now gates on this specific document being
|
||||||
it does not guarantee that *this specific* document is visible yet
|
retrievable via the MCP semantic-search tool, but Astrolabe's own search
|
||||||
(observed on nc32 where deck-card seed data indexes first and the new
|
endpoint is a distinct read path (its own JWT + query handler), so we still
|
||||||
note arrives in Qdrant a few seconds later). Poll until the unique term
|
poll it here until the unique term returns our note — or fail loudly with
|
||||||
returns our note, or fail loudly with the last response we saw.
|
the last response we saw.
|
||||||
"""
|
"""
|
||||||
deadline = time.monotonic() + timeout_seconds
|
deadline = time.monotonic() + timeout_seconds
|
||||||
last_results: list | None = None
|
last_results: list | None = None
|
||||||
@@ -170,9 +170,16 @@ async def test_chunk_context_endpoint_uses_app_password(
|
|||||||
assert note_id is not None
|
assert note_id is not None
|
||||||
|
|
||||||
sync_complete, status = await wait_for_vector_sync(
|
sync_complete, status = await wait_for_vector_sync(
|
||||||
mcp_client, initial_count, timeout_seconds=90
|
mcp_client,
|
||||||
|
initial_count,
|
||||||
|
timeout_seconds=90,
|
||||||
|
search_term=unique_term,
|
||||||
|
note_id=note_id,
|
||||||
|
)
|
||||||
|
assert sync_complete, (
|
||||||
|
f"Note {note_id} ({unique_term}) never became searchable "
|
||||||
|
f"within timeout. Last sync status: {status}"
|
||||||
)
|
)
|
||||||
assert sync_complete, f"Vector sync did not complete: {status}"
|
|
||||||
|
|
||||||
# Use the browser's session to drive Astrolabe end-to-end, the way a
|
# Use the browser's session to drive Astrolabe end-to-end, the way a
|
||||||
# real user would: this exercises astrolabe's OAuth token retrieval
|
# real user would: this exercises astrolabe's OAuth token retrieval
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ from playwright.async_api import Page
|
|||||||
|
|
||||||
# Import helper functions from existing test
|
# Import helper functions from existing test
|
||||||
from tests.conftest import create_mcp_client_session
|
from tests.conftest import create_mcp_client_session
|
||||||
|
from tests.integration._search_helpers import document_is_searchable
|
||||||
from tests.integration.test_astrolabe_multi_user_background_sync import (
|
from tests.integration.test_astrolabe_multi_user_background_sync import (
|
||||||
complete_astrolabe_authorization,
|
complete_astrolabe_authorization,
|
||||||
login_to_nextcloud,
|
login_to_nextcloud,
|
||||||
@@ -39,14 +40,40 @@ pytestmark = [pytest.mark.integration, pytest.mark.multi_user_basic]
|
|||||||
|
|
||||||
|
|
||||||
async def wait_for_vector_sync(
|
async def wait_for_vector_sync(
|
||||||
mcp_client, initial_indexed_count: int, timeout_seconds: int = 60
|
mcp_client,
|
||||||
|
initial_indexed_count: int,
|
||||||
|
timeout_seconds: int = 60,
|
||||||
|
*,
|
||||||
|
search_term: str | None = None,
|
||||||
|
note_id: int | None = None,
|
||||||
) -> tuple[bool, dict | None]:
|
) -> tuple[bool, dict | None]:
|
||||||
"""Wait for vector sync to index new content.
|
"""Wait for vector sync to index newly-created content.
|
||||||
|
|
||||||
|
Completion signal:
|
||||||
|
|
||||||
|
- When ``search_term`` is provided (preferred), poll ``nc_semantic_search``
|
||||||
|
until the new document is actually retrievable. This is robust against
|
||||||
|
full-corpus re-scan churn and doubles as a real end-to-end check — it is
|
||||||
|
exactly what callers assert downstream.
|
||||||
|
- Otherwise, fall back to the legacy gauge-delta predicate.
|
||||||
|
|
||||||
|
Why the gauge delta is unreliable: under ``VECTOR_SYNC_SCAN_INTERVAL`` the
|
||||||
|
background sync re-queues the whole corpus every scan, so the corpus-wide
|
||||||
|
``indexed_count`` is *non-monotonic* — it can be re-counted downward
|
||||||
|
mid-scan. ``indexed_count > initial_indexed_count`` can therefore never hold
|
||||||
|
even though the new document is indexed and the status has settled to
|
||||||
|
``idle`` / ``pending_count == 0``. That false failure was the dominant
|
||||||
|
multi-user-basic CI flake (``test_astrolabe_plotly_visualization`` /
|
||||||
|
``test_astrolabe_chunk_context``).
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
mcp_client: MCP client session
|
mcp_client: MCP client session
|
||||||
initial_indexed_count: Initial indexed document count before creating content
|
initial_indexed_count: Indexed document count before creating content
|
||||||
|
(only used by the legacy gauge-delta fallback)
|
||||||
timeout_seconds: Maximum time to wait for sync
|
timeout_seconds: Maximum time to wait for sync
|
||||||
|
search_term: Unique term contained in the new document; enables the
|
||||||
|
robust searchability-based completion signal
|
||||||
|
note_id: ID of the new document, used to match search results exactly
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple of (success, status_data)
|
Tuple of (success, status_data)
|
||||||
@@ -73,7 +100,14 @@ async def wait_for_vector_sync(
|
|||||||
status_data.get("status"),
|
status_data.get("status"),
|
||||||
)
|
)
|
||||||
|
|
||||||
if indexed_count > initial_indexed_count and pending_count == 0:
|
if search_term is not None:
|
||||||
|
if await document_is_searchable(mcp_client, search_term, note_id):
|
||||||
|
logger.info(
|
||||||
|
"✓ Sync complete: document %s retrievable via semantic search",
|
||||||
|
note_id,
|
||||||
|
)
|
||||||
|
return True, status_data
|
||||||
|
elif indexed_count > initial_indexed_count and pending_count == 0:
|
||||||
logger.info(
|
logger.info(
|
||||||
"✓ Sync complete: %s documents indexed (was %s)",
|
"✓ Sync complete: %s documents indexed (was %s)",
|
||||||
indexed_count,
|
indexed_count,
|
||||||
@@ -197,24 +231,41 @@ The visualization should show this document as a point in PCA-reduced space.
|
|||||||
|
|
||||||
# Phase 4: Wait for vector indexing
|
# Phase 4: Wait for vector indexing
|
||||||
sync_complete, status = await wait_for_vector_sync(
|
sync_complete, status = await wait_for_vector_sync(
|
||||||
alice_mcp_client, initial_count, timeout_seconds=90
|
alice_mcp_client,
|
||||||
|
initial_count,
|
||||||
|
timeout_seconds=90,
|
||||||
|
search_term=unique_term,
|
||||||
|
note_id=note_id,
|
||||||
|
)
|
||||||
|
assert sync_complete, (
|
||||||
|
f"Note {note_id} ({unique_term}) never became searchable "
|
||||||
|
f"within timeout. Last sync status: {status}"
|
||||||
)
|
)
|
||||||
assert sync_complete, f"Vector sync did not complete in time: {status}"
|
|
||||||
|
|
||||||
# Phase 5: Navigate to Astrolabe and perform search
|
# Phase 5: Navigate to Astrolabe and perform search
|
||||||
await navigate_to_astrolabe_main(page)
|
await navigate_to_astrolabe_main(page)
|
||||||
|
|
||||||
# Fill search query - find the Astrolabe search input specifically
|
# Find the Astrolabe search field. The published app differs across
|
||||||
# The NcTextField component wraps the input in a div with class mcp-search-input
|
# the NC matrix: NC31 pulls astrolabe <=0.24 (NcTextField -> <input>,
|
||||||
search_input = page.locator(".mcp-search-input input")
|
# submits on Enter); NC32 pulls astrolabe >=0.25 (NcTextArea ->
|
||||||
await search_input.wait_for(timeout=10000, state="visible")
|
# <textarea>, submits on Ctrl/Cmd+Enter). Match either element so the
|
||||||
|
# test isn't pinned to one frontend revision.
|
||||||
|
# 30s (not 10s): the SPA can be slow to mount on a loaded CI runner.
|
||||||
|
search_input = page.locator(
|
||||||
|
".mcp-search-input textarea, .mcp-search-input input"
|
||||||
|
).first
|
||||||
|
await search_input.wait_for(timeout=30000, state="visible")
|
||||||
await search_input.fill(unique_term)
|
await search_input.fill(unique_term)
|
||||||
logger.info("Entered search query: %s", unique_term)
|
logger.info("Entered search query: %s", unique_term)
|
||||||
|
|
||||||
# Trigger search by pressing Enter on the input field
|
# Trigger search. NcTextField submits on Enter; NcTextArea inserts a
|
||||||
# This is wired to performSearch via @keyup.enter in the Vue component
|
# newline on Enter and submits on Ctrl/Cmd+Enter — so key off the tag.
|
||||||
await search_input.press("Enter")
|
field_tag = await search_input.evaluate("el => el.tagName.toLowerCase()")
|
||||||
logger.info("Pressed Enter to trigger search")
|
if field_tag == "textarea":
|
||||||
|
await search_input.press("Control+Enter")
|
||||||
|
else:
|
||||||
|
await search_input.press("Enter")
|
||||||
|
logger.info("Triggered search via %s submit", field_tag)
|
||||||
|
|
||||||
# Wait for loading to complete - watch for loading indicator to disappear
|
# Wait for loading to complete - watch for loading indicator to disappear
|
||||||
loading_indicator = page.locator(".mcp-loading")
|
loading_indicator = page.locator(".mcp-loading")
|
||||||
|
|||||||
@@ -21,17 +21,46 @@ HTTP with BasicAuth (which establishes a Nextcloud session for the request) —
|
|||||||
no browser needed.
|
no browser needed.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
|
|
||||||
|
import anyio
|
||||||
import httpx
|
import httpx
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
pytestmark = [pytest.mark.integration, pytest.mark.login_flow]
|
pytestmark = [pytest.mark.integration, pytest.mark.login_flow]
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
NEXTCLOUD_URL = "http://localhost:8080"
|
NEXTCLOUD_URL = "http://localhost:8080"
|
||||||
ASTROLABE_API = f"{NEXTCLOUD_URL}/apps/astrolabe/api"
|
ASTROLABE_API = f"{NEXTCLOUD_URL}/apps/astrolabe/api"
|
||||||
_HEADERS = {"OCS-APIRequest": "true"}
|
_HEADERS = {"OCS-APIRequest": "true"}
|
||||||
|
|
||||||
|
# The first /search after container start is slow: astrolabe mints a JWT and
|
||||||
|
# the MCP server runs a semantic search that may cold-load the embedding model.
|
||||||
|
# A single 30s read timeout was a CI flake (httpx.ReadTimeout); give the search
|
||||||
|
# path a generous budget and one retry on transient transport errors.
|
||||||
|
_SEARCH_TIMEOUT = httpx.Timeout(90.0)
|
||||||
|
|
||||||
|
|
||||||
|
async def _get_with_retry(
|
||||||
|
client: httpx.AsyncClient, url: str, *, max_attempts: int = 2, **kwargs
|
||||||
|
) -> httpx.Response:
|
||||||
|
"""GET, retrying on transient transport errors (timeouts/conn resets)."""
|
||||||
|
last_exc: httpx.TransportError | None = None
|
||||||
|
for attempt in range(1, max_attempts + 1):
|
||||||
|
try:
|
||||||
|
return await client.get(url, **kwargs)
|
||||||
|
except httpx.TransportError as e: # covers timeouts + connect/read errors
|
||||||
|
last_exc = e
|
||||||
|
logger.warning(
|
||||||
|
"GET %s failed (attempt %s/%s): %s", url, attempt, max_attempts, e
|
||||||
|
)
|
||||||
|
if attempt < max_attempts:
|
||||||
|
await anyio.sleep(2) # no point sleeping before we give up
|
||||||
|
assert last_exc is not None # loop ran at least once, so this is set
|
||||||
|
raise last_exc
|
||||||
|
|
||||||
|
|
||||||
async def _astrolabe_configured(client: httpx.AsyncClient, auth) -> bool:
|
async def _astrolabe_configured(client: httpx.AsyncClient, auth) -> bool:
|
||||||
"""Readiness probe: astrolabe must be able to reach its MCP server."""
|
"""Readiness probe: astrolabe must be able to reach its MCP server."""
|
||||||
@@ -46,6 +75,7 @@ async def _astrolabe_configured(client: httpx.AsyncClient, auth) -> bool:
|
|||||||
return bool(resp.json().get("success"))
|
return bool(resp.json().get("success"))
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.timeout(300) # cold model load + retry can exceed the 180s default
|
||||||
async def test_session_user_searches_without_provisioning(test_users_setup):
|
async def test_session_user_searches_without_provisioning(test_users_setup):
|
||||||
"""A non-admin session user searches with no OAuth/provisioning step.
|
"""A non-admin session user searches with no OAuth/provisioning step.
|
||||||
|
|
||||||
@@ -66,11 +96,13 @@ async def test_session_user_searches_without_provisioning(test_users_setup):
|
|||||||
"precondition: bob has not opted into background indexing"
|
"precondition: bob has not opted into background indexing"
|
||||||
)
|
)
|
||||||
|
|
||||||
resp = await client.get(
|
resp = await _get_with_retry(
|
||||||
|
client,
|
||||||
f"{ASTROLABE_API}/search",
|
f"{ASTROLABE_API}/search",
|
||||||
params={"query": "quarterly planning", "limit": 3},
|
params={"query": "quarterly planning", "limit": 3},
|
||||||
auth=auth,
|
auth=auth,
|
||||||
headers=_HEADERS,
|
headers=_HEADERS,
|
||||||
|
timeout=_SEARCH_TIMEOUT,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert resp.status_code == 200, resp.text
|
assert resp.status_code == 200, resp.text
|
||||||
@@ -81,6 +113,7 @@ async def test_session_user_searches_without_provisioning(test_users_setup):
|
|||||||
assert "results" in body and "algorithm_used" in body
|
assert "results" in body and "algorithm_used" in body
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.timeout(300) # cold model load + retry can exceed the 180s default
|
||||||
async def test_admin_session_search_succeeds():
|
async def test_admin_session_search_succeeds():
|
||||||
"""The same JWT-mint path works for the admin session user."""
|
"""The same JWT-mint path works for the admin session user."""
|
||||||
admin_pw = os.environ["NEXTCLOUD_PASSWORD"]
|
admin_pw = os.environ["NEXTCLOUD_PASSWORD"]
|
||||||
@@ -88,11 +121,13 @@ async def test_admin_session_search_succeeds():
|
|||||||
async with httpx.AsyncClient(timeout=30) as client:
|
async with httpx.AsyncClient(timeout=30) as client:
|
||||||
if not await _astrolabe_configured(client, auth):
|
if not await _astrolabe_configured(client, auth):
|
||||||
pytest.skip("Astrolabe not wired to an MCP server in this stack")
|
pytest.skip("Astrolabe not wired to an MCP server in this stack")
|
||||||
resp = await client.get(
|
resp = await _get_with_retry(
|
||||||
|
client,
|
||||||
f"{ASTROLABE_API}/search",
|
f"{ASTROLABE_API}/search",
|
||||||
params={"query": "infrastructure", "limit": 3},
|
params={"query": "infrastructure", "limit": 3},
|
||||||
auth=auth,
|
auth=auth,
|
||||||
headers=_HEADERS,
|
headers=_HEADERS,
|
||||||
|
timeout=_SEARCH_TIMEOUT,
|
||||||
)
|
)
|
||||||
assert resp.status_code == 200, resp.text
|
assert resp.status_code == 200, resp.text
|
||||||
assert resp.json()["success"] is True
|
assert resp.json()["success"] is True
|
||||||
|
|||||||
@@ -174,16 +174,22 @@ async def indexed_manual_pdf(nc_client, nc_mcp_client):
|
|||||||
content = json.loads(result.content[0].text) if result.content else {}
|
content = json.loads(result.content[0].text) if result.content else {}
|
||||||
indexed = content.get("indexed_count", 0)
|
indexed = content.get("indexed_count", 0)
|
||||||
pending = content.get("pending_count", 1)
|
pending = content.get("pending_count", 1)
|
||||||
|
status = content.get("status")
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
"Attempt %s/%s: indexed=%s, pending=%s",
|
"Attempt %s/%s: indexed=%s, pending=%s, status=%s",
|
||||||
attempt,
|
attempt,
|
||||||
max_attempts,
|
max_attempts,
|
||||||
indexed,
|
indexed,
|
||||||
pending,
|
pending,
|
||||||
|
status,
|
||||||
)
|
)
|
||||||
|
|
||||||
if indexed > 0 and pending == 0:
|
# Require indexed > 0 (the manual must actually be indexed —
|
||||||
|
# idle/pending==0 is also the *initial* empty state) AND a
|
||||||
|
# settled idle scan so we don't break during a transient
|
||||||
|
# pending==0 window mid re-scan churn.
|
||||||
|
if indexed > 0 and pending == 0 and status == "idle":
|
||||||
logger.info(
|
logger.info(
|
||||||
"Vector indexing complete: %s documents indexed", indexed
|
"Vector indexing complete: %s documents indexed", indexed
|
||||||
)
|
)
|
||||||
@@ -399,27 +405,53 @@ async def test_retrieval_quality_all_queries(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
async def test_no_results_for_unrelated_query(nc_mcp_client, indexed_manual_pdf):
|
async def _top_score(nc_mcp_client: Any, query: str) -> float | None:
|
||||||
"""Test that completely unrelated queries return low/no scores.
|
"""Return the best fusion score for ``query``, or None if no results."""
|
||||||
|
|
||||||
The Nextcloud manual shouldn't have relevant content for
|
|
||||||
quantum physics queries.
|
|
||||||
"""
|
|
||||||
result = await nc_mcp_client.call_tool(
|
result = await nc_mcp_client.call_tool(
|
||||||
"nc_semantic_search",
|
"nc_semantic_search",
|
||||||
arguments={
|
arguments={"query": query, "limit": 5, "score_threshold": 0.0},
|
||||||
"query": "quantum entanglement hadron collider particle physics",
|
)
|
||||||
"limit": 5,
|
assert result.isError is False, result.content
|
||||||
"score_threshold": 0.5, # Higher threshold to filter irrelevant
|
data = json.loads(result.content[0].text)
|
||||||
},
|
results = data.get("results", [])
|
||||||
|
if not results: # guard the list directly, not via total_found
|
||||||
|
return None
|
||||||
|
return max(r["score"] for r in results)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_no_results_for_unrelated_query(nc_mcp_client, indexed_manual_pdf):
|
||||||
|
"""An unrelated query must not out-rank a genuinely relevant one.
|
||||||
|
|
||||||
|
The Nextcloud manual has no quantum-physics content, so a physics query
|
||||||
|
must not look *more* relevant than a real manual query.
|
||||||
|
|
||||||
|
We deliberately do NOT assert on an absolute score magnitude. Fusion scores
|
||||||
|
(RRF/DBSF) are rank-based, not calibrated relevance: the top hit saturates
|
||||||
|
near the high end of the range regardless of true relevance, so a hardcoded
|
||||||
|
``max_score < 0.8`` check was a CI flake (it tripped whenever the unrelated
|
||||||
|
query happened to retrieve any chunk at all). Comparing against a relevant
|
||||||
|
query on the same corpus is self-calibrating and stable.
|
||||||
|
"""
|
||||||
|
# No results for the nonsense query is the ideal outcome — treat as score
|
||||||
|
# 0.0 and fall through, so the comparison (and the manual-is-indexed check
|
||||||
|
# below) still runs instead of the test silently skipping every time the
|
||||||
|
# physics query finds nothing.
|
||||||
|
unrelated = (
|
||||||
|
await _top_score(
|
||||||
|
nc_mcp_client, "quantum entanglement hadron collider particle physics"
|
||||||
|
)
|
||||||
|
or 0.0
|
||||||
)
|
)
|
||||||
|
|
||||||
assert result.isError is False
|
relevant = await _top_score(
|
||||||
data = json.loads(result.content[0].text)
|
nc_mcp_client, "how do I enable two-factor authentication"
|
||||||
|
)
|
||||||
|
assert relevant is not None, (
|
||||||
|
"Relevant control query returned nothing — manual not indexed?"
|
||||||
|
)
|
||||||
|
|
||||||
# Should have few or no high-scoring results
|
# The unrelated query must not appear more relevant than the real one.
|
||||||
# Low score threshold means we might get some results, but they should be low quality
|
assert unrelated <= relevant, (
|
||||||
if data["total_found"] > 0:
|
f"Unrelated query scored {unrelated}, higher than the relevant "
|
||||||
# If results exist, they should have low scores
|
f"control query's {relevant} — retrieval is not discriminating."
|
||||||
max_score = max(r["score"] for r in data["results"])
|
)
|
||||||
assert max_score < 0.8, f"Unexpected high score {max_score} for unrelated query"
|
|
||||||
|
|||||||
@@ -20,6 +20,8 @@ import anyio
|
|||||||
import pytest
|
import pytest
|
||||||
from mcp.types import CreateMessageResult, TextContent
|
from mcp.types import CreateMessageResult, TextContent
|
||||||
|
|
||||||
|
from tests.integration._search_helpers import document_is_searchable
|
||||||
|
|
||||||
pytestmark = pytest.mark.integration
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
|
|
||||||
@@ -27,6 +29,8 @@ async def wait_for_vector_sync(
|
|||||||
nc_mcp_client,
|
nc_mcp_client,
|
||||||
*,
|
*,
|
||||||
initial_indexed_count: int | None = None,
|
initial_indexed_count: int | None = None,
|
||||||
|
search_term: str | None = None,
|
||||||
|
note_id: int | None = None,
|
||||||
max_wait: int = 90,
|
max_wait: int = 90,
|
||||||
wait_interval: int = 1,
|
wait_interval: int = 1,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
@@ -34,9 +38,15 @@ async def wait_for_vector_sync(
|
|||||||
|
|
||||||
Args:
|
Args:
|
||||||
nc_mcp_client: MCP client to poll status with.
|
nc_mcp_client: MCP client to poll status with.
|
||||||
initial_indexed_count: If set, wait until indexed_count exceeds this
|
search_term: If set (preferred), wait until a document matching this
|
||||||
value and pending_count reaches 0. Otherwise wait for idle with
|
term is retrievable via ``nc_semantic_search``. Robust against
|
||||||
no pending work.
|
full-corpus re-scan churn, where the corpus-wide ``indexed_count``
|
||||||
|
gauge is non-monotonic and ``indexed_count > initial`` can never
|
||||||
|
hold even though the document is indexed.
|
||||||
|
note_id: Optional exact-match document id paired with ``search_term``.
|
||||||
|
initial_indexed_count: Legacy gauge-delta fallback when no search_term
|
||||||
|
is given: wait until indexed_count exceeds this value and
|
||||||
|
pending_count reaches 0.
|
||||||
max_wait: Maximum seconds to wait before failing.
|
max_wait: Maximum seconds to wait before failing.
|
||||||
wait_interval: Seconds between status polls.
|
wait_interval: Seconds between status polls.
|
||||||
|
|
||||||
@@ -49,18 +59,32 @@ async def wait_for_vector_sync(
|
|||||||
sync_status = await nc_mcp_client.call_tool(
|
sync_status = await nc_mcp_client.call_tool(
|
||||||
"nc_get_vector_sync_status", arguments={}
|
"nc_get_vector_sync_status", arguments={}
|
||||||
)
|
)
|
||||||
status_data = json.loads(sync_status.content[0].text)
|
try:
|
||||||
|
status_data = json.loads(sync_status.content[0].text)
|
||||||
|
except (AttributeError, IndexError, ValueError):
|
||||||
|
# transient empty/error response — keep polling. .get() defaults
|
||||||
|
# below also keep an empty dict from triggering a false break.
|
||||||
|
status_data = {}
|
||||||
|
|
||||||
if initial_indexed_count is not None:
|
if search_term is not None:
|
||||||
# Wait for new document(s) to be indexed
|
# Robust signal: wait for the specific document to be retrievable
|
||||||
|
if await document_is_searchable(nc_mcp_client, search_term, note_id):
|
||||||
|
break
|
||||||
|
elif initial_indexed_count is not None:
|
||||||
|
# Legacy: wait for new document(s) to be indexed (gauge delta)
|
||||||
if (
|
if (
|
||||||
status_data["indexed_count"] > initial_indexed_count
|
status_data.get("indexed_count", 0) > initial_indexed_count
|
||||||
and status_data["pending_count"] == 0
|
and status_data.get("pending_count", 1) == 0
|
||||||
):
|
):
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
# Wait for all pending work to complete
|
# NOTE: idle + pending==0 is also the *initial empty* state, so this
|
||||||
if status_data["status"] == "idle" and status_data["pending_count"] == 0:
|
# can break before a caller's work is even enqueued — prefer passing
|
||||||
|
# search_term. Kept only for callers that just need a settled corpus.
|
||||||
|
if (
|
||||||
|
status_data.get("status") == "idle"
|
||||||
|
and status_data.get("pending_count", 1) == 0
|
||||||
|
):
|
||||||
break
|
break
|
||||||
|
|
||||||
await anyio.sleep(wait_interval)
|
await anyio.sleep(wait_interval)
|
||||||
@@ -117,14 +141,6 @@ async def test_semantic_search_answer_successful_sampling(
|
|||||||
"""
|
"""
|
||||||
await require_vector_sync_tools(nc_mcp_client)
|
await require_vector_sync_tools(nc_mcp_client)
|
||||||
|
|
||||||
# Get initial indexed count before creating note
|
|
||||||
|
|
||||||
initial_sync = await nc_mcp_client.call_tool(
|
|
||||||
"nc_get_vector_sync_status", arguments={}
|
|
||||||
)
|
|
||||||
initial_indexed_count = json.loads(initial_sync.content[0].text)["indexed_count"]
|
|
||||||
print(f"Initial indexed count: {initial_indexed_count}")
|
|
||||||
|
|
||||||
# Create a note with content about Python async
|
# Create a note with content about Python async
|
||||||
_note = await temporary_note_factory(
|
_note = await temporary_note_factory(
|
||||||
title="Python Async Guide",
|
title="Python Async Guide",
|
||||||
@@ -142,12 +158,13 @@ Avoid blocking operations in async code.""",
|
|||||||
)
|
)
|
||||||
print(f"Created note ID: {_note['id']}")
|
print(f"Created note ID: {_note['id']}")
|
||||||
|
|
||||||
# Wait for vector indexing to complete
|
# Wait for vector indexing to complete. Gate on the new note actually
|
||||||
status_data = await wait_for_vector_sync(
|
# being retrievable rather than on the corpus-wide indexed_count gauge,
|
||||||
nc_mcp_client, initial_indexed_count=initial_indexed_count
|
# which is non-monotonic under re-scan churn (see wait_for_vector_sync).
|
||||||
)
|
await wait_for_vector_sync(
|
||||||
assert status_data["indexed_count"] > initial_indexed_count, (
|
nc_mcp_client,
|
||||||
f"New note was not indexed (count stayed at {initial_indexed_count})"
|
search_term="Python Async Programming coroutines",
|
||||||
|
note_id=_note["id"],
|
||||||
)
|
)
|
||||||
|
|
||||||
# Mock the sampling call
|
# Mock the sampling call
|
||||||
@@ -267,8 +284,12 @@ async def test_semantic_search_answer_with_limit(nc_mcp_client, temporary_note_f
|
|||||||
category="Development",
|
category="Development",
|
||||||
)
|
)
|
||||||
|
|
||||||
# Wait for vector indexing to complete
|
# Wait until the batch is indexed — gate on the last note being searchable
|
||||||
await wait_for_vector_sync(nc_mcp_client)
|
# rather than a bare idle signal, which can fire before the new notes are
|
||||||
|
# even enqueued.
|
||||||
|
await wait_for_vector_sync(
|
||||||
|
nc_mcp_client, search_term="async context managers", note_id=_note3["id"]
|
||||||
|
)
|
||||||
|
|
||||||
call_result = await nc_mcp_client.call_tool(
|
call_result = await nc_mcp_client.call_tool(
|
||||||
"nc_semantic_search_answer",
|
"nc_semantic_search_answer",
|
||||||
@@ -308,8 +329,10 @@ async def test_semantic_search_answer_score_threshold(
|
|||||||
category="Test",
|
category="Test",
|
||||||
)
|
)
|
||||||
|
|
||||||
# Wait for vector indexing to complete
|
# Gate on the new note being searchable (not a bare idle signal).
|
||||||
await wait_for_vector_sync(nc_mcp_client)
|
await wait_for_vector_sync(
|
||||||
|
nc_mcp_client, search_term="widget manufacturing", note_id=_note["id"]
|
||||||
|
)
|
||||||
|
|
||||||
# Query with exact match
|
# Query with exact match
|
||||||
call_result = await nc_mcp_client.call_tool(
|
call_result = await nc_mcp_client.call_tool(
|
||||||
@@ -355,8 +378,10 @@ async def test_semantic_search_answer_max_tokens(nc_mcp_client, temporary_note_f
|
|||||||
category="Test",
|
category="Test",
|
||||||
)
|
)
|
||||||
|
|
||||||
# Wait for vector indexing to complete
|
# Gate on the new note being searchable (not a bare idle signal).
|
||||||
await wait_for_vector_sync(nc_mcp_client)
|
await wait_for_vector_sync(
|
||||||
|
nc_mcp_client, search_term="Long Document content", note_id=_note["id"]
|
||||||
|
)
|
||||||
|
|
||||||
call_result = await nc_mcp_client.call_tool(
|
call_result = await nc_mcp_client.call_tool(
|
||||||
"nc_semantic_search_answer",
|
"nc_semantic_search_answer",
|
||||||
|
|||||||
@@ -105,7 +105,7 @@ async def get_oauth_token_with_client(
|
|||||||
|
|
||||||
# Wait for callback
|
# Wait for callback
|
||||||
logger.info("Waiting for OAuth callback...")
|
logger.info("Waiting for OAuth callback...")
|
||||||
timeout_seconds = 30
|
timeout_seconds = 60 # was 30; too tight for consent+redirect on loaded CI
|
||||||
start_time = time.time()
|
start_time = time.time()
|
||||||
while state not in auth_states:
|
while state not in auth_states:
|
||||||
if time.time() - start_time > timeout_seconds:
|
if time.time() - start_time > timeout_seconds:
|
||||||
|
|||||||
@@ -161,7 +161,7 @@ async def get_oauth_token_with_client(
|
|||||||
|
|
||||||
# Wait for callback
|
# Wait for callback
|
||||||
logger.info("Waiting for OAuth callback...")
|
logger.info("Waiting for OAuth callback...")
|
||||||
timeout_seconds = 30
|
timeout_seconds = 60 # was 30; too tight for consent+redirect on loaded CI
|
||||||
start_time = time.time()
|
start_time = time.time()
|
||||||
while state not in auth_states:
|
while state not in auth_states:
|
||||||
if time.time() - start_time > timeout_seconds:
|
if time.time() - start_time > timeout_seconds:
|
||||||
|
|||||||
@@ -241,7 +241,7 @@ async def _obtain_token_for_client(
|
|||||||
|
|
||||||
# Wait for callback server to receive auth code
|
# Wait for callback server to receive auth code
|
||||||
logger.info("Waiting for callback server to receive auth code...")
|
logger.info("Waiting for callback server to receive auth code...")
|
||||||
timeout_seconds = 30
|
timeout_seconds = 60 # was 30; too tight for consent+redirect on loaded CI
|
||||||
start_time = time.time()
|
start_time = time.time()
|
||||||
while state not in auth_states:
|
while state not in auth_states:
|
||||||
if time.time() - start_time > timeout_seconds:
|
if time.time() - start_time > timeout_seconds:
|
||||||
|
|||||||
Reference in New Issue
Block a user