Merge pull request #921 from cbcoutinho/fix/vector-sync-test-reliability
test(integration): fix vector-sync flake by gating on document searchability
This commit is contained in:
+2
-2
@@ -1987,7 +1987,7 @@ async def playwright_oauth_token(
|
||||
# Wait for callback server to receive the auth code
|
||||
# Browser will be redirected to localhost:8081 which will capture the code
|
||||
logger.info("Waiting for callback server to receive auth code...")
|
||||
timeout_seconds = 30
|
||||
timeout_seconds = 60 # was 30; too tight for consent+redirect on loaded CI
|
||||
start_time = time.time()
|
||||
while state not in auth_states:
|
||||
if time.time() - start_time > timeout_seconds:
|
||||
@@ -2696,7 +2696,7 @@ async def _get_oauth_token_for_user(
|
||||
logger.info(
|
||||
"Waiting for callback server to receive auth code for %s...", username
|
||||
)
|
||||
timeout_seconds = 30
|
||||
timeout_seconds = 60 # was 30; too tight for consent+redirect on loaded CI
|
||||
start_time = time.time()
|
||||
while state not in auth_states:
|
||||
if time.time() - start_time > timeout_seconds:
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Shared helpers for asserting vector-sync visibility in integration tests.
|
||||
|
||||
Kept dependency-light (no Playwright) so both the multi-user-basic UI tests and
|
||||
the single-user sampling tests can import it.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def document_is_searchable(
|
||||
mcp_client: Any, search_term: str, note_id: int | None = None
|
||||
) -> bool:
|
||||
"""Return True once a freshly-created document is retrievable.
|
||||
|
||||
Polls ``nc_semantic_search`` (hybrid: an exact unique term reliably matches
|
||||
on the keyword side) and matches by ``note_id`` when provided, otherwise by
|
||||
the term appearing in a result's title/excerpt. Transient errors return
|
||||
False so callers can keep polling.
|
||||
"""
|
||||
try:
|
||||
search = await mcp_client.call_tool(
|
||||
"nc_semantic_search",
|
||||
# limit is generous: a fresh note can sit below seed data (e.g. deck
|
||||
# cards) in a crowded corpus, and the query is cheap.
|
||||
arguments={"query": search_term, "limit": 50, "score_threshold": 0.0},
|
||||
)
|
||||
except Exception as e: # transient transport/availability blip — keep polling
|
||||
logger.debug("Semantic search poll failed: %s", e)
|
||||
return False
|
||||
if search.isError:
|
||||
logger.debug("Semantic search poll error: %s", search)
|
||||
return False
|
||||
|
||||
try:
|
||||
results = json.loads(search.content[0].text).get("results", [])
|
||||
except (IndexError, ValueError) as e: # empty content / malformed JSON
|
||||
logger.debug("Semantic search parse failed: %s", e)
|
||||
return False
|
||||
|
||||
# Token match (not contiguous substring) so multi-word terms work in the
|
||||
# note_id-less fallback path.
|
||||
tokens = search_term.lower().split()
|
||||
for r in results:
|
||||
if note_id is not None:
|
||||
# str-coerce both sides: nc_semantic_search returns int ids today,
|
||||
# but the Astrolabe API serialises some ids as strings — match the
|
||||
# defensive comparison in _poll_astrolabe_search_for_note so a future
|
||||
# schema change can't silently break the match.
|
||||
if str(r.get("id")) == str(note_id):
|
||||
if r.get("doc_type") == "note":
|
||||
return True
|
||||
# id matched but not a note — surface possible schema drift at
|
||||
# WARNING (CI runs --log-cli-level=WARN) instead of letting the
|
||||
# caller time out with a generic message.
|
||||
logger.warning(
|
||||
"search hit id=%s has doc_type=%s (expected note)",
|
||||
note_id,
|
||||
r.get("doc_type"),
|
||||
)
|
||||
else:
|
||||
haystack = f"{r.get('title', '')} {r.get('excerpt', '')}".lower()
|
||||
if tokens and all(t in haystack for t in tokens):
|
||||
return True
|
||||
return False
|
||||
@@ -56,11 +56,11 @@ async def _poll_astrolabe_search_for_note(
|
||||
) -> dict:
|
||||
"""Poll Astrolabe's search endpoint until `note_id` shows up in results.
|
||||
|
||||
`wait_for_vector_sync` only waits for the total indexed count to grow —
|
||||
it does not guarantee that *this specific* document is visible yet
|
||||
(observed on nc32 where deck-card seed data indexes first and the new
|
||||
note arrives in Qdrant a few seconds later). Poll until the unique term
|
||||
returns our note, or fail loudly with the last response we saw.
|
||||
`wait_for_vector_sync` now gates on this specific document being
|
||||
retrievable via the MCP semantic-search tool, but Astrolabe's own search
|
||||
endpoint is a distinct read path (its own JWT + query handler), so we still
|
||||
poll it here until the unique term returns our note — or fail loudly with
|
||||
the last response we saw.
|
||||
"""
|
||||
deadline = time.monotonic() + timeout_seconds
|
||||
last_results: list | None = None
|
||||
@@ -170,9 +170,16 @@ async def test_chunk_context_endpoint_uses_app_password(
|
||||
assert note_id is not None
|
||||
|
||||
sync_complete, status = await wait_for_vector_sync(
|
||||
mcp_client, initial_count, timeout_seconds=90
|
||||
mcp_client,
|
||||
initial_count,
|
||||
timeout_seconds=90,
|
||||
search_term=unique_term,
|
||||
note_id=note_id,
|
||||
)
|
||||
assert sync_complete, (
|
||||
f"Note {note_id} ({unique_term}) never became searchable "
|
||||
f"within timeout. Last sync status: {status}"
|
||||
)
|
||||
assert sync_complete, f"Vector sync did not complete: {status}"
|
||||
|
||||
# Use the browser's session to drive Astrolabe end-to-end, the way a
|
||||
# real user would: this exercises astrolabe's OAuth token retrieval
|
||||
|
||||
@@ -28,6 +28,7 @@ from playwright.async_api import Page
|
||||
|
||||
# Import helper functions from existing test
|
||||
from tests.conftest import create_mcp_client_session
|
||||
from tests.integration._search_helpers import document_is_searchable
|
||||
from tests.integration.test_astrolabe_multi_user_background_sync import (
|
||||
complete_astrolabe_authorization,
|
||||
login_to_nextcloud,
|
||||
@@ -39,14 +40,40 @@ pytestmark = [pytest.mark.integration, pytest.mark.multi_user_basic]
|
||||
|
||||
|
||||
async def wait_for_vector_sync(
|
||||
mcp_client, initial_indexed_count: int, timeout_seconds: int = 60
|
||||
mcp_client,
|
||||
initial_indexed_count: int,
|
||||
timeout_seconds: int = 60,
|
||||
*,
|
||||
search_term: str | None = None,
|
||||
note_id: int | None = None,
|
||||
) -> tuple[bool, dict | None]:
|
||||
"""Wait for vector sync to index new content.
|
||||
"""Wait for vector sync to index newly-created content.
|
||||
|
||||
Completion signal:
|
||||
|
||||
- When ``search_term`` is provided (preferred), poll ``nc_semantic_search``
|
||||
until the new document is actually retrievable. This is robust against
|
||||
full-corpus re-scan churn and doubles as a real end-to-end check — it is
|
||||
exactly what callers assert downstream.
|
||||
- Otherwise, fall back to the legacy gauge-delta predicate.
|
||||
|
||||
Why the gauge delta is unreliable: under ``VECTOR_SYNC_SCAN_INTERVAL`` the
|
||||
background sync re-queues the whole corpus every scan, so the corpus-wide
|
||||
``indexed_count`` is *non-monotonic* — it can be re-counted downward
|
||||
mid-scan. ``indexed_count > initial_indexed_count`` can therefore never hold
|
||||
even though the new document is indexed and the status has settled to
|
||||
``idle`` / ``pending_count == 0``. That false failure was the dominant
|
||||
multi-user-basic CI flake (``test_astrolabe_plotly_visualization`` /
|
||||
``test_astrolabe_chunk_context``).
|
||||
|
||||
Args:
|
||||
mcp_client: MCP client session
|
||||
initial_indexed_count: Initial indexed document count before creating content
|
||||
initial_indexed_count: Indexed document count before creating content
|
||||
(only used by the legacy gauge-delta fallback)
|
||||
timeout_seconds: Maximum time to wait for sync
|
||||
search_term: Unique term contained in the new document; enables the
|
||||
robust searchability-based completion signal
|
||||
note_id: ID of the new document, used to match search results exactly
|
||||
|
||||
Returns:
|
||||
Tuple of (success, status_data)
|
||||
@@ -73,7 +100,14 @@ async def wait_for_vector_sync(
|
||||
status_data.get("status"),
|
||||
)
|
||||
|
||||
if indexed_count > initial_indexed_count and pending_count == 0:
|
||||
if search_term is not None:
|
||||
if await document_is_searchable(mcp_client, search_term, note_id):
|
||||
logger.info(
|
||||
"✓ Sync complete: document %s retrievable via semantic search",
|
||||
note_id,
|
||||
)
|
||||
return True, status_data
|
||||
elif indexed_count > initial_indexed_count and pending_count == 0:
|
||||
logger.info(
|
||||
"✓ Sync complete: %s documents indexed (was %s)",
|
||||
indexed_count,
|
||||
@@ -197,24 +231,41 @@ The visualization should show this document as a point in PCA-reduced space.
|
||||
|
||||
# Phase 4: Wait for vector indexing
|
||||
sync_complete, status = await wait_for_vector_sync(
|
||||
alice_mcp_client, initial_count, timeout_seconds=90
|
||||
alice_mcp_client,
|
||||
initial_count,
|
||||
timeout_seconds=90,
|
||||
search_term=unique_term,
|
||||
note_id=note_id,
|
||||
)
|
||||
assert sync_complete, (
|
||||
f"Note {note_id} ({unique_term}) never became searchable "
|
||||
f"within timeout. Last sync status: {status}"
|
||||
)
|
||||
assert sync_complete, f"Vector sync did not complete in time: {status}"
|
||||
|
||||
# Phase 5: Navigate to Astrolabe and perform search
|
||||
await navigate_to_astrolabe_main(page)
|
||||
|
||||
# Fill search query - find the Astrolabe search input specifically
|
||||
# The NcTextField component wraps the input in a div with class mcp-search-input
|
||||
search_input = page.locator(".mcp-search-input input")
|
||||
await search_input.wait_for(timeout=10000, state="visible")
|
||||
# Find the Astrolabe search field. The published app differs across
|
||||
# the NC matrix: NC31 pulls astrolabe <=0.24 (NcTextField -> <input>,
|
||||
# submits on Enter); NC32 pulls astrolabe >=0.25 (NcTextArea ->
|
||||
# <textarea>, submits on Ctrl/Cmd+Enter). Match either element so the
|
||||
# test isn't pinned to one frontend revision.
|
||||
# 30s (not 10s): the SPA can be slow to mount on a loaded CI runner.
|
||||
search_input = page.locator(
|
||||
".mcp-search-input textarea, .mcp-search-input input"
|
||||
).first
|
||||
await search_input.wait_for(timeout=30000, state="visible")
|
||||
await search_input.fill(unique_term)
|
||||
logger.info("Entered search query: %s", unique_term)
|
||||
|
||||
# Trigger search by pressing Enter on the input field
|
||||
# This is wired to performSearch via @keyup.enter in the Vue component
|
||||
await search_input.press("Enter")
|
||||
logger.info("Pressed Enter to trigger search")
|
||||
# Trigger search. NcTextField submits on Enter; NcTextArea inserts a
|
||||
# newline on Enter and submits on Ctrl/Cmd+Enter — so key off the tag.
|
||||
field_tag = await search_input.evaluate("el => el.tagName.toLowerCase()")
|
||||
if field_tag == "textarea":
|
||||
await search_input.press("Control+Enter")
|
||||
else:
|
||||
await search_input.press("Enter")
|
||||
logger.info("Triggered search via %s submit", field_tag)
|
||||
|
||||
# Wait for loading to complete - watch for loading indicator to disappear
|
||||
loading_indicator = page.locator(".mcp-loading")
|
||||
|
||||
@@ -21,17 +21,46 @@ HTTP with BasicAuth (which establishes a Nextcloud session for the request) —
|
||||
no browser needed.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
import anyio
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
pytestmark = [pytest.mark.integration, pytest.mark.login_flow]
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
NEXTCLOUD_URL = "http://localhost:8080"
|
||||
ASTROLABE_API = f"{NEXTCLOUD_URL}/apps/astrolabe/api"
|
||||
_HEADERS = {"OCS-APIRequest": "true"}
|
||||
|
||||
# The first /search after container start is slow: astrolabe mints a JWT and
|
||||
# the MCP server runs a semantic search that may cold-load the embedding model.
|
||||
# A single 30s read timeout was a CI flake (httpx.ReadTimeout); give the search
|
||||
# path a generous budget and one retry on transient transport errors.
|
||||
_SEARCH_TIMEOUT = httpx.Timeout(90.0)
|
||||
|
||||
|
||||
async def _get_with_retry(
|
||||
client: httpx.AsyncClient, url: str, *, max_attempts: int = 2, **kwargs
|
||||
) -> httpx.Response:
|
||||
"""GET, retrying on transient transport errors (timeouts/conn resets)."""
|
||||
last_exc: httpx.TransportError | None = None
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
try:
|
||||
return await client.get(url, **kwargs)
|
||||
except httpx.TransportError as e: # covers timeouts + connect/read errors
|
||||
last_exc = e
|
||||
logger.warning(
|
||||
"GET %s failed (attempt %s/%s): %s", url, attempt, max_attempts, e
|
||||
)
|
||||
if attempt < max_attempts:
|
||||
await anyio.sleep(2) # no point sleeping before we give up
|
||||
assert last_exc is not None # loop ran at least once, so this is set
|
||||
raise last_exc
|
||||
|
||||
|
||||
async def _astrolabe_configured(client: httpx.AsyncClient, auth) -> bool:
|
||||
"""Readiness probe: astrolabe must be able to reach its MCP server."""
|
||||
@@ -46,6 +75,7 @@ async def _astrolabe_configured(client: httpx.AsyncClient, auth) -> bool:
|
||||
return bool(resp.json().get("success"))
|
||||
|
||||
|
||||
@pytest.mark.timeout(300) # cold model load + retry can exceed the 180s default
|
||||
async def test_session_user_searches_without_provisioning(test_users_setup):
|
||||
"""A non-admin session user searches with no OAuth/provisioning step.
|
||||
|
||||
@@ -66,11 +96,13 @@ async def test_session_user_searches_without_provisioning(test_users_setup):
|
||||
"precondition: bob has not opted into background indexing"
|
||||
)
|
||||
|
||||
resp = await client.get(
|
||||
resp = await _get_with_retry(
|
||||
client,
|
||||
f"{ASTROLABE_API}/search",
|
||||
params={"query": "quarterly planning", "limit": 3},
|
||||
auth=auth,
|
||||
headers=_HEADERS,
|
||||
timeout=_SEARCH_TIMEOUT,
|
||||
)
|
||||
|
||||
assert resp.status_code == 200, resp.text
|
||||
@@ -81,6 +113,7 @@ async def test_session_user_searches_without_provisioning(test_users_setup):
|
||||
assert "results" in body and "algorithm_used" in body
|
||||
|
||||
|
||||
@pytest.mark.timeout(300) # cold model load + retry can exceed the 180s default
|
||||
async def test_admin_session_search_succeeds():
|
||||
"""The same JWT-mint path works for the admin session user."""
|
||||
admin_pw = os.environ["NEXTCLOUD_PASSWORD"]
|
||||
@@ -88,11 +121,13 @@ async def test_admin_session_search_succeeds():
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
if not await _astrolabe_configured(client, auth):
|
||||
pytest.skip("Astrolabe not wired to an MCP server in this stack")
|
||||
resp = await client.get(
|
||||
resp = await _get_with_retry(
|
||||
client,
|
||||
f"{ASTROLABE_API}/search",
|
||||
params={"query": "infrastructure", "limit": 3},
|
||||
auth=auth,
|
||||
headers=_HEADERS,
|
||||
timeout=_SEARCH_TIMEOUT,
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert resp.json()["success"] is True
|
||||
|
||||
@@ -174,16 +174,22 @@ async def indexed_manual_pdf(nc_client, nc_mcp_client):
|
||||
content = json.loads(result.content[0].text) if result.content else {}
|
||||
indexed = content.get("indexed_count", 0)
|
||||
pending = content.get("pending_count", 1)
|
||||
status = content.get("status")
|
||||
|
||||
logger.info(
|
||||
"Attempt %s/%s: indexed=%s, pending=%s",
|
||||
"Attempt %s/%s: indexed=%s, pending=%s, status=%s",
|
||||
attempt,
|
||||
max_attempts,
|
||||
indexed,
|
||||
pending,
|
||||
status,
|
||||
)
|
||||
|
||||
if indexed > 0 and pending == 0:
|
||||
# Require indexed > 0 (the manual must actually be indexed —
|
||||
# idle/pending==0 is also the *initial* empty state) AND a
|
||||
# settled idle scan so we don't break during a transient
|
||||
# pending==0 window mid re-scan churn.
|
||||
if indexed > 0 and pending == 0 and status == "idle":
|
||||
logger.info(
|
||||
"Vector indexing complete: %s documents indexed", indexed
|
||||
)
|
||||
@@ -399,27 +405,53 @@ async def test_retrieval_quality_all_queries(
|
||||
)
|
||||
|
||||
|
||||
async def test_no_results_for_unrelated_query(nc_mcp_client, indexed_manual_pdf):
|
||||
"""Test that completely unrelated queries return low/no scores.
|
||||
|
||||
The Nextcloud manual shouldn't have relevant content for
|
||||
quantum physics queries.
|
||||
"""
|
||||
async def _top_score(nc_mcp_client: Any, query: str) -> float | None:
|
||||
"""Return the best fusion score for ``query``, or None if no results."""
|
||||
result = await nc_mcp_client.call_tool(
|
||||
"nc_semantic_search",
|
||||
arguments={
|
||||
"query": "quantum entanglement hadron collider particle physics",
|
||||
"limit": 5,
|
||||
"score_threshold": 0.5, # Higher threshold to filter irrelevant
|
||||
},
|
||||
arguments={"query": query, "limit": 5, "score_threshold": 0.0},
|
||||
)
|
||||
assert result.isError is False, result.content
|
||||
data = json.loads(result.content[0].text)
|
||||
results = data.get("results", [])
|
||||
if not results: # guard the list directly, not via total_found
|
||||
return None
|
||||
return max(r["score"] for r in results)
|
||||
|
||||
|
||||
async def test_no_results_for_unrelated_query(nc_mcp_client, indexed_manual_pdf):
|
||||
"""An unrelated query must not out-rank a genuinely relevant one.
|
||||
|
||||
The Nextcloud manual has no quantum-physics content, so a physics query
|
||||
must not look *more* relevant than a real manual query.
|
||||
|
||||
We deliberately do NOT assert on an absolute score magnitude. Fusion scores
|
||||
(RRF/DBSF) are rank-based, not calibrated relevance: the top hit saturates
|
||||
near the high end of the range regardless of true relevance, so a hardcoded
|
||||
``max_score < 0.8`` check was a CI flake (it tripped whenever the unrelated
|
||||
query happened to retrieve any chunk at all). Comparing against a relevant
|
||||
query on the same corpus is self-calibrating and stable.
|
||||
"""
|
||||
# No results for the nonsense query is the ideal outcome — treat as score
|
||||
# 0.0 and fall through, so the comparison (and the manual-is-indexed check
|
||||
# below) still runs instead of the test silently skipping every time the
|
||||
# physics query finds nothing.
|
||||
unrelated = (
|
||||
await _top_score(
|
||||
nc_mcp_client, "quantum entanglement hadron collider particle physics"
|
||||
)
|
||||
or 0.0
|
||||
)
|
||||
|
||||
assert result.isError is False
|
||||
data = json.loads(result.content[0].text)
|
||||
relevant = await _top_score(
|
||||
nc_mcp_client, "how do I enable two-factor authentication"
|
||||
)
|
||||
assert relevant is not None, (
|
||||
"Relevant control query returned nothing — manual not indexed?"
|
||||
)
|
||||
|
||||
# Should have few or no high-scoring results
|
||||
# Low score threshold means we might get some results, but they should be low quality
|
||||
if data["total_found"] > 0:
|
||||
# If results exist, they should have low scores
|
||||
max_score = max(r["score"] for r in data["results"])
|
||||
assert max_score < 0.8, f"Unexpected high score {max_score} for unrelated query"
|
||||
# The unrelated query must not appear more relevant than the real one.
|
||||
assert unrelated <= relevant, (
|
||||
f"Unrelated query scored {unrelated}, higher than the relevant "
|
||||
f"control query's {relevant} — retrieval is not discriminating."
|
||||
)
|
||||
|
||||
@@ -20,6 +20,8 @@ import anyio
|
||||
import pytest
|
||||
from mcp.types import CreateMessageResult, TextContent
|
||||
|
||||
from tests.integration._search_helpers import document_is_searchable
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
@@ -27,6 +29,8 @@ async def wait_for_vector_sync(
|
||||
nc_mcp_client,
|
||||
*,
|
||||
initial_indexed_count: int | None = None,
|
||||
search_term: str | None = None,
|
||||
note_id: int | None = None,
|
||||
max_wait: int = 90,
|
||||
wait_interval: int = 1,
|
||||
) -> dict:
|
||||
@@ -34,9 +38,15 @@ async def wait_for_vector_sync(
|
||||
|
||||
Args:
|
||||
nc_mcp_client: MCP client to poll status with.
|
||||
initial_indexed_count: If set, wait until indexed_count exceeds this
|
||||
value and pending_count reaches 0. Otherwise wait for idle with
|
||||
no pending work.
|
||||
search_term: If set (preferred), wait until a document matching this
|
||||
term is retrievable via ``nc_semantic_search``. Robust against
|
||||
full-corpus re-scan churn, where the corpus-wide ``indexed_count``
|
||||
gauge is non-monotonic and ``indexed_count > initial`` can never
|
||||
hold even though the document is indexed.
|
||||
note_id: Optional exact-match document id paired with ``search_term``.
|
||||
initial_indexed_count: Legacy gauge-delta fallback when no search_term
|
||||
is given: wait until indexed_count exceeds this value and
|
||||
pending_count reaches 0.
|
||||
max_wait: Maximum seconds to wait before failing.
|
||||
wait_interval: Seconds between status polls.
|
||||
|
||||
@@ -49,18 +59,32 @@ async def wait_for_vector_sync(
|
||||
sync_status = await nc_mcp_client.call_tool(
|
||||
"nc_get_vector_sync_status", arguments={}
|
||||
)
|
||||
status_data = json.loads(sync_status.content[0].text)
|
||||
try:
|
||||
status_data = json.loads(sync_status.content[0].text)
|
||||
except (AttributeError, IndexError, ValueError):
|
||||
# transient empty/error response — keep polling. .get() defaults
|
||||
# below also keep an empty dict from triggering a false break.
|
||||
status_data = {}
|
||||
|
||||
if initial_indexed_count is not None:
|
||||
# Wait for new document(s) to be indexed
|
||||
if search_term is not None:
|
||||
# Robust signal: wait for the specific document to be retrievable
|
||||
if await document_is_searchable(nc_mcp_client, search_term, note_id):
|
||||
break
|
||||
elif initial_indexed_count is not None:
|
||||
# Legacy: wait for new document(s) to be indexed (gauge delta)
|
||||
if (
|
||||
status_data["indexed_count"] > initial_indexed_count
|
||||
and status_data["pending_count"] == 0
|
||||
status_data.get("indexed_count", 0) > initial_indexed_count
|
||||
and status_data.get("pending_count", 1) == 0
|
||||
):
|
||||
break
|
||||
else:
|
||||
# Wait for all pending work to complete
|
||||
if status_data["status"] == "idle" and status_data["pending_count"] == 0:
|
||||
# NOTE: idle + pending==0 is also the *initial empty* state, so this
|
||||
# can break before a caller's work is even enqueued — prefer passing
|
||||
# search_term. Kept only for callers that just need a settled corpus.
|
||||
if (
|
||||
status_data.get("status") == "idle"
|
||||
and status_data.get("pending_count", 1) == 0
|
||||
):
|
||||
break
|
||||
|
||||
await anyio.sleep(wait_interval)
|
||||
@@ -117,14 +141,6 @@ async def test_semantic_search_answer_successful_sampling(
|
||||
"""
|
||||
await require_vector_sync_tools(nc_mcp_client)
|
||||
|
||||
# Get initial indexed count before creating note
|
||||
|
||||
initial_sync = await nc_mcp_client.call_tool(
|
||||
"nc_get_vector_sync_status", arguments={}
|
||||
)
|
||||
initial_indexed_count = json.loads(initial_sync.content[0].text)["indexed_count"]
|
||||
print(f"Initial indexed count: {initial_indexed_count}")
|
||||
|
||||
# Create a note with content about Python async
|
||||
_note = await temporary_note_factory(
|
||||
title="Python Async Guide",
|
||||
@@ -142,12 +158,13 @@ Avoid blocking operations in async code.""",
|
||||
)
|
||||
print(f"Created note ID: {_note['id']}")
|
||||
|
||||
# Wait for vector indexing to complete
|
||||
status_data = await wait_for_vector_sync(
|
||||
nc_mcp_client, initial_indexed_count=initial_indexed_count
|
||||
)
|
||||
assert status_data["indexed_count"] > initial_indexed_count, (
|
||||
f"New note was not indexed (count stayed at {initial_indexed_count})"
|
||||
# Wait for vector indexing to complete. Gate on the new note actually
|
||||
# being retrievable rather than on the corpus-wide indexed_count gauge,
|
||||
# which is non-monotonic under re-scan churn (see wait_for_vector_sync).
|
||||
await wait_for_vector_sync(
|
||||
nc_mcp_client,
|
||||
search_term="Python Async Programming coroutines",
|
||||
note_id=_note["id"],
|
||||
)
|
||||
|
||||
# Mock the sampling call
|
||||
@@ -267,8 +284,12 @@ async def test_semantic_search_answer_with_limit(nc_mcp_client, temporary_note_f
|
||||
category="Development",
|
||||
)
|
||||
|
||||
# Wait for vector indexing to complete
|
||||
await wait_for_vector_sync(nc_mcp_client)
|
||||
# Wait until the batch is indexed — gate on the last note being searchable
|
||||
# rather than a bare idle signal, which can fire before the new notes are
|
||||
# even enqueued.
|
||||
await wait_for_vector_sync(
|
||||
nc_mcp_client, search_term="async context managers", note_id=_note3["id"]
|
||||
)
|
||||
|
||||
call_result = await nc_mcp_client.call_tool(
|
||||
"nc_semantic_search_answer",
|
||||
@@ -308,8 +329,10 @@ async def test_semantic_search_answer_score_threshold(
|
||||
category="Test",
|
||||
)
|
||||
|
||||
# Wait for vector indexing to complete
|
||||
await wait_for_vector_sync(nc_mcp_client)
|
||||
# Gate on the new note being searchable (not a bare idle signal).
|
||||
await wait_for_vector_sync(
|
||||
nc_mcp_client, search_term="widget manufacturing", note_id=_note["id"]
|
||||
)
|
||||
|
||||
# Query with exact match
|
||||
call_result = await nc_mcp_client.call_tool(
|
||||
@@ -355,8 +378,10 @@ async def test_semantic_search_answer_max_tokens(nc_mcp_client, temporary_note_f
|
||||
category="Test",
|
||||
)
|
||||
|
||||
# Wait for vector indexing to complete
|
||||
await wait_for_vector_sync(nc_mcp_client)
|
||||
# Gate on the new note being searchable (not a bare idle signal).
|
||||
await wait_for_vector_sync(
|
||||
nc_mcp_client, search_term="Long Document content", note_id=_note["id"]
|
||||
)
|
||||
|
||||
call_result = await nc_mcp_client.call_tool(
|
||||
"nc_semantic_search_answer",
|
||||
|
||||
@@ -105,7 +105,7 @@ async def get_oauth_token_with_client(
|
||||
|
||||
# Wait for callback
|
||||
logger.info("Waiting for OAuth callback...")
|
||||
timeout_seconds = 30
|
||||
timeout_seconds = 60 # was 30; too tight for consent+redirect on loaded CI
|
||||
start_time = time.time()
|
||||
while state not in auth_states:
|
||||
if time.time() - start_time > timeout_seconds:
|
||||
|
||||
@@ -161,7 +161,7 @@ async def get_oauth_token_with_client(
|
||||
|
||||
# Wait for callback
|
||||
logger.info("Waiting for OAuth callback...")
|
||||
timeout_seconds = 30
|
||||
timeout_seconds = 60 # was 30; too tight for consent+redirect on loaded CI
|
||||
start_time = time.time()
|
||||
while state not in auth_states:
|
||||
if time.time() - start_time > timeout_seconds:
|
||||
|
||||
@@ -241,7 +241,7 @@ async def _obtain_token_for_client(
|
||||
|
||||
# Wait for callback server to receive auth code
|
||||
logger.info("Waiting for callback server to receive auth code...")
|
||||
timeout_seconds = 30
|
||||
timeout_seconds = 60 # was 30; too tight for consent+redirect on loaded CI
|
||||
start_time = time.time()
|
||||
while state not in auth_states:
|
||||
if time.time() - start_time > timeout_seconds:
|
||||
|
||||
Reference in New Issue
Block a user