diff --git a/tests/conftest.py b/tests/conftest.py index eb5fedbe..453e1122 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1987,7 +1987,7 @@ async def playwright_oauth_token( # Wait for callback server to receive the auth code # Browser will be redirected to localhost:8081 which will capture the code logger.info("Waiting for callback server to receive auth code...") - timeout_seconds = 30 + timeout_seconds = 60 # was 30; too tight for consent+redirect on loaded CI start_time = time.time() while state not in auth_states: if time.time() - start_time > timeout_seconds: @@ -2696,7 +2696,7 @@ async def _get_oauth_token_for_user( logger.info( "Waiting for callback server to receive auth code for %s...", username ) - timeout_seconds = 30 + timeout_seconds = 60 # was 30; too tight for consent+redirect on loaded CI start_time = time.time() while state not in auth_states: if time.time() - start_time > timeout_seconds: diff --git a/tests/integration/_search_helpers.py b/tests/integration/_search_helpers.py new file mode 100644 index 00000000..404d6835 --- /dev/null +++ b/tests/integration/_search_helpers.py @@ -0,0 +1,68 @@ +"""Shared helpers for asserting vector-sync visibility in integration tests. + +Kept dependency-light (no Playwright) so both the multi-user-basic UI tests and +the single-user sampling tests can import it. +""" + +import json +import logging +from typing import Any + +logger = logging.getLogger(__name__) + + +async def document_is_searchable( + mcp_client: Any, search_term: str, note_id: int | None = None +) -> bool: + """Return True once a freshly-created document is retrievable. + + Polls ``nc_semantic_search`` (hybrid: an exact unique term reliably matches + on the keyword side) and matches by ``note_id`` when provided, otherwise by + the term appearing in a result's title/excerpt. Transient errors return + False so callers can keep polling. + """ + try: + search = await mcp_client.call_tool( + "nc_semantic_search", + # limit is generous: a fresh note can sit below seed data (e.g. deck + # cards) in a crowded corpus, and the query is cheap. + arguments={"query": search_term, "limit": 50, "score_threshold": 0.0}, + ) + except Exception as e: # transient transport/availability blip — keep polling + logger.debug("Semantic search poll failed: %s", e) + return False + if search.isError: + logger.debug("Semantic search poll error: %s", search) + return False + + try: + results = json.loads(search.content[0].text).get("results", []) + except (IndexError, ValueError) as e: # empty content / malformed JSON + logger.debug("Semantic search parse failed: %s", e) + return False + + # Token match (not contiguous substring) so multi-word terms work in the + # note_id-less fallback path. + tokens = search_term.lower().split() + for r in results: + if note_id is not None: + # str-coerce both sides: nc_semantic_search returns int ids today, + # but the Astrolabe API serialises some ids as strings — match the + # defensive comparison in _poll_astrolabe_search_for_note so a future + # schema change can't silently break the match. + if str(r.get("id")) == str(note_id): + if r.get("doc_type") == "note": + return True + # id matched but not a note — surface possible schema drift at + # WARNING (CI runs --log-cli-level=WARN) instead of letting the + # caller time out with a generic message. + logger.warning( + "search hit id=%s has doc_type=%s (expected note)", + note_id, + r.get("doc_type"), + ) + else: + haystack = f"{r.get('title', '')} {r.get('excerpt', '')}".lower() + if tokens and all(t in haystack for t in tokens): + return True + return False diff --git a/tests/integration/test_astrolabe_chunk_context.py b/tests/integration/test_astrolabe_chunk_context.py index 0fcc259b..5d6fb5ef 100644 --- a/tests/integration/test_astrolabe_chunk_context.py +++ b/tests/integration/test_astrolabe_chunk_context.py @@ -56,11 +56,11 @@ async def _poll_astrolabe_search_for_note( ) -> dict: """Poll Astrolabe's search endpoint until `note_id` shows up in results. - `wait_for_vector_sync` only waits for the total indexed count to grow — - it does not guarantee that *this specific* document is visible yet - (observed on nc32 where deck-card seed data indexes first and the new - note arrives in Qdrant a few seconds later). Poll until the unique term - returns our note, or fail loudly with the last response we saw. + `wait_for_vector_sync` now gates on this specific document being + retrievable via the MCP semantic-search tool, but Astrolabe's own search + endpoint is a distinct read path (its own JWT + query handler), so we still + poll it here until the unique term returns our note — or fail loudly with + the last response we saw. """ deadline = time.monotonic() + timeout_seconds last_results: list | None = None @@ -170,9 +170,16 @@ async def test_chunk_context_endpoint_uses_app_password( assert note_id is not None sync_complete, status = await wait_for_vector_sync( - mcp_client, initial_count, timeout_seconds=90 + mcp_client, + initial_count, + timeout_seconds=90, + search_term=unique_term, + note_id=note_id, + ) + assert sync_complete, ( + f"Note {note_id} ({unique_term}) never became searchable " + f"within timeout. Last sync status: {status}" ) - assert sync_complete, f"Vector sync did not complete: {status}" # Use the browser's session to drive Astrolabe end-to-end, the way a # real user would: this exercises astrolabe's OAuth token retrieval diff --git a/tests/integration/test_astrolabe_plotly_visualization.py b/tests/integration/test_astrolabe_plotly_visualization.py index 33607174..d8f7775f 100644 --- a/tests/integration/test_astrolabe_plotly_visualization.py +++ b/tests/integration/test_astrolabe_plotly_visualization.py @@ -28,6 +28,7 @@ from playwright.async_api import Page # Import helper functions from existing test from tests.conftest import create_mcp_client_session +from tests.integration._search_helpers import document_is_searchable from tests.integration.test_astrolabe_multi_user_background_sync import ( complete_astrolabe_authorization, login_to_nextcloud, @@ -39,14 +40,40 @@ pytestmark = [pytest.mark.integration, pytest.mark.multi_user_basic] async def wait_for_vector_sync( - mcp_client, initial_indexed_count: int, timeout_seconds: int = 60 + mcp_client, + initial_indexed_count: int, + timeout_seconds: int = 60, + *, + search_term: str | None = None, + note_id: int | None = None, ) -> tuple[bool, dict | None]: - """Wait for vector sync to index new content. + """Wait for vector sync to index newly-created content. + + Completion signal: + + - When ``search_term`` is provided (preferred), poll ``nc_semantic_search`` + until the new document is actually retrievable. This is robust against + full-corpus re-scan churn and doubles as a real end-to-end check — it is + exactly what callers assert downstream. + - Otherwise, fall back to the legacy gauge-delta predicate. + + Why the gauge delta is unreliable: under ``VECTOR_SYNC_SCAN_INTERVAL`` the + background sync re-queues the whole corpus every scan, so the corpus-wide + ``indexed_count`` is *non-monotonic* — it can be re-counted downward + mid-scan. ``indexed_count > initial_indexed_count`` can therefore never hold + even though the new document is indexed and the status has settled to + ``idle`` / ``pending_count == 0``. That false failure was the dominant + multi-user-basic CI flake (``test_astrolabe_plotly_visualization`` / + ``test_astrolabe_chunk_context``). Args: mcp_client: MCP client session - initial_indexed_count: Initial indexed document count before creating content + initial_indexed_count: Indexed document count before creating content + (only used by the legacy gauge-delta fallback) timeout_seconds: Maximum time to wait for sync + search_term: Unique term contained in the new document; enables the + robust searchability-based completion signal + note_id: ID of the new document, used to match search results exactly Returns: Tuple of (success, status_data) @@ -73,7 +100,14 @@ async def wait_for_vector_sync( status_data.get("status"), ) - if indexed_count > initial_indexed_count and pending_count == 0: + if search_term is not None: + if await document_is_searchable(mcp_client, search_term, note_id): + logger.info( + "✓ Sync complete: document %s retrievable via semantic search", + note_id, + ) + return True, status_data + elif indexed_count > initial_indexed_count and pending_count == 0: logger.info( "✓ Sync complete: %s documents indexed (was %s)", indexed_count, @@ -197,24 +231,41 @@ The visualization should show this document as a point in PCA-reduced space. # Phase 4: Wait for vector indexing sync_complete, status = await wait_for_vector_sync( - alice_mcp_client, initial_count, timeout_seconds=90 + alice_mcp_client, + initial_count, + timeout_seconds=90, + search_term=unique_term, + note_id=note_id, + ) + assert sync_complete, ( + f"Note {note_id} ({unique_term}) never became searchable " + f"within timeout. Last sync status: {status}" ) - assert sync_complete, f"Vector sync did not complete in time: {status}" # Phase 5: Navigate to Astrolabe and perform search await navigate_to_astrolabe_main(page) - # Fill search query - find the Astrolabe search input specifically - # The NcTextField component wraps the input in a div with class mcp-search-input - search_input = page.locator(".mcp-search-input input") - await search_input.wait_for(timeout=10000, state="visible") + # Find the Astrolabe search field. The published app differs across + # the NC matrix: NC31 pulls astrolabe <=0.24 (NcTextField -> , + # submits on Enter); NC32 pulls astrolabe >=0.25 (NcTextArea -> + #