From ca313e7271fb9d434fd9bdcf7ce810b6a649f1c1 Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Wed, 17 Jun 2026 23:19:41 +0200 Subject: [PATCH] =?UTF-8?q?test(integration):=20address=20round-7=20review?= =?UTF-8?q?=20=E2=80=94=20keep=20RAG=20assertion=20live,=20fix=20races?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - test_no_results_for_unrelated_query: replace pytest.skip with `unrelated = ... or 0.0` and fall through. The physics query almost always returns nothing on this corpus, so the skip meant the comparison (and the manual-is-indexed check) never ran. Treating no-results as score 0.0 keeps the test live and vacuously satisfies `0.0 <= relevant`. - test_sampling: the three limit/threshold/max-tokens tests now gate on a representative created note being searchable (search_term + note_id) instead of a bare idle signal that can fire before the new notes are enqueued. - _get_with_retry: only sleep between attempts, not before giving up. - _search_helpers: log the id/doc_type schema-drift mismatch at WARNING (CI runs --log-cli-level=WARN) so it surfaces instead of hiding behind a timeout. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/integration/_search_helpers.py | 7 ++++--- .../test_astrolabe_session_jwt_search.py | 3 ++- tests/integration/test_rag.py | 15 ++++++++------ tests/integration/test_sampling.py | 20 +++++++++++++------ 4 files changed, 29 insertions(+), 16 deletions(-) diff --git a/tests/integration/_search_helpers.py b/tests/integration/_search_helpers.py index 198ceea9..b7414095 100644 --- a/tests/integration/_search_helpers.py +++ b/tests/integration/_search_helpers.py @@ -49,9 +49,10 @@ async def document_is_searchable( if r.get("id") == note_id: if r.get("doc_type") == "note": return True - # id matched but not a note — surface possible schema drift - # rather than silently timing out. - logger.debug( + # id matched but not a note — surface possible schema drift at + # WARNING (CI runs --log-cli-level=WARN) instead of letting the + # caller time out with a generic message. + logger.warning( "search hit id=%s has doc_type=%s (expected note)", note_id, r.get("doc_type"), diff --git a/tests/integration/test_astrolabe_session_jwt_search.py b/tests/integration/test_astrolabe_session_jwt_search.py index 782a0b9b..f3b288b2 100644 --- a/tests/integration/test_astrolabe_session_jwt_search.py +++ b/tests/integration/test_astrolabe_session_jwt_search.py @@ -56,7 +56,8 @@ async def _get_with_retry( logger.warning( "GET %s failed (attempt %s/%s): %s", url, attempt, max_attempts, e ) - await anyio.sleep(2) + if attempt < max_attempts: + await anyio.sleep(2) # no point sleeping before we give up assert last_exc is not None # loop ran at least once, so this is set raise last_exc diff --git a/tests/integration/test_rag.py b/tests/integration/test_rag.py index aaf99fd8..d36a2d59 100644 --- a/tests/integration/test_rag.py +++ b/tests/integration/test_rag.py @@ -432,13 +432,16 @@ async def test_no_results_for_unrelated_query(nc_mcp_client, indexed_manual_pdf) query happened to retrieve any chunk at all). Comparing against a relevant query on the same corpus is self-calibrating and stable. """ - unrelated = await _top_score( - nc_mcp_client, "quantum entanglement hadron collider particle physics" + # No results for the nonsense query is the ideal outcome — treat as score + # 0.0 and fall through, so the comparison (and the manual-is-indexed check + # below) still runs instead of the test silently skipping every time the + # physics query finds nothing. + unrelated = ( + await _top_score( + nc_mcp_client, "quantum entanglement hadron collider particle physics" + ) + or 0.0 ) - if unrelated is None: - # No results for the nonsense query is the ideal outcome; skip (rather - # than a silent pass) so the test report shows the path was taken. - pytest.skip("No results for nonsense physics query — the ideal outcome") relevant = await _top_score( nc_mcp_client, "how do I enable two-factor authentication" diff --git a/tests/integration/test_sampling.py b/tests/integration/test_sampling.py index 293e5dc7..78340996 100644 --- a/tests/integration/test_sampling.py +++ b/tests/integration/test_sampling.py @@ -274,8 +274,12 @@ async def test_semantic_search_answer_with_limit(nc_mcp_client, temporary_note_f category="Development", ) - # Wait for vector indexing to complete - await wait_for_vector_sync(nc_mcp_client) + # Wait until the batch is indexed — gate on the last note being searchable + # rather than a bare idle signal, which can fire before the new notes are + # even enqueued. + await wait_for_vector_sync( + nc_mcp_client, search_term="async context managers", note_id=_note3["id"] + ) call_result = await nc_mcp_client.call_tool( "nc_semantic_search_answer", @@ -315,8 +319,10 @@ async def test_semantic_search_answer_score_threshold( category="Test", ) - # Wait for vector indexing to complete - await wait_for_vector_sync(nc_mcp_client) + # Gate on the new note being searchable (not a bare idle signal). + await wait_for_vector_sync( + nc_mcp_client, search_term="widget manufacturing", note_id=_note["id"] + ) # Query with exact match call_result = await nc_mcp_client.call_tool( @@ -362,8 +368,10 @@ async def test_semantic_search_answer_max_tokens(nc_mcp_client, temporary_note_f category="Test", ) - # Wait for vector indexing to complete - await wait_for_vector_sync(nc_mcp_client) + # Gate on the new note being searchable (not a bare idle signal). + await wait_for_vector_sync( + nc_mcp_client, search_term="Long Document content", note_id=_note["id"] + ) call_result = await nc_mcp_client.call_tool( "nc_semantic_search_answer",