Merge remote-tracking branch 'origin/master' into feat/tier2-incluster-ocr

This commit is contained in:
Chris Coutinho
2026-06-18 01:43:26 +02:00
10 changed files with 297 additions and 79 deletions
+2 -2
View File
@@ -1987,7 +1987,7 @@ async def playwright_oauth_token(
# Wait for callback server to receive the auth code
# Browser will be redirected to localhost:8081 which will capture the code
logger.info("Waiting for callback server to receive auth code...")
timeout_seconds = 30
timeout_seconds = 60 # was 30; too tight for consent+redirect on loaded CI
start_time = time.time()
while state not in auth_states:
if time.time() - start_time > timeout_seconds:
@@ -2696,7 +2696,7 @@ async def _get_oauth_token_for_user(
logger.info(
"Waiting for callback server to receive auth code for %s...", username
)
timeout_seconds = 30
timeout_seconds = 60 # was 30; too tight for consent+redirect on loaded CI
start_time = time.time()
while state not in auth_states:
if time.time() - start_time > timeout_seconds:
+68
View File
@@ -0,0 +1,68 @@
"""Shared helpers for asserting vector-sync visibility in integration tests.
Kept dependency-light (no Playwright) so both the multi-user-basic UI tests and
the single-user sampling tests can import it.
"""
import json
import logging
from typing import Any
logger = logging.getLogger(__name__)
async def document_is_searchable(
mcp_client: Any, search_term: str, note_id: int | None = None
) -> bool:
"""Return True once a freshly-created document is retrievable.
Polls ``nc_semantic_search`` (hybrid: an exact unique term reliably matches
on the keyword side) and matches by ``note_id`` when provided, otherwise by
the term appearing in a result's title/excerpt. Transient errors return
False so callers can keep polling.
"""
try:
search = await mcp_client.call_tool(
"nc_semantic_search",
# limit is generous: a fresh note can sit below seed data (e.g. deck
# cards) in a crowded corpus, and the query is cheap.
arguments={"query": search_term, "limit": 50, "score_threshold": 0.0},
)
except Exception as e: # transient transport/availability blip — keep polling
logger.debug("Semantic search poll failed: %s", e)
return False
if search.isError:
logger.debug("Semantic search poll error: %s", search)
return False
try:
results = json.loads(search.content[0].text).get("results", [])
except (IndexError, ValueError) as e: # empty content / malformed JSON
logger.debug("Semantic search parse failed: %s", e)
return False
# Token match (not contiguous substring) so multi-word terms work in the
# note_id-less fallback path.
tokens = search_term.lower().split()
for r in results:
if note_id is not None:
# str-coerce both sides: nc_semantic_search returns int ids today,
# but the Astrolabe API serialises some ids as strings — match the
# defensive comparison in _poll_astrolabe_search_for_note so a future
# schema change can't silently break the match.
if str(r.get("id")) == str(note_id):
if r.get("doc_type") == "note":
return True
# id matched but not a note — surface possible schema drift at
# WARNING (CI runs --log-cli-level=WARN) instead of letting the
# caller time out with a generic message.
logger.warning(
"search hit id=%s has doc_type=%s (expected note)",
note_id,
r.get("doc_type"),
)
else:
haystack = f"{r.get('title', '')} {r.get('excerpt', '')}".lower()
if tokens and all(t in haystack for t in tokens):
return True
return False
@@ -56,11 +56,11 @@ async def _poll_astrolabe_search_for_note(
) -> dict:
"""Poll Astrolabe's search endpoint until `note_id` shows up in results.
`wait_for_vector_sync` only waits for the total indexed count to grow —
it does not guarantee that *this specific* document is visible yet
(observed on nc32 where deck-card seed data indexes first and the new
note arrives in Qdrant a few seconds later). Poll until the unique term
returns our note, or fail loudly with the last response we saw.
`wait_for_vector_sync` now gates on this specific document being
retrievable via the MCP semantic-search tool, but Astrolabe's own search
endpoint is a distinct read path (its own JWT + query handler), so we still
poll it here until the unique term returns our note — or fail loudly with
the last response we saw.
"""
deadline = time.monotonic() + timeout_seconds
last_results: list | None = None
@@ -170,9 +170,16 @@ async def test_chunk_context_endpoint_uses_app_password(
assert note_id is not None
sync_complete, status = await wait_for_vector_sync(
mcp_client, initial_count, timeout_seconds=90
mcp_client,
initial_count,
timeout_seconds=90,
search_term=unique_term,
note_id=note_id,
)
assert sync_complete, (
f"Note {note_id} ({unique_term}) never became searchable "
f"within timeout. Last sync status: {status}"
)
assert sync_complete, f"Vector sync did not complete: {status}"
# Use the browser's session to drive Astrolabe end-to-end, the way a
# real user would: this exercises astrolabe's OAuth token retrieval
@@ -28,6 +28,7 @@ from playwright.async_api import Page
# Import helper functions from existing test
from tests.conftest import create_mcp_client_session
from tests.integration._search_helpers import document_is_searchable
from tests.integration.test_astrolabe_multi_user_background_sync import (
complete_astrolabe_authorization,
login_to_nextcloud,
@@ -39,14 +40,40 @@ pytestmark = [pytest.mark.integration, pytest.mark.multi_user_basic]
async def wait_for_vector_sync(
mcp_client, initial_indexed_count: int, timeout_seconds: int = 60
mcp_client,
initial_indexed_count: int,
timeout_seconds: int = 60,
*,
search_term: str | None = None,
note_id: int | None = None,
) -> tuple[bool, dict | None]:
"""Wait for vector sync to index new content.
"""Wait for vector sync to index newly-created content.
Completion signal:
- When ``search_term`` is provided (preferred), poll ``nc_semantic_search``
until the new document is actually retrievable. This is robust against
full-corpus re-scan churn and doubles as a real end-to-end check — it is
exactly what callers assert downstream.
- Otherwise, fall back to the legacy gauge-delta predicate.
Why the gauge delta is unreliable: under ``VECTOR_SYNC_SCAN_INTERVAL`` the
background sync re-queues the whole corpus every scan, so the corpus-wide
``indexed_count`` is *non-monotonic* — it can be re-counted downward
mid-scan. ``indexed_count > initial_indexed_count`` can therefore never hold
even though the new document is indexed and the status has settled to
``idle`` / ``pending_count == 0``. That false failure was the dominant
multi-user-basic CI flake (``test_astrolabe_plotly_visualization`` /
``test_astrolabe_chunk_context``).
Args:
mcp_client: MCP client session
initial_indexed_count: Initial indexed document count before creating content
initial_indexed_count: Indexed document count before creating content
(only used by the legacy gauge-delta fallback)
timeout_seconds: Maximum time to wait for sync
search_term: Unique term contained in the new document; enables the
robust searchability-based completion signal
note_id: ID of the new document, used to match search results exactly
Returns:
Tuple of (success, status_data)
@@ -73,7 +100,14 @@ async def wait_for_vector_sync(
status_data.get("status"),
)
if indexed_count > initial_indexed_count and pending_count == 0:
if search_term is not None:
if await document_is_searchable(mcp_client, search_term, note_id):
logger.info(
"✓ Sync complete: document %s retrievable via semantic search",
note_id,
)
return True, status_data
elif indexed_count > initial_indexed_count and pending_count == 0:
logger.info(
"✓ Sync complete: %s documents indexed (was %s)",
indexed_count,
@@ -197,24 +231,41 @@ The visualization should show this document as a point in PCA-reduced space.
# Phase 4: Wait for vector indexing
sync_complete, status = await wait_for_vector_sync(
alice_mcp_client, initial_count, timeout_seconds=90
alice_mcp_client,
initial_count,
timeout_seconds=90,
search_term=unique_term,
note_id=note_id,
)
assert sync_complete, (
f"Note {note_id} ({unique_term}) never became searchable "
f"within timeout. Last sync status: {status}"
)
assert sync_complete, f"Vector sync did not complete in time: {status}"
# Phase 5: Navigate to Astrolabe and perform search
await navigate_to_astrolabe_main(page)
# Fill search query - find the Astrolabe search input specifically
# The NcTextField component wraps the input in a div with class mcp-search-input
search_input = page.locator(".mcp-search-input input")
await search_input.wait_for(timeout=10000, state="visible")
# Find the Astrolabe search field. The published app differs across
# the NC matrix: NC31 pulls astrolabe <=0.24 (NcTextField -> <input>,
# submits on Enter); NC32 pulls astrolabe >=0.25 (NcTextArea ->
# <textarea>, submits on Ctrl/Cmd+Enter). Match either element so the
# test isn't pinned to one frontend revision.
# 30s (not 10s): the SPA can be slow to mount on a loaded CI runner.
search_input = page.locator(
".mcp-search-input textarea, .mcp-search-input input"
).first
await search_input.wait_for(timeout=30000, state="visible")
await search_input.fill(unique_term)
logger.info("Entered search query: %s", unique_term)
# Trigger search by pressing Enter on the input field
# This is wired to performSearch via @keyup.enter in the Vue component
await search_input.press("Enter")
logger.info("Pressed Enter to trigger search")
# Trigger search. NcTextField submits on Enter; NcTextArea inserts a
# newline on Enter and submits on Ctrl/Cmd+Enter — so key off the tag.
field_tag = await search_input.evaluate("el => el.tagName.toLowerCase()")
if field_tag == "textarea":
await search_input.press("Control+Enter")
else:
await search_input.press("Enter")
logger.info("Triggered search via %s submit", field_tag)
# Wait for loading to complete - watch for loading indicator to disappear
loading_indicator = page.locator(".mcp-loading")
@@ -21,17 +21,46 @@ HTTP with BasicAuth (which establishes a Nextcloud session for the request) —
no browser needed.
"""
import logging
import os
import anyio
import httpx
import pytest
pytestmark = [pytest.mark.integration, pytest.mark.login_flow]
logger = logging.getLogger(__name__)
NEXTCLOUD_URL = "http://localhost:8080"
ASTROLABE_API = f"{NEXTCLOUD_URL}/apps/astrolabe/api"
_HEADERS = {"OCS-APIRequest": "true"}
# The first /search after container start is slow: astrolabe mints a JWT and
# the MCP server runs a semantic search that may cold-load the embedding model.
# A single 30s read timeout was a CI flake (httpx.ReadTimeout); give the search
# path a generous budget and one retry on transient transport errors.
_SEARCH_TIMEOUT = httpx.Timeout(90.0)
async def _get_with_retry(
client: httpx.AsyncClient, url: str, *, max_attempts: int = 2, **kwargs
) -> httpx.Response:
"""GET, retrying on transient transport errors (timeouts/conn resets)."""
last_exc: httpx.TransportError | None = None
for attempt in range(1, max_attempts + 1):
try:
return await client.get(url, **kwargs)
except httpx.TransportError as e: # covers timeouts + connect/read errors
last_exc = e
logger.warning(
"GET %s failed (attempt %s/%s): %s", url, attempt, max_attempts, e
)
if attempt < max_attempts:
await anyio.sleep(2) # no point sleeping before we give up
assert last_exc is not None # loop ran at least once, so this is set
raise last_exc
async def _astrolabe_configured(client: httpx.AsyncClient, auth) -> bool:
"""Readiness probe: astrolabe must be able to reach its MCP server."""
@@ -46,6 +75,7 @@ async def _astrolabe_configured(client: httpx.AsyncClient, auth) -> bool:
return bool(resp.json().get("success"))
@pytest.mark.timeout(300) # cold model load + retry can exceed the 180s default
async def test_session_user_searches_without_provisioning(test_users_setup):
"""A non-admin session user searches with no OAuth/provisioning step.
@@ -66,11 +96,13 @@ async def test_session_user_searches_without_provisioning(test_users_setup):
"precondition: bob has not opted into background indexing"
)
resp = await client.get(
resp = await _get_with_retry(
client,
f"{ASTROLABE_API}/search",
params={"query": "quarterly planning", "limit": 3},
auth=auth,
headers=_HEADERS,
timeout=_SEARCH_TIMEOUT,
)
assert resp.status_code == 200, resp.text
@@ -81,6 +113,7 @@ async def test_session_user_searches_without_provisioning(test_users_setup):
assert "results" in body and "algorithm_used" in body
@pytest.mark.timeout(300) # cold model load + retry can exceed the 180s default
async def test_admin_session_search_succeeds():
"""The same JWT-mint path works for the admin session user."""
admin_pw = os.environ["NEXTCLOUD_PASSWORD"]
@@ -88,11 +121,13 @@ async def test_admin_session_search_succeeds():
async with httpx.AsyncClient(timeout=30) as client:
if not await _astrolabe_configured(client, auth):
pytest.skip("Astrolabe not wired to an MCP server in this stack")
resp = await client.get(
resp = await _get_with_retry(
client,
f"{ASTROLABE_API}/search",
params={"query": "infrastructure", "limit": 3},
auth=auth,
headers=_HEADERS,
timeout=_SEARCH_TIMEOUT,
)
assert resp.status_code == 200, resp.text
assert resp.json()["success"] is True
+53 -21
View File
@@ -174,16 +174,22 @@ async def indexed_manual_pdf(nc_client, nc_mcp_client):
content = json.loads(result.content[0].text) if result.content else {}
indexed = content.get("indexed_count", 0)
pending = content.get("pending_count", 1)
status = content.get("status")
logger.info(
"Attempt %s/%s: indexed=%s, pending=%s",
"Attempt %s/%s: indexed=%s, pending=%s, status=%s",
attempt,
max_attempts,
indexed,
pending,
status,
)
if indexed > 0 and pending == 0:
# Require indexed > 0 (the manual must actually be indexed —
# idle/pending==0 is also the *initial* empty state) AND a
# settled idle scan so we don't break during a transient
# pending==0 window mid re-scan churn.
if indexed > 0 and pending == 0 and status == "idle":
logger.info(
"Vector indexing complete: %s documents indexed", indexed
)
@@ -399,27 +405,53 @@ async def test_retrieval_quality_all_queries(
)
async def test_no_results_for_unrelated_query(nc_mcp_client, indexed_manual_pdf):
"""Test that completely unrelated queries return low/no scores.
The Nextcloud manual shouldn't have relevant content for
quantum physics queries.
"""
async def _top_score(nc_mcp_client: Any, query: str) -> float | None:
"""Return the best fusion score for ``query``, or None if no results."""
result = await nc_mcp_client.call_tool(
"nc_semantic_search",
arguments={
"query": "quantum entanglement hadron collider particle physics",
"limit": 5,
"score_threshold": 0.5, # Higher threshold to filter irrelevant
},
arguments={"query": query, "limit": 5, "score_threshold": 0.0},
)
assert result.isError is False, result.content
data = json.loads(result.content[0].text)
results = data.get("results", [])
if not results: # guard the list directly, not via total_found
return None
return max(r["score"] for r in results)
async def test_no_results_for_unrelated_query(nc_mcp_client, indexed_manual_pdf):
"""An unrelated query must not out-rank a genuinely relevant one.
The Nextcloud manual has no quantum-physics content, so a physics query
must not look *more* relevant than a real manual query.
We deliberately do NOT assert on an absolute score magnitude. Fusion scores
(RRF/DBSF) are rank-based, not calibrated relevance: the top hit saturates
near the high end of the range regardless of true relevance, so a hardcoded
``max_score < 0.8`` check was a CI flake (it tripped whenever the unrelated
query happened to retrieve any chunk at all). Comparing against a relevant
query on the same corpus is self-calibrating and stable.
"""
# No results for the nonsense query is the ideal outcome — treat as score
# 0.0 and fall through, so the comparison (and the manual-is-indexed check
# below) still runs instead of the test silently skipping every time the
# physics query finds nothing.
unrelated = (
await _top_score(
nc_mcp_client, "quantum entanglement hadron collider particle physics"
)
or 0.0
)
assert result.isError is False
data = json.loads(result.content[0].text)
relevant = await _top_score(
nc_mcp_client, "how do I enable two-factor authentication"
)
assert relevant is not None, (
"Relevant control query returned nothing — manual not indexed?"
)
# Should have few or no high-scoring results
# Low score threshold means we might get some results, but they should be low quality
if data["total_found"] > 0:
# If results exist, they should have low scores
max_score = max(r["score"] for r in data["results"])
assert max_score < 0.8, f"Unexpected high score {max_score} for unrelated query"
# The unrelated query must not appear more relevant than the real one.
assert unrelated <= relevant, (
f"Unrelated query scored {unrelated}, higher than the relevant "
f"control query's {relevant} — retrieval is not discriminating."
)
+55 -30
View File
@@ -20,6 +20,8 @@ import anyio
import pytest
from mcp.types import CreateMessageResult, TextContent
from tests.integration._search_helpers import document_is_searchable
pytestmark = pytest.mark.integration
@@ -27,6 +29,8 @@ async def wait_for_vector_sync(
nc_mcp_client,
*,
initial_indexed_count: int | None = None,
search_term: str | None = None,
note_id: int | None = None,
max_wait: int = 90,
wait_interval: int = 1,
) -> dict:
@@ -34,9 +38,15 @@ async def wait_for_vector_sync(
Args:
nc_mcp_client: MCP client to poll status with.
initial_indexed_count: If set, wait until indexed_count exceeds this
value and pending_count reaches 0. Otherwise wait for idle with
no pending work.
search_term: If set (preferred), wait until a document matching this
term is retrievable via ``nc_semantic_search``. Robust against
full-corpus re-scan churn, where the corpus-wide ``indexed_count``
gauge is non-monotonic and ``indexed_count > initial`` can never
hold even though the document is indexed.
note_id: Optional exact-match document id paired with ``search_term``.
initial_indexed_count: Legacy gauge-delta fallback when no search_term
is given: wait until indexed_count exceeds this value and
pending_count reaches 0.
max_wait: Maximum seconds to wait before failing.
wait_interval: Seconds between status polls.
@@ -49,18 +59,32 @@ async def wait_for_vector_sync(
sync_status = await nc_mcp_client.call_tool(
"nc_get_vector_sync_status", arguments={}
)
status_data = json.loads(sync_status.content[0].text)
try:
status_data = json.loads(sync_status.content[0].text)
except (AttributeError, IndexError, ValueError):
# transient empty/error response — keep polling. .get() defaults
# below also keep an empty dict from triggering a false break.
status_data = {}
if initial_indexed_count is not None:
# Wait for new document(s) to be indexed
if search_term is not None:
# Robust signal: wait for the specific document to be retrievable
if await document_is_searchable(nc_mcp_client, search_term, note_id):
break
elif initial_indexed_count is not None:
# Legacy: wait for new document(s) to be indexed (gauge delta)
if (
status_data["indexed_count"] > initial_indexed_count
and status_data["pending_count"] == 0
status_data.get("indexed_count", 0) > initial_indexed_count
and status_data.get("pending_count", 1) == 0
):
break
else:
# Wait for all pending work to complete
if status_data["status"] == "idle" and status_data["pending_count"] == 0:
# NOTE: idle + pending==0 is also the *initial empty* state, so this
# can break before a caller's work is even enqueued — prefer passing
# search_term. Kept only for callers that just need a settled corpus.
if (
status_data.get("status") == "idle"
and status_data.get("pending_count", 1) == 0
):
break
await anyio.sleep(wait_interval)
@@ -117,14 +141,6 @@ async def test_semantic_search_answer_successful_sampling(
"""
await require_vector_sync_tools(nc_mcp_client)
# Get initial indexed count before creating note
initial_sync = await nc_mcp_client.call_tool(
"nc_get_vector_sync_status", arguments={}
)
initial_indexed_count = json.loads(initial_sync.content[0].text)["indexed_count"]
print(f"Initial indexed count: {initial_indexed_count}")
# Create a note with content about Python async
_note = await temporary_note_factory(
title="Python Async Guide",
@@ -142,12 +158,13 @@ Avoid blocking operations in async code.""",
)
print(f"Created note ID: {_note['id']}")
# Wait for vector indexing to complete
status_data = await wait_for_vector_sync(
nc_mcp_client, initial_indexed_count=initial_indexed_count
)
assert status_data["indexed_count"] > initial_indexed_count, (
f"New note was not indexed (count stayed at {initial_indexed_count})"
# Wait for vector indexing to complete. Gate on the new note actually
# being retrievable rather than on the corpus-wide indexed_count gauge,
# which is non-monotonic under re-scan churn (see wait_for_vector_sync).
await wait_for_vector_sync(
nc_mcp_client,
search_term="Python Async Programming coroutines",
note_id=_note["id"],
)
# Mock the sampling call
@@ -267,8 +284,12 @@ async def test_semantic_search_answer_with_limit(nc_mcp_client, temporary_note_f
category="Development",
)
# Wait for vector indexing to complete
await wait_for_vector_sync(nc_mcp_client)
# Wait until the batch is indexed — gate on the last note being searchable
# rather than a bare idle signal, which can fire before the new notes are
# even enqueued.
await wait_for_vector_sync(
nc_mcp_client, search_term="async context managers", note_id=_note3["id"]
)
call_result = await nc_mcp_client.call_tool(
"nc_semantic_search_answer",
@@ -308,8 +329,10 @@ async def test_semantic_search_answer_score_threshold(
category="Test",
)
# Wait for vector indexing to complete
await wait_for_vector_sync(nc_mcp_client)
# Gate on the new note being searchable (not a bare idle signal).
await wait_for_vector_sync(
nc_mcp_client, search_term="widget manufacturing", note_id=_note["id"]
)
# Query with exact match
call_result = await nc_mcp_client.call_tool(
@@ -355,8 +378,10 @@ async def test_semantic_search_answer_max_tokens(nc_mcp_client, temporary_note_f
category="Test",
)
# Wait for vector indexing to complete
await wait_for_vector_sync(nc_mcp_client)
# Gate on the new note being searchable (not a bare idle signal).
await wait_for_vector_sync(
nc_mcp_client, search_term="Long Document content", note_id=_note["id"]
)
call_result = await nc_mcp_client.call_tool(
"nc_semantic_search_answer",
@@ -105,7 +105,7 @@ async def get_oauth_token_with_client(
# Wait for callback
logger.info("Waiting for OAuth callback...")
timeout_seconds = 30
timeout_seconds = 60 # was 30; too tight for consent+redirect on loaded CI
start_time = time.time()
while state not in auth_states:
if time.time() - start_time > timeout_seconds:
@@ -161,7 +161,7 @@ async def get_oauth_token_with_client(
# Wait for callback
logger.info("Waiting for OAuth callback...")
timeout_seconds = 30
timeout_seconds = 60 # was 30; too tight for consent+redirect on loaded CI
start_time = time.time()
while state not in auth_states:
if time.time() - start_time > timeout_seconds:
@@ -241,7 +241,7 @@ async def _obtain_token_for_client(
# Wait for callback server to receive auth code
logger.info("Waiting for callback server to receive auth code...")
timeout_seconds = 30
timeout_seconds = 60 # was 30; too tight for consent+redirect on loaded CI
start_time = time.time()
while state not in auth_states:
if time.time() - start_time > timeout_seconds: