Merge pull request #836 from cbcoutinho/feat/183-procrastinate-ingest-queue
feat: replace NATS ingest with procrastinate Postgres queue (#183)
This commit is contained in:
Chris Coutinho
@@ -0,0 +1,139 @@
|
||||
"""End-to-end Postgres smoke for the procrastinate ingest queue (Deck #183).
|
||||
|
||||
Validates the queue mechanics the in-memory connector can't: real
|
||||
``queueing_lock`` partial-unique dedup, idempotent schema apply, and the
|
||||
``list_queues`` stats the status surface reads. Opt-in like
|
||||
``test_storage_postgres.py``::
|
||||
|
||||
docker compose --profile postgres up -d postgres-test
|
||||
export TEST_DATABASE_URL=postgresql+asyncpg://mcp:mcp@localhost:5433/mcp
|
||||
uv run pytest tests/integration/test_ingest_queue_postgres.py -v -m postgres
|
||||
|
||||
Skipped when ``TEST_DATABASE_URL`` is unset or the service is unreachable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import socket
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import pytest
|
||||
|
||||
import nextcloud_mcp_server.config as config_module
|
||||
from nextcloud_mcp_server.vector.queue.procrastinate import (
|
||||
INGEST_QUEUE_NAME,
|
||||
ProcrastinateTaskProducer,
|
||||
apply_ingest_queue_schema,
|
||||
build_app_for_url,
|
||||
get_ingest_job_counts,
|
||||
)
|
||||
from nextcloud_mcp_server.vector.scanner import DocumentTask
|
||||
|
||||
pytestmark = [pytest.mark.integration, pytest.mark.postgres]
|
||||
|
||||
|
||||
def _postgres_url() -> str | None:
|
||||
return os.environ.get("TEST_DATABASE_URL") or None
|
||||
|
||||
|
||||
def _reachable(url: str) -> bool:
|
||||
parsed = urlparse(url)
|
||||
try:
|
||||
with socket.create_connection(
|
||||
(parsed.hostname or "localhost", parsed.port or 5432), timeout=1.0
|
||||
):
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def postgres_url() -> str:
|
||||
url = _postgres_url()
|
||||
if not url:
|
||||
pytest.skip(
|
||||
"TEST_DATABASE_URL not set — run "
|
||||
"`docker compose --profile postgres up -d postgres-test` and export "
|
||||
"TEST_DATABASE_URL=postgresql+asyncpg://mcp:mcp@localhost:5433/mcp"
|
||||
)
|
||||
# pytest.skip raises, but ty doesn't model it as NoReturn — narrow explicitly.
|
||||
assert url is not None
|
||||
if not _reachable(url):
|
||||
pytest.skip(f"Postgres at {url} is not reachable")
|
||||
return url
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def fresh_app(postgres_url: str, monkeypatch: pytest.MonkeyPatch):
|
||||
"""Drop+recreate the public schema, then apply procrastinate's schema."""
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
|
||||
engine = create_async_engine(postgres_url, future=True)
|
||||
try:
|
||||
async with engine.begin() as conn:
|
||||
await conn.execute(text("DROP SCHEMA public CASCADE"))
|
||||
await conn.execute(text("CREATE SCHEMA public"))
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
# build_app_for_url passes the URL explicitly to get_procrastinate_conninfo,
|
||||
# so only the ssl lookup (which reads settings) needs pinning here.
|
||||
monkeypatch.setattr(config_module, "get_database_ssl", lambda: None)
|
||||
|
||||
app = build_app_for_url(postgres_url)
|
||||
await apply_ingest_queue_schema(app)
|
||||
return app
|
||||
|
||||
|
||||
def _task(doc_id: str, doc_type: str = "note") -> DocumentTask:
|
||||
return DocumentTask(
|
||||
user_id="alice",
|
||||
doc_id=doc_id,
|
||||
doc_type=doc_type,
|
||||
operation="index",
|
||||
modified_at=100,
|
||||
etag=f"etag-{doc_id}",
|
||||
)
|
||||
|
||||
|
||||
async def test_ingest_queue_end_to_end(fresh_app):
|
||||
"""One self-contained smoke against real Postgres.
|
||||
|
||||
Kept as a single test so each assertion runs against the same freshly-applied
|
||||
schema — splitting across functions reintroduces the inter-test ``DROP
|
||||
SCHEMA`` that confuses pooled psycopg connections' cached prepared statements
|
||||
(a test-harness artifact, not a production path: prod never drops the schema).
|
||||
"""
|
||||
# 1. Schema is present and a second apply is a no-op (idempotent).
|
||||
await apply_ingest_queue_schema(fresh_app)
|
||||
|
||||
async with fresh_app.open_async():
|
||||
present = await fresh_app.connector.execute_query_one_async(
|
||||
"SELECT to_regclass('procrastinate_jobs') IS NOT NULL AS present"
|
||||
)
|
||||
assert present["present"] is True
|
||||
|
||||
# 2. Defer + real queueing_lock dedup (one todo per doc).
|
||||
producer = ProcrastinateTaskProducer(fresh_app)
|
||||
await producer.send(_task("1"))
|
||||
await producer.send(_task("1")) # deduped by queueing_lock
|
||||
await producer.send(_task("2"))
|
||||
|
||||
rows = await fresh_app.connector.execute_query_all_async(
|
||||
"SELECT count(*) AS n FROM procrastinate_jobs "
|
||||
"WHERE queue_name = %(q)s AND status = 'todo'",
|
||||
q=INGEST_QUEUE_NAME,
|
||||
)
|
||||
assert rows[0]["n"] == 2
|
||||
|
||||
# 3. The status-surface counts read agrees.
|
||||
counts = await get_ingest_job_counts(fresh_app)
|
||||
assert counts.get("todo") == 2
|
||||
|
||||
# 4. Fresh todo jobs are not "doing", so none are stalled.
|
||||
stalled = await fresh_app.job_manager.get_stalled_jobs(
|
||||
queue=INGEST_QUEUE_NAME, seconds_since_heartbeat=0
|
||||
)
|
||||
assert list(stalled) == []
|
||||
@@ -16,6 +16,8 @@ import logging
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.login_flow
|
||||
@@ -65,8 +67,6 @@ async def test_basicauth_shows_all_tools(nc_mcp_client):
|
||||
async def test_read_only_token_filters_write_tools(nc_mcp_login_flow_client_read_only):
|
||||
"""Test that a token with only read scopes filters out write tools."""
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Connect with token that has only "notes.read" scope
|
||||
result = await nc_mcp_login_flow_client_read_only.list_tools()
|
||||
assert result is not None
|
||||
@@ -114,8 +114,6 @@ async def test_read_only_token_filters_write_tools(nc_mcp_login_flow_client_read
|
||||
async def test_write_only_token_filters_read_tools(nc_mcp_login_flow_client_write_only):
|
||||
"""Test that a token with only write scopes filters out read tools."""
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Connect with token that has only "notes.write" scope
|
||||
result = await nc_mcp_login_flow_client_write_only.list_tools()
|
||||
assert result is not None
|
||||
@@ -163,8 +161,6 @@ async def test_write_only_token_filters_read_tools(nc_mcp_login_flow_client_writ
|
||||
async def test_full_access_token_shows_all_tools(nc_mcp_login_flow_client_full_access):
|
||||
"""Test that a token with both read and write scopes scopes can see all tools."""
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Connect with token that has both "notes.read" and "notes.write" scopes
|
||||
result = await nc_mcp_login_flow_client_full_access.list_tools()
|
||||
assert result is not None
|
||||
@@ -305,7 +301,11 @@ async def test_tools_have_scope_decorators(nc_mcp_client):
|
||||
@pytest.mark.integration
|
||||
async def test_scope_classification():
|
||||
"""Test that our scope classification correctly identifies read vs write operations."""
|
||||
from scripts.add_scope_decorators_simple import classify_function
|
||||
# `scripts/` is a dev-only helper dir (not an installed package); resolved
|
||||
# at runtime via the repo root on sys.path, so ty can't see it.
|
||||
from scripts.add_scope_decorators_simple import ( # ty: ignore[unresolved-import]
|
||||
classify_function,
|
||||
)
|
||||
|
||||
# Test read operations
|
||||
assert classify_function("nc_notes_get_note") == "notes.read"
|
||||
@@ -336,7 +336,11 @@ async def test_scope_classification():
|
||||
@pytest.mark.integration
|
||||
async def test_all_tools_classified():
|
||||
"""Verify that all tools can be properly classified as read or write."""
|
||||
from scripts.add_scope_decorators_simple import classify_function
|
||||
# `scripts/` is a dev-only helper dir (not an installed package); resolved
|
||||
# at runtime via the repo root on sys.path, so ty can't see it.
|
||||
from scripts.add_scope_decorators_simple import ( # ty: ignore[unresolved-import]
|
||||
classify_function,
|
||||
)
|
||||
|
||||
# List of all tool names (extracted from our implementation)
|
||||
all_tools = [
|
||||
@@ -407,8 +411,6 @@ async def test_jwt_with_no_custom_scopes_returns_zero_tools(
|
||||
so users can provision Nextcloud access after authentication
|
||||
"""
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Connect with JWT token that has NO custom scopes (only openid, profile, email)
|
||||
result = await nc_mcp_login_flow_client_no_custom_scopes.list_tools()
|
||||
assert result is not None
|
||||
@@ -451,8 +453,6 @@ async def test_jwt_consent_scenarios_read_only(nc_mcp_login_flow_client_read_onl
|
||||
Expected: Should see read tools but not write tools.
|
||||
"""
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
result = await nc_mcp_login_flow_client_read_only.list_tools()
|
||||
assert result is not None
|
||||
assert len(result.tools) > 0
|
||||
@@ -490,8 +490,6 @@ async def test_jwt_consent_scenarios_write_only(nc_mcp_login_flow_client_write_o
|
||||
Expected: Should see write tools but not read-only tools.
|
||||
"""
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
result = await nc_mcp_login_flow_client_write_only.list_tools()
|
||||
assert result is not None
|
||||
assert len(result.tools) > 0
|
||||
@@ -529,8 +527,6 @@ async def test_jwt_consent_scenarios_full_access(nc_mcp_login_flow_client_full_a
|
||||
Expected: Should see all 90+ tools (both read and write).
|
||||
"""
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
result = await nc_mcp_login_flow_client_full_access.list_tools()
|
||||
assert result is not None
|
||||
assert len(result.tools) > 0
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Regression test for the lifespan context `task_producer` exposure (Deck #183).
|
||||
|
||||
`nc_get_vector_sync_status` reads `lifespan_ctx.task_producer` for postgres-backend
|
||||
job counts. It was previously a snapshot dataclass field the per-session yields
|
||||
forgot to populate, so the tool always reported `pending=0` on the postgres
|
||||
backend. It is now a `@property` that reads the module singleton live (like
|
||||
`eviction_task_group`); these tests pin that contract.
|
||||
"""
|
||||
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
|
||||
import nextcloud_mcp_server.app as app_module
|
||||
from nextcloud_mcp_server.app import AppContext, OAuthAppContext
|
||||
from nextcloud_mcp_server.client import NextcloudClient
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def test_app_context_task_producer_reads_vector_sync_state(monkeypatch):
|
||||
sentinel = object()
|
||||
monkeypatch.setattr(app_module._vector_sync_state, "task_producer", sentinel)
|
||||
ctx = AppContext(client=cast(NextcloudClient, None))
|
||||
assert ctx.task_producer is sentinel
|
||||
|
||||
|
||||
def test_oauth_app_context_task_producer_reads_vector_sync_state(monkeypatch):
|
||||
sentinel = object()
|
||||
monkeypatch.setattr(app_module._vector_sync_state, "task_producer", sentinel)
|
||||
ctx = OAuthAppContext(
|
||||
nextcloud_host="https://example.test", token_verifier=object()
|
||||
)
|
||||
assert ctx.task_producer is sentinel
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Tests for the MCP decomposition hook-point settings (design §10).
|
||||
"""Tests for the MCP decomposition hook-point settings (design §10, Deck #183).
|
||||
|
||||
Every default must reproduce the monolith; the opt-in settings are validated
|
||||
in ``Settings.__post_init__``.
|
||||
@@ -6,6 +6,7 @@ in ``Settings.__post_init__``.
|
||||
|
||||
import pytest
|
||||
|
||||
import nextcloud_mcp_server.config as config_module
|
||||
from nextcloud_mcp_server.canonical import canonical_json
|
||||
from nextcloud_mcp_server.config import Settings
|
||||
|
||||
@@ -16,23 +17,21 @@ class TestDecompositionDefaults:
|
||||
def test_defaults_are_monolith(self):
|
||||
s = Settings()
|
||||
assert s.embedding_provider == "autodetect"
|
||||
assert s.ingest_mode == "local"
|
||||
assert s.status_backend == "local"
|
||||
# SQLite/dev default → the in-process memory queue.
|
||||
assert s.ingest_queue == "memory"
|
||||
assert s.mcp_role == "all"
|
||||
assert s.collection_metadata_source == "qdrant"
|
||||
assert s.fact_event_emitter == "none"
|
||||
assert s.ingest_bus_url is None
|
||||
assert s.embedding_gateway_url is None
|
||||
assert s.tenant_id is None
|
||||
assert s.ingest_bus_num_replicas == 1
|
||||
|
||||
def test_enum_values_normalized(self):
|
||||
# Mixed case / surrounding whitespace is normalized before validation.
|
||||
s = Settings(
|
||||
collection_metadata_source=" QDRANT ",
|
||||
fact_event_emitter="NONE",
|
||||
mcp_role=" API ",
|
||||
)
|
||||
assert s.collection_metadata_source == "qdrant"
|
||||
assert s.fact_event_emitter == "none"
|
||||
assert s.mcp_role == "api"
|
||||
|
||||
|
||||
class TestEnumValidation:
|
||||
@@ -40,58 +39,48 @@ class TestEnumValidation:
|
||||
"field,value",
|
||||
[
|
||||
("embedding_provider", "openai"),
|
||||
("ingest_mode", "remote"),
|
||||
("status_backend", "redis"),
|
||||
("collection_metadata_source", "postgres"),
|
||||
("fact_event_emitter", "kafka"),
|
||||
("mcp_role", "leader"),
|
||||
("collection_metadata_source", "redis"),
|
||||
],
|
||||
)
|
||||
def test_invalid_enum_rejected(self, field, value):
|
||||
with pytest.raises(ValueError, match=field.upper()):
|
||||
Settings(**{field: value})
|
||||
|
||||
def test_invalid_ingest_queue_rejected(self):
|
||||
with pytest.raises(ValueError, match="INGEST_QUEUE"):
|
||||
Settings(ingest_queue="kafka")
|
||||
|
||||
class TestFailFast:
|
||||
def test_external_with_local_status_crashes(self):
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="STATUS_BACKEND=local is incompatible with INGEST_MODE=external",
|
||||
):
|
||||
Settings(
|
||||
ingest_mode="external",
|
||||
status_backend="local",
|
||||
ingest_bus_url="nats://nats:4222",
|
||||
tenant_id="tenant-uuid",
|
||||
)
|
||||
|
||||
class TestIngestQueueResolution:
|
||||
def test_postgres_requires_postgres_url(self):
|
||||
# Explicit postgres against the default SQLite DATABASE_URL is a
|
||||
# misconfiguration (procrastinate is Postgres-only).
|
||||
with pytest.raises(ValueError, match="INGEST_QUEUE=postgres requires"):
|
||||
Settings(ingest_queue="postgres")
|
||||
|
||||
def test_auto_postgres_when_database_url_is_postgres(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
config_module,
|
||||
"get_database_url",
|
||||
lambda: "postgresql+asyncpg://mcp:mcp@db/mcp",
|
||||
)
|
||||
assert Settings().ingest_queue == "postgres"
|
||||
|
||||
def test_explicit_memory_on_postgres_url(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
config_module,
|
||||
"get_database_url",
|
||||
lambda: "postgresql+asyncpg://mcp:mcp@db/mcp",
|
||||
)
|
||||
assert Settings(ingest_queue="memory").ingest_queue == "memory"
|
||||
|
||||
|
||||
class TestConditionalRequired:
|
||||
def test_external_requires_bus_url(self):
|
||||
with pytest.raises(ValueError, match="INGEST_BUS_URL is required"):
|
||||
Settings(ingest_mode="external", status_backend="bus", tenant_id="t1")
|
||||
|
||||
def test_external_requires_tenant_id(self):
|
||||
with pytest.raises(ValueError, match="TENANT_ID is required"):
|
||||
Settings(
|
||||
ingest_mode="external",
|
||||
status_backend="bus",
|
||||
ingest_bus_url="nats://nats:4222",
|
||||
)
|
||||
|
||||
def test_gateway_requires_gateway_url(self):
|
||||
with pytest.raises(ValueError, match="EMBEDDING_GATEWAY_URL is required"):
|
||||
Settings(embedding_provider="gateway")
|
||||
|
||||
def test_external_happy_path(self):
|
||||
s = Settings(
|
||||
ingest_mode="external",
|
||||
status_backend="bus",
|
||||
ingest_bus_url="nats://nats:4222",
|
||||
tenant_id="0a1b2c3d-0000-0000-0000-000000000000",
|
||||
)
|
||||
assert s.ingest_mode == "external"
|
||||
assert s.status_backend == "bus"
|
||||
|
||||
def test_gateway_happy_path(self):
|
||||
s = Settings(
|
||||
embedding_provider="gateway",
|
||||
@@ -100,24 +89,84 @@ class TestConditionalRequired:
|
||||
assert s.embedding_provider == "gateway"
|
||||
|
||||
|
||||
class TestTenantIdSubjectToken:
|
||||
@pytest.mark.parametrize(
|
||||
"tenant_id",
|
||||
["a.b", "a*b", "a>b", "a b", "a\tb"],
|
||||
)
|
||||
def test_illegal_subject_chars_rejected(self, tenant_id):
|
||||
with pytest.raises(ValueError, match="TENANT_ID must not contain"):
|
||||
Settings(tenant_id=tenant_id)
|
||||
|
||||
def test_uuid_form_accepted(self):
|
||||
class TestTenantId:
|
||||
def test_arbitrary_tenant_id_accepted(self):
|
||||
# The old NATS-subject charset restriction was dropped with NATS
|
||||
# (Deck #183); tenant_id is now just an opaque per-tenant identity.
|
||||
s = Settings(tenant_id="0a1b2c3d-0000-0000-0000-000000000000")
|
||||
assert s.tenant_id == "0a1b2c3d-0000-0000-0000-000000000000"
|
||||
|
||||
|
||||
class TestReplicas:
|
||||
def test_zero_replicas_rejected(self):
|
||||
with pytest.raises(ValueError, match="INGEST_BUS_NUM_REPLICAS must be >= 1"):
|
||||
Settings(ingest_bus_num_replicas=0)
|
||||
class TestProcrastinateConninfo:
|
||||
@pytest.mark.parametrize(
|
||||
"url,expected_sslmode",
|
||||
[
|
||||
("postgresql+asyncpg://mcp:p%40ss@db:5432/mcp", None),
|
||||
],
|
||||
)
|
||||
def test_conninfo_round_trips_password(self, monkeypatch, url, expected_sslmode):
|
||||
from psycopg.conninfo import conninfo_to_dict
|
||||
|
||||
monkeypatch.setattr(config_module, "get_database_url", lambda: url)
|
||||
# No SSL settings → sslmode omitted (libpq default ``prefer``).
|
||||
monkeypatch.setattr(config_module, "get_database_ssl", lambda: None)
|
||||
parsed = conninfo_to_dict(config_module.get_procrastinate_conninfo())
|
||||
assert parsed["password"] == "p@ss"
|
||||
assert parsed["host"] == "db"
|
||||
assert parsed["dbname"] == "mcp"
|
||||
assert parsed.get("sslmode") == expected_sslmode
|
||||
|
||||
def test_conninfo_connect_timeout_defaults_to_10(self, monkeypatch):
|
||||
from psycopg.conninfo import conninfo_to_dict
|
||||
|
||||
monkeypatch.setattr(
|
||||
config_module,
|
||||
"get_database_url",
|
||||
lambda: "postgresql+asyncpg://mcp:s@db/mcp",
|
||||
)
|
||||
monkeypatch.setattr(config_module, "get_database_ssl", lambda: None)
|
||||
parsed = conninfo_to_dict(config_module.get_procrastinate_conninfo())
|
||||
assert parsed["connect_timeout"] == "10"
|
||||
|
||||
def test_conninfo_honors_url_connect_timeout(self, monkeypatch):
|
||||
from psycopg.conninfo import conninfo_to_dict
|
||||
|
||||
monkeypatch.setattr(
|
||||
config_module,
|
||||
"get_database_url",
|
||||
lambda: "postgresql+asyncpg://mcp:s@db/mcp?connect_timeout=3",
|
||||
)
|
||||
monkeypatch.setattr(config_module, "get_database_ssl", lambda: None)
|
||||
parsed = conninfo_to_dict(config_module.get_procrastinate_conninfo())
|
||||
assert parsed["connect_timeout"] == "3"
|
||||
|
||||
def test_conninfo_ssl_mapping(self, monkeypatch):
|
||||
from psycopg.conninfo import conninfo_to_dict
|
||||
|
||||
monkeypatch.setattr(
|
||||
config_module,
|
||||
"get_database_url",
|
||||
lambda: "postgresql+asyncpg://mcp:s@db/mcp",
|
||||
)
|
||||
# verify off → encrypt without verifying.
|
||||
monkeypatch.setattr(config_module, "get_database_ssl", lambda: False)
|
||||
assert (
|
||||
conninfo_to_dict(config_module.get_procrastinate_conninfo())["sslmode"]
|
||||
== "require"
|
||||
)
|
||||
# verify on → verify-full.
|
||||
monkeypatch.setattr(config_module, "get_database_ssl", lambda: True)
|
||||
assert (
|
||||
conninfo_to_dict(config_module.get_procrastinate_conninfo())["sslmode"]
|
||||
== "verify-full"
|
||||
)
|
||||
|
||||
def test_conninfo_rejects_non_postgres(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
config_module, "get_database_url", lambda: "sqlite+aiosqlite:///x.db"
|
||||
)
|
||||
with pytest.raises(ValueError, match="requires a PostgreSQL DATABASE_URL"):
|
||||
config_module.get_procrastinate_conninfo()
|
||||
|
||||
|
||||
class TestCanonicalJson:
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Unit tests for the shared ingest-status read model (Deck #183)."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from nextcloud_mcp_server.vector.ingest_status import get_ingest_pending
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
class TestGetIngestPending:
|
||||
async def test_postgres_reads_job_counts(self):
|
||||
producer = AsyncMock()
|
||||
producer.job_counts.return_value = {"todo": 5, "doing": 2, "failed": 1}
|
||||
|
||||
result = await get_ingest_pending(
|
||||
task_producer=producer,
|
||||
document_receive_stream=None,
|
||||
ingest_queue="postgres",
|
||||
)
|
||||
assert result.pending == 7 # todo + doing
|
||||
assert result.job_counts == {"todo": 5, "doing": 2, "failed": 1}
|
||||
|
||||
async def test_postgres_degrades_to_zero_on_error(self):
|
||||
producer = AsyncMock()
|
||||
producer.job_counts.side_effect = RuntimeError("db down")
|
||||
|
||||
result = await get_ingest_pending(
|
||||
task_producer=producer,
|
||||
document_receive_stream=None,
|
||||
ingest_queue="postgres",
|
||||
)
|
||||
assert result.pending == 0
|
||||
assert result.job_counts == {}
|
||||
|
||||
async def test_memory_reads_stream_buffer(self):
|
||||
stream = SimpleNamespace(
|
||||
statistics=lambda: SimpleNamespace(current_buffer_used=3)
|
||||
)
|
||||
result = await get_ingest_pending(
|
||||
task_producer=None,
|
||||
document_receive_stream=stream,
|
||||
ingest_queue="memory",
|
||||
)
|
||||
assert result.pending == 3
|
||||
assert result.job_counts is None
|
||||
|
||||
async def test_memory_without_stream_is_zero(self):
|
||||
result = await get_ingest_pending(
|
||||
task_producer=None,
|
||||
document_receive_stream=None,
|
||||
ingest_queue="memory",
|
||||
)
|
||||
assert result.pending == 0
|
||||
assert result.job_counts is None
|
||||
@@ -1,155 +0,0 @@
|
||||
"""NATS ingest producer: DocumentTask → IngestMessage + dedup header (§3.4)."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from nextcloud_mcp_server.canonical import canonical_json
|
||||
from nextcloud_mcp_server.vector.queue.factory import _transport_for
|
||||
from nextcloud_mcp_server.vector.queue.nats import (
|
||||
NatsTaskProducer,
|
||||
_modified_at_rfc3339,
|
||||
msg_id,
|
||||
warn_if_insecure_nats_url,
|
||||
)
|
||||
from nextcloud_mcp_server.vector.queue.postgres import PostgresTaskProducer
|
||||
from nextcloud_mcp_server.vector.scanner import DocumentTask
|
||||
|
||||
FIXTURE = Path(__file__).parents[2] / "fixtures" / "ingest_message_example.json"
|
||||
TENANT = "00000000-0000-0000-0000-000000000001"
|
||||
|
||||
|
||||
def _producer(mocker, tenant_id=TENANT):
|
||||
return NatsTaskProducer(
|
||||
nc=mocker.MagicMock(), js=mocker.AsyncMock(), tenant_id=tenant_id
|
||||
)
|
||||
|
||||
|
||||
def test_ingest_message_translation(mocker):
|
||||
p = _producer(mocker)
|
||||
task = DocumentTask(
|
||||
user_id="alice",
|
||||
doc_id="12345",
|
||||
doc_type="file",
|
||||
operation="index",
|
||||
modified_at=1700000000,
|
||||
file_path="/Documents/report.pdf",
|
||||
etag="etag-abc123",
|
||||
)
|
||||
msg = p.ingest_message(task)
|
||||
assert msg["tenant_id"] == TENANT # from settings, not the task
|
||||
assert msg["content_hash"] == "etag-abc123" # etag wins
|
||||
assert msg["user_id"] == "alice"
|
||||
assert msg["doc_type"] == "file"
|
||||
assert msg["operation"] == "index"
|
||||
assert msg["file_path"] == "/Documents/report.pdf"
|
||||
|
||||
|
||||
def test_content_hash_falls_back_to_modified_at(mocker):
|
||||
p = _producer(mocker)
|
||||
task = DocumentTask(
|
||||
user_id="u", doc_id="d", doc_type="note", operation="delete", modified_at=0
|
||||
)
|
||||
assert p.ingest_message(task)["content_hash"] == "0"
|
||||
|
||||
|
||||
async def test_send_publishes_with_dedup_header(mocker):
|
||||
p = _producer(mocker)
|
||||
task = DocumentTask(
|
||||
user_id="alice",
|
||||
doc_id="12345",
|
||||
doc_type="file",
|
||||
operation="index",
|
||||
modified_at=1700000000,
|
||||
etag="e",
|
||||
)
|
||||
await p.send(task)
|
||||
p._js.publish.assert_awaited_once()
|
||||
args = p._js.publish.await_args.args
|
||||
kwargs = p._js.publish.await_args.kwargs
|
||||
assert args[0] == f"mcp.ingest.requested.{TENANT}"
|
||||
expected_mid = msg_id(TENANT, "12345", _modified_at_rfc3339(1700000000))
|
||||
assert kwargs["headers"]["Nats-Msg-Id"] == expected_mid
|
||||
assert json.loads(args[1])["doc_id"] == "12345"
|
||||
|
||||
|
||||
def test_msg_id_known_vector():
|
||||
mid = msg_id("t", "d", "2026-01-01T00:00:00+00:00")
|
||||
expected = hashlib.sha256(
|
||||
canonical_json(
|
||||
{
|
||||
"tenant_id": "t",
|
||||
"doc_id": "d",
|
||||
"modified_at": "2026-01-01T00:00:00+00:00",
|
||||
}
|
||||
)
|
||||
).hexdigest()
|
||||
assert mid == expected
|
||||
|
||||
|
||||
def test_publisher_matches_shared_fixture(mocker):
|
||||
# The same fixture is validated as an IngestMessage in the processor repo.
|
||||
# Here we assert the publisher emits exactly the fixture's key set + stable
|
||||
# field values (modified_at format is allowed to differ — epoch→ISO).
|
||||
fixture = json.loads(FIXTURE.read_text(encoding="utf-8"))
|
||||
p = _producer(mocker, tenant_id=fixture["tenant_id"])
|
||||
task = DocumentTask(
|
||||
user_id=fixture["user_id"],
|
||||
doc_id=fixture["doc_id"],
|
||||
doc_type=fixture["doc_type"],
|
||||
operation=fixture["operation"],
|
||||
modified_at=1764201600,
|
||||
file_path=fixture["file_path"],
|
||||
etag=fixture["content_hash"],
|
||||
)
|
||||
msg = p.ingest_message(task)
|
||||
assert set(msg.keys()) == set(fixture.keys())
|
||||
for key in (
|
||||
"tenant_id",
|
||||
"doc_id",
|
||||
"content_hash",
|
||||
"doc_type",
|
||||
"operation",
|
||||
"user_id",
|
||||
"file_path",
|
||||
):
|
||||
assert msg[key] == fixture[key]
|
||||
assert msg["modified_at"] # non-empty ISO timestamp
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"url,expected",
|
||||
[
|
||||
("nats://nats:4222", "nats"),
|
||||
("postgres://h/db", "postgres"),
|
||||
("postgresql://h/db", "postgres"),
|
||||
("https://elsewhere", "nats"),
|
||||
],
|
||||
)
|
||||
def test_transport_for(url, expected):
|
||||
assert _transport_for(url) == expected
|
||||
|
||||
|
||||
async def test_postgres_producer_is_a_seam():
|
||||
with pytest.raises(NotImplementedError, match="documented seam"):
|
||||
await PostgresTaskProducer.connect(object())
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"url,should_warn",
|
||||
[
|
||||
("nats://nats:4222", True),
|
||||
("ws://nats:8080", True),
|
||||
("tls://nats:4222", False),
|
||||
("wss://nats:8080", False),
|
||||
],
|
||||
)
|
||||
def test_warn_if_insecure_nats_url(url, should_warn, caplog):
|
||||
import logging
|
||||
|
||||
with caplog.at_level(logging.WARNING):
|
||||
warn_if_insecure_nats_url(url)
|
||||
warned = any("unencrypted transport" in r.getMessage() for r in caplog.records)
|
||||
assert warned is should_warn
|
||||
@@ -0,0 +1,234 @@
|
||||
"""Unit tests for the procrastinate ingest producer + task (Deck #183).
|
||||
|
||||
Uses procrastinate's in-memory connector so no live Postgres is required.
|
||||
"""
|
||||
|
||||
from typing import cast
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
from procrastinate import App, JobContext, testing
|
||||
|
||||
import nextcloud_mcp_server.vector.queue.procrastinate as pq
|
||||
from nextcloud_mcp_server.vector.scanner import DocumentTask
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app():
|
||||
"""An App bound to the in-memory connector with the ingest tasks."""
|
||||
return pq.build_app(testing.InMemoryConnector())
|
||||
|
||||
|
||||
def _task(doc_id="42", doc_type="note", operation="index"):
|
||||
return DocumentTask(
|
||||
user_id="alice",
|
||||
doc_id=doc_id,
|
||||
doc_type=doc_type,
|
||||
operation=operation,
|
||||
modified_at=100,
|
||||
etag="etag-abc",
|
||||
)
|
||||
|
||||
|
||||
class TestProcrastinateTaskProducer:
|
||||
async def test_send_defers_with_correct_job_shape(self, app):
|
||||
async with app.open_async():
|
||||
producer = pq.ProcrastinateTaskProducer(app)
|
||||
await producer.send(_task())
|
||||
|
||||
jobs = list(app.connector.jobs.values())
|
||||
assert len(jobs) == 1
|
||||
job = jobs[0]
|
||||
assert job["task_name"] == pq.INGEST_TASK_NAME
|
||||
assert job["queue_name"] == pq.INGEST_QUEUE_NAME
|
||||
assert job["queueing_lock"] == "alice:note:42"
|
||||
assert job["lock"] is None # no execution lock (crash-deadlock guard)
|
||||
assert job["args"]["doc_id"] == "42"
|
||||
assert job["args"]["etag"] == "etag-abc"
|
||||
|
||||
async def test_duplicate_send_is_deduped(self, app):
|
||||
async with app.open_async():
|
||||
producer = pq.ProcrastinateTaskProducer(app)
|
||||
await producer.send(_task())
|
||||
# Same doc again → AlreadyEnqueued, swallowed; still one job.
|
||||
await producer.send(_task())
|
||||
assert len(app.connector.jobs) == 1
|
||||
|
||||
async def test_distinct_docs_create_separate_jobs(self, app):
|
||||
async with app.open_async():
|
||||
producer = pq.ProcrastinateTaskProducer(app)
|
||||
await producer.send(_task(doc_id="1"))
|
||||
await producer.send(_task(doc_id="2"))
|
||||
assert len(app.connector.jobs) == 2
|
||||
|
||||
def test_clone_returns_self(self, app):
|
||||
producer = pq.ProcrastinateTaskProducer(app)
|
||||
assert producer.clone() is producer
|
||||
|
||||
async def test_connect_opens_pool_and_drain_closes(self, app, monkeypatch):
|
||||
# connect() resolves the process-wide app; point it at our in-memory one.
|
||||
monkeypatch.setattr(pq, "get_procrastinate_app", lambda: app)
|
||||
|
||||
producer = await pq.ProcrastinateTaskProducer.connect()
|
||||
# `await app.open_async()` must actually open the connector (regression
|
||||
# guard for the await-vs-`async with` form on the long-lived pool).
|
||||
assert app.connector.states == ["open_async"]
|
||||
|
||||
# An open pool means send() works end-to-end.
|
||||
await producer.send(_task())
|
||||
assert len(app.connector.jobs) == 1
|
||||
|
||||
await producer.drain()
|
||||
assert "closed_async" in app.connector.states
|
||||
|
||||
|
||||
class TestProcessDocumentTask:
|
||||
async def test_runs_pipeline_and_closes_client(self, monkeypatch):
|
||||
captured = {}
|
||||
|
||||
fake_client = AsyncMock()
|
||||
|
||||
async def fake_resolve(user_id):
|
||||
captured["user_id"] = user_id
|
||||
return fake_client
|
||||
|
||||
async def fake_process(task, nc_client, *, max_retries):
|
||||
captured["task"] = task
|
||||
captured["nc_client"] = nc_client
|
||||
captured["max_retries"] = max_retries
|
||||
|
||||
monkeypatch.setattr(pq, "_resolve_client", fake_resolve)
|
||||
monkeypatch.setattr(
|
||||
"nextcloud_mcp_server.vector.processor.process_document", fake_process
|
||||
)
|
||||
|
||||
# Calling the Task runs its wrapped function in-process.
|
||||
await pq.process_document_task(
|
||||
user_id="alice",
|
||||
doc_id="42",
|
||||
doc_type="note",
|
||||
operation="index",
|
||||
modified_at=100,
|
||||
etag="e1",
|
||||
)
|
||||
|
||||
assert captured["user_id"] == "alice"
|
||||
assert isinstance(captured["task"], DocumentTask)
|
||||
assert captured["task"].doc_id == "42"
|
||||
assert captured["task"].etag == "e1"
|
||||
# Worker disables the in-process retry loop; durable retry is the queue's.
|
||||
assert captured["max_retries"] == 1
|
||||
fake_client.close.assert_awaited_once()
|
||||
|
||||
async def test_pipeline_error_propagates_and_closes_client(self, monkeypatch):
|
||||
# A non-credential failure must propagate (so procrastinate's
|
||||
# RetryStrategy picks it up) and still close the client via finally.
|
||||
fake_client = AsyncMock()
|
||||
|
||||
async def fake_resolve(user_id):
|
||||
return fake_client
|
||||
|
||||
async def fake_process(task, nc_client, *, max_retries):
|
||||
raise RuntimeError("transient qdrant failure")
|
||||
|
||||
monkeypatch.setattr(pq, "_resolve_client", fake_resolve)
|
||||
monkeypatch.setattr(
|
||||
"nextcloud_mcp_server.vector.processor.process_document", fake_process
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="transient qdrant failure"):
|
||||
await pq.process_document_task(
|
||||
user_id="alice",
|
||||
doc_id="42",
|
||||
doc_type="note",
|
||||
operation="index",
|
||||
modified_at=100,
|
||||
)
|
||||
fake_client.close.assert_awaited_once()
|
||||
|
||||
async def test_skips_on_missing_credentials(self, monkeypatch):
|
||||
from nextcloud_mcp_server.vector.oauth_sync import NotProvisionedError
|
||||
|
||||
async def fake_resolve(user_id):
|
||||
raise NotProvisionedError("no app password")
|
||||
|
||||
called = False
|
||||
|
||||
async def fake_process(*args, **kwargs):
|
||||
nonlocal called
|
||||
called = True
|
||||
|
||||
monkeypatch.setattr(pq, "_resolve_client", fake_resolve)
|
||||
monkeypatch.setattr(
|
||||
"nextcloud_mcp_server.vector.processor.process_document", fake_process
|
||||
)
|
||||
|
||||
# Returns cleanly (job succeeds as a no-op); pipeline never runs.
|
||||
await pq.process_document_task(
|
||||
user_id="ghost",
|
||||
doc_id="9",
|
||||
doc_type="note",
|
||||
operation="index",
|
||||
modified_at=0,
|
||||
)
|
||||
assert called is False
|
||||
|
||||
|
||||
class TestReclaimStalledJobs:
|
||||
async def test_reclaims_each_stalled_job(self):
|
||||
from datetime import datetime
|
||||
|
||||
retried: list[int] = []
|
||||
|
||||
class Job:
|
||||
def __init__(self, id):
|
||||
self.id = id
|
||||
|
||||
class FakeManager:
|
||||
async def get_stalled_jobs(self, queue=None, seconds_since_heartbeat=0):
|
||||
assert queue == pq.INGEST_QUEUE_NAME
|
||||
return [Job(1), Job(2), Job(None)] # None id is skipped
|
||||
|
||||
async def retry_job_by_id_async(self, job_id, retry_at):
|
||||
assert isinstance(retry_at, datetime)
|
||||
retried.append(job_id)
|
||||
|
||||
class FakeApp:
|
||||
job_manager = FakeManager()
|
||||
|
||||
class Ctx:
|
||||
app = FakeApp()
|
||||
|
||||
await pq.reclaim_stalled_ingest_jobs(cast(JobContext, Ctx()), timestamp=0)
|
||||
assert retried == [1, 2]
|
||||
|
||||
|
||||
class TestGetIngestJobCounts:
|
||||
async def test_aggregates_stats_rows(self):
|
||||
class FakeManager:
|
||||
async def list_queues_async(self, queue=None):
|
||||
assert queue == pq.INGEST_QUEUE_NAME
|
||||
# procrastinate flattens per-status stats into top-level keys.
|
||||
return [
|
||||
{
|
||||
"name": "ingest",
|
||||
"jobs_count": 6,
|
||||
"todo": 3,
|
||||
"doing": 1,
|
||||
"succeeded": 0,
|
||||
"failed": 2,
|
||||
"cancelled": 0,
|
||||
"aborted": 0,
|
||||
}
|
||||
]
|
||||
|
||||
class FakeApp:
|
||||
job_manager = FakeManager()
|
||||
|
||||
counts = await pq.get_ingest_job_counts(cast(App, FakeApp()))
|
||||
assert counts["todo"] == 3
|
||||
assert counts["doing"] == 1
|
||||
assert counts["failed"] == 2
|
||||
assert counts["succeeded"] == 0
|
||||
@@ -1,167 +0,0 @@
|
||||
"""StatusStore + NATS status message handling (design §10.1, STATUS_BACKEND=bus)."""
|
||||
|
||||
import json
|
||||
|
||||
from nextcloud_mcp_server.vector.queue.status import (
|
||||
NatsStatusSubscriber,
|
||||
StatusStore,
|
||||
state_from_subject,
|
||||
)
|
||||
|
||||
|
||||
def test_store_records_and_counts():
|
||||
store = StatusStore()
|
||||
store.record("d1", "ready", content_hash="h1")
|
||||
store.record("d2", "failed")
|
||||
store.record("d1", "ready", content_hash="h1") # idempotent overwrite
|
||||
assert len(store) == 2
|
||||
assert store.counts() == {"ready": 1, "failed": 1}
|
||||
assert store.get("d1")["content_hash"] == "h1"
|
||||
|
||||
|
||||
def test_store_is_bounded_lru():
|
||||
store = StatusStore(max_size=2)
|
||||
store.record("d1", "ready")
|
||||
store.record("d2", "ready")
|
||||
store.record("d3", "ready") # evicts d1
|
||||
assert len(store) == 2
|
||||
assert store.get("d1") is None
|
||||
assert store.get("d3") is not None
|
||||
|
||||
|
||||
def test_state_from_subject():
|
||||
assert state_from_subject("mcp.document.ready.tenant-1") == "ready"
|
||||
assert state_from_subject("mcp.document.failed.tenant-1") == "failed"
|
||||
assert state_from_subject("mcp.document.reparsed.tenant-1") == "reparsed"
|
||||
assert state_from_subject("mcp.document.bogus.tenant-1") is None
|
||||
assert state_from_subject("mcp.ingest.requested.tenant-1") is None
|
||||
|
||||
|
||||
def test_handle_message_records_state():
|
||||
store = StatusStore()
|
||||
events = []
|
||||
sub = NatsStatusSubscriber(
|
||||
nc=None,
|
||||
js=None,
|
||||
tenant_id="t1",
|
||||
store=store,
|
||||
on_event=lambda d, s: events.append((d, s)),
|
||||
)
|
||||
payload = json.dumps(
|
||||
{
|
||||
"tenant_id": "t1",
|
||||
"doc_id": "doc-9",
|
||||
"content_hash": "abc",
|
||||
"transitioned_at": "2026-05-27T00:00:00Z",
|
||||
}
|
||||
).encode()
|
||||
sub.handle_message("mcp.document.ready.t1", payload)
|
||||
entry = store.get("doc-9")
|
||||
assert entry["state"] == "ready"
|
||||
assert entry["content_hash"] == "abc"
|
||||
assert events == [("doc-9", "ready")]
|
||||
|
||||
|
||||
def test_handle_message_ignores_bad_payload_and_subject():
|
||||
store = StatusStore()
|
||||
sub = NatsStatusSubscriber(nc=None, js=None, tenant_id="t1", store=store)
|
||||
sub.handle_message("mcp.document.ready.t1", b"not json")
|
||||
sub.handle_message("mcp.ingest.requested.t1", b'{"doc_id":"x"}')
|
||||
assert len(store) == 0
|
||||
|
||||
|
||||
async def test_run_signals_started_then_retries_subscribe(mocker, monkeypatch):
|
||||
"""run() signals started before subscribing, retries a failed subscribe,
|
||||
and consumes messages once subscribed."""
|
||||
import anyio
|
||||
|
||||
# Make backoff sleeps instant so the retry path doesn't stall the test.
|
||||
async def _no_sleep(*_a, **_k):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(anyio, "sleep", _no_sleep)
|
||||
|
||||
store = StatusStore()
|
||||
js = mocker.AsyncMock()
|
||||
|
||||
# First subscribe attempt fails (broker not ready), second succeeds.
|
||||
fake_sub = mocker.AsyncMock()
|
||||
js.pull_subscribe.side_effect = [ConnectionError("broker not ready"), fake_sub]
|
||||
|
||||
shutdown = anyio.Event()
|
||||
msg = mocker.Mock()
|
||||
msg.subject = "mcp.document.ready.t1"
|
||||
msg.data = json.dumps({"doc_id": "d1", "content_hash": "h1"}).encode()
|
||||
msg.ack = mocker.AsyncMock()
|
||||
|
||||
fetches = {"n": 0}
|
||||
|
||||
async def _fetch(*_a, **_k):
|
||||
fetches["n"] += 1
|
||||
if fetches["n"] == 1:
|
||||
return [msg]
|
||||
shutdown.set() # stop the loop after the first batch is handled
|
||||
return []
|
||||
|
||||
fake_sub.fetch.side_effect = _fetch
|
||||
|
||||
task_status = mocker.Mock()
|
||||
subscriber = NatsStatusSubscriber(
|
||||
nc=mocker.AsyncMock(), js=js, tenant_id="t1", store=store
|
||||
)
|
||||
|
||||
await subscriber.run(shutdown, task_status=task_status)
|
||||
|
||||
# started() fires before any subscribe attempt and exactly once.
|
||||
task_status.started.assert_called_once()
|
||||
# The failed first subscribe was retried (two attempts total).
|
||||
assert js.pull_subscribe.call_count == 2
|
||||
# The message from the successful subscription was recorded + acked.
|
||||
assert store.get("d1") == {
|
||||
"state": "ready",
|
||||
"content_hash": "h1",
|
||||
"transitioned_at": None,
|
||||
}
|
||||
msg.ack.assert_awaited_once()
|
||||
|
||||
|
||||
async def test_run_resubscribes_after_fetch_error(mocker, monkeypatch):
|
||||
"""A non-timeout fetch error drops the subscription and re-subscribes."""
|
||||
import anyio
|
||||
import nats.errors
|
||||
|
||||
async def _no_sleep(*_a, **_k):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(anyio, "sleep", _no_sleep)
|
||||
|
||||
store = StatusStore()
|
||||
js = mocker.AsyncMock()
|
||||
first_sub = mocker.AsyncMock()
|
||||
second_sub = mocker.AsyncMock()
|
||||
js.pull_subscribe.side_effect = [first_sub, second_sub]
|
||||
|
||||
shutdown = anyio.Event()
|
||||
|
||||
# first_sub.fetch raises a real broker error → re-subscribe.
|
||||
first_sub.fetch.side_effect = ConnectionResetError("broker dropped")
|
||||
|
||||
# second_sub.fetch idles once (timeout) then stops the loop.
|
||||
fetches = {"n": 0}
|
||||
|
||||
async def _second_fetch(*_a, **_k):
|
||||
fetches["n"] += 1
|
||||
if fetches["n"] == 1:
|
||||
raise nats.errors.TimeoutError
|
||||
shutdown.set()
|
||||
return []
|
||||
|
||||
second_sub.fetch.side_effect = _second_fetch
|
||||
|
||||
subscriber = NatsStatusSubscriber(
|
||||
nc=mocker.AsyncMock(), js=js, tenant_id="t1", store=store
|
||||
)
|
||||
await subscriber.run(shutdown)
|
||||
|
||||
# Re-subscribed after the fetch error (two subscriptions used).
|
||||
assert js.pull_subscribe.call_count == 2
|
||||
Reference in New Issue
Block a user