Chris CoutinhoandClaude Opus 4.8
5d205fcaab
Make per-tier bottlenecks in the document-processing pipeline
(scan -> fetch -> parse -> chunk -> embed -> Qdrant upsert) visible via
metrics, traces, and structured logs. Today the document_processors layer
emits only a logger.info line: no metric, no span, and page counts live only
inside a log string. The single processing-duration histogram is unlabeled and
whole-document, so it cannot isolate parse vs embed vs upsert.
New astrolabe_* metric family (distinct from the mcp_* protocol metrics):
- astrolabe_document_parse_{duration_seconds,total} + pages/chars/bytes counters
recorded at the ProcessorRegistry.process() boundary (covers all current and
future processors uniformly)
- astrolabe_document_escalation_total (dormant; tiered-pipeline readiness)
- astrolabe_embedding_{duration_seconds,requests_total,chunks_total,chars_total}
- astrolabe_document_chunks_total, astrolabe_documents_indexed_total{source,status}
Tracing: new document_processor.parse child span + enriched embed/chunk span
attributes (provider/model/batch_size/chunk_count). Structured logs gain a
consistent field vocabulary (doc_id, doc_type, processor, tier, pages, chars,
byte_size, chunks, duration_ms, status) so Loki can aggregate without regex.
Tier-readiness: processor/tier are labels from day one and a tier property is
added to DocumentProcessor, so adding docling/OCR/LLM tiers later is additive
(new label values, never new metrics). Tenant comes from the kube namespace
label; mime_type/model are span attributes only (cardinality). Existing
mcp_vector_sync_*/mcp_qdrant_* are left untouched.
Refs Deck #175 (superset of #173 Phase 2). Dashboard/recording-rules follow-up
tracked on #175 for homelab-argocd.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
123 lines
4.1 KiB
Python
123 lines
4.1 KiB
Python
"""Unit tests for embedding observability.
|
|
|
|
Covers:
|
|
1. ``Settings.get_embedding_provider_family()`` — the single source of truth for
|
|
the ``provider`` metric label / span attribute — across provider configs.
|
|
2. The ``record_embedding`` helper — that it increments the right
|
|
``astrolabe_embedding_*`` series and skips the throughput counters on error.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
from prometheus_client import REGISTRY
|
|
|
|
from nextcloud_mcp_server.config import Settings
|
|
from nextcloud_mcp_server.observability.metrics import record_embedding
|
|
|
|
pytestmark = pytest.mark.unit
|
|
|
|
|
|
def _sample(name: str, labels: dict[str, str]) -> float:
|
|
return REGISTRY.get_sample_value(name, labels) or 0.0
|
|
|
|
|
|
class TestProviderFamily:
|
|
"""Provider-family detection mirrors ProviderRegistry priority."""
|
|
|
|
def test_bedrock(self):
|
|
assert (
|
|
Settings(aws_region="us-east-1").get_embedding_provider_family()
|
|
== "bedrock"
|
|
)
|
|
|
|
def test_openai(self):
|
|
settings = Settings(
|
|
openai_api_key="sk-test",
|
|
aws_region=None,
|
|
bedrock_embedding_model=None,
|
|
bedrock_generation_model=None,
|
|
)
|
|
assert settings.get_embedding_provider_family() == "openai"
|
|
|
|
def test_mistral(self):
|
|
settings = Settings(
|
|
mistral_api_key="m-test",
|
|
aws_region=None,
|
|
bedrock_embedding_model=None,
|
|
bedrock_generation_model=None,
|
|
openai_api_key=None,
|
|
)
|
|
assert settings.get_embedding_provider_family() == "mistral"
|
|
|
|
def test_ollama(self):
|
|
settings = Settings(
|
|
ollama_base_url="http://localhost:11434",
|
|
aws_region=None,
|
|
bedrock_embedding_model=None,
|
|
bedrock_generation_model=None,
|
|
openai_api_key=None,
|
|
mistral_api_key=None,
|
|
)
|
|
assert settings.get_embedding_provider_family() == "ollama"
|
|
|
|
def test_simple_fallback(self):
|
|
settings = Settings(
|
|
aws_region=None,
|
|
bedrock_embedding_model=None,
|
|
bedrock_generation_model=None,
|
|
openai_api_key=None,
|
|
mistral_api_key=None,
|
|
ollama_base_url=None,
|
|
)
|
|
assert settings.get_embedding_provider_family() == "simple"
|
|
|
|
def test_gateway_uses_model_prefix(self):
|
|
settings = Settings(
|
|
embedding_provider="gateway",
|
|
embedding_gateway_url="http://gateway:8080",
|
|
embedding_gateway_model="mistral/mistral-embed",
|
|
)
|
|
assert settings.get_embedding_provider_family() == "mistral"
|
|
|
|
|
|
class TestRecordEmbedding:
|
|
def test_dense_success_increments_throughput(self):
|
|
labels = {"kind": "dense", "provider": "uttest-prov"}
|
|
before_chunks = _sample("astrolabe_embedding_chunks_total", labels)
|
|
before_chars = _sample("astrolabe_embedding_chars_total", labels)
|
|
before_req = _sample(
|
|
"astrolabe_embedding_requests_total", {**labels, "status": "success"}
|
|
)
|
|
|
|
record_embedding("dense", "uttest-prov", 0.42, chunks=12, chars=3400)
|
|
|
|
assert _sample("astrolabe_embedding_chunks_total", labels) == (
|
|
before_chunks + 12
|
|
)
|
|
assert _sample("astrolabe_embedding_chars_total", labels) == (
|
|
before_chars + 3400
|
|
)
|
|
assert _sample(
|
|
"astrolabe_embedding_requests_total", {**labels, "status": "success"}
|
|
) == (before_req + 1)
|
|
assert (
|
|
_sample(
|
|
"astrolabe_embedding_duration_seconds_count",
|
|
{**labels, "status": "success"},
|
|
)
|
|
>= 1
|
|
)
|
|
|
|
def test_sparse_error_skips_throughput(self):
|
|
labels = {"kind": "sparse", "provider": "bm25-uttest"}
|
|
record_embedding(
|
|
"sparse", "bm25-uttest", 0.1, chunks=5, chars=100, status="error"
|
|
)
|
|
assert _sample("astrolabe_embedding_chunks_total", labels) == 0.0
|
|
assert _sample("astrolabe_embedding_chars_total", labels) == 0.0
|
|
assert (
|
|
_sample("astrolabe_embedding_requests_total", {**labels, "status": "error"})
|
|
== 1.0
|
|
)
|