feat(observability): astrolabe_* metrics + traces for the document pipeline
Make per-tier bottlenecks in the document-processing pipeline
(scan -> fetch -> parse -> chunk -> embed -> Qdrant upsert) visible via
metrics, traces, and structured logs. Today the document_processors layer
emits only a logger.info line: no metric, no span, and page counts live only
inside a log string. The single processing-duration histogram is unlabeled and
whole-document, so it cannot isolate parse vs embed vs upsert.
New astrolabe_* metric family (distinct from the mcp_* protocol metrics):
- astrolabe_document_parse_{duration_seconds,total} + pages/chars/bytes counters
recorded at the ProcessorRegistry.process() boundary (covers all current and
future processors uniformly)
- astrolabe_document_escalation_total (dormant; tiered-pipeline readiness)
- astrolabe_embedding_{duration_seconds,requests_total,chunks_total,chars_total}
- astrolabe_document_chunks_total, astrolabe_documents_indexed_total{source,status}
Tracing: new document_processor.parse child span + enriched embed/chunk span
attributes (provider/model/batch_size/chunk_count). Structured logs gain a
consistent field vocabulary (doc_id, doc_type, processor, tier, pages, chars,
byte_size, chunks, duration_ms, status) so Loki can aggregate without regex.
Tier-readiness: processor/tier are labels from day one and a tier property is
added to DocumentProcessor, so adding docling/OCR/LLM tiers later is additive
(new label values, never new metrics). Tenant comes from the kube namespace
label; mime_type/model are span attributes only (cardinality). Existing
mcp_vector_sync_*/mcp_qdrant_* are left untouched.
Refs Deck #175 (superset of #173 Phase 2). Dashboard/recording-rules follow-up
tracked on #175 for homelab-argocd.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
7e4b83dc94
commit
5d205fcaab
@@ -0,0 +1,122 @@
|
||||
"""Unit tests for embedding observability.
|
||||
|
||||
Covers:
|
||||
1. ``Settings.get_embedding_provider_family()`` — the single source of truth for
|
||||
the ``provider`` metric label / span attribute — across provider configs.
|
||||
2. The ``record_embedding`` helper — that it increments the right
|
||||
``astrolabe_embedding_*`` series and skips the throughput counters on error.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from prometheus_client import REGISTRY
|
||||
|
||||
from nextcloud_mcp_server.config import Settings
|
||||
from nextcloud_mcp_server.observability.metrics import record_embedding
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def _sample(name: str, labels: dict[str, str]) -> float:
|
||||
return REGISTRY.get_sample_value(name, labels) or 0.0
|
||||
|
||||
|
||||
class TestProviderFamily:
|
||||
"""Provider-family detection mirrors ProviderRegistry priority."""
|
||||
|
||||
def test_bedrock(self):
|
||||
assert (
|
||||
Settings(aws_region="us-east-1").get_embedding_provider_family()
|
||||
== "bedrock"
|
||||
)
|
||||
|
||||
def test_openai(self):
|
||||
settings = Settings(
|
||||
openai_api_key="sk-test",
|
||||
aws_region=None,
|
||||
bedrock_embedding_model=None,
|
||||
bedrock_generation_model=None,
|
||||
)
|
||||
assert settings.get_embedding_provider_family() == "openai"
|
||||
|
||||
def test_mistral(self):
|
||||
settings = Settings(
|
||||
mistral_api_key="m-test",
|
||||
aws_region=None,
|
||||
bedrock_embedding_model=None,
|
||||
bedrock_generation_model=None,
|
||||
openai_api_key=None,
|
||||
)
|
||||
assert settings.get_embedding_provider_family() == "mistral"
|
||||
|
||||
def test_ollama(self):
|
||||
settings = Settings(
|
||||
ollama_base_url="http://localhost:11434",
|
||||
aws_region=None,
|
||||
bedrock_embedding_model=None,
|
||||
bedrock_generation_model=None,
|
||||
openai_api_key=None,
|
||||
mistral_api_key=None,
|
||||
)
|
||||
assert settings.get_embedding_provider_family() == "ollama"
|
||||
|
||||
def test_simple_fallback(self):
|
||||
settings = Settings(
|
||||
aws_region=None,
|
||||
bedrock_embedding_model=None,
|
||||
bedrock_generation_model=None,
|
||||
openai_api_key=None,
|
||||
mistral_api_key=None,
|
||||
ollama_base_url=None,
|
||||
)
|
||||
assert settings.get_embedding_provider_family() == "simple"
|
||||
|
||||
def test_gateway_uses_model_prefix(self):
|
||||
settings = Settings(
|
||||
embedding_provider="gateway",
|
||||
embedding_gateway_url="http://gateway:8080",
|
||||
embedding_gateway_model="mistral/mistral-embed",
|
||||
)
|
||||
assert settings.get_embedding_provider_family() == "mistral"
|
||||
|
||||
|
||||
class TestRecordEmbedding:
|
||||
def test_dense_success_increments_throughput(self):
|
||||
labels = {"kind": "dense", "provider": "uttest-prov"}
|
||||
before_chunks = _sample("astrolabe_embedding_chunks_total", labels)
|
||||
before_chars = _sample("astrolabe_embedding_chars_total", labels)
|
||||
before_req = _sample(
|
||||
"astrolabe_embedding_requests_total", {**labels, "status": "success"}
|
||||
)
|
||||
|
||||
record_embedding("dense", "uttest-prov", 0.42, chunks=12, chars=3400)
|
||||
|
||||
assert _sample("astrolabe_embedding_chunks_total", labels) == (
|
||||
before_chunks + 12
|
||||
)
|
||||
assert _sample("astrolabe_embedding_chars_total", labels) == (
|
||||
before_chars + 3400
|
||||
)
|
||||
assert _sample(
|
||||
"astrolabe_embedding_requests_total", {**labels, "status": "success"}
|
||||
) == (before_req + 1)
|
||||
assert (
|
||||
_sample(
|
||||
"astrolabe_embedding_duration_seconds_count",
|
||||
{**labels, "status": "success"},
|
||||
)
|
||||
>= 1
|
||||
)
|
||||
|
||||
def test_sparse_error_skips_throughput(self):
|
||||
labels = {"kind": "sparse", "provider": "bm25-uttest"}
|
||||
record_embedding(
|
||||
"sparse", "bm25-uttest", 0.1, chunks=5, chars=100, status="error"
|
||||
)
|
||||
assert _sample("astrolabe_embedding_chunks_total", labels) == 0.0
|
||||
assert _sample("astrolabe_embedding_chars_total", labels) == 0.0
|
||||
assert (
|
||||
_sample("astrolabe_embedding_requests_total", {**labels, "status": "error"})
|
||||
== 1.0
|
||||
)
|
||||
Reference in New Issue
Block a user