test: add integration tests for semantic search with in-process embeddings

Adds comprehensive integration tests for vector database semantic search that
work without external dependencies (Ollama), making them suitable for CI/CD.

Changes:
- Add SimpleEmbeddingProvider: in-process TF-IDF-like embeddings using feature hashing
- Make Ollama optional: embedding service now falls back to SimpleEmbeddingProvider
- Add 6 integration tests covering semantic search, filtering, and batch operations
- Downgrade urllib3 to 1.26.x for qdrant-client compatibility
- Update docker-compose.yml to comment out Ollama configuration (optional)

The SimpleEmbeddingProvider generates deterministic, normalized embeddings
suitable for testing semantic similarity without requiring external services.
Tests validate that similar texts have higher cosine similarity and that
semantic search correctly ranks results by relevance.

Test coverage:
- Deterministic embedding generation
- Semantic similarity between texts
- Full search flow with Qdrant (in-memory)
- Category filtering
- Empty result handling
- Batch embedding generation

All tests pass and can run in GitHub CI without Ollama infrastructure.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
2025-11-08 22:13:33 +01:00
co-authored by Claude
parent fdd82f59e2
commit 7b8c3f93a8
8 changed files with 500 additions and 20 deletions
+2 -1
View File
@@ -1,5 +1,6 @@
"""Embedding service package for generating vector embeddings."""
from .service import EmbeddingService, get_embedding_service
from .simple_provider import SimpleEmbeddingProvider
__all__ = ["EmbeddingService", "get_embedding_service"]
__all__ = ["EmbeddingService", "get_embedding_service", "SimpleEmbeddingProvider"]
+17 -8
View File
@@ -5,6 +5,7 @@ import os
from .base import EmbeddingProvider
from .ollama_provider import OllamaEmbeddingProvider
from .simple_provider import SimpleEmbeddingProvider
logger = logging.getLogger(__name__)
@@ -21,27 +22,35 @@ class EmbeddingService:
Auto-detect available embedding provider.
Checks environment variables in order:
1. OLLAMA_BASE_URL - Use Ollama provider
1. OLLAMA_BASE_URL - Use Ollama provider (production)
2. OPENAI_API_KEY - Use OpenAI provider (future)
3. Fallback to SimpleEmbeddingProvider (testing/development)
Returns:
Configured embedding provider
Raises:
ValueError: If no embedding provider is configured
"""
# Ollama provider (for this deployment)
# Ollama provider (production)
ollama_url = os.getenv("OLLAMA_BASE_URL")
if ollama_url:
logger.info(f"Using Ollama embedding provider: {ollama_url}")
return OllamaEmbeddingProvider(
base_url=ollama_url,
model=os.getenv("OLLAMA_EMBEDDING_MODEL", "nomic-embed-text"),
verify_ssl=os.getenv("OLLAMA_VERIFY_SSL", "true").lower() == "true",
)
raise ValueError(
"No embedding provider configured. "
"Set OLLAMA_BASE_URL environment variable."
# OpenAI provider (future implementation)
# openai_key = os.getenv("OPENAI_API_KEY")
# if openai_key:
# return OpenAIEmbeddingProvider(api_key=openai_key)
# Fallback to simple provider for development/testing
logger.warning(
"No embedding provider configured (OLLAMA_BASE_URL or OPENAI_API_KEY not set). "
"Using SimpleEmbeddingProvider for testing/development. "
"For production, configure an external embedding service."
)
return SimpleEmbeddingProvider(dimension=384)
async def embed(self, text: str) -> list[float]:
"""
@@ -0,0 +1,123 @@
"""Simple in-process embedding provider for testing.
This provider uses a basic TF-IDF-like approach with feature hashing to generate
deterministic embeddings without requiring external services. Suitable for testing
but not for production use.
"""
import hashlib
import math
import re
from collections import Counter
from .base import EmbeddingProvider
class SimpleEmbeddingProvider(EmbeddingProvider):
"""Simple deterministic embedding provider using feature hashing.
This implementation:
- Tokenizes text into words
- Uses feature hashing to map words to fixed-size vectors
- Applies TF-IDF-like weighting
- Normalizes vectors to unit length
Not suitable for production but good for testing semantic search infrastructure.
"""
def __init__(self, dimension: int = 384):
"""Initialize simple embedding provider.
Args:
dimension: Embedding dimension (default: 384)
"""
self.dimension = dimension
def _tokenize(self, text: str) -> list[str]:
"""Tokenize text into lowercase words.
Args:
text: Input text
Returns:
List of lowercase word tokens
"""
# Simple word tokenization
text = text.lower()
words = re.findall(r"\b\w+\b", text)
return words
def _hash_word(self, word: str) -> int:
"""Hash word to dimension index.
Args:
word: Word to hash
Returns:
Index in range [0, dimension)
"""
hash_bytes = hashlib.md5(word.encode()).digest()
hash_int = int.from_bytes(hash_bytes[:4], byteorder="big")
return hash_int % self.dimension
def _embed_single(self, text: str) -> list[float]:
"""Generate embedding for single text.
Args:
text: Input text
Returns:
Normalized embedding vector
"""
tokens = self._tokenize(text)
if not tokens:
return [0.0] * self.dimension
# Count term frequencies
term_freq = Counter(tokens)
# Initialize vector
vector = [0.0] * self.dimension
# Apply TF weighting with feature hashing
for word, count in term_freq.items():
idx = self._hash_word(word)
# Simple TF weighting: log(1 + count)
vector[idx] += math.log1p(count)
# Normalize to unit length
norm = math.sqrt(sum(x * x for x in vector))
if norm > 0:
vector = [x / norm for x in vector]
return vector
async def embed(self, text: str) -> list[float]:
"""Generate embedding vector for text.
Args:
text: Input text to embed
Returns:
Vector embedding as list of floats
"""
return self._embed_single(text)
async def embed_batch(self, texts: list[str]) -> list[list[float]]:
"""Generate embeddings for multiple texts.
Args:
texts: List of texts to embed
Returns:
List of vector embeddings
"""
return [self._embed_single(text) for text in texts]
def get_dimension(self) -> int:
"""Get embedding dimension.
Returns:
Vector dimension
"""
return self.dimension