feat: add the redisvl[otel] observability foundation (FDE-86)

.github/workflows/test.yml

@@ -120,7 +120,7 @@
         run: |
           set -euo pipefail
 
-          uv sync --all-extras --frozen
+          uv sync --all-extras --group otel-test --frozen
 
           # Must cover every redis-py-version matrix value; unhandled values fail.
           case "$REDIS_PY_VERSION" in
@@ -140,7 +140,63 @@
         run: |
           make test
 
+  test-without-otel:
+    # OpenTelemetry is optional (`redisvl[otel]`), but --all-extras installs it
+    # on every other leg: mistralai pulls in the SDK and fastmcp the API. This
+    # leg runs the default suite without it.
+    name: Tests without OpenTelemetry
+    runs-on: ubuntu-latest
+    # Ordered after the HF model cache is primed, not gated on it; see `test`.
+    needs: service-tests
+    if: ${{ !cancelled() }}
+    env:
+      HF_HOME: ${{ github.workspace }}/hf_cache
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v6
+
+      - name: Cache HuggingFace Models
+        uses: actions/cache@v4
+        with:
+          path: hf_cache
+          key: ${{ runner.os }}-hf-cache
+
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v10.0.0
+        with:
+          version: ${{ env.UV_VERSION }}
+          enable-cache: true
+          python-version: ${{ env.PYTHON_VERSION }}   # sets UV_PYTHON
+          cache-dependency-glob: |
+            pyproject.toml
+            uv.lock
+
+      - name: Install dependencies without OpenTelemetry
+        run: |
+          set -euo pipefail
+
+          # Every extra except mistralai and mcp, the two that pull it in.
+          uv sync --frozen \
+            --extra openai --extra cohere --extra voyageai \
+            --extra sentence-transformers --extra langcache --extra vertexai \
+            --extra google-genai --extra bedrock --extra pillow \
+            --extra sql-redis --extra ollama
+          if uv run python -c "import opentelemetry" 2>/dev/null; then
+            echo "::error title=OpenTelemetry installed::the without-OpenTelemetry leg can import opentelemetry"
+            exit 1
+          fi
+
+      - name: Run tests
+        run: |
+          # The MCP server needs fastmcp, which always brings the API with it.
+          make test ARGS="--ignore=tests/unit/test_mcp --ignore=tests/integration/test_mcp"
+
   notebooks:
     name: Notebook Validation
     runs-on: ubuntu-latest
     # Ordered after the HF cache is primed, not gated on it; see the `test` job.

Makefile

@@ -6,7 +6,7 @@ ARGS ?=
 
 install: ## Install the project and all dependencies
 	@echo "🚀 Installing project dependencies with uv"
-	uv sync --all-extras
+	uv sync --all-extras --group otel-test
 
 redis-start: ## Start Redis in Docker
 	@echo "🐳 Starting Redis"

pyproject.toml

@@ -81,6 +81,12 @@ pillow = [
 sql-redis = [
     "sql-redis>=0.7.1",
 ]
+# API only: the application owns the SDK and exporters. 1.30.0 is the first
+# release that accepts histogram bucket advice
+# (explicit_bucket_boundaries_advisory).
+otel = [
+    "opentelemetry-api>=1.30.0",
+]
 all = [
     "mistralai>=1.0.0,<2",
     "openai>=1.1.0",
@@ -96,6 +102,7 @@ all = [
     "pillow>=11.3.0",
     "sql-redis>=0.7.1",
     "ollama>=0.5.4",
+    "opentelemetry-api>=1.30.0",
 ]
 ollama = [
     "ollama>=0.5.4",
@@ -132,6 +139,10 @@ dev = [
     "codespell>=2.4.1,<3",
     "langcache>=0.9.0",
 ]
+# Not a default group, so a plain `uv sync` stays free of OpenTelemetry.
+otel-test = [
+    "opentelemetry-sdk>=1.30.0",
+]
 docs = [
     "sphinx>=4.4.0",
     "sphinx-book-theme>=1.0.0",

redisvl/utils/otel.py

@@ -0,0 +1,142 @@
+"""OpenTelemetry plumbing for RedisVL's own instrumentation. Internal.
+
+RedisVL depends only on the OpenTelemetry API, through the optional
+``redisvl[otel]`` extra; the application owns the SDK and exporters. The tracer
+and meter come from the global providers, so they do nothing until the
+application configures an SDK. When the API is not installed, or
+``REDISVL_OTEL_DISABLED`` is ``true``, stand-ins with the same interface are
+returned instead, so instrumented code never needs to check.
+
+The stand-in span leaves the active context alone, so write to a span through
+the handle from ``start_as_current_span(...) as span``, never through
+``trace.get_current_span()``, which would reach the caller's span instead.
+
+RedisVL never constructs a provider and never initialises redis-py's own
+observability; both belong to the application.
+"""
+
+import os
+from contextlib import ContextDecorator
+from enum import Enum
+from functools import lru_cache
+from typing import TYPE_CHECKING, Any
+
+from redisvl.version import __version__
+
+try:
+    from opentelemetry import metrics, trace
+    from opentelemetry.trace import SpanKind
+except ImportError:  # the redisvl[otel] extra is not installed
+    metrics = trace = None  # type: ignore[assignment]
+
+    class SpanKind(Enum):  # type: ignore[no-redef]
+        """Stand-in for ``opentelemetry.trace.SpanKind``."""
+
+        INTERNAL = 0
+        SERVER = 1
+        CLIENT = 2
+        PRODUCER = 3
+        CONSUMER = 4
+
+
+if TYPE_CHECKING:
+    from opentelemetry.metrics import Meter
+    from opentelemetry.trace import Tracer
+
+INSTRUMENTATION_SCOPE = "redisvl"
+# The semantic-conventions release the standard names below follow. Bump the
+# two together.
+SCHEMA_URL = "https://opentelemetry.io/schemas/1.44.0"
+# Read once, on first use. Same convention as OTEL_SDK_DISABLED: only "true",
+# in any case, disables.
+DISABLED_ENV_VAR = "REDISVL_OTEL_DISABLED"
+
+SEMANTIC_CACHE_OPERATION_DURATION = "redisvl.semantic_cache.operation.duration"
+SEMANTIC_CACHE_DISTANCE = "redisvl.semantic_cache.distance"
+GEN_AI_CLIENT_OPERATION_DURATION = "gen_ai.client.operation.duration"
+
+# Metric and attribute names. ``redisvl.*`` names are RedisVL's own; the rest come from the
+# OpenTelemetry semantic conventions, where ``gen_ai.*`` is still in
+# Development and may be renamed upstream.
+CACHE_NAME = "redisvl.cache.name"
+CACHE_OPERATION = "redisvl.cache.operation"
+CACHE_RESULT = "redisvl.cache.result"
+CACHE_DISTANCE = "redisvl.cache.distance"
+CACHE_DISTANCE_THRESHOLD = "redisvl.cache.distance_threshold"
+CACHE_NUM_RESULTS = "redisvl.cache.num_results"
+EMBEDDINGS_CACHE_RESULT = "redisvl.embeddings_cache.result"
+ERROR_TYPE = "error.type"
+DB_SYSTEM_NAME = "db.system.name"
+DB_COLLECTION_NAME = "db.collection.name"
+GEN_AI_OPERATION_NAME = "gen_ai.operation.name"
+GEN_AI_PROVIDER_NAME = "gen_ai.provider.name"
+GEN_AI_REQUEST_MODEL = "gen_ai.request.model"
+GEN_AI_EMBEDDINGS_DIMENSION_COUNT = "gen_ai.embeddings.dimension.count"
+GEN_AI_DATA_SOURCE_ID = "gen_ai.data_source.id"
+GEN_AI_RETRIEVAL_TOP_K = "gen_ai.retrieval.top_k"
+
+
+class _NoOpSpan(ContextDecorator):
+    """A span that records nothing, usable as its own context manager or
+    decorator, like ``start_as_current_span``'s result."""
+
+    def __enter__(self) -> "_NoOpSpan":
+        return self
+
+    def __exit__(self, *exc_info: Any) -> None:
+        return None
+
+    def is_recording(self) -> bool:
+        return False
+
+    def set_attribute(self, key: str, value: Any) -> None:
+        pass
+
+    def set_attributes(self, attributes: Any) -> None:
+        pass
+
+    def set_status(self, *args: Any, **kwargs: Any) -> None:
+        pass
+
+    def record_exception(self, *args: Any, **kwargs: Any) -> None:
+        pass
+
+
+class _NoOpTracer:
+    _span = _NoOpSpan()
+
+    def start_as_current_span(self, *args: Any, **kwargs: Any) -> _NoOpSpan:
+        return self._span
+
+
+class _NoOpHistogram:
+    def record(self, *args: Any, **kwargs: Any) -> None:
+        pass
+
+
+class _NoOpMeter:
+    _histogram = _NoOpHistogram()
+
+    def create_histogram(self, *args: Any, **kwargs: Any) -> _NoOpHistogram:
+        return self._histogram
+
+
+def _enabled() -> bool:
+    disabled = os.environ.get(DISABLED_ENV_VAR, "").strip().lower() == "true"
+    return trace is not None and not disabled
+
+
+@lru_cache(maxsize=None)
+def get_tracer() -> "Tracer":
+    """Return RedisVL's tracer, or a no-op stand-in."""
+    if not _enabled():
+        return _NoOpTracer()  # type: ignore[return-value]
+    return trace.get_tracer(INSTRUMENTATION_SCOPE, __version__, schema_url=SCHEMA_URL)
+
+
+@lru_cache(maxsize=None)
+def get_meter() -> "Meter":
+    """Return RedisVL's meter, or a no-op stand-in."""
+    if not _enabled():
+        return _NoOpMeter()  # type: ignore[return-value]
+    return metrics.get_meter(INSTRUMENTATION_SCOPE, __version__, schema_url=SCHEMA_URL)

tests/conftest.py

@@ -297,6 +297,51 @@ def cluster_client(redis_cluster_url):
     yield conn
 
 
+@pytest.fixture
+def otel_sdk():
+    """Configure OpenTelemetry SDK providers with in-memory exporters.
+
+    Yields ``(span_exporter, metric_reader)``. The API lets a process set its
+    global providers only once, so teardown resets those private globals, as
+    opentelemetry-test-utils does.
+    """
+    pytest.importorskip("opentelemetry.sdk")
+    from opentelemetry import metrics, trace
+    from opentelemetry.metrics import _internal as metrics_internal
+    from opentelemetry.sdk.metrics import MeterProvider
+    from opentelemetry.sdk.metrics.export import InMemoryMetricReader
+    from opentelemetry.sdk.trace import TracerProvider
+    from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+    from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
+        InMemorySpanExporter,
+    )
+    from opentelemetry.util._once import Once
+
+    from redisvl.utils import otel
+
+    def reset():
+        trace._TRACER_PROVIDER_SET_ONCE = Once()
+        trace._TRACER_PROVIDER = None
+        trace._PROXY_TRACER_PROVIDER = trace.ProxyTracerProvider()
+        metrics_internal._METER_PROVIDER_SET_ONCE = Once()
+        metrics_internal._METER_PROVIDER = None
+        metrics_internal._PROXY_METER_PROVIDER = metrics_internal._ProxyMeterProvider()
+        otel.get_tracer.cache_clear()
+        otel.get_meter.cache_clear()
+
+    spans, reader = InMemorySpanExporter(), InMemoryMetricReader()
+    tracer_provider = TracerProvider()
+    tracer_provider.add_span_processor(SimpleSpanProcessor(spans))
+    meter_provider = MeterProvider(metric_readers=[reader])
+    reset()
+    trace.set_tracer_provider(tracer_provider)
+    metrics.set_meter_provider(meter_provider)
+    yield spans, reader
+    tracer_provider.shutdown()
+    meter_provider.shutdown()
+    reset()
+
+
 @pytest.fixture(scope="session")
 def hf_vectorizer():
     return HFTextVectorizer(