From 59e062727ddca88d3502c1d5f0abacf03ba1b4b8 Mon Sep 17 00:00:00 2001 From: Stephen Wang Date: Tue, 15 Sep 2026 15:20:51 -0700 Subject: [PATCH 1/6] feat(obs): wire sgp-obs from the SDK for traces, metrics and logs (#518) Co-authored-by: Claude Opus 5 --- adk/pyproject.toml | 18 + .../lib/cli/templates/PRIVATE_INDEX.md | 62 +++ .../default-claude-code/Dockerfile-uv.j2 | 20 + .../default-claude-code/Dockerfile.j2 | 13 +- .../templates/default-codex/Dockerfile-uv.j2 | 20 + .../cli/templates/default-codex/Dockerfile.j2 | 13 +- .../default-langgraph/Dockerfile-uv.j2 | 20 + .../templates/default-langgraph/Dockerfile.j2 | 13 +- .../default-openai-agents/Dockerfile-uv.j2 | 20 + .../default-openai-agents/Dockerfile.j2 | 13 +- .../default-openai-agents/project/acp.py.j2 | 17 +- .../default-pydantic-ai/Dockerfile-uv.j2 | 20 + .../default-pydantic-ai/Dockerfile.j2 | 13 +- .../cli/templates/default/Dockerfile-uv.j2 | 20 + .../lib/cli/templates/default/Dockerfile.j2 | 13 +- .../sync-claude-code/Dockerfile-uv.j2 | 20 + .../templates/sync-claude-code/Dockerfile.j2 | 13 +- .../cli/templates/sync-codex/Dockerfile-uv.j2 | 20 + .../cli/templates/sync-codex/Dockerfile.j2 | 13 +- .../templates/sync-langgraph/Dockerfile-uv.j2 | 20 + .../templates/sync-langgraph/Dockerfile.j2 | 13 +- .../Dockerfile-uv.j2 | 20 + .../Dockerfile.j2 | 13 +- .../project/agent.py.j2 | 17 +- .../sync-openai-agents/Dockerfile-uv.j2 | 20 + .../sync-openai-agents/Dockerfile.j2 | 13 +- .../sync-openai-agents/project/acp.py.j2 | 19 +- .../sync-pydantic-ai/Dockerfile-uv.j2 | 20 + .../templates/sync-pydantic-ai/Dockerfile.j2 | 13 +- .../lib/cli/templates/sync/Dockerfile-uv.j2 | 20 + .../lib/cli/templates/sync/Dockerfile.j2 | 13 +- .../temporal-claude-code/Dockerfile-uv.j2 | 20 + .../temporal-claude-code/Dockerfile.j2 | 13 +- .../templates/temporal-codex/Dockerfile-uv.j2 | 20 + .../templates/temporal-codex/Dockerfile.j2 | 13 +- .../temporal-langgraph/Dockerfile-uv.j2 | 20 + .../temporal-langgraph/Dockerfile.j2 | 13 +- .../temporal-openai-agents/Dockerfile-uv.j2 | 20 + .../temporal-openai-agents/Dockerfile.j2 | 13 +- .../project/workflow.py.j2 | 19 +- .../temporal-pydantic-ai/Dockerfile-uv.j2 | 20 + .../temporal-pydantic-ai/Dockerfile.j2 | 13 +- .../cli/templates/temporal/Dockerfile-uv.j2 | 20 + .../lib/cli/templates/temporal/Dockerfile.j2 | 13 +- src/agentex/lib/cli/tests/__init__.py | 0 .../lib/cli/tests/test_template_tracing.py | 57 +++ .../lib/core/adapters/llm/_genai_metrics.py | 133 ++++++ .../lib/core/adapters/llm/adapter_litellm.py | 21 +- .../lib/core/adapters/llm/tests/__init__.py | 0 .../adapters/llm/tests/test_genai_metrics.py | 174 +++++++ .../lib/core/observability/sgp_obs_setup.py | 350 ++++++++++++++ .../observability/tests/test_sgp_obs_setup.py | 447 ++++++++++++++++++ .../lib/core/temporal/workers/worker.py | 23 +- .../core/tracing/tracing_processor_manager.py | 104 ++++ .../lib/sdk/fastacp/base/base_acp_server.py | 27 ++ .../lib/sdk/fastacp/base/tests/__init__.py | 0 .../fastacp/base/tests/test_shutdown_hooks.py | 302 ++++++++++++ src/agentex/lib/utils/logging.py | 83 ++++ src/agentex/lib/utils/tests/__init__.py | 0 .../lib/utils/tests/test_logging_handover.py | 182 +++++++ 60 files changed, 2635 insertions(+), 47 deletions(-) create mode 100644 src/agentex/lib/cli/templates/PRIVATE_INDEX.md create mode 100644 src/agentex/lib/cli/tests/__init__.py create mode 100644 src/agentex/lib/cli/tests/test_template_tracing.py create mode 100644 src/agentex/lib/core/adapters/llm/_genai_metrics.py create mode 100644 src/agentex/lib/core/adapters/llm/tests/__init__.py create mode 100644 src/agentex/lib/core/adapters/llm/tests/test_genai_metrics.py create mode 100644 src/agentex/lib/core/observability/sgp_obs_setup.py create mode 100644 src/agentex/lib/core/observability/tests/test_sgp_obs_setup.py create mode 100644 src/agentex/lib/sdk/fastacp/base/tests/__init__.py create mode 100644 src/agentex/lib/sdk/fastacp/base/tests/test_shutdown_hooks.py create mode 100644 src/agentex/lib/utils/tests/__init__.py create mode 100644 src/agentex/lib/utils/tests/test_logging_handover.py diff --git a/adk/pyproject.toml b/adk/pyproject.toml index b42b50e11..88125a8ce 100644 --- a/adk/pyproject.toml +++ b/adk/pyproject.toml @@ -65,6 +65,7 @@ dependencies = [ # agentex/lib/* uses `from typing import override` (3.12+) in 19 files. # The slim agentex-client keeps 3.11 support. requires-python = ">= 3.12,<4" + classifiers = [ "Typing :: Typed", "Intended Audience :: Developers", @@ -76,6 +77,23 @@ classifiers = [ "License :: OSI Approved :: Apache Software License", ] +# No `obs` extra, deliberately — do not add one for sgp-obs. +# +# sgp-obs is not on public PyPI (it is served from Scale's curated CodeArtifact +# mirror), and declaring it in [project.optional-dependencies] makes THIS repo's uv +# workspace unresolvable: `uv sync` re-locks, locking must resolve every declared +# optional dependency of every workspace member, and there is no way to exempt one. +# Measured: `uv lock --check`, `uv sync --all-extras`, plain `uv sync` with no extras, +# and `uv sync --all-extras --no-extra obs` all fail (`--no-extra` filters what is +# installed, not what is resolved); `uv lock` has no `--no-extra`; and +# `[tool.uv] override-dependencies` does not exempt it either. Only `--frozen` works, +# which would leave nobody able to re-lock this repo again. +# +# So the dependency is the AGENT's to declare — `sgp-obs[genai-auto,http,otlp]` +# against the mirror — and the SDK wires it when it is importable. See +# agentex/lib/core/observability/sgp_obs_setup.py; nothing imports sgp_obs outside a +# try, so a plain `pip install agentex-sdk` is unaffected either way. + [project.urls] Homepage = "https://github.com/scaleapi/scale-agentex-python" Repository = "https://github.com/scaleapi/scale-agentex-python" diff --git a/src/agentex/lib/cli/templates/PRIVATE_INDEX.md b/src/agentex/lib/cli/templates/PRIVATE_INDEX.md new file mode 100644 index 000000000..922107f9e --- /dev/null +++ b/src/agentex/lib/cli/templates/PRIVATE_INDEX.md @@ -0,0 +1,62 @@ +# The private package index in scaffold Dockerfiles + +Every scaffold Dockerfile mounts a build secret named `codeartifact-pip-conf`. It lets an agent +install Scale-internal packages — `sgp-obs`, for instance — that are not on public PyPI, without the +build holding any registry credential of its own. The control-plane broker mints a short-lived +CodeArtifact token per build and injects it as that secret. + +- Design: [Private Package Access for Customer Agents (PRD)](https://app.notion.com/p/Private-Package-Access-for-Customer-Agents-PRD-3ad904d6e6cb802cb091df1c25e230bc) +- Tracking: [SGPINF-1568](https://linear.app/scale-epd/issue/SGPINF-1568/provide-scale-internal-packages-to-agentex-agents-in-customer) + +## It is inert by default + +The mount is `required=false` and guarded by `[ -s ... ]`, so with no secret injected the build is +byte-identical to one without any of this. That covers every local build, every CI build, and every +agent that never opts in. An empty secret file is skipped too. + +## Opting in + +Add the index to the agent's `pyproject.toml`: + +```toml +[[tool.uv.index]] +name = "scale-pypi" +url = "" +default = true +``` + +The name must be exactly `scale-pypi`. uv applies `UV_INDEX_SCALE_PYPI_USERNAME` / +`UV_INDEX_SCALE_PYPI_PASSWORD` to the index of that name, so renaming it makes the credentials +silently stop applying. Setting `UV_INDEX_URL` instead does not authenticate a *named* index at +all, and the resolve fails with a 401. + +## Three things that are easy to get wrong + +**The token arrives percent-encoded.** The buildspec URL-encodes it to embed it in the pip config's +URL userinfo, so a token containing `+`, `/` or `=` arrives as `%2B`, `%2F`, `%3D`. The `uv sync` +templates decode it before exporting it as a password. Passing it through still-encoded sends a +different string and the resolve 401s. + +**The credential must not follow project-controlled configuration.** uv binds credentials by index +*name*, and the name-to-URL mapping would otherwise come from the agent's own `pyproject.toml` — so a +project that pointed `scale-pypi` at another host would receive the token. Verified against a local +server: the rogue host receives `Authorization: Basic aws:` and the real index is never +contacted. The templates therefore export `UV_INDEX` to re-bind the name to the URL the *broker* +supplied, which overrides whatever the project declared. With that in place the rogue host is never +contacted. The pinned URL carries no userinfo; the token still travels only in +`UV_INDEX_SCALE_PYPI_PASSWORD`. + +The case this defends is not a malicious agent author — they also write the Dockerfile and could read +the mounted secret directly. It is a *contributed* change to a project file, where a one-line URL edit +is far less conspicuous in review than an exfiltration command in a Dockerfile. + +**The two template variants work differently, deliberately.** + +| Template | Install step | How the credential is supplied | +| --- | --- | --- | +| `Dockerfile-uv.j2` | `uv sync` against the agent's `pyproject.toml` | Named index `scale-pypi`, pinned via `UV_INDEX`, token decoded into `UV_INDEX_SCALE_PYPI_PASSWORD` | +| `Dockerfile.j2` | `uv pip install -r requirements.txt` | No pyproject is present, so there is no named index to bind to. The credentialed URL is used directly via `UV_DEFAULT_INDEX` | + +The `requirements.txt` variant does **not** decode the token, and that is the point: it stays inside +the URL, already encoded for exactly that use. Decoding it there would corrupt it. It is also not +exposed to the redirection problem above, because the URL comes wholly from the injected secret. diff --git a/src/agentex/lib/cli/templates/default-claude-code/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default-claude-code/Dockerfile-uv.j2 index 93d0f82d1..8a22d0f89 100644 --- a/src/agentex/lib/cli/templates/default-claude-code/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default-claude-code/Dockerfile-uv.j2 @@ -34,7 +34,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -42,6 +55,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/default-claude-code/Dockerfile.j2 b/src/agentex/lib/cli/templates/default-claude-code/Dockerfile.j2 index d714d96f9..3556f6dfd 100644 --- a/src/agentex/lib/cli/templates/default-claude-code/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/default-claude-code/Dockerfile.j2 @@ -33,8 +33,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/default-codex/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default-codex/Dockerfile-uv.j2 index 02860b9b9..b3c03c988 100644 --- a/src/agentex/lib/cli/templates/default-codex/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default-codex/Dockerfile-uv.j2 @@ -34,7 +34,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -42,6 +55,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/default-codex/Dockerfile.j2 b/src/agentex/lib/cli/templates/default-codex/Dockerfile.j2 index 1a8eb1484..c0b3fc385 100644 --- a/src/agentex/lib/cli/templates/default-codex/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/default-codex/Dockerfile.j2 @@ -33,8 +33,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/default-langgraph/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default-langgraph/Dockerfile-uv.j2 index dd3035f7b..9b4f8d25b 100644 --- a/src/agentex/lib/cli/templates/default-langgraph/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default-langgraph/Dockerfile-uv.j2 @@ -30,7 +30,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -38,6 +51,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/default-langgraph/Dockerfile.j2 b/src/agentex/lib/cli/templates/default-langgraph/Dockerfile.j2 index 0395caf74..7f148e274 100644 --- a/src/agentex/lib/cli/templates/default-langgraph/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/default-langgraph/Dockerfile.j2 @@ -29,8 +29,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile-uv.j2 index dd3035f7b..9b4f8d25b 100644 --- a/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile-uv.j2 @@ -30,7 +30,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -38,6 +51,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile.j2 b/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile.j2 index 056d60b96..0a416aa38 100644 --- a/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile.j2 @@ -29,8 +29,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/default-openai-agents/project/acp.py.j2 b/src/agentex/lib/cli/templates/default-openai-agents/project/acp.py.j2 index 66ee31243..ad8b6e41d 100644 --- a/src/agentex/lib/cli/templates/default-openai-agents/project/acp.py.j2 +++ b/src/agentex/lib/cli/templates/default-openai-agents/project/acp.py.j2 @@ -20,7 +20,7 @@ from dotenv import load_dotenv load_dotenv() -from agents import Agent, Runner, function_tool, set_tracing_disabled +from agents import Agent, Runner, function_tool, set_trace_processors from agentex.lib import adk from agentex.lib.types.acp import SendEventParams, CancelTaskParams, CreateTaskParams @@ -34,10 +34,17 @@ from agentex.lib.core.harness.emitter import UnifiedEmitter from agentex.lib.adk import OpenAITurn from agentex.lib.core.tracing.tracing_processor_manager import add_tracing_processor_config -# Disable the openai-agents SDK's native tracer so it doesn't ship traces to -# api.openai.com using OPENAI_API_KEY (which may be a LiteLLM proxy key). -# SGP tracing below still runs via the Agentex tracing manager. -set_tracing_disabled(True) +# Drop the openai-agents SDK's own exporter, so it can't ship traces to +# api.openai.com using OPENAI_API_KEY (which may be a gateway/proxy key and would 401). +# +# Clearing the processor list rather than disabling tracing outright: disabling stops +# spans being produced AT ALL, which silently starves any processor added later — +# including the sgp-obs bridge the SDK installs when observability is on, so a Runner +# turn would contribute no model spans. Clearing instead removes the OpenAI exporter +# (which otherwise stays registered and is merely never fed) while leaving the +# machinery alive for the bridge to attach to. +# Agentex/SGP tracing still runs via the tracing manager. +set_trace_processors([]) logger = make_logger(__name__) diff --git a/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile-uv.j2 index dd3035f7b..9b4f8d25b 100644 --- a/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile-uv.j2 @@ -30,7 +30,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -38,6 +51,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile.j2 b/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile.j2 index 0395caf74..7f148e274 100644 --- a/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile.j2 @@ -29,8 +29,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/default/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default/Dockerfile-uv.j2 index dd3035f7b..9b4f8d25b 100644 --- a/src/agentex/lib/cli/templates/default/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default/Dockerfile-uv.j2 @@ -30,7 +30,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -38,6 +51,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/default/Dockerfile.j2 b/src/agentex/lib/cli/templates/default/Dockerfile.j2 index 0395caf74..7f148e274 100644 --- a/src/agentex/lib/cli/templates/default/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/default/Dockerfile.j2 @@ -29,8 +29,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile-uv.j2 index 93d0f82d1..8a22d0f89 100644 --- a/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile-uv.j2 @@ -34,7 +34,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -42,6 +55,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile.j2 b/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile.j2 index 6cdc70799..cd0338d18 100644 --- a/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile.j2 @@ -33,8 +33,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/sync-codex/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-codex/Dockerfile-uv.j2 index 02860b9b9..b3c03c988 100644 --- a/src/agentex/lib/cli/templates/sync-codex/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-codex/Dockerfile-uv.j2 @@ -34,7 +34,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -42,6 +55,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/sync-codex/Dockerfile.j2 b/src/agentex/lib/cli/templates/sync-codex/Dockerfile.j2 index afa4470d9..79293756d 100644 --- a/src/agentex/lib/cli/templates/sync-codex/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/sync-codex/Dockerfile.j2 @@ -33,8 +33,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile-uv.j2 index dd3035f7b..9b4f8d25b 100644 --- a/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile-uv.j2 @@ -30,7 +30,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -38,6 +51,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile.j2 b/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile.j2 index 4d9f41d45..d0c204e47 100644 --- a/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile.j2 @@ -29,8 +29,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile-uv.j2 index dd3035f7b..9b4f8d25b 100644 --- a/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile-uv.j2 @@ -30,7 +30,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -38,6 +51,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile.j2 b/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile.j2 index 4d9f41d45..d0c204e47 100644 --- a/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile.j2 @@ -29,8 +29,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/project/agent.py.j2 b/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/project/agent.py.j2 index 07546bffb..315c5a6ae 100644 --- a/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/project/agent.py.j2 +++ b/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/project/agent.py.j2 @@ -15,7 +15,7 @@ from __future__ import annotations from datetime import datetime -from agents import Runner, set_tracing_disabled +from agents import Runner, set_trace_processors from agents.sandbox import SandboxAgent, SandboxRunConfig from agents.run_config import RunConfig from agents.sandbox.sandboxes.unix_local import ( @@ -25,10 +25,17 @@ from agents.sandbox.sandboxes.unix_local import ( from project.tools import get_capabilities -# Disable the openai-agents SDK's native tracer so it doesn't ship traces to -# api.openai.com using OPENAI_API_KEY (which may be a gateway/proxy key and would -# 401). Agentex tracing still runs via the tracing manager configured in acp.py. -set_tracing_disabled(True) +# Drop the openai-agents SDK's own exporter, so it can't ship traces to +# api.openai.com using OPENAI_API_KEY (which may be a gateway/proxy key and would 401). +# +# Clearing the processor list rather than disabling tracing outright: disabling stops +# spans being produced AT ALL, which silently starves any processor added later — +# including the sgp-obs bridge the SDK installs when observability is on, so a Runner +# turn would contribute no model spans. Clearing instead removes the OpenAI exporter +# (which otherwise stays registered and is merely never fed) while leaving the +# machinery alive for the bridge to attach to. +# Agentex/SGP tracing still runs via the tracing manager. +set_trace_processors([]) MODEL_NAME = "gpt-4o-mini" INSTRUCTIONS = """You are a local sandbox assistant. diff --git a/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile-uv.j2 index dd3035f7b..9b4f8d25b 100644 --- a/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile-uv.j2 @@ -30,7 +30,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -38,6 +51,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile.j2 b/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile.j2 index 4d9f41d45..d0c204e47 100644 --- a/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile.j2 @@ -29,8 +29,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/sync-openai-agents/project/acp.py.j2 b/src/agentex/lib/cli/templates/sync-openai-agents/project/acp.py.j2 index 41029f2ce..07849e81d 100644 --- a/src/agentex/lib/cli/templates/sync-openai-agents/project/acp.py.j2 +++ b/src/agentex/lib/cli/templates/sync-openai-agents/project/acp.py.j2 @@ -13,12 +13,19 @@ from agentex.types.task_message_update import TaskMessageUpdate, StreamTaskMessa from agentex.types.task_message_content import TaskMessageContent from agentex.types.text_content import TextContent from agentex.lib.utils.logging import make_logger -from agents import Agent, Runner, RunConfig, function_tool, set_tracing_disabled - -# Disable the openai-agents SDK's native tracer so it doesn't ship traces to -# api.openai.com using OPENAI_API_KEY (which may be a LiteLLM proxy key). -# SGP tracing below still runs via the Agentex tracing manager. -set_tracing_disabled(True) +from agents import Agent, Runner, RunConfig, function_tool, set_trace_processors + +# Drop the openai-agents SDK's own exporter, so it can't ship traces to +# api.openai.com using OPENAI_API_KEY (which may be a gateway/proxy key and would 401). +# +# Clearing the processor list rather than disabling tracing outright: disabling stops +# spans being produced AT ALL, which silently starves any processor added later — +# including the sgp-obs bridge the SDK installs when observability is on, so a Runner +# turn would contribute no model spans. Clearing instead removes the OpenAI exporter +# (which otherwise stays registered and is merely never fed) while leaving the +# machinery alive for the bridge to attach to. +# Agentex/SGP tracing still runs via the tracing manager. +set_trace_processors([]) logger = make_logger(__name__) diff --git a/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile-uv.j2 index dd3035f7b..9b4f8d25b 100644 --- a/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile-uv.j2 @@ -30,7 +30,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -38,6 +51,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile.j2 b/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile.j2 index 4d9f41d45..d0c204e47 100644 --- a/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile.j2 @@ -29,8 +29,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/sync/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync/Dockerfile-uv.j2 index dd3035f7b..9b4f8d25b 100644 --- a/src/agentex/lib/cli/templates/sync/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync/Dockerfile-uv.j2 @@ -30,7 +30,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -38,6 +51,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/sync/Dockerfile.j2 b/src/agentex/lib/cli/templates/sync/Dockerfile.j2 index 4d9f41d45..d0c204e47 100644 --- a/src/agentex/lib/cli/templates/sync/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/sync/Dockerfile.j2 @@ -29,8 +29,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile-uv.j2 index f8746c573..1665bceb1 100644 --- a/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile-uv.j2 @@ -42,7 +42,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -50,6 +63,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile.j2 b/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile.j2 index 225863607..1297b7bd7 100644 --- a/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile.j2 @@ -41,8 +41,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/temporal-codex/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal-codex/Dockerfile-uv.j2 index 7e31387fa..41d83e31c 100644 --- a/src/agentex/lib/cli/templates/temporal-codex/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal-codex/Dockerfile-uv.j2 @@ -42,7 +42,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -50,6 +63,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/temporal-codex/Dockerfile.j2 b/src/agentex/lib/cli/templates/temporal-codex/Dockerfile.j2 index 0ae4e2079..d77d8073f 100644 --- a/src/agentex/lib/cli/templates/temporal-codex/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/temporal-codex/Dockerfile.j2 @@ -41,8 +41,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile-uv.j2 index 6746869df..56b4d949c 100644 --- a/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile-uv.j2 @@ -36,7 +36,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -44,6 +57,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile.j2 b/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile.j2 index ba47485a9..5bb133a22 100644 --- a/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile.j2 @@ -35,8 +35,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile-uv.j2 index 0d9801016..a674d7d35 100644 --- a/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile-uv.j2 @@ -36,7 +36,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -44,6 +57,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile.j2 b/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile.j2 index 4c1798c42..a9a63757d 100644 --- a/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile.j2 @@ -35,8 +35,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/temporal-openai-agents/project/workflow.py.j2 b/src/agentex/lib/cli/templates/temporal-openai-agents/project/workflow.py.j2 index af8b7a299..1b2ae547c 100644 --- a/src/agentex/lib/cli/templates/temporal-openai-agents/project/workflow.py.j2 +++ b/src/agentex/lib/cli/templates/temporal-openai-agents/project/workflow.py.j2 @@ -10,12 +10,19 @@ from agentex.lib.core.temporal.types.workflow import SignalName from agentex.lib.utils.logging import make_logger from agentex.types.text_content import TextContent from agentex.lib.environment_variables import EnvironmentVariables -from agents import Agent, Runner, set_tracing_disabled - -# Disable the openai-agents SDK's native tracer so it doesn't ship traces to -# api.openai.com using OPENAI_API_KEY (which may be a LiteLLM proxy key). -# SGP tracing below still runs via the Agentex tracing manager. -set_tracing_disabled(True) +from agents import Agent, Runner, set_trace_processors + +# Drop the openai-agents SDK's own exporter, so it can't ship traces to +# api.openai.com using OPENAI_API_KEY (which may be a gateway/proxy key and would 401). +# +# Clearing the processor list rather than disabling tracing outright: disabling stops +# spans being produced AT ALL, which silently starves any processor added later — +# including the sgp-obs bridge the SDK installs when observability is on, so a Runner +# turn would contribute no model spans. Clearing instead removes the OpenAI exporter +# (which otherwise stays registered and is merely never fed) while leaving the +# machinery alive for the bridge to attach to. +# Agentex/SGP tracing still runs via the tracing manager. +set_trace_processors([]) from agentex.lib.core.temporal.plugins.openai_agents.hooks.hooks import TemporalStreamingHooks from pydantic import BaseModel diff --git a/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile-uv.j2 index 0d9801016..a674d7d35 100644 --- a/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile-uv.j2 @@ -36,7 +36,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -44,6 +57,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile.j2 b/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile.j2 index 4c1798c42..a9a63757d 100644 --- a/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile.j2 @@ -35,8 +35,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/temporal/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal/Dockerfile-uv.j2 index 0d9801016..a674d7d35 100644 --- a/src/agentex/lib/cli/templates/temporal/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal/Dockerfile-uv.j2 @@ -36,7 +36,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -44,6 +57,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/temporal/Dockerfile.j2 b/src/agentex/lib/cli/templates/temporal/Dockerfile.j2 index 4c1798c42..a9a63757d 100644 --- a/src/agentex/lib/cli/templates/temporal/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/temporal/Dockerfile.j2 @@ -35,8 +35,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/tests/__init__.py b/src/agentex/lib/cli/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/agentex/lib/cli/tests/test_template_tracing.py b/src/agentex/lib/cli/tests/test_template_tracing.py new file mode 100644 index 000000000..adb76fd08 --- /dev/null +++ b/src/agentex/lib/cli/tests/test_template_tracing.py @@ -0,0 +1,57 @@ +"""The openai-agents scaffolds must not disable tracing outright. + +`set_tracing_disabled(True)` stops openai-agents producing spans AT ALL, which +silently starves any processor registered later — including the sgp-obs bridge the SDK +installs when observability is on. The bridge still reports itself installed, so a +Runner turn contributes no model spans and nothing says why. + +Measured against a spy processor: + + set_tracing_disabled(True) -> processors ['BatchTraceProcessor', 'Spy'], spy saw 0 + set_trace_processors([]) -> processors ['Spy'], spy saw 1 + +Note the first row: disabling tracing leaves the OpenAI exporter REGISTERED, merely +never fed. Clearing the list actually removes it, so the replacement is strictly better +at the thing the original was trying to do — keep traces away from api.openai.com. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +TEMPLATES = Path(__file__).resolve().parents[1] / "templates" + + +def _templates_using_agents_tracing() -> list[Path]: + return sorted( + p for p in TEMPLATES.rglob("*.j2") if "set_trace_processors" in p.read_text() + ) + + +def test_some_templates_were_found(): + """Guards the glob itself: if the templates move, the assertions below would + vacuously pass on an empty list.""" + assert _templates_using_agents_tracing(), f"no templates found under {TEMPLATES}" + + +@pytest.mark.parametrize( + "template", _templates_using_agents_tracing(), ids=lambda p: p.parent.parent.name +) +class TestOpenAIAgentsScaffolds: + def test_does_not_disable_tracing(self, template: Path): + text = template.read_text() + assert "set_tracing_disabled(" not in text, ( + f"{template} disables openai-agents tracing, which starves the sgp-obs bridge" + ) + + def test_clears_the_processor_list_instead(self, template: Path): + assert "set_trace_processors([])" in template.read_text() + + def test_imports_what_it_calls(self, template: Path): + text = template.read_text() + assert "set_trace_processors" in text.split("\n\n")[0] or any( + "import" in line and "set_trace_processors" in line + for line in text.splitlines() + ), f"{template} calls set_trace_processors without importing it" diff --git a/src/agentex/lib/core/adapters/llm/_genai_metrics.py b/src/agentex/lib/core/adapters/llm/_genai_metrics.py new file mode 100644 index 000000000..3c2293cd6 --- /dev/null +++ b/src/agentex/lib/core/adapters/llm/_genai_metrics.py @@ -0,0 +1,133 @@ +"""GenAI metrics for the litellm gateway, via ``sgp_obs.metrics.genai.call()``. + +Why the SDK does this rather than leaving it to zero-code instrumentation: + +Most model calls in the fleet reach the wire through the ``openai`` client, and for +those, patching that one client covers everything with no code — ``Runner.run``, the +ADK's openai provider, and litellm pointed at an OpenAI-compatible proxy. The client +patch cannot help in two situations, and this gateway hits both: + +1. **litellm routing natively** to Anthropic, Bedrock, Vertex or Azure never touches + the ``openai`` client, so nothing records it at all. +2. Even in proxy mode, the patch sits *inside* the OpenAI client, so it reports + ``gen_ai.provider.name="openai"`` — the protocol. It cannot know that the caller + asked for ``claude-sonnet-4``. This gateway chose the vendor, so it can say so. + +``transport=`` resolves the overlap between the two: when the call is going out over +the OpenAI client, we name that, and ``call()`` stands down if the client instrumentor +is already recording. When litellm routes natively there is no such overlap, so we +record. That decision is made per call, from the model string, in +:func:`_split_model`. + +Everything here is fail-open: sgp-obs is an optional dependency and a telemetry problem +must never fail a model call. If the import fails, :func:`inference_call` returns an +object that records nothing and costs nothing. +""" + +from __future__ import annotations + +from typing import Any + +from agentex.lib.utils.logging import make_logger + +logger = make_logger(__name__) + +# litellm's directive for "send this to the configured OpenAI-compatible proxy". It is +# a routing instruction, not a vendor, so it is stripped before reading the vendor. +_PROXY_PREFIX = "litellm_proxy/" + +# A bare model name with no "/" prefix is OpenAI, per litellm's own default. +_DEFAULT_VENDOR = "openai" + +_warned = False + + +def _split_model(model: str) -> tuple[str, bool]: + """``(vendor, goes_out_over_the_openai_client)`` for a litellm model string. + + ``"litellm_proxy/anthropic/claude-sonnet-4"`` -> ``("anthropic", True)`` + ``"anthropic/claude-sonnet-4"`` -> ``("anthropic", False)`` + ``"gpt-4o"`` -> ``("openai", True)`` + + A bare name is OpenAI, and litellm reaches OpenAI through the ``openai`` + client, so the client instrumentor already sees it and we stand down. + """ + proxied = model.startswith(_PROXY_PREFIX) + rest = model[len(_PROXY_PREFIX):] if proxied else model + vendor = rest.split("/", 1)[0] if "/" in rest else _DEFAULT_VENDOR + # Proxy mode always leaves over the OpenAI client. So does a native openai/* call. + return (vendor or _DEFAULT_VENDOR), proxied or vendor == _DEFAULT_VENDOR + + +def resolve_model(args: tuple[Any, ...], kwargs: dict[str, Any]) -> str: + """The model for a litellm call, whether it arrived by keyword or positionally. + + ``litellm.acompletion`` takes ``model`` as its FIRST positional argument, and the + gateway forwards ``*args`` untouched, so ``gateway.acompletion("anthropic/claude- + sonnet-4", messages)`` is a legal call that puts the model in ``args[0]``. + + Reading only ``kwargs`` there does not merely mislabel the vendor, it loses the + measurement: an empty model resolves to the default vendor "openai", which sets + ``transport=OPENAI``, which makes ``call()`` stand down for the OpenAI client + instrumentor — while litellm routes natively to Anthropic and never touches that + client. Nothing records it and nothing says so. + """ + model = kwargs.get("model") + if not model and args: + model = args[0] + # Positional args are forwarded verbatim, so args[0] is whatever the caller passed; + # only a string can be a litellm model name. + return model if isinstance(model, str) else "" + + +def inference_call(kwargs: dict[str, Any], args: tuple[Any, ...] = ()) -> Any: + """Begin recording one litellm call. Never raises, never returns None.""" + try: + # See sgp_obs_setup.py: optional, not publicly installable, absent in CI. + from sgp_obs.metrics import genai # type: ignore[import-not-found] + except Exception: + global _warned + if not _warned: + _warned = True + logger.debug( + "sgp-obs is not available; GenAI metrics are off for litellm calls" + ) + return _NULL_CALL + + try: + model = resolve_model(args, kwargs) + vendor, over_openai_client = _split_model(model) + return genai.call( + provider=vendor, + operation=genai.CHAT, + model=model, + # litellm normalises every vendor's response onto the OpenAI shape, so one + # parser reads them all — which is exactly what `spec` separates from the + # `provider` label. + spec=genai.OPENAI_SPEC, + transport=genai.OPENAI if over_openai_client else "", + ) + except Exception: + logger.debug("could not start a GenAI metrics record", exc_info=True) + return _NULL_CALL + + +class _NullCall: + """What call sites get when sgp-obs is absent. Records nothing, costs nothing.""" + + def observe(self, response: Any) -> Any: + return response + + # Underscored like __aexit__'s params below: present for parity with the real + # sgp-obs call object, never read here. + def failed(self, _error: BaseException) -> None: + return + + async def __aenter__(self) -> "_NullCall": + return self + + async def __aexit__(self, _exc_type: Any, _exc: Any, _tb: Any) -> bool: + return False # never suppress the caller's exception + + +_NULL_CALL = _NullCall() diff --git a/src/agentex/lib/core/adapters/llm/adapter_litellm.py b/src/agentex/lib/core/adapters/llm/adapter_litellm.py index 7935f5f49..8fb1602aa 100644 --- a/src/agentex/lib/core/adapters/llm/adapter_litellm.py +++ b/src/agentex/lib/core/adapters/llm/adapter_litellm.py @@ -6,6 +6,7 @@ from agentex.lib.utils.logging import make_logger from agentex.lib.types.llm_messages import Completion from agentex.lib.core.adapters.llm.port import LLMGateway +from agentex.lib.core.adapters.llm._genai_metrics import inference_call logger = make_logger(__name__) @@ -36,9 +37,13 @@ async def acompletion(self, *args, **kwargs) -> Completion: "Please use self.acompletion_stream instead of self.acompletion to stream responses" ) - # Return a single completion for non-streaming - response = await llm.acompletion(*args, **kwargs) - return Completion.model_validate(response) + # `async with`, not try/except: asyncio.CancelledError is a BaseException, so a + # caller that disappears mid-flight would skip an `except Exception` handler and + # the record would be silently dropped. + async with inference_call(kwargs, args) as call: + # Return a single completion for non-streaming + response = call.observe(await llm.acompletion(*args, **kwargs)) + return Completion.model_validate(response) @override async def acompletion_stream( @@ -47,5 +52,11 @@ async def acompletion_stream( if not kwargs.get("stream"): raise ValueError("To use streaming, please set stream=True in the kwargs") - async for chunk in await llm.acompletion(*args, **kwargs): # type: ignore[misc] - yield Completion.model_validate(chunk) + async with inference_call(kwargs, args) as call: + # observe() takes ownership of the stream and yields the same chunks, so it + # can read time-to-first-chunk and the token totals off the last chunk. + # Wrapping only the `await` would return before the first chunk arrived and + # record zero tokens for every streamed call. + stream = call.observe(await llm.acompletion(*args, **kwargs)) + async for chunk in stream: # type: ignore[misc] + yield Completion.model_validate(chunk) diff --git a/src/agentex/lib/core/adapters/llm/tests/__init__.py b/src/agentex/lib/core/adapters/llm/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/agentex/lib/core/adapters/llm/tests/test_genai_metrics.py b/src/agentex/lib/core/adapters/llm/tests/test_genai_metrics.py new file mode 100644 index 000000000..b4276fdb1 --- /dev/null +++ b/src/agentex/lib/core/adapters/llm/tests/test_genai_metrics.py @@ -0,0 +1,174 @@ +"""Tests for ``agentex.lib.core.adapters.llm._genai_metrics``. + +The important property is the one that holds in every environment today: with +``sgp-obs`` absent, :func:`inference_call` must hand back something the litellm +gateway can drive as an async context manager, whose ``observe()`` returns the +response untouched and which never swallows the caller's exception. That is the +path every agent without the ``obs`` extra takes on every model call, so a +regression here breaks model calls rather than just losing a metric. +""" + +from __future__ import annotations + +import sys +import builtins + +import pytest + +from agentex.lib.core.adapters.llm import _genai_metrics +from agentex.lib.core.adapters.llm._genai_metrics import ( + _split_model, + resolve_model, + inference_call, +) + + +class TestSplitModel: + """``(vendor, goes_out_over_the_openai_client)``. The boolean decides whether + ``call()`` stands down for the OpenAI client instrumentor or records itself, so + getting it wrong either double-counts a call or loses it.""" + + @pytest.mark.parametrize( + ("model", "vendor", "over_openai_client"), + [ + # Proxy mode: litellm sends this to an OpenAI-compatible proxy over the + # openai client, but the caller asked for a non-OpenAI vendor. + ("litellm_proxy/anthropic/claude-sonnet-4", "anthropic", True), + ("litellm_proxy/gpt-4o", "openai", True), + # Native routing: litellm's own handler, no openai client involved. + ("anthropic/claude-sonnet-4", "anthropic", False), + ("bedrock/anthropic.claude-v2", "bedrock", False), + ("vertex_ai/gemini-2.0-flash", "vertex_ai", False), + # A bare name is OpenAI per litellm's default, and reaches OpenAI + # through the openai client — so the instrumentor already sees it. + ("gpt-4o", "openai", True), + ("openai/gpt-4o", "openai", True), + ], + ) + def test_vendor_and_transport(self, model, vendor, over_openai_client): + assert _split_model(model) == (vendor, over_openai_client) + + def test_empty_model_does_not_raise(self): + """kwargs.get("model") is "" when a caller passes model positionally. + Falling back to litellm's own default is right, and must not blow up.""" + assert _split_model("") == ("openai", True) + + +class TestFailsOpenWithoutSgpObs: + @staticmethod + def _hide_sgp_obs(monkeypatch): + for name in [m for m in sys.modules if m.startswith("sgp_obs")]: + monkeypatch.delitem(sys.modules, name, raising=False) + real_import = builtins.__import__ + + def no_sgp_obs(name, *args, **kwargs): + if name == "sgp_obs" or name.startswith("sgp_obs."): + raise ImportError("No module named 'sgp_obs'") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", no_sgp_obs) + # The "already warned" latch is module state; reset so the path is exercised. + monkeypatch.setattr(_genai_metrics, "_warned", False) + + def test_returns_a_usable_recorder_not_none(self, monkeypatch): + self._hide_sgp_obs(monkeypatch) + assert inference_call({"model": "gpt-4o"}) is _genai_metrics._NULL_CALL + + async def test_observe_returns_the_response_unchanged(self, monkeypatch): + """The gateway does `call.observe(await acompletion(...))`, so an observe() + that returned None would turn every completion into None.""" + self._hide_sgp_obs(monkeypatch) + sentinel = object() + async with inference_call({"model": "gpt-4o"}) as call: + assert call.observe(sentinel) is sentinel + + async def test_does_not_suppress_the_callers_exception(self, monkeypatch): + """__aexit__ must return falsey. Suppressing here would make a failed model + call look like a successful one that returned nothing.""" + self._hide_sgp_obs(monkeypatch) + with pytest.raises(ValueError, match="upstream"): + async with inference_call({"model": "gpt-4o"}): + raise ValueError("upstream blew up") + + async def test_cancellation_still_propagates(self, monkeypatch): + """CancelledError is a BaseException; the `async with` in the gateway exists + so a disappearing caller is not silently dropped.""" + import asyncio + + self._hide_sgp_obs(monkeypatch) + with pytest.raises(asyncio.CancelledError): + async with inference_call({"model": "gpt-4o"}): + raise asyncio.CancelledError() + + def test_a_broken_sgp_obs_does_not_break_a_model_call(self, monkeypatch): + """Not just ImportError: anything raised while starting a record must fall + back to the null recorder.""" + module = type(sys)("sgp_obs.metrics") + genai = type(sys)("genai") + + def exploding(**_kwargs): + raise RuntimeError("sgp-obs internals changed") + + genai.call = exploding + genai.CHAT = "chat" + genai.OPENAI_SPEC = "openai" + genai.OPENAI = "openai" + module.genai = genai + monkeypatch.setitem(sys.modules, "sgp_obs", type(sys)("sgp_obs")) + monkeypatch.setitem(sys.modules, "sgp_obs.metrics", module) + assert inference_call({"model": "gpt-4o"}) is _genai_metrics._NULL_CALL + + +class TestResolveModel: + """litellm takes `model` as its FIRST positional argument and the gateway forwards + *args untouched, so a positional call is legal and must still be measured. + + Reading only kwargs does not merely mislabel the vendor: an empty model resolves to + the default vendor "openai", which sets transport=OPENAI, which makes call() stand + down for the OpenAI client instrumentor — while litellm routes natively to Anthropic + and never touches that client. Nothing records it and nothing says so. + """ + + def test_keyword_model(self): + assert resolve_model((), {"model": "gpt-4o"}) == "gpt-4o" + + def test_positional_model(self): + assert resolve_model(("anthropic/claude-sonnet-4",), {}) == "anthropic/claude-sonnet-4" + + def test_keyword_wins_over_positional(self): + """litellm itself would reject both, but if it ever resolved one, the keyword is + the explicit intent.""" + assert resolve_model(("a/b",), {"model": "c/d"}) == "c/d" + + def test_no_model_at_all(self): + assert resolve_model((), {}) == "" + + def test_a_non_string_first_arg_is_not_a_model(self): + """*args is forwarded verbatim, so args[0] is whatever the caller passed.""" + assert resolve_model(([{"role": "user"}],), {}) == "" + + def test_positional_native_vendor_does_not_stand_down(self, monkeypatch): + """The regression this guards: a positional Anthropic model must be recorded by + the gateway, because nothing else will.""" + seen = {} + + class _Genai: + CHAT = "chat" + OPENAI_SPEC = "openai" + OPENAI = "openai" + + @staticmethod + def call(**kwargs): + seen.update(kwargs) + return _genai_metrics._NULL_CALL + + module = type(sys)("sgp_obs.metrics") + module.genai = _Genai + monkeypatch.setitem(sys.modules, "sgp_obs", type(sys)("sgp_obs")) + monkeypatch.setitem(sys.modules, "sgp_obs.metrics", module) + + inference_call({}, ("anthropic/claude-sonnet-4",)) + assert seen["model"] == "anthropic/claude-sonnet-4" + assert seen["provider"] == "anthropic" + # Empty transport == "no OpenAI-client overlap, so record it here". + assert seen["transport"] == "" diff --git a/src/agentex/lib/core/observability/sgp_obs_setup.py b/src/agentex/lib/core/observability/sgp_obs_setup.py new file mode 100644 index 000000000..5b76766f5 --- /dev/null +++ b/src/agentex/lib/core/observability/sgp_obs_setup.py @@ -0,0 +1,350 @@ +"""Optional sgp-obs wiring: traces, metrics and logs, switched on by environment. + +Why this lives in the SDK rather than in each agent: the fleet is ~147 agent repos, +and their deployments pin an exact SDK version. Doing the wiring here means an agent +adopts observability by installing ``sgp-obs`` and setting environment, instead of +carrying the wiring code — including the two parts that are easy to get wrong and +fail silently (where ``init()`` is called from, and flushing on the way out). + +``sgp-obs`` is NOT declared as a dependency or an extra of this package. It is not on +public PyPI, and declaring it would make this repo's own uv workspace unresolvable: +``uv sync`` re-locks, locking must resolve every declared optional dependency, and +neither ``--no-extra`` nor ``[tool.uv] override-dependencies`` exempts one. So the +contract is inverted — an agent declares ``sgp-obs[genai-auto,http,otlp]`` itself, +against Scale's curated mirror, and this module wires it if it is importable. Nothing +here imports ``sgp_obs`` outside a ``try``, so a plain ``pip install agentex-sdk`` +behaves exactly as it did before this module existed. + +TWO gates, both of which must pass before anything is recorded: + +1. ``sgp-obs`` must be importable. If it is not, this returns ``"not_installed"``. +2. The environment must ask for it. As of sgp-obs 0.16.0 every signal is opt-in + TWICE: the master switch ``SGP_OBS_ENABLED=true``, AND that signal's + ``*_DISABLED`` variable set to an explicit ``false``. An unset ``*_DISABLED`` + leaves the signal OFF. So the master switch on its own wires nothing at all — + measured on 0.16.0, ``SGP_OBS_ENABLED=true`` alone returns zero handles. All + three signals together need:: + + SGP_OBS_ENABLED=true + SGP_METRICS_DISABLED=false + SGP_TRACES_DISABLED=false + SGP_LOGS_DISABLED=false + + That inverts the advice written against 0.15.0, where traces came on with the + master switch and had to be turned off. This module does not second-guess the + gate — it calls ``init()`` and reports which signals came back — but it does + warn when the master switch is on and nothing wired, because that combination + is otherwise completely silent. + +Metrics additionally need an OTLP endpoint. sgp-obs never builds a MeterProvider +from nothing; in a cluster the OTel Operator's auto-instrumentation normally +supplies one, and agent pods get no injection, so ``OTEL_EXPORTER_OTLP_ENDPOINT`` +has to be on the pod spec. + +Fail-open is absolute: this is telemetry, and no failure here may stop an agent from +starting or serving. Every path returns a status string instead of raising. +""" + +from __future__ import annotations + +import os +from typing import Any + +from agentex.lib.utils.logging import ( + make_logger, + _reset_for_tests as _logging_reset_for_tests, + route_agentex_loggers_to_root, +) + +logger = make_logger(__name__) + +_status: str | None = None + +# sgp-obs' own truthy set (sgp_obs.env._TRUTHY), so "is the master switch on?" is +# answered the same way here as in the library deciding whether to wire. +_TRUTHY = {"1", "true", "yes", "on"} + +# The logs-profile selector. The SDK knows the runtime is agentex; an agent author +# would have to know to pass it. It stamps agent_id (from AGENT_ID) and task_id (from +# the SDK's streaming contextvar) onto every log record. +_SOURCE = "agentex" + + +def _master_switch_on() -> bool: + return (os.getenv("SGP_OBS_ENABLED") or "").strip().lower() in _TRUTHY + + +def init_sgp_obs(app: Any = None) -> str: + """Wire sgp-obs if it is installed and enabled. Returns a status; never raises. + + Statuses: ``"not_installed"``, ``"disabled"``, ``"wired:"``, ``"error"``. + + ``app`` is the ACP server. Passing it is what adds ``http.server.*`` for the + agent's own entry point — without it the agent is observable only from the + model call outwards, and its own latency and error rate cannot be alerted on. + It is also what installs the trace-context ingress middleware, so an incoming + ``traceparent`` continues into the agent's spans rather than starting a new trace. + """ + global _status + if _status is not None: + # init() is not meant to run twice, and a Temporal worker plus an ACP + # server can both reach this in one process. + return _status + + try: + # Not resolvable in a normal env: sgp-obs is not a dependency of this + # package and is not on public PyPI. That is the case this branch exists for. + import sgp_obs # type: ignore[import-not-found] + except ImportError: + if _master_switch_on(): + # The operator asked for observability and the package is absent. Silence + # here is the worst outcome, so say what is missing and how to fix it. + logger.warning( + "SGP_OBS_ENABLED is set but sgp-obs is not installed, so no telemetry " + "will be produced. Add sgp-obs[genai-auto,http,otlp] to this agent's " + "dependencies (it resolves from Scale's curated mirror, not public PyPI)." + ) + _status = "not_installed" + return _status + except Exception: # pragma: no cover - a broken install must not stop startup + logger.debug("sgp-obs import failed unexpectedly", exc_info=True) + _status = "error" + return _status + + try: + handles = sgp_obs.init( + app=app, + # Fills OTEL_SERVICE_NAME only when the deployment left it unset or + # blank; the deployment always outranks this. Without either, every + # signal is attributed to service.name="unknown". + service_name=(os.getenv("AGENT_NAME") or "").strip() or None, + source=_SOURCE, + ) + except Exception: # pragma: no cover - sgp_obs.init is itself fail-open + # One deliberate exception to its fail-open rule: under the standard CI + # variable, any logs misconfiguration raises so a build cannot pass while + # logging is broken. Swallowed here regardless — an agent must still serve. + logger.warning("sgp-obs initialization failed; continuing without it", exc_info=True) + _status = "error" + return _status + + if not handles: + if _master_switch_on(): + # 0.16.0's double opt-in: the master switch alone wires nothing, and + # sgp-obs says nothing about it. Name the variables that are missing. + logger.warning( + "SGP_OBS_ENABLED is set but no sgp-obs signal is enabled, so nothing " + "will be exported. Each signal is opt-in separately: set " + "SGP_METRICS_DISABLED=false, SGP_TRACES_DISABLED=false and " + "SGP_LOGS_DISABLED=false for the signals you want. An unset " + "*_DISABLED leaves that signal off." + ) + # Otherwise expected, and the default: an agent with sgp-obs installed still + # records nothing until someone sets the environment. + _status = "disabled" + return _status + + if "logs" in handles: + _hand_agentex_logging_to_the_pipeline() + + if "traces" in handles: + _install_openai_agents_bridge() + _warn_if_correlation_backend_mismatched() + + _status = "wired:" + ",".join(sorted(handles)) + logger.info("sgp-obs wired (%s)", _status) + return _status + + +def _hand_agentex_logging_to_the_pipeline() -> None: + """Stop agentex's own loggers printing a second, ungoverned copy of every record. + + ``agentex.lib.utils.logging.make_logger`` attaches a handler to each module's own + (leaf) logger. sgp-obs' logs pipeline replaces the handlers on the ROOT logger and + deliberately leaves named loggers alone, because a named logger's handler may be + there on purpose. The two are individually correct and together print everything + twice: once in agentex's plain-text format from the leaf, once as pipeline JSON + from root. Measured on sgp-obs 0.16.0, one ``logger.info()`` gave two stdout lines, + and sgp-obs' boot warning named 63 loggers. + + The duplicate is not merely redundant: it is emitted before the pipeline's filters, + so it carries no ``agent_id``/``task_id``, is not governed by the allowlist, and is + not truncated. + + Only agentex's loggers are handed over — see + :func:`~agentex.lib.utils.logging.route_agentex_loggers_to_root` for why by prefix, + why ``capture_loggers=`` is not the mechanism, and why a third party's handler is + left where it is. + """ + try: + cleared = route_agentex_loggers_to_root() + except Exception: # pragma: no cover - telemetry must never break startup + logger.debug("could not hand agentex logging to the sgp-obs pipeline", exc_info=True) + return + + if cleared: + # sgp-obs has already logged its "bypass log governance" warning by this point, + # naming loggers this call has just fixed. Say so, or the two lines read as a + # contradiction to whoever is looking at the pod's first second of output. + logger.info( + "routed %d agentex logger(s) through the sgp-obs logs pipeline; any " + "'bypass log governance' warning above that names agentex.* loggers was " + "emitted before this ran and no longer applies to them", + cleared, + ) + + +def _install_openai_agents_bridge() -> bool: + """Register sgp-obs' openai-agents trace processor, so a ``Runner`` turn produces + logical model-operation spans. + + This is the one piece of traces wiring ``sgp_obs.init()`` does NOT do for itself. + Measured on 0.16.0 after a plain ``init()`` with the traces signal on: + + GenAI attempt span processor installed + litellm logical adapter installed + httpx / aiohttp egress instrumented + openai-agents bridge NOT installed + + which is why the obs-test agents each carry a hand-written bootstrap that calls it. + It matters more than the others here: roughly 83% of model-calling agents reach the + model through the openai-agents ``Runner``, so without this the dominant path + contributes no logical spans and "traces on" looks like it does nothing. + + Unconditional because ``openai-agents`` is a hard dependency of this SDK, so the + ``agents`` package is importable in every agent. The call is idempotent and returns + False rather than raising when the SDK is somehow absent. + """ + try: + from sgp_obs.traces import install_openai_agents_bridge # type: ignore[import-not-found] + + installed = bool(install_openai_agents_bridge()) + if installed: + logger.debug("sgp-obs openai-agents bridge installed") + _warn_if_openai_agents_tracing_disabled() + else: + # Only reachable if `agents` is not importable, which should not happen + # while openai-agents is a hard dependency — so say so rather than shrug. + logger.warning( + "sgp-obs openai-agents bridge did not install; Runner turns will " + "produce no logical model-operation spans." + ) + return installed + except Exception: # pragma: no cover - telemetry must never break startup + logger.debug("sgp-obs openai-agents bridge unavailable", exc_info=True) + return False + + +def _warn_if_openai_agents_tracing_disabled() -> None: + """Warn when the bridge is installed but openai-agents tracing is switched off. + + ``install_openai_agents_bridge()`` returns True as soon as it registers itself as a + trace processor — it cannot tell whether the provider will ever feed it. If the + agent called ``set_tracing_disabled(True)``, no spans are produced at all, so the + bridge is registered and permanently idle, and nothing says so. + + That is not hypothetical: it is what the openai-agents scaffolds used to do, so + agents generated before this change carry it. Those scaffolds now clear the + processor list instead, which removes the OpenAI exporter (the thing they were + actually trying to avoid) while leaving spans flowing to the bridge. + + Reads a private attribute, so it is fully guarded: a diagnostic must never be the + reason startup fails, and if upstream renames it we simply stop warning. + """ + try: + from agents.tracing import get_trace_provider + + if getattr(get_trace_provider(), "_disabled", False): + logger.warning( + "The sgp-obs openai-agents bridge is installed but openai-agents " + "tracing is disabled, so Runner turns will produce no model spans. " + "Replace set_tracing_disabled(True) with set_trace_processors([]): " + "that still stops traces reaching api.openai.com, but keeps spans " + "flowing to the bridge." + ) + except Exception: # pragma: no cover - a diagnostic must never break startup + logger.debug("could not determine openai-agents tracing state", exc_info=True) + + +def _warn_if_correlation_backend_mismatched() -> None: + """Warn when sgp-obs is exporting OTel traces but the SDK's business-span + correlation is still reading ddtrace. + + The SDK has had its own correlation for a while (core/tracing/obs_span.py). It + writes BOTH directions of the link between a business span and an obs span: + + forward — obs_trace_id / obs_span_id onto the business span's data, so the + SGP tracing UI can pivot to Tempo + backward — agentex.business_span_id / agentex.business_trace_id onto the OTel + span, so Tempo can pivot back + + Which backend it opens that span in is chosen by SGP_OBS_MODE, which defaults to + ``dd_only``. In that mode it opens a ddtrace span, and only if a ddtrace trace is + already active — which on a bare-uvicorn agent it never is. So the wrapper is + never opened, the correlation dict comes back empty, and BOTH edges vanish + silently while the traces signal still reports itself as wired. + + Measured on sgp-obs 0.16.0 with a real business span: mode unset gives zero + exported spans and no ids in either direction; SGP_OBS_MODE=lgtm gives the + span, both tags, and a round trip that closes (the business span's obs_span_id + equals the exported span's span id, and the span's agentex.business_span_id + equals the business span's id). + + Warn rather than set it: SGP_OBS_MODE also steers correlation reads elsewhere, + and an agent genuinely running ddtrace (the Centipede family) would be misread + if this flipped underneath it. The operator picks; this only makes the silent + case audible. + """ + try: + from agentex.lib.core.tracing.obs_ids import LGTM, get_obs_mode + + if get_obs_mode() != LGTM: + logger.warning( + "sgp-obs wired the traces signal (OpenTelemetry), but SGP_OBS_MODE is " + "%r, so this SDK's business-span correlation still targets ddtrace and " + "will not link anything. Set SGP_OBS_MODE=lgtm to get both edges: " + "obs_trace_id/obs_span_id on the business span, and " + "agentex.business_span_id/agentex.business_trace_id on the OTel span.", + get_obs_mode(), + ) + except Exception: # pragma: no cover - a diagnostic must never break startup + logger.debug("could not check SGP_OBS_MODE", exc_info=True) + + +async def shutdown_sgp_obs() -> None: + """Flush the providers ``init()`` built. Never raises. + + Without this, whatever is sitting in a periodic exporter's buffer when the pod + stops is dropped — which for a short-lived or scaled-to-zero agent can be most + of what it recorded. sgp-obs only flushes providers it OWNS; one adopted from + the runtime is left to its owner, so this is safe under operator injection. + + Run in a thread: the flush blocks up to the SDK export timeout per owned signal, + and this is called from an async lifespan. + """ + if _status is None or not _status.startswith("wired"): + return + + try: + import asyncio + + import sgp_obs # type: ignore[import-not-found] + + # Added in sgp-obs 0.16.0. Feature-detected rather than version-pinned, + # because this package does not depend on sgp-obs and so cannot set a floor. + shutdown = getattr(sgp_obs, "shutdown", None) + if shutdown is None: + logger.debug("sgp-obs has no shutdown(); needs 0.16.0+ to flush on exit") + return + await asyncio.to_thread(shutdown) + except Exception: # pragma: no cover - a failed flush must not fail shutdown + logger.debug("sgp-obs shutdown failed", exc_info=True) + + +def _reset_for_tests() -> None: + global _status + _status = None + # The logging hand-over is a process-wide latch too, and a test that wired the + # logs signal would otherwise leave make_logger attaching nothing for the rest + # of the session. + _logging_reset_for_tests() diff --git a/src/agentex/lib/core/observability/tests/test_sgp_obs_setup.py b/src/agentex/lib/core/observability/tests/test_sgp_obs_setup.py new file mode 100644 index 000000000..34d2ef3ad --- /dev/null +++ b/src/agentex/lib/core/observability/tests/test_sgp_obs_setup.py @@ -0,0 +1,447 @@ +"""Tests for ``agentex.lib.core.observability.sgp_obs_setup``. + +The property under test is that this can never hurt a caller: whatever the state of +sgp-obs or the environment, ``init_sgp_obs`` returns a status string and does not +raise, and ``shutdown_sgp_obs`` does not raise. Both gates get a test, plus the +failure modes, the two silent-misconfiguration warnings, and the flush. + +These never import the real sgp-obs — it is absent in CI by design — so every test +installs a stand-in whose ``init`` is under the test's control. +""" + +from __future__ import annotations + +import sys +import builtins +from contextlib import contextmanager + +import pytest + +from agentex.lib.core.observability import sgp_obs_setup +from agentex.lib.core.observability.sgp_obs_setup import init_sgp_obs, shutdown_sgp_obs + +_SWITCHES = ( + "SGP_OBS_ENABLED", + "SGP_METRICS_DISABLED", + "SGP_TRACES_DISABLED", + "SGP_LOGS_DISABLED", + "AGENT_NAME", +) + + +@pytest.fixture(autouse=True) +def _reset(monkeypatch): + """The status is cached process-wide, so every test starts from unset. The + environment is cleared too: two code paths branch on the master switch, and a + developer with SGP_OBS_ENABLED exported would otherwise flip those tests.""" + for name in _SWITCHES: + monkeypatch.delenv(name, raising=False) + sgp_obs_setup._reset_for_tests() + yield + sgp_obs_setup._reset_for_tests() + + +@contextmanager +def caplog_at(monkeypatch): + """Collect sgp_obs_setup's WARNING messages regardless of root config.""" + records: list[str] = [] + monkeypatch.setattr( + sgp_obs_setup.logger, "warning", + lambda msg, *a, **_k: records.append(msg % a if a else msg), + ) + yield records + + +def _fake_sgp_obs(monkeypatch, init=None, shutdown=None, bridge=None): + """Install a stand-in ``sgp_obs`` module whose entry points we control. + + ``bridge`` stands in for ``sgp_obs.traces.install_openai_agents_bridge``; it lives + on a fake ``sgp_obs.traces`` submodule because that is how the SDK imports it. + """ + module = type(sys)("sgp_obs") + module.init = init if init is not None else (lambda **_kwargs: {"metrics": object()}) + if shutdown is not None: + module.shutdown = shutdown + monkeypatch.setitem(sys.modules, "sgp_obs", module) + + traces = type(sys)("sgp_obs.traces") + traces.install_openai_agents_bridge = bridge if bridge is not None else (lambda: True) + monkeypatch.setitem(sys.modules, "sgp_obs.traces", traces) + return module + + +def _block_sgp_obs_import(monkeypatch, exc=None): + monkeypatch.delitem(sys.modules, "sgp_obs", raising=False) + real_import = builtins.__import__ + error = exc or ImportError("No module named 'sgp_obs'") + + def blocked(name, *args, **kwargs): + if name == "sgp_obs" or name.startswith("sgp_obs."): + raise error + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", blocked) + + +class TestGateOneSgpObsNotInstalled: + def test_missing_package_is_reported_not_raised(self, monkeypatch): + _block_sgp_obs_import(monkeypatch) + assert init_sgp_obs() == "not_installed" + + def test_a_broken_install_does_not_stop_startup(self, monkeypatch): + """An ImportError is ordinary; anything else is a broken install, not a + missing one, and must still be swallowed.""" + _block_sgp_obs_import(monkeypatch, RuntimeError("half-installed wheel")) + assert init_sgp_obs() == "error" + + def test_silence_is_expected_when_nobody_asked(self, monkeypatch, caplog): + """sgp-obs is not a dependency, so absent-and-unasked-for is the normal + case for every agent. It must not warn.""" + _block_sgp_obs_import(monkeypatch) + with caplog.at_level("WARNING"): + assert init_sgp_obs() == "not_installed" + assert caplog.records == [] + + def test_enabled_but_missing_says_what_to_install(self, monkeypatch, caplog): + """The one case that must be loud: the operator asked for observability and + the package is not there. Silence would look like working instrumentation.""" + monkeypatch.setenv("SGP_OBS_ENABLED", "true") + _block_sgp_obs_import(monkeypatch) + with caplog.at_level("WARNING"): + assert init_sgp_obs() == "not_installed" + assert len(caplog.records) == 1 + assert "sgp-obs is not installed" in caplog.text + assert "genai-auto,http,otlp" in caplog.text + + +class TestGateTwoEnvironmentSwitches: + def test_no_handles_means_disabled(self, monkeypatch): + """sgp_obs.init() returns an empty dict when the master switch or every + per-signal switch is off. That is the DEFAULT: sgp-obs installed, and + recording nothing until someone sets the environment.""" + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {}) + assert init_sgp_obs() == "disabled" + + def test_disabled_and_unasked_for_is_quiet(self, monkeypatch, caplog): + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {}) + with caplog.at_level("WARNING"): + assert init_sgp_obs() == "disabled" + assert caplog.records == [] + + def test_master_switch_on_but_nothing_wired_names_the_variables( + self, monkeypatch, caplog + ): + """sgp-obs 0.16.0 made every signal opt-in twice: the master switch plus an + explicit *_DISABLED=false. So SGP_OBS_ENABLED on its own wires nothing and + says nothing, which is the single easiest way to believe an agent is + instrumented when it is not.""" + monkeypatch.setenv("SGP_OBS_ENABLED", "true") + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {}) + with caplog.at_level("WARNING"): + assert init_sgp_obs() == "disabled" + assert len(caplog.records) == 1 + for var in ("SGP_METRICS_DISABLED", "SGP_TRACES_DISABLED", "SGP_LOGS_DISABLED"): + assert var in caplog.text + + @pytest.mark.parametrize("raw", ["1", "true", "TRUE", "yes", "on"]) + def test_master_switch_truthy_forms(self, monkeypatch, caplog, raw): + """Matched to sgp_obs.env._TRUTHY, so this module's idea of "on" is the + same as the library's. A mismatch would put the warning on the wrong side.""" + monkeypatch.setenv("SGP_OBS_ENABLED", raw) + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {}) + with caplog.at_level("WARNING"): + init_sgp_obs() + assert len(caplog.records) == 1 + + def test_traces_without_lgtm_mode_warns_that_correlation_is_dead( + self, monkeypatch + ): + """SGP_OBS_MODE defaults to dd_only, where the SDK's business-span wrapper + only opens if a ddtrace trace is already active — never true on a + bare-uvicorn agent. So both correlation edges vanish while the traces + signal still reports itself wired. Measured: mode unset -> zero exported + spans and no ids either way; lgtm -> both edges, round trip closes.""" + monkeypatch.delenv("SGP_OBS_MODE", raising=False) + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {"traces": object()}) + with caplog_at(monkeypatch) as records: + assert init_sgp_obs() == "wired:traces" + assert any("SGP_OBS_MODE" in r for r in records) + + def test_traces_with_lgtm_mode_is_quiet(self, monkeypatch, caplog): + monkeypatch.setenv("SGP_OBS_MODE", "lgtm") + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {"traces": object()}) + with caplog.at_level("WARNING"): + assert init_sgp_obs() == "wired:traces" + assert caplog.records == [] + + def test_metrics_only_does_not_warn_about_the_mode(self, monkeypatch, caplog): + """The correlation edges are a traces concern. A metrics-only agent has no + business-span linking to lose, so the warning would be noise.""" + monkeypatch.delenv("SGP_OBS_MODE", raising=False) + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {"metrics": object()}) + with caplog.at_level("WARNING"): + assert init_sgp_obs() == "wired:metrics" + assert caplog.records == [] + + def test_all_three_signals_are_named_in_the_status(self, monkeypatch): + _fake_sgp_obs( + monkeypatch, + lambda **_kwargs: {"logs": object(), "metrics": object(), "traces": object()}, + ) + assert init_sgp_obs() == "wired:logs,metrics,traces" + + +class TestWhatIsPassedToSgpObs: + @staticmethod + def _capture(monkeypatch): + seen = {} + + def capture(**kwargs): + seen.update(kwargs) + return {"metrics": object()} + + _fake_sgp_obs(monkeypatch, capture) + return seen + + def test_app_reaches_sgp_obs(self, monkeypatch): + """Passing the ACP server is what adds http.server.* for the agent's own + entry point and installs the trace-context ingress, so it must not be + silently dropped.""" + seen = self._capture(monkeypatch) + sentinel = object() + init_sgp_obs(app=sentinel) + assert seen["app"] is sentinel + + def test_source_is_agentex(self, monkeypatch): + """The SDK knows the runtime; an agent author would have to know to pass it. + It is what stamps agent_id and task_id onto log records.""" + seen = self._capture(monkeypatch) + init_sgp_obs() + assert seen["source"] == "agentex" + + def test_agent_name_is_offered_as_the_service_name(self, monkeypatch): + """sgp-obs fills OTEL_SERVICE_NAME from this only when the deployment left + it unset; without either, every signal is attributed to "unknown".""" + monkeypatch.setenv("AGENT_NAME", "compass-sleep-agent") + seen = self._capture(monkeypatch) + init_sgp_obs() + assert seen["service_name"] == "compass-sleep-agent" + + @pytest.mark.parametrize("raw", ["", " "]) + def test_blank_agent_name_is_passed_as_none(self, monkeypatch, raw): + """Blank is the Helm rendered-empty idiom. Forwarding "" would have sgp-obs + set OTEL_SERVICE_NAME to an empty string rather than leave it alone.""" + monkeypatch.setenv("AGENT_NAME", raw) + seen = self._capture(monkeypatch) + init_sgp_obs() + assert seen["service_name"] is None + + +class TestFailOpen: + def test_an_exception_from_init_is_swallowed(self, monkeypatch): + def boom(**_kwargs): + raise ValueError("boom") + + _fake_sgp_obs(monkeypatch, boom) + assert init_sgp_obs() == "error" + + def test_a_ci_logs_misconfiguration_still_does_not_stop_startup(self, monkeypatch): + """sgp_obs.init has one deliberate exception to its own fail-open rule: under + the CI variable, a logs misconfiguration raises. An agent must still serve.""" + + def strict(**_kwargs): + raise RuntimeError("MisconfigurationError: drop mode without an allowlist") + + _fake_sgp_obs(monkeypatch, strict) + assert init_sgp_obs() == "error" + + def test_status_is_computed_once(self, monkeypatch): + """A Temporal worker and an ACP server can both reach this in one process; + sgp_obs.init() is not meant to run twice.""" + calls = [] + + def counting(**kwargs): + calls.append(kwargs) + return {"metrics": object()} + + _fake_sgp_obs(monkeypatch, counting) + assert init_sgp_obs() == "wired:metrics" + assert init_sgp_obs() == "wired:metrics" + assert len(calls) == 1 + + +class TestShutdown: + async def test_flushes_when_wired(self, monkeypatch): + """Without this the periodic exporter's buffer is dropped when the pod + stops, which for a short-lived agent can be most of what it recorded.""" + called = [] + _fake_sgp_obs(monkeypatch, shutdown=lambda: called.append(True)) + assert init_sgp_obs() == "wired:metrics" + await shutdown_sgp_obs() + assert called == [True] + + async def test_no_flush_when_never_wired(self, monkeypatch): + called = [] + _fake_sgp_obs( + monkeypatch, init=lambda **_kwargs: {}, shutdown=lambda: called.append(True) + ) + assert init_sgp_obs() == "disabled" + await shutdown_sgp_obs() + assert called == [] + + async def test_no_flush_before_init(self, monkeypatch): + """Called from the lifespan's finally, which runs even if startup failed + before the constructor's init_sgp_obs ever ran.""" + called = [] + _fake_sgp_obs(monkeypatch, shutdown=lambda: called.append(True)) + await shutdown_sgp_obs() + assert called == [] + + async def test_an_older_sgp_obs_without_shutdown_is_tolerated(self, monkeypatch): + """shutdown() arrived in 0.16.0. This package declares no dependency on + sgp-obs and so cannot set a floor, hence feature detection.""" + _fake_sgp_obs(monkeypatch) # no shutdown attribute + assert init_sgp_obs() == "wired:metrics" + await shutdown_sgp_obs() # must not raise + + async def test_a_failing_flush_does_not_fail_shutdown(self, monkeypatch): + def boom(): + raise RuntimeError("exporter timed out") + + _fake_sgp_obs(monkeypatch, shutdown=boom) + assert init_sgp_obs() == "wired:metrics" + await shutdown_sgp_obs() # must not raise + + +class TestAnAgentStillServesWithoutSgpObs: + """Nitesh's verification item, startup half: an account not yet on the + CodeArtifact allowlist gets an image with no ``sgp_obs`` in it. The gate + returning ``not_installed`` is necessary but not sufficient — what has to hold + is that the ACP server still constructs and still answers requests. This + exercises the real constructor, which is where ``init_sgp_obs`` is called. + """ + + def test_acp_server_constructs_and_serves_healthz(self, monkeypatch): + from fastapi.testclient import TestClient + + from agentex.lib.sdk.fastacp.base.base_acp_server import BaseACPServer + + # Import first, unpatched, so the deep FastACP dependency chain loads + # cleanly; only sgp_obs is hidden, and only while the constructor runs. + _block_sgp_obs_import(monkeypatch) + + server = BaseACPServer() + assert sgp_obs_setup._status == "not_installed" + + # No `with`: that would run the lifespan, which registers the agent + # against a live control plane. + response = TestClient(server).get("/healthz") + assert response.status_code == 200 + assert response.json() == {"status": "healthy"} + + def test_the_json_rpc_route_is_still_mounted(self, monkeypatch): + """A server that answers /healthz but lost /api would pass a liveness probe + and fail every actual request.""" + from agentex.lib.sdk.fastacp.base.base_acp_server import BaseACPServer + + _block_sgp_obs_import(monkeypatch) + routes = {getattr(r, "path", None) for r in BaseACPServer().routes} + assert {"/healthz", "/api"} <= routes + + +class TestOpenAIAgentsBridge: + """sgp_obs.init() installs the GenAI attempt processor, the litellm adapter and the + egress instrumentors by itself, but NOT the openai-agents bridge (measured on + 0.16.0). That is the path ~83% of model-calling agents take, so the SDK installs it + — otherwise "traces on" produces no logical model-operation spans for most agents. + """ + + def test_installed_when_traces_are_wired(self, monkeypatch): + calls = [] + _fake_sgp_obs( + monkeypatch, + init=lambda **_kwargs: {"traces": object()}, + bridge=lambda: calls.append(True) or True, + ) + monkeypatch.setenv("SGP_OBS_MODE", "lgtm") + assert init_sgp_obs() == "wired:traces" + assert calls == [True] + + def test_not_installed_without_the_traces_signal(self, monkeypatch): + """A metrics-only agent has no span pipeline to feed, so installing an + openai-agents trace processor would be pointless work at startup.""" + calls = [] + _fake_sgp_obs( + monkeypatch, + init=lambda **_kwargs: {"metrics": object()}, + bridge=lambda: calls.append(True) or True, + ) + assert init_sgp_obs() == "wired:metrics" + assert calls == [] + + def test_a_bridge_that_declines_is_reported(self, monkeypatch, caplog): + """False means the `agents` SDK was not importable. openai-agents is a hard + dependency of this package, so that should be impossible — say so rather than + swallow it.""" + monkeypatch.setenv("SGP_OBS_MODE", "lgtm") + _fake_sgp_obs( + monkeypatch, init=lambda **_kwargs: {"traces": object()}, bridge=lambda: False + ) + with caplog.at_level("WARNING"): + assert init_sgp_obs() == "wired:traces" + assert "openai-agents bridge" in caplog.text + + def test_a_raising_bridge_does_not_stop_startup(self, monkeypatch): + def boom(): + raise RuntimeError("sgp-obs internals moved") + + monkeypatch.setenv("SGP_OBS_MODE", "lgtm") + _fake_sgp_obs( + monkeypatch, init=lambda **_kwargs: {"traces": object()}, bridge=boom + ) + assert init_sgp_obs() == "wired:traces" + + +class TestLoggingHandover: + """agentex's make_logger attaches a handler to each module's own logger; sgp-obs' + logs pipeline owns the ROOT logger and deliberately leaves named loggers alone. Both + then print, so every record appears twice — and the agentex copy is emitted before + the pipeline's filters, so it carries no agent_id/task_id, is not governed by the + allowlist, and is not truncated. + """ + + @staticmethod + def _spy(monkeypatch): + calls = [] + monkeypatch.setattr( + sgp_obs_setup, "route_agentex_loggers_to_root", lambda: calls.append(True) or 1 + ) + return calls + + def test_handover_runs_when_the_logs_signal_is_wired(self, monkeypatch): + calls = self._spy(monkeypatch) + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {"logs": object()}) + assert init_sgp_obs() == "wired:logs" + assert calls == [True] + + def test_no_handover_when_logs_are_not_wired(self, monkeypatch): + """Nothing owns the root logger in that case, so stripping the leaf handlers + would send agentex's records nowhere at all.""" + calls = self._spy(monkeypatch) + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {"metrics": object()}) + assert init_sgp_obs() == "wired:metrics" + assert calls == [] + + def test_no_handover_when_sgp_obs_is_absent(self, monkeypatch): + calls = self._spy(monkeypatch) + _block_sgp_obs_import(monkeypatch) + assert init_sgp_obs() == "not_installed" + assert calls == [] + + def test_a_failing_handover_does_not_stop_startup(self, monkeypatch): + def boom(): + raise RuntimeError("logging registry is in a strange state") + + monkeypatch.setattr(sgp_obs_setup, "route_agentex_loggers_to_root", boom) + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {"logs": object()}) + assert init_sgp_obs() == "wired:logs" diff --git a/src/agentex/lib/core/temporal/workers/worker.py b/src/agentex/lib/core/temporal/workers/worker.py index 72631917f..651286fbe 100644 --- a/src/agentex/lib/core/temporal/workers/worker.py +++ b/src/agentex/lib/core/temporal/workers/worker.py @@ -32,6 +32,8 @@ from agentex.lib.core.tracing.temporal import temporal_tracing_interceptors from agentex.lib.environment_variables import EnvironmentVariables from agentex.lib.core.compat.version_guard import assert_backend_compatible +from agentex.lib.core.observability.sgp_obs_setup import init_sgp_obs, shutdown_sgp_obs +from agentex.lib.core.tracing.tracing_processor_manager import shutdown_sync_tracing_processors logger = make_logger(__name__) @@ -222,6 +224,18 @@ async def run( workflow: type | None = None, workflows: list[type] | None = None, ): + # A Temporal agent runs its model calls HERE, in a separate process from the + # ACP server, and this process never constructs a BaseACPServer — so without + # this call an agent that installed sgp-obs and set the documented environment + # would still get no metrics, traces or structured logs from its worker, which + # is where the interesting work happens. + # + # No `app=`: there is no ASGI application in this process. The health-check + # server is aiohttp, which sgp-obs' ASGI middleware does not apply to, so the + # worker contributes model and egress telemetry but no http.server.* — correct, + # since nothing here serves agent traffic. + init_sgp_obs() + await self.start_health_check_server() await self._register_agent() @@ -267,7 +281,14 @@ async def run( # Eagerly set the worker status to healthy self.healthy = True logger.info(f"Running workers for task queue: {self.task_queue}") - await worker.run() + try: + await worker.run() + finally: + # Same drains as the ACP lifespan, for the same reason: whatever is still + # queued when the pod stops is otherwise dropped. Both are bounded and + # fail-open, so neither can stop the worker exiting. + await shutdown_sync_tracing_processors() + await shutdown_sgp_obs() async def _health_check(self): return web.json_response(self.healthy) diff --git a/src/agentex/lib/core/tracing/tracing_processor_manager.py b/src/agentex/lib/core/tracing/tracing_processor_manager.py index 07c440313..5227e891c 100644 --- a/src/agentex/lib/core/tracing/tracing_processor_manager.py +++ b/src/agentex/lib/core/tracing/tracing_processor_manager.py @@ -1,5 +1,8 @@ from __future__ import annotations +import asyncio +import logging +import threading from typing import TYPE_CHECKING from threading import Lock @@ -78,3 +81,104 @@ def get_sync_tracing_processors(): def get_async_tracing_processors(): return GLOBAL_TRACING_PROCESSOR_MANAGER.get_async_processors() + + +_logger = logging.getLogger(__name__) + +# Total wall-clock budget for draining every sync tracing processor. A pod's +# terminationGracePeriodSeconds (30s by default) is shared with the OTel flush that +# follows this, so the drain takes a small slice of it. +SYNC_TRACING_SHUTDOWN_BUDGET_S = 5.0 + + +async def shutdown_sync_tracing_processors( + budget_s: float = SYNC_TRACING_SHUTDOWN_BUDGET_S, +) -> None: + """Drain the sync tracing processors' queues at shutdown. Never raises. + + Nothing used to call this. The ACP lifespan drained ``shutdown_default_span_queue``, + which is the ASYNC path only, so a sync agent dropped whatever business spans were + still queued when the pod stopped. That matters beyond the lost spans: the business + span is what an obs span's ``agentex.business_trace_id`` resolves to, so losing it + breaks the pivot from Tempo back to the SGP store. + + ``SGPSyncTracingProcessor.shutdown`` calls ``flush_queue()``, a BLOCKING HTTP flush + with retries, so three properties have to hold at once: + + **Off the calling loop.** Awaiting it inline stalls the lifespan, so a slow + collector could burn the pod's whole termination grace period and stop the OTel + flush that runs after this — trading a few business spans for all of the OTel ones. + + **Concurrent.** Every processor is started at once and they share one deadline. A + sequential loop would let the first stalled processor spend the entire budget, so + later processors were skipped even when they would have finished instantly. + + **On DAEMON threads, not the default executor.** This is the subtle one. + ``asyncio.wait_for`` stops *awaiting* a thread; it cannot stop the thread. And + ``asyncio.run`` calls ``loop.shutdown_default_executor()``, which JOINS the default + executor — as does a private ``ThreadPoolExecutor``, via its atexit hook. So a + timed-out ``asyncio.to_thread`` flush leaves the process blocked on the very export + the deadline was meant to escape. Measured: a 10s stalled flush under a 0.25s budget + returns in 0.25s but the process exits at 10.0s with ``to_thread``, and at 0.25s on + a daemon thread. A daemon thread is abandoned at interpreter exit, which is what the + budget promises. + """ + try: + processors = get_sync_tracing_processors() + except Exception: # pragma: no cover - nothing to drain + _logger.debug("sync tracing processors unavailable at shutdown", exc_info=True) + return + + if not processors: + return + + loop = asyncio.get_running_loop() + finished: list[threading.Event] = [] + all_done = asyncio.Event() + + def _note_finished() -> None: + if all(event.is_set() for event in finished): + all_done.set() + + def _flush(processor: SyncTracingProcessor, event: threading.Event) -> None: + try: + processor.shutdown() + except Exception: + _logger.warning( + "%s raised while flushing on shutdown; some business spans may be lost", + type(processor).__name__, + exc_info=True, + ) + finally: + event.set() + # The loop may already be closed if we timed out and shutdown raced ahead; + # abandoning the notification is fine, nobody is waiting on it any more. + try: + loop.call_soon_threadsafe(_note_finished) + except RuntimeError: # pragma: no cover - loop already closed + pass + + for index, processor in enumerate(processors): + event = threading.Event() + finished.append(event) + threading.Thread( + target=_flush, + args=(processor, event), + daemon=True, + name=f"agentex-span-flush-{index}", + ).start() + + try: + await asyncio.wait_for(all_done.wait(), budget_s) + except (TimeoutError, asyncio.TimeoutError): + stalled = [ + type(processor).__name__ + for processor, event in zip(processors, finished) + if not event.is_set() + ] + _logger.warning( + "sync tracing shutdown budget of %.1fs expired with %s still flushing; " + "their business spans are lost, but shutdown continues", + budget_s, + ", ".join(stalled) or "unknown processors", + ) diff --git a/src/agentex/lib/sdk/fastacp/base/base_acp_server.py b/src/agentex/lib/sdk/fastacp/base/base_acp_server.py index 864b466d0..06bc2595c 100644 --- a/src/agentex/lib/sdk/fastacp/base/base_acp_server.py +++ b/src/agentex/lib/sdk/fastacp/base/base_acp_server.py @@ -39,6 +39,8 @@ FASTACP_HEADER_SKIP_EXACT, FASTACP_HEADER_SKIP_PREFIXES, ) +from agentex.lib.core.observability.sgp_obs_setup import init_sgp_obs, shutdown_sgp_obs +from agentex.lib.core.tracing.tracing_processor_manager import shutdown_sync_tracing_processors logger = make_logger(__name__) @@ -139,6 +141,20 @@ def __init__(self): # Method handlers # this just adds a request ID to the request and response headers self.add_middleware(RequestIDMiddleware) + + # Optional observability (traces, metrics, logs), off unless sgp-obs is + # installed AND the SGP_OBS_* environment switches ask for it — see + # observability/sgp_obs_setup.py for the two gates. sgp-obs is deliberately + # not a dependency of this package; the agent declares it. Returns a status + # instead of raising: a telemetry problem must never stop an agent starting. + # + # Here rather than in the lifespan, deliberately: sgp-obs installs ASGI + # instrumentation via add_middleware, and Starlette raises "Cannot add middleware + # after an application has started" once the lifespan is running. Wiring it there + # loses http.server.* for the agent's own entry point — and loses it QUIETLY, + # because sgp-obs fails open. + init_sgp_obs(app=self) + self._handlers: dict[RPCMethod, Callable] = {} # Agent info to return in healthz @@ -176,9 +192,20 @@ async def lifespan_context(app: FastAPI): # noqa: ARG001 yield finally: await shutdown_default_span_queue() + # The queue above is the ASYNC path only. Sync tracing processors + # hold their own queue and nothing ever drained it, so a sync ACP + # agent lost whatever business spans were still queued when the pod + # stopped — including the ones the obs correlation points at. + await shutdown_sync_tracing_processors() + # Flush whatever sgp-obs still holds. A periodic exporter's buffer + # is otherwise dropped when the pod stops, which for a short-lived + # or scaled-to-zero agent can be most of what it recorded. No-op + # when sgp-obs is absent or was never wired. + await shutdown_sgp_obs() return lifespan_context + async def _healthz(self): """Health check endpoint""" result = {"status": "healthy"} diff --git a/src/agentex/lib/sdk/fastacp/base/tests/__init__.py b/src/agentex/lib/sdk/fastacp/base/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/agentex/lib/sdk/fastacp/base/tests/test_shutdown_hooks.py b/src/agentex/lib/sdk/fastacp/base/tests/test_shutdown_hooks.py new file mode 100644 index 000000000..fe9cfbe28 --- /dev/null +++ b/src/agentex/lib/sdk/fastacp/base/tests/test_shutdown_hooks.py @@ -0,0 +1,302 @@ +"""Tests for the ACP lifespan's shutdown drains. + +``shutdown_default_span_queue`` covers the async span path. The SYNC tracing +processors keep their own queue, and nothing in the SDK ever shut them down, so a +sync ACP agent dropped whatever business spans were still queued when the pod +stopped. That is worse than the spans themselves: the business span is what an obs +span's ``agentex.business_trace_id`` resolves to, so losing it breaks the pivot from +Tempo back to the SGP store. +""" + +from __future__ import annotations + +from agentex.lib.sdk.fastacp.base import base_acp_server +from agentex.lib.core.tracing.tracing_processor_manager import ( + shutdown_sync_tracing_processors, +) + + +def _block_sgp_obs_import(monkeypatch): + """Make `import sgp_obs` fail, i.e. the image a tokenless build produces.""" + import sys + import builtins + + monkeypatch.delitem(sys.modules, "sgp_obs", raising=False) + real_import = builtins.__import__ + + def blocked(name, *args, **kwargs): + if name == "sgp_obs" or name.startswith("sgp_obs."): + raise ImportError("No module named 'sgp_obs'") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", blocked) + + +class _Processor: + def __init__(self, explode: bool = False) -> None: + self.calls = 0 + self._explode = explode + + def shutdown(self) -> None: + self.calls += 1 + if self._explode: + raise RuntimeError("flush timed out") + + +def _patch_processors(monkeypatch, processors): + import agentex.lib.core.tracing.tracing_processor_manager as mgr + + monkeypatch.setattr(mgr, "get_sync_tracing_processors", lambda: processors) + + +class TestSyncProcessorDrain: + async def test_every_processor_is_flushed(self, monkeypatch): + a, b = _Processor(), _Processor() + _patch_processors(monkeypatch, [a, b]) + await shutdown_sync_tracing_processors() + assert (a.calls, b.calls) == (1, 1) + + async def test_one_failure_does_not_stop_the_others(self, monkeypatch): + """A processor that hangs or raises must not strand the spans held by the + ones after it in the list.""" + bad, good = _Processor(explode=True), _Processor() + _patch_processors(monkeypatch, [bad, good]) + await shutdown_sync_tracing_processors() + assert good.calls == 1 + + async def test_no_processors_is_a_no_op(self, monkeypatch): + _patch_processors(monkeypatch, []) + await shutdown_sync_tracing_processors() # must not raise + + async def test_an_unreadable_processor_list_does_not_fail_shutdown(self, monkeypatch): + """Nothing here may stop the pod from shutting down. + + This used to block the import of ``tracing_processor_manager``, which tested + nothing once the drain moved INTO that module: it reads + ``get_sync_tracing_processors`` as a module global, so the import never runs and + the ``except`` branch was never reached. Make the lookup itself raise instead.""" + import agentex.lib.core.tracing.tracing_processor_manager as mgr + + def boom(): + raise RuntimeError("processor registry unavailable") + + monkeypatch.setattr(mgr, "get_sync_tracing_processors", boom) + await shutdown_sync_tracing_processors() # must not raise + + def test_the_lifespan_calls_it(self): + """Pin the wiring, not just the helper: a drain nothing calls is worthless.""" + import inspect + + source = inspect.getsource(base_acp_server.BaseACPServer.get_lifespan_function) + assert "shutdown_sync_tracing_processors()" in source + assert "shutdown_sgp_obs()" in source + + +class TestTheDrainIsBounded: + """`SGPSyncTracingProcessor.shutdown` does a BLOCKING HTTP flush with retries. If + the drain waited on it inline and without a limit, a slow or unreachable collector + would burn the pod's whole termination grace period and the OTel flush that runs + after it would never happen — trading a few business spans for all of the OTel ones. + """ + + async def test_a_stalled_processor_does_not_hang_shutdown(self, monkeypatch): + import time + import asyncio + + class Stalled: + def shutdown(self): + time.sleep(2) # blocking, like a retrying HTTP flush + + _patch_processors(monkeypatch, [Stalled()]) + started = asyncio.get_running_loop().time() + await shutdown_sync_tracing_processors(budget_s=0.25) + elapsed = asyncio.get_running_loop().time() - started + assert elapsed < 1, f"drain took {elapsed:.1f}s against a 0.25s budget" + + async def test_the_budget_is_shared_so_a_stall_cannot_starve_the_rest(self, monkeypatch): + """A shared deadline means the drain as a whole is bounded, not each processor + separately — N stalled processors must not cost N * budget.""" + import time + import asyncio + + class Stalled: + def shutdown(self): + time.sleep(2) + + _patch_processors(monkeypatch, [Stalled(), Stalled(), Stalled()]) + started = asyncio.get_running_loop().time() + await shutdown_sync_tracing_processors(budget_s=0.25) + elapsed = asyncio.get_running_loop().time() - started + assert elapsed < 1, f"drain took {elapsed:.1f}s for 3 stalled processors" + + async def test_it_does_not_block_the_event_loop(self, monkeypatch): + """The flush must run off-loop: other lifespan work has to keep progressing + while a processor is stuck.""" + import time + import asyncio + + class Stalled: + def shutdown(self): + time.sleep(2) + + _patch_processors(monkeypatch, [Stalled()]) + ticks = 0 + + async def heartbeat(): + nonlocal ticks + while True: + await asyncio.sleep(0.01) + ticks += 1 + + beat = asyncio.create_task(heartbeat()) + await shutdown_sync_tracing_processors(budget_s=0.25) + beat.cancel() + assert ticks > 0, "the event loop was blocked during the drain" + + +class TestTheTemporalWorkerIsWiredToo: + """A Temporal agent runs its model calls in the worker process, which never + constructs a BaseACPServer. Without its own init the documented environment leaves + that process — the one doing the interesting work — completely unwired. + """ + + def test_the_worker_inits_and_drains(self): + import inspect + + from agentex.lib.core.temporal.workers.worker import AgentexWorker + + source = inspect.getsource(AgentexWorker.run) + assert "init_sgp_obs()" in source + assert "shutdown_sgp_obs()" in source + assert "shutdown_sync_tracing_processors()" in source + + def test_the_worker_does_not_pass_an_app(self): + """There is no ASGI application in the worker process. The health-check server + is aiohttp, which sgp-obs' ASGI middleware does not apply to, so passing it + would be wrong rather than merely useless.""" + import inspect + + from agentex.lib.core.temporal.workers.worker import AgentexWorker + + source = inspect.getsource(AgentexWorker.run) + assert "init_sgp_obs(app=" not in source + + +class TestConcurrencyAndProcessExit: + """Two properties the budget only really has if these hold.""" + + async def test_a_fast_processor_finishes_even_when_another_stalls(self, monkeypatch): + """Flushes start concurrently under ONE shared deadline. Draining them in + sequence let the first stalled processor spend the whole budget, so every + processor after it was skipped even when it would have returned instantly.""" + import time + + class Stalled: + def shutdown(self): + time.sleep(2) + + class Fast: + def __init__(self): + self.flushed = False + + def shutdown(self): + self.flushed = True + + fast = Fast() + # Stalled FIRST: in a sequential drain it would eat the budget and `fast` + # would never be asked. + _patch_processors(monkeypatch, [Stalled(), fast]) + await shutdown_sync_tracing_processors(budget_s=0.5) + assert fast.flushed, "a fast processor was starved by a stalled one" + + def test_a_stalled_flush_does_not_delay_process_exit(self): + """The property the deadline actually promises, and the one it did NOT have. + + `asyncio.wait_for` stops awaiting a thread; it cannot stop the thread. And + `asyncio.run` joins the default executor on the way out (as does a private + ThreadPoolExecutor, via its atexit hook), so a timed-out `asyncio.to_thread` + flush left the process blocked on the very export the budget was meant to + escape — measured at 10.0s against a 0.25s budget. Daemon threads are abandoned + at interpreter exit, which is what the budget promises. + + A subprocess, because this is about interpreter shutdown: it cannot be observed + from inside the test process. + """ + import os + import sys + import time + import textwrap + import subprocess + from pathlib import Path + + # tests/base/fastacp/sdk/lib/agentex/src -> parents[6] is the src root. + src = Path(__file__).resolve().parents[6] + program = textwrap.dedent( + """ + import asyncio, sys, time + from agentex.lib.core.tracing.tracing_processor_manager import ( + shutdown_sync_tracing_processors, + ) + import agentex.lib.core.tracing.tracing_processor_manager as mgr + + class Stalled: + def shutdown(self): + time.sleep(30) + + mgr.get_sync_tracing_processors = lambda: [Stalled()] + asyncio.run(shutdown_sync_tracing_processors(budget_s=0.25)) + """ + ) + started = time.monotonic() + proc = subprocess.run( + [sys.executable, "-c", program], + capture_output=True, + text=True, + timeout=30, + # Inherit the environment: replacing it wholesale breaks the + # interpreter's own bootstrap before the test can run. + env={**os.environ, "PYTHONPATH": str(src)}, + ) + elapsed = time.monotonic() - started + assert proc.returncode == 0, proc.stderr[-2000:] + assert elapsed < 10, ( + f"process took {elapsed:.1f}s to exit with a 30s stalled flush and a " + "0.25s budget; the flush thread is blocking interpreter shutdown" + ) + + +class TestTheWorkerObsPathRunsWithoutSgpObs: + """The image a build with NO broker token produces has no sgp-obs in it, and a + Temporal agent's model calls happen in this process. + + The two tests above pin that ``run()`` *calls* these, by reading its source. That + cannot catch a call that is written correctly and then raises, so this exercises the + sequence for real. Together: one proves the wiring exists, the other proves it is + harmless. + """ + + def test_the_worker_module_imports_and_constructs(self): + from agentex.lib.core.temporal.workers.worker import AgentexWorker + + # port 0 so nothing binds a real health port during the test + assert AgentexWorker(task_queue="probe", health_check_port=0) is not None + + async def test_init_and_both_drains_are_inert(self, monkeypatch): + """Exactly what ``run()`` does: init at entry, both drains in its finally — + with nothing wired, which is every agent that has not adopted.""" + from agentex.lib.core.observability import sgp_obs_setup + from agentex.lib.core.observability.sgp_obs_setup import ( + init_sgp_obs, + shutdown_sgp_obs, + ) + + monkeypatch.delenv("SGP_OBS_ENABLED", raising=False) + sgp_obs_setup._reset_for_tests() + try: + _block_sgp_obs_import(monkeypatch) + assert init_sgp_obs() == "not_installed" + # Neither drain may raise just because nothing was ever wired. + await shutdown_sync_tracing_processors() + await shutdown_sgp_obs() + finally: + sgp_obs_setup._reset_for_tests() diff --git a/src/agentex/lib/utils/logging.py b/src/agentex/lib/utils/logging.py index a0d39331b..fd97fc072 100644 --- a/src/agentex/lib/utils/logging.py +++ b/src/agentex/lib/utils/logging.py @@ -13,6 +13,25 @@ DEFAULT_LOG_LEVEL = logging.INFO +# Every logger this module hands out is a LEAF (``make_logger(__name__)``), and until +# now each one carried its own handler. That is fine on its own, but an observability +# pipeline that owns the ROOT logger -- sgp-obs replaces the root handler list -- then +# prints a SECOND copy of every record: once here, and once more when the record +# propagates to root. Measured on sgp-obs 0.16.0: one ``logger.info()`` produced two +# stdout lines, and sgp-obs' own boot warning named 63 loggers "bypassing log +# governance". The plain-text copy also skips the pipeline's enrichment (agent_id, +# task_id), its allowlist and its truncation, so it is not merely redundant. +# +# While this is True, ``make_logger`` attaches nothing and the record reaches the root +# pipeline by propagation alone. ``sgp_obs_setup`` sets it via +# :func:`route_agentex_loggers_to_root` -- nothing else may. +_ROOT_PIPELINE_OWNS_LOGGING = False + +# Handlers are cleared by prefix rather than by an enumerated list: the names are +# module paths, several agentex modules are imported LAZILY, and any list would be a +# snapshot that goes stale the moment one of them loads. +_PACKAGE_ROOT = "agentex" + def resolve_log_level() -> int: """Read the log level from ``LOG_LEVEL``, falling back to INFO. @@ -72,6 +91,13 @@ def make_logger(name: str) -> logging.Logger: logger = logging.getLogger(name) logger.setLevel(resolve_log_level()) + if _ROOT_PIPELINE_OWNS_LOGGING: + # A handler here would be the second one on this record's path to stdout. + # The level above is deliberately still applied: LOG_LEVEL is what agent + # authors set, and letting the pipeline's own threshold silently replace it + # would change behaviour nobody asked to change. + return logger + environment = os.getenv("ENVIRONMENT") if environment == "local": console = Console() @@ -96,3 +122,60 @@ def make_logger(name: str) -> logging.Logger: logger.addHandler(stream_handler) # Create a logger object with the name of the current module return logger + + +def route_agentex_loggers_to_root() -> int: + """Hand agentex's logging over to whatever owns the root logger. Returns the + number of loggers cleared. + + Two halves, and BOTH are needed -- measured, one line per ``logger.info()`` only + when they run together: + + * the sweep below fixes the loggers that ALREADY exist, i.e. every agentex module + imported before this ran; + * the flag fixes every logger created AFTER it, which a sweep cannot reach. + agentex imports several modules lazily (the adk ``_claude_code_sync`` / + ``_codex_sync`` / ``_pydantic_ai_sync`` harnesses among them), so their + ``make_logger`` call happens later and would attach a fresh duplicate handler. + + sgp-obs offers ``capture_loggers=`` for the first half, and it is deliberately not + used: it matches EXACT logger names, not prefixes (measured -- passing + ``("agentex",)`` still produced two lines), so it would mean enumerating ~60 module + paths; and passing anything at all replaces its uvicorn default, which would put + uvicorn's access log back to printing twice. + + Only agentex's own loggers are touched. A third party's handler may be there on + purpose -- which is exactly why sgp-obs warns about them rather than stripping them + -- so litellm's three loggers and anything else keep whatever they have. + """ + global _ROOT_PIPELINE_OWNS_LOGGING + _ROOT_PIPELINE_OWNS_LOGGING = True + + cleared = 0 + # list() snapshots the registry: a getLogger() on another thread would otherwise + # mutate the dict mid-iteration. + for name, existing in list(logging.Logger.manager.loggerDict.items()): + if not isinstance(existing, logging.Logger): + continue # a PlaceHolder for a name whose children exist but itself does not + if name != _PACKAGE_ROOT and not name.startswith(_PACKAGE_ROOT + "."): + continue + if not existing.handlers: + continue + if not existing.propagate: + # Deliberately cut off from root, so nothing of its reaches the pipeline. + # Clearing its handlers would send its records NOWHERE -- worse than a + # duplicate. Leave it exactly as its owner set it up. + continue + for handler in list(existing.handlers): + try: + handler.flush() # a buffering handler must not lose records on removal + except Exception: + pass + existing.removeHandler(handler) + cleared += 1 + return cleared + + +def _reset_for_tests() -> None: + global _ROOT_PIPELINE_OWNS_LOGGING + _ROOT_PIPELINE_OWNS_LOGGING = False diff --git a/src/agentex/lib/utils/tests/__init__.py b/src/agentex/lib/utils/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/agentex/lib/utils/tests/test_logging_handover.py b/src/agentex/lib/utils/tests/test_logging_handover.py new file mode 100644 index 000000000..4a7205afb --- /dev/null +++ b/src/agentex/lib/utils/tests/test_logging_handover.py @@ -0,0 +1,182 @@ +"""Tests for handing agentex's loggers over to a root logging pipeline. + +``make_logger`` attaches a handler to each module's OWN (leaf) logger. sgp-obs' logs +pipeline replaces the handlers on the ROOT logger and deliberately leaves named loggers +alone, on the grounds that a named logger's handler may be there on purpose. Each is +defensible; together they print every record twice — once in agentex's plain text from +the leaf, once as pipeline JSON from root. Measured on sgp-obs 0.16.0: one +``logger.info()`` produced two stdout lines and sgp-obs named 63 loggers as "bypassing +log governance". + +The duplicate is not merely redundant. It is emitted before the pipeline's filters, so +it carries no ``agent_id``/``task_id``, is not governed by the allowlist, and is not +truncated. + +The fix has two halves and needs both, which is what the subprocess tests pin: + +* the sweep clears loggers that ALREADY exist when it runs; +* the latch stops ``make_logger`` attaching to loggers created AFTERWARDS. + +A sweep alone misses the second: agentex imports several harness modules lazily, so +their ``make_logger`` runs later and would attach a fresh duplicate. +""" + +from __future__ import annotations + +import os +import sys +import logging +import textwrap +import subprocess +from typing import override +from pathlib import Path + +import pytest + +from agentex.lib.utils import logging as agentex_logging +from agentex.lib.utils.logging import make_logger, route_agentex_loggers_to_root + +_SRC = Path(__file__).resolve().parents[4] + + +@pytest.fixture(autouse=True) +def _restore_logging(): + """The latch and the loggers are process-wide; put both back.""" + saved = { + name: (obj.handlers[:], obj.propagate) + for name, obj in logging.Logger.manager.loggerDict.items() + if isinstance(obj, logging.Logger) + } + try: + yield + finally: + agentex_logging._reset_for_tests() + for name, (handlers, propagate) in saved.items(): + existing = logging.Logger.manager.loggerDict.get(name) + if isinstance(existing, logging.Logger): + existing.handlers[:] = handlers + existing.propagate = propagate + + +def _run(handover: bool) -> str: + """One trial in its own process — root-logger state is global and cannot be + isolated within a test session. Returns stdout+stderr.""" + program = textwrap.dedent( + f""" + import logging, sys + from agentex.lib.utils.logging import make_logger, route_agentex_loggers_to_root + + # Exists BEFORE the handover, like any eagerly-imported agentex module. + before = make_logger("agentex.lib.probe.before") + + # Stand in for sgp-obs' pipeline: a single handler on ROOT. + root = logging.getLogger() + root.handlers[:] = [logging.StreamHandler(sys.stdout)] + root.setLevel(logging.INFO) + + if {handover!r}: + route_agentex_loggers_to_root() + + # Created AFTER, like one of the lazily-imported harness modules. + after = make_logger("agentex.lib.probe.after") + + before.info("MARKER-BEFORE") + after.info("MARKER-AFTER") + """ + ) + proc = subprocess.run( + [sys.executable, "-c", program], + capture_output=True, + text=True, + timeout=60, + env={**os.environ, "PYTHONPATH": str(_SRC), "LOG_LEVEL": "INFO", "ENVIRONMENT": "production"}, + ) + assert proc.returncode == 0, proc.stderr[-2000:] + return proc.stdout + proc.stderr + + +class TestEveryRecordIsPrintedOnce: + def test_without_the_handover_everything_doubles(self): + """The bug, pinned. If this ever reads 1, the other two tests below have + stopped proving anything.""" + out = _run(handover=False) + assert out.count("MARKER-BEFORE") == 2 + assert out.count("MARKER-AFTER") == 2 + + def test_a_logger_created_before_the_handover_prints_once(self): + out = _run(handover=True) + assert out.count("MARKER-BEFORE") == 1 + + def test_a_logger_created_after_the_handover_prints_once(self): + """The half a sweep cannot reach: agentex imports harness modules lazily, so + their make_logger runs after init and would attach a fresh duplicate.""" + out = _run(handover=True) + assert out.count("MARKER-AFTER") == 1 + + +class TestTheSweepIsNarrow: + def test_it_clears_an_agentex_logger_that_has_a_handler(self): + lg = logging.getLogger("agentex.lib.probe.sweep") + lg.addHandler(logging.NullHandler()) + assert route_agentex_loggers_to_root() >= 1 + assert lg.handlers == [] + + def test_it_leaves_other_packages_alone(self): + """A third party's handler may be deliberate — which is exactly why sgp-obs + warns about them rather than stripping them.""" + other = logging.getLogger("litellm.probe") + handler = logging.NullHandler() + other.addHandler(handler) + route_agentex_loggers_to_root() + assert other.handlers == [handler] + + def test_it_leaves_a_non_propagating_agentex_logger_alone(self): + """Cut off from root on purpose, so nothing of its reaches the pipeline. + Clearing its handlers would send its records NOWHERE — worse than a duplicate.""" + lg = logging.getLogger("agentex.lib.probe.isolated") + handler = logging.NullHandler() + lg.addHandler(handler) + lg.propagate = False + route_agentex_loggers_to_root() + assert lg.handlers == [handler] + + def test_a_prefix_lookalike_is_not_swept(self): + """`agentexfoo` is a different package, not a child of `agentex`.""" + lg = logging.getLogger("agentexfoo.probe") + handler = logging.NullHandler() + lg.addHandler(handler) + route_agentex_loggers_to_root() + assert lg.handlers == [handler] + + def test_handlers_are_flushed_before_removal(self): + """A buffering handler would otherwise lose whatever it was holding.""" + flushed = [] + + class Recording(logging.NullHandler): + @override + def flush(self): + flushed.append(True) + + lg = logging.getLogger("agentex.lib.probe.flush") + lg.addHandler(Recording()) + route_agentex_loggers_to_root() + assert flushed == [True] + + +class TestMakeLoggerRespectsTheLatch: + def test_it_attaches_nothing_once_the_pipeline_owns_logging(self): + route_agentex_loggers_to_root() + assert make_logger("agentex.lib.probe.after_latch").handlers == [] + + def test_it_still_attaches_when_nothing_owns_logging(self): + """The non-negotiable half: an agent without sgp-obs must log exactly as it + did before any of this existed.""" + agentex_logging._reset_for_tests() + assert make_logger("agentex.lib.probe.no_latch").handlers != [] + + def test_the_level_is_applied_either_way(self, monkeypatch): + """LOG_LEVEL is what agent authors set; letting the pipeline's own threshold + silently replace it would change behaviour nobody asked to change.""" + monkeypatch.setenv("LOG_LEVEL", "DEBUG") + route_agentex_loggers_to_root() + assert make_logger("agentex.lib.probe.level").level == logging.DEBUG From 5271bb9ff979e68a67698079fd88baf7081fbdf7 Mon Sep 17 00:00:00 2001 From: Sirui Wang Date: Tue, 15 Sep 2026 15:31:47 -0700 Subject: [PATCH 2/6] chore(release): prepare 0.27.0b1 --- .release-please-manifest.json | 4 ++-- adk/pyproject.toml | 2 +- pyproject.toml | 2 +- src/agentex/_version.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 95c44cfb4..486e1262d 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,4 +1,4 @@ { - ".": "0.26.0", - "adk": "0.26.0" + ".": "0.27.0-b1", + "adk": "0.27.0-b1" } diff --git a/adk/pyproject.toml b/adk/pyproject.toml index 88125a8ce..9ea724a05 100644 --- a/adk/pyproject.toml +++ b/adk/pyproject.toml @@ -4,7 +4,7 @@ # (agentex/{__init__.py, _*.py, types/, resources/}) ships from the slim # sibling package `agentex-client` which is pinned as a runtime dep. name = "agentex-sdk" -version = "0.26.0" +version = "0.27.0b1" description = "Agent Development Kit (ADK) overlay for the Agentex API — FastACP server, Temporal workflows, LLM provider integrations, observability" license = "Apache-2.0" authors = [ diff --git a/pyproject.toml b/pyproject.toml index 8fb1f97cf..b2f5f05cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ # overlay (formerly `src/agentex/lib/*`) now lives in `adk/` and ships # as the sibling `agentex-sdk` package — see `adk/pyproject.toml`. name = "agentex-client" -version = "0.26.0" +version = "0.27.0b1" description = "The official Python REST client for the Agentex API" dynamic = ["readme"] license = "Apache-2.0" diff --git a/src/agentex/_version.py b/src/agentex/_version.py index 34aa48bd8..c35f6e82b 100644 --- a/src/agentex/_version.py +++ b/src/agentex/_version.py @@ -1,4 +1,4 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. __title__ = "agentex" -__version__ = "0.26.0" # x-release-please-version +__version__ = "0.27.0b1" # x-release-please-version From 95ff466ef1c307889b7e33d629bab9339968dc35 Mon Sep 17 00:00:00 2001 From: Sirui Wang Date: Wed, 16 Sep 2026 11:49:01 -0700 Subject: [PATCH 3/6] fix(obs): hand an agent's own loggers to the logs pipeline, not just agentex's The hand-over added in #518 had two halves, and only one of them covered an agent's own modules. The latch in `make_logger` never looked at the logger's name, so anything created after init was fine. The sweep matched the `agentex` prefix, so a logger created BEFORE init under any other name kept its handler and went on printing a second, ungoverned copy of every record. That is not a corner case: agents call `make_logger(__name__)` from their own modules, and `project.acp` -- the module that builds the ACP server, in every scaffold -- logs at import, which is necessarily before `init_sgp_obs` runs. Measured on dbt-assistant running 0.27.0b1: 123 of 3361 log lines were the second copy, each 80 microseconds after its governed twin, carrying `name`/`request_id` but no `trace_id`, `span_id`, `source` or `agent_id`. Since it is emitted before the pipeline's filters, it also escapes the allowlist and the truncation. The SDK cannot know an agent's package name, so `make_logger` now marks each handler it attaches and the sweep takes back exactly those, on a logger of any name. Prefix matching stays for `agentex.*` itself, where every handler is ours by definition. A handler this module did not attach is still left alone -- litellm's three loggers and anything else keep what their owner set up, which is why sgp-obs warns about them rather than stripping them. Renamed `route_agentex_loggers_to_root` to `route_loggers_to_root`, since "agentex loggers" is what the bug was. Co-authored-by: Claude Opus 5 --- .../lib/core/observability/sgp_obs_setup.py | 34 ++++---- .../observability/tests/test_sgp_obs_setup.py | 14 ++-- src/agentex/lib/utils/logging.py | 81 +++++++++++++----- .../lib/utils/tests/test_logging_handover.py | 84 +++++++++++++++---- 4 files changed, 156 insertions(+), 57 deletions(-) diff --git a/src/agentex/lib/core/observability/sgp_obs_setup.py b/src/agentex/lib/core/observability/sgp_obs_setup.py index 5b76766f5..b487852cf 100644 --- a/src/agentex/lib/core/observability/sgp_obs_setup.py +++ b/src/agentex/lib/core/observability/sgp_obs_setup.py @@ -53,7 +53,7 @@ from agentex.lib.utils.logging import ( make_logger, _reset_for_tests as _logging_reset_for_tests, - route_agentex_loggers_to_root, + route_loggers_to_root, ) logger = make_logger(__name__) @@ -145,7 +145,7 @@ def init_sgp_obs(app: Any = None) -> str: return _status if "logs" in handles: - _hand_agentex_logging_to_the_pipeline() + _hand_logging_to_the_pipeline() if "traces" in handles: _install_openai_agents_bridge() @@ -156,8 +156,8 @@ def init_sgp_obs(app: Any = None) -> str: return _status -def _hand_agentex_logging_to_the_pipeline() -> None: - """Stop agentex's own loggers printing a second, ungoverned copy of every record. +def _hand_logging_to_the_pipeline() -> None: + """Stop a second, ungoverned copy of every log record being printed. ``agentex.lib.utils.logging.make_logger`` attaches a handler to each module's own (leaf) logger. sgp-obs' logs pipeline replaces the handlers on the ROOT logger and @@ -169,17 +169,21 @@ def _hand_agentex_logging_to_the_pipeline() -> None: The duplicate is not merely redundant: it is emitted before the pipeline's filters, so it carries no ``agent_id``/``task_id``, is not governed by the allowlist, and is - not truncated. - - Only agentex's loggers are handed over — see - :func:`~agentex.lib.utils.logging.route_agentex_loggers_to_root` for why by prefix, - why ``capture_loggers=`` is not the mechanism, and why a third party's handler is - left where it is. + not truncated. Measured on dbt-assistant running 0.27.0b1: 123 of 3361 log lines + were the second copy, each one 80 microseconds after its governed twin. + + An agent's OWN modules are covered, not just the SDK's. They call + ``make_logger(__name__)`` too, under the agent's package name, and that is where + the dbt-assistant duplicate came from. See + :func:`~agentex.lib.utils.logging.route_loggers_to_root` for how a handler is + recognised as the SDK's on a logger whose name the SDK cannot predict, why + ``capture_loggers=`` is not the mechanism, and why a third party's handler is left + where it is. """ try: - cleared = route_agentex_loggers_to_root() + cleared = route_loggers_to_root() except Exception: # pragma: no cover - telemetry must never break startup - logger.debug("could not hand agentex logging to the sgp-obs pipeline", exc_info=True) + logger.debug("could not hand agentex logging to the sgp-obs logs pipeline", exc_info=True) return if cleared: @@ -187,9 +191,9 @@ def _hand_agentex_logging_to_the_pipeline() -> None: # naming loggers this call has just fixed. Say so, or the two lines read as a # contradiction to whoever is looking at the pod's first second of output. logger.info( - "routed %d agentex logger(s) through the sgp-obs logs pipeline; any " - "'bypass log governance' warning above that names agentex.* loggers was " - "emitted before this ran and no longer applies to them", + "routed %d logger(s) through the sgp-obs logs pipeline; any 'bypass log " + "governance' warning above that names an agentex.* logger, or one of this " + "agent's own, was emitted before this ran and no longer applies to it", cleared, ) diff --git a/src/agentex/lib/core/observability/tests/test_sgp_obs_setup.py b/src/agentex/lib/core/observability/tests/test_sgp_obs_setup.py index 34d2ef3ad..290068368 100644 --- a/src/agentex/lib/core/observability/tests/test_sgp_obs_setup.py +++ b/src/agentex/lib/core/observability/tests/test_sgp_obs_setup.py @@ -403,18 +403,18 @@ def boom(): class TestLoggingHandover: - """agentex's make_logger attaches a handler to each module's own logger; sgp-obs' - logs pipeline owns the ROOT logger and deliberately leaves named loggers alone. Both - then print, so every record appears twice — and the agentex copy is emitted before - the pipeline's filters, so it carries no agent_id/task_id, is not governed by the - allowlist, and is not truncated. + """agentex's make_logger attaches a handler to each module's own logger — the + agent's modules as well as the SDK's; sgp-obs' logs pipeline owns the ROOT logger and + deliberately leaves named loggers alone. Both then print, so every record appears + twice — and the leaf copy is emitted before the pipeline's filters, so it carries no + agent_id/task_id, is not governed by the allowlist, and is not truncated. """ @staticmethod def _spy(monkeypatch): calls = [] monkeypatch.setattr( - sgp_obs_setup, "route_agentex_loggers_to_root", lambda: calls.append(True) or 1 + sgp_obs_setup, "route_loggers_to_root", lambda: calls.append(True) or 1 ) return calls @@ -442,6 +442,6 @@ def test_a_failing_handover_does_not_stop_startup(self, monkeypatch): def boom(): raise RuntimeError("logging registry is in a strange state") - monkeypatch.setattr(sgp_obs_setup, "route_agentex_loggers_to_root", boom) + monkeypatch.setattr(sgp_obs_setup, "route_loggers_to_root", boom) _fake_sgp_obs(monkeypatch, lambda **_kwargs: {"logs": object()}) assert init_sgp_obs() == "wired:logs" diff --git a/src/agentex/lib/utils/logging.py b/src/agentex/lib/utils/logging.py index fd97fc072..e2d5d5cb4 100644 --- a/src/agentex/lib/utils/logging.py +++ b/src/agentex/lib/utils/logging.py @@ -24,7 +24,7 @@ # # While this is True, ``make_logger`` attaches nothing and the record reaches the root # pipeline by propagation alone. ``sgp_obs_setup`` sets it via -# :func:`route_agentex_loggers_to_root` -- nothing else may. +# :func:`route_loggers_to_root` -- nothing else may. _ROOT_PIPELINE_OWNS_LOGGING = False # Handlers are cleared by prefix rather than by an enumerated list: the names are @@ -32,6 +32,21 @@ # snapshot that goes stale the moment one of them loads. _PACKAGE_ROOT = "agentex" +# ``make_logger`` stamps every handler it attaches, so the hand-over can find its own +# handlers again on a logger of ANY name. +# +# The prefix above cannot reach them all, and that gap was a measured duplicate rather +# than a theoretical one: agents call ``make_logger(__name__)`` from their own modules, +# whose names come from the agent's package (``project.acp`` in every scaffold), so the +# prefix does not match and the leaf handler stayed attached. On dbt-assistant, 123 of +# 3361 log lines were a second, ungoverned copy carrying ``name``/``request_id`` but no +# ``trace_id``, ``span_id``, ``source`` or ``agent_id``. The SDK cannot know an agent's +# package name, so ownership is recorded on the handler at the moment it is attached. +# +# Marking the handler rather than keeping a registry of logger names means there is no +# bookkeeping to go stale, and a handler moved to another logger is still recognised. +_OWNED_BY_MAKE_LOGGER = "_agentex_make_logger_owned" + def resolve_log_level() -> int: """Read the log level from ``LOG_LEVEL``, falling back to INFO. @@ -81,6 +96,17 @@ def json_record(self, message: str, extra: dict, record: logging.LogRecord) -> d return extra + +def _attach(logger: logging.Logger, handler: logging.Handler) -> None: + """Attach ``handler`` and record that this module owns it. + + The mark is what lets :func:`route_loggers_to_root` take this handler back off a + logger whose name it could not have predicted. + """ + setattr(handler, _OWNED_BY_MAKE_LOGGER, True) + logger.addHandler(handler) + + def make_logger(name: str) -> logging.Logger: """ Creates a logger object with a RichHandler to print colored text. @@ -102,13 +128,15 @@ def make_logger(name: str) -> logging.Logger: if environment == "local": console = Console() # Add the RichHandler to the logger to print colored text - handler = RichHandler( - console=console, - show_level=False, - show_path=False, - show_time=False, + _attach( + logger, + RichHandler( + console=console, + show_level=False, + show_path=False, + show_time=False, + ), ) - logger.addHandler(handler) return logger stream_handler = logging.StreamHandler() @@ -119,21 +147,22 @@ def make_logger(name: str) -> logging.Logger: logging.Formatter("%(asctime)s %(levelname)s [%(name)s] [%(filename)s:%(lineno)d] - %(message)s") ) - logger.addHandler(stream_handler) + _attach(logger, stream_handler) # Create a logger object with the name of the current module return logger -def route_agentex_loggers_to_root() -> int: - """Hand agentex's logging over to whatever owns the root logger. Returns the - number of loggers cleared. +def route_loggers_to_root() -> int: + """Hand logging over to whatever owns the root logger. Returns the number of + loggers a handler was taken off. Two halves, and BOTH are needed -- measured, one line per ``logger.info()`` only when they run together: - * the sweep below fixes the loggers that ALREADY exist, i.e. every agentex module - imported before this ran; - * the flag fixes every logger created AFTER it, which a sweep cannot reach. + * the sweep below fixes the loggers that ALREADY exist, i.e. every module whose + ``make_logger`` call ran before this did -- the whole of an agent's own code, + since the ACP server is constructed from a module that logs; + * the latch fixes every logger created AFTER it, which a sweep cannot reach. agentex imports several modules lazily (the adk ``_claude_code_sync`` / ``_codex_sync`` / ``_pydantic_ai_sync`` harnesses among them), so their ``make_logger`` call happens later and would attach a fresh duplicate handler. @@ -144,9 +173,17 @@ def route_agentex_loggers_to_root() -> int: paths; and passing anything at all replaces its uvicorn default, which would put uvicorn's access log back to printing twice. - Only agentex's own loggers are touched. A third party's handler may be there on - purpose -- which is exactly why sgp-obs warns about them rather than stripping them - -- so litellm's three loggers and anything else keep whatever they have. + A handler is taken off only when it is ours, on one of two grounds: + + * anything under the ``agentex`` prefix is this package's own logger, so every + handler on it is ours to move; + * on a logger of any other name -- an agent's ``project.acp``, or any third + party's -- only a handler carrying :data:`_OWNED_BY_MAKE_LOGGER` is touched. + + That second rule is the fix for the duplicate measured on dbt-assistant, and it is + narrow on purpose. A third party's handler may be there deliberately -- which is + exactly why sgp-obs warns about them rather than stripping them -- so litellm's + three loggers and anything else keep whatever they set up themselves. """ global _ROOT_PIPELINE_OWNS_LOGGING _ROOT_PIPELINE_OWNS_LOGGING = True @@ -157,8 +194,6 @@ def route_agentex_loggers_to_root() -> int: for name, existing in list(logging.Logger.manager.loggerDict.items()): if not isinstance(existing, logging.Logger): continue # a PlaceHolder for a name whose children exist but itself does not - if name != _PACKAGE_ROOT and not name.startswith(_PACKAGE_ROOT + "."): - continue if not existing.handlers: continue if not existing.propagate: @@ -166,13 +201,19 @@ def route_agentex_loggers_to_root() -> int: # Clearing its handlers would send its records NOWHERE -- worse than a # duplicate. Leave it exactly as its owner set it up. continue + ours = name == _PACKAGE_ROOT or name.startswith(_PACKAGE_ROOT + ".") + removed = 0 for handler in list(existing.handlers): + if not ours and not getattr(handler, _OWNED_BY_MAKE_LOGGER, False): + continue try: handler.flush() # a buffering handler must not lose records on removal except Exception: pass existing.removeHandler(handler) - cleared += 1 + removed += 1 + if removed: + cleared += 1 return cleared diff --git a/src/agentex/lib/utils/tests/test_logging_handover.py b/src/agentex/lib/utils/tests/test_logging_handover.py index 4a7205afb..265237322 100644 --- a/src/agentex/lib/utils/tests/test_logging_handover.py +++ b/src/agentex/lib/utils/tests/test_logging_handover.py @@ -1,4 +1,4 @@ -"""Tests for handing agentex's loggers over to a root logging pipeline. +"""Tests for handing the process's loggers over to a root logging pipeline. ``make_logger`` attaches a handler to each module's OWN (leaf) logger. sgp-obs' logs pipeline replaces the handlers on the ROOT logger and deliberately leaves named loggers @@ -19,6 +19,14 @@ A sweep alone misses the second: agentex imports several harness modules lazily, so their ``make_logger`` runs later and would attach a fresh duplicate. + +Both halves have to cover an agent's OWN loggers, not just ``agentex.*``. The agent +calls ``make_logger(__name__)`` from modules named for its own package, and a sweep that +matched only the ``agentex`` prefix left those printing twice: measured on dbt-assistant +running 0.27.0b1, 123 of 3361 log lines were the ungoverned copy, all of them from +``project.acp``. The latch already covered them (it does not look at the name); the +sweep did not, because ``project.acp``'s ``make_logger`` runs at import, before the ACP +server is constructed and ``init_sgp_obs`` runs. """ from __future__ import annotations @@ -34,7 +42,7 @@ import pytest from agentex.lib.utils import logging as agentex_logging -from agentex.lib.utils.logging import make_logger, route_agentex_loggers_to_root +from agentex.lib.utils.logging import make_logger, route_loggers_to_root _SRC = Path(__file__).resolve().parents[4] @@ -64,23 +72,28 @@ def _run(handover: bool) -> str: program = textwrap.dedent( f""" import logging, sys - from agentex.lib.utils.logging import make_logger, route_agentex_loggers_to_root + from agentex.lib.utils.logging import make_logger, route_loggers_to_root # Exists BEFORE the handover, like any eagerly-imported agentex module. before = make_logger("agentex.lib.probe.before") + # The agent's own module, which is where the measured duplicate came from: + # its make_logger runs at import, so it always predates the handover. + agent = make_logger("project.acp") + # Stand in for sgp-obs' pipeline: a single handler on ROOT. root = logging.getLogger() root.handlers[:] = [logging.StreamHandler(sys.stdout)] root.setLevel(logging.INFO) if {handover!r}: - route_agentex_loggers_to_root() + route_loggers_to_root() # Created AFTER, like one of the lazily-imported harness modules. after = make_logger("agentex.lib.probe.after") before.info("MARKER-BEFORE") + agent.info("MARKER-AGENT") after.info("MARKER-AFTER") """ ) @@ -97,16 +110,24 @@ def _run(handover: bool) -> str: class TestEveryRecordIsPrintedOnce: def test_without_the_handover_everything_doubles(self): - """The bug, pinned. If this ever reads 1, the other two tests below have - stopped proving anything.""" + """The bug, pinned. If this ever reads 1, the other tests below have stopped + proving anything.""" out = _run(handover=False) assert out.count("MARKER-BEFORE") == 2 + assert out.count("MARKER-AGENT") == 2 assert out.count("MARKER-AFTER") == 2 def test_a_logger_created_before_the_handover_prints_once(self): out = _run(handover=True) assert out.count("MARKER-BEFORE") == 1 + def test_an_agents_own_logger_prints_once(self): + """The regression measured on dbt-assistant: ``project.acp`` is not under the + ``agentex`` prefix, so a prefix-only sweep left its handler attached and every + record it logged was printed twice.""" + out = _run(handover=True) + assert out.count("MARKER-AGENT") == 1 + def test_a_logger_created_after_the_handover_prints_once(self): """The half a sweep cannot reach: agentex imports harness modules lazily, so their make_logger runs after init and would attach a fresh duplicate.""" @@ -118,7 +139,15 @@ class TestTheSweepIsNarrow: def test_it_clears_an_agentex_logger_that_has_a_handler(self): lg = logging.getLogger("agentex.lib.probe.sweep") lg.addHandler(logging.NullHandler()) - assert route_agentex_loggers_to_root() >= 1 + assert route_loggers_to_root() >= 1 + assert lg.handlers == [] + + def test_it_clears_our_own_handler_from_a_logger_of_any_name(self): + """``make_logger`` marks what it attaches, which is the only way to find it + again on a logger named for the agent's package rather than for agentex.""" + lg = make_logger("project.acp") + assert lg.handlers != [] + assert route_loggers_to_root() >= 1 assert lg.handlers == [] def test_it_leaves_other_packages_alone(self): @@ -127,9 +156,18 @@ def test_it_leaves_other_packages_alone(self): other = logging.getLogger("litellm.probe") handler = logging.NullHandler() other.addHandler(handler) - route_agentex_loggers_to_root() + route_loggers_to_root() assert other.handlers == [handler] + def test_it_takes_only_its_own_handler_off_a_shared_logger(self): + """An agent may have added a handler of its own next to ours. Ours goes, the + agent's stays exactly where it put it.""" + lg = make_logger("project.shared") + theirs = logging.NullHandler() + lg.addHandler(theirs) + route_loggers_to_root() + assert lg.handlers == [theirs] + def test_it_leaves_a_non_propagating_agentex_logger_alone(self): """Cut off from root on purpose, so nothing of its reaches the pipeline. Clearing its handlers would send its records NOWHERE — worse than a duplicate.""" @@ -137,15 +175,25 @@ def test_it_leaves_a_non_propagating_agentex_logger_alone(self): handler = logging.NullHandler() lg.addHandler(handler) lg.propagate = False - route_agentex_loggers_to_root() + route_loggers_to_root() assert lg.handlers == [handler] - def test_a_prefix_lookalike_is_not_swept(self): - """`agentexfoo` is a different package, not a child of `agentex`.""" + def test_it_leaves_a_non_propagating_logger_of_ours_alone(self): + """Same reasoning, for a logger the agent cut off from root after asking us for + it: our handler is the only route its records have.""" + lg = make_logger("project.isolated") + ours = lg.handlers[:] + lg.propagate = False + route_loggers_to_root() + assert lg.handlers == ours + + def test_a_prefix_lookalike_gets_no_blanket_sweep(self): + """`agentexfoo` is a different package, not a child of `agentex`, so only a + handler of ours would be taken off it — and this one is not.""" lg = logging.getLogger("agentexfoo.probe") handler = logging.NullHandler() lg.addHandler(handler) - route_agentex_loggers_to_root() + route_loggers_to_root() assert lg.handlers == [handler] def test_handlers_are_flushed_before_removal(self): @@ -159,15 +207,21 @@ def flush(self): lg = logging.getLogger("agentex.lib.probe.flush") lg.addHandler(Recording()) - route_agentex_loggers_to_root() + route_loggers_to_root() assert flushed == [True] class TestMakeLoggerRespectsTheLatch: def test_it_attaches_nothing_once_the_pipeline_owns_logging(self): - route_agentex_loggers_to_root() + route_loggers_to_root() assert make_logger("agentex.lib.probe.after_latch").handlers == [] + def test_it_attaches_nothing_for_an_agents_own_logger_either(self): + """The latch never looked at the name, so this half already covered the agent's + lazily-imported modules; pinned so it stays that way.""" + route_loggers_to_root() + assert make_logger("project.after_latch").handlers == [] + def test_it_still_attaches_when_nothing_owns_logging(self): """The non-negotiable half: an agent without sgp-obs must log exactly as it did before any of this existed.""" @@ -178,5 +232,5 @@ def test_the_level_is_applied_either_way(self, monkeypatch): """LOG_LEVEL is what agent authors set; letting the pipeline's own threshold silently replace it would change behaviour nobody asked to change.""" monkeypatch.setenv("LOG_LEVEL", "DEBUG") - route_agentex_loggers_to_root() + route_loggers_to_root() assert make_logger("agentex.lib.probe.level").level == logging.DEBUG From 6656c8d9e030b546c9075f06763517afe999f55a Mon Sep 17 00:00:00 2001 From: Sirui Wang Date: Wed, 16 Sep 2026 11:49:15 -0700 Subject: [PATCH 4/6] fix(obs): keep request_id in the logs once the leaf handler is gone `request_id` had exactly one writer: `CustomJSONFormatter`, on the handler `make_logger` attaches to each module's own logger. Removing that handler is what stops the duplicate line -- and it takes the field's only writer with it, so without this the id would not move to the governed copy, it would disappear. Measured on dbt-assistant: `request_id` was on 5.2% of lines, which were exactly the ungoverned copies, against `trace_id` on 90.6%. The ACP middleware now binds its id into sgp-obs' shared correlation context, which the logs pipeline enriches every record from. sgp-obs can also fill that context from its own `RequestIdMiddleware`; binding the SDK's id instead keeps ONE generator for the value, so the id in the logs is the id `ctx_var_request_id` gives application code and the id `x-request-id` carried in. Deliberately not written onto the record here. The pipeline's enrich stage runs on a copy of the record at handler time and treats a hand-set value as authoritative, which cannot collide with a caller's own field. Setting the attribute up front does collide: with `request_id` already on the record, the stdlib raises `KeyError: Attempt to overwrite 'request_id' in LogRecord` from `logger.info(..., extra={"request_id": ...})` -- measured, not theoretical, and an unacceptable way for telemetry to reach an agent. Fail-open throughout, and the optional import is resolved once: Python does not cache a failed import, so attempting one per request would re-walk sys.path for the majority of agents that never install sgp-obs. Co-authored-by: Claude Opus 5 --- .../lib/sdk/fastacp/base/base_acp_server.py | 70 ++++++++ tests/test_request_id_correlation.py | 150 ++++++++++++++++++ 2 files changed, 220 insertions(+) create mode 100644 tests/test_request_id_correlation.py diff --git a/src/agentex/lib/sdk/fastacp/base/base_acp_server.py b/src/agentex/lib/sdk/fastacp/base/base_acp_server.py index 06bc2595c..50c304c92 100644 --- a/src/agentex/lib/sdk/fastacp/base/base_acp_server.py +++ b/src/agentex/lib/sdk/fastacp/base/base_acp_server.py @@ -86,6 +86,71 @@ def _attach_incoming_otel_context(scope_headers: list[tuple[bytes, bytes]]) -> o return None +# sgp-obs is an optional install (see ``sgp_obs_setup``), and Python does not cache a +# FAILED import, so attempting one per request would re-walk sys.path for the majority +# of agents that do not have it. Resolved once, to the module or to None. +_OBS_CONTEXT_UNRESOLVED = object() +_obs_context_module: Any = _OBS_CONTEXT_UNRESOLVED + + +def _sgp_obs_context() -> Any | None: + global _obs_context_module + if _obs_context_module is _OBS_CONTEXT_UNRESOLVED: + try: + from sgp_obs import context as obs_context # type: ignore[import-not-found] + + _obs_context_module = obs_context + except Exception: # pragma: no cover - the normal case: sgp-obs is not installed + _obs_context_module = None + return _obs_context_module + + +def _bind_request_id_for_telemetry(request_id: str) -> object | None: + """Put the request id where a logs pipeline reads it from. + + Until the logging hand-over, ``request_id`` reached the logs through exactly one + writer: ``CustomJSONFormatter``, on the handler ``make_logger`` attaches to each + module's own logger. That handler is taken off once a pipeline owns the root logger, + because it was printing a second, ungoverned copy of every record -- and it was the + field's only writer, so without this the request id would not move to the governed + copy, it would disappear. Measured on dbt-assistant: ``request_id`` appeared on 5.2% + of log lines, which were exactly the ungoverned copies. + + sgp-obs reads the id from its shared correlation context -- the one place all three + signals take correlation ids from -- and stamps it onto each record in a stage that + runs on a COPY of the record at handler time. That is why the id is handed over + rather than written onto the record here: ``extra={"request_id": ...}`` from a + caller and an attribute set before the call would collide, and the stdlib raises + ``KeyError`` for that collision at the ``logger.info()`` call site. + + sgp-obs can also fill this context from its own ``RequestIdMiddleware``. Binding the + SDK's id here instead keeps ONE generator for the value, so the id in the logs is + the same one ``ctx_var_request_id`` gives application code and the same one + ``x-request-id`` carried in. + + Returns a reset token (or None); fail-open. + """ + obs_context = _sgp_obs_context() + if obs_context is None: + return None + try: + return obs_context.bind(request_id=request_id) + except Exception: # pragma: no cover - obs must never break a request + return None + + +def _unbind_request_id_for_telemetry(token: object | None) -> None: + if token is None: + return + obs_context = _sgp_obs_context() + if obs_context is None: + return + try: + obs_context.reset(token) + except Exception: # pragma: no cover - best-effort + pass + + def _detach_otel_context(token: object | None) -> None: if token is None: return @@ -105,12 +170,16 @@ def __init__(self, app: ASGIApp) -> None: async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: otel_token: object | None = None + obs_request_token: object | None = None if scope["type"] == "http": scope_headers = scope.get("headers", []) headers = dict(scope_headers) raw_request_id = headers.get(b"x-request-id", b"") request_id = raw_request_id.decode() if raw_request_id else uuid.uuid4().hex ctx_var_request_id.set(request_id) + # Keep the id in the logs once the leaf handler that used to write it is + # gone; see _bind_request_id_for_telemetry. + obs_request_token = _bind_request_id_for_telemetry(request_id) # Continue the ingress trace for this request (and its background # Temporal dispatch); see _attach_incoming_otel_context. otel_token = _attach_incoming_otel_context(scope_headers) @@ -118,6 +187,7 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: await self.app(scope, receive, send) finally: _detach_otel_context(otel_token) + _unbind_request_id_for_telemetry(obs_request_token) class BaseACPServer(FastAPI): diff --git a/tests/test_request_id_correlation.py b/tests/test_request_id_correlation.py new file mode 100644 index 000000000..b281cf94e --- /dev/null +++ b/tests/test_request_id_correlation.py @@ -0,0 +1,150 @@ +"""Unit tests for handing the ACP request id to the observability pipeline. + +``request_id`` used to reach the logs through exactly one writer: ``CustomJSONFormatter`` +on the handler ``make_logger`` attaches to each module's own logger. That handler is +taken off once a logs pipeline owns the root logger, because it was printing a second, +ungoverned copy of every record — so the id has to be handed over, or it is simply lost. +Measured on dbt-assistant running 0.27.0b1, ``request_id`` was on 5.2% of log lines, +which were exactly the ungoverned copies. + +The hand-over target is sgp-obs' shared correlation context (``sgp_obs.context``: +``bind(**fields) -> Token``, ``reset(token)``, ``current()``), stubbed here because +sgp-obs is an optional install and is deliberately not a dependency of this package. +""" + +from __future__ import annotations + +from typing import Any +from contextvars import ContextVar + +import pytest + +from agentex.lib.utils.logging import ctx_var_request_id +from agentex.lib.sdk.fastacp.base import base_acp_server +from agentex.lib.sdk.fastacp.base.base_acp_server import ( + RequestIDMiddleware, + _bind_request_id_for_telemetry, + _unbind_request_id_for_telemetry, +) + + +class StubObsContext: + """The shape of ``sgp_obs.context`` that this SDK uses, over a real ContextVar so + "was the id in scope while the request ran?" is a real question.""" + + def __init__(self) -> None: + self._var: ContextVar[str | None] = ContextVar("stub_request_id", default=None) + self.binds: list[dict[str, Any]] = [] + self.resets = 0 + + def bind(self, **fields: Any) -> object: + self.binds.append(fields) + return self._var.set(fields.get("request_id")) + + def reset(self, token: Any) -> None: + self.resets += 1 + self._var.reset(token) + + def current(self) -> str | None: + return self._var.get() + + +@pytest.fixture +def obs(monkeypatch: pytest.MonkeyPatch) -> StubObsContext: + stub = StubObsContext() + # The memo is the seam: the helpers resolve `sgp_obs.context` once per process. + monkeypatch.setattr(base_acp_server, "_obs_context_module", stub) + return stub + + +def test_the_request_id_is_bound_for_the_pipeline(obs: StubObsContext) -> None: + token = _bind_request_id_for_telemetry("req-abc") + try: + assert obs.binds == [{"request_id": "req-abc"}] + assert obs.current() == "req-abc" + finally: + _unbind_request_id_for_telemetry(token) + + +def test_it_is_unbound_again(obs: StubObsContext) -> None: + _unbind_request_id_for_telemetry(_bind_request_id_for_telemetry("req-abc")) + assert obs.resets == 1 + assert obs.current() is None + + +def test_an_absent_sgp_obs_is_fail_open(monkeypatch: pytest.MonkeyPatch) -> None: + """The normal case for an agent that has not installed it.""" + monkeypatch.setattr(base_acp_server, "_obs_context_module", None) + assert _bind_request_id_for_telemetry("req-abc") is None + _unbind_request_id_for_telemetry(None) # must be safe + + +def test_a_raising_bind_does_not_break_the_request(monkeypatch: pytest.MonkeyPatch) -> None: + """``bind`` rejects unknown field names, so a future rename must degrade to no + correlation rather than to a failed request.""" + + class Raising: + def bind(self, **_fields: Any) -> object: + raise TypeError("unexpected keyword argument") + + def reset(self, _token: Any) -> None: + raise AssertionError("nothing to reset") + + monkeypatch.setattr(base_acp_server, "_obs_context_module", Raising()) + assert _bind_request_id_for_telemetry("req-abc") is None + + +@pytest.mark.asyncio +async def test_the_middleware_binds_the_same_id_it_gives_application_code( + obs: StubObsContext, +) -> None: + """One generator for the value: the id in the logs is the id the SDK's own + contextvar hands to the agent, and the id ``x-request-id`` carried in.""" + seen: dict[str, Any] = {} + + async def app(_scope: Any, _receive: Any, _send: Any) -> None: + seen["sdk"] = ctx_var_request_id.get(None) + seen["obs"] = obs.current() + + scope = {"type": "http", "headers": [(b"x-request-id", b"req-from-the-gateway")]} + await RequestIDMiddleware(app)(scope, None, None) # type: ignore[arg-type] + + assert seen["obs"] == "req-from-the-gateway" + assert seen["sdk"] == seen["obs"] + # Bound for the request only, so a later record cannot inherit a stale id. + assert obs.current() is None + assert obs.resets == 1 + + +@pytest.mark.asyncio +async def test_a_generated_id_is_bound_when_the_header_is_absent(obs: StubObsContext) -> None: + seen: dict[str, Any] = {} + + async def app(_scope: Any, _receive: Any, _send: Any) -> None: + seen["obs"] = obs.current() + + await RequestIDMiddleware(app)({"type": "http", "headers": []}, None, None) # type: ignore[arg-type] + assert seen["obs"] + + +@pytest.mark.asyncio +async def test_a_non_http_scope_binds_nothing(obs: StubObsContext) -> None: + """Lifespan and websocket scopes have no request id to bind.""" + + async def app(_scope: Any, _receive: Any, _send: Any) -> None: + return None + + await RequestIDMiddleware(app)({"type": "lifespan"}, None, None) # type: ignore[arg-type] + assert obs.binds == [] + assert obs.resets == 0 + + +@pytest.mark.asyncio +async def test_it_is_unbound_even_when_the_request_raises(obs: StubObsContext) -> None: + async def app(_scope: Any, _receive: Any, _send: Any) -> None: + raise RuntimeError("handler blew up") + + with pytest.raises(RuntimeError): + await RequestIDMiddleware(app)({"type": "http", "headers": []}, None, None) # type: ignore[arg-type] + assert obs.resets == 1 + assert obs.current() is None From ad3b44cb7c6b915aec9b5151835ca22129453e06 Mon Sep 17 00:00:00 2001 From: Sirui Wang Date: Wed, 16 Sep 2026 11:49:16 -0700 Subject: [PATCH 5/6] chore(release): supersede 0.27.0b1 with 0.27.0b2 0.27.0b1 is not just prepared, it is installed: the six obs-test agents pin `agentex-sdk==0.27.0b1` and dbt-assistant is running it, which is how the log duplication was measured. Leaving the two fixes above under the same version would make "0.27.0b1" name two different artifacts. Still does not publish anything. Further fixes on this branch can ride b2 while it is unpublished; once it is published, the next one bumps again. Co-authored-by: Claude Opus 5 --- .release-please-manifest.json | 4 ++-- adk/pyproject.toml | 2 +- pyproject.toml | 2 +- src/agentex/_version.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 486e1262d..7bf483f3f 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,4 +1,4 @@ { - ".": "0.27.0-b1", - "adk": "0.27.0-b1" + ".": "0.27.0-b2", + "adk": "0.27.0-b2" } diff --git a/adk/pyproject.toml b/adk/pyproject.toml index 9ea724a05..14ad401fa 100644 --- a/adk/pyproject.toml +++ b/adk/pyproject.toml @@ -4,7 +4,7 @@ # (agentex/{__init__.py, _*.py, types/, resources/}) ships from the slim # sibling package `agentex-client` which is pinned as a runtime dep. name = "agentex-sdk" -version = "0.27.0b1" +version = "0.27.0b2" description = "Agent Development Kit (ADK) overlay for the Agentex API — FastACP server, Temporal workflows, LLM provider integrations, observability" license = "Apache-2.0" authors = [ diff --git a/pyproject.toml b/pyproject.toml index b2f5f05cf..9c1954d19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ # overlay (formerly `src/agentex/lib/*`) now lives in `adk/` and ships # as the sibling `agentex-sdk` package — see `adk/pyproject.toml`. name = "agentex-client" -version = "0.27.0b1" +version = "0.27.0b2" description = "The official Python REST client for the Agentex API" dynamic = ["readme"] license = "Apache-2.0" diff --git a/src/agentex/_version.py b/src/agentex/_version.py index c35f6e82b..c190753a8 100644 --- a/src/agentex/_version.py +++ b/src/agentex/_version.py @@ -1,4 +1,4 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. __title__ = "agentex" -__version__ = "0.27.0b1" # x-release-please-version +__version__ = "0.27.0b2" # x-release-please-version From 51a2bb3c02332b13e80a32e87798308f03fd135b Mon Sep 17 00:00:00 2001 From: Sirui Wang Date: Thu, 17 Sep 2026 09:44:17 -0700 Subject: [PATCH 6/6] fix(obs): resolve the optional sgp-obs import once, not per model call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Python does not cache a FAILED import, so `from sgp_obs.metrics import genai` inside inference_call re-walked sys.path on every single litellm call — for the majority of agents, which are the ones that do not have sgp-obs installed. Measured on the published 0.27.0b2 wheel in a clean venv with no sgp-obs, best of 5x500 calls against a mocked transport: 0.26.0 11.6 us/call 0.27.0b2 as published 84.0 us/call 0.27.0b2 + this fix 11.8 us/call The bare failed import is 62us of that, with only five sys.path entries; a container image has more. Negligible beside a real model call, but it is pure waste on the hot path, and the module docstring claimed the fallback "costs nothing", which was the one part of it that was not true. base_acp_server.py already solved exactly this for sgp_obs.context with an _OBS_CONTEXT_UNRESOLVED sentinel, and said why in a comment. This applies the same shape so the two files agree. The one-time debug line moves into the resolver, whose body now runs exactly once — which retires the separate _warned latch rather than leaving two latches for one fact. Adds _reset_for_tests(), matching sgp_obs_setup.py and utils/logging.py. The handle is process-wide state, so without it the first test to run with sgp-obs absent would cache None for the rest of the session and every later test that injects a fake sgp_obs.metrics would silently exercise the null path instead of the one it means to. Tests: 22 pass, up from 19. The new ones count import attempts under a counting __import__ hook — 50 calls give 50 attempts on the old body and 1 on this one — and pin that a present sgp-obs is still used on calls two and three, so the cache cannot degrade a working install to the null path. Co-Authored-By: Claude Opus 5 --- .../lib/core/adapters/llm/_genai_metrics.py | 60 ++++++++--- .../adapters/llm/tests/test_genai_metrics.py | 102 +++++++++++++++++- 2 files changed, 148 insertions(+), 14 deletions(-) diff --git a/src/agentex/lib/core/adapters/llm/_genai_metrics.py b/src/agentex/lib/core/adapters/llm/_genai_metrics.py index 3c2293cd6..0c399ddfb 100644 --- a/src/agentex/lib/core/adapters/llm/_genai_metrics.py +++ b/src/agentex/lib/core/adapters/llm/_genai_metrics.py @@ -21,7 +21,8 @@ Everything here is fail-open: sgp-obs is an optional dependency and a telemetry problem must never fail a model call. If the import fails, :func:`inference_call` returns an -object that records nothing and costs nothing. +object that records nothing, and the failure is remembered so that later calls cost an +identity check rather than another walk of sys.path. """ from __future__ import annotations @@ -39,7 +40,39 @@ # A bare model name with no "/" prefix is OpenAI, per litellm's own default. _DEFAULT_VENDOR = "openai" -_warned = False +# sgp-obs is an optional install (see ``sgp_obs_setup``), and Python does NOT cache a +# FAILED import, so importing inside ``inference_call`` re-walked sys.path on every +# single model call for the majority of agents that do not have it. Measured on +# 0.27.0b2 in a venv with five sys.path entries (a container image has more): 62us per +# attempt, which took the gateway's own per-call overhead from 12us to 84us. Resolved +# once, to the module or to None -- the shape ``base_acp_server`` already uses for +# ``sgp_obs.context``, for the same reason. +_GENAI_UNRESOLVED = object() +_genai_module: Any = _GENAI_UNRESOLVED + + +def _genai() -> Any | None: + """The sgp-obs GenAI metrics module, or None when it is not installed. + + Resolved on first use rather than at import time, so that importing the litellm + adapter does not pay for it and the answer is read after startup has run. + + The debug line is here rather than at the call site because this body runs exactly + once, which is the only place a "said it once" latch is not needed. + """ + global _genai_module + if _genai_module is _GENAI_UNRESOLVED: + try: + # See sgp_obs_setup.py: optional, not publicly installable, absent in CI. + from sgp_obs.metrics import genai # type: ignore[import-not-found] + + _genai_module = genai + except Exception: + _genai_module = None + logger.debug( + "sgp-obs is not available; GenAI metrics are off for litellm calls" + ) + return _genai_module def _split_model(model: str) -> tuple[str, bool]: @@ -82,16 +115,8 @@ def resolve_model(args: tuple[Any, ...], kwargs: dict[str, Any]) -> str: def inference_call(kwargs: dict[str, Any], args: tuple[Any, ...] = ()) -> Any: """Begin recording one litellm call. Never raises, never returns None.""" - try: - # See sgp_obs_setup.py: optional, not publicly installable, absent in CI. - from sgp_obs.metrics import genai # type: ignore[import-not-found] - except Exception: - global _warned - if not _warned: - _warned = True - logger.debug( - "sgp-obs is not available; GenAI metrics are off for litellm calls" - ) + genai = _genai() + if genai is None: return _NULL_CALL try: @@ -131,3 +156,14 @@ async def __aexit__(self, _exc_type: Any, _exc: Any, _tb: Any) -> bool: _NULL_CALL = _NullCall() + + +def _reset_for_tests() -> None: + """Forget the resolved module, so a test can present a different sgp-obs. + + The handle is a process-wide latch: without this, the first test to run with + sgp-obs absent would cache None for the rest of the session and every later test + that injects a fake ``sgp_obs.metrics`` would silently exercise the null path. + """ + global _genai_module + _genai_module = _GENAI_UNRESOLVED diff --git a/src/agentex/lib/core/adapters/llm/tests/test_genai_metrics.py b/src/agentex/lib/core/adapters/llm/tests/test_genai_metrics.py index b4276fdb1..8719b0198 100644 --- a/src/agentex/lib/core/adapters/llm/tests/test_genai_metrics.py +++ b/src/agentex/lib/core/adapters/llm/tests/test_genai_metrics.py @@ -23,6 +23,20 @@ ) +@pytest.fixture(autouse=True) +def _forget_resolved_sgp_obs(): + """Clear the resolved-once module handle around every test. + + It is process-wide state, so without this the first test to run with sgp-obs + absent would cache None for the rest of the session and every later test that + injects a fake ``sgp_obs.metrics`` would silently exercise the null path instead + of the one it means to. + """ + _genai_metrics._reset_for_tests() + yield + _genai_metrics._reset_for_tests() + + class TestSplitModel: """``(vendor, goes_out_over_the_openai_client)``. The boolean decides whether ``call()`` stands down for the OpenAI client instrumentor or records itself, so @@ -67,8 +81,9 @@ def no_sgp_obs(name, *args, **kwargs): return real_import(name, *args, **kwargs) monkeypatch.setattr(builtins, "__import__", no_sgp_obs) - # The "already warned" latch is module state; reset so the path is exercised. - monkeypatch.setattr(_genai_metrics, "_warned", False) + # Resolution is cached, so drop anything a previous call resolved -- otherwise + # hiding the module here would have no effect. + _genai_metrics._reset_for_tests() def test_returns_a_usable_recorder_not_none(self, monkeypatch): self._hide_sgp_obs(monkeypatch) @@ -119,6 +134,89 @@ def exploding(**_kwargs): assert inference_call({"model": "gpt-4o"}) is _genai_metrics._NULL_CALL +class TestTheImportIsResolvedOnce: + """Python does not cache a FAILED import, so importing inside ``inference_call`` + re-walked sys.path on every model call. Measured at 62us per attempt with five + sys.path entries, which was most of the gateway's per-call overhead for the + majority of agents -- the ones with no sgp-obs installed.""" + + def test_a_missing_sgp_obs_is_looked_up_once_not_per_call(self, monkeypatch): + attempts = [] + real_import = builtins.__import__ + + def counting_import(name, *args, **kwargs): + if name == "sgp_obs" or name.startswith("sgp_obs."): + attempts.append(name) + raise ImportError("No module named 'sgp_obs'") + return real_import(name, *args, **kwargs) + + for name in [m for m in sys.modules if m.startswith("sgp_obs")]: + monkeypatch.delitem(sys.modules, name, raising=False) + monkeypatch.setattr(builtins, "__import__", counting_import) + + for _ in range(50): + assert inference_call({"model": "gpt-4o"}) is _genai_metrics._NULL_CALL + + assert len(attempts) == 1, f"expected one import attempt, got {len(attempts)}" + + def test_a_present_sgp_obs_is_looked_up_once_too(self, monkeypatch): + """The handle must cache the module as well as the failure, or an agent that + DOES have sgp-obs keeps paying for a lookup it already did.""" + attempts = [] + + class _Genai: + CHAT = "chat" + OPENAI_SPEC = "openai" + OPENAI = "openai" + + @staticmethod + def call(**_kwargs): + return _genai_metrics._NULL_CALL + + module = type(sys)("sgp_obs.metrics") + module.genai = _Genai + monkeypatch.setitem(sys.modules, "sgp_obs", type(sys)("sgp_obs")) + monkeypatch.setitem(sys.modules, "sgp_obs.metrics", module) + + real_import = builtins.__import__ + + def counting_import(name, *args, **kwargs): + if name == "sgp_obs" or name.startswith("sgp_obs."): + attempts.append(name) + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", counting_import) + + for _ in range(50): + inference_call({"model": "gpt-4o"}) + + assert len(attempts) == 1, f"expected one import attempt, got {len(attempts)}" + + def test_the_recorder_is_still_the_real_one_after_caching(self, monkeypatch): + """Caching must not turn a working sgp-obs into the null path on call two.""" + seen = [] + + class _Genai: + CHAT = "chat" + OPENAI_SPEC = "openai" + OPENAI = "openai" + + @staticmethod + def call(**kwargs): + seen.append(kwargs["model"]) + return _genai_metrics._NULL_CALL + + module = type(sys)("sgp_obs.metrics") + module.genai = _Genai + monkeypatch.setitem(sys.modules, "sgp_obs", type(sys)("sgp_obs")) + monkeypatch.setitem(sys.modules, "sgp_obs.metrics", module) + + for index in range(3): + inference_call({"model": f"anthropic/claude-{index}"}) + + assert seen == ["anthropic/claude-0", "anthropic/claude-1", "anthropic/claude-2"] + + class TestResolveModel: """litellm takes `model` as its FIRST positional argument and the gateway forwards *args untouched, so a positional call is legal and must still be measured.