braintrust-evals.yml

name: Braintrust Evals

on:
  workflow_dispatch:
  push:
    branches:
      - main
  pull_request:
    types:
      - labeled
      - synchronize

jobs:
  run-braintrust-evals:
    name: Run Braintrust Evals
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: write
    if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'braintrust-evals')
    env:
      BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
      VOYAGE_API_KEY: ${{ secrets.TEST_MDB_MCP_VOYAGE_API_KEY }}
      EMBEDDING_PROVIDER_ENDPOINT: "https://api.voyageai.com/v1/embeddings"
      GIT_BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
      # Limits concurrent evals to avoid slowdowns from simultaneous database or text/vector index creation per eval case
      EVAL_MAX_CONCURRENCY: 25
    steps:
      - uses: GitHubSecurityLab/actions-permissions/monitor@v1
      - uses: actions/checkout@v7.0.1
        with:
          persist-credentials: false
      - uses: pnpm/action-setup@v5
      - uses: actions/setup-node@v7
        with:
          # Braintrust push accepts Node 18–22 only
          # https://www.braintrust.dev/docs/evaluate/remote-evals#2-register-your-sandbox
          node-version: "22"
          cache: "pnpm"
      - name: Install dependencies
        run: pnpm install --frozen-lockfile
      - name: Build
        run: pnpm run build
      - name: Push eval to Braintrust
        if: github.ref == 'refs/heads/main'
        run: pnpm run eval:push
      - name: Start local MongoDB
        run: pnpm run eval:db-start
      - name: Run Braintrust eval suite
        run: pnpm run eval:ci:run
      - name: Stop local MongoDB
        if: always()
        run: pnpm run eval:db-stop
      - name: Comment summary on PR
        if: github.event_name == 'pull_request'
        uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5
        with:
          hide_and_recreate: true
          hide_classify: "OUTDATED"
          path: .eval/ci-report.md