From 5b1d56d277fc59f698b8967e2972f3de4ce1893f Mon Sep 17 00:00:00 2001 From: yoobinshin Date: Thu, 22 Oct 2026 14:14:19 -0400 Subject: [PATCH] fix: pin Arc chart image repositories to the resolved MCR path The workflow resolves image tags from aks/v2/fleet but packages the chart from the plain release tag. When that tag predates a change to the chart's image repositories, the published chart pairs the new tags with the old path: 0.18.16-3 was packaged from v0.18.16, one commit before the move to aks/v2/fleet, and referenced aks/fleet/*:v0.18.16-3, which does not exist. Every install hit ImagePullBackOff and the canary rollout gate failed. Overwrite each image repository with MCR_FLEET_REPO before packaging, so the chart always points where the tags were resolved, regardless of the release tag's values.yaml. Then render the chart and check every referenced image against MCR, failing before publish if any is missing. Co-Authored-By: Claude Signed-off-by: yoobinshin --- .github/workflows/build-publish-mcr.yml | 50 ++++++++++++++++++++++--- charts/member-agent-arc/README.md | 5 +++ 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-publish-mcr.yml b/.github/workflows/build-publish-mcr.yml index 552fa8c6e..4f2514af2 100644 --- a/.github/workflows/build-publish-mcr.yml +++ b/.github/workflows/build-publish-mcr.yml @@ -104,10 +104,55 @@ jobs: runs-on: labels: [self-hosted, "1ES.Pool=1es-aks-fleet-pool-ubuntu"] needs: prepare-variables + env: + MEMBER_AGENT_IMAGE_VERSION: ${{ needs.prepare-variables.outputs.release_tag }} + REFRESH_TOKEN_IMAGE_VERSION: ${{ needs.prepare-variables.outputs.release_tag }} + CRD_INSTALLER_IMAGE_VERSION: ${{ needs.prepare-variables.outputs.release_tag }} + MCS_CONTROLLER_IMAGE_VERSION: ${{ needs.prepare-variables.outputs.fleet_networking_version }} + MEMBER_NET_CONTROLLER_IMAGE_VERSION: ${{ needs.prepare-variables.outputs.fleet_networking_version }} steps: - uses: actions/checkout@v6 with: ref: ${{ needs.prepare-variables.outputs.git_ref }} + # The chart comes from the release tag, which can predate a move of the image repository, so pin it to + # the repository the tags were resolved from. + - name: Point the chart at the resolved image repository + run: | + set -euo pipefail + values=charts/member-agent-arc/values.yaml + for pair in memberagent:member-agent refreshtoken:refresh-token crdinstaller:crd-installer \ + mcscontrollermanager:mcs-controller-manager membernetcontrollermanager:member-net-controller-manager; do + key="${pair%%:*}" + repo="mcr.microsoft.com/${MCR_FLEET_REPO}/${pair#*:}" + sed -i "/^${key}:/,/^[^[:space:]]/ s|^\([[:space:]]*repository:\).*|\1 ${repo}|" "$values" + if ! grep -q "repository: ${repo}$" "$values"; then + echo "Failed to set ${key}.repository in ${values}." >&2 + exit 1 + fi + done + - name: Verify every chart image is published + run: | + set -euo pipefail + envsubst < charts/member-agent-arc/values.yaml > "$RUNNER_TEMP/values.yaml" + images=$(helm template charts/member-agent-arc -f "$RUNNER_TEMP/values.yaml" \ + | sed -n 's/^[[:space:]]*image:[[:space:]]*"\{0,1\}\([^"]*\)"\{0,1\}[[:space:]]*$/\1/p' | sort -u) + if [ -z "$images" ]; then + echo "The rendered chart references no images." >&2 + exit 1 + fi + missing=0 + for image in $images; do + ref="${image#mcr.microsoft.com/}" + if [ "$ref" = "$image" ] || ! curl -sfI --retry 3 --retry-delay 5 --retry-all-errors --max-time 30 \ + -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json' \ + "https://mcr.microsoft.com/v2/${ref%:*}/manifests/${ref##*:}" > /dev/null; then + echo "Image not found in MCR: ${image}" >&2 + missing=1 + else + echo "Found ${image}" + fi + done + exit "$missing" - name: 'Login the ACR' run: | az login --identity @@ -119,9 +164,4 @@ jobs: make helm-package-arc-member-cluster-agents env: ARC_MEMBER_AGENT_HELMCHART_VERSION: ${{ needs.prepare-variables.outputs.arc_helmchart_version }} - MEMBER_AGENT_IMAGE_VERSION: ${{ needs.prepare-variables.outputs.release_tag }} - REFRESH_TOKEN_IMAGE_VERSION: ${{ needs.prepare-variables.outputs.release_tag }} - CRD_INSTALLER_IMAGE_VERSION: ${{ needs.prepare-variables.outputs.release_tag }} - MCS_CONTROLLER_IMAGE_VERSION: ${{ needs.prepare-variables.outputs.fleet_networking_version }} - MEMBER_NET_CONTROLLER_IMAGE_VERSION: ${{ needs.prepare-variables.outputs.fleet_networking_version }} REGISTRY: ${{ secrets.AZURE_REGISTRY }}/${{ env.ARC_REGISTRY_REPO}} diff --git a/charts/member-agent-arc/README.md b/charts/member-agent-arc/README.md index a4863168f..9fb5f370c 100644 --- a/charts/member-agent-arc/README.md +++ b/charts/member-agent-arc/README.md @@ -28,6 +28,11 @@ Images are published to MCR as `-` (for example `v0.18.16 on every rebuild of the same source. The workflow selects the highest revision and derives the chart version from it by stripping the leading `v`, so image tag `v0.18.16-1` produces chart version `0.18.16-1`. +The chart is checked out at the plain release tag, which can predate a change to `values.yaml`. The workflow therefore +overwrites each image `repository` with the path the tags were resolved from (`MCR_FLEET_REPO`), and fails before +publishing if any image the rendered chart references is missing from MCR. The repositories in `values.yaml` only +take effect when the chart is packaged outside the workflow. + The image tags in `values.yaml` are placeholders of the form `${MEMBER_AGENT_IMAGE_VERSION}`, substituted by `envsubst` during `make helm-package-arc-member-cluster-agents`. That target rewrites `values.yaml` in place, so after running it locally restore the file with `git checkout charts/member-agent-arc/values.yaml`.