From 5b1d56d277fc59f698b8967e2972f3de4ce1893f Mon Sep 17 00:00:00 2001 From: yoobinshin Date: Thu, 22 Oct 2026 14:14:19 -0400 Subject: [PATCH 1/4] fix: pin Arc chart image repositories to the resolved MCR path The workflow resolves image tags from aks/v2/fleet but packages the chart from the plain release tag. When that tag predates a change to the chart's image repositories, the published chart pairs the new tags with the old path: 0.18.16-3 was packaged from v0.18.16, one commit before the move to aks/v2/fleet, and referenced aks/fleet/*:v0.18.16-3, which does not exist. Every install hit ImagePullBackOff and the canary rollout gate failed. Overwrite each image repository with MCR_FLEET_REPO before packaging, so the chart always points where the tags were resolved, regardless of the release tag's values.yaml. Then render the chart and check every referenced image against MCR, failing before publish if any is missing. Co-Authored-By: Claude Signed-off-by: yoobinshin --- .github/workflows/build-publish-mcr.yml | 50 ++++++++++++++++++++++--- charts/member-agent-arc/README.md | 5 +++ 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-publish-mcr.yml b/.github/workflows/build-publish-mcr.yml index 552fa8c6e..4f2514af2 100644 --- a/.github/workflows/build-publish-mcr.yml +++ b/.github/workflows/build-publish-mcr.yml @@ -104,10 +104,55 @@ jobs: runs-on: labels: [self-hosted, "1ES.Pool=1es-aks-fleet-pool-ubuntu"] needs: prepare-variables + env: + MEMBER_AGENT_IMAGE_VERSION: ${{ needs.prepare-variables.outputs.release_tag }} + REFRESH_TOKEN_IMAGE_VERSION: ${{ needs.prepare-variables.outputs.release_tag }} + CRD_INSTALLER_IMAGE_VERSION: ${{ needs.prepare-variables.outputs.release_tag }} + MCS_CONTROLLER_IMAGE_VERSION: ${{ needs.prepare-variables.outputs.fleet_networking_version }} + MEMBER_NET_CONTROLLER_IMAGE_VERSION: ${{ needs.prepare-variables.outputs.fleet_networking_version }} steps: - uses: actions/checkout@v6 with: ref: ${{ needs.prepare-variables.outputs.git_ref }} + # The chart comes from the release tag, which can predate a move of the image repository, so pin it to + # the repository the tags were resolved from. + - name: Point the chart at the resolved image repository + run: | + set -euo pipefail + values=charts/member-agent-arc/values.yaml + for pair in memberagent:member-agent refreshtoken:refresh-token crdinstaller:crd-installer \ + mcscontrollermanager:mcs-controller-manager membernetcontrollermanager:member-net-controller-manager; do + key="${pair%%:*}" + repo="mcr.microsoft.com/${MCR_FLEET_REPO}/${pair#*:}" + sed -i "/^${key}:/,/^[^[:space:]]/ s|^\([[:space:]]*repository:\).*|\1 ${repo}|" "$values" + if ! grep -q "repository: ${repo}$" "$values"; then + echo "Failed to set ${key}.repository in ${values}." >&2 + exit 1 + fi + done + - name: Verify every chart image is published + run: | + set -euo pipefail + envsubst < charts/member-agent-arc/values.yaml > "$RUNNER_TEMP/values.yaml" + images=$(helm template charts/member-agent-arc -f "$RUNNER_TEMP/values.yaml" \ + | sed -n 's/^[[:space:]]*image:[[:space:]]*"\{0,1\}\([^"]*\)"\{0,1\}[[:space:]]*$/\1/p' | sort -u) + if [ -z "$images" ]; then + echo "The rendered chart references no images." >&2 + exit 1 + fi + missing=0 + for image in $images; do + ref="${image#mcr.microsoft.com/}" + if [ "$ref" = "$image" ] || ! curl -sfI --retry 3 --retry-delay 5 --retry-all-errors --max-time 30 \ + -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json' \ + "https://mcr.microsoft.com/v2/${ref%:*}/manifests/${ref##*:}" > /dev/null; then + echo "Image not found in MCR: ${image}" >&2 + missing=1 + else + echo "Found ${image}" + fi + done + exit "$missing" - name: 'Login the ACR' run: | az login --identity @@ -119,9 +164,4 @@ jobs: make helm-package-arc-member-cluster-agents env: ARC_MEMBER_AGENT_HELMCHART_VERSION: ${{ needs.prepare-variables.outputs.arc_helmchart_version }} - MEMBER_AGENT_IMAGE_VERSION: ${{ needs.prepare-variables.outputs.release_tag }} - REFRESH_TOKEN_IMAGE_VERSION: ${{ needs.prepare-variables.outputs.release_tag }} - CRD_INSTALLER_IMAGE_VERSION: ${{ needs.prepare-variables.outputs.release_tag }} - MCS_CONTROLLER_IMAGE_VERSION: ${{ needs.prepare-variables.outputs.fleet_networking_version }} - MEMBER_NET_CONTROLLER_IMAGE_VERSION: ${{ needs.prepare-variables.outputs.fleet_networking_version }} REGISTRY: ${{ secrets.AZURE_REGISTRY }}/${{ env.ARC_REGISTRY_REPO}} diff --git a/charts/member-agent-arc/README.md b/charts/member-agent-arc/README.md index a4863168f..9fb5f370c 100644 --- a/charts/member-agent-arc/README.md +++ b/charts/member-agent-arc/README.md @@ -28,6 +28,11 @@ Images are published to MCR as `-` (for example `v0.18.16 on every rebuild of the same source. The workflow selects the highest revision and derives the chart version from it by stripping the leading `v`, so image tag `v0.18.16-1` produces chart version `0.18.16-1`. +The chart is checked out at the plain release tag, which can predate a change to `values.yaml`. The workflow therefore +overwrites each image `repository` with the path the tags were resolved from (`MCR_FLEET_REPO`), and fails before +publishing if any image the rendered chart references is missing from MCR. The repositories in `values.yaml` only +take effect when the chart is packaged outside the workflow. + The image tags in `values.yaml` are placeholders of the form `${MEMBER_AGENT_IMAGE_VERSION}`, substituted by `envsubst` during `make helm-package-arc-member-cluster-agents`. That target rewrites `values.yaml` in place, so after running it locally restore the file with `git checkout charts/member-agent-arc/values.yaml`. From 83dd5ab70699840ba08f7229ffd116c3d09d5140 Mon Sep 17 00:00:00 2001 From: yoobinshin Date: Thu, 22 Oct 2026 14:39:17 -0400 Subject: [PATCH 2/4] fix: drop curl --retry-all-errors from the image check The self-hosted 1ES runner's curl predates --retry-all-errors, so every manifest probe exited on the unknown option and each image was reported missing. Read the HTTP status instead of relying on -f, which also surfaces the status in the failure message. Co-Authored-By: Claude Signed-off-by: yoobinshin --- .github/workflows/build-publish-mcr.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-publish-mcr.yml b/.github/workflows/build-publish-mcr.yml index 4f2514af2..8d12fee03 100644 --- a/.github/workflows/build-publish-mcr.yml +++ b/.github/workflows/build-publish-mcr.yml @@ -143,13 +143,20 @@ jobs: missing=0 for image in $images; do ref="${image#mcr.microsoft.com/}" - if [ "$ref" = "$image" ] || ! curl -sfI --retry 3 --retry-delay 5 --retry-all-errors --max-time 30 \ - -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json' \ - "https://mcr.microsoft.com/v2/${ref%:*}/manifests/${ref##*:}" > /dev/null; then - echo "Image not found in MCR: ${image}" >&2 + if [ "$ref" = "$image" ]; then + echo "Image is not hosted on MCR: ${image}" >&2 missing=1 - else + continue + fi + # The self-hosted runner's curl predates --retry-all-errors. + status=$(curl -s -o /dev/null -w '%{http_code}' -I --retry 3 --retry-delay 5 --max-time 30 \ + -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json' \ + "https://mcr.microsoft.com/v2/${ref%:*}/manifests/${ref##*:}") || true + if [ "$status" = "200" ]; then echo "Found ${image}" + else + echo "Image not found in MCR (HTTP ${status:-none}): ${image}" >&2 + missing=1 fi done exit "$missing" From d5b9207a1a34c5a6b937a61dcea1598c1e4470f9 Mon Sep 17 00:00:00 2001 From: yoobinshin Date: Thu, 22 Oct 2026 14:45:37 -0400 Subject: [PATCH 3/4] refactor: rewrite Arc chart image repositories with one sed Each repository already ends in its image name, so swapping the prefix needs no key-to-image map, and the image check that follows fails the run if a rewrite misses. Co-Authored-By: Claude Signed-off-by: yoobinshin --- .github/workflows/build-publish-mcr.yml | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/.github/workflows/build-publish-mcr.yml b/.github/workflows/build-publish-mcr.yml index 8d12fee03..6abee694a 100644 --- a/.github/workflows/build-publish-mcr.yml +++ b/.github/workflows/build-publish-mcr.yml @@ -117,19 +117,7 @@ jobs: # The chart comes from the release tag, which can predate a move of the image repository, so pin it to # the repository the tags were resolved from. - name: Point the chart at the resolved image repository - run: | - set -euo pipefail - values=charts/member-agent-arc/values.yaml - for pair in memberagent:member-agent refreshtoken:refresh-token crdinstaller:crd-installer \ - mcscontrollermanager:mcs-controller-manager membernetcontrollermanager:member-net-controller-manager; do - key="${pair%%:*}" - repo="mcr.microsoft.com/${MCR_FLEET_REPO}/${pair#*:}" - sed -i "/^${key}:/,/^[^[:space:]]/ s|^\([[:space:]]*repository:\).*|\1 ${repo}|" "$values" - if ! grep -q "repository: ${repo}$" "$values"; then - echo "Failed to set ${key}.repository in ${values}." >&2 - exit 1 - fi - done + run: sed -i "s|^\([[:space:]]*repository:\).*/\([^/]*\)$|\1 mcr.microsoft.com/${MCR_FLEET_REPO}/\2|" charts/member-agent-arc/values.yaml - name: Verify every chart image is published run: | set -euo pipefail From 517c3c77146051c74e1c56d73ac598638cc09b20 Mon Sep 17 00:00:00 2001 From: yoobinshin Date: Thu, 22 Oct 2026 16:27:07 -0400 Subject: [PATCH 4/4] fix: rewrite only the legacy Arc chart image prefix Replace mcr.microsoft.com/aks/fleet/ rather than every repository value, so charts already on the current path and images hosted elsewhere are left untouched. The step is a compatibility shim for publishing new image revisions from release tags whose chart predates the move to aks/v2/fleet. Co-Authored-By: Claude Signed-off-by: yoobinshin --- .github/workflows/build-publish-mcr.yml | 8 ++++---- charts/member-agent-arc/README.md | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-publish-mcr.yml b/.github/workflows/build-publish-mcr.yml index 6abee694a..a9e90c9b6 100644 --- a/.github/workflows/build-publish-mcr.yml +++ b/.github/workflows/build-publish-mcr.yml @@ -114,10 +114,10 @@ jobs: - uses: actions/checkout@v6 with: ref: ${{ needs.prepare-variables.outputs.git_ref }} - # The chart comes from the release tag, which can predate a move of the image repository, so pin it to - # the repository the tags were resolved from. - - name: Point the chart at the resolved image repository - run: sed -i "s|^\([[:space:]]*repository:\).*/\([^/]*\)$|\1 mcr.microsoft.com/${MCR_FLEET_REPO}/\2|" charts/member-agent-arc/values.yaml + # Compatibility for release tags whose chart still uses the legacy MCR prefix. + # Keep while publishing new image revisions from those tags is supported. + - name: Update legacy chart image repositories + run: sed -i "s|^\([[:space:]]*repository:[[:space:]]*\)mcr\.microsoft\.com/aks/fleet/|\1mcr.microsoft.com/${MCR_FLEET_REPO}/|" charts/member-agent-arc/values.yaml - name: Verify every chart image is published run: | set -euo pipefail diff --git a/charts/member-agent-arc/README.md b/charts/member-agent-arc/README.md index 9fb5f370c..6d01c2f63 100644 --- a/charts/member-agent-arc/README.md +++ b/charts/member-agent-arc/README.md @@ -28,10 +28,10 @@ Images are published to MCR as `-` (for example `v0.18.16 on every rebuild of the same source. The workflow selects the highest revision and derives the chart version from it by stripping the leading `v`, so image tag `v0.18.16-1` produces chart version `0.18.16-1`. -The chart is checked out at the plain release tag, which can predate a change to `values.yaml`. The workflow therefore -overwrites each image `repository` with the path the tags were resolved from (`MCR_FLEET_REPO`), and fails before -publishing if any image the rendered chart references is missing from MCR. The repositories in `values.yaml` only -take effect when the chart is packaged outside the workflow. +The chart is checked out at the plain release tag, which can predate the move to `aks/v2/fleet`. The workflow rewrites +the legacy `mcr.microsoft.com/aks/fleet/` prefix to `MCR_FLEET_REPO`, which is needed only while new image revisions +are published from release tags whose chart still uses that prefix. It then fails before publishing if any image the +rendered chart references is missing from MCR. The image tags in `values.yaml` are placeholders of the form `${MEMBER_AGENT_IMAGE_VERSION}`, substituted by `envsubst` during `make helm-package-arc-member-cluster-agents`. That target rewrites `values.yaml` in place, so