[3.16.2] Cherry picks from develop to release-3.16

.github/workflows/bump_version.yml

@@ -6,7 +6,7 @@ on:
     # Inputs the workflow accepts.
     inputs:
       pcluster-version:
-        description: 'The target version of ParallelCluster CLI'
+        description: 'The target version of ParallelCluster Node'
         required: true
         type: string
       branch:

.github/workflows/cherry-pick-to-branch.yml

@@ -0,0 +1,201 @@
+name: Cherry-pick to branch
+
+on:
+  workflow_dispatch:
+    inputs:
+      pr_numbers:
+        description: "Space-separated list of PR numbers to cherry-pick"
+        required: true
+        type: string
+      target_branches:
+        description: "Space-separated list of target branches to cherry-pick into"
+        required: true
+        type: string
+
+jobs:
+  cherry-pick:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+      - name: Get PR details
+        id: pr-info
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PR_NUMBERS: ${{ inputs.pr_numbers }}
+          TARGET_BRANCHES: ${{ inputs.target_branches }}
+          REPO: ${{ github.repository }}
+        run: |
+          # workflow_dispatch has no PR event payload, so fetch everything we need
+          # about each PR from the API.
+
+          # Space-separated list of PR numbers. PRs are processed in the order
+          # they are provided, so commits are cherry-picked in that same order.
+          PR_LIST=$(echo "${PR_NUMBERS}" | tr -s ' ' | sed 's/^ *//;s/ *$//')
+          for PR in ${PR_LIST}; do
+            if ! echo "${PR}" | grep -qE '^[0-9]+$'; then
+              echo "::error::Invalid PR number '${PR}' — must be numeric"
+              exit 1
+            fi
+          done
+
+          if [ -z "${PR_LIST// }" ]; then
+            echo "::error::No PR numbers provided"
+            exit 1
+          fi
+
+          # Validate target branches contain only safe characters
+          for BRANCH in ${TARGET_BRANCHES}; do
+            if ! echo "${BRANCH}" | grep -qE '^[a-zA-Z0-9._/-]+$'; then
+              echo "::error::Invalid branch name '${BRANCH}' — must match [a-zA-Z0-9._/-]+"
+              exit 1
+            fi
+          done
+
+          ALL_COMMITS=""
+          PR_DETAILS=""
+          PR_DISPLAY=""
+          FIRST_TITLE=""
+          for PR in ${PR_LIST}; do
+            PR_JSON=$(gh pr view "${PR}" --repo "${REPO}" \
+              --json baseRefName,title,author)
+            BASE_BRANCH=$(echo "${PR_JSON}" | jq -r '.baseRefName')
+            PR_TITLE=$(echo "${PR_JSON}" | jq -r '.title')
+            PR_AUTHOR=$(echo "${PR_JSON}" | jq -r '.author.login')
+
+            # A PR can contain merges from its base branch. Those commits should not be cherry-picked.
+            # The types of commit are distinguished by the number of parents:
+            # 1 parent = non-merge commits, 2 parents = merge commit
+            COMMITS=$(gh api --paginate \
+              "repos/${REPO}/pulls/${PR}/commits?per_page=100" \
+              --jq '.[] | select((.parents | length) < 2) | .sha' | tr '\n' ' ')
+
+            if [ -z "${COMMITS// }" ]; then
+              echo "::error::PR #${PR} contains no non-merge commits to cherry-pick"
+              exit 1
+            fi
+
+            ALL_COMMITS="${ALL_COMMITS} ${COMMITS}"
+            PR_DETAILS="${PR_DETAILS}- #${PR} \"${PR_TITLE}\" by @${PR_AUTHOR} (from \`${BASE_BRANCH}\`)"$'\n'
+
+            if [ -z "${PR_DISPLAY}" ]; then
+              PR_DISPLAY="#${PR}"
+              FIRST_TITLE="${PR_TITLE}"
+            else
+              PR_DISPLAY="${PR_DISPLAY}, #${PR}"
+            fi
+          done
+
+          # Trim leading whitespace from the accumulated commit list
+          ALL_COMMITS=$(echo "${ALL_COMMITS}" | sed 's/^ *//')
+
+          # Drop the trailing newline so the rendered body has no blank last line
+          PR_DETAILS="${PR_DETAILS%$'\n'}"
+
+          # A single PR (no space in the list) keeps its title in the PR name;
+          # multiple PRs reference the first one followed by "and more".
+          FIRST_PR="${PR_LIST%% *}"
+          case "${PR_LIST}" in
+            *" "*) NEW_PR_TITLE="Cherry-pick #${FIRST_PR} and more" ;;
+            *)     NEW_PR_TITLE="Cherry-pick ${PR_DISPLAY}: ${FIRST_TITLE}" ;;
+          esac
+
+          echo "commits=${ALL_COMMITS}" >> "$GITHUB_OUTPUT"
+          echo "pr_numbers=${PR_LIST}" >> "$GITHUB_OUTPUT"
+          echo "pr_display=${PR_DISPLAY}" >> "$GITHUB_OUTPUT"
+          echo "pr_title=${NEW_PR_TITLE}" >> "$GITHUB_OUTPUT"
+          {
+            echo "pr_details<<CHERRY_PICK_EOF"
+            echo "${PR_DETAILS}"
+            echo "CHERRY_PICK_EOF"
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Configure git
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+      - name: Cherry-pick and create PR(s)
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COMMITS: ${{ steps.pr-info.outputs.commits }}
+          PR_NUMBERS: ${{ steps.pr-info.outputs.pr_numbers }}
+          PR_DISPLAY: ${{ steps.pr-info.outputs.pr_display }}
+          PR_TITLE: ${{ steps.pr-info.outputs.pr_title }}
+          PR_DETAILS: ${{ steps.pr-info.outputs.pr_details }}
+          TARGET_BRANCHES: ${{ inputs.target_branches }}
+          REPO: ${{ github.repository }}
+        run: |
+          # Fetch each PR's commits. They may live in a fork (different account),
+          # so they are not present in the base repo checkout. The refs/pull/*/head
+          # ref makes them available regardless of which fork the PR came from.
+          for PR in ${PR_NUMBERS}; do
+            git fetch origin "refs/pull/${PR}/head"
+          done
+
+          # Name the branch after the first and last PR in the set (a single PR
+          # keeps the plain pr-<n> form) so it stays within git's ref length limit
+          # regardless of how many PRs are cherry-picked.
+          FIRST_PR="${PR_NUMBERS%% *}"
+          LAST_PR="${PR_NUMBERS##* }"
+          case "${PR_NUMBERS}" in
+            *" "*) BRANCH_PR="pr-${FIRST_PR}-and-${LAST_PR}" ;;
+            *)     BRANCH_PR="pr-${FIRST_PR}" ;;
+          esac
+
+          HAS_FAILURE=false
+          for TARGET_BRANCH in ${TARGET_BRANCHES}; do
+            CHERRY_PICK_BRANCH="cherry-pick/${BRANCH_PR}-to-${TARGET_BRANCH}"
+
+            echo "Cherry-picking commits from PR(s) ${PR_DISPLAY} to ${TARGET_BRANCH}"
+            echo "Commits: ${COMMITS}"
+
+            git fetch origin "${TARGET_BRANCH}"
+            git checkout -b "${CHERRY_PICK_BRANCH}" "origin/${TARGET_BRANCH}"
+
+            # Cherry-pick all commits in order (works for both squash and rebase)
+            CHERRY_PICK_FAILED=false
+            for COMMIT in ${COMMITS}; do
+              if ! git cherry-pick "${COMMIT}" --no-edit; then
+                CHERRY_PICK_FAILED=true
+                git cherry-pick --abort
+                break
+              fi
+            done
+
+            if [ "${CHERRY_PICK_FAILED}" = "false" ]; then
+              git push origin "${CHERRY_PICK_BRANCH}"
+              PR_BODY=$(printf '%s\n\nIncluded PRs:\n%s' \
+                "Automated cherry-pick of ${PR_DISPLAY} to \`${TARGET_BRANCH}\`." \
+                "${PR_DETAILS}")
+              NEW_PR_URL=$(gh pr create \
+                --base "${TARGET_BRANCH}" \
+                --head "${CHERRY_PICK_BRANCH}" \
+                --title "${PR_TITLE}" \
+                --body "${PR_BODY}" \
+                --label "cherry-pick")
+              for PR in ${PR_NUMBERS}; do
+                gh pr comment "${PR}" --repo "${REPO}" \
+                  --body "✅ Cherry-pick to \`${TARGET_BRANCH}\` opened: ${NEW_PR_URL}"
+              done
+              echo "✅ Cherry-pick PR to ${TARGET_BRANCH} created successfully: ${NEW_PR_URL}"
+            else
+              HAS_FAILURE=true
+              echo "::error::Cherry-pick to ${TARGET_BRANCH} failed due to conflicts."
+            fi
+
+            # Clean up for next iteration
+            git checkout --detach 2>/dev/null || true
+            git branch -D "${CHERRY_PICK_BRANCH}" 2>/dev/null || true
+          done
+
+          if [ "${HAS_FAILURE}" = "true" ]; then
+            exit 1
+          fi

.github/workflows/ci.yml

@@ -26,40 +26,20 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         name:
-          - Python 3.9 Tests
-          - Python 3.10 Tests
-          - Python 3.11 Tests
-          - Python 3.12 Tests
-          - Python 3.13 Tests
-          - Python 3.12 Tests Coverage
+          - Python 3.14 Tests
+          - Python 3.14 Tests Coverage
           - Code Checks
         include:
-          - name: Python 3.9 Tests
-            python: 3.9
+          - name: Python 3.14 Tests
+            python: '3.14'
             toxdir: cli
-            toxenv: py39-nocov
-          - name: Python 3.10 Tests
-            python: '3.10'
+            toxenv: py314-nocov
+          - name: Python 3.14 Tests Coverage
+            python: '3.14'
             toxdir: cli
-            toxenv: py310-nocov
-          - name: Python 3.11 Tests
-            python: '3.11'
-            toxdir: cli
-            toxenv: py311-nocov
-          - name: Python 3.12 Tests
-            python: '3.12'
-            toxdir: cli
-            toxenv: py312-nocov
-          - name: Python 3.13 Tests
-            python: '3.13'
-            toxdir: cli
-            toxenv: py313-nocov
-          - name: Python 3.12 Tests Coverage
-            python: 3.12
-            toxdir: cli
-            toxenv: py312-cov
+            toxenv: py314-cov
           - name: Code Checks
-            python: 3.12
+            python: '3.14'
             toxdir: cli
             toxenv: code-linters
 

CHANGELOG.md

@@ -3,6 +3,15 @@ aws-parallelcluster-node CHANGELOG
 
 This file is used to list changes made in each version of the aws-parallelcluster-node package.
 
+3.16.2
+------
+
+**BUG FIXES**
+- Fix an issue where EC2 throttling during compute node launch is not retried and is reported as insufficient capacity
+  when using Multiple Instance Types or multiple subnets.
+- Fix an issue where the Comment and Extra fields of Slurm nodes and the Comment field of Slurm reservations
+  could alter the node and reservation information read by ParallelCluster.
+
 3.16.1
 ------
 

setup.py

@@ -44,7 +44,7 @@ def read(fname):
     license="Apache License 2.0",
     packages=find_packages("src", exclude=["tests"]),
     package_dir={"": "src"},
-    python_requires=">=3.9",
+    python_requires=">=3.14",
     install_requires=requires,
     entry_points=dict(console_scripts=console_scripts),
     zip_safe=False,

src/common/schedulers/slurm_commands.py

@@ -75,10 +75,15 @@
     r"|(LastBusyTime=\S+)"
     r"|(ReservationName=\S+)"
     r"|(InstanceId=\S+)"
-    r"|(Reason=.*)",
+    r"|(Reason=.*)"
+    r"|(Comment=.*)"
+    r"|(Extra=.*)",
     re.MULTILINE,
 )
 
+# Fields matched up to the end of the line and discarded, to prevent their free-text content from being parsed as fields
+SCONTROL_DISCARDED_FIELDS = ("Comment", "Extra")
+
 # Fields extracted from `scontrol show partitions` output. `(?<!Next)` ensures `State` is matched but
 # `NextState` is not.
 SCONTROL_PARTITION_INFO_FIELD_REGEX = re.compile(
@@ -426,6 +431,8 @@ def _extract_scontrol_records(raw_output: str, field_regex) -> List[Dict[str, st
         fields = {}
         for match in field_regex.finditer(record):
             key, _, value = match.group(0).partition("=")
+            if key in SCONTROL_DISCARDED_FIELDS:
+                continue
             fields[key] = value
         if fields:
             records.append(fields)

src/common/schedulers/slurm_reservation_commands.py

@@ -39,7 +39,7 @@
 # Fields extracted from raw `scontrol show reservations` output. Only `ReservationName` is anchored at the
 # start of a line; `(?<!Next)` ensures `State` is matched but `NextState` is not.
 SCONTROL_RESERVATION_INFO_FIELD_REGEX = re.compile(
-    r"^(ReservationName=\S+)" r"|(?<!Next)(State=\S+)" r"|(Users=\S+)" r"|(Nodes=\S+)",
+    r"^(ReservationName=\S+)" r"|(?<!Next)(State=\S+)" r"|(Users=\S+)" r"|(Nodes=\S+)" r"|(Comment=.*)",
     re.MULTILINE,
 )