From 012b2c5377d1e90ada85e2cc9cd0376b72d2a685 Mon Sep 17 00:00:00 2001 From: hanwenli Date: Tue, 13 Oct 2026 05:19:14 -0700 Subject: [PATCH] Initial Support for Slurm 26.05 Slurm 26.05 adds two keys to each job in the `SLURM_RESUME_FILE`: - [exclusive](https://github.com/SchedMD/slurm/commit/7f63a8cfe24e79d478b4882bfdf1c47839e630c2) - [priority](https://github.com/SchedMD/slurm/commit/219d033b5e80c9f8164577919315aa77d95d23c6) Changelog: [26.05.0rc1](https://github.com/SchedMD/slurm/blob/476de143d0fd4137fd4606882a68443965529ba3/CHANGELOG/slurm-26.05.md?plain=1#L332) ("Add job priority value to SLURM_RESUME_FILE"), [L465](https://github.com/SchedMD/slurm/blob/476de143d0fd4137fd4606882a68443965529ba3/CHANGELOG/slurm-26.05.md?plain=1#L465) ("Add exclusive to the SLURM_RESUME_FILE"). Code: [power_save.c](https://github.com/SchedMD/slurm/blob/slurm-26-05-5-1/src/slurmctld/power_save.c#L650-L671). `SlurmResumeJob(**job)` raised `TypeError: unexpected keyword argument 'exclusive'`, which failed the whole resume. As a result, ParallelCluster could not launch dynamic nodes with Slurm 26.05. This commit: - [Must have] Accepts `exclusive` and `priority`, and ignores any key a future Slurm adds, so new Resume File fields no longer break scaling. - [Cleanup BTW] Removes `SlurmResumeJob.is_exclusive()`. It had no caller since #574 (job-level scaling for all jobs). Backward compatible: Slurm 25.11 doesn't emit either key, so both default to `None` and parsing is unchanged. Note: this is an initial support for Slurm 26.05 to allow ParallelCluster scale correctly. This commit doesn't consider incorporating new Slurm features. --- CHANGELOG.md | 3 + src/slurm_plugin/slurm_resources.py | 17 ++++-- .../slurm_resources/test_slurm_resources.py | 55 +++++++++++++++++-- 3 files changed, 65 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b5c8031..a52d76a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ This file is used to list changes made in each version of the aws-parallelcluste 3.17.0 ------ +**ENHANCEMENTS** +- Support Slurm 26.05. Previous ParallelCluster versions are not compatible with Slurm 26.05. + **BUG FIXES** - Fix an issue where EC2 throttling during compute node launch is not retried and is reported as insufficient capacity when using Multiple Instance Types or multiple subnets. diff --git a/src/slurm_plugin/slurm_resources.py b/src/slurm_plugin/slurm_resources.py index 19ed0c50..77d9b472 100644 --- a/src/slurm_plugin/slurm_resources.py +++ b/src/slurm_plugin/slurm_resources.py @@ -126,8 +126,16 @@ def __init__( reservation="", features="", extra="", + exclusive=None, + priority=None, + **unknown_fields, ): - """Initialize Slurm job with attributes.""" + """ + Initialize Slurm job with attributes. + + Slurm adds fields to the jobs of the Resume File over time (exclusive and priority in 26.05), so fields this + class does not know are ignored rather than rejected: rejecting them fails the resume of every node. + """ super().__init__( job_id, ) @@ -139,9 +147,10 @@ def __init__( self.reservation = reservation self.features = features self.extra = extra - - def is_exclusive(self): - return self.oversubscribe == JobOversubscribe.NO + self.exclusive = exclusive + self.priority = priority + if unknown_fields: + logger.debug("Ignoring unknown fields of job %s in Slurm Resume File: %s", job_id, unknown_fields) def __eq__(self, other): """Compare 2 SlurmJob objects.""" diff --git a/tests/slurm_plugin/slurm_resources/test_slurm_resources.py b/tests/slurm_plugin/slurm_resources/test_slurm_resources.py index 13b1a41a..73454fd1 100644 --- a/tests/slurm_plugin/slurm_resources/test_slurm_resources.py +++ b/tests/slurm_plugin/slurm_resources/test_slurm_resources.py @@ -18,6 +18,7 @@ DynamicNode, EC2InstanceHealthState, InvalidNodenameError, + JobOversubscribe, MissingInstance, SlurmPartition, SlurmResumeJob, @@ -1528,7 +1529,7 @@ def test_get_node_list(nodenames, expected_node_list, expected_exception): class TestSlurmJob: @pytest.mark.parametrize( - "job_json, expected_nodes_alloc, expected_nodes_resume, expected_is_exclusive, expected_exception", + "job_json, expected_nodes_alloc, expected_nodes_resume, expected_oversubscribe, expected_exception", [ ( { @@ -1543,7 +1544,7 @@ class TestSlurmJob: }, ["queue1-st-c5xlarge-1", "queue1-st-c5xlarge-3", "queue1-st-c5xlarge-4", "queue1-st-c5xlarge-5"], ["queue1-st-c5xlarge-7", "queue1-st-c5xlarge-8"], - True, + JobOversubscribe.NO, False, ), ( @@ -1559,7 +1560,7 @@ class TestSlurmJob: }, ["queue1-st-c5xlarge-1", "queue1-st-c5xlarge-3", "queue1-st-c5xlarge-4", "queue1-st-c5xlarge-5"], ["queue1-st-c5xlarge-7", "queue1-st-c5xlarge-8"], - False, + JobOversubscribe.YES, False, ), ( @@ -1607,13 +1608,51 @@ class TestSlurmJob: }, ["queue1-st-c5xlarge-1"], ["queue1-st-c5xlarge-1"], + JobOversubscribe.OK, + False, + ), + # Slurm 26.05 adds exclusive and priority + ( + { + "exclusive": "NODE", + "extra": None, + "job_id": 140816, + "priority": 1, + "features": None, + "nodes_alloc": "queue1-st-c5xlarge-1", + "nodes_resume": "queue1-st-c5xlarge-1", + "oversubscribe": "NO", + "partition": "cloud_exclusive", + "reservation": None, + }, + ["queue1-st-c5xlarge-1"], + ["queue1-st-c5xlarge-1"], + JobOversubscribe.NO, False, + ), + # Fields a future Slurm adds to the Resume File must not break the resume + ( + { + "exclusive": "TOPO", + "extra": None, + "job_id": 140816, + "features": None, + "nodes_alloc": "queue1-st-c5xlarge-1", + "nodes_resume": "queue1-st-c5xlarge-1", + "oversubscribe": "NO", + "partition": "cloud", + "reservation": None, + "field_added_by_a_future_slurm": "value", + }, + ["queue1-st-c5xlarge-1"], + ["queue1-st-c5xlarge-1"], + JobOversubscribe.NO, False, ), ], ) def test_slurm_job( - self, job_json, expected_nodes_alloc, expected_nodes_resume, expected_is_exclusive, expected_exception + self, job_json, expected_nodes_alloc, expected_nodes_resume, expected_oversubscribe, expected_exception ): if expected_exception: with pytest.raises(InvalidNodenameError): @@ -1622,7 +1661,9 @@ def test_slurm_job( job = SlurmResumeJob(**job_json) assert_that(job.nodes_alloc).is_equal_to(expected_nodes_alloc) assert_that(job.nodes_resume).is_equal_to(expected_nodes_resume) - assert_that(job.is_exclusive()).is_equal_to(expected_is_exclusive) + assert_that(job.oversubscribe).is_equal_to(expected_oversubscribe) + assert_that(job.exclusive).is_equal_to(job_json.get("exclusive")) + assert_that(job.priority).is_equal_to(job_json.get("priority")) @pytest.mark.parametrize( "jobs_json, expected_equal", @@ -1788,7 +1829,9 @@ def test_slurm_job_hash( "partition='cloud_exclusive', " "reservation=None, " "features=None, " - "extra=None)", + "extra=None, " + "exclusive=None, " + "priority=None)", ), ], )