diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b5c8031..a52d76a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ This file is used to list changes made in each version of the aws-parallelcluste 3.17.0 ------ +**ENHANCEMENTS** +- Support Slurm 26.05. Previous ParallelCluster versions are not compatible with Slurm 26.05. + **BUG FIXES** - Fix an issue where EC2 throttling during compute node launch is not retried and is reported as insufficient capacity when using Multiple Instance Types or multiple subnets. diff --git a/src/slurm_plugin/slurm_resources.py b/src/slurm_plugin/slurm_resources.py index 19ed0c50..77d9b472 100644 --- a/src/slurm_plugin/slurm_resources.py +++ b/src/slurm_plugin/slurm_resources.py @@ -126,8 +126,16 @@ def __init__( reservation="", features="", extra="", + exclusive=None, + priority=None, + **unknown_fields, ): - """Initialize Slurm job with attributes.""" + """ + Initialize Slurm job with attributes. + + Slurm adds fields to the jobs of the Resume File over time (exclusive and priority in 26.05), so fields this + class does not know are ignored rather than rejected: rejecting them fails the resume of every node. + """ super().__init__( job_id, ) @@ -139,9 +147,10 @@ def __init__( self.reservation = reservation self.features = features self.extra = extra - - def is_exclusive(self): - return self.oversubscribe == JobOversubscribe.NO + self.exclusive = exclusive + self.priority = priority + if unknown_fields: + logger.debug("Ignoring unknown fields of job %s in Slurm Resume File: %s", job_id, unknown_fields) def __eq__(self, other): """Compare 2 SlurmJob objects.""" diff --git a/tests/slurm_plugin/slurm_resources/test_slurm_resources.py b/tests/slurm_plugin/slurm_resources/test_slurm_resources.py index 13b1a41a..73454fd1 100644 --- a/tests/slurm_plugin/slurm_resources/test_slurm_resources.py +++ b/tests/slurm_plugin/slurm_resources/test_slurm_resources.py @@ -18,6 +18,7 @@ DynamicNode, EC2InstanceHealthState, InvalidNodenameError, + JobOversubscribe, MissingInstance, SlurmPartition, SlurmResumeJob, @@ -1528,7 +1529,7 @@ def test_get_node_list(nodenames, expected_node_list, expected_exception): class TestSlurmJob: @pytest.mark.parametrize( - "job_json, expected_nodes_alloc, expected_nodes_resume, expected_is_exclusive, expected_exception", + "job_json, expected_nodes_alloc, expected_nodes_resume, expected_oversubscribe, expected_exception", [ ( { @@ -1543,7 +1544,7 @@ class TestSlurmJob: }, ["queue1-st-c5xlarge-1", "queue1-st-c5xlarge-3", "queue1-st-c5xlarge-4", "queue1-st-c5xlarge-5"], ["queue1-st-c5xlarge-7", "queue1-st-c5xlarge-8"], - True, + JobOversubscribe.NO, False, ), ( @@ -1559,7 +1560,7 @@ class TestSlurmJob: }, ["queue1-st-c5xlarge-1", "queue1-st-c5xlarge-3", "queue1-st-c5xlarge-4", "queue1-st-c5xlarge-5"], ["queue1-st-c5xlarge-7", "queue1-st-c5xlarge-8"], - False, + JobOversubscribe.YES, False, ), ( @@ -1607,13 +1608,51 @@ class TestSlurmJob: }, ["queue1-st-c5xlarge-1"], ["queue1-st-c5xlarge-1"], + JobOversubscribe.OK, + False, + ), + # Slurm 26.05 adds exclusive and priority + ( + { + "exclusive": "NODE", + "extra": None, + "job_id": 140816, + "priority": 1, + "features": None, + "nodes_alloc": "queue1-st-c5xlarge-1", + "nodes_resume": "queue1-st-c5xlarge-1", + "oversubscribe": "NO", + "partition": "cloud_exclusive", + "reservation": None, + }, + ["queue1-st-c5xlarge-1"], + ["queue1-st-c5xlarge-1"], + JobOversubscribe.NO, False, + ), + # Fields a future Slurm adds to the Resume File must not break the resume + ( + { + "exclusive": "TOPO", + "extra": None, + "job_id": 140816, + "features": None, + "nodes_alloc": "queue1-st-c5xlarge-1", + "nodes_resume": "queue1-st-c5xlarge-1", + "oversubscribe": "NO", + "partition": "cloud", + "reservation": None, + "field_added_by_a_future_slurm": "value", + }, + ["queue1-st-c5xlarge-1"], + ["queue1-st-c5xlarge-1"], + JobOversubscribe.NO, False, ), ], ) def test_slurm_job( - self, job_json, expected_nodes_alloc, expected_nodes_resume, expected_is_exclusive, expected_exception + self, job_json, expected_nodes_alloc, expected_nodes_resume, expected_oversubscribe, expected_exception ): if expected_exception: with pytest.raises(InvalidNodenameError): @@ -1622,7 +1661,9 @@ def test_slurm_job( job = SlurmResumeJob(**job_json) assert_that(job.nodes_alloc).is_equal_to(expected_nodes_alloc) assert_that(job.nodes_resume).is_equal_to(expected_nodes_resume) - assert_that(job.is_exclusive()).is_equal_to(expected_is_exclusive) + assert_that(job.oversubscribe).is_equal_to(expected_oversubscribe) + assert_that(job.exclusive).is_equal_to(job_json.get("exclusive")) + assert_that(job.priority).is_equal_to(job_json.get("priority")) @pytest.mark.parametrize( "jobs_json, expected_equal", @@ -1788,7 +1829,9 @@ def test_slurm_job_hash( "partition='cloud_exclusive', " "reservation=None, " "features=None, " - "extra=None)", + "extra=None, " + "exclusive=None, " + "priority=None)", ), ], )