From 8944ba0e120b238109ec1066bf59761ee142df27 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Mon, 12 Oct 2026 09:33:26 -0600 Subject: [PATCH 1/6] fix(slurm): match whole lines in the epilog SSH access cleanup (#1407) 41-lastuserjob-ssh used 'grep -w' for the localusers.backup exemption and an unanchored 'sed "/${SLURM_JOB_USER}/d"' on /etc/localusers. Both match substrings and regular expressions: a job user 'al' removed the SSH allow entry of 'alice' and 'halley', a dot matched any character, and 'je' was treated as exempt when 'je.kim' was listed. Match the exact line, literally, in both places (the same -x -F form scripts 40 and 42 already use). Adds six cases to tests/slurm-epilog/run-tests.sh. Signed-off-by: Doug Holt --- .../etc/slurm/epilog.d/41-lastuserjob-ssh | 15 +++- tests/slurm-epilog/run-tests.sh | 79 +++++++++++++++++++ 2 files changed, 92 insertions(+), 2 deletions(-) diff --git a/roles/slurm/templates/etc/slurm/epilog.d/41-lastuserjob-ssh b/roles/slurm/templates/etc/slurm/epilog.d/41-lastuserjob-ssh index d25cb7117..90dcbba10 100755 --- a/roles/slurm/templates/etc/slurm/epilog.d/41-lastuserjob-ssh +++ b/roles/slurm/templates/etc/slurm/epilog.d/41-lastuserjob-ssh @@ -1,8 +1,19 @@ #!/usr/bin/env bash set -ex -if grep -q -w "$SLURM_JOB_USER" /etc/slurm/localusers.backup ; then +# Both lookups below act on whole lines. `grep -w` and an unanchored +# `sed "/${user}/d"` match substrings and regular expressions instead: with a +# job user `al`, the sed deleted the SSH allow entry of `alice`, `halley` and +# every other account containing "al", and a dot in a name (`je.kim`) matched +# any character. Match the exact line, literally, in both places. +localusers_backup=/etc/slurm/localusers.backup +localusers_live=/etc/localusers + +if grep -qxF -- "$SLURM_JOB_USER" "$localusers_backup" ; then exit 0 # don't revoke access for these users fi -sed -i "/${SLURM_JOB_USER}/d" /etc/localusers +# Escape the characters that are special in a sed regular expression (and the +# `/` delimiter) so the account name is matched literally. +user_pattern=$(printf '%s' "$SLURM_JOB_USER" | sed -e 's/[][\.*^$/]/\\&/g') +sed -i "/^${user_pattern}\$/d" "$localusers_live" diff --git a/tests/slurm-epilog/run-tests.sh b/tests/slurm-epilog/run-tests.sh index 7b6f707b3..98fc992ec 100755 --- a/tests/slurm-epilog/run-tests.sh +++ b/tests/slurm-epilog/run-tests.sh @@ -608,6 +608,85 @@ pathname_case "a mount point containing a tab is decoded" $'data fi # identity and /proc/self/mountinfo +# --------------------------------------------------------------------------- +# 41-lastuserjob-ssh: which SSH allow entries are removed +# --------------------------------------------------------------------------- +# +# The script edits two files, the live /etc/localusers and the localusers.backup +# exemption list. Both lookups used to match substrings and regular +# expressions: `grep -w` treated `je` as listed when `je.kim` was (the dot is +# not a word character), and `sed "/${user}/d"` deleted every entry that merely +# contained the name. The cases below pin whole-line, literal matching. +echo +echo "41-lastuserjob-ssh -- exact match" + +setup_ssh() { + local dir="$work/ssh" + rm -rf "$dir" + mkdir -p "$dir" + write_logger_stub "$dir" + render "epilog.d/41-lastuserjob-ssh" "$dir/41-ssh" || return 1 + sed -i "s#^localusers_backup=.*#localusers_backup='$dir/localusers.backup'#" "$dir/41-ssh" + sed -i "s#^localusers_live=.*#localusers_live='$dir/localusers'#" "$dir/41-ssh" + if ! grep -qF "localusers_backup='$dir/localusers.backup'" "$dir/41-ssh" \ + || ! grep -qF "localusers_live='$dir/localusers'" "$dir/41-ssh"; then + printf ' FAIL harness: the 41 paths were not redirected into the fixture\n' >&2 + return 1 + fi + printf '%s' "$dir" +} + +ssh_case() { + # $1 = label, $2 = job user, $3 = backup list, $4 = live list, + # $5 = expected live list afterwards + local label="$1" user="$2" backup="$3" live="$4" want="$5" dir + dir="$(setup_ssh)" || { no "$label" "harness setup failed"; return; } + printf '%s\n' "$backup" > "$dir/localusers.backup" + printf '%s\n' "$live" > "$dir/localusers" + local out rc got + out=$(PATH="$dir/bin:$PATH" SLURM_JOB_USER="$user" "$dir/41-ssh" 2>&1); rc=$? + got="$(cat "$dir/localusers")" + if [ "$rc" != 0 ]; then + no "$label" "rc=$rc out=$(printf '%s' "$out" | tail -1)" + elif [ "$got" != "$want" ]; then + no "$label" "live list is '$(printf '%s' "$got" | tr '\n' ' ')', wanted '$(printf '%s' "$want" | tr '\n' ' ')'" + else + ok "$label" + fi +} + +ssh_case "the job user's own entry is removed" \ + alice "root" "root +alice +bob" "root +bob" +ssh_case "a name contained in other entries removes only its own line" \ + al "root" "root +al +alice +halley" "root +alice +halley" +ssh_case "a dot in the name is not a wildcard" \ + je.kim "root" "root +je.kim +jeXkim" "root +jeXkim" +ssh_case "a name with sed metacharacters is matched literally" \ + 'a*b' "root" "root +a*b +aab" "root +aab" +ssh_case "a listed operator keeps the entry" \ + alice "alice" "root +alice" "root +alice" +ssh_case "a name that is only a prefix of a listed operator is not exempt" \ + je "je.kim" "root +je +je.kim" "root +je.kim" + echo printf '%d passed, %d failed\n' "$pass" "$fail" [ "$fail" = 0 ] From c8960e7d80652dcc07cc983e07dcaa14d1ea8f20 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Mon, 12 Oct 2026 10:14:26 -0600 Subject: [PATCH 2/6] feat(slurm): job-scoped epilog cleanup, default on, with a user escape hatch (#1407) The *-lastuserjob-* epilog scripts decided what to remove from squeue alone and killed every process the user owned, so a job of the same user placed on the node just after squeue answered (CompleteWait=0), or one the controller was slow to report, could lose its processes. New role variable slurm_epilog_cleanup_scope, rendered at deploy time: job (default) The dispatcher also scans /proc//cgroup for the user's processes in any job_ cgroup other than the ending job's (cgroup v1 and v2 layouts) and skips the last-user-job scripts when it finds one. 40-lastuserjob-processes replaces killall -9 -u with a PID list classified by cgroup: the ending job's processes are killed, another job's never, orphans only when the user has no other job. 42 skips when job_container/tmpfs is configured. Anything that cannot be read or recognised is treated as 'another job may be here': the cleanup is deferred, never guessed, and the epilog never fails. user DEPRECATED, removed in 27.01. Renders the 26.09 scripts byte-for-byte. The playbook validates the value and warns when it is user. Signed-off-by: Doug Holt --- roles/slurm/defaults/main.yml | 13 ++ roles/slurm/tasks/compute.yml | 52 +++++++ .../slurm/epilog.d/40-lastuserjob-processes | 96 +++++++++++++ .../etc/slurm/epilog.d/42-lastuserjob-cleanup | 17 +++ .../etc/slurm/shared/bin/run-parts.sh | 64 +++++++++ .../etc/slurm/shared/lib/epilog-job-scope.sh | 133 ++++++++++++++++++ 6 files changed, 375 insertions(+) create mode 100644 roles/slurm/templates/etc/slurm/shared/lib/epilog-job-scope.sh diff --git a/roles/slurm/defaults/main.yml b/roles/slurm/defaults/main.yml index b96f63491..601c61951 100644 --- a/roles/slurm/defaults/main.yml +++ b/roles/slurm/defaults/main.yml @@ -91,6 +91,19 @@ slurm_enable_prolog_epilog: true # Useful to delete old files (e.g. from previous versions of deepops) slurm_clear_old_prolog_epilog: false +# How the epilog's "last job of the user" cleanup (scripts 40-lastuserjob-* +# through 50-lastuserjob-*) decides what to remove when a job ends. +# job: (default) job-scoped. The cleanup also checks the node's own cgroups +# and never touches processes of another job of the same user; files and +# enroot directories are removed only when the user has no other job on +# the node. Requires cgroup process tracking (ProctrackType=proctrack/cgroup, +# the default in this role); cgroup v1 and v2 are both understood. +# user: DEPRECATED. The 26.09 behaviour, rendered byte-for-byte: the cleanup +# is decided by squeue alone and kills every process the user owns. +# Kept for one release train as an escape hatch; removed in 27.01. +# See docs/slurm-cluster/epilog-job-scoped-cleanup.md +slurm_epilog_cleanup_scope: job + # Fileglob patterns used to copy prolog and epilog files prolog_fileglob: - "templates/etc/slurm/prolog.d/*" diff --git a/roles/slurm/tasks/compute.yml b/roles/slurm/tasks/compute.yml index 242bc914e..0f640df42 100644 --- a/roles/slurm/tasks/compute.yml +++ b/roles/slurm/tasks/compute.yml @@ -82,6 +82,17 @@ notify: restart slurmd tags: config +- name: validate slurm_epilog_cleanup_scope + assert: + that: + - slurm_epilog_cleanup_scope in ['job', 'user'] + fail_msg: >- + slurm_epilog_cleanup_scope must be 'job' or 'user' + (got '{{ slurm_epilog_cleanup_scope }}'). + tags: + - prolog + - epilog + - name: copy shared scripts template: src: "{{ item }}" @@ -93,6 +104,47 @@ - prolog - epilog +- name: warn that slurm_epilog_cleanup_scope user is deprecated + debug: + msg: >- + slurm_epilog_cleanup_scope: user restores the 26.09 epilog cleanup and is + deprecated. It will be removed in 27.01; see + docs/slurm-cluster/epilog-job-scoped-cleanup.md. + when: slurm_epilog_cleanup_scope == 'user' + tags: + - prolog + - epilog + +- name: create shared library directory for job-scoped epilog cleanup + file: + path: "{{ slurm_config_dir }}/shared/lib" + state: directory + owner: slurm + mode: 0755 + when: slurm_epilog_cleanup_scope == 'job' + tags: + - prolog + - epilog + +- name: copy job-scoped epilog helpers + template: + src: etc/slurm/shared/lib/epilog-job-scope.sh + dest: "{{ slurm_config_dir }}/shared/lib/epilog-job-scope.sh" + mode: 0644 + when: slurm_epilog_cleanup_scope == 'job' + tags: + - prolog + - epilog + +- name: remove job-scoped epilog helpers when the scope is user + file: + path: "{{ slurm_config_dir }}/shared/lib/epilog-job-scope.sh" + state: absent + when: slurm_epilog_cleanup_scope == 'user' + tags: + - prolog + - epilog + - name: include prolog/epilog include_tasks: prolog_epilog.yml when: slurm_enable_prolog_epilog diff --git a/roles/slurm/templates/etc/slurm/epilog.d/40-lastuserjob-processes b/roles/slurm/templates/etc/slurm/epilog.d/40-lastuserjob-processes index b8d82f19c..492cf80ce 100755 --- a/roles/slurm/templates/etc/slurm/epilog.d/40-lastuserjob-processes +++ b/roles/slurm/templates/etc/slurm/epilog.d/40-lastuserjob-processes @@ -36,8 +36,104 @@ case "$lookup_rc" in ;; esac +{% if slurm_epilog_cleanup_scope == 'job' %} +# Job-scoped cleanup (slurm_epilog_cleanup_scope: job). +# +# `killall -9 -u` reaps every process the user owns on the node, whichever job +# it belongs to. Here each process is classified by the job cgroup it sits in +# (/proc//cgroup, see shared/lib/epilog-job-scope.sh) and only the ones +# that are this job's, or that belong to no job and the user has no job left, +# are killed: +# +# in the ending job's cgroup kill (Slurm does this too; belt and braces) +# in any other job's cgroup never kill, whatever squeue said +# in no job cgroup (orphan) kill only when the dispatcher found no other +# job of this user, by squeue and by cgroup +# cgroup line not understood skip and log +# +# Anything that cannot be established is a skip, never a kill, and the script +# never fails the epilog: a failing epilog drains the node. +kill_bin=kill +epilog_proc_root=/proc +job_scope_lib='{{ slurm_config_dir }}/shared/lib/epilog-job-scope.sh' + +if [ "$SLURM_JOB_USER" = root ]; then + exit 0 +fi + +if [ ! -r "$job_scope_lib" ] || ! . "$job_scope_lib"; then + logger -s -t slurm-epilog \ + "cannot load ${job_scope_lib}; skipping process cleanup" + exit 0 +fi + +if ! epilog_resolve_uid || [ "$epilog_uid" = 0 ]; then + logger -s -t slurm-epilog \ + "no usable uid for ${SLURM_JOB_USER}; skipping process cleanup" + exit 0 +fi + +if ! pids=$(epilog_user_pids "$epilog_uid"); then + logger -s -t slurm-epilog \ + "cannot list processes under ${epilog_proc_root}; skipping process cleanup" + exit 0 +fi + +# The dispatcher exports 1 only when neither squeue nor the cgroup scan found +# another job of this user. Run on its own the script treats orphans as live. +last_user_job="${EPILOG_LAST_USER_JOB:-0}" + +killed=0 +spared=0 +unsure=0 +for pid in $pids; do + [ "$pid" != "$$" ] || continue + + rc=0 + jobs=$(epilog_pid_jobs "$pid") || rc=$? + case "$rc" in + 0) ;; + 1) continue ;; # exited since the listing + *) + logger -s -t slurm-epilog \ + "cannot classify pid ${pid} (unreadable or unrecognised cgroup data); leaving it alone" + unsure=$((unsure + 1)) + continue + ;; + esac + + in_ending=0 + in_other=0 + for job in $jobs; do + if [ "$job" = "$SLURM_JOB_ID" ]; then + in_ending=1 + else + in_other=1 + fi + done + + if [ "$in_other" = 1 ]; then + spared=$((spared + 1)) + continue + fi + + if [ "$in_ending" = 0 ] && [ "$last_user_job" != 1 ]; then + # An orphan, but the user may still be running here. + spared=$((spared + 1)) + continue + fi + + if "$kill_bin" -9 -- "$pid" 2>/dev/null; then + killed=$((killed + 1)) + fi +done + +logger -s -t slurm-epilog \ + "Job-scoped process cleanup for ${SLURM_JOB_USER} job ${SLURM_JOB_ID}: killed ${killed}, spared ${spared}, unclassified ${unsure}" +{% else %} if [ "$SLURM_JOB_USER" != root ]; then if killall -9 -u "$SLURM_JOB_USER" ; then logger -s -t slurm-epilog 'Killed residual user processes' fi fi +{% endif %} diff --git a/roles/slurm/templates/etc/slurm/epilog.d/42-lastuserjob-cleanup b/roles/slurm/templates/etc/slurm/epilog.d/42-lastuserjob-cleanup index 33f9f90b9..5318c0e1a 100755 --- a/roles/slurm/templates/etc/slurm/epilog.d/42-lastuserjob-cleanup +++ b/roles/slurm/templates/etc/slurm/epilog.d/42-lastuserjob-cleanup @@ -33,6 +33,23 @@ case "$lookup_rc" in ;; esac +{% if slurm_epilog_cleanup_scope == 'job' %} +# Job-scoped cleanup (slurm_epilog_cleanup_scope: job). +# +# Files carry a uid, never a job id, so deleting "this job's files" from +# ownership alone is not possible: this script stays per user, and the +# dispatcher only runs it when neither squeue nor the node's cgroups show +# another job of the user. When Slurm's job_container/tmpfs plugin is active, +# /tmp and /dev/shm are private to each job and vanish with it, so this +# cleanup has nothing to add and would only reach into the host's copies. +slurm_conf='{{ slurm_config_dir }}/slurm.conf' +if grep -qE '^[[:space:]]*JobContainerType[[:space:]]*=[[:space:]]*job_container/tmpfs' "$slurm_conf" 2>/dev/null; then + logger -s -t slurm-epilog \ + "job_container/tmpfs is configured; per-job /tmp and /dev/shm are removed with the job, skipping file cleanup" + exit 0 +fi + +{% endif %} if [[ -n "$SLURM_JOB_USER" && "$SLURM_JOB_USER" != "root" ]]; then # `find -user` needs the account to resolve. When the directory service is # unavailable find exits non-zero without printing anything, and the old diff --git a/roles/slurm/templates/etc/slurm/shared/bin/run-parts.sh b/roles/slurm/templates/etc/slurm/shared/bin/run-parts.sh index 19b857fcc..08903c419 100755 --- a/roles/slurm/templates/etc/slurm/shared/bin/run-parts.sh +++ b/roles/slurm/templates/etc/slurm/shared/bin/run-parts.sh @@ -73,6 +73,56 @@ else log "[WARN] squeue failed while counting jobs for ${SLURM_JOB_USER} on ${HOSTNAME}; not running the last-user-job scripts." fi +{% if slurm_epilog_cleanup_scope == 'job' %} +# Job-scoped cleanup (slurm_epilog_cleanup_scope: job). Only meaningful where +# there are *-lastuserjob-* scripts, i.e. the epilog; the prolog skips it. +# +# squeue is the controller's opinion of the node, and it can be behind it: with +# the default CompleteWait=0 the controller may place a new job of this user +# here after squeue answered and before the cleanup below runs. The kernel's +# view cannot be: every process of a Slurm job sits in a job_ cgroup, so a +# process of this user in the cgroup of any job other than the one ending is +# direct, node-local evidence that the user is not done here. Either source +# saying "another job is here" is enough to skip the *-lastuserjob-* scripts. +# +# When the local view cannot be established (no numeric uid, unreadable or +# unrecognised /proc//cgroup data) the answer is the safe one: another job +# may be here. The cleanup is deferred to the next job's epilog; it is never +# guessed at. +epilog_proc_root=/proc +job_scope_lib='{{ slurm_config_dir }}/shared/lib/epilog-job-scope.sh' + +# node_view: clear (no other job's process found), other_job, or unknown. +node_view=clear +if compgen -G "$parts_dir/*-lastuserjob-*" >/dev/null; then + node_view=unknown + # `.` on a missing file must not take the whole epilog down under set -e. + if [ -r "$job_scope_lib" ] && . "$job_scope_lib"; then + if epilog_resolve_uid; then + node_rc=0 + epilog_other_job_present "$epilog_uid" "$SLURM_JOB_ID" || node_rc=$? + case "$node_rc" in + 0) node_view=other_job + log "Another job of ${SLURM_JOB_USER} has processes on this node (found by cgroup); not running the last-user-job scripts." + ;; + 1) node_view=clear ;; + *) log "[WARN] could not read the cgroup membership of the processes of ${SLURM_JOB_USER}; assuming another job may be here." ;; + esac + else + log "[WARN] no numeric uid for ${SLURM_JOB_USER}; assuming another job may be here." + fi + else + log "[WARN] cannot load ${job_scope_lib}; assuming another job of ${SLURM_JOB_USER} may be here." + fi +fi +if [ "$node_view" != clear ]; then + last_user_job=0 +fi +# Scripts that decide per process (40-lastuserjob-processes) read this to know +# whether the user has no other job here, which is when orphans may be reaped. +export EPILOG_LAST_USER_JOB="$last_user_job" + +{% endif %} # Re-implement run-parts since on centos it is just a bash script with no useful flags. failed=0 for script in "$parts_dir"/*; do @@ -84,6 +134,20 @@ for script in "$parts_dir"/*; do log "Skipping $script because the job was not run in exclusive mode." continue fi +{% if slurm_epilog_cleanup_scope == 'job' %} + # 40-lastuserjob-processes classifies each process by its own job cgroup + # and applies the last-user-job rule to orphans itself, so it must also run + # when another job is present: it is what spares that job's processes and + # still reaps the ending job's. + if [[ "$script" == *"-lastuserjob-processes" ]]; then + log "Running $script ..." + if ! $script >>/var/log/slurm/prolog-epilog 2>&1; then + log "[ERROR] $script failed. Check the log at /var/log/slurm/prolog-epilog for more details." + failed=1 + fi + continue + fi +{% endif %} if [[ "$script" == *"-lastuserjob-"* ]] && [ "$last_user_job" = 0 ]; then log "Skipping $script because there is still another job running on this node for the same user." continue diff --git a/roles/slurm/templates/etc/slurm/shared/lib/epilog-job-scope.sh b/roles/slurm/templates/etc/slurm/shared/lib/epilog-job-scope.sh new file mode 100644 index 000000000..45de17046 --- /dev/null +++ b/roles/slurm/templates/etc/slurm/shared/lib/epilog-job-scope.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# Helpers for job-scoped epilog cleanup (slurm_epilog_cleanup_scope: job). +# +# Sourced by shared/bin/run-parts.sh and epilog.d/40-lastuserjob-processes. +# Deployed only when the scope is "job". +# +# The question these answer is "whose is this process?", asked of the node +# instead of the controller. squeue reports what the controller believes, which +# can lag the node and cannot see a job placed here a moment ago. The kernel's +# /proc//cgroup cannot be out of date: Slurm puts every process of a job +# in a cgroup named job_ (cgroup v1: /slurm/uid_N/job_M/..., cgroup v2: +# .../slurmstepd.scope/job_M/...), so the job id is read straight off the path. +# +# Everything here fails closed. "Could not tell" is reported as such and the +# caller treats it as "another job may be here": cleanup is skipped and logged, +# never guessed. A skipped cleanup is a delay, since it runs again when the +# next job of the user ends. A wrong guess destroys a running job. +# +# The caller sets epilog_proc_root (normally /proc) before sourcing, so that +# the unit tests can supply a fake process tree. It is set by the script, not +# read from the environment. + +: "${epilog_proc_root:?epilog_proc_root must be set by the caller}" + +# epilog_resolve_uid +# Sets epilog_uid from SLURM_JOB_UID, or from the account name when that is +# missing. Returns 1 when no numeric uid can be established. +epilog_resolve_uid() { + epilog_uid="" + if [[ "${SLURM_JOB_UID:-}" =~ ^[0-9]+$ ]]; then + epilog_uid="$SLURM_JOB_UID" + return 0 + fi + local resolved + if resolved=$(id -u -- "${SLURM_JOB_USER:-}" 2>/dev/null) && [[ "$resolved" =~ ^[0-9]+$ ]]; then + epilog_uid="$resolved" + return 0 + fi + return 1 +} + +# epilog_pid_owned_by UID PID +# True when the real or effective uid of PID is UID. /proc//status is +# used rather than the owner of /proc/, which turns root-owned for a +# process that has made itself non-dumpable. +# Returns 0 owned, 1 not owned or gone. +epilog_pid_owned_by() { + local uid="$1" pid="$2" key real eff rest + local status="$epilog_proc_root/$pid/status" + [ -r "$status" ] || return 1 + while read -r key real eff rest; do + if [ "$key" = "Uid:" ]; then + [ "$real" = "$uid" ] || [ "$eff" = "$uid" ] + return + fi + done <"$status" 2>/dev/null + return 1 +} + +# epilog_pid_jobs PID +# Prints the job id of every job_ cgroup component in +# /proc//cgroup, one per line (none printed means the process is in no +# Slurm job cgroup). +# Returns 0 parsed, 1 the process is gone, 2 unreadable or not understood. +epilog_pid_jobs() { + local pid="$1" file content line path + file="$epilog_proc_root/$pid/cgroup" + # The redirect has to wrap the whole command: bash reports a failed read of + # $(/dev/null; then + [ -e "$file" ] || return 1 + return 2 + fi + [ -n "$content" ] || return 2 + while IFS= read -r line; do + [ -n "$line" ] || continue + # hierarchy-id:controllers:path, with an empty controller list on v2. + if [[ ! "$line" =~ ^[0-9]+:[^:]*:/ ]]; then + return 2 + fi + path="${line#*:*:}" + if [[ "$path" =~ (^|/)job_([0-9]+)(/|$) ]]; then + printf '%s\n' "${BASH_REMATCH[2]}" + fi + done <<<"$content" + return 0 +} + +# epilog_user_pids UID +# Prints the pid of every process owned by UID. +# Returns 0 on success and 2 when the process table is unusable (nothing +# listed at all), which the caller must treat as "cannot tell". +epilog_user_pids() { + local uid="$1" entry pid listed=0 + for entry in "$epilog_proc_root"/[0-9]*; do + [ -d "$entry" ] || continue + pid="${entry##*/}" + [[ "$pid" =~ ^[0-9]+$ ]] || continue + listed=1 + if epilog_pid_owned_by "$uid" "$pid"; then + printf '%s\n' "$pid" + fi + done + [ "$listed" = 1 ] || return 2 + return 0 +} + +# epilog_other_job_present UID ENDING_JOBID +# Is any process of UID inside the cgroup of a job other than ENDING_JOBID? +# Returns 0 yes, 1 no, 2 cannot tell (unreadable or unrecognised cgroup +# data, or no process table). +epilog_other_job_present() { + local uid="$1" ending="$2" pid pids jobs job rc unsure=0 + if ! pids=$(epilog_user_pids "$uid"); then + return 2 + fi + for pid in $pids; do + rc=0 + jobs=$(epilog_pid_jobs "$pid") || rc=$? + case "$rc" in + 0) ;; + 1) continue ;; + *) unsure=1; continue ;; + esac + for job in $jobs; do + if [ "$job" != "$ending" ]; then + return 0 + fi + done + done + [ "$unsure" = 0 ] || return 2 + return 1 +} From 22b68a23fc3f6394b1178508523d9dfa62d9334e Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Mon, 12 Oct 2026 10:14:26 -0600 Subject: [PATCH 3/6] test(slurm): cover job-scoped epilog cleanup and the user-scope snapshot (#1407) Extends tests/slurm-epilog/run-tests.sh with a fake /proc and a recording kill so the job-scoped paths run unprivileged: dispatcher guard (cgroup v1 and v2, other user, orphan, unrecognised, empty, missing library, no uid), script 40 per-PID classification, 42 job_container/tmpfs skip, 50 behind the guard, and a snapshot check that scope=user renders the 26.09 scripts byte-for-byte. render.py takes the scope and matches Ansible's trim_blocks. Signed-off-by: Doug Holt --- tests/slurm-epilog/render.py | 18 +- tests/slurm-epilog/run-tests.sh | 417 +++++++++++++++++- .../user-scope/40-lastuserjob-processes | 43 ++ .../user-scope/42-lastuserjob-cleanup | 146 ++++++ .../user-scope/50-lastuserjob-all-enroot-dirs | 10 + .../snapshots/user-scope/epilog.sh | 6 + .../snapshots/user-scope/run-parts.sh | 101 +++++ 7 files changed, 736 insertions(+), 5 deletions(-) create mode 100644 tests/slurm-epilog/snapshots/user-scope/40-lastuserjob-processes create mode 100644 tests/slurm-epilog/snapshots/user-scope/42-lastuserjob-cleanup create mode 100644 tests/slurm-epilog/snapshots/user-scope/50-lastuserjob-all-enroot-dirs create mode 100644 tests/slurm-epilog/snapshots/user-scope/epilog.sh create mode 100644 tests/slurm-epilog/snapshots/user-scope/run-parts.sh diff --git a/tests/slurm-epilog/render.py b/tests/slurm-epilog/render.py index 198371184..56ce83f84 100644 --- a/tests/slurm-epilog/render.py +++ b/tests/slurm-epilog/render.py @@ -21,12 +21,18 @@ "enroot_data_path": "/tmp/enroot-data/user-$(id -u)", } +SCOPES = ("job", "user") + def main(): - if len(sys.argv) != 3: - sys.exit("usage: render.py ") + if len(sys.argv) not in (3, 4): + sys.exit("usage: render.py [job|user]") template_path, output_path = sys.argv[1], sys.argv[2] + # slurm_epilog_cleanup_scope. "job" is the role default. + scope = sys.argv[3] if len(sys.argv) == 4 else "job" + if scope not in SCOPES: + sys.exit("scope must be one of: " + ", ".join(SCOPES)) # keep_trailing_newline so the rendered script ends the way the original # does; a missing final newline changes nothing functionally but makes @@ -48,9 +54,15 @@ def main(): loader=FileSystemLoader("/"), undefined=StrictUndefined, keep_trailing_newline=True, + # Ansible's template module sets trim_blocks, so a block tag on a line + # of its own leaves no blank line behind. Match it, or the rendered + # scripts here differ from the deployed ones. + trim_blocks=True, autoescape=False, ) - rendered = env.get_template(template_path.lstrip("/")).render(**VARS) + rendered = env.get_template(template_path.lstrip("/")).render( + slurm_epilog_cleanup_scope=scope, **VARS + ) with open(output_path, "w") as handle: handle.write(rendered) diff --git a/tests/slurm-epilog/run-tests.sh b/tests/slurm-epilog/run-tests.sh index 98fc992ec..abdeb3050 100755 --- a/tests/slurm-epilog/run-tests.sh +++ b/tests/slurm-epilog/run-tests.sh @@ -56,7 +56,7 @@ STUB render() { # $1 = template path relative to $templates, $2 = output path - python3 "$here/render.py" "$templates/$1" "$2" || return 1 + python3 "$here/render.py" "$templates/$1" "$2" "${RENDER_SCOPE:-job}" || return 1 chmod +x "$2" # A rendered script that does not parse would fail every behavioural case # below with the same opaque message; check the syntax separately so the @@ -64,6 +64,27 @@ render() { bash -n "$2" } +# A fake /proc: //status carries the Uid line and //cgroup +# the cgroup membership, which is all the job-scoped helpers read. Paths are +# written the way the kernel writes them, for cgroup v1 and v2. +fake_proc() { + # $1 = root, $2 = pid, $3 = uid, $4 = contents of /proc//cgroup + mkdir -p "$1/$2" + printf 'Name:\tfixture\nUid:\t%s\t%s\t%s\t%s\n' "$3" "$3" "$3" "$3" > "$1/$2/status" + if [ -n "${4-}" ]; then + printf '%s\n' "$4" > "$1/$2/cgroup" + else + : > "$1/$2/cgroup" + fi +} + +# cgroup lines as Slurm lays them out. v1 has one line per controller +# hierarchy under /slurm/uid_N/job_M; v2 has one unified line under the +# slurmstepd scope. +cg_v1_job() { printf '4:cpuset:/slurm/uid_%s/job_%s/step_0/task_0\n3:memory:/slurm/uid_%s/job_%s/step_0\n1:name=systemd:/system.slice/slurmd.service' "$1" "$2" "$1" "$2"; } +cg_v2_job() { printf '0::/system.slice/slurmstepd.scope/job_%s/step_0/user/task_0' "$1"; } +cg_none() { printf '0::/user.slice/user-%s.slice/session-3.scope' "$1"; } + # --------------------------------------------------------------------------- # run-parts.sh: which jobs count as "the user still has work on this node" # --------------------------------------------------------------------------- @@ -79,6 +100,26 @@ setup_runparts() { render "shared/bin/run-parts.sh" "$dir/run-parts.sh" || return 1 + # Job scope: the dispatcher also reads the node's own process table. Point + # it at a fake /proc and at the helper library rendered into the fixture, + # and check the redirects took, as the cleanup setup does: a template edit + # that changes those lines would otherwise hand the real /proc back. + mkdir -p "$dir/proc" + if [ "${RENDER_SCOPE:-job}" = job ]; then + mkdir -p "$dir/lib" + render "shared/lib/epilog-job-scope.sh" "$dir/lib/epilog-job-scope.sh" || return 1 + sed -i "s#^epilog_proc_root=.*#epilog_proc_root='$dir/proc'#" "$dir/run-parts.sh" + sed -i "s#^job_scope_lib=.*#job_scope_lib='$dir/lib/epilog-job-scope.sh'#" "$dir/run-parts.sh" + if ! grep -qF "epilog_proc_root='$dir/proc'" "$dir/run-parts.sh" \ + || ! grep -qF "job_scope_lib='$dir/lib/epilog-job-scope.sh'" "$dir/run-parts.sh"; then + printf ' FAIL harness: the dispatcher /proc root was not redirected into the fixture\n' >&2 + return 1 + fi + # Every real /proc has processes. One owned by another account keeps an + # otherwise empty fixture from reading as "no process table". + fake_proc "$dir/proc" 1 0 '0::/init.scope' + fi + # The script calls squeue twice: once for the exclusive check (-o "%C %D"), # once for the last-user-job count. Answer the first with a non-exclusive # allocation so only the second is under test. @@ -160,12 +201,14 @@ runparts_case() { local label="$1" expect="$2" jobs="${3-}" rc="${4-0}" local dir dir="$(setup_runparts)" || { no "$label" "harness setup failed"; return; } + # $5 = optional function that builds the fake /proc under $dir/proc + if [ -n "${5-}" ]; then "$5" "$dir/proc"; fi rm -f "$dir/RAN" local out out=$( PATH="$dir/bin:$PATH" \ - SLURM_JOB_USER=alice SLURM_JOB_ID=1000 SLURMD_NODENAME=node1 \ + SLURM_JOB_USER=alice SLURM_JOB_UID=1001 SLURM_JOB_ID=1000 SLURMD_NODENAME=node1 \ HOSTNAME=node1 SLURM_JOBID=1000 \ SQUEUE_JOBS="$jobs" SQUEUE_RC="$rc" \ "$dir/run-parts.sh" "$dir/parts" 2>&1 @@ -253,6 +296,11 @@ setup_cleanup() { # before any deletion -- test safety cannot rest on the code under test. sed -i "s#^ for dir in /tmp /dev/shm ; do# for dir in $dir/roots/tmp $dir/roots/shm ; do#" "$dir/42-cleanup" + # Job scope also reads slurm.conf (job_container/tmpfs). Keep that off the + # host's real file too; the fixture has none unless a case writes one. The + # user-scope rendering has no such line, so a miss here is not an error. + sed -i "s#^slurm_conf=.*#slurm_conf='$dir/etc/slurm/slurm.conf'#" "$dir/42-cleanup" + # And the redirect is verified rather than assumed: a template edit that # changes that line would silently hand the real roots back. if ! grep -qF "for dir in $dir/roots/tmp $dir/roots/shm ; do" "$dir/42-cleanup"; then @@ -608,6 +656,371 @@ pathname_case "a mount point containing a tab is decoded" $'data fi # identity and /proc/self/mountinfo +# --------------------------------------------------------------------------- +# Job scope: dispatcher guard, script 40, script 42 and script 50 +# --------------------------------------------------------------------------- +# +# Everything here runs against a fake /proc (fake_proc above) and a recording +# `kill`, so it needs neither Slurm, cgroups nor root. The ending job is 1000 +# and the user is uid 1001 throughout; "squeue says clear" is the stub with no +# jobs. + +# Fake process tables for the dispatcher cases. +proc_other_job_v1() { fake_proc "$1" 2001 1001 "$(cg_v1_job 1001 1002)"; } +proc_other_job_v2() { fake_proc "$1" 2001 1001 "$(cg_v2_job 1002)"; } +proc_own_job_only() { fake_proc "$1" 2001 1001 "$(cg_v2_job 1000)"; } +proc_other_job_other_user() { fake_proc "$1" 2001 1555 "$(cg_v2_job 1002)"; } +proc_unrecognised() { fake_proc "$1" 2001 1001 'this is not a cgroup file'; } +proc_unreadable() { fake_proc "$1" 2001 1001 ''; } +proc_orphan_only() { fake_proc "$1" 2001 1001 "$(cg_none 1001)"; } +proc_own_and_other() { fake_proc "$1" 2001 1001 "$(cg_v2_job 1000)"; fake_proc "$1" 2002 1001 "$(cg_v2_job 1003)"; } + +echo +echo "run-parts.sh -- job-scoped guard (scope=job)" +RENDER_SCOPE=job +runparts_case "node scan: another job's process (cgroup v1) blocks though squeue is clear" skipped "" 0 proc_other_job_v1 +runparts_case "node scan: another job's process (cgroup v2) blocks though squeue is clear" skipped "" 0 proc_other_job_v2 +runparts_case "node scan: only the ending job's processes do not block" ran "" 0 proc_own_job_only +runparts_case "node scan: another user's job does not block" ran "" 0 proc_other_job_other_user +runparts_case "node scan: an orphan alone does not block" ran "" 0 proc_orphan_only +runparts_case "node scan: own and other job together block" skipped "" 0 proc_own_and_other +runparts_case "node scan: an unrecognised cgroup layout fails closed" skipped "" 0 proc_unrecognised +runparts_case "node scan: an empty cgroup file fails closed" skipped "" 0 proc_unreadable +runparts_case "squeue still wins when the node scan is clear" skipped "1001:running" 0 proc_orphan_only + +# No process table at all: a /proc with nothing in it must read as "cannot +# tell", not as "no other job". +proc_empty() { rm -rf "$1"; mkdir -p "$1"; } +runparts_case "node scan: an empty process table fails closed" skipped "" 0 proc_empty + +# A dispatcher that cannot load its helper library must defer, not run on +# squeue alone. +dir="$(setup_runparts)" || dir="" +if [ -z "$dir" ]; then + no "a missing helper library fails closed" "harness setup failed" +else + rm -f "$dir/lib/epilog-job-scope.sh" "$dir/RAN" + out=$(PATH="$dir/bin:$PATH" SLURM_JOB_USER=alice SLURM_JOB_UID=1001 SLURM_JOB_ID=1000 \ + HOSTNAME=node1 SLURM_JOBID=1000 SQUEUE_JOBS="" SQUEUE_RC=0 \ + "$dir/run-parts.sh" "$dir/parts" 2>&1); rc=$? + if [ "$rc" = 0 ] && [ ! -e "$dir/RAN" ] && printf '%s' "$out" | grep -q 'cannot load'; then + ok "a missing helper library fails closed" + else + no "a missing helper library fails closed" "rc=$rc ran=$([ -e "$dir/RAN" ] && echo yes || echo no) out=$(printf '%s' "$out" | tail -1)" + fi +fi + +# A job without SLURM_JOB_UID resolves the account instead; with neither it +# defers. +dir="$(setup_runparts)" || dir="" +if [ -z "$dir" ]; then + no "no resolvable uid fails closed" "harness setup failed" +else + rm -f "$dir/RAN" + out=$(PATH="$dir/bin:$PATH" SLURM_JOB_USER="nosuchuser-$$" SLURM_JOB_UID="" SLURM_JOB_ID=1000 \ + HOSTNAME=node1 SLURM_JOBID=1000 SQUEUE_JOBS="" SQUEUE_RC=0 \ + "$dir/run-parts.sh" "$dir/parts" 2>&1); rc=$? + if [ "$rc" = 0 ] && [ ! -e "$dir/RAN" ] && printf '%s' "$out" | grep -q 'no numeric uid'; then + ok "no resolvable uid fails closed" + else + no "no resolvable uid fails closed" "rc=$rc out=$(printf '%s' "$out" | tail -1)" + fi +fi + +echo +echo "run-parts.sh -- scope=user ignores the node scan" +RENDER_SCOPE=user +runparts_case "user scope: another job's cgroup does not block (squeue only)" ran "" 0 proc_other_job_v2 +RENDER_SCOPE=job + +# --- 40-lastuserjob-processes ------------------------------------------------ +echo +echo "40-lastuserjob-processes -- PID classification (scope=job)" + +setup_procs() { + local dir="$work/procs" + rm -rf "$dir" + mkdir -p "$dir/proc" "$dir/lib" "$dir/etc" + write_logger_stub "$dir" + render "epilog.d/40-lastuserjob-processes" "$dir/40-processes" || return 1 + render "shared/lib/epilog-job-scope.sh" "$dir/lib/epilog-job-scope.sh" || return 1 + sed -i "s#^localusers_backup=.*#localusers_backup='$dir/etc/localusers.backup'#" "$dir/40-processes" + sed -i "s#^epilog_proc_root=.*#epilog_proc_root='$dir/proc'#" "$dir/40-processes" + sed -i "s#^job_scope_lib=.*#job_scope_lib='$dir/lib/epilog-job-scope.sh'#" "$dir/40-processes" + # A recording kill: the harness must never signal a real process. + cat > "$dir/bin/kill" <> "$dir/kill.log" +STUB + chmod +x "$dir/bin/kill" + sed -i "s#^kill_bin=.*#kill_bin='$dir/bin/kill'#" "$dir/40-processes" + for want in "localusers_backup='$dir/etc/localusers.backup'" "epilog_proc_root='$dir/proc'" \ + "job_scope_lib='$dir/lib/epilog-job-scope.sh'" "kill_bin='$dir/bin/kill'"; do + if ! grep -qF "$want" "$dir/40-processes"; then + printf ' FAIL harness: 40 was not redirected into the fixture (%s)\n' "$want" >&2 + return 1 + fi + done + : > "$dir/etc/localusers.backup" + : > "$dir/kill.log" + printf '%s' "$dir" +} + +# $1 label, $2 builder, $3 "pid pid ..." expected killed, $4 EPILOG_LAST_USER_JOB +procs_case() { + local label="$1" build="$2" want="$3" last="${4-0}" dir + dir="$(setup_procs)" || { no "$label" "harness setup failed"; return; } + "$build" "$dir/proc" + local out rc + out=$(PATH="$dir/bin:$PATH" SLURM_JOB_USER=alice SLURM_JOB_UID=1001 SLURM_JOB_ID=1000 \ + EPILOG_LAST_USER_JOB="$last" "$dir/40-processes" 2>&1); rc=$? + local got + got=$(sed -e 's/^-9 -- //' "$dir/kill.log" | sort -n | tr '\n' ' ' | sed 's/ $//') + if [ "$rc" != 0 ]; then + no "$label" "rc=$rc out=$(printf '%s' "$out" | tail -1)" + elif [ "$got" != "$want" ]; then + no "$label" "killed '$got', wanted '$want'" + else + ok "$label" + fi +} + +# One PID per row of the classification table in the design note, on both +# layouts. 3001 is the ending job's, 3002 another job's, 3003 an orphan, 3004 +# unparsable; 3005 belongs to someone else's job and is never ours to touch. +mixed_v1() { fake_proc "$1" 3001 1001 "$(cg_v1_job 1001 1000)"; fake_proc "$1" 3002 1001 "$(cg_v1_job 1001 1002)" + fake_proc "$1" 3003 1001 "$(cg_none 1001)"; fake_proc "$1" 3004 1001 'garbage' + fake_proc "$1" 3005 1777 "$(cg_v1_job 1777 1050)"; } +mixed_v2() { fake_proc "$1" 3001 1001 "$(cg_v2_job 1000)"; fake_proc "$1" 3002 1001 "$(cg_v2_job 1002)" + fake_proc "$1" 3003 1001 "$(cg_none 1001)"; fake_proc "$1" 3004 1001 'garbage' + fake_proc "$1" 3005 1777 "$(cg_v2_job 1050)"; } + +procs_case "cgroup v1: ending job killed, other job spared, orphan spared while another job exists" mixed_v1 "3001" 0 +procs_case "cgroup v2: ending job killed, other job spared, orphan spared while another job exists" mixed_v2 "3001" 0 +procs_case "cgroup v1: orphan killed only when the user has no other job" mixed_v1 "3001 3003" 1 +procs_case "cgroup v2: orphan killed only when the user has no other job" mixed_v2 "3001 3003" 1 +# Even when the dispatcher says "last job", a process in another job's cgroup +# is never killed: that is the race squeue cannot see. +procs_case "another job's process is spared even when the dispatcher says last job" mixed_v2 "3001 3003" 1 + +exited_between() { fake_proc "$1" 3010 1001 "$(cg_v2_job 1000)"; rm -f "$1/3010/cgroup"; } +procs_case "a process that exits between listing and classifying is skipped" exited_between "" 1 + +# A vanished process is routine and must not put bash errors in the epilog log. +vanished_quiet_dir="$(setup_procs)" || vanished_quiet_dir="" +if [ -z "$vanished_quiet_dir" ]; then + no "a vanished process produces no shell error output" "harness setup failed" +else + fake_proc "$vanished_quiet_dir/proc" 3012 1001 "$(cg_v2_job 1000)" + rm -f "$vanished_quiet_dir/proc/3012/cgroup" + out=$(PATH="$vanished_quiet_dir/bin:$PATH" SLURM_JOB_USER=alice SLURM_JOB_UID=1001 SLURM_JOB_ID=1000 \ + EPILOG_LAST_USER_JOB=1 "$vanished_quiet_dir/40-processes" 2>&1) + if ! printf '%s' "$out" | grep -qi 'no such file\|line [0-9]*:'; then + ok "a vanished process produces no shell error output" + else + no "a vanished process produces no shell error output" "$(printf '%s' "$out" | head -2 | tr '\n' ' ')" + fi +fi + +unparsable_only() { fake_proc "$1" 3011 1001 'x:y'; } +procs_case "an unparsable cgroup line is skipped, not killed" unparsable_only "" 1 + +# Nested job paths (a step cgroup under the job) classify by the job_ component. +nested_step() { fake_proc "$1" 3020 1001 '0::/system.slice/slurmstepd.scope/job_1000/step_batch/user/task_0'; } +procs_case "a step cgroup under the ending job is the ending job's" nested_step "3020" 0 + +# A job id that is only a prefix of the ending job's must not match. +prefix_job() { fake_proc "$1" 3021 1001 "$(cg_v2_job 10001)"; } +procs_case "job 10001 is not job 1000" prefix_job "" 1 + +# --- exemptions and edge users ------------------------------------------------- +exempt_dir="$(setup_procs)" || exempt_dir="" +if [ -z "$exempt_dir" ]; then + no "a user in localusers.backup is spared" "harness setup failed" +else + mixed_v2 "$exempt_dir/proc" + printf 'alice\n' > "$exempt_dir/etc/localusers.backup" + PATH="$exempt_dir/bin:$PATH" SLURM_JOB_USER=alice SLURM_JOB_UID=1001 SLURM_JOB_ID=1000 \ + EPILOG_LAST_USER_JOB=1 "$exempt_dir/40-processes" >/dev/null 2>&1; rc=$? + if [ "$rc" = 0 ] && [ ! -s "$exempt_dir/kill.log" ]; then + ok "a user in localusers.backup is spared" + else + no "a user in localusers.backup is spared" "rc=$rc kills=$(tr '\n' ' ' < "$exempt_dir/kill.log")" + fi +fi + +root_dir="$(setup_procs)" || root_dir="" +if [ -z "$root_dir" ]; then + no "root is never touched" "harness setup failed" +else + fake_proc "$root_dir/proc" 3030 0 "$(cg_v2_job 1000)" + PATH="$root_dir/bin:$PATH" SLURM_JOB_USER=root SLURM_JOB_UID=0 SLURM_JOB_ID=1000 \ + EPILOG_LAST_USER_JOB=1 "$root_dir/40-processes" >/dev/null 2>&1; rc=$? + if [ "$rc" = 0 ] && [ ! -s "$root_dir/kill.log" ]; then + ok "root is never touched" + else + no "root is never touched" "rc=$rc kills=$(tr '\n' ' ' < "$root_dir/kill.log")" + fi +fi + +lookup_dir="$(setup_procs)" || lookup_dir="" +if [ -z "$lookup_dir" ]; then + no "an unreadable localusers.backup stops the process cleanup" "harness setup failed" +else + mixed_v2 "$lookup_dir/proc" + rm -f "$lookup_dir/etc/localusers.backup" + out=$(PATH="$lookup_dir/bin:$PATH" SLURM_JOB_USER=alice SLURM_JOB_UID=1001 SLURM_JOB_ID=1000 \ + EPILOG_LAST_USER_JOB=1 "$lookup_dir/40-processes" 2>&1); rc=$? + if [ "$rc" = 0 ] && [ ! -s "$lookup_dir/kill.log" ] && printf '%s' "$out" | grep -q 'localusers lookup failed'; then + ok "an unreadable localusers.backup stops the process cleanup" + else + no "an unreadable localusers.backup stops the process cleanup" "rc=$rc out=$(printf '%s' "$out" | tail -1)" + fi +fi + +# Without the dispatcher's word that this was the last job, an orphan stays. +standalone_dir="$(setup_procs)" || standalone_dir="" +if [ -z "$standalone_dir" ]; then + no "run without the dispatcher, orphans are spared" "harness setup failed" +else + mixed_v2 "$standalone_dir/proc" + PATH="$standalone_dir/bin:$PATH" SLURM_JOB_USER=alice SLURM_JOB_UID=1001 SLURM_JOB_ID=1000 \ + "$standalone_dir/40-processes" >/dev/null 2>&1; rc=$? + got=$(sed -e 's/^-9 -- //' "$standalone_dir/kill.log" | tr '\n' ' ' | sed 's/ $//') + if [ "$rc" = 0 ] && [ "$got" = 3001 ]; then + ok "run without the dispatcher, orphans are spared" + else + no "run without the dispatcher, orphans are spared" "rc=$rc killed='$got'" + fi +fi + +# And in scope=user the original killall is what runs. +user40="$work/user40"; rm -rf "$user40"; mkdir -p "$user40/bin" "$user40/etc" +write_logger_stub "$user40" +if RENDER_SCOPE=user render "epilog.d/40-lastuserjob-processes" "$user40/40" \ + && grep -q 'killall -9 -u' "$user40/40" && ! grep -q 'epilog-job-scope' "$user40/40"; then + ok "user scope renders the original killall script" +else + no "user scope renders the original killall script" "see $user40" +fi + +# --- 42-lastuserjob-cleanup: job_container/tmpfs --------------------------------- +echo +echo "42-lastuserjob-cleanup -- job_container/tmpfs (scope=job)" +jc_identity="$(cleanup_identity)" || jc_identity="" +if [ -z "$jc_identity" ]; then + printf ' skip job_container cases (no resolvable unprivileged account)\n' +else + jc_case() { + # $1 = label, $2 = slurm.conf contents ("" = none), $3 = expect gone|kept + local label="$1" conf="$2" expect="$3" dir + dir="$(setup_cleanup)" || { no "$label" "harness setup failed"; return; } + : > "$dir/etc/slurm/localusers.backup" + sed -i "s#^slurm_conf=.*#slurm_conf='$dir/etc/slurm/slurm.conf'#" "$dir/42-cleanup" + grep -qF "slurm_conf='$dir/etc/slurm/slurm.conf'" "$dir/42-cleanup" \ + || { no "$label" "harness could not redirect slurm.conf"; return; } + [ -z "$conf" ] || printf '%s\n' "$conf" > "$dir/etc/slurm/slurm.conf" + own_file "$jc_identity" "$dir/roots/tmp/scratch" || { no "$label" "could not create the fixture file"; return; } + local out rc state=kept + out=$(PATH="$dir/bin:$PATH" SLURM_JOB_USER="$jc_identity" "$dir/42-cleanup" 2>&1); rc=$? + [ -e "$dir/roots/tmp/scratch" ] || state=gone + if [ "$rc" != 0 ]; then + no "$label" "rc=$rc out=$(printf '%s' "$out" | tail -1)" + elif [ "$state" != "$expect" ]; then + no "$label" "file is $state, wanted $expect" + elif [ "$expect" = kept ] && ! printf '%s' "$out" | grep -q 'job_container/tmpfs is configured'; then + no "$label" "skipped without logging why: $(printf '%s' "$out" | tail -1)" + else + ok "$label" + fi + } + jc_case "job_container/tmpfs configured: file cleanup is skipped and logged" \ + "JobContainerType=job_container/tmpfs" kept + jc_case "job_container/tmpfs with spacing is still detected" \ + " JobContainerType = job_container/tmpfs" kept + jc_case "a commented-out JobContainerType does not skip the cleanup" \ + "#JobContainerType=job_container/tmpfs" gone + jc_case "no JobContainerType: cleanup runs" "ProctrackType=proctrack/cgroup" gone + jc_case "no slurm.conf readable: cleanup runs as before" "" gone +fi + +# --- 50-lastuserjob-all-enroot-dirs ----------------------------------------------- +# The enroot script has no guard of its own; it is protected by the dispatcher. +# The dispatcher cases above run a -lastuserjob- marker, which is the same gate +# 50 sits behind, so what is pinned here is the rendering: the paths survive +# in both scopes, and nothing in the script changes with the scope. +echo +echo "50-lastuserjob-all-enroot-dirs -- same removal in both scopes, behind the stronger guard" +e50j="$work/e50j"; e50u="$work/e50u"; rm -rf "$e50j" "$e50u"; mkdir -p "$e50j" "$e50u" +if RENDER_SCOPE=job render "epilog.d/50-lastuserjob-all-enroot-dirs" "$e50j/50" \ + && RENDER_SCOPE=user render "epilog.d/50-lastuserjob-all-enroot-dirs" "$e50u/50" \ + && cmp -s "$e50j/50" "$e50u/50"; then + ok "script 50 is identical in both scopes; the dispatcher guard is what changes" +else + no "script 50 is identical in both scopes; the dispatcher guard is what changes" "rendered output differs" +fi +# The dispatcher test that matters for 50: a -lastuserjob-all-enroot-dirs +# script is skipped when the node scan finds another job. +dir="$(setup_runparts)" || dir="" +if [ -z "$dir" ]; then + no "enroot cleanup is skipped when another job is present" "harness setup failed" +else + rm -f "$dir/parts/"* + for n in 50-lastuserjob-all-enroot-dirs; do + printf '#!/usr/bin/env bash\ntouch "%s/RAN-50"\n' "$dir" > "$dir/parts/$n"; chmod +x "$dir/parts/$n" + done + proc_other_job_v2 "$dir/proc" + PATH="$dir/bin:$PATH" SLURM_JOB_USER=alice SLURM_JOB_UID=1001 SLURM_JOB_ID=1000 HOSTNAME=node1 \ + SLURM_JOBID=1000 SQUEUE_JOBS="" SQUEUE_RC=0 "$dir/run-parts.sh" "$dir/parts" >/dev/null 2>&1 + skipped_ok=1; [ ! -e "$dir/RAN-50" ] || skipped_ok=0 + rm -f "$dir/proc/2001/cgroup"; cg_none 1001 > "$dir/proc/2001/cgroup" + PATH="$dir/bin:$PATH" SLURM_JOB_USER=alice SLURM_JOB_UID=1001 SLURM_JOB_ID=1000 HOSTNAME=node1 \ + SLURM_JOBID=1000 SQUEUE_JOBS="" SQUEUE_RC=0 "$dir/run-parts.sh" "$dir/parts" >/dev/null 2>&1 + if [ "$skipped_ok" = 1 ] && [ -e "$dir/RAN-50" ]; then + ok "enroot cleanup is skipped when another job is present and runs when last" + else + no "enroot cleanup is skipped when another job is present and runs when last" \ + "skipped_when_other=$skipped_ok ran_when_last=$([ -e "$dir/RAN-50" ] && echo yes || echo no)" + fi +fi + +# --- 26.09 snapshot ----------------------------------------------------------------- +# scope=user must render byte-for-byte what 26.09 deployed. The snapshots are the +# 26.09 templates rendered with the same fixture values; script 41 is excluded +# because its match fix applies in both scopes and is covered above. +echo +echo "scope=user -- byte-identical to the 26.09 scripts" +snap_ok() { + # $1 = template relative path, $2 = snapshot file name + local out="$work/snap-$2" + if RENDER_SCOPE=user render "$1" "$out" && cmp -s "$out" "$here/snapshots/user-scope/$2"; then + ok "scope=user renders $2 identically to 26.09" + else + no "scope=user renders $2 identically to 26.09" "$(diff "$here/snapshots/user-scope/$2" "$out" 2>&1 | head -5 | tr '\n' ' ')" + fi +} +snap_ok "epilog.d/40-lastuserjob-processes" 40-lastuserjob-processes +snap_ok "epilog.d/42-lastuserjob-cleanup" 42-lastuserjob-cleanup +snap_ok "epilog.d/50-lastuserjob-all-enroot-dirs" 50-lastuserjob-all-enroot-dirs +snap_ok "shared/bin/run-parts.sh" run-parts.sh +snap_ok "epilog.sh" epilog.sh + +# Every template must render and parse in both scopes; StrictUndefined fails +# the render if a template references a variable the role does not define. +for scope in job user; do + bad="" + for t in epilog.sh epilog.d/40-lastuserjob-processes epilog.d/41-lastuserjob-ssh \ + epilog.d/42-lastuserjob-cleanup epilog.d/50-lastuserjob-all-enroot-dirs \ + shared/bin/run-parts.sh shared/lib/epilog-job-scope.sh; do + RENDER_SCOPE=$scope render "$t" "$work/parse-$scope" 2>/dev/null || bad="$bad $t" + done + if [ -z "$bad" ]; then + ok "all epilog templates render and parse with scope=$scope" + else + no "all epilog templates render and parse with scope=$scope" "failed:$bad" + fi +done + # --------------------------------------------------------------------------- # 41-lastuserjob-ssh: which SSH allow entries are removed # --------------------------------------------------------------------------- diff --git a/tests/slurm-epilog/snapshots/user-scope/40-lastuserjob-processes b/tests/slurm-epilog/snapshots/user-scope/40-lastuserjob-processes new file mode 100644 index 000000000..aff60eb81 --- /dev/null +++ b/tests/slurm-epilog/snapshots/user-scope/40-lastuserjob-processes @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -ex + +# Sibling 41-lastuserjob-ssh already consults this list before it touches a +# user, so that operator accounts do not lose access when their job ends. +# `killall -9 -u` is broader than anything 41 does -- it reaps every process +# the user owns on this node, including their interactive login shell and its +# sshd session -- so it has to honour the same exemption. +# +# Without this an operator who submits from a login shell on a compute node is +# disconnected the moment the job's epilog runs. +# +# The list is one account per line, so match whole lines literally (-x -F). +# Accounts here routinely contain dots (`je.kim`), which a regex word match +# would treat as "any character". +localusers_backup='/etc/slurm/localusers.backup' + +# grep exits 0 when listed, 1 when definitely not listed, and >1 when the +# lookup itself failed (missing or unreadable file, I/O error). A failed +# lookup is not evidence of absence, so it must not authorise the killall. +# `|| lookup_rc=$?` keeps the non-zero status from tripping `set -e`. +lookup_rc=0 +grep -qxF -- "$SLURM_JOB_USER" "$localusers_backup" || lookup_rc=$? + +case "$lookup_rc" in + 0) + exit 0 # listed: don't kill these users' login sessions + ;; + 1) + : # definitely not listed: fall through to the cleanup below + ;; + *) + logger -s -t slurm-epilog \ + "localusers lookup failed (grep rc=${lookup_rc}, file=${localusers_backup}); skipping process cleanup" + exit 0 + ;; +esac + +if [ "$SLURM_JOB_USER" != root ]; then + if killall -9 -u "$SLURM_JOB_USER" ; then + logger -s -t slurm-epilog 'Killed residual user processes' + fi +fi diff --git a/tests/slurm-epilog/snapshots/user-scope/42-lastuserjob-cleanup b/tests/slurm-epilog/snapshots/user-scope/42-lastuserjob-cleanup new file mode 100644 index 000000000..239273cc3 --- /dev/null +++ b/tests/slurm-epilog/snapshots/user-scope/42-lastuserjob-cleanup @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +set -ex + +# Sibling 40-lastuserjob-processes spares the accounts listed in +# localusers.backup so that operators keep their login session when a job of +# theirs ends. Deleting every file they own under /tmp and /dev/shm undoes that +# from the other side: the session survives but loses the editor swap files, +# sockets and scratch it is holding open. Honour the same exemption. +# +# The list is one account per line, so match whole lines literally (-x -F). +# Accounts here routinely contain dots (`je.kim`), which a regex word match +# would treat as "any character". +localusers_backup='/etc/slurm/localusers.backup' + +# grep exits 0 when listed, 1 when definitely not listed, and >1 when the +# lookup itself failed (missing or unreadable file, I/O error). A failed lookup +# is not evidence of absence, so it must not authorise the deletion. +# `|| lookup_rc=$?` keeps the non-zero status from tripping `set -e`. +lookup_rc=0 +grep -qxF -- "$SLURM_JOB_USER" "$localusers_backup" || lookup_rc=$? + +case "$lookup_rc" in + 0) + exit 0 # listed: leave these users' files alone + ;; + 1) + : # definitely not listed: fall through to the cleanup below + ;; + *) + logger -s -t slurm-epilog \ + "localusers lookup failed (grep rc=${lookup_rc}, file=${localusers_backup}); skipping file cleanup" + exit 0 + ;; +esac + +if [[ -n "$SLURM_JOB_USER" && "$SLURM_JOB_USER" != "root" ]]; then + # `find -user` needs the account to resolve. When the directory service is + # unavailable find exits non-zero without printing anything, and the old + # `||:` turned that into a silent no-op. Worse, a partial listing would + # still be piped into `rm -fr`. Resolve the name once, up front, and skip + # the whole step when it fails: an unresolvable owner is not evidence that + # nothing is owned. + if ! id -u -- "$SLURM_JOB_USER" >/dev/null 2>&1; then + logger -s -t slurm-epilog \ + "cannot resolve ${SLURM_JOB_USER}; skipping file cleanup rather than deleting on a partial listing" + exit 0 + fi + + for dir in /tmp /dev/shm ; do + [ -d "$dir" ] || continue + + # -xdev keeps the walk on this filesystem. Without it the walk descends + # into anything mounted underneath -- a job_container/tmpfs private + # /tmp, a user sshfs, a bind mount of shared storage -- and deletes + # files that belong to another node's still-running job. + # + # It is not enough on its own, because it bounds the walk and not the + # deletion that follows: + # + # - a user-owned directory holding a mount is still selected, and + # `rm -fr` then descends into the mount the walk refused to enter; + # - a user-owned mount root can be selected in its own right, and the + # recursive delete empties the mounted filesystem; + # - a bind mount from this same filesystem is not a boundary to -xdev + # at all, so the walk goes straight in. + # + # So the boundary is taken from the kernel's mount table rather than + # inferred from device numbers. Every mount under $dir is pruned from + # the walk, and a selected directory that still has one beneath it is + # left alone. Its siblings are deleted as before, so cleanup continues + # around a mount instead of stopping at the first one. + if [ ! -r /proc/self/mountinfo ]; then + logger -s -t slurm-epilog \ + "cannot read /proc/self/mountinfo; skipping cleanup under ${dir} rather than deleting across an unknown mount boundary" + continue + fi + + # Field 5 is the mount point. Spaces, tabs, newlines and backslashes in + # it are written as octal escapes, which printf %b turns back into the + # real path. The kernel writes exactly three octal digits (`\040`), but + # %b reads `\0` followed by up to three digits, so `\0401` -- a space + # and then a "1" -- would decode as `\001`. Prefixing each escape with + # a `0` pins the digit count. awk's status is checked for the same + # reason find's is: an + # empty mount list reads as "nothing is mounted here", which would put + # the recursive delete back exactly where it was. + mountlist=$(mktemp) || continue + if ! awk '{print $5}' /proc/self/mountinfo >"$mountlist" 2>/dev/null; then + logger -s -t slurm-epilog \ + "could not read the mount table; skipping cleanup under ${dir}" + rm -f "$mountlist" + continue + fi + + mounts=() + prune=() + while IFS= read -r field; do + printf -v mount_point '%b' "${field//\\/\\0}" + case "$mount_point" in + "$dir"/*) + mounts+=( "$mount_point" ) + pattern=${mount_point//\\/\\\\} + pattern=${pattern//\*/\\*} + pattern=${pattern//\?/\\?} + pattern=${pattern//\[/\\[} + prune+=( -path "$pattern" -prune -o ) + ;; + esac + done <"$mountlist" + rm -f "$mountlist" + + # find's exit status is checked instead of being swallowed: a failed or + # partial walk must not authorise deleting whatever it did manage to + # list. The listing is materialised first for the same reason; piping + # straight into xargs hides find's status behind xargs'. + victims=$(mktemp) || continue + if find "$dir" -xdev "${prune[@]}" -user "$SLURM_JOB_USER" -print0 >"$victims" 2>/dev/null; then + while IFS= read -r -d '' victim; do + nested="" + # The array-length form would open a Jinja comment in this + # template and break rendering, so emptiness is tested through + # the joined elements. A mount point is never an empty string. + if [ -d "$victim" ] && [ -n "${mounts[*]}" ]; then + for mount_point in "${mounts[@]}"; do + case "$mount_point" in + "$victim"/*) nested="$mount_point" ; break ;; + esac + done + fi + + if [ -n "$nested" ]; then + logger -s -t slurm-epilog \ + "leaving ${victim} alone: ${nested} is mounted under it" + continue + fi + + rm -fr -- "$victim" ||: + done <"$victims" + logger -s -t slurm-epilog "Removed residual user files under ${dir}" + else + logger -s -t slurm-epilog \ + "find failed under ${dir} for ${SLURM_JOB_USER}; leaving it alone" + fi + rm -f "$victims" + done +fi diff --git a/tests/slurm-epilog/snapshots/user-scope/50-lastuserjob-all-enroot-dirs b/tests/slurm-epilog/snapshots/user-scope/50-lastuserjob-all-enroot-dirs new file mode 100644 index 000000000..247a3d410 --- /dev/null +++ b/tests/slurm-epilog/snapshots/user-scope/50-lastuserjob-all-enroot-dirs @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -ex + +command -v enroot >/dev/null || exit 0 # enroot not installed + +runtime_path="$(sudo -u "$SLURM_JOB_USER" sh -c 'echo "/run/enroot/user-$(id -u)"')" +rm -rf "$runtime_path" + +data_path="$(sudo -u "$SLURM_JOB_USER" sh -c 'echo "/tmp/enroot-data/user-$(id -u)"')" +rm -rf "$data_path" diff --git a/tests/slurm-epilog/snapshots/user-scope/epilog.sh b/tests/slurm-epilog/snapshots/user-scope/epilog.sh new file mode 100644 index 000000000..2ae4725b8 --- /dev/null +++ b/tests/slurm-epilog/snapshots/user-scope/epilog.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -e + +logger -s -t slurm-epilog "START user=$SLURM_JOB_USER job=$SLURM_JOB_ID" +/etc/slurm/shared/bin/run-parts.sh /etc/slurm/epilog.d +logger -s -t slurm-epilog "END user=$SLURM_JOB_USER job=$SLURM_JOB_ID" diff --git a/tests/slurm-epilog/snapshots/user-scope/run-parts.sh b/tests/slurm-epilog/snapshots/user-scope/run-parts.sh new file mode 100644 index 000000000..cadffcc42 --- /dev/null +++ b/tests/slurm-epilog/snapshots/user-scope/run-parts.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# This could all be done with run-parts using regexes on ubuntu. +# However, centos' version of run-parts is just a simple bash script with no useful flags. +set -e + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 parts_dir" + exit 1 +fi +parts_dir=$1 + +log () { + logger -s -t slurm "$@" +} + +# Use an absolute path: slurmd's PATH does not include a custom +# slurm_install_prefix, and with an empty command result both sides of the +# comparison below were "" and every job was treated as exclusive. +squeue_bin="/usr/bin/squeue" + +# Find out if we are running in exclusive mode. +# Ask squeue for allocated CPUs and node count directly instead of parsing +# "scontrol show job": on recent Slurm the pattern TRES=cpu= matched both the +# ReqTRES= and AllocTRES= lines, yielding a multi-line value that never +# compared equal, so exclusive jobs were never detected. +# +# The lookup runs inside a conditional on purpose. This script has "set -e", so +# a bare assignment from a failing squeue aborts the whole prolog/epilog run and +# fails the job; redirecting stderr does not suppress the exit status. A failed, +# empty or non-numeric lookup must instead leave exclusive=0, which only skips +# the *-exclusive-* scripts and lets every other script run. +exclusive=0 +if job_alloc=$("$squeue_bin" -h -j "$SLURM_JOBID" -o "%C %D" 2>/dev/null); then + read -r numcpus_job numnodes_job <<<"$job_alloc" || true + if [[ "$numcpus_job" =~ ^[0-9]+$ ]] && [[ "$numnodes_job" =~ ^[0-9]+$ ]]; then + numcpus_sys=$(( $(grep -c ^processor /proc/cpuinfo) * numnodes_job )) + if [ "$numcpus_sys" -eq "$numcpus_job" ]; then + exclusive=1 + fi + else + log "[WARN] squeue returned no usable allocation for job ${SLURM_JOBID} ('${job_alloc}'); treating the job as non-exclusive." + fi +else + log "[WARN] squeue failed for job ${SLURM_JOBID}; treating the job as non-exclusive." +fi + +# Find out if there are any more jobs on this node for this user. +# Same reasoning as above with one difference: a failed lookup must not be read +# as "no other jobs", because that would run the *-lastuserjob-* cleanup scripts +# while another job of the same user is still on the node. Piping squeue into +# "wc -l" also hides its exit status, so the status is captured explicitly. +# +# RUNNING alone is too narrow. A job that is SUSPENDED (scontrol suspend, or +# preempted with SuspendTime) or STOPPED still owns its processes, its files in +# /tmp and /dev/shm, and its enroot directories; a job that is CONFIGURING or +# RESIZING has an allocation on this node and is about to. None of those appear +# under "-t running", so the *-lastuserjob-* scripts would run and reap them. +# +# COMPLETING is deliberately absent. The epilog runs while its own job is in +# that state, so including it would make every job find itself, leave +# last_user_job at 0, and disable the cleanup entirely. +lastuserjob_states='running,suspended,stopped,configuring,resizing' +last_user_job=0 +if user_jobs=$("$squeue_bin" -h -u "$SLURM_JOB_USER" -w "$HOSTNAME" -t "$lastuserjob_states" -o '%i' 2>/dev/null); then + # Drop this job's own entry defensively. It should not be in the states + # above, but an id match is cheap and keeps a future state addition from + # silently switching the cleanup off. + other_jobs=$(printf '%s\n' "$user_jobs" | grep -vxF "$SLURM_JOB_ID" || :) + if [ -z "$other_jobs" ]; then + last_user_job=1 + fi +else + log "[WARN] squeue failed while counting jobs for ${SLURM_JOB_USER} on ${HOSTNAME}; not running the last-user-job scripts." +fi + +# Re-implement run-parts since on centos it is just a bash script with no useful flags. +failed=0 +for script in "$parts_dir"/*; do + if [ ! -x "$script" ]; then + log "Skipping $script because it is not executable." + continue + fi + if [[ "$script" == *"-exclusive-"* ]] && [ "$exclusive" = 0 ]; then + log "Skipping $script because the job was not run in exclusive mode." + continue + fi + if [[ "$script" == *"-lastuserjob-"* ]] && [ "$last_user_job" = 0 ]; then + log "Skipping $script because there is still another job running on this node for the same user." + continue + fi + log "Running $script ..." + if ! $script >>/var/log/slurm/prolog-epilog 2>&1; then + log "[ERROR] $script failed. Check the log at /var/log/slurm/prolog-epilog for more details." + failed=1 + fi +done + +if [ "$failed" = "1" ]; then + log "One or more scripts failed." + exit 1 +fi From ea6d473e299ee3bdf0920a30b993a88cec393327 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Mon, 12 Oct 2026 10:14:27 -0600 Subject: [PATCH 4/6] docs(slurm): job-scoped epilog cleanup behaviour, user switch and job_container/tmpfs (#1407) Signed-off-by: Doug Holt --- docs/slurm-cluster/README.md | 2 + .../epilog-job-scoped-cleanup.md | 127 ++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 docs/slurm-cluster/epilog-job-scoped-cleanup.md diff --git a/docs/slurm-cluster/README.md b/docs/slurm-cluster/README.md index 382759074..1242928e7 100644 --- a/docs/slurm-cluster/README.md +++ b/docs/slurm-cluster/README.md @@ -195,6 +195,8 @@ Read through the [slurm usage guide](slurm-usage.md) for more information. The default Slurm deployment includes a collection of prolog and epilog scripts that should be modified to suit a particular system. For more information, see the [prolog/epilog documentation](slurm-prolog-epilog/README.md). +From release 26.11 the cleanup that runs when a user's job ends is job-scoped by default and no longer touches the user's other running jobs on the node; `slurm_epilog_cleanup_scope: user` restores the previous behaviour for one release. See [Job-scoped epilog cleanup](epilog-job-scoped-cleanup.md). + ### Lua plugins Slurm's Lua plugins (for example `JobSubmitPlugins=lua` or `CliFilterPlugins=lua`) only load when Slurm was built with Lua support. DeepOps does not request Lua support or install its development package by default; Slurm may still detect a preinstalled Lua library. To enable it, set the following in `config/group_vars/slurm-cluster.yml` and run the Slurm playbook: diff --git a/docs/slurm-cluster/epilog-job-scoped-cleanup.md b/docs/slurm-cluster/epilog-job-scoped-cleanup.md new file mode 100644 index 000000000..cf4e637ea --- /dev/null +++ b/docs/slurm-cluster/epilog-job-scoped-cleanup.md @@ -0,0 +1,127 @@ +# Job-scoped epilog cleanup + +Starting with release 26.11, the Slurm epilog cleans up after a finished job +without touching other jobs of the same user that are running on the same node. +This page explains what changed, why, and how to go back to the old behaviour +for one release. + +## What changed + +When a job ends, the epilog (`/etc/slurm/epilog.d/*-lastuserjob-*`) removes what +the user left behind on the node: stray processes, files in `/tmp` and +`/dev/shm`, enroot containers, and the user's SSH access entry. + +Before 26.11 this was decided from the controller's view alone: the epilog asked +`squeue` whether the user had another job on the node, and if not it killed +every process the user owned (`killall -9 -u`) and deleted everything they owned +in `/tmp` and `/dev/shm`. Two things could go wrong: + +- A new job of the same user could be placed on the node just after `squeue` + answered, and have its processes killed a moment later. +- The controller can lag behind the node, and it cannot see processes that left + the job's cgroup. + +With `slurm_epilog_cleanup_scope: job` (the new default) the epilog also looks +at the node itself. Every process of a Slurm job lives in a `job_` cgroup, so +the epilog reads `/proc//cgroup` (cgroup v1 and v2 are both understood) and +decides per process: + +| Process is in… | Epilog does | +|---|---| +| the ending job's cgroup | kills it (Slurm does too; this is a backstop) | +| another job's cgroup | never touches it, whatever `squeue` said | +| no job cgroup (an orphan, for example a leftover daemon) | kills it only when the user has no other job on the node | +| a cgroup entry it cannot read or recognise | leaves it alone and logs it | + +The file cleanup (`/tmp`, `/dev/shm`), enroot directory removal, and SSH access +removal still happen per user, because files carry a user id and not a job id. +They now run only when neither `squeue` nor the node's cgroups show another job +of that user. If that check cannot be completed, the cleanup is skipped and +logged, then happens when the user's next job ends. A skipped cleanup is a delay, +not a loss. + +In short: **less is deleted, never more.** Nothing that was spared before is +removed now. + +### What you may notice + +- If a user's job is still running when another of their jobs ends, that job's + processes, `/tmp` files and enroot containers now survive. Previously they + could be removed. +- After the user's last job ends, leftovers are still removed as before. +- `/tmp` cleanup can be delayed on a node where the cgroup data cannot be read. + The epilog logs a `[WARN]` line (`slurm` tag in the journal, also in + `/var/log/slurm/prolog-epilog`) when it defers. +- Any extra `*-lastuserjob-*` script you added under `/etc/slurm/epilog.d` + follows the same, stricter guard and therefore runs less often. +- Accounts listed in `/etc/slurm/localusers.backup`, and root, are exempt as + before. +- Requires cgroup-based process tracking (`ProctrackType=proctrack/cgroup`, the + DeepOps default). With a different process tracker the epilog cannot find job + cgroups and will defer cleanup; use `user` (below) or fix the tracker. + +### Related fix: SSH access entry + +The script that removes a user's SSH access entry (`41-lastuserjob-ssh`) matched +names as patterns rather than whole lines: ending a job for user `al` could also +remove the entries for `alice` or `halley`, and a dot in a name matched any +character. It now matches the exact line. This applies in both scopes. + +## Going back to the 26.09 behaviour + +If you need the old behaviour, set this in `config/group_vars/slurm-cluster.yml` +and re-run the Slurm playbook (`ansible-playbook playbooks/slurm-cluster.yml`) so the epilog scripts are re-rendered on the compute nodes: + +```yaml +slurm_epilog_cleanup_scope: user +``` + +The scripts for `user` are the 26.09 scripts byte-for-byte (apart from the SSH +match fix above). The playbook prints a deprecation notice when `user` is set. + +**`user` is deprecated and will be removed in release 27.01.** It exists for one +release so a site with an unusual setup, for example a custom epilog, a process +tracker other than cgroup, or user daemons that live outside any job cgroup, can +keep the old behaviour while it reports the gap. If you need `user`, please open +an issue at https://github.com/NVIDIA/deepops/issues saying why. + +## Per-job `/tmp` and `/dev/shm`: `job_container/tmpfs` + +The complete answer for per-job temporary files is Slurm's own +`job_container/tmpfs` plugin, which gives each job a private `/tmp` and +`/dev/shm` that disappear when the job ends. DeepOps does not configure it for +you, but if you enable it in `slurm.conf` the epilog notices +(`JobContainerType=job_container/tmpfs`) and skips its own `/tmp` and `/dev/shm` +file cleanup, since the plugin already handles it: + +``` +JobContainerType=job_container/tmpfs +``` + +and add a `job_container.conf` next to `slurm.conf`. See the Slurm documentation +for `job_container.conf` (the `BasePath` must exist and be writable by slurmd on +each compute node). Wiring this into the role is tracked separately. + +## Enroot and Pyxis + +The enroot cleanup removes the user's runtime and data directories, which are +shared by all of that user's jobs on the node. Per-job enroot directories were +considered and not adopted, because people rely on containers persisting between +jobs. If you want containers that disappear with the job, Pyxis has a +container scope setting for job-lifetime containers; see the Pyxis documentation +for your version. + +## Checking what the epilog did + +Each epilog logs through `logger -t slurm-epilog` and `-t slurm`. Typical lines: + +``` +slurm-epilog: Job-scoped process cleanup for alice job 4711: killed 3, spared 12, unclassified 0 +slurm: Another job of alice has processes on this node (found by cgroup); not running the last-user-job scripts. +``` + +## Testing + +The unprivileged unit tests are in `tests/slurm-epilog/run-tests.sh` and run in CI. +An overlapping-jobs fixture for a disposable cluster is in +`tests/slurm-epilog/overlapping-jobs/`. From b777748f1feb97b2869cb337c531de384929b0c0 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Mon, 12 Oct 2026 10:14:34 -0600 Subject: [PATCH 5/6] docs(slurm): drop reference to a fixture not in this change (#1407) Signed-off-by: Doug Holt --- docs/slurm-cluster/epilog-job-scoped-cleanup.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/slurm-cluster/epilog-job-scoped-cleanup.md b/docs/slurm-cluster/epilog-job-scoped-cleanup.md index cf4e637ea..5d04044d3 100644 --- a/docs/slurm-cluster/epilog-job-scoped-cleanup.md +++ b/docs/slurm-cluster/epilog-job-scoped-cleanup.md @@ -123,5 +123,4 @@ slurm: Another job of alice has processes on this node (found by cgroup); not ru ## Testing The unprivileged unit tests are in `tests/slurm-epilog/run-tests.sh` and run in CI. -An overlapping-jobs fixture for a disposable cluster is in -`tests/slurm-epilog/overlapping-jobs/`. +They use a fake `/proc` and cover both cgroup layouts. From 7b529ca0590f2d83052e8c52bdb89d4b285c23fc Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Thu, 22 Oct 2026 06:33:59 -0600 Subject: [PATCH 6/6] docs(slurm): describe non-cgroup tracker fallback and processes started outside Slurm (#1407) --- docs/slurm-cluster/epilog-job-scoped-cleanup.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/slurm-cluster/epilog-job-scoped-cleanup.md b/docs/slurm-cluster/epilog-job-scoped-cleanup.md index 5d04044d3..759869222 100644 --- a/docs/slurm-cluster/epilog-job-scoped-cleanup.md +++ b/docs/slurm-cluster/epilog-job-scoped-cleanup.md @@ -57,8 +57,14 @@ removed now. - Accounts listed in `/etc/slurm/localusers.backup`, and root, are exempt as before. - Requires cgroup-based process tracking (`ProctrackType=proctrack/cgroup`, the - DeepOps default). With a different process tracker the epilog cannot find job - cgroups and will defer cleanup; use `user` (below) or fix the tracker. + DeepOps default). With a different process tracker the epilog cannot tell + which job a process belongs to, so it treats every process as unassigned and + falls back to asking `squeue`: if the user has another job on the node nothing + is cleaned up, otherwise all of the user's processes are removed. It does not + defer. Use `user` (below) or fix the tracker if that is not what you want. +- Processes started outside Slurm while a job is finishing, for example over + SSH, are not protected: they are not part of any job, so they count as + unassigned and can be removed when the user's last job ends. ### Related fix: SSH access entry