Crash handler logging, documentation

.github/workflows/ci.yml

@@ -169,6 +169,14 @@ jobs:
         run: |
           docker exec node-1 bash -c 'cd Spindle-build/testsuite && flux alloc --nodes=${workers} ./run_crash_tests.sh --launcher=flux --nodes=${workers} --scratch=/shared/spindle_crash_test'
 
+      - name: Run spindle-flux-ubuntu session crash test
+        run: |
+          docker exec node-1 bash -c 'cd Spindle-build/testsuite && flux alloc --nodes=${workers} ./run_crash_tests.sh --launcher=flux --nodes=${workers} --scratch=/shared/spindle_crash_test --session'
+
+      - name: Run spindle-flux-ubuntu cross-executable crash test
+        run: |
+          docker exec node-1 bash -c 'cd Spindle-build/testsuite && flux alloc --nodes=${workers} ./run_crash_tests.sh --launcher=flux --nodes=${workers} --scratch=/shared/spindle_crash_test --cross-exe'
+
       - name: Bring spindle-flux-ubuntu down
         id: flux-ubuntu-down
         if: ${{ always() }}
@@ -366,6 +374,14 @@ jobs:
         run: |
           docker exec slurm-plugin-head bash -c 'cd Spindle-build/testsuite && ./run_crash_tests.sh --launcher=slurm-plugin --nodes=${workers} --scratch=/shared/spindle_crash_test'
 
+      - name: Run spindle-slurm-plugin-ubuntu session crash test
+        run: |
+          docker exec slurm-plugin-head bash -c 'cd Spindle-build/testsuite && ./run_crash_tests.sh --launcher=slurm-plugin --nodes=${workers} --scratch=/shared/spindle_crash_test --session'
+
+      - name: Run spindle-slurm-plugin-ubuntu cross-executable crash test
+        run: |
+          docker exec slurm-plugin-head bash -c 'cd Spindle-build/testsuite && ./run_crash_tests.sh --launcher=slurm-plugin --nodes=${workers} --scratch=/shared/spindle_crash_test --cross-exe'
+
       - name: Bring spindle-slurm-plugin-ubuntu down
         id: slurm-ubuntu-down
         if: ${{ always() }}

README

@@ -56,6 +56,10 @@ Spindle can trigger scalable loading of dlopened libraries, dependent
 library, executables, python modules and specified application data
 files.
 
+Spindle can also coordinate coredump creation when a large job
+crashes.  Rather than letting every process write a coredump, Spindle
+selects one representative process per unique crash location, and can
+write a log of which ranks crashed where.
 
 Compilation:
 ============

containers/spindle-slurm-ubuntu/testing-srun/Dockerfile

@@ -2,9 +2,17 @@ ARG BASE_VERSION=latest
 FROM ghcr.io/llnl/spindle-slurm-base:${BASE_VERSION}
 ARG replicas=4
 ENV workers=${replicas}
+ENV TMPDIR=/tmp
+RUN echo 'TMPDIR="/tmp"' >> /etc/environment
+ENV SPINDLE_TEST_CONTAINER=1
 
 ARG BUILD_ROOT=containers/spindle-slurm-ubuntu/testing-srun
 
+RUN apt-get update \
+    && DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \
+    gdb \
+    libc6-dbg
+
 # Slurm daemons run as $SLURM_USER
 ARG SLURM_USER=slurm
 

containers/spindle-slurm-ubuntu/testing-srun/docker-compose.yml

@@ -35,13 +35,19 @@ networks:
   slurm:
     driver: bridge
 
+volumes:
+  crash_scratch:
+
 # Common parameters for all nodes.
 x-shared-node-parameters: &shared-node-parameters
   build: *shared-build-context
   networks:
     - slurm
+  volumes:
+    - crash_scratch:/shared
   cap_add:
     - SYS_NICE # Required for libnuma
+    - SYS_RESOURCE # Allow changing ulimit
 
 x-healthcheck-parameters: &healthcheck-parameters
   start_period: 3s

containers/spindle-slurm-ubuntu/testing-srun/scripts/entrypoint.sh

@@ -3,7 +3,12 @@
 echo "SLURM_ROLE: ${SLURM_ROLE}"
 
 echo "Starting munged..."
-sudo -u munge /usr/sbin/munged 
+sudo -u munge /usr/sbin/munged
+
+if [ -d /shared ]; then
+    sudo chown -R "$(id -un):$(id -gn)" /shared
+    sudo chmod 755 /shared
+fi
 
 if [ "${SLURM_ROLE}" = "db" ]; then
     echo "Starting slurmdbd..."
@@ -13,7 +18,7 @@ elif [ "${SLURM_ROLE}" = "ctl" ] ; then
     sudo -u slurm /usr/sbin/slurmctld -i -Dvvv
 elif [ "${SLURM_ROLE}" = "worker" ] ; then
     echo "Starting slurmd..."
-    sudo /usr/sbin/slurmd -Dvvv
+    sudo bash -c 'ulimit -c unlimited; exec /usr/sbin/slurmd -Dvvv'
 fi
 
 sleep inf

doc/spindle.1

@@ -116,6 +116,18 @@ By default Spindle will attempt to auto-detect the MPI implementation from the l
 \fB\-d \fIyes\fR|\fIno\fR, \fR\-\-debug=\fIyes\fR|\fIno\fR
 If yes, Spindle will adjust its operations so that debuggers can also attach to spindle-controlled processes.  Note that there may be other factors outside of Spindle's control that may still prevent debuggers from working on Spindle-controlled processes.  As of this writing, \fB\-\-debug=yes\fR will allow gdb to attach to a Spindle process, but not TotalView.  This option may also cause extra overhead when starting processes.  This option defaults to no.
 
+.TP
+\fB\-\-crash\-dedup=\fIyes\fR|\fIno\fR
+If yes, Spindle will coordinate coredump creation when application processes crash.  Processes that crash at the same location are grouped together, with one process for each unique crash site allowed to write a coredump while the others are suppressed. Default is no.
+
+.TP
+\fB\-\-crash\-altstack=\fIyes\fR|\fIno\fR
+If yes, Spindle's crash handling signal handlers will run on an alternate stack.  This allows crashes caused by stack overflow on the main thread to be handled and deduplicated, but this may alter application behavior.  Default is no.
+
+.TP
+\fB\-\-crash\-log\fR[=\fIPATH\fR]
+Write a log file listing the ranks that crashed at each crash location.  This option implies \fB\-\-crash\-dedup\fR.  If \fIPATH\fR is omitted, the log is written to spindle-crash-log.NUMBER in the working directory.  If \fIPATH\fR is an existing directory, the log is written into that directory. Environment variables can be used in \fIPATH\fR by prefixing them with a '$' character.
+
 .TP
 \fB\-e \fIFILE\fR, \fB\-\-preload=\fIFILE\fR
 Provides a text file containing white-space separated filenames.  Spindle will preload the files in \fIFILE\fR onto each node before starting process execution.  

doc/spindle_crash_handler_README.md

@@ -0,0 +1,125 @@
+Spindle Crash Handler
+=====================
+
+This README describes Spindle's crash handling features, which control
+coredump creation and report crash locations when an application crashes.
+
+Overview
+--------
+
+When a large parallel application hits a bug, many processes tend to
+crash at the same place.  If coredumps are enabled, every crashing
+process writes its own coredump, which can flood a shared file system
+with thousands of near-identical files.
+
+With crash handling enabled, each crashing process reports its crash
+location to Spindle.  Spindle groups the reports by crash site and
+selects one process per unique site to write a coredump; coredumps
+from the other processes at that site are suppressed. Spindle can 
+also write a crash log summarizing the crash sites and which ranks
+crashed at each.
+
+Crash handling covers crashes from SIGSEGV, SIGBUS, SIGFPE, SIGILL,
+and SIGABRT. 
+
+Usage
+-----
+
+Pass `--crash-dedup` to the spindle command to enable coredump
+deduplication:
+
+    spindle --crash-dedup srun -n 512 ./my_app
+
+Add `--crash-log` to also write a crash log.  Since `--crash-log`
+implies `--crash-dedup`, it can be used on its own:
+
+    spindle --crash-log=/p/lustre/me/logs srun -n 512 ./my_app
+
+As with other coredumps, the usual system settings apply.  Make sure
+the core file size limit (`ulimit -c`) allows coredumps on the compute
+nodes, or the selected process will not be able to write one.
+
+When Spindle runs through a resource manager plugin rather than the
+spindle command, pass the same options through the plugin.  With the
+Slurm plugin:
+
+    srun --spindle="--crash-log" -n 512 ./my_app
+
+With a Slurm plugin session, give the options when the session starts
+and they apply to every job run in that session:
+
+    salloc -N4 --spindle-session="--crash-log" ./my_jobs.sh
+
+With the Flux plugin, use the corresponding shell options:
+
+    flux run -o spindle.crash-dedup -o spindle.crash-log -N4 -n512 ./my_app
+
+Crash sites
+-----------
+
+A crash site identifies where the application crashed.  For most
+signals it is the executable plus the library and offset of the
+faulting instruction, such as:
+
+    exe: /home/me/my_app
+    site: libfoo.so.1+0x2f10
+
+For SIGABRT crashes that carry a glibc abort message, such as a failed
+`assert()` or a heap corruption report, the abort message is used as
+the crash site:
+
+    exe: /home/me/my_app
+    site: abort:my_app: solver.c:88: solve: Assertion `n > 0' failed.
+
+The crash log
+-------------
+
+The `--crash-log[=PATH]` option writes a log describing every crash in
+the job.  If `PATH` is omitted, the log is written to
+`spindle-crash-log.$NUMBER` in the working directory, where NUMBER is
+Spindle's session number.  `PATH` may name the log file itself or an
+existing directory to place the default filename in.  Environment
+variables can be used in `PATH` by prefixing them with a `$`
+character.
+
+The log contains one entry per crash site, separated by blank lines:
+
+    exe: /home/me/my_app
+    site: libsolver.so.1+0x2f10
+    exemplar: 4
+    count: 12
+    ranks: 4-15
+
+    exe: /home/me/my_app
+    site: abort:my_app: solver.c:88: solve: Assertion `n > 0' failed.
+    exemplar: 0
+    count: 4
+    ranks: 0-3
+
+The `exemplar` is the rank that was selected to write the coredump for 
+that crash site.  The `ranks` field lists every rank that crashed there.
+The log is written when the job exits, or at session end when running
+in session mode.  If no process crashed, no log file is created.
+
+Interaction with application signal handlers
+--------------------------------------------
+
+Applications that install their own handlers for crash signals keep
+working under Spindle.  When a crash signal arrives, Spindle invokes
+the application's handler first.  Some runtimes handle faults as part
+of normal operation, such as garbage collectors that trap writes
+to protected pages. If the application's handler resolves the fault,
+Spindle does not handle the signal and execution continues.
+
+Stack overflows
+---------------
+
+A crash caused by stack overflow cannot normally run a signal
+handler, because the handler has no stack to run on.  The
+`--crash-altstack` option makes Spindle's crash handling run on an
+alternate signal stack, so stack-overflow crashes on the
+application's main thread can be handled and deduplicated like any
+other crash. This will cause any application-registered signal handlers for
+SIGSEGV, SIGBUS, SIGFPE, SIGILL, or SIGABRT to also run on the alternate
+stack, which may alter application behavior. This option is off by default.
+

doc/spindle_launch_README.md

@@ -126,6 +126,13 @@ we have to break ABI compatibility.
         Spindle deduplicates the reports by crash site, selecting one
         representative per unique site, permitting only those representatives
         to write a coredump. 
+   -    'OPT_CRASH_LOG' - Write a log at job end listing which ranks
+        crashed at which crash sites.  The log path is given in the
+        `crash_log` field of `spindle_args_t`.  Requires
+        `OPT_CRASH_HANDLER`.
+   -    'OPT_CRASH_ALTSTACK' - The crash handler registers an alternate
+        signal stack on the application's main thread, allowing it to
+        handle crashes caused by stack overflow.
 
 - `typedef struct { ... } spindle_args_t`
 
@@ -190,6 +197,9 @@ we have to break ABI compatibility.
     -   `char *preloadfile` - Points to a file containing a white-space
         separated list of files that should be staged onto every node in the
         job before the application runs.
+    -   `char *crash_log` - When `OPT_CRASH_LOG` is set, the path of the
+        crash log file.  Should be an absolute path on a file system
+        accessible by the Spindle servers.
 
 The FrontEnd API
 ----------------