diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7e4708d5..41dd3ca5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -169,6 +169,14 @@ jobs: run: | docker exec node-1 bash -c 'cd Spindle-build/testsuite && flux alloc --nodes=${workers} ./run_crash_tests.sh --launcher=flux --nodes=${workers} --scratch=/shared/spindle_crash_test' + - name: Run spindle-flux-ubuntu session crash test + run: | + docker exec node-1 bash -c 'cd Spindle-build/testsuite && flux alloc --nodes=${workers} ./run_crash_tests.sh --launcher=flux --nodes=${workers} --scratch=/shared/spindle_crash_test --session' + + - name: Run spindle-flux-ubuntu cross-executable crash test + run: | + docker exec node-1 bash -c 'cd Spindle-build/testsuite && flux alloc --nodes=${workers} ./run_crash_tests.sh --launcher=flux --nodes=${workers} --scratch=/shared/spindle_crash_test --cross-exe' + - name: Bring spindle-flux-ubuntu down id: flux-ubuntu-down if: ${{ always() }} @@ -366,6 +374,14 @@ jobs: run: | docker exec slurm-plugin-head bash -c 'cd Spindle-build/testsuite && ./run_crash_tests.sh --launcher=slurm-plugin --nodes=${workers} --scratch=/shared/spindle_crash_test' + - name: Run spindle-slurm-plugin-ubuntu session crash test + run: | + docker exec slurm-plugin-head bash -c 'cd Spindle-build/testsuite && ./run_crash_tests.sh --launcher=slurm-plugin --nodes=${workers} --scratch=/shared/spindle_crash_test --session' + + - name: Run spindle-slurm-plugin-ubuntu cross-executable crash test + run: | + docker exec slurm-plugin-head bash -c 'cd Spindle-build/testsuite && ./run_crash_tests.sh --launcher=slurm-plugin --nodes=${workers} --scratch=/shared/spindle_crash_test --cross-exe' + - name: Bring spindle-slurm-plugin-ubuntu down id: slurm-ubuntu-down if: ${{ always() }} diff --git a/README b/README index 8af82804..2518dc95 100644 --- a/README +++ b/README @@ -56,6 +56,10 @@ Spindle can trigger scalable loading of dlopened libraries, dependent library, executables, python modules and specified application data files. +Spindle can also coordinate coredump creation when a large job +crashes. Rather than letting every process write a coredump, Spindle +selects one representative process per unique crash location, and can +write a log of which ranks crashed where. Compilation: ============ diff --git a/containers/spindle-slurm-ubuntu/testing-srun/Dockerfile b/containers/spindle-slurm-ubuntu/testing-srun/Dockerfile index 5b281c3f..acd403a7 100644 --- a/containers/spindle-slurm-ubuntu/testing-srun/Dockerfile +++ b/containers/spindle-slurm-ubuntu/testing-srun/Dockerfile @@ -2,9 +2,17 @@ ARG BASE_VERSION=latest FROM ghcr.io/llnl/spindle-slurm-base:${BASE_VERSION} ARG replicas=4 ENV workers=${replicas} +ENV TMPDIR=/tmp +RUN echo 'TMPDIR="/tmp"' >> /etc/environment +ENV SPINDLE_TEST_CONTAINER=1 ARG BUILD_ROOT=containers/spindle-slurm-ubuntu/testing-srun +RUN apt-get update \ + && DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \ + gdb \ + libc6-dbg + # Slurm daemons run as $SLURM_USER ARG SLURM_USER=slurm diff --git a/containers/spindle-slurm-ubuntu/testing-srun/docker-compose.yml b/containers/spindle-slurm-ubuntu/testing-srun/docker-compose.yml index 39332e3e..6c19a745 100644 --- a/containers/spindle-slurm-ubuntu/testing-srun/docker-compose.yml +++ b/containers/spindle-slurm-ubuntu/testing-srun/docker-compose.yml @@ -35,13 +35,19 @@ networks: slurm: driver: bridge +volumes: + crash_scratch: + # Common parameters for all nodes. x-shared-node-parameters: &shared-node-parameters build: *shared-build-context networks: - slurm + volumes: + - crash_scratch:/shared cap_add: - SYS_NICE # Required for libnuma + - SYS_RESOURCE # Allow changing ulimit x-healthcheck-parameters: &healthcheck-parameters start_period: 3s diff --git a/containers/spindle-slurm-ubuntu/testing-srun/scripts/entrypoint.sh b/containers/spindle-slurm-ubuntu/testing-srun/scripts/entrypoint.sh index 5b7b8c17..ffcdb67e 100755 --- a/containers/spindle-slurm-ubuntu/testing-srun/scripts/entrypoint.sh +++ b/containers/spindle-slurm-ubuntu/testing-srun/scripts/entrypoint.sh @@ -3,7 +3,12 @@ echo "SLURM_ROLE: ${SLURM_ROLE}" echo "Starting munged..." -sudo -u munge /usr/sbin/munged +sudo -u munge /usr/sbin/munged + +if [ -d /shared ]; then + sudo chown -R "$(id -un):$(id -gn)" /shared + sudo chmod 755 /shared +fi if [ "${SLURM_ROLE}" = "db" ]; then echo "Starting slurmdbd..." @@ -13,7 +18,7 @@ elif [ "${SLURM_ROLE}" = "ctl" ] ; then sudo -u slurm /usr/sbin/slurmctld -i -Dvvv elif [ "${SLURM_ROLE}" = "worker" ] ; then echo "Starting slurmd..." - sudo /usr/sbin/slurmd -Dvvv + sudo bash -c 'ulimit -c unlimited; exec /usr/sbin/slurmd -Dvvv' fi sleep inf diff --git a/doc/spindle.1 b/doc/spindle.1 index 11c4182c..68716383 100644 --- a/doc/spindle.1 +++ b/doc/spindle.1 @@ -116,6 +116,18 @@ By default Spindle will attempt to auto-detect the MPI implementation from the l \fB\-d \fIyes\fR|\fIno\fR, \fR\-\-debug=\fIyes\fR|\fIno\fR If yes, Spindle will adjust its operations so that debuggers can also attach to spindle-controlled processes. Note that there may be other factors outside of Spindle's control that may still prevent debuggers from working on Spindle-controlled processes. As of this writing, \fB\-\-debug=yes\fR will allow gdb to attach to a Spindle process, but not TotalView. This option may also cause extra overhead when starting processes. This option defaults to no. +.TP +\fB\-\-crash\-dedup=\fIyes\fR|\fIno\fR +If yes, Spindle will coordinate coredump creation when application processes crash. Processes that crash at the same location are grouped together, with one process for each unique crash site allowed to write a coredump while the others are suppressed. Default is no. + +.TP +\fB\-\-crash\-altstack=\fIyes\fR|\fIno\fR +If yes, Spindle's crash handling signal handlers will run on an alternate stack. This allows crashes caused by stack overflow on the main thread to be handled and deduplicated, but this may alter application behavior. Default is no. + +.TP +\fB\-\-crash\-log\fR[=\fIPATH\fR] +Write a log file listing the ranks that crashed at each crash location. This option implies \fB\-\-crash\-dedup\fR. If \fIPATH\fR is omitted, the log is written to spindle-crash-log.NUMBER in the working directory. If \fIPATH\fR is an existing directory, the log is written into that directory. Environment variables can be used in \fIPATH\fR by prefixing them with a '$' character. + .TP \fB\-e \fIFILE\fR, \fB\-\-preload=\fIFILE\fR Provides a text file containing white-space separated filenames. Spindle will preload the files in \fIFILE\fR onto each node before starting process execution. diff --git a/doc/spindle_crash_handler_README.md b/doc/spindle_crash_handler_README.md new file mode 100644 index 00000000..0e838e66 --- /dev/null +++ b/doc/spindle_crash_handler_README.md @@ -0,0 +1,125 @@ +Spindle Crash Handler +===================== + +This README describes Spindle's crash handling features, which control +coredump creation and report crash locations when an application crashes. + +Overview +-------- + +When a large parallel application hits a bug, many processes tend to +crash at the same place. If coredumps are enabled, every crashing +process writes its own coredump, which can flood a shared file system +with thousands of near-identical files. + +With crash handling enabled, each crashing process reports its crash +location to Spindle. Spindle groups the reports by crash site and +selects one process per unique site to write a coredump; coredumps +from the other processes at that site are suppressed. Spindle can +also write a crash log summarizing the crash sites and which ranks +crashed at each. + +Crash handling covers crashes from SIGSEGV, SIGBUS, SIGFPE, SIGILL, +and SIGABRT. + +Usage +----- + +Pass `--crash-dedup` to the spindle command to enable coredump +deduplication: + + spindle --crash-dedup srun -n 512 ./my_app + +Add `--crash-log` to also write a crash log. Since `--crash-log` +implies `--crash-dedup`, it can be used on its own: + + spindle --crash-log=/p/lustre/me/logs srun -n 512 ./my_app + +As with other coredumps, the usual system settings apply. Make sure +the core file size limit (`ulimit -c`) allows coredumps on the compute +nodes, or the selected process will not be able to write one. + +When Spindle runs through a resource manager plugin rather than the +spindle command, pass the same options through the plugin. With the +Slurm plugin: + + srun --spindle="--crash-log" -n 512 ./my_app + +With a Slurm plugin session, give the options when the session starts +and they apply to every job run in that session: + + salloc -N4 --spindle-session="--crash-log" ./my_jobs.sh + +With the Flux plugin, use the corresponding shell options: + + flux run -o spindle.crash-dedup -o spindle.crash-log -N4 -n512 ./my_app + +Crash sites +----------- + +A crash site identifies where the application crashed. For most +signals it is the executable plus the library and offset of the +faulting instruction, such as: + + exe: /home/me/my_app + site: libfoo.so.1+0x2f10 + +For SIGABRT crashes that carry a glibc abort message, such as a failed +`assert()` or a heap corruption report, the abort message is used as +the crash site: + + exe: /home/me/my_app + site: abort:my_app: solver.c:88: solve: Assertion `n > 0' failed. + +The crash log +------------- + +The `--crash-log[=PATH]` option writes a log describing every crash in +the job. If `PATH` is omitted, the log is written to +`spindle-crash-log.$NUMBER` in the working directory, where NUMBER is +Spindle's session number. `PATH` may name the log file itself or an +existing directory to place the default filename in. Environment +variables can be used in `PATH` by prefixing them with a `$` +character. + +The log contains one entry per crash site, separated by blank lines: + + exe: /home/me/my_app + site: libsolver.so.1+0x2f10 + exemplar: 4 + count: 12 + ranks: 4-15 + + exe: /home/me/my_app + site: abort:my_app: solver.c:88: solve: Assertion `n > 0' failed. + exemplar: 0 + count: 4 + ranks: 0-3 + +The `exemplar` is the rank that was selected to write the coredump for +that crash site. The `ranks` field lists every rank that crashed there. +The log is written when the job exits, or at session end when running +in session mode. If no process crashed, no log file is created. + +Interaction with application signal handlers +-------------------------------------------- + +Applications that install their own handlers for crash signals keep +working under Spindle. When a crash signal arrives, Spindle invokes +the application's handler first. Some runtimes handle faults as part +of normal operation, such as garbage collectors that trap writes +to protected pages. If the application's handler resolves the fault, +Spindle does not handle the signal and execution continues. + +Stack overflows +--------------- + +A crash caused by stack overflow cannot normally run a signal +handler, because the handler has no stack to run on. The +`--crash-altstack` option makes Spindle's crash handling run on an +alternate signal stack, so stack-overflow crashes on the +application's main thread can be handled and deduplicated like any +other crash. This will cause any application-registered signal handlers for +SIGSEGV, SIGBUS, SIGFPE, SIGILL, or SIGABRT to also run on the alternate +stack, which may alter application behavior. This option is off by default. + diff --git a/doc/spindle_launch_README.md b/doc/spindle_launch_README.md index a30c8df8..66ba3c4f 100644 --- a/doc/spindle_launch_README.md +++ b/doc/spindle_launch_README.md @@ -126,6 +126,13 @@ we have to break ABI compatibility. Spindle deduplicates the reports by crash site, selecting one representative per unique site, permitting only those representatives to write a coredump. + - 'OPT_CRASH_LOG' - Write a log at job end listing which ranks + crashed at which crash sites. The log path is given in the + `crash_log` field of `spindle_args_t`. Requires + `OPT_CRASH_HANDLER`. + - 'OPT_CRASH_ALTSTACK' - The crash handler registers an alternate + signal stack on the application's main thread, allowing it to + handle crashes caused by stack overflow. - `typedef struct { ... } spindle_args_t` @@ -190,6 +197,9 @@ we have to break ABI compatibility. - `char *preloadfile` - Points to a file containing a white-space separated list of files that should be staged onto every node in the job before the application runs. + - `char *crash_log` - When `OPT_CRASH_LOG` is set, the path of the + crash log file. Should be an absolute path on a file system + accessible by the Spindle servers. The FrontEnd API ---------------- diff --git a/src/client/crash_handler/crash_handler.c b/src/client/crash_handler/crash_handler.c index 8482a30c..35016808 100644 --- a/src/client/crash_handler/crash_handler.c +++ b/src/client/crash_handler/crash_handler.c @@ -21,6 +21,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include #include #include +#include #include #include #include @@ -38,15 +39,17 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "crash_sigchain.h" #define CRASH_ALTSTACK_SIZE 65536 -#define CRASH_SITE_BUF_SIZE (PATH_MAX + 32) +/* Payload has to fit in server's receive buffer which is size MAX_PATH_LEN */ +#define CRASH_SITE_BUF_SIZE (MAX_PATH_LEN - 3 * sizeof(int32_t)) #define CRASH_REQ_BUF_SIZE \ - (sizeof(ldcs_message_header_t) + 2 * sizeof(int32_t) + CRASH_SITE_BUF_SIZE) + (sizeof(ldcs_message_header_t) + 3 * sizeof(int32_t) + CRASH_SITE_BUF_SIZE) #define CRASH_ABORT_MSG_MAX (64u * 1024u) -static int crash_global_rank = -1; -static int crash_read_fd = -1; -static int crash_write_fd = -1; -static int crash_installed = 0; +static int crash_global_rank = -1; +static int crash_display_rank = -1; +static int crash_read_fd = -1; +static int crash_write_fd = -1; +static int crash_installed = 0; static char *crash_altstack_buf = NULL; static char crash_site_buf[CRASH_SITE_BUF_SIZE]; @@ -72,12 +75,12 @@ static void resolve_pc_to_crash_site(unsigned long pc, char *buf, size_t buflen) (void) crash_fmt_lib_offset(buf, buflen, "", pc); } -/* Builds the crash report message */ +/* Builds the crash report message. */ static size_t build_crash_report(char *buf, size_t buflen, int rank, - const char *site) + int display_rank, const char *site) { size_t name_len = strlen(site) + 1; - size_t payload_len = 2 * sizeof(int32_t) + name_len; + size_t payload_len = 3 * sizeof(int32_t) + name_len; size_t total_len = sizeof(ldcs_message_header_t) + payload_len; if (total_len > buflen) return 0; @@ -89,10 +92,12 @@ static size_t build_crash_report(char *buf, size_t buflen, int rank, char *payload = buf + sizeof(hdr); int32_t rank32 = (int32_t) rank; + int32_t drank32 = (int32_t) display_rank; int32_t nlen32 = (int32_t) name_len; memcpy(payload, &rank32, sizeof(rank32)); - memcpy(payload + sizeof(int32_t), &nlen32, sizeof(nlen32)); - memcpy(payload + 2 * sizeof(int32_t), site, name_len); + memcpy(payload + sizeof(int32_t), &drank32, sizeof(drank32)); + memcpy(payload + 2 * sizeof(int32_t), &nlen32, sizeof(nlen32)); + memcpy(payload + 3 * sizeof(int32_t), site, name_len); return total_len; } @@ -121,21 +126,59 @@ static size_t read_abort_msg(char *buf, size_t buflen) return n; } -/* Builds the crash site string according to the signal type. - For SIGABRT, use the abort_msg; otherwise, +. */ +/* Prefix crash site with executable to distinguish crashes at the same site + but from different executables within different jobs of the same session. */ +static size_t crash_write_exe_prefix(char *buf, size_t buflen) +{ + static const char trunc_mark[] = "..."; + const size_t mark_len = sizeof(trunc_mark) - 1; + const char *exe = crash_lib_offset_exe_path(); + size_t exe_len = strlen(exe); + size_t max_exe = buflen / 2; + size_t pos = 0; + + if (max_exe <= mark_len + 1) + return 0; + max_exe -= 1; /* room for '|' */ + + if (exe_len > max_exe) { + memcpy(buf, trunc_mark, mark_len); + pos = mark_len; + exe += exe_len - (max_exe - mark_len); + exe_len = max_exe - mark_len; + } + memcpy(buf + pos, exe, exe_len); + pos += exe_len; + buf[pos++] = '|'; + return pos; +} + +/* Builds the crash site string |, where is the + abort_msg for SIGABRT and + otherwise. */ static void crash_build_site(int sig, unsigned long pc, char *buf, size_t buflen) { + if (buflen == 0) return; + buf[0] = '\0'; + + size_t prefix_len = crash_write_exe_prefix(buf, buflen); + char *site = buf + prefix_len; + size_t site_buflen = buflen - prefix_len; + if (sig == SIGABRT) { - static const char prefix[] = "abort:"; - const size_t prefix_len = sizeof(prefix) - 1; - memcpy(buf, prefix, prefix_len); - size_t n = read_abort_msg(buf + prefix_len, buflen - prefix_len); + static const char abort_prefix[] = "abort:"; + const size_t abort_len = sizeof(abort_prefix) - 1; + size_t n = 0; + + if (site_buflen > abort_len) { + memcpy(site, abort_prefix, abort_len); + n = read_abort_msg(site + abort_len, site_buflen - abort_len); + } // if we failed to get the abort string, fall back to + if (n == 0) - resolve_pc_to_crash_site(pc, buf, buflen); + resolve_pc_to_crash_site(pc, site, site_buflen); } else { - resolve_pc_to_crash_site(pc, buf, buflen); + resolve_pc_to_crash_site(pc, site, site_buflen); } } @@ -145,7 +188,8 @@ static void crash_build_site(int sig, unsigned long pc, static int crash_query_server(const char *site, int rank, int32_t *winner) { char req_buf[CRASH_REQ_BUF_SIZE]; - size_t req_len = build_crash_report(req_buf, sizeof req_buf, rank, site); + size_t req_len = build_crash_report(req_buf, sizeof req_buf, rank, + crash_display_rank, site); if (req_len == 0) return -1; @@ -243,6 +287,30 @@ static void crash_handler_entry(int sig, siginfo_t *info, void *uctx) signal(sig, SIG_DFL); } +/* Resolves a display rank for use in crash logging from launcher/MPI env vars. */ +static int resolve_display_rank(int fallback) +{ + static const char *const rank_vars[] = { + "PMIX_RANK", "OMPI_COMM_WORLD_RANK", "PMI_RANK", "JSM_NAMESPACE_RANK", + "FLUX_TASK_RANK", "MV2_COMM_WORLD_RANK", "PALS_RANKID", "ALPS_APP_PE", + "SLURM_PROCID" + }; + for (size_t i = 0; i < sizeof(rank_vars) / sizeof(rank_vars[0]); i++) { + const char *val = getenv(rank_vars[i]); + if (val == NULL || val[0] == '\0') + continue; + char *end = NULL; + long rank = strtol(val, &end, 10); + if (*end != '\0' || rank < 0 || rank > INT32_MAX) + continue; + debug_printf2("display rank %ld from %s\n", + rank, rank_vars[i]); + return (int) rank; + } + debug_printf2("could not detect MPI rank from environment\n"); + return fallback; +} + /* Performs setup and installs the signal handler. */ int crash_handler_install(int global_rank, int ldcsid_in) { @@ -252,6 +320,7 @@ int crash_handler_install(int global_rank, int ldcsid_in) crash_sigchain_init(); crash_global_rank = global_rank; + crash_display_rank = resolve_display_rank(global_rank); if (client_get_raw_fds(ldcsid_in, &crash_read_fd, &crash_write_fd) != 0 || crash_read_fd < 0 || crash_write_fd < 0) { @@ -270,26 +339,35 @@ int crash_handler_install(int global_rank, int ldcsid_in) tmp, sizeof tmp); } - /* Set up the altstack. + /* Set up the altstack if enabled. If the reason for a segfault is a stack overflow, the signal handler itself will have no stack available. We handle this by registering an alternate stack - for the signal handler. However, note that this is per-thread, and currently - we do not register an alternate stack on any thread other than the main thread. + for the signal handler if requested with --crash-altstack. + However, note that this is per-thread, and currently we do not register an + alternate stack on any thread other than the main thread. TODO: handle alternate stack on other threads */ - crash_altstack_buf = mmap(NULL, CRASH_ALTSTACK_SIZE, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (crash_altstack_buf == MAP_FAILED) { - crash_altstack_buf = NULL; - } else { - stack_t ss; - memset(&ss, 0, sizeof ss); - ss.ss_sp = crash_altstack_buf; - ss.ss_size = CRASH_ALTSTACK_SIZE; - ss.ss_flags = 0; - if (sigaltstack(&ss, NULL) != 0) { - munmap(crash_altstack_buf, CRASH_ALTSTACK_SIZE); + if (opts & OPT_CRASH_ALTSTACK) { + crash_altstack_buf = mmap(NULL, CRASH_ALTSTACK_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (crash_altstack_buf == MAP_FAILED) { crash_altstack_buf = NULL; + debug_printf2("crash handler: failed to mmap altstack\n"); + } else { + stack_t ss; + memset(&ss, 0, sizeof ss); + ss.ss_sp = crash_altstack_buf; + ss.ss_size = CRASH_ALTSTACK_SIZE; + ss.ss_flags = 0; + if (sigaltstack(&ss, NULL) != 0) { + munmap(crash_altstack_buf, CRASH_ALTSTACK_SIZE); + crash_altstack_buf = NULL; + debug_printf2("crash handler: failed to register altstack\n"); + } else { + debug_printf2("crash handler: registered altstack\n"); + } } + } else { + crash_altstack_buf = NULL; } /* Install the signal handler. */ diff --git a/src/client/crash_handler/crash_lib_offset.c b/src/client/crash_handler/crash_lib_offset.c index f6c45eac..286440cb 100644 --- a/src/client/crash_handler/crash_lib_offset.c +++ b/src/client/crash_handler/crash_lib_offset.c @@ -161,3 +161,8 @@ void crash_lib_offset_prime(void) exe_auxv_phdrs = (const ElfW(Phdr) *) getauxval(AT_PHDR); exe_auxv_phnum = getauxval(AT_PHNUM); } + +const char *crash_lib_offset_exe_path(void) +{ + return get_executable_path(); +} diff --git a/src/client/crash_handler/crash_lib_offset.h b/src/client/crash_handler/crash_lib_offset.h index a50bbe69..d7d19e8d 100644 --- a/src/client/crash_handler/crash_lib_offset.h +++ b/src/client/crash_handler/crash_lib_offset.h @@ -21,6 +21,8 @@ Place, Suite 330, Boston, MA 02111-1307 USA void crash_lib_offset_prime(void); +const char *crash_lib_offset_exe_path(void); + int crash_lib_offset_get_signal_safe(unsigned long pc, char *buf, size_t buflen); diff --git a/src/fe/startup/config_mgr.cc b/src/fe/startup/config_mgr.cc index e1368642..be74a684 100644 --- a/src/fe/startup/config_mgr.cc +++ b/src/fe/startup/config_mgr.cc @@ -18,6 +18,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "config_parser.h" #include "spindle_debug.h" #include "config.h" +#include "parseloc.h" #include #include @@ -148,6 +149,8 @@ using namespace std; #define DEFAULT_CRASH_DEDUP_STR "false" #endif +#define DEFAULT_CRASH_ALTSTACK_STR "false" + #if defined(PYTHON_INST_PREFIX) #define PYTHON_PREFIX_DEFAULT PYTHON_INST_PREFIX #else @@ -302,7 +305,11 @@ void initOptionsList() { confRshCommand, "rsh-command", shortRSHCmd, groupMisc, cvString, {}, RSHCMD_STR, "The command to run rsh/ssh, when doing RSH startup mode." }, { confCrashDedup, "crash-dedup", shortCrashDedup, groupMisc, cvBool, {}, DEFAULT_CRASH_DEDUP_STR, - "Deduplicate coredumps by crash site, emitting only one coredump per unique site." } + "Deduplicate coredumps by crash site, emitting only one coredump per unique site." }, + { confCrashAltstack, "crash-altstack", shortCrashAltstack, groupMisc, cvBool, {}, DEFAULT_CRASH_ALTSTACK_STR, + "Registers an alternate stack for signal handlers used by crash deduplication." }, + { confCrashLog, "crash-log", shortCrashLog, groupMisc, cvStringOptional, {}, "", + "Log deduplicated crash sites, showing which ranks crashed at which sites." } } ); } @@ -1009,6 +1016,17 @@ bool ConfigMap::toSpindleArgs(spindle_args_t &args, bool alloc_strs) const case confCrashDedup: setopt(args.opts, OPT_CRASH_HANDLER, boolresult); break; + case confCrashAltstack: + setopt(args.opts, OPT_CRASH_ALTSTACK, boolresult); + break; + case confCrashLog: + args.crash_log = resolve_crash_log_path(strresult.empty() ? NULL : strresult.c_str(), + getNumber()); + if (!args.crash_log) + return false; + setopt(args.opts, OPT_CRASH_LOG, true); + setopt(args.opts, OPT_CRASH_HANDLER, true); + break; } } diff --git a/src/fe/startup/config_mgr.h b/src/fe/startup/config_mgr.h index 14f18b7a..1ed2d645 100644 --- a/src/fe/startup/config_mgr.h +++ b/src/fe/startup/config_mgr.h @@ -69,7 +69,9 @@ enum SpindleConfigID { confEndSession, confRunSession, confPatchLdso, - confCrashDedup + confCrashDedup, + confCrashAltstack, + confCrashLog }; enum CmdlineShortOptions { @@ -130,6 +132,8 @@ enum CmdlineShortOptions { shortPatchLdso = 299, shortCachePaths = 300, shortCrashDedup = 301, + shortCrashAltstack = 302, + shortCrashLog = 303, }; enum CmdlineGroups { diff --git a/src/fe/startup/spindle_fe.cc b/src/fe/startup/spindle_fe.cc index f1daab9e..66e8361e 100644 --- a/src/fe/startup/spindle_fe.cc +++ b/src/fe/startup/spindle_fe.cc @@ -80,7 +80,8 @@ static int pack_data(spindle_args_t *args, void* &buffer, unsigned &buffer_size) buffer_size += args->local_prefixes ? strlen(args->local_prefixes) + 1 : 1; buffer_size += args->session_key ? strlen(args->session_key) + 1 : 1; buffer_size += args->exec_excludes ? strlen(args->exec_excludes) + 1 : 1; - + buffer_size += args->crash_log ? strlen(args->crash_log) + 1 : 1; + unsigned int pos = 0; char *buf = (char *) malloc(buffer_size); pack_param(args->number, buf, pos); @@ -103,6 +104,7 @@ static int pack_data(spindle_args_t *args, void* &buffer, unsigned &buffer_size) pack_param(args->local_prefixes, buf, pos); pack_param(args->session_key, buf, pos); pack_param(args->exec_excludes, buf, pos); + pack_param(args->crash_log, buf, pos); assert(pos == buffer_size); buffer = (void *) buf; @@ -351,6 +353,8 @@ static void printSpindleFlags(opt_t opts) { printFlag(opts, OPT_OFF, "OPT_OFF", ss); printFlag(opts, OPT_PATCHLDSO, "OPT_PATCHLDSO", ss); printFlag(opts, OPT_CRASH_HANDLER, "OPT_CRASH_HANDLER", ss); + printFlag(opts, OPT_CRASH_LOG, "OPT_CRASH_LOG", ss); + printFlag(opts, OPT_CRASH_ALTSTACK, "OPT_CRASH_ALTSTACK", ss); ss << ", "; if (OPT_GET_SEC(opts) == OPT_SEC_MUNGE) ss << "OPT_SEC_MUNGE"; if (OPT_GET_SEC(opts) == OPT_SEC_KEYLMON) ss << "OPT_SEC_KEYLMON"; @@ -400,12 +404,13 @@ int spindleInitFE(const char **hosts, spindle_args_t *params) debug_printf("spindle_args_t { number = %lu; port = %u; num_ports = %u; opts = %lu; unique_id = %lu; " "use_launcher = %u; startup_type = %u; shm_cache_size = %u; commpath = %s; " "cachepaths = %s; " - "pythonprefix = %s; preloadfile = %s; bundle_timeout_ms = %u; bundle_cachesize_kb = %u }\n", + "pythonprefix = %s; preloadfile = %s; bundle_timeout_ms = %u; bundle_cachesize_kb = %u; " + "crash_log = %s }\n", (unsigned long) params->number, params->port, params->num_ports, params->opts, params->unique_id, params->use_launcher, params->startup_type, params->shm_cache_size, params->commpath, params->candidate_cachepaths, params->pythonprefix, params->preloadfile, params->bundle_timeout_ms, - params->bundle_cachesize_kb); + params->bundle_cachesize_kb, params->crash_log); printSpindleFlags(params->opts); debug_printf("Starting FE servers with hostlist of size %u on port %u\n", hosts_size, params->port); ldcs_audit_server_fe_md_open(const_cast(hosts), hosts_size, diff --git a/src/flux/Makefile.am b/src/flux/Makefile.am index 717c2104..986b232a 100644 --- a/src/flux/Makefile.am +++ b/src/flux/Makefile.am @@ -12,6 +12,7 @@ libspindleflux_la_SOURCES = \ libspindleflux_la_CPPFLAGS = \ -I$(top_srcdir)/src/include \ + -I$(top_srcdir)/src/utils \ $(FLUX_CORE_CFLAGS) \ $(FLUX_HOSTLIST_CFLAGS) diff --git a/src/flux/Makefile.in b/src/flux/Makefile.in index e6733b24..744f1624 100644 --- a/src/flux/Makefile.in +++ b/src/flux/Makefile.in @@ -436,6 +436,7 @@ libspindleflux_la_SOURCES = \ libspindleflux_la_CPPFLAGS = \ -I$(top_srcdir)/src/include \ + -I$(top_srcdir)/src/utils \ $(FLUX_CORE_CFLAGS) \ $(FLUX_HOSTLIST_CFLAGS) diff --git a/src/flux/flux-spindle.c b/src/flux/flux-spindle.c index a3abeef7..f722ddfd 100644 --- a/src/flux/flux-spindle.c +++ b/src/flux/flux-spindle.c @@ -34,6 +34,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "spindle_launch.h" #include "fluxmgr.h" +#include "parseloc.h" #define debug_printf(PRIORITY, FORMAT, ...) \ do { \ @@ -178,6 +179,10 @@ static int spindle_is_enabled(struct spindle_ctx *ctx) { char *spindle_env; + if (!ctx) { + return 0; + } + spindle_env = getenv("SPINDLE"); if (spindle_env) { if (strcasecmp(spindle_env, "false") == 0 || strcmp(spindle_env, "0") == 0) { @@ -380,6 +385,8 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx) int had_error = 0; int numa = 0; int crash_dedup = 0; + int crash_altstack = 0; + json_t *crash_log = NULL; const char *relocaout = NULL, *reloclibs = NULL, *relocexec = NULL, *relocpython = NULL; const char *followfork = NULL, *preload = NULL, *level = NULL; const char *pyprefix = NULL, *commpath = NULL; @@ -405,7 +412,7 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx) * supplied by the user, but not unpacked (This handles typos, etc). */ if (json_unpack_ex (opts, &error, JSON_STRICT, - "{s?i s?i s?i s?i s?s s?s s?s s?s s?s s?s s?s s?i s?s s?s s?s s?s s?i}", + "{s?i s?i s?i s?i s?s s?s s?s s?s s?s s?s s?s s?i s?s s?s s?s s?s s?i s?i s?o}", "noclean", &noclean, "nostrip", &nostrip, "push", &push, @@ -422,7 +429,9 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx) "preload", &preload, "level", &level, "cachepaths", &cachepaths, - "crash-dedup", &crash_dedup) < 0) + "crash-dedup", &crash_dedup, + "crash-altstack", &crash_altstack, + "crash-log", &crash_log) < 0) logerrno_printf_and_return(1, "Error in spindle option: %s\n", error.text); if (noclean) @@ -474,6 +483,25 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx) if (crash_dedup) { ctx->params.opts |= OPT_CRASH_HANDLER; } + if (crash_altstack) { + ctx->params.opts |= OPT_CRASH_ALTSTACK; + } + if (crash_log) { + /* --crash-log can be a path or true; if true, use the default path. */ + const char *value = NULL; + char *abspath; + if (json_is_string (crash_log)) + value = json_string_value (crash_log); + else if (!json_is_true (crash_log) + && !(json_is_integer (crash_log) && json_integer_value (crash_log) > 0)) + logerrno_printf_and_return(1, "Error in spindle option: crash-log must be a path or true\n"); + abspath = resolve_crash_log_path (value, ctx->params.number); + if (!abspath) + logerrno_printf_and_return(1, "unable to expand crash-log path\n"); + ctx->params.crash_log = abspath; + ctx->params.opts |= OPT_CRASH_LOG; + ctx->params.opts |= OPT_CRASH_HANDLER; + } if (level) { if (strcmp(level, "high") == 0) { ctx->params.opts |= OPT_RELOCAOUT; diff --git a/src/include/ldcs_api.h b/src/include/ldcs_api.h index 220eb349..4fb6b60a 100644 --- a/src/include/ldcs_api.h +++ b/src/include/ldcs_api.h @@ -89,6 +89,7 @@ typedef enum { LDCS_MSG_CHOSEN_CACHEPATH, LDCS_MSG_CRASH_REPORT, LDCS_MSG_CRASH_RESPONSE, + LDCS_MSG_CRASH_LOG, LDCS_MSG_UNKNOWN } ldcs_message_ids_t; diff --git a/src/include/spindle_launch.h b/src/include/spindle_launch.h index 98121396..05e89680 100644 --- a/src/include/spindle_launch.h +++ b/src/include/spindle_launch.h @@ -59,8 +59,10 @@ extern "C" { #define OPT_STOPRELOC (1 << 28) /* Stops spindle from relocating file contents, but still allow it to intercept file-not-found attempts */ #define OPT_NUMA (1 << 29) /* Enables file replication across NUMA domains */ #define OPT_OFF (1 << 30) /* Turns spindle off, disabling everything */ -#define OPT_PATCHLDSO (1 << 31) /* Enables patching of ld.so to intercept stat calls */ +#define OPT_PATCHLDSO ((opt_t) 1 << 31) /* Enables patching of ld.so to intercept stat calls */ #define OPT_CRASH_HANDLER ((opt_t) 1 << 32) /* Enables crash handler with coredump deduplication */ +#define OPT_CRASH_LOG ((opt_t) 1 << 33) /* Write a log of crash sites and ranks at job end */ +#define OPT_CRASH_ALTSTACK ((opt_t) 1 << 34) /* Crash handler registers an alternate signal stack */ #define OPT_SET_SEC(OPT, X) OPT |= (X << 19) #define OPT_GET_SEC(OPT) ((OPT >> 19) & 7) @@ -161,6 +163,9 @@ typedef struct { /* Executable names to exclude from spindle and not run on */ char *exec_excludes; + + /* Path of the crash log file written by the root server */ + char *crash_log; } spindle_args_t; /* Functions used to startup Spindle on the front-end. Init returns after finishing start-up, diff --git a/src/server/auditserver/Makefile.am b/src/server/auditserver/Makefile.am index 8823ead6..b567d8dd 100644 --- a/src/server/auditserver/Makefile.am +++ b/src/server/auditserver/Makefile.am @@ -5,7 +5,7 @@ AM_CPPFLAGS = -I$(top_srcdir)/comlib -I$(top_srcdir)/cache -I$(top_srcdir)/../co LDADD = $(top_builddir)/cache/libldcs_cache.la -lrt #AM_LDFLAGS = -all-static -libserverbase_la_SOURCES = ldcs_audit_server_client_cb.c ldcs_audit_server_server_cb.c ldcs_audit_server_process.c ldcs_audit_server_filemngt.c ldcs_audit_server_handlers.c ldcs_audit_server_crash_handler.c ldcs_elf_read.c ldcs_audit_server_requestors.c ldcs_audit_server_numa.c msgbundle.c parse_mounts.cc cleanup_proc.cc translate_maps.c force_exit.c $(top_srcdir)/../utils/fileutil.c +libserverbase_la_SOURCES = ldcs_audit_server_client_cb.c ldcs_audit_server_server_cb.c ldcs_audit_server_process.c ldcs_audit_server_filemngt.c ldcs_audit_server_handlers.c ldcs_audit_server_crash_handler.c ldcs_audit_server_crash_log.c ldcs_elf_read.c ldcs_audit_server_requestors.c ldcs_audit_server_numa.c msgbundle.c parse_mounts.cc cleanup_proc.cc translate_maps.c force_exit.c $(top_srcdir)/../utils/fileutil.c libserverbase_la_LIBADD = -lpthread #libaudit_server_msocket_la_SOURCES = ldcs_audit_server_md_msocket.c ldcs_audit_server_md_msocket_util.c ldcs_audit_server_md_msocket_topo.c diff --git a/src/server/auditserver/Makefile.in b/src/server/auditserver/Makefile.in index cede46ff..a3fcbe43 100644 --- a/src/server/auditserver/Makefile.in +++ b/src/server/auditserver/Makefile.in @@ -123,7 +123,8 @@ am__dirstamp = $(am__leading_dot)dirstamp am_libserverbase_la_OBJECTS = ldcs_audit_server_client_cb.lo \ ldcs_audit_server_server_cb.lo ldcs_audit_server_process.lo \ ldcs_audit_server_filemngt.lo ldcs_audit_server_handlers.lo \ - ldcs_audit_server_crash_handler.lo ldcs_elf_read.lo \ + ldcs_audit_server_crash_handler.lo \ + ldcs_audit_server_crash_log.lo ldcs_elf_read.lo \ ldcs_audit_server_requestors.lo ldcs_audit_server_numa.lo \ msgbundle.lo parse_mounts.lo cleanup_proc.lo translate_maps.lo \ force_exit.lo $(top_builddir)/../utils/fileutil.lo @@ -147,6 +148,7 @@ am__depfiles_remade = $(top_builddir)/../utils/$(DEPDIR)/fileutil.Plo \ ./$(DEPDIR)/cleanup_proc.Plo ./$(DEPDIR)/force_exit.Plo \ ./$(DEPDIR)/ldcs_audit_server_client_cb.Plo \ ./$(DEPDIR)/ldcs_audit_server_crash_handler.Plo \ + ./$(DEPDIR)/ldcs_audit_server_crash_log.Plo \ ./$(DEPDIR)/ldcs_audit_server_filemngt.Plo \ ./$(DEPDIR)/ldcs_audit_server_handlers.Plo \ ./$(DEPDIR)/ldcs_audit_server_md_cobo.Plo \ @@ -381,7 +383,7 @@ noinst_LTLIBRARIES = libaudit_server_cobo.la libserverbase.la AM_CPPFLAGS = -I$(top_srcdir)/comlib -I$(top_srcdir)/cache -I$(top_srcdir)/../cobo -I$(top_srcdir)/../logging -I$(top_srcdir)/../include -I$(top_srcdir)/../utils -DLIBEXECDIR=\"$(pkglibexecdir)\" LDADD = $(top_builddir)/cache/libldcs_cache.la -lrt #AM_LDFLAGS = -all-static -libserverbase_la_SOURCES = ldcs_audit_server_client_cb.c ldcs_audit_server_server_cb.c ldcs_audit_server_process.c ldcs_audit_server_filemngt.c ldcs_audit_server_handlers.c ldcs_audit_server_crash_handler.c ldcs_elf_read.c ldcs_audit_server_requestors.c ldcs_audit_server_numa.c msgbundle.c parse_mounts.cc cleanup_proc.cc translate_maps.c force_exit.c $(top_srcdir)/../utils/fileutil.c +libserverbase_la_SOURCES = ldcs_audit_server_client_cb.c ldcs_audit_server_server_cb.c ldcs_audit_server_process.c ldcs_audit_server_filemngt.c ldcs_audit_server_handlers.c ldcs_audit_server_crash_handler.c ldcs_audit_server_crash_log.c ldcs_elf_read.c ldcs_audit_server_requestors.c ldcs_audit_server_numa.c msgbundle.c parse_mounts.cc cleanup_proc.cc translate_maps.c force_exit.c $(top_srcdir)/../utils/fileutil.c libserverbase_la_LIBADD = -lpthread #libaudit_server_msocket_la_SOURCES = ldcs_audit_server_md_msocket.c ldcs_audit_server_md_msocket_util.c ldcs_audit_server_md_msocket_topo.c @@ -463,6 +465,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/force_exit.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ldcs_audit_server_client_cb.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ldcs_audit_server_crash_handler.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ldcs_audit_server_crash_log.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ldcs_audit_server_filemngt.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ldcs_audit_server_handlers.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ldcs_audit_server_md_cobo.Plo@am__quote@ # am--include-marker @@ -668,6 +671,7 @@ distclean: distclean-am -rm -f ./$(DEPDIR)/force_exit.Plo -rm -f ./$(DEPDIR)/ldcs_audit_server_client_cb.Plo -rm -f ./$(DEPDIR)/ldcs_audit_server_crash_handler.Plo + -rm -f ./$(DEPDIR)/ldcs_audit_server_crash_log.Plo -rm -f ./$(DEPDIR)/ldcs_audit_server_filemngt.Plo -rm -f ./$(DEPDIR)/ldcs_audit_server_handlers.Plo -rm -f ./$(DEPDIR)/ldcs_audit_server_md_cobo.Plo @@ -729,6 +733,7 @@ maintainer-clean: maintainer-clean-am -rm -f ./$(DEPDIR)/force_exit.Plo -rm -f ./$(DEPDIR)/ldcs_audit_server_client_cb.Plo -rm -f ./$(DEPDIR)/ldcs_audit_server_crash_handler.Plo + -rm -f ./$(DEPDIR)/ldcs_audit_server_crash_log.Plo -rm -f ./$(DEPDIR)/ldcs_audit_server_filemngt.Plo -rm -f ./$(DEPDIR)/ldcs_audit_server_handlers.Plo -rm -f ./$(DEPDIR)/ldcs_audit_server_md_cobo.Plo diff --git a/src/server/auditserver/ldcs_audit_server_crash_handler.c b/src/server/auditserver/ldcs_audit_server_crash_handler.c index bdc25871..a99e6689 100644 --- a/src/server/auditserver/ldcs_audit_server_crash_handler.c +++ b/src/server/auditserver/ldcs_audit_server_crash_handler.c @@ -26,12 +26,11 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "spindle_launch.h" #include "msgbundle.h" -static crash_site_entry_t *crash_site_find(ldcs_process_data_t *procdata, - const char *site, size_t site_len); -static crash_site_entry_t *crash_site_insert(ldcs_process_data_t *procdata, - const char *site, size_t site_len); static char *crash_pack(int32_t rank, const char *site, size_t site_len, size_t *out_len); +static char *crash_pack_report(int32_t dedup_rank, int32_t display_rank, + const char *site, size_t site_len, + size_t *out_len); static void send_crash_response_to_waiter(ldcs_process_data_t *procdata, crash_waiter_t *w, int winning_rank, @@ -40,18 +39,24 @@ static int crash_resolve(ldcs_process_data_t *procdata, crash_site_entry_t *entry, int winning_rank); static int forward_crash_query_up(ldcs_process_data_t *procdata, const char *site, size_t site_len, - int first_waiter_rank); + int first_waiter_rank, + int first_waiter_display_rank); static int crash_parse_report(ldcs_message_t *msg, const char *err_str, - int32_t *rank, + int32_t *dedup_rank, int32_t *display_rank, const char **site, size_t *site_len); +static int crash_parse_response(ldcs_message_t *msg, const char *err_str, + int32_t *rank, + const char **site, size_t *site_len); static int crash_report_common(ldcs_process_data_t *procdata, crash_waiter_t *w, int reporter_rank, - const char *site, size_t site_len); + int reporter_display_rank, + const char *site, size_t site_len, + crash_site_entry_t **entry_out); /* INTERNAL HELPER FUNCTIONS */ -static crash_site_entry_t *crash_site_find(ldcs_process_data_t *procdata, - const char *site, size_t site_len) +crash_site_entry_t *crash_site_find(ldcs_process_data_t *procdata, + const char *site, size_t site_len) { int i; for (i = 0; i < procdata->crash_sites_count; ++i) { @@ -64,8 +69,8 @@ static crash_site_entry_t *crash_site_find(ldcs_process_data_t *procdata, return NULL; } -static crash_site_entry_t *crash_site_insert(ldcs_process_data_t *procdata, - const char *site, size_t site_len) +crash_site_entry_t *crash_site_insert(ldcs_process_data_t *procdata, + const char *site, size_t site_len) { if (procdata->crash_sites_count >= procdata->crash_sites_cap) { int new_cap = procdata->crash_sites_cap ? procdata->crash_sites_cap * 2 : 8; @@ -77,9 +82,10 @@ static crash_site_entry_t *crash_site_insert(ldcs_process_data_t *procdata, memcpy(copy, site, site_len); copy[site_len] = '\0'; crash_site_entry_t *e = &procdata->crash_sites[procdata->crash_sites_count++]; + memset(e, 0, sizeof(*e)); e->site = copy; e->site_len = site_len; - e->resolved = 0; + e->exemplar_rank = -1; return e; } @@ -96,6 +102,21 @@ static char *crash_pack(int32_t rank, const char *site, size_t site_len, return buf; } +static char *crash_pack_report(int32_t dedup_rank, int32_t display_rank, + const char *site, size_t site_len, + size_t *out_len) +{ + size_t total = 3 * sizeof(int32_t) + site_len; + char *buf = malloc(total); + int32_t name_len_v = (int32_t) site_len; + memcpy(buf, &dedup_rank, sizeof(int32_t)); + memcpy(buf + 1 * sizeof(int32_t), &display_rank, sizeof(int32_t)); + memcpy(buf + 2 * sizeof(int32_t), &name_len_v, sizeof(int32_t)); + memcpy(buf + 3 * sizeof(int32_t), site, site_len); + *out_len = total; + return buf; +} + static void send_crash_response_to_waiter(ldcs_process_data_t *procdata, crash_waiter_t *w, int winning_rank, @@ -142,10 +163,13 @@ static int crash_resolve(ldcs_process_data_t *procdata, static int forward_crash_query_up(ldcs_process_data_t *procdata, const char *site, size_t site_len, - int first_waiter_rank) + int first_waiter_rank, + int first_waiter_display_rank) { size_t total = 0; - char *buf = crash_pack((int32_t) first_waiter_rank, site, site_len, &total); + char *buf = crash_pack_report((int32_t) first_waiter_rank, + (int32_t) first_waiter_display_rank, + site, site_len, &total); ldcs_message_t msg; msg.header.type = LDCS_MSG_CRASH_REPORT; msg.header.len = total; @@ -156,14 +180,35 @@ static int forward_crash_query_up(ldcs_process_data_t *procdata, } static int crash_parse_report(ldcs_message_t *msg, const char *err_str, - int32_t *rank, + int32_t *dedup_rank, int32_t *display_rank, const char **site, size_t *site_len) { - if (msg->header.len < 2 * sizeof(int32_t)) { + if (msg->header.len < 3 * sizeof(int32_t)) { err_printf("malformed crash report in %s\n", err_str); return -1; } int32_t name_len; + memcpy(dedup_rank, msg->data, sizeof(int32_t)); + memcpy(display_rank, msg->data + 1 * sizeof(int32_t), sizeof(int32_t)); + memcpy(&name_len, msg->data + 2 * sizeof(int32_t), sizeof(int32_t)); + if (name_len <= 0) { + err_printf("bad name_len %d in %s\n", (int) name_len, err_str); + return -1; + } + *site = msg->data + 3 * sizeof(int32_t); + *site_len = (size_t) name_len; + return 0; +} + +static int crash_parse_response(ldcs_message_t *msg, const char *err_str, + int32_t *rank, + const char **site, size_t *site_len) +{ + if (msg->header.len < 2 * sizeof(int32_t)) { + err_printf("malformed crash response in %s\n", err_str); + return -1; + } + int32_t name_len; memcpy(rank, msg->data, sizeof(int32_t)); memcpy(&name_len, msg->data + 1 * sizeof(int32_t), sizeof(int32_t)); if (name_len <= 0) { @@ -177,17 +222,21 @@ static int crash_parse_report(ldcs_message_t *msg, const char *err_str, static int crash_report_common(ldcs_process_data_t *procdata, crash_waiter_t *w, int reporter_rank, - const char *site, size_t site_len) + int reporter_display_rank, + const char *site, size_t site_len, + crash_site_entry_t **entry_out) { // If we have already seen this crash site before, then we know // it can't be the winner and can short-circuit and respond // immediately that this rank was not selected. crash_site_entry_t *e = crash_site_find(procdata, site, site_len); if (e) { - debug_printf2("known crash site '%s' (%s); suppressing %s reporter rank=%d\n", + if (entry_out) + *entry_out = e; + debug_printf2("known crash site '%s' (%s); suppressing %s reporter rank=%d display=%d\n", e->site, e->resolved ? "resolved" : "in flight", w->kind == CRASH_WAITER_LOCAL ? "local" : "child", - reporter_rank); + reporter_rank, reporter_display_rank); send_crash_response_to_waiter(procdata, w, -1, site, site_len); return 0; } @@ -196,19 +245,24 @@ static int crash_report_common(ldcs_process_data_t *procdata, // seen it and forward up the tree for resolution. e = crash_site_insert(procdata, site, site_len); e->waiter = *w; + if (entry_out) + *entry_out = e; // If we reached the root without finding a decision already made, // then this was the first instance of this crash site; // select this rank to produce the exemplar coredump. if (ldcs_audit_server_md_is_responsible(procdata, "")) { - debug_printf2("new crash site '%s' at root; selecting rank %d\n", - e->site, reporter_rank); + debug_printf2("new crash site '%s' at root; selecting rank %d (display %d)\n", + e->site, reporter_rank, reporter_display_rank); + if (procdata->opts & OPT_CRASH_LOG) + e->exemplar_rank = reporter_display_rank; return crash_resolve(procdata, e, reporter_rank); } - debug_printf2("new crash site '%s' at interior; forwarding upward %d\n", - e->site, reporter_rank); - return forward_crash_query_up(procdata, site, site_len, reporter_rank); + debug_printf2("new crash site '%s' at interior; forwarding upward %d (display %d)\n", + e->site, reporter_rank, reporter_display_rank); + return forward_crash_query_up(procdata, site, site_len, reporter_rank, + reporter_display_rank); } /* PUBLIC API */ @@ -225,9 +279,10 @@ int handle_client_crash_report(ldcs_process_data_t *procdata, nc, (unsigned long) msg->header.len); int32_t rank_raw; + int32_t display_rank; const char *site; size_t site_len; - if (crash_parse_report(msg, "crash report from local client", &rank_raw, &site, &site_len) != 0) + if (crash_parse_report(msg, "crash report from local client", &rank_raw, &display_rank, &site, &site_len) != 0) return -1; crash_waiter_t w; @@ -235,7 +290,17 @@ int handle_client_crash_report(ldcs_process_data_t *procdata, w.nc = nc; w.global_rank = (int) rank_raw; w.peer = NULL; - return crash_report_common(procdata, &w, (int) rank_raw, site, site_len); + crash_site_entry_t *e = NULL; + int result = crash_report_common(procdata, &w, (int) rank_raw, (int) display_rank, + site, site_len, &e); + + if ((procdata->opts & OPT_CRASH_LOG) && e) { + crash_log_append_rank(e, display_rank); + debug_printf2("crash log: recorded local display rank %d at site '%s' (%d ranks)\n", + (int) display_rank, e->site, e->log_ranks_count); + crash_log_updated(procdata); + } + return result; } // Act on a crash report passed to us from a child @@ -243,9 +308,10 @@ int handle_crash_report_recv(ldcs_process_data_t *procdata, node_peer_t peer, ldcs_message_t *msg) { int32_t first_waiter_rank; + int32_t first_waiter_display_rank; const char *site; size_t site_len; - if (crash_parse_report(msg, "crash report from child", &first_waiter_rank, &site, &site_len) != 0) + if (crash_parse_report(msg, "crash report from child", &first_waiter_rank, &first_waiter_display_rank, &site, &site_len) != 0) return -1; crash_waiter_t w; @@ -253,7 +319,8 @@ int handle_crash_report_recv(ldcs_process_data_t *procdata, w.nc = -1; w.global_rank = -1; w.peer = peer; - return crash_report_common(procdata, &w, (int) first_waiter_rank, site, site_len); + return crash_report_common(procdata, &w, (int) first_waiter_rank, + (int) first_waiter_display_rank, site, site_len, NULL); } // Act on a crash response passed to us from our parent @@ -268,7 +335,7 @@ int handle_crash_response_recv(ldcs_process_data_t *procdata, int32_t selected_rank; const char *site; size_t site_len; - if (crash_parse_report(msg, "crash response from parent", &selected_rank, &site, &site_len) != 0) + if (crash_parse_response(msg, "crash response from parent", &selected_rank, &site, &site_len) != 0) return -1; crash_site_entry_t *e = crash_site_find(procdata, site, site_len); @@ -292,6 +359,7 @@ void crash_free_tables(ldcs_process_data_t *procdata) if (procdata->crash_sites) { for (i = 0; i < procdata->crash_sites_count; ++i) { free(procdata->crash_sites[i].site); + free(procdata->crash_sites[i].log_ranks); } free(procdata->crash_sites); procdata->crash_sites = NULL; diff --git a/src/server/auditserver/ldcs_audit_server_crash_handler.h b/src/server/auditserver/ldcs_audit_server_crash_handler.h index 1a419930..3377b057 100644 --- a/src/server/auditserver/ldcs_audit_server_crash_handler.h +++ b/src/server/auditserver/ldcs_audit_server_crash_handler.h @@ -33,6 +33,28 @@ int handle_crash_report_recv(ldcs_process_data_t *procdata, int handle_crash_response_recv(ldcs_process_data_t *procdata, node_peer_t peer, ldcs_message_t *msg); +/* CRASH_LOG table from a child */ +int handle_crash_log_recv(ldcs_process_data_t *procdata, + node_peer_t peer, ldcs_message_t *msg); + +/* Crash-site table shared by the dedup handler and the crash log */ +crash_site_entry_t *crash_site_find(ldcs_process_data_t *procdata, + const char *site, size_t site_len); +crash_site_entry_t *crash_site_insert(ldcs_process_data_t *procdata, + const char *site, size_t site_len); + +/* Add a display rank to a site's crash-log rank list */ +void crash_log_append_rank(crash_site_entry_t *e, int32_t rank); + +/* Send accumulated crash-log ranks to the parent. */ +void crash_log_flush_to_parent(ldcs_process_data_t *procdata); + +/* Forward or write pending crash-log data */ +void crash_log_updated(ldcs_process_data_t *procdata); + +/* Write crash log file */ +void crash_log_root_write(ldcs_process_data_t *procdata); + /* Free the crash-site table at shutdown */ void crash_free_tables(ldcs_process_data_t *procdata); diff --git a/src/server/auditserver/ldcs_audit_server_crash_log.c b/src/server/auditserver/ldcs_audit_server_crash_log.c new file mode 100644 index 00000000..31758c15 --- /dev/null +++ b/src/server/auditserver/ldcs_audit_server_crash_log.c @@ -0,0 +1,348 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#include +#include +#include +#include +#include +#include + +#include "ldcs_api.h" +#include "ldcs_audit_server_process.h" +#include "ldcs_audit_server_md.h" +#include "ldcs_audit_server_crash_handler.h" +#include "spindle_launch.h" +#include "msgbundle.h" + +typedef struct { + const char *site; + int32_t site_len; + int32_t exemplar; + int32_t nranks; + const char *ranks; +} crash_log_entry_t; + +/* Appends a display rank to a site's crash-log rank list. */ +void crash_log_append_rank(crash_site_entry_t *e, int32_t rank) +{ + if (e->log_ranks_count >= e->log_ranks_cap) { + int new_cap = e->log_ranks_cap ? e->log_ranks_cap * 2 : 8; + e->log_ranks = realloc(e->log_ranks, new_cap * sizeof(*e->log_ranks)); + e->log_ranks_cap = new_cap; + } + e->log_ranks[e->log_ranks_count++] = rank; +} + +static size_t crash_log_entry_size(crash_site_entry_t *e) +{ + return 3 * sizeof(int32_t) + e->site_len + e->log_ranks_count * sizeof(int32_t); +} + +static size_t crash_log_entry_pack(char *buf, crash_site_entry_t *e) +{ + size_t pos = 0; + int32_t site_len32 = (int32_t) e->site_len; + int32_t exemplar32 = (int32_t) e->exemplar_rank; + int32_t nranks32 = (int32_t) e->log_ranks_count; + memcpy(buf + pos, &site_len32, sizeof(int32_t)); + pos += sizeof(int32_t); + memcpy(buf + pos, e->site, e->site_len); + pos += e->site_len; + memcpy(buf + pos, &exemplar32, sizeof(int32_t)); + pos += sizeof(int32_t); + memcpy(buf + pos, &nranks32, sizeof(int32_t)); + pos += sizeof(int32_t); + memcpy(buf + pos, e->log_ranks, e->log_ranks_count * sizeof(int32_t)); + pos += e->log_ranks_count * sizeof(int32_t); + return pos; +} + +static int crash_log_parse_entry(char *data, size_t len, size_t *pos, + crash_log_entry_t *out) +{ + size_t p = *pos; + if (p + sizeof(int32_t) > len) + return -1; + memcpy(&out->site_len, data + p, sizeof(int32_t)); + p += sizeof(int32_t); + if (out->site_len <= 0 || p + (size_t) out->site_len > len) + return -1; + out->site = data + p; + p += (size_t) out->site_len; + if (p + 2 * sizeof(int32_t) > len) + return -1; + memcpy(&out->exemplar, data + p, sizeof(int32_t)); + p += sizeof(int32_t); + memcpy(&out->nranks, data + p, sizeof(int32_t)); + p += sizeof(int32_t); + if (out->nranks < 0 || p + (size_t) out->nranks * sizeof(int32_t) > len) + return -1; + out->ranks = data + p; + p += (size_t) out->nranks * sizeof(int32_t); + *pos = p; + return 0; +} + +static void crash_log_clear_pending(ldcs_process_data_t *procdata) +{ + int i; + for (i = 0; i < procdata->crash_sites_count; ++i) { + crash_site_entry_t *e = &procdata->crash_sites[i]; + free(e->log_ranks); + e->log_ranks = NULL; + e->log_ranks_count = 0; + e->log_ranks_cap = 0; + } +} + +void crash_log_flush_to_parent(ldcs_process_data_t *procdata) +{ + int i; + + if (!(procdata->opts & OPT_CRASH_LOG)) + return; + if (ldcs_audit_server_md_is_responsible(procdata, "")) + return; + + int32_t site_count = 0; + size_t total = sizeof(int32_t); + for (i = 0; i < procdata->crash_sites_count; ++i) { + crash_site_entry_t *e = &procdata->crash_sites[i]; + if (e->log_ranks_count == 0) + continue; + site_count++; + total += crash_log_entry_size(e); + } + if (site_count == 0) + return; + + char *buf = malloc(total); + memcpy(buf, &site_count, sizeof(int32_t)); + size_t pos = sizeof(int32_t); + for (i = 0; i < procdata->crash_sites_count; ++i) { + crash_site_entry_t *e = &procdata->crash_sites[i]; + if (e->log_ranks_count == 0) + continue; + pos += crash_log_entry_pack(buf + pos, e); + } + + ldcs_message_t msg; + msg.header.type = LDCS_MSG_CRASH_LOG; + msg.header.len = total; + msg.data = buf; + debug_printf2("flushing %d crash sites to parent (%lu bytes)\n", + (int) site_count, (unsigned long) total); + int rc = spindle_forward_query(procdata, &msg); + free(buf); + if (rc == -1) + return; + + crash_log_clear_pending(procdata); +} + +/* Non-root servers push pending log data to the parent; + the root defers the file write until teardown. */ +void crash_log_updated(ldcs_process_data_t *procdata) +{ + if (ldcs_audit_server_md_is_responsible(procdata, "")) { + if (procdata->crash_log_teardown) + crash_log_root_write(procdata); + } + else { + crash_log_flush_to_parent(procdata); + } +} + +static void crash_log_merge_entry(ldcs_process_data_t *procdata, + crash_log_entry_t *ent) +{ + int j; + crash_site_entry_t *e = crash_site_find(procdata, ent->site, (size_t) ent->site_len); + if (!e) { + e = crash_site_insert(procdata, ent->site, (size_t) ent->site_len); + e->resolved = 1; + } + if (ent->exemplar != -1) + e->exemplar_rank = (int) ent->exemplar; + for (j = 0; j < (int) ent->nranks; ++j) { + int32_t r; + memcpy(&r, ent->ranks + j * sizeof(int32_t), sizeof(int32_t)); + crash_log_append_rank(e, r); + } + debug_printf2("crash log merged site '%s': now %d ranks, exemplar %d\n", + e->site, e->log_ranks_count, e->exemplar_rank); +} + +int handle_crash_log_recv(ldcs_process_data_t *procdata, + node_peer_t peer, ldcs_message_t *msg) +{ + char *data = msg->data; + size_t len = msg->header.len; + size_t pos = 0; + int32_t site_count; + int i; + + if (ldcs_audit_server_md_is_parent(peer)) { + err_printf("unexpectedly got CRASH_LOG from peer other than a child\n"); + return -1; + } + + if (len < sizeof(int32_t)) + goto malformed; + memcpy(&site_count, data, sizeof(int32_t)); + pos = sizeof(int32_t); + if (site_count < 0) + goto malformed; + + debug_printf2("crash log merging %d sites from child\n", (int) site_count); + for (i = 0; i < site_count; ++i) { + crash_log_entry_t ent; + if (crash_log_parse_entry(data, len, &pos, &ent) != 0) + goto malformed; + crash_log_merge_entry(procdata, &ent); + } + + crash_log_updated(procdata); + return 0; + +malformed: + err_printf("malformed CRASH_LOG message from child\n"); + return -1; +} + +static int rank_cmp(const void *a, const void *b) +{ + int32_t ra = *(const int32_t *) a; + int32_t rb = *(const int32_t *) b; + if (ra < rb) return -1; + if (ra > rb) return 1; + return 0; +} + +static void write_escaped(FILE *f, const char *s, size_t len) +{ + size_t i; + for (i = 0; i < len; i++) { + switch (s[i]) { + case '\\': + fputs("\\\\", f); + break; + case '\n': + fputs("\\n", f); + break; + default: + fputc(s[i], f); + } + } +} + +/* Write rank list as comma-separated values, with runs of three or more + contiguous ranks as ranges, e.g., "1,3-8,16,17,20" */ +static void write_compressed_ranks(FILE *f, const int32_t *ranks, int count) +{ + int i = 0; + int first = 1; + while (i < count) { + int j = i; + while (j + 1 < count && + (ranks[j+1] == ranks[j] + 1 || ranks[j+1] == ranks[j])) + j++; + if (!first) + fputc(',', f); + if (ranks[j] >= ranks[i] + 2) + fprintf(f, "%d-%d", (int) ranks[i], (int) ranks[j]); + else if (ranks[j] == ranks[i] + 1) + fprintf(f, "%d,%d", (int) ranks[i], (int) ranks[j]); + else + fprintf(f, "%d", (int) ranks[i]); + first = 0; + i = j + 1; + } + fputc('\n', f); +} + +/* Write the crash log from the crash log accumulated at the root */ +void crash_log_root_write(ldcs_process_data_t *procdata) +{ + int i, nsites = 0; + char *tmppath; + + if (!(procdata->opts & OPT_CRASH_LOG)) + return; + if (!ldcs_audit_server_md_is_responsible(procdata, "")) + return; + if (!procdata->crash_log || procdata->crash_log[0] == '\0') { + err_printf("OPT_CRASH_LOG set but no crash-log path present\n"); + return; + } + + for (i = 0; i < procdata->crash_sites_count; ++i) { + if (procdata->crash_sites[i].log_ranks_count > 0) + nsites++; + } + if (nsites == 0) + return; + + tmppath = malloc(strlen(procdata->crash_log) + 5); + sprintf(tmppath, "%s.tmp", procdata->crash_log); + FILE *f = fopen(tmppath, "w"); + if (!f) { + err_printf("Could not open crash log temp file %s for writing: %s\n", + tmppath, strerror(errno)); + free(tmppath); + return; + } + + int first_site = 1; + for (i = 0; i < procdata->crash_sites_count; ++i) { + crash_site_entry_t *e = &procdata->crash_sites[i]; + if (e->log_ranks_count == 0) + continue; + qsort(e->log_ranks, e->log_ranks_count, sizeof(*e->log_ranks), rank_cmp); + if (!first_site) + fputc('\n', f); + first_site = 0; + const char *sep = strchr(e->site, '|'); + const char *site = sep ? sep + 1 : e->site; + fprintf(f, "exe: "); + if (sep) + write_escaped(f, e->site, (size_t) (sep - e->site)); + fprintf(f, "\nsite: "); + write_escaped(f, site, strlen(site)); + fprintf(f, "\nexemplar: %d\n", e->exemplar_rank); + fprintf(f, "count: %d\n", e->log_ranks_count); + fprintf(f, "ranks: "); + write_compressed_ranks(f, e->log_ranks, e->log_ranks_count); + } + + if (fclose(f) != 0) { + err_printf("Error writing crash log %s: %s\n", + tmppath, strerror(errno)); + unlink(tmppath); + free(tmppath); + return; + } + if (rename(tmppath, procdata->crash_log) != 0) { + err_printf("Could not rename crash log %s to %s: %s\n", + tmppath, procdata->crash_log, strerror(errno)); + unlink(tmppath); + free(tmppath); + return; + } + free(tmppath); + debug_printf("crash log: wrote %d sites to %s\n", nsites, procdata->crash_log); +} diff --git a/src/server/auditserver/ldcs_audit_server_handlers.c b/src/server/auditserver/ldcs_audit_server_handlers.c index ebfdef13..bdbbf847 100644 --- a/src/server/auditserver/ldcs_audit_server_handlers.c +++ b/src/server/auditserver/ldcs_audit_server_handlers.c @@ -1265,6 +1265,10 @@ static int handle_exit_broadcast(ldcs_process_data_t *procdata) ldcs_message_t out_msg; debug_printf("Setting up Exiting after receiving exit bcast message\n"); + /* Write crash logs before exiting */ + procdata->crash_log_teardown = 1; + crash_log_updated(procdata); + out_msg.header.type = LDCS_MSG_EXIT; out_msg.header.len = 0; out_msg.data = NULL; @@ -2003,6 +2007,8 @@ int handle_server_message(ldcs_process_data_t *procdata, node_peer_t peer, ldcs_ return handle_crash_report_recv(procdata, peer, msg); case LDCS_MSG_CRASH_RESPONSE: return handle_crash_response_recv(procdata, peer, msg); + case LDCS_MSG_CRASH_LOG: + return handle_crash_log_recv(procdata, peer, msg); default: err_printf("Received unexpected message from node: %d\n", (int) msg->header.type); assert(0); @@ -3142,6 +3148,7 @@ static int handle_send_exit_ready_if_done(ldcs_process_data_t *procdata) return handle_exit_broadcast(procdata); } else { + crash_log_flush_to_parent(procdata); debug_printf2("Sending exit ready message to parent\n"); result = spindle_forward_query(procdata, &msg); msgbundle_force_flush(procdata); diff --git a/src/server/auditserver/ldcs_audit_server_process.c b/src/server/auditserver/ldcs_audit_server_process.c index 8cd7b936..2109c2ff 100644 --- a/src/server/auditserver/ldcs_audit_server_process.c +++ b/src/server/auditserver/ldcs_audit_server_process.c @@ -149,6 +149,7 @@ int ldcs_audit_server_process(spindle_args_t *args) ldcs_process_data.cachepath_bitidx = 0; ldcs_process_data.number = args->number; ldcs_process_data.pythonprefix = args->pythonprefix; + ldcs_process_data.crash_log = args->crash_log; ldcs_process_data.localprefix = args->local_prefixes; ldcs_process_data.exec_excludes = args->exec_excludes; ldcs_process_data.numa_substrs = args->numa_files; @@ -176,6 +177,7 @@ int ldcs_audit_server_process(spindle_args_t *args) ldcs_process_data.crash_sites = NULL; ldcs_process_data.crash_sites_count = 0; ldcs_process_data.crash_sites_cap = 0; + ldcs_process_data.crash_log_teardown = 0; if (ldcs_process_data.opts & OPT_PULL) { debug_printf("Using PULL model\n"); diff --git a/src/server/auditserver/ldcs_audit_server_process.h b/src/server/auditserver/ldcs_audit_server_process.h index 4a1734ae..fc552d5c 100644 --- a/src/server/auditserver/ldcs_audit_server_process.h +++ b/src/server/auditserver/ldcs_audit_server_process.h @@ -131,6 +131,10 @@ typedef struct crash_site_entry_t { size_t site_len; int resolved; crash_waiter_t waiter; + int exemplar_rank; + int32_t *log_ranks; + int log_ranks_count; + int log_ranks_cap; } crash_site_entry_t; struct ldcs_process_data_struct @@ -157,6 +161,7 @@ struct ldcs_process_data_struct uint64_t cachepath_bitidx; /* Bit index of valid cachepaths on a given server. */ char *hostname; char *pythonprefix; + char *crash_log; /* Path of the crash-site log written by the root server. */ char *localprefix; char *exec_excludes; char *numa_substrs; @@ -187,6 +192,7 @@ struct ldcs_process_data_struct crash_site_entry_t *crash_sites; int crash_sites_count; int crash_sites_cap; + int crash_log_teardown; /* multi daemon support */ int md_rank; diff --git a/src/server/comlib/ldcs_api_util.c b/src/server/comlib/ldcs_api_util.c index e9c6cb89..a9e40ff0 100644 --- a/src/server/comlib/ldcs_api_util.c +++ b/src/server/comlib/ldcs_api_util.c @@ -95,6 +95,7 @@ char* _message_type_to_str (ldcs_message_ids_t type) { STR_CASE(LDCS_MSG_CHOSEN_CACHEPATH); STR_CASE(LDCS_MSG_CRASH_REPORT); STR_CASE(LDCS_MSG_CRASH_RESPONSE); + STR_CASE(LDCS_MSG_CRASH_LOG); STR_CASE(LDCS_MSG_UNKNOWN); } return "unknown"; diff --git a/src/server/startup/spindle_be.cc b/src/server/startup/spindle_be.cc index d5a234a1..a04105de 100644 --- a/src/server/startup/spindle_be.cc +++ b/src/server/startup/spindle_be.cc @@ -70,6 +70,7 @@ static int unpack_data(spindle_args_t *args, void *buffer, int buffer_size) unpack_param(args->local_prefixes, buf, pos); unpack_param(args->session_key, buf, pos); unpack_param(args->exec_excludes, buf, pos); + unpack_param(args->crash_log, buf, pos); assert(pos == buffer_size); assert( args->candidate_cachepaths ); return 0; diff --git a/src/slurm_plugin/plugin_utils.c b/src/slurm_plugin/plugin_utils.c index f77531e4..765eac2e 100644 --- a/src/slurm_plugin/plugin_utils.c +++ b/src/slurm_plugin/plugin_utils.c @@ -961,7 +961,13 @@ void decodeCmdArgs(char *cmd, int *sargc, char ***sargv) else { c = cmd[i]; } - target[t++] = c; + target[t++] = c; + } + + /* Flush the final token */ + if (t) { + target[t] = '\0'; + argv[argc++] = strdup(target); } if (!argc) { diff --git a/src/slurm_plugin/slurm_plugin.c b/src/slurm_plugin/slurm_plugin.c index 0989299c..24da9ffd 100644 --- a/src/slurm_plugin/slurm_plugin.c +++ b/src/slurm_plugin/slurm_plugin.c @@ -36,6 +36,8 @@ Place, Suite 330, Boston, MA 02111-1307 USA #define SPINDLE_USE_SESSION "SPINDLE_USE_SESSION" #define SPANK_SPINDLE_USE_SESSION "SPANK_" SPINDLE_USE_SESSION +#define SPINDLE_SESSION_OPTS "SPINDLE_SESSION_OPTS" +#define SPANK_SPINDLE_SESSION_OPTS "SPANK_" SPINDLE_SESSION_OPTS SPINDLE_EXPORT extern const char plugin_name[]; SPINDLE_EXPORT extern const char plugin_type[]; @@ -100,6 +102,7 @@ static pid_t pidBE = 0; static pid_t pidFE = 0; static __thread spank_t current_spank; static const char *user_options = NULL; +static const char *session_options = NULL; static int enable_spindle = 0; static int start_session = 0; @@ -119,8 +122,8 @@ struct spank_option spank_options[] = // CLI options for salloc and sbatch struct spank_option session_option = { - "spindle-session", NULL, - "Start a Spindle session for this allocation", 0, 0, + "spindle-session", "[spindle options]", + "Start a Spindle session for this allocation", 2, 0, (spank_opt_cb_f) spindle_session_options }; @@ -152,16 +155,56 @@ static int should_use_session(spank_t spank) { spank_option_getopt (but this doesn't work in remote context which is why we need cases for both remote and job script). */ if (context == S_CTX_JOB_SCRIPT) { + char *optval = NULL; session_env = getenv(SPANK_SPINDLE_USE_SESSION); - if (session_env) + if (session_env) return 1; - err = spank_option_getopt(spank, &session_option, NULL); - return (err == ESPANK_SUCCESS); + err = spank_option_getopt(spank, &session_option, &optval); + return (err == ESPANK_SUCCESS); } return 0; } +/* Slurm stores an absent optional argument as the string "(null)" */ +static const char *filter_session_optval(const char *val) +{ + if (!val || val[0] == '\0' || strcmp(val, "(null)") == 0) + return NULL; + return val; +} + +/* Retrieves the optional argument of --spindle-session */ +static const char *get_session_options(spank_t spank) +{ + spank_context_t context; + char *val = NULL; + + context = spank_context(); + if (context == S_CTX_LOCAL || context == S_CTX_ALLOCATOR) + return filter_session_optval(session_options ? session_options + : getenv(SPANK_SPINDLE_SESSION_OPTS)); + if (context == S_CTX_REMOTE) + return filter_session_optval(readSpankEnv(spank, SPANK_SPINDLE_SESSION_OPTS)); + if (context == S_CTX_JOB_SCRIPT) { + val = getenv(SPANK_SPINDLE_SESSION_OPTS); + if (!val && spank_option_getopt(spank, &session_option, &val) != ESPANK_SUCCESS) + val = NULL; + return filter_session_optval(val); + } + return NULL; +} + +/* In session mode, point process_spindle_args' merge at the session's options. */ +static void apply_session_options(spank_t spank) +{ + const char *opts = get_session_options(spank); + if (opts) { + sdprintf(2, "Using spindle session options '%s'\n", opts); + user_options = opts; + } +} + int slurm_spank_init(spank_t spank, int ac, char *argv[]) { spank_context_t context; context = spank_context(); @@ -185,6 +228,8 @@ int slurm_spank_init_post_opt(spank_t spank, int ac, char *argv[]) { } if (start_session) { setenv(SPANK_SPINDLE_USE_SESSION, "1", 1); + if (session_options) + setenv(SPANK_SPINDLE_SESSION_OPTS, session_options, 1); } } return 0; @@ -208,6 +253,14 @@ static int forward_environment_to_job_control(spank_t spank) /* Forward session status from srun to job prolog. */ err = spank_job_control_setenv(spank, SPINDLE_USE_SESSION, "1", 1); if (err != ESPANK_SUCCESS) return -1; + /* Forward the session's spindle options (inherited from salloc's + environment) so the epilog can see them; SPANK env vars do not + otherwise propagate to the epilog. */ + envVal = getenv(SPANK_SPINDLE_SESSION_OPTS); + if (envVal) { + err = spank_job_control_setenv(spank, SPINDLE_SESSION_OPTS, envVal, 1); + if (err != ESPANK_SUCCESS) return -1; + } } envVal = getenv("SPINDLE_DEBUG"); @@ -550,6 +603,8 @@ static int handleStart(void *params, char **output_str) sdprintf(1, "In handleStart\n"); use_session = should_use_session(spank); + if (use_session) + apply_session_options(spank); result = process_spindle_args(spank, site_argc, site_argv, &args, NULL, NULL, use_session); if (result == -1) { sdprintf(1, "ERROR: Could not process spindle args in handlestart\n"); @@ -1091,10 +1146,42 @@ static int spindle_options(int val, const char *optarg, int remote) return 0; } +/* Validate session options so that we can fail immediately at salloc/sbatch + if they are invalid, rather than failing only when we get to srun */ +static int validate_session_options(const char *opts) +{ + spindle_args_t args = {0}; + char *copy, *err_string = NULL; + char **argv = NULL; + int argc = 0, result, i; + + copy = strdup(opts); + decodeCmdArgs(copy, &argc, &argv); + result = fillInSpindleArgsCmdlineFE(&args, SPINDLE_FILLARGS_NOUNIQUEID | SPINDLE_FILLARGS_NONUMBER, + argc, argv, &err_string); + if (result == -1 && err_string && err_string[0]) { + slurm_error("Error processing Spindle session arguments: %s", err_string); + } + for (i = 0; i < argc; i++) + free(argv[i]); + free(argv); + free(copy); + return result; +} + /* Handles arguments to salloc and sbatch */ static int spindle_session_options(int val, const char *optarg, int remote) { + spank_context_t context; + start_session = 1; + if (optarg) { + context = spank_context(); + if ((context == S_CTX_ALLOCATOR || context == S_CTX_LOCAL) && + validate_session_options(optarg) == -1) + return -1; + session_options = optarg; + } return 0; } @@ -1257,6 +1344,8 @@ static int handleExit(void *params, char **output_str) sdprintf(1, "In handleExit\n"); use_session = should_use_session(spank); + if (use_session) + apply_session_options(spank); result = process_spindle_args(spank, site_argc, site_argv, &args, NULL, NULL, use_session); if (result == -1) { sdprintf(1, "ERROR: Could not process spindle args in handleExit\n"); diff --git a/src/utils/parseloc.c b/src/utils/parseloc.c index bdf5ca1f..94c2baf6 100644 --- a/src/utils/parseloc.c +++ b/src/utils/parseloc.c @@ -155,6 +155,57 @@ char *parse_location_noerr(char *loc, number_t number) return parse_location_impl(loc, number, 0); } +/* + * Resolve the crash-log path to an absolute file path. Defaults to + * "spindle-crash-log.$NUMBER" in the working directory; a user-provided + * path may name the file or a directory to place the log in. + */ +char *resolve_crash_log_path(const char *value, number_t number) +{ + char default_template[] = "spindle-crash-log.$NUMBER"; + char *path; + struct stat sb; + + path = parse_location(value && value[0] != '\0' ? (char *) value : default_template, + number); + if (!path) { + err_printf("Could not expand crash log path %s\n", + value ? value : default_template); + return NULL; + } + + if (path[0] != '/') { + char cwd[MAX_PATH_LEN+1]; + if (getcwd(cwd, sizeof(cwd)) == NULL) { + err_printf("Could not get working directory to resolve crash log path %s\n", path); + free(path); + return NULL; + } + char *abspath = malloc(strlen(cwd) + strlen(path) + 2); + sprintf(abspath, "%s/%s", cwd, path); + free(path); + path = abspath; + } + + /* If the path is a directory, we write the file into that directory; + if not, we use it as the full path to the log file. */ + if (stat(path, &sb) == 0 && S_ISDIR(sb.st_mode)) { + char *filename = parse_location(default_template, number); + if (!filename) { + err_printf("Could not expand crash log filename\n"); + free(path); + return NULL; + } + int add_slash = path[strlen(path)-1] != '/'; + char *fullpath = malloc(strlen(path) + add_slash + strlen(filename) + 1); + sprintf(fullpath, "%s%s%s", path, add_slash ? "/" : "", filename); + free(filename); + free(path); + path = fullpath; + } + return path; +} + /** * Realize takes the 'realpath' of a non-existant location. * If later directories in the path don't exist, it'll cut them diff --git a/src/utils/parseloc.h b/src/utils/parseloc.h index a99409c3..103ee08a 100644 --- a/src/utils/parseloc.h +++ b/src/utils/parseloc.h @@ -25,6 +25,7 @@ extern "C" { char *parse_location(char *loc, number_t number); char *parse_location_noerr(char *loc, number_t number); +char *resolve_crash_log_path(const char *value, number_t number); char *realize(char *path); char **parse_colonsep_prefixes(char *colonsep_list, number_t number); int is_local_prefix(const char *path, char **local_prefixes); diff --git a/testsuite/run_crash_tests_template.sh b/testsuite/run_crash_tests_template.sh index 3d4b71c9..75203366 100755 --- a/testsuite/run_crash_tests_template.sh +++ b/testsuite/run_crash_tests_template.sh @@ -1,9 +1,14 @@ #!/usr/bin/env bash # Usage: -# ./run_crash_tests.sh [--launcher=serial|flux|slurm|slurm-plugin] --nodes=N [--scratch=DIR] +# ./run_crash_tests.sh [--launcher=serial|flux|slurm|slurm-plugin] --nodes=N +# [--scratch=DIR] [--modes=LIST] [--session | --cross-exe] # -# If running the tests from a non-shared filesystem, set --scratch to a shared +# By default, runs all normal crash tests; use --modes to specify a subset to run. +# --session and --cross-exe instead run the session-based tests; +# these are separate because they require fresh sessions +# +# If running the tests from a non-shared filesystem, set --scratch to a shared # filesystem so that the script can count all corefiles produced across all nodes. # The CI containers' main filesystem is not shared, so a shared volume should be # mounted across all containers and specified in --scratch @@ -23,7 +28,11 @@ die() { echo "FAIL: $*" >&2; exit 1; } # Fields: # mode: --mode to pass to crash test runner # cores: expected number of cores produced; if N, then equal to total number of ranks -# flags: if "multi-rank", skip when running on only one rank; if "clean", expect test to NOT crash +# crashers: expected number of crashing ranks (== total ranks in the crash log); +# N = all ranks, E = even ranks (ceil(N/2)), or a literal count +# flags (comma-separated): "multi-rank" skips the mode on a single rank; +# "clean" expects the test to NOT crash; "altstack" runs the mode with +# --crash-altstack. # top_frame_regex: regex that should match the top frame in produced coredumps. # Note that all threads will be checked, so in multithreaded examples the regex should # also match anything that could be on threads other than the one that faulted. @@ -32,41 +41,46 @@ die() { echo "FAIL: $*" >&2; exit 1; } # binary: optional alternate executable to run in place of the default crash_test. # crash_mode: optional alternate crash mode argument to executable CRASH_TESTS=( -# mode ; cores ; flags ; top_frame_regex ; site_regex ; binary ; crash_mode - 'all-same ; 1 ; ; crash_function_A ; crash_test\+0x' - 'fixed-address-exe ; 1 ; ; crash_function_A ; crash_test_fixedaddr\+0x ; crash_test_fixedaddr ; all-same' - 'pie-exe ; 1 ; ; crash_function_A ; crash_test_pie\+0x ; crash_test_pie ; all-same' - 'all-different ; N ; ; crash_function_[0-9]+' - 'two-groups ; 2 ; ; crash_function_(A|B)' - 'one-crashes ; 1 ; ; crash_function_A' - 'partial ; 1 ; ; crash_function_A' - 'late-straggler ; 1 ; ; crash_function_A' - 'in-library ; 1 ; ; crash_in_library ; libcrashfuncs\.so\+0x' - 'in-dlmopen-library ; 1 ; ; crash_in_library ; libcrashfuncs\.so\+0x' - 'in-fixed-library ; 1 ; ; crash_in_fixed_library ; libcrashfixed\.so\+0x' - 'in-fixed-dlmopen-library ; 1 ; ; crash_in_fixed_library ; libcrashfixed\.so\+0x' - 'in-library-ctor ; 1 ; ; ctor_crash' - 'sigabrt ; 1 ; ; (__GI_)?raise|abort|pthread_kill' - 'assert ; 1 ; ; (__GI_)?raise|abort|pthread_kill ; abort:.*Assertion' - 'mixed-abort-segv ; 2 ; multi-rank ; (__GI_)?raise|abort|pthread_kill|do_mixed_abort_segv' - 'span-read ; 1 ; ; do_span_read' - 'safepoint ; 0 ; clean ; -' - 'safepoint-then-crash ; 1 ; ; crash_function_A' - 'safepoint-bad ; 1 ; ; do_safepoint_bad' - 'safepoint-bad-write ; 1 ; ; do_safepoint_bad_write' - 'safepoint-fix-write ; 0 ; clean ; -' - 'safepoint-longjmp ; 0 ; clean ; -' - 'safepoint-span-read ; 0 ; clean ; -' - 'safepoint-span-write ; 0 ; clean ; -' - 'safepoint-span-bad-read ; 1 ; ; do_safepoint_span_bad_read' - 'safepoint-span-bad-write ; 1 ; ; do_safepoint_span_bad_write' - 'mmap-sigbus-bad ; 1 ; ; do_mmap_sigbus_bad' - 'mmap-sigbus-fixed ; 0 ; clean ; -' - 'no-crash ; 0 ; clean ; -' +# mode ; cores ; crashers ; flags ; top_frame_regex ; site_regex ; binary ; crash_mode + 'all-same ; 1 ; N ; ; crash_function_A ; crash_test\+0x ' + 'fixed-address-exe ; 1 ; N ; ; crash_function_A ; crash_test_fixedaddr\+0x ; crash_test_fixedaddr ; all-same' + 'pie-exe ; 1 ; N ; ; crash_function_A ; crash_test_pie\+0x ; crash_test_pie ; all-same' + 'all-different ; N ; N ; ; crash_function_[0-9]+' + 'two-groups ; 2 ; N ; ; crash_function_(A|B)' + 'one-crashes ; 1 ; 1 ; ; crash_function_A' + 'partial ; 1 ; E ; ; crash_function_A' + 'late-straggler ; 1 ; N ; ; crash_function_A' + 'in-library ; 1 ; N ; ; crash_in_library ; libcrashfuncs\.so\+0x ' + 'in-dlmopen-library ; 1 ; N ; ; crash_in_library ; libcrashfuncs\.so\+0x ' + 'in-fixed-library ; 1 ; N ; ; crash_in_fixed_library ; libcrashfixed\.so\+0x ' + 'in-fixed-dlmopen-library ; 1 ; N ; ; crash_in_fixed_library ; libcrashfixed\.so\+0x ' + 'in-library-ctor ; 1 ; 1 ; ; ctor_crash' + 'sigabrt ; 1 ; N ; ; (__GI_)?raise|abort|pthread_kill' + 'assert ; 1 ; N ; ; (__GI_)?raise|abort|pthread_kill ; ^abort:.*Assertion ' + 'mixed-abort-segv ; 2 ; N ; multi-rank ; (__GI_)?raise|abort|pthread_kill|do_mixed_abort_segv' + 'span-read ; 1 ; N ; ; do_span_read' + 'safepoint ; 0 ; 0 ; clean ; -' + 'safepoint-then-crash ; 1 ; N ; ; crash_function_A' + 'safepoint-bad ; 1 ; N ; ; do_safepoint_bad' + 'safepoint-bad-write ; 1 ; N ; ; do_safepoint_bad_write' + 'safepoint-fix-write ; 0 ; 0 ; clean ; -' + 'safepoint-fix-write-altstack ; 0 ; 0 ; clean,altstack ; - ; ; ; safepoint-fix-write' + 'safepoint-longjmp ; 0 ; 0 ; clean ; -' + 'safepoint-span-read ; 0 ; 0 ; clean ; -' + 'safepoint-span-write ; 0 ; 0 ; clean ; -' + 'safepoint-span-bad-read ; 1 ; N ; ; do_safepoint_span_bad_read' + 'safepoint-span-bad-write ; 1 ; N ; ; do_safepoint_span_bad_write' + 'mmap-sigbus-bad ; 1 ; N ; ; do_mmap_sigbus_bad' + 'mmap-sigbus-fixed ; 0 ; 0 ; clean ; -' + 'no-crash ; 0 ; 0 ; clean ; -' ) -declare -A TEST_CORES TEST_FLAGS TEST_TOPFRAME TEST_SITE TEST_BINARY TEST_CRASHMODE +declare -A TEST_CORES TEST_CRASHERS TEST_FLAGS TEST_TOPFRAME TEST_SITE TEST_BINARY TEST_CRASHMODE DEFAULT_MODES=() +SESSION=0 +CROSS_EXE=0 + +# ---------------- test table parsing ---------------- trim() { local s="$1" @@ -76,12 +90,13 @@ trim() { } parse_table() { - local row mode cores flags top site bin cmode + local row mode cores crashers flags top site bin cmode for row in "${CRASH_TESTS[@]}"; do - IFS=';' read -r mode cores flags top site bin cmode <<<"$row" + IFS=';' read -r mode cores crashers flags top site bin cmode <<<"$row" mode=$(trim "$mode") [ -n "$mode" ] || continue cores=$(trim "$cores") + crashers=$(trim "$crashers") flags=$(trim "$flags") top=$(trim "$top") site=$(trim "$site") @@ -90,6 +105,7 @@ parse_table() { [ -n "$bin" ] || bin="crash_test" [ -n "$cmode" ] || cmode="$mode" TEST_CORES[$mode]="$cores" + TEST_CRASHERS[$mode]="$crashers" TEST_FLAGS[$mode]="$flags" TEST_TOPFRAME[$mode]="$top" TEST_SITE[$mode]="$site" @@ -117,33 +133,47 @@ resolve_cores() { printf '%s' "$val" } +# Expected number of crashing ranks +resolve_crashers() { + local mode="$1" val + val="${TEST_CRASHERS[$mode]}" + case "$val" in + N) val="$NODES" ;; + E) val=$(( (NODES + 1) / 2 )) ;; + esac + printf '%s' "$val" +} + +# ---------------- arguments ---------------- + usage() { local prog prog=$(basename "$0") cat <&2 + rc=1 + continue + fi + for ((r = lo; r <= hi; r++)); do + printf '%s\n' "$r" + done + done + return $rc +} + +# Verify the crash log file contains the expected crash sites. +verify_crash_log() { local mode="$1" local dir="$2" + local expected_sites="$3" + local log="$dir/crash.log" + local expected_total + expected_total=$(resolve_crashers "$mode") + + if [ ! -f "$log" ]; then + echo " crash log $log missing" >&2 + return 1 + fi + local expected_site="${TEST_SITE[$mode]:-}" + [ "$expected_site" = "-" ] && expected_site="" + + local binary_name="${TEST_BINARY[$mode]:-crash_test}" + local expected_exe="${binary_name}\$" + + local rc=0 sites=0 total=0 + local line exe site exemplar="" count="" + local -A seen=() + local ranks r ranks_count exemplar_listed + + # Parse the crash log + while IFS= read -r line; do + case "$line" in + "exe: "*) + exe="${line#exe: }" + sites=$((sites + 1)) + exemplar="" + count="" + if ! [[ "$exe" =~ $expected_exe ]]; then + echo " exe '$exe' does not match executable '$binary_name'" >&2 + rc=1 + fi + ;; + "site: "*) + site="${line#site: }" + if [ -n "$expected_site" ] && ! [[ "$site" =~ $expected_site ]]; then + echo " site '$site' does not match '$expected_site'" >&2 + rc=1 + fi + ;; + "exemplar: "*) + exemplar="${line#exemplar: }" + ;; + "count: "*) + count="${line#count: }" + ;; + "ranks: "*) + ranks_count=0 + exemplar_listed=0 + ranks=$(expand_rank_list "${line#ranks: }") || rc=1 + for r in $ranks; do + ranks_count=$((ranks_count + 1)) + if [ "$r" -ge "$NODES" ]; then + echo " rank $r outside [0,$NODES)" >&2 + rc=1 + fi + if [ -n "${seen[$r]:-}" ]; then + echo " rank $r repeated" >&2 + rc=1 + fi + seen[$r]=1 + [ "$r" = "$exemplar" ] && exemplar_listed=1 + done + if [ "$count" != "$ranks_count" ]; then + echo " count $count != $ranks_count listed ranks" >&2 + rc=1 + fi + if [ "$exemplar_listed" != "1" ]; then + echo " exemplar '$exemplar' not in ranks list" >&2 + rc=1 + fi + total=$((total + ranks_count)) + ;; + esac + done <"$log" - # If this test doesn't have a crash site specified, there's nothing to do - [ "$expected_site" = "-" ] && return 0 - [ -z "$expected_site" ] && return 0 + if [ "$sites" != "$expected_sites" ]; then + echo " $sites crashsites, expected $expected_sites" >&2 + rc=1 + fi + if [ "$total" != "$expected_total" ]; then + echo " $total total ranks, expected $expected_total" >&2 + rc=1 + fi + return $rc +} - local binary="$TESTDIR/${TEST_BINARY[$mode]:-crash_test}" - local core cores site +# Verify that each coredump recorded in the log actually exists on disk. +verify_exemplar_cores() { + local mode="$1" + local dir="$2" + local log="$dir/crash.log" + + local -A rank_pid + local line r p f + for f in "$dir/stdout.log" "$dir/stderr.log"; do + [ -f "$f" ] || continue + while IFS= read -r line; do + if [[ "$line" =~ rank=([0-9]+)\ .*pid=([0-9]+) ]]; then + r="${BASH_REMATCH[1]}" + p="${BASH_REMATCH[2]}" + rank_pid[$r]="$p" + fi + done <"$f" + done + + local core cores cores=$(core_files "$dir") - for core in $cores; do - site=$(read_crash_site "$core" "$binary") - if [ -z "$site" ]; then - echo " core $core: could not read crash site" >&2 - return 1 + + local ex found + while IFS= read -r line; do + case "$line" in "exemplar: "*) ;; *) continue ;; esac + ex="${line#exemplar: }" + p="${rank_pid[$ex]:-}" + if [ -z "$p" ]; then + echo " no pid banner for exemplar rank $ex; skipping core check" >&2 + continue fi - if ! printf '%s' "$site" | grep -qE -- "$expected_site"; then - echo " core $core: crash dedup key '$site' does not match '$expected_site'" >&2 + found=0 + for core in $cores; do + case "$core" in + */rank_"$ex"/*) ;; + *) continue ;; + esac + if [[ "${core##*/}" =~ (^|[^0-9])$p([^0-9]|$) ]]; then + found=1 + break + fi + done + if [ "$found" != "1" ]; then + echo " no coredump found for exemplar rank $ex (pid $p)" >&2 return 1 fi - done + done <"$log" return 0 } +# Verify that logged crash site matches what is recorded in the coredump +verify_log_matches_core() { + local mode="$1" + local dir="$2" + local log="$dir/crash.log" + local binary="$TESTDIR/${TEST_BINARY[$mode]:-crash_test}" + + local core log_exe log_site core_site + core=$(core_files "$dir" | head -1) + [ -n "$core" ] || { echo " no core for gdb cross-check" >&2; return 1; } + log_exe=$(log_values "$log" exe | head -1) + log_site=$(log_values "$log" site | head -1) + core_site=$(read_crash_site "$core" "$binary") + if [ -z "$core_site" ]; then + echo " could not read crash site from $core" >&2 + return 1 + fi + if [ "$core_site" != "$log_exe|$log_site" ]; then + echo " core site '$core_site' != logged site '$log_exe|$log_site'" >&2 + return 1 + fi + return 0 +} + +# ---------------- session tests ---------------- + +# Common setup for the session-based tests. Sets three globals consumed by +# session_test_launch and the run_*_test functions: +# SESSION_DIR per-test scratch directory +# SESSION_LOG crash log path inside SESSION_DIR +# SESSION_RUN launcher command that runs one program inside the session +session_test_setup() { + local name="$1" + case "$LAUNCHER" in + slurm-plugin|flux) ;; + *) die "--$name requires --launcher=slurm-plugin or --launcher=flux" ;; + esac + + mkdir -p "$CRASH_TEST_SCRATCH" || die "can't create scratch dir '$CRASH_TEST_SCRATCH'" + SESSION_DIR=$(mktemp -d "$CRASH_TEST_SCRATCH/$name.XXXXXX") || die "can't create test dir" + SESSION_LOG="$SESSION_DIR/crash.log" + + case "$LAUNCHER" in + slurm-plugin) + SESSION_RUN="srun --spindle" + ;; + flux) + SESSION_RUN="flux run -o userrc=$SPINDLE_RC -o spindle --env=LD_LIBRARY_PATH -N$NODES -n$NODES --" + ;; + esac +} + +session_test_launch() { + local session_opts="--crash-dedup --crash-log=$SESSION_LOG" + chmod +x "$SESSION_DIR/inner.sh" + ulimit -c unlimited + case "$LAUNCHER" in + slurm-plugin) + ( cd "$SESSION_DIR" && salloc -N"$NODES" -n"$NODES" \ + --spindle-session="$session_opts" "$SESSION_DIR/inner.sh" ) \ + >"$SESSION_DIR/stdout.log" 2>"$SESSION_DIR/stderr.log" + ;; + flux) + # The session's crash options must be given at session start + local sid + sid=$("$SPINDLE" --start-session $session_opts 2>"$SESSION_DIR/session.log") || \ + die "spindle --start-session failed (see $SESSION_DIR/session.log)" + ( cd "$SESSION_DIR" && "$SESSION_DIR/inner.sh" ) \ + >"$SESSION_DIR/stdout.log" 2>"$SESSION_DIR/stderr.log" + if [ -n "$sid" ]; then + "$SPINDLE" --end-session="$sid" >>"$SESSION_DIR/session.log" 2>&1 + else + "$SPINDLE" --end-session >>"$SESSION_DIR/session.log" 2>&1 + fi + ;; + esac + # Wait briefly for the server to shut down and write the log + sleep 2 +} + +# Session-mode crash-log test +# two crashing runs inside one spindle session share a crash log. +run_session_test() { + session_test_setup session + local dir="$SESSION_DIR" log="$SESSION_LOG" + + # inner.sh is the script that gets run inside the session + cat >"$dir/inner.sh" < "$dir/log_after_run1" +$SESSION_RUN "$TESTDIR/crash_test" --crash-mode sigabrt +sleep 3 +if [ -e "$log" ]; then echo present; else echo absent; fi > "$dir/log_after_run2" +EOF + session_test_launch + + local after1 after2 sites=0 total=0 c + after1=$(cat "$dir/log_after_run1" 2>/dev/null || echo missing) + after2=$(cat "$dir/log_after_run2" 2>/dev/null || echo missing) + if [ -f "$log" ]; then + sites=$(log_values "$log" site | wc -l) + for c in $(log_values "$log" count); do + total=$((total + c)) + done + fi + + local ok=1 + [ "$after1" = "absent" ] || { echo "FAIL session: log $after1 after run 1"; ok=0; } + [ "$after2" = "absent" ] || { echo "FAIL session: log $after2 after run 2"; ok=0; } + [ "$sites" = "2" ] || { echo "FAIL session: $sites sites after session end"; ok=0; } + [ "$total" = "$((2 * NODES))" ] || \ + { echo "FAIL session: $total total ranks in final log (expected $((2 * NODES)))"; ok=0; } + + [ "$ok" = "1" ] || exit 1 + echo "PASS session" +} + +# Cross-executable dedup test: two different executables crashing +# at the same offset in the same shared library inside one session +# should not be deduplicated +run_cross_exe_test() { + session_test_setup cross-exe + local dir="$SESSION_DIR" + + # inner.sh is the script that gets run inside the session + cat >"$dir/inner.sh" <"$dir/stdout.log" 2>"$dir/stderr.log" || launch_rc=$? + [ "$LAUNCHER" = "serial" ] || sleep 1 + + if has_flag "$mode" clean; then + verify_clean_mode "$mode" "$dir" "$launch_rc" + else + verify_crashed_mode "$mode" "$dir" + fi +} + main() { - parse_args "$@" parse_table + parse_args "$@" if [ -z "$CRASH_TEST_SCRATCH" ]; then if [ -n "${SPINDLE_TEST_CONTAINER:-}" ]; then @@ -336,6 +804,16 @@ main() { fi check_prereqs + if [ "$SESSION" = "1" ]; then + run_session_test + return + fi + + if [ "$CROSS_EXE" = "1" ]; then + run_cross_exe_test + return + fi + local pass=0 fail=0 local modes_to_run @@ -345,8 +823,7 @@ main() { modes_to_run=() IFS=',' read -ra specified_modes <<< "$MODES" for m in "${specified_modes[@]}"; do - m=$(echo "$m" | xargs) - modes_to_run+=("$m") + modes_to_run+=("$(trim "$m")") done fi @@ -361,64 +838,11 @@ main() { continue fi - # Run each test in per-test directory so all the coredumps run in one place - # and we can count them - mkdir -p "$CRASH_TEST_SCRATCH" || die "can't create scratch dir '$CRASH_TEST_SCRATCH'" - local dir - dir=$(mktemp -d "$CRASH_TEST_SCRATCH/$mode.XXXXXX") || die "can't create test dir under '$CRASH_TEST_SCRATCH'" - local launch_rc=0 - ( cd "$dir" && launch "$mode" ) >"$dir/stdout.log" 2>"$dir/stderr.log" || launch_rc=$? - [ "$LAUNCHER" = "serial" ] || sleep 1 - - # If this test is not supposed to crash, verify that it didn't crash - # and exited normally - if has_flag "$mode" clean; then - local actual_cores - actual_cores=$(count_cores "$dir") - if [ "$actual_cores" != "0" ]; then - echo "FAIL $mode: expected 0 dumps, got $actual_cores" - fail=$((fail+1)) - continue - fi - if [ "$launch_rc" != "0" ]; then - echo "FAIL $mode: launcher exited $launch_rc (expected 0)" - fail=$((fail+1)) - continue - fi - echo "PASS $mode (clean exit)" + if run_one_mode "$mode"; then pass=$((pass+1)) - continue - fi - - # Otherwise, if this test is supposed to crash, verify that we - # got the number of core files that we expect - local want - want=$(resolve_cores "$mode") - local actual - actual=$(count_cores "$dir") - - if [ "$actual" != "$want" ]; then - echo "FAIL $mode: expected $want coredumps, got $actual" - fail=$((fail+1)) - continue - fi - - # And verify that the core files show the expected crash sites - if ! verify_top_frames "$mode" "$dir"; then - echo "FAIL $mode: dump top-frame verification failed" + else fail=$((fail+1)) - continue fi - - # Verify the crash handler's + dedup key - if ! verify_crash_site "$mode" "$dir"; then - echo "FAIL $mode: crash site dedup key verification failed" - fail=$((fail+1)) - continue - fi - - echo "PASS $mode ($actual dumps)" - pass=$((pass+1)) done echo