diff --git a/.github/workflows/ci-flux-artifacts.yml b/.github/workflows/ci-flux-artifacts.yml new file mode 100644 index 00000000..3d35772c --- /dev/null +++ b/.github/workflows/ci-flux-artifacts.yml @@ -0,0 +1,79 @@ +name: ci-flux-artifacts + +on: + workflow_dispatch: + +permissions: + contents: read + packages: read + +jobs: + spindle-flux-ubuntu: + name: Testsuite (Flux, Ubuntu) + environment: Spindle CI + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out Spindle + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + + - name: Setup Docker Compose + uses: docker/setup-compose-action@16feee727cbdc83b6a014e6cc26fec4a79bcf30c + with: + version: latest + + - name: Build spindle-flux-ubuntu image + id: flux-ubuntu-build + run: | + cd containers/spindle-flux-ubuntu + docker compose --progress=plain build + + - name: Bring spindle-flux-ubuntu up + id: flux-ubuntu-up + run: | + cd containers/spindle-flux-ubuntu + docker compose up -d --wait --wait-timeout 60 + + - name: Verify munge works in spindle-flux-ubuntu + id: flux-ubuntu-munge + run: | + docker exec node-1 bash -c 'munge -n | unmunge' + + - name: Run spindle-flux-ubuntu testsuite + id: flux-ubuntu-testsuite + run: | + docker exec node-1 bash -c 'cd Spindle-build/testsuite && SPINDLE_DEBUG=3 flux alloc --nodes=${workers} ./runTests --nodes=${workers} --tasks-per-node=3' + +# - name: Configure host core_pattern for coredumps +# if: ${{ !env.ACT }} +# run: | +# sudo sysctl -w kernel.core_pattern='core.%p' +# +# - name: Run spindle-flux-ubuntu crash tests +# run: | +# docker exec node-1 bash -c 'cd Spindle-build/testsuite && SPINDLE_DEBUG=3 flux alloc --nodes=${workers} ./run_crash_tests.sh --launcher=flux --nodes=${workers} --scratch=/shared/spindle_crash_test' + + - name: Collect Spindle logs from container + if: ${{ always() }} + continue-on-error: true + run: | + mkdir -p spindle-logs + docker cp node-1:/Spindle-build/testsuite/. spindle-logs/ || echo "Log collection failed" + ls -la spindle-logs/ + + - name: Upload Spindle logs + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: spindle-logs-flux + path: spindle-logs/spindle_output.* + retention-days: 7 + if-no-files-found: warn + + - name: Bring spindle-flux-ubuntu down + id: flux-ubuntu-down + if: ${{ always() }} + continue-on-error: true + run: | + cd containers/spindle-flux-ubuntu + docker compose down diff --git a/.github/workflows/ci-serial-arm64-artifacts.yml b/.github/workflows/ci-serial-arm64-artifacts.yml new file mode 100644 index 00000000..dd6fea1b --- /dev/null +++ b/.github/workflows/ci-serial-arm64-artifacts.yml @@ -0,0 +1,79 @@ +name: ci-serial-arm64-artifacts + +on: + workflow_dispatch: + +permissions: + contents: read + packages: read + +jobs: + spindle-serial-ubuntu_arm64: + name: Testsuite (Serial, Ubuntu, arm64) + environment: Spindle CI + runs-on: ubuntu-24.04-arm + timeout-minutes: 30 + steps: + - name: Check out Spindle + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + + - name: Setup Docker Compose + uses: docker/setup-compose-action@16feee727cbdc83b6a014e6cc26fec4a79bcf30c + with: + version: latest + + - name: Build spindle-serial-ubuntu image + id: serial-ubuntu-build + run: | + cd containers/spindle-serial-ubuntu + docker compose --progress=plain build + + - name: Bring spindle-serial-ubuntu up + id: serial-ubuntu-up + run: | + cd containers/spindle-serial-ubuntu + docker compose up -d + + - name: Verify munge works in spindle-serial-ubuntu + id: serial-ubuntu-munge + run: | + docker exec spindlenode bash -c 'munge -n | unmunge' + + - name: Run spindle-serial-ubuntu testsuite + id: serial-ubuntu-testsuite + run: | + docker exec spindlenode bash -c 'cd Spindle-build/testsuite && SPINDLE_DEBUG=3 ./runTests' + +# - name: Configure host core_pattern for coredumps +# if: ${{ !env.ACT }} +# run: | +# sudo sysctl -w kernel.core_pattern='core.%p' +# +# - name: Run spindle-serial-ubuntu crash tests +# run: | +# docker exec spindlenode bash -c 'cd Spindle-build/testsuite && SPINDLE_DEBUG=3 ./run_crash_tests.sh --launcher=serial --scratch=/tmp/spindle_crash_test' + + - name: Collect Spindle logs from container + if: ${{ always() }} + continue-on-error: true + run: | + mkdir -p spindle-logs + docker cp spindlenode:/Spindle-build/testsuite/. spindle-logs/ || echo "Log collection failed" + ls -la spindle-logs/ + + - name: Upload Spindle logs + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: spindle-logs-serial-arm64 + path: spindle-logs/spindle_output.* + retention-days: 7 + if-no-files-found: warn + + - name: Bring spindle-serial-ubuntu down + id: serial-ubuntu-down + if: ${{ always() }} + continue-on-error: true + run: | + cd containers/spindle-serial-ubuntu + docker compose down diff --git a/.github/workflows/ci-serial-x86-artifacts.yml b/.github/workflows/ci-serial-x86-artifacts.yml new file mode 100644 index 00000000..d8dd05a3 --- /dev/null +++ b/.github/workflows/ci-serial-x86-artifacts.yml @@ -0,0 +1,79 @@ +name: ci-serial-x86-artifacts + +on: + workflow_dispatch: + +permissions: + contents: read + packages: read + +jobs: + spindle-serial-ubuntu_x86: + name: Testsuite (Serial, Ubuntu, x86) + environment: Spindle CI + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out Spindle + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + + - name: Setup Docker Compose + uses: docker/setup-compose-action@16feee727cbdc83b6a014e6cc26fec4a79bcf30c + with: + version: latest + + - name: Build spindle-serial-ubuntu image + id: serial-ubuntu-build + run: | + cd containers/spindle-serial-ubuntu + docker compose --progress=plain build + + - name: Bring spindle-serial-ubuntu up + id: serial-ubuntu-up + run: | + cd containers/spindle-serial-ubuntu + docker compose up -d + + - name: Verify munge works in spindle-serial-ubuntu + id: serial-ubuntu-munge + run: | + docker exec spindlenode bash -c 'munge -n | unmunge' + + - name: Run spindle-serial-ubuntu testsuite + id: serial-ubuntu-testsuite + run: | + docker exec spindlenode bash -c 'cd Spindle-build/testsuite && SPINDLE_DEBUG=3 ./runTests' + +# - name: Configure host core_pattern for coredumps +# if: ${{ !env.ACT }} +# run: | +# sudo sysctl -w kernel.core_pattern='core.%p' +# +# - name: Run spindle-serial-ubuntu crash tests +# run: | +# docker exec spindlenode bash -c 'cd Spindle-build/testsuite && SPINDLE_DEBUG=3 ./run_crash_tests.sh --launcher=serial --scratch=/tmp/spindle_crash_test' + + - name: Collect Spindle logs from container + if: ${{ always() }} + continue-on-error: true + run: | + mkdir -p spindle-logs + docker cp spindlenode:/Spindle-build/testsuite/. spindle-logs/ || echo "Log collection failed" + ls -la spindle-logs/ + + - name: Upload Spindle logs + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: spindle-logs-serial-x86 + path: spindle-logs/spindle_output.* + retention-days: 7 + if-no-files-found: warn + + - name: Bring spindle-serial-ubuntu down + id: serial-ubuntu-down + if: ${{ always() }} + continue-on-error: true + run: | + cd containers/spindle-serial-ubuntu + docker compose down diff --git a/.github/workflows/ci-slurm-artifacts.yml b/.github/workflows/ci-slurm-artifacts.yml new file mode 100644 index 00000000..bc96ac16 --- /dev/null +++ b/.github/workflows/ci-slurm-artifacts.yml @@ -0,0 +1,93 @@ +name: ci-slurm-artifacts + +on: + workflow_dispatch: + +permissions: + contents: read + packages: read + +jobs: + spindle-slurm-ubuntu: + name: Testsuite (Slurm rshlaunch, Ubuntu) + environment: Spindle CI + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out Spindle + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + + - name: Setup Docker Compose + uses: docker/setup-compose-action@16feee727cbdc83b6a014e6cc26fec4a79bcf30c + with: + version: latest + + - name: Login to GitHub Container Registry + if: ${{ !env.ACT }} + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Generate MariaDB configuration + id: slurm-ubuntu-mariadb + run: | + cd containers/spindle-slurm-ubuntu/testing + ./generate_config.sh + + - name: Build spindle-slurm-ubuntu image + id: slurm-ubuntu-build + run: | + cd containers/spindle-slurm-ubuntu/testing + docker compose --progress=plain build + + - name: Bring spindle-slurm-ubuntu up + id: slurm-ubuntu-up + run: | + cd containers/spindle-slurm-ubuntu/testing + docker compose up -d --wait --wait-timeout 120 + + - name: Verify munge works in spindle-slurm-ubuntu + id: slurm-ubuntu-munge + run: | + docker exec slurm-head bash -c 'munge -n | unmunge' + + - name: Run spindle-slurm-ubuntu testsuite + id: slurm-ubuntu-testsuite + run: | + docker exec slurm-head bash -c 'cd Spindle-build/testsuite && salloc -n${workers} -N${workers} SPINDLE_DEBUG=3 ./runTests ${workers}' + +# - name: Configure host core_pattern for coredumps +# if: ${{ !env.ACT }} +# run: | +# sudo sysctl -w kernel.core_pattern='core.%p' +# +# - name: Run spindle-slurm-ubuntu crash tests +# run: | +# docker exec slurm-head bash -c 'cd Spindle-build/testsuite && SPINDLE_DEBUG=3 ./run_crash_tests.sh --launcher=slurm --nodes=${workers} --scratch=/shared/spindle_crash_test' + + - name: Collect Spindle logs from container + if: ${{ always() }} + continue-on-error: true + run: | + mkdir -p spindle-logs + docker cp slurm-head:/Spindle-build/testsuite/. spindle-logs/ || echo "Log collection failed" + ls -la spindle-logs/ + + - name: Upload Spindle logs + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: spindle-logs-slurm + path: spindle-logs/spindle_output.* + retention-days: 7 + if-no-files-found: warn + + - name: Bring spindle-slurm-ubuntu down + id: slurm-ubuntu-down + if: ${{ always() }} + continue-on-error: true + run: | + cd containers/spindle-slurm-ubuntu/testing + docker compose down diff --git a/.github/workflows/ci-slurm-plugin-artifacts.yml b/.github/workflows/ci-slurm-plugin-artifacts.yml new file mode 100644 index 00000000..0f38a480 --- /dev/null +++ b/.github/workflows/ci-slurm-plugin-artifacts.yml @@ -0,0 +1,98 @@ +name: ci-slurm-plugin-artifacts + +on: + workflow_dispatch: + +permissions: + contents: read + packages: read + +jobs: + spindle-slurm-plugin-ubuntu: + name: Testsuite (Slurm Plugin, Ubuntu) + environment: Spindle CI + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out Spindle + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + + - name: Setup Docker Compose + uses: docker/setup-compose-action@16feee727cbdc83b6a014e6cc26fec4a79bcf30c + with: + version: latest + + - name: Login to GitHub Container Registry + if: ${{ !env.ACT }} + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Generate MariaDB configuration + id: slurm-ubuntu-mariadb + run: | + cd containers/spindle-slurm-ubuntu/testing-plugin + ./generate_config.sh + + - name: Build spindle-slurm-plugin-ubuntu image + id: slurm-ubuntu-build + run: | + cd containers/spindle-slurm-ubuntu/testing-plugin + docker compose --progress=plain build + + - name: Bring spindle-slurm-plugin-ubuntu up + id: slurm-ubuntu-up + run: | + cd containers/spindle-slurm-ubuntu/testing-plugin + docker compose up -d --wait --wait-timeout 120 + + - name: Verify munge works in spindle-slurm-plugin-ubuntu + id: slurm-ubuntu-munge + run: | + docker exec slurm-plugin-head bash -c 'munge -n | unmunge' + + - name: Run spindle-slurm-plugin-ubuntu testsuite + id: slurm-ubuntu-testsuite + run: | + docker exec slurm-plugin-head bash -c 'cd Spindle-build/testsuite && salloc -n${workers} -N${workers} SPINDLE_DEBUG=3 ./runTests ${workers}' + + - name: Run spindle-slurm-plugin-ubuntu session testsuite + id: slurm-ubuntu-testsuite-sessions + run: | + docker exec slurm-plugin-head bash -c 'cd Spindle-build/testsuite && salloc -n${workers} -N${workers} --spindle-session SPINDLE_DEBUG=3 ./runTests ${workers}' + +# - name: Configure host core_pattern for coredumps +# if: ${{ !env.ACT }} +# run: | +# sudo sysctl -w kernel.core_pattern='core.%p' +# +# - name: Run spindle-slurm-plugin-ubuntu crash tests +# run: | +# docker exec slurm-plugin-head bash -c 'cd Spindle-build/testsuite && SPINDLE_DEBUG=3 ./run_crash_tests.sh --launcher=slurm-plugin --nodes=${workers} --scratch=/shared/spindle_crash_test' + + - name: Collect Spindle logs from container + if: ${{ always() }} + continue-on-error: true + run: | + mkdir -p spindle-logs + docker cp slurm-plugin-head:/Spindle-build/testsuite/. spindle-logs/ || echo "Log collection failed" + ls -la spindle-logs/ + + - name: Upload Spindle logs + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: spindle-logs-slurm-plugin + path: spindle-logs/spindle_output.* + retention-days: 7 + if-no-files-found: warn + + - name: Bring spindle-slurm-plugin-ubuntu down + id: slurm-ubuntu-down + if: ${{ always() }} + continue-on-error: true + run: | + cd containers/spindle-slurm-ubuntu/testing-plugin + docker compose down diff --git a/.github/workflows/ci-slurm-srun-artifacts.yml b/.github/workflows/ci-slurm-srun-artifacts.yml new file mode 100644 index 00000000..068fb06c --- /dev/null +++ b/.github/workflows/ci-slurm-srun-artifacts.yml @@ -0,0 +1,84 @@ +name: ci-slurm-srun-artifacts + +on: + workflow_dispatch: + +permissions: + contents: read + packages: read + +jobs: + spindle-slurm-srun-ubuntu: + name: Testsuite (Slurm srun, Ubuntu) + environment: Spindle CI + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out Spindle + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + + - name: Setup Docker Compose + uses: docker/setup-compose-action@16feee727cbdc83b6a014e6cc26fec4a79bcf30c + with: + version: latest + + - name: Login to GitHub Container Registry + if: ${{ !env.ACT }} + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Generate MariaDB configuration + id: slurm-ubuntu-mariadb + run: | + cd containers/spindle-slurm-ubuntu/testing-srun + ./generate_config.sh + + - name: Build spindle-slurm-ubuntu image + id: slurm-ubuntu-build + run: | + cd containers/spindle-slurm-ubuntu/testing-srun + docker compose --progress=plain build + + - name: Bring spindle-slurm-ubuntu up + id: slurm-ubuntu-up + run: | + cd containers/spindle-slurm-ubuntu/testing-srun + docker compose up -d --wait --wait-timeout 120 + + - name: Verify munge works in spindle-slurm-srun-ubuntu + id: slurm-ubuntu-munge + run: | + docker exec slurm-srun-head bash -c 'munge -n | unmunge' + + - name: Run spindle-slurm-srun-ubuntu testsuite + id: slurm-ubuntu-testsuite + run: | + docker exec slurm-srun-head bash -c 'cd Spindle-build/testsuite && salloc -n${workers} -N${workers} SPINDLE_DEBUG=3 ./runTests ${workers}' + + - name: Collect Spindle logs from container + if: ${{ always() }} + continue-on-error: true + run: | + mkdir -p spindle-logs + docker cp slurm-srun-head:/Spindle-build/testsuite/. spindle-logs/ || echo "Log collection failed" + ls -la spindle-logs/ + + - name: Upload Spindle logs + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: spindle-logs-slurm-srun + path: spindle-logs/spindle_output.* + retention-days: 7 + if-no-files-found: warn + + - name: Bring spindle-slurm-ubuntu down + id: slurm-ubuntu-down + if: ${{ always() }} + continue-on-error: true + run: | + cd containers/spindle-slurm-ubuntu/testing-srun + docker compose down diff --git a/config.h.in b/config.h.in index 121c551f..54fa292b 100644 --- a/config.h.in +++ b/config.h.in @@ -9,8 +9,9 @@ /* Colon-separated list of potential back-end cache directories */ #undef CACHEPATHS -/* Back-end directory for communication and housekeeping */ -#undef COMMPATH +/* Colon-separated list of candidate paths for back-end communication and + housekeeping */ +#undef COMMPATHS /* Define if were using biter for client/server communication */ #undef COMM_BITER diff --git a/configure b/configure index ac8d1434..1503b981 100755 --- a/configure +++ b/configure @@ -850,7 +850,7 @@ with_default_num_ports with_localstorage with_cachepaths with_cachepath -with_commpath +with_commpaths with_default_local_prefix with_testrm with_rm @@ -1598,8 +1598,8 @@ Optional Packages: --with-cachepaths=DIR Colon-separated list of potential back-end cache directories , - --with-compath=DIR Back-end directory for communication and - housekeeping + --with-compaths=DIR Colon-separated list of candidate paths for back-end + communication and housekeeping --with-default-local-prefix=DIRS Colon-seperated list of directories that Spindle will not cache files out of @@ -16691,11 +16691,11 @@ if test "${with_cachepath+set}" = set; then : fi -# Check whether --with-commpath was given. -if test "${with_commpath+set}" = set; then : - withval=$with_commpath; COMMPATH=${withval} +# Check whether --with-commpaths was given. +if test "${with_commpaths+set}" = set; then : + withval=$with_commpaths; COMMPATHS=${withval} else - COMMPATH=$DEFAULT_LOC + COMMPATHS=$DEFAULT_LOC fi @@ -16703,7 +16703,7 @@ fi if test "${with_default_local_prefix+set}" = set; then : withval=$with_default_local_prefix; SPINDLE_LOCAL_PREFIX=${withval} else - SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH" + SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATHS" fi @@ -16723,7 +16723,7 @@ _ACEOF cat >>confdefs.h <<_ACEOF -#define COMMPATH "$COMMPATH" +#define COMMPATHS "$COMMPATHS" _ACEOF diff --git a/configure.common.ac b/configure.common.ac index baa9c0a9..010b050f 100644 --- a/configure.common.ac +++ b/configure.common.ac @@ -29,18 +29,18 @@ AC_ARG_WITH(cachepath, [[],[]], [AC_MSG_ERROR(use --with-cachepaths=DIRS (plural) instead of --with-cachepath=DIR to specify one or more cache paths)], []) -AC_ARG_WITH(commpath, - [AS_HELP_STRING([--with-compath=DIR],[Back-end directory for communication and housekeeping])], - [COMMPATH=${withval}], - [COMMPATH=$DEFAULT_LOC]) +AC_ARG_WITH(commpaths, + [AS_HELP_STRING([--with-compaths=DIR],[Colon-separated list of candidate paths for back-end communication and housekeeping])], + [COMMPATHS=${withval}], + [COMMPATHS=$DEFAULT_LOC]) AC_ARG_WITH(default-local-prefix, [AS_HELP_STRING([--with-default-local-prefix=DIRS],[Colon-seperated list of directories that Spindle will not cache files out of])], [SPINDLE_LOCAL_PREFIX=${withval}], - [SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH"]) + [SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATHS"]) AC_DEFINE_UNQUOTED([SPINDLE_PORT],[$SPINDLE_PORT],[The default port for Spindle]) AC_DEFINE_UNQUOTED([NUM_COBO_PORTS],[$NUM_COBO_PORTS],[Number of ports for COBO to search for an open port]) AC_DEFINE_UNQUOTED([SPINDLE_MAX_PORT],[$(($SPINDLE_PORT + $NUM_COBO_PORTS - 1))],[The maximum port value]) -AC_DEFINE_UNQUOTED([COMMPATH],"[$COMMPATH]",[Back-end directory for communication and housekeeping]) +AC_DEFINE_UNQUOTED([COMMPATHS],"[$COMMPATHS]",[Colon-separated list of candidate paths for back-end communication and housekeeping]) AC_DEFINE_UNQUOTED([CACHEPATHS],"[$CACHEPATHS]",[Colon-separated list of potential back-end cache directories]) AC_DEFINE_UNQUOTED([SPINDLE_LOCAL_PREFIX],"[$SPINDLE_LOCAL_PREFIX]",[The default colon-separated list of directories that Spindle will not cache files out of]) diff --git a/containers/spindle-flux-ubuntu/scripts/build_spindle.sh b/containers/spindle-flux-ubuntu/scripts/build_spindle.sh index a63b5d8d..454bca88 100755 --- a/containers/spindle-flux-ubuntu/scripts/build_spindle.sh +++ b/containers/spindle-flux-ubuntu/scripts/build_spindle.sh @@ -4,7 +4,7 @@ set -euxo pipefail mkdir -p /home/${USER}/Spindle-build cd /home/${USER}/Spindle-build -/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=flux --enable-flux-plugin --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" +/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=flux --enable-flux-plugin --with-cachepaths=/bad_cachepath:/tmp/commpath/cachepath --with-commpaths=/bad_commpath:/tmp/commpaths CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" make -j$(nproc) make install diff --git a/containers/spindle-serial-ubuntu/scripts/build_spindle.sh b/containers/spindle-serial-ubuntu/scripts/build_spindle.sh index 1022ba14..c18c202d 100755 --- a/containers/spindle-serial-ubuntu/scripts/build_spindle.sh +++ b/containers/spindle-serial-ubuntu/scripts/build_spindle.sh @@ -4,7 +4,7 @@ set -euxo pipefail mkdir -p /home/${USER}/Spindle-build cd /home/${USER}/Spindle-build -/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=serial --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" +/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=serial --with-cachepaths=/bad_cachepath:/tmp/commpath/cachepath --with-commpaths=/bad_commpath:/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" make -j$(nproc) make install diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh index 2b36be90..c214630d 100755 --- a/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh +++ b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh @@ -3,7 +3,7 @@ set -euxo pipefail mkdir -p /home/${USER}/Spindle-build cd /home/${USER}/Spindle-build -/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm-plugin --enable-slurm-plugin --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" +/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm-plugin --enable-slurm-plugin --with-cachepaths=/bad_cachepath:/tmp/commpath/cachepath --with-commpaths=/bad_commpath:/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" make -j$(nproc) make install diff --git a/containers/spindle-slurm-ubuntu/testing-srun/scripts/build_spindle.sh b/containers/spindle-slurm-ubuntu/testing-srun/scripts/build_spindle.sh index 125fe2eb..165de7c9 100755 --- a/containers/spindle-slurm-ubuntu/testing-srun/scripts/build_spindle.sh +++ b/containers/spindle-slurm-ubuntu/testing-srun/scripts/build_spindle.sh @@ -3,7 +3,7 @@ set -euxo pipefail mkdir -p /home/${USER}/Spindle-build cd /home/${USER}/Spindle-build -/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" +/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm --with-cachepaths=/bad_cachepath:/tmp/commpath/cachepath --with-commpaths=/bad_commpath:/tmp/commpath:/bar CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" make -j$(nproc) make install diff --git a/containers/spindle-slurm-ubuntu/testing/scripts/build_spindle.sh b/containers/spindle-slurm-ubuntu/testing/scripts/build_spindle.sh index 7fcb48e3..7cdf58dd 100755 --- a/containers/spindle-slurm-ubuntu/testing/scripts/build_spindle.sh +++ b/containers/spindle-slurm-ubuntu/testing/scripts/build_spindle.sh @@ -3,7 +3,7 @@ set -euxo pipefail mkdir -p /home/${USER}/Spindle-build cd /home/${USER}/Spindle-build -/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm --with-rsh-launch --with-rsh-cmd=/usr/bin/ssh --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" +/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm --with-rsh-launch --with-rsh-cmd=/usr/bin/ssh --with-cachepaths=/bad_cachepath:/tmp/commpath/cachepath --with-commpaths=/bad_commpath:/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" make -j$(nproc) make install diff --git a/src/client/beboot/spindle_bootstrap.c b/src/client/beboot/spindle_bootstrap.c index e769fe4c..b814c5e2 100644 --- a/src/client/beboot/spindle_bootstrap.c +++ b/src/client/beboot/spindle_bootstrap.c @@ -33,6 +33,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "client_api.h" #include "exec_util.h" #include "shmcache.h" +#include "parseloc.h" #include "config.h" @@ -53,7 +54,7 @@ static int rankinfo[4]={-1,-1,-1,-1}; number_t number; static int use_cache; static unsigned int cachesize; -static char *commpath, *number_s, *symbolic_commpath; +static char *commpath, *number_s, *commpaths; static char **cmdline; static char *executable; static char *client_lib; @@ -168,7 +169,7 @@ static int parse_cmdline(int argc, char *argv[]) daemon_args[i - 3] = NULL; } - symbolic_commpath = argv[i++]; + commpaths = argv[i++]; i++; // Skip over candidate_cachepaths. number_s = argv[i++]; number = (number_t) strtoul(number_s, NULL, 0); @@ -182,7 +183,7 @@ static int parse_cmdline(int argc, char *argv[]) return 0; } -static void launch_daemon(char *commpath) +static void launch_daemon( void ) { /*grand-child fork, then execv daemon. By grand-child forking we ensure that the app won't get confused by seeing an unknown process as a child. */ @@ -192,11 +193,6 @@ static void launch_daemon(char *commpath) char unique_file[MAX_PATH_LEN+1]; char buffer[32]; - result = spindle_mkdir(commpath); - if (result == -1) { - debug_printf("Exiting due to spindle_mkdir error\n"); - exit(-1); - } snprintf(unique_file, MAX_PATH_LEN, "%s/spindle_daemon_pid", commpath); unique_file[MAX_PATH_LEN] = '\0'; fd = open(unique_file, O_CREAT | O_EXCL | O_WRONLY, 0600); @@ -352,14 +348,12 @@ int main(int argc, char *argv[]) } } - char *orig_commpath = parse_location(symbolic_commpath, number); - if (!orig_commpath) { - return -1; + if( -1 == getFirstValidPath( commpaths, &commpath, number ) ){ + return -1; } - commpath = realize(orig_commpath); if (daemon_args) { - launch_daemon(commpath); + launch_daemon(); } result = establish_connection(); diff --git a/src/client/client_comlib/client_api.c b/src/client/client_comlib/client_api.c index cec31051..07735226 100644 --- a/src/client/client_comlib/client_api.c +++ b/src/client/client_comlib/client_api.c @@ -40,42 +40,31 @@ static struct lock_t comm_lock; int send_cachepath_query( int fd, char **chosen_realized_cachepath, char **chosen_parsed_cachepath){ - int retries = 0, max_retries = 1000, rc = 0; - struct timespec delay_between_retries = { .tv_sec = 0, .tv_nsec = 1000000 }; + int rc = 0; ldcs_message_t message; - char buffer[MAX_PATH_LEN+1]; + char buffer[2*(MAX_PATH_LEN+1)]; buffer[MAX_PATH_LEN] = '\0'; - do{ - message.header.type = LDCS_MSG_CHOSEN_CACHEPATH_REQUEST; - message.header.len = 0; - message.data = buffer; - - COMM_LOCK; - - debug_printf3("sending message of type: CHOSEN_CACHEPATH_REQUEST.\n" ); - rc = client_send_msg(fd, &message); - if( rc != 0 ){ - return rc; - } - rc = client_recv_msg_static(fd, &message, LDCS_READ_BLOCK); - if( rc != 0 ){ - return rc; - } - - COMM_UNLOCK; - - if( message.header.type == LDCS_MSG_NO_CACHEPATH_CONSENSUS_YET ){ - if( retries++ >= max_retries ){ - break; - } - nanosleep( &delay_between_retries, NULL ); - continue; - } - break; + message.header.type = LDCS_MSG_CHOSEN_CACHEPATH_REQUEST; + message.header.len = 0; + message.data = buffer; + + debug_printf3("sending message of type: CHOSEN_CACHEPATH_REQUEST.\n" ); + COMM_LOCK; - }while( 1 ); + rc = client_send_msg(fd, &message); + if( rc != 0 ){ + COMM_UNLOCK; + return rc; + } + rc = client_recv_msg_static(fd, &message, LDCS_READ_BLOCK); + if( rc != 0 ){ + COMM_UNLOCK; + return rc; + } + + COMM_UNLOCK; if (message.header.type != LDCS_MSG_CHOSEN_CACHEPATH || message.header.len > MAX_PATH_LEN) { err_printf("Got unexpected message of type %d\n", (int) message.header.type); diff --git a/src/client/config.h.in b/src/client/config.h.in index 044ca9e1..2869ccd7 100644 --- a/src/client/config.h.in +++ b/src/client/config.h.in @@ -9,8 +9,9 @@ /* Colon-separated list of potential back-end cache directories */ #undef CACHEPATHS -/* Back-end directory for communication and housekeeping */ -#undef COMMPATH +/* Colon-separated list of candidate paths for back-end communication and + housekeeping */ +#undef COMMPATHS /* Define if were using biter for client/server communication */ #undef COMM_BITER diff --git a/src/client/configure b/src/client/configure index eb84ee07..7a59cba7 100755 --- a/src/client/configure +++ b/src/client/configure @@ -813,7 +813,7 @@ with_default_num_ports with_localstorage with_cachepaths with_cachepath -with_commpath +with_commpaths with_default_local_prefix with_testrm with_rm @@ -1540,8 +1540,8 @@ Optional Packages: --with-cachepaths=DIR Colon-separated list of potential back-end cache directories , - --with-compath=DIR Back-end directory for communication and - housekeeping + --with-compaths=DIR Colon-separated list of candidate paths for back-end + communication and housekeeping --with-default-local-prefix=DIRS Colon-seperated list of directories that Spindle will not cache files out of @@ -12616,11 +12616,11 @@ if test "${with_cachepath+set}" = set; then : fi -# Check whether --with-commpath was given. -if test "${with_commpath+set}" = set; then : - withval=$with_commpath; COMMPATH=${withval} +# Check whether --with-commpaths was given. +if test "${with_commpaths+set}" = set; then : + withval=$with_commpaths; COMMPATHS=${withval} else - COMMPATH=$DEFAULT_LOC + COMMPATHS=$DEFAULT_LOC fi @@ -12628,7 +12628,7 @@ fi if test "${with_default_local_prefix+set}" = set; then : withval=$with_default_local_prefix; SPINDLE_LOCAL_PREFIX=${withval} else - SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH" + SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATHS" fi @@ -12648,7 +12648,7 @@ _ACEOF cat >>confdefs.h <<_ACEOF -#define COMMPATH "$COMMPATH" +#define COMMPATHS "$COMMPATHS" _ACEOF diff --git a/src/fe/config.h.in b/src/fe/config.h.in index 615997be..2ab5066c 100644 --- a/src/fe/config.h.in +++ b/src/fe/config.h.in @@ -9,8 +9,9 @@ /* Colon-separated list of potential back-end cache directories */ #undef CACHEPATHS -/* Back-end directory for communication and housekeeping */ -#undef COMMPATH +/* Colon-separated list of candidate paths for back-end communication and + housekeeping */ +#undef COMMPATHS /* Define if were using biter for client/server communication */ #undef COMM_BITER diff --git a/src/fe/configure b/src/fe/configure index 4c11bdc7..43bec26e 100755 --- a/src/fe/configure +++ b/src/fe/configure @@ -834,7 +834,7 @@ with_default_num_ports with_localstorage with_cachepaths with_cachepath -with_commpath +with_commpaths with_default_local_prefix with_testrm with_rm @@ -1578,8 +1578,8 @@ Optional Packages: --with-cachepaths=DIR Colon-separated list of potential back-end cache directories , - --with-compath=DIR Back-end directory for communication and - housekeeping + --with-compaths=DIR Colon-separated list of candidate paths for back-end + communication and housekeeping --with-default-local-prefix=DIRS Colon-seperated list of directories that Spindle will not cache files out of @@ -16466,11 +16466,11 @@ if test "${with_cachepath+set}" = set; then : fi -# Check whether --with-commpath was given. -if test "${with_commpath+set}" = set; then : - withval=$with_commpath; COMMPATH=${withval} +# Check whether --with-commpaths was given. +if test "${with_commpaths+set}" = set; then : + withval=$with_commpaths; COMMPATHS=${withval} else - COMMPATH=$DEFAULT_LOC + COMMPATHS=$DEFAULT_LOC fi @@ -16478,7 +16478,7 @@ fi if test "${with_default_local_prefix+set}" = set; then : withval=$with_default_local_prefix; SPINDLE_LOCAL_PREFIX=${withval} else - SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH" + SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATHS" fi @@ -16498,7 +16498,7 @@ _ACEOF cat >>confdefs.h <<_ACEOF -#define COMMPATH "$COMMPATH" +#define COMMPATHS "$COMMPATHS" _ACEOF diff --git a/src/fe/startup/config_mgr.cc b/src/fe/startup/config_mgr.cc index 1b9d9ade..02235648 100644 --- a/src/fe/startup/config_mgr.cc +++ b/src/fe/startup/config_mgr.cc @@ -50,10 +50,10 @@ using namespace std; #define SPINDLE_NUM_PORTS_STR "250" #endif -#if defined(COMMPATH) -#define SPINDLE_COMMPATH_STR COMMPATH +#if defined(COMMPATHS) +#define SPINDLE_COMMPATHS_STR COMMPATHS #else -#define SPINDLE_COMMPATH_STR "$TMPDIR" +#define SPINDLE_COMMPATHS_STR "$TMPDIR" #endif #if defined(CACHEPATHS) @@ -273,7 +273,7 @@ void initOptionsList() "Provides a text file containing a white-space separated list of files that should be relocated to each node before execution begins" }, { confStrip, "strip", shortStrip, groupMisc, cvBool, {}, "true", "Strip debug and symbol information from binaries before distributing them." }, - { confCommPath, "commpath", shortCommPath, groupMisc, cvString, {}, SPINDLE_COMMPATH_STR, + { confCommPaths, "commpaths", shortCommPaths, groupMisc, cvString, {}, SPINDLE_COMMPATHS_STR, "Back-end directory communication and housekeeping. Should be a non-shared location such as a ramdisk." }, { confCachePaths, "cachepaths", shortCachePaths, groupMisc, cvString, {}, SPINDLE_CACHEPATHS_STR, "Colon-separated list of candidate paths for cached libraries."}, @@ -743,11 +743,8 @@ bool ConfigMap::toSpindleArgs(spindle_args_t &args, bool alloc_strs) const case confNumPorts: args.num_ports = numresult; break; - case confCommPath: { - string path = strresult + "/spindle.$NUMBER"; - args.commpath = strdup(path.c_str()); - break; - } + case confCommPaths: + __attribute__((fallthrough)); // gcc-specific case confCachePaths:{ // Paramemter values are colon-separated lists of paths. // Append "/spindle.$NUMBER" to each path in the list. @@ -760,10 +757,15 @@ bool ConfigMap::toSpindleArgs(spindle_args_t &args, bool alloc_strs) const idx = paths.find(":", idx + number_var_with_colon.size()); }; paths += number_var_without_colon; - args.candidate_cachepaths = strdup(paths.c_str()); + if( name == confCommPaths ){ + args.commpaths = strdup(paths.c_str()); + }else if( name == confCachePaths ){ + args.candidate_cachepaths = strdup(paths.c_str()); + } break; } case confCachePrefix: + __attribute__((fallthrough)); // gcc-specific case confPythonPrefix: if (args.pythonprefix) args.pythonprefix = getstr(string(args.pythonprefix) + string(":") + strresult, true); diff --git a/src/fe/startup/config_mgr.h b/src/fe/startup/config_mgr.h index ec3c8135..f679a5d5 100644 --- a/src/fe/startup/config_mgr.h +++ b/src/fe/startup/config_mgr.h @@ -29,7 +29,7 @@ enum SpindleConfigID { confCmdlineNewgroup, confPort, confNumPorts, - confCommPath, + confCommPaths, confCachePaths, confCachePrefix, confPythonPrefix, @@ -83,7 +83,7 @@ enum CmdlineShortOptions { shortAuditType = 'k', shortRelocSO = 'l', shortNoClean = 'n', - shortCommPath = 'o', + shortCommPaths = 'o', shortPush = 'p', shortPull = 'q', shortPythonPrefix = 'r', diff --git a/src/fe/startup/parse_launcher.cc b/src/fe/startup/parse_launcher.cc index 4484e998..e8226b73 100644 --- a/src/fe/startup/parse_launcher.cc +++ b/src/fe/startup/parse_launcher.cc @@ -26,7 +26,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include #include #include - +#include "parseloc.h" #include "spindle_launch.h" #include "spindle_debug.h" #include "config.h" @@ -292,6 +292,7 @@ void ModifyArgv::modifyCmdLine() snprintf(options_str, 32, "%lu", (unsigned long) params->opts); string options(options_str); + getFirstValidPath( params->commpaths, &( params->commpath ), params->number ); string commpath(params->commpath); char number_str[32]; diff --git a/src/fe/startup/spindle_fe.cc b/src/fe/startup/spindle_fe.cc index a038e201..9e5b7577 100644 --- a/src/fe/startup/spindle_fe.cc +++ b/src/fe/startup/spindle_fe.cc @@ -41,7 +41,6 @@ static const char *logging_file = NULL; #endif static const char spindle_bootstrap[] = LIBEXECDIR "/spindle_bootstrap"; static bool sendAndWaitForAlive(); -static void determineCachepathConsensus(); #define STARTUP_TIMEOUT 60 @@ -71,7 +70,7 @@ static int pack_data(spindle_args_t *args, void* &buffer, unsigned &buffer_size) buffer_size += sizeof(number_t); buffer_size += sizeof(opt_t); buffer_size += sizeof(unique_id_t); - buffer_size += args->commpath ? strlen(args->commpath) + 1 : 1; + buffer_size += args->commpaths ? strlen(args->commpaths) + 1 : 1; buffer_size += args->candidate_cachepaths ? strlen(args->candidate_cachepaths) + 1 : 1; buffer_size += args->pythonprefix ? strlen(args->pythonprefix) + 1 : 1; buffer_size += args->preloadfile ? strlen(args->preloadfile) + 1 : 1; @@ -92,7 +91,7 @@ static int pack_data(spindle_args_t *args, void* &buffer, unsigned &buffer_size) pack_param(args->use_launcher, buf, pos); pack_param(args->startup_type, buf, pos); pack_param(args->shm_cache_size, buf, pos); - pack_param(args->commpath, buf, pos); + pack_param(args->commpaths, buf, pos); pack_param(args->candidate_cachepaths, buf, pos); pack_param(args->pythonprefix, buf, pos); pack_param(args->preloadfile, buf, pos); @@ -232,7 +231,7 @@ int getApplicationArgsFE(spindle_args_t *params, int *spindle_argc, char ***spin (*spindle_argv)[n++] = strdup(numports_s); (*spindle_argv)[n++] = strdup(uniqueid_s); } - (*spindle_argv)[n++] = strdup(params->commpath); + (*spindle_argv)[n++] = strdup(params->commpaths); (*spindle_argv)[n++] = strdup(params->candidate_cachepaths); (*spindle_argv)[n++] = strdup(number_s); (*spindle_argv)[n++] = strdup(opt_s); @@ -398,11 +397,11 @@ int spindleInitFE(const char **hosts, spindle_args_t *params) /* Start FE server */ debug_printf("spindle_args_t { number = %lu; port = %u; num_ports = %u; opts = %lu; unique_id = %lu; " - "use_launcher = %u; startup_type = %u; shm_cache_size = %u; commpath = %s; " + "use_launcher = %u; startup_type = %u; shm_cache_size = %u; commpaths = %s; " "cachepaths = %s; " "pythonprefix = %s; preloadfile = %s; bundle_timeout_ms = %u; bundle_cachesize_kb = %u }\n", (unsigned long) params->number, params->port, params->num_ports, params->opts, params->unique_id, - params->use_launcher, params->startup_type, params->shm_cache_size, params->commpath, + params->use_launcher, params->startup_type, params->shm_cache_size, params->commpaths, params->candidate_cachepaths, params->pythonprefix, params->preloadfile, params->bundle_timeout_ms, params->bundle_cachesize_kb); @@ -433,7 +432,6 @@ int spindleInitFE(const char **hosts, spindle_args_t *params) /* Wait for servers to indicate startup */ sendAndWaitForAlive(); - determineCachepathConsensus(); return 0; } @@ -490,17 +488,6 @@ void markRSHPidReapedFE() clear_fe_rsh_pid(); } -static void determineCachepathConsensus( void ){ - ldcs_message_t consensus_req_msg; - consensus_req_msg.header.type = LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS; - consensus_req_msg.header.len = 0; - consensus_req_msg.data = NULL; - int result = ldcs_audit_server_fe_broadcast(&consensus_req_msg, NULL); - if (result == -1) { - debug_printf("Failure sending cachepath consensus message\n"); - } -} - static bool sendAndWaitForAlive() { int result; diff --git a/src/flux/flux-spindle.c b/src/flux/flux-spindle.c index dd00aa32..9517e523 100644 --- a/src/flux/flux-spindle.c +++ b/src/flux/flux-spindle.c @@ -381,7 +381,7 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx) int numa = 0; const char *relocaout = NULL, *reloclibs = NULL, *relocexec = NULL, *relocpython = NULL; const char *followfork = NULL, *preload = NULL, *level = NULL; - const char *pyprefix = NULL, *commpath = NULL; + const char *pyprefix = NULL, *commpaths = NULL; char *numafiles = NULL, *cachepaths = NULL; if (flux_shell_getopt_unpack (shell, "spindle", "o", &opts) < 0) @@ -415,7 +415,7 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx) "reloc-exec", &relocexec, "reloc-python", &relocpython, "python-prefix", &pyprefix, - "commpath", &commpath, + "commpaths", &commpaths, "numa", &numa, "numa-files", &numafiles, "preload", &preload, @@ -466,8 +466,8 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx) if( cachepaths ){ ctx->params.candidate_cachepaths = cachepaths; } - if (commpath) { - ctx->params.commpath = (char *) commpath; + if (commpaths) { + ctx->params.commpaths = (char *) commpaths; } if (level) { if (strcmp(level, "high") == 0) { diff --git a/src/flux/sessionmgr.c b/src/flux/sessionmgr.c index 17027163..a936c46b 100644 --- a/src/flux/sessionmgr.c +++ b/src/flux/sessionmgr.c @@ -106,27 +106,14 @@ char **strip_start_from_argv(int argc, char **argv) return new_argv; } -extern char *parse_location(char *loc, int number); -extern int spindle_mkdir(char *orig_path); - -#if !defined(COMMPATH) -#error COMMPATH must be defined in config.h -#endif +#define SESSIONPATH "/tmp/spindle/session" const char *get_session_dir() { - int result; - char *dir; - dir = parse_location((char *) (COMMPATH "/spindle_session"), 0); - if (!dir) { - spindle_debug_printf(1, "ERROR: Could not parse directory for spindle session location from %s/spindle_session\n", COMMPATH); - return NULL; - } - - result = spindle_mkdir(dir); - if (result == -1) { - spindle_debug_printf(1, "ERROR: Could not mkdir spindle session location at %s\n", dir); - free(dir); - return NULL; - } - return dir; + /* This function was originally designed to return a realized version of COMMPATH. + * Since then, COMMPATH has been replaced by COMMPATHS, which can be modified by + * the user on a per-job basis and uses the number_t number as part of the path + * name. Sessions are designed to span multiple jobs, and thus shouldn't be + * piggybacking on COMMPATHS. For now, we'll put in a hardcoded directory. + */ + return strdup(SESSIONPATH); } diff --git a/src/include/ldcs_api.h b/src/include/ldcs_api.h index 0bcd8f40..2b261a56 100644 --- a/src/include/ldcs_api.h +++ b/src/include/ldcs_api.h @@ -85,10 +85,8 @@ typedef enum { LDCS_MSG_PICKONE_RESP, LDCS_MSG_ALIVE_REQ, LDCS_MSG_ALIVE_RESP, - LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS, LDCS_MSG_CHOSEN_CACHEPATH_REQUEST, LDCS_MSG_CHOSEN_CACHEPATH, - LDCS_MSG_NO_CACHEPATH_CONSENSUS_YET, LDCS_MSG_UNKNOWN } ldcs_message_ids_t; diff --git a/src/include/spindle_launch.h b/src/include/spindle_launch.h index ca7b8d3d..ac256a39 100644 --- a/src/include/spindle_launch.h +++ b/src/include/spindle_launch.h @@ -126,6 +126,9 @@ typedef struct { /* The local-disk location for communication and housekeeping. */ char *commpath; + /* The initial colon-separated list of commpath candidates. */ + char *commpaths; + /* Path[s] for cached libraries. */ char *candidate_cachepaths; /* Colon-separated list of candidate paths (max 64) */ diff --git a/src/server/auditserver/ldcs_audit_server_handlers.c b/src/server/auditserver/ldcs_audit_server_handlers.c index 1afdf2e8..6b3a6b1c 100644 --- a/src/server/auditserver/ldcs_audit_server_handlers.c +++ b/src/server/auditserver/ldcs_audit_server_handlers.c @@ -181,7 +181,6 @@ static int handle_setup_alias(ldcs_process_data_t *procdata, char *pathname, cha static int handle_client_dirlists_req(ldcs_process_data_t *procdata, int nc); static int handle_close_client_query(ldcs_process_data_t *procdata, int nc); static int handle_alive_msg(ldcs_process_data_t *procdata, ldcs_message_t *msg); -static int handle_cachepath_consensus(ldcs_process_data_t *procdata, ldcs_message_t *msg); static int handle_chosen_cachepath_request(ldcs_process_data_t *procdata, int nc); extern void getValidCachePathByIndex( uint64_t validBitIdx, char **realizedCachePath, char **parsedCachePath, char **symbolicCachePath ); @@ -1997,8 +1996,6 @@ int handle_server_message(ldcs_process_data_t *procdata, node_peer_t peer, ldcs_ case LDCS_MSG_ALIVE_REQ: case LDCS_MSG_ALIVE_RESP: return handle_alive_msg(procdata, msg); - case LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS: - return handle_cachepath_consensus(procdata, msg); default: err_printf("Received unexpected message from node: %d\n", (int) msg->header.type); assert(0); @@ -2961,35 +2958,26 @@ static int handle_client_pickone_msg(ldcs_process_data_t *procdata, int nc, ldcs } /** - * Handle LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS to determine which cachepaths are - * available across all of the servers. + * Determine which cachepaths are available across all of the servers. */ -static int cachepath_consensus_reached; -static int handle_cachepath_consensus(ldcs_process_data_t *procdata, ldcs_message_t *msg){ - +int handle_cachepath_consensus(ldcs_process_data_t *procdata) +{ int num_children = ldcs_audit_server_md_get_num_children(procdata); - debug_printf( "Processing REQUEST_CACHEPATH_CONSENSUS.\n" ); + debug_printf( "Calculating cachepath consensus.\n" ); debug_printf3( " procdata->cachepath_bitidx = %#"PRIx64"\n", procdata->cachepath_bitidx ); debug_printf3( " procdata->cachepaths = %s\n", procdata->cachepaths ); debug_printf3( " procdata->cachepath = %s [should be null]\n", procdata->cachepath ); debug_printf3( " procdata->commpath = %s\n", procdata->commpath ); debug_printf3( " num_children = %d\n", num_children ); - if (num_children) { - spindle_broadcast(procdata, msg); - debug_printf3( "Successfully broadcast REQUEST_CACHEPATH_CONSENSUS\n" ); - msgbundle_force_flush(procdata); - debug_printf3( "Successfully flushed the broadcast of REQUEST_CACHEPATH_CONSENSUS\n" ); - } - ldcs_audit_server_md_allreduce_AND( &procdata->cachepath_bitidx ); debug_printf3( "The consensus value for procdata->cachepath_bitidx is: %#"PRIx64"\n", procdata->cachepath_bitidx ); - if( procdata->cachepath_bitidx == 0 ){ - err_printf("No valid cachepath path available. Falling back to \"commpath\" path (%s).\n", procdata->commpath); + if( procdata->cachepath_bitidx == 0 ) { + debug_printf("No valid cachepath path available. Falling back to \"commpath\" path (%s).\n", procdata->commpath); procdata->cachepath = procdata->commpath; - }else{ + } else { getValidCachePathByIndex( procdata->cachepath_bitidx, &procdata->cachepath, &procdata->parsed_cachepath, @@ -3003,14 +2991,10 @@ static int handle_cachepath_consensus(ldcs_process_data_t *procdata, ldcs_messag ldcs_audit_server_filemngt_init(procdata->cachepath, procdata->commpath); test_printf(" cachepath=%s\n", procdata->cachepath); - cachepath_consensus_reached = 1; return 0; } -/** - * Handle LDCS_MSG_CHOSEN_CACHEPATH_REQUEST - */ -static int handle_chosen_cachepath_request(ldcs_process_data_t *procdata, int nc){ +static int handle_chosen_cachepath_request(ldcs_process_data_t *procdata, int nc) { ldcs_message_t msg; int connid; ldcs_client_t *client; @@ -3022,18 +3006,12 @@ static int handle_chosen_cachepath_request(ldcs_process_data_t *procdata, int nc return 0; - if( cachepath_consensus_reached ){ - msg.header.type = LDCS_MSG_CHOSEN_CACHEPATH; - msg.header.len = strlen(procdata->cachepath) + 1 + strlen(procdata->parsed_cachepath) + 1; - msg.data = calloc( 1, msg.header.len ); - strcpy( msg.data, procdata->cachepath ); - strcpy( &msg.data[ strlen(procdata->cachepath)+1 ], procdata->parsed_cachepath ); - }else{ - msg.header.type = LDCS_MSG_NO_CACHEPATH_CONSENSUS_YET; - msg.header.len = 0; - msg.data = NULL; - } - + msg.header.type = LDCS_MSG_CHOSEN_CACHEPATH; + msg.header.len = strlen(procdata->cachepath) + 1 + strlen(procdata->parsed_cachepath) + 1; + msg.data = calloc( 1, msg.header.len ); + strcpy( msg.data, procdata->cachepath ); + strcpy( &msg.data[ strlen(procdata->cachepath)+1 ], procdata->parsed_cachepath ); + ldcs_send_msg(connid, &msg); free( msg.data ); procdata->server_stat.clientmsg.cnt++; diff --git a/src/server/auditserver/ldcs_audit_server_handlers.h b/src/server/auditserver/ldcs_audit_server_handlers.h index 8702316c..790e52b3 100644 --- a/src/server/auditserver/ldcs_audit_server_handlers.h +++ b/src/server/auditserver/ldcs_audit_server_handlers.h @@ -25,6 +25,7 @@ int handle_server_error(ldcs_process_data_t *procdata, node_peer_t peer); int handle_client_message(ldcs_process_data_t *procdata, int nc, ldcs_message_t *msg); int handle_client_start(ldcs_process_data_t *procdata, int nc); int handle_client_end(ldcs_process_data_t *procdata, int nc); +int handle_cachepath_consensus(ldcs_process_data_t *procdata); int exit_note_cb(int infd, int serverid, void *data); diff --git a/src/server/auditserver/ldcs_audit_server_process.c b/src/server/auditserver/ldcs_audit_server_process.c index b6ecdbff..7e32337d 100644 --- a/src/server/auditserver/ldcs_audit_server_process.c +++ b/src/server/auditserver/ldcs_audit_server_process.c @@ -137,7 +137,7 @@ void stopprofile() int ldcs_audit_server_process(spindle_args_t *args) { - int serverid, fd; + int serverid, fd, result; startprofile(args); @@ -198,7 +198,20 @@ int ldcs_audit_server_process(spindle_args_t *args) if (ldcs_process_data.opts & OPT_PROCCLEAN) init_cleanup_proc(ldcs_process_data.cachepath, ldcs_process_data.commpath); - debug_printf3("Initializing connections for clients at %s and %lu\n", + /* Calculate location of cache */ + debug_printf2("Calculating cache path location\n"); + determineValidCachePaths( + &ldcs_process_data.cachepath_bitidx, + ldcs_process_data.cachepaths, + ldcs_process_data.number ); + result = handle_cachepath_consensus(&ldcs_process_data); + if (result == -1) { + err_printf("Could not determine cachepath consensus\n"); + return -1; + } + + /* Setup connections for clients to start connecting */ + debug_printf2("Initializing connections for clients at %s and %lu\n", ldcs_process_data.commpath, (unsigned long) ldcs_process_data.number); serverid = ldcs_create_server(ldcs_process_data.commpath, ldcs_process_data.number); if (serverid == -1) { @@ -231,10 +244,7 @@ int ldcs_audit_server_process(spindle_args_t *args) if (fd != -1) { ldcs_listen_register_fd(fd, serverid, forceExitCB, (void *) &ldcs_process_data); } - determineValidCachePaths( - &ldcs_process_data.cachepath_bitidx, - ldcs_process_data.cachepaths, - ldcs_process_data.number ); + return 0; } diff --git a/src/server/comlib/ldcs_api_util.c b/src/server/comlib/ldcs_api_util.c index af101815..28a3d90c 100644 --- a/src/server/comlib/ldcs_api_util.c +++ b/src/server/comlib/ldcs_api_util.c @@ -91,10 +91,8 @@ char* _message_type_to_str (ldcs_message_ids_t type) { STR_CASE(LDCS_MSG_PICKONE_RESP); STR_CASE(LDCS_MSG_ALIVE_REQ); STR_CASE(LDCS_MSG_ALIVE_RESP); - STR_CASE(LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS); STR_CASE(LDCS_MSG_CHOSEN_CACHEPATH_REQUEST); STR_CASE(LDCS_MSG_CHOSEN_CACHEPATH); - STR_CASE(LDCS_MSG_NO_CACHEPATH_CONSENSUS_YET); STR_CASE(LDCS_MSG_UNKNOWN); } return "unknown"; diff --git a/src/server/config.h.in b/src/server/config.h.in index 24040627..97bc7ce9 100644 --- a/src/server/config.h.in +++ b/src/server/config.h.in @@ -9,8 +9,9 @@ /* Colon-separated list of potential back-end cache directories */ #undef CACHEPATHS -/* Back-end directory for communication and housekeeping */ -#undef COMMPATH +/* Colon-separated list of candidate paths for back-end communication and + housekeeping */ +#undef COMMPATHS /* Define if were using biter for client/server communication */ #undef COMM_BITER diff --git a/src/server/configure b/src/server/configure index 83bff908..1b9bfd96 100755 --- a/src/server/configure +++ b/src/server/configure @@ -840,7 +840,7 @@ with_default_num_ports with_localstorage with_cachepaths with_cachepath -with_commpath +with_commpaths with_default_local_prefix with_testrm with_rm @@ -1575,8 +1575,8 @@ Optional Packages: --with-cachepaths=DIR Colon-separated list of potential back-end cache directories , - --with-compath=DIR Back-end directory for communication and - housekeeping + --with-compaths=DIR Colon-separated list of candidate paths for back-end + communication and housekeeping --with-default-local-prefix=DIRS Colon-seperated list of directories that Spindle will not cache files out of @@ -16463,11 +16463,11 @@ if test "${with_cachepath+set}" = set; then : fi -# Check whether --with-commpath was given. -if test "${with_commpath+set}" = set; then : - withval=$with_commpath; COMMPATH=${withval} +# Check whether --with-commpaths was given. +if test "${with_commpaths+set}" = set; then : + withval=$with_commpaths; COMMPATHS=${withval} else - COMMPATH=$DEFAULT_LOC + COMMPATHS=$DEFAULT_LOC fi @@ -16475,7 +16475,7 @@ fi if test "${with_default_local_prefix+set}" = set; then : withval=$with_default_local_prefix; SPINDLE_LOCAL_PREFIX=${withval} else - SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH" + SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATHS" fi @@ -16495,7 +16495,7 @@ _ACEOF cat >>confdefs.h <<_ACEOF -#define COMMPATH "$COMMPATH" +#define COMMPATHS "$COMMPATHS" _ACEOF diff --git a/src/server/startup/spindle_be.cc b/src/server/startup/spindle_be.cc index d5a234a1..17e072f6 100644 --- a/src/server/startup/spindle_be.cc +++ b/src/server/startup/spindle_be.cc @@ -58,7 +58,7 @@ static int unpack_data(spindle_args_t *args, void *buffer, int buffer_size) unpack_param(args->use_launcher, buf, pos); unpack_param(args->startup_type, buf, pos); unpack_param(args->shm_cache_size, buf, pos); - unpack_param(args->commpath, buf, pos); + unpack_param(args->commpaths, buf, pos); unpack_param(args->candidate_cachepaths, buf, pos); unpack_param(args->pythonprefix, buf, pos); unpack_param(args->preloadfile, buf, pos); @@ -146,17 +146,14 @@ int spindleRunBE(unsigned int port, unsigned int num_ports, unique_id_t unique_i assert(args.port == port); - /* Expand environment variables in commpath. */ - char *new_commpath = parse_location(args.commpath, args.number); - if (!new_commpath) { - err_printf("Failed to convert commpath %s\n", args.commpath); - if (args.startup_type == startup_external) - LOGGING_FINI; - return -1; + /* Find the first valid commpath in the commpaths list. */ + if( -1 == getFirstValidPath( args.commpaths, &( args.commpath ), args.number ) ){ + // Failed. + err_printf("Failed to find valid commpath in %s\n", args.commpaths); + if (args.startup_type == startup_external) + LOGGING_FINI; + return -1; } - debug_printf("Translated commpath from %s to %s\n", args.commpath, new_commpath); - free(args.commpath); - args.commpath = strdup(new_commpath); result = ldcs_audit_server_process(&args); if (result == -1) { diff --git a/src/slurm_plugin/Makefile.am b/src/slurm_plugin/Makefile.am index 489dffcb..f712463b 100644 --- a/src/slurm_plugin/Makefile.am +++ b/src/slurm_plugin/Makefile.am @@ -3,7 +3,7 @@ lib_LTLIBRARIES = libspindleslurm.la libver=`$(top_srcdir)/LIB_VERSION spindleslurm` libspindleslurm_la_SOURCES = encode_decode.c plugin_utils.c slurm_plugin.c $(top_srcdir)/src/utils/spindle_mkdir.c $(top_srcdir)/src/utils/parseloc.c -libspindleslurm_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/include -I$(top_srcdir)/src/logging -DUSE_PLUGIN_DEBUG -DDEBUG -DCUSTOM_GETENV -DCUSTOM_GETENV_FREE -DSCONTROL_BIN="$(SCONTROL_ABSPATH)" -DSINFO_BIN="$(SINFO_ABSPATH)" -DSPINDLE_DO_EXPORT +libspindleslurm_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/utils -I$(top_srcdir)/src/include -I$(top_srcdir)/src/logging -DUSE_PLUGIN_DEBUG -DDEBUG -DCUSTOM_GETENV -DCUSTOM_GETENV_FREE -DSCONTROL_BIN="$(SCONTROL_ABSPATH)" -DSINFO_BIN="$(SINFO_ABSPATH)" -DSPINDLE_DO_EXPORT libspindleslurm_la_LDFLAGS = $(AM_LDFLAGS) -ldl -version-info $(libver) libspindleslurm_la_LIBADD = $(top_builddir)/src/server/startup/libspindlebe.la $(top_builddir)/src/fe/startup/libspindlefe.la libspindleslurm_la_CFLAGS = $(CFLAGS) -fvisibility=hidden diff --git a/src/slurm_plugin/Makefile.in b/src/slurm_plugin/Makefile.in index a7fe188f..8d3f3643 100644 --- a/src/slurm_plugin/Makefile.in +++ b/src/slurm_plugin/Makefile.in @@ -383,7 +383,7 @@ top_srcdir = @top_srcdir@ lib_LTLIBRARIES = libspindleslurm.la libver = `$(top_srcdir)/LIB_VERSION spindleslurm` libspindleslurm_la_SOURCES = encode_decode.c plugin_utils.c slurm_plugin.c $(top_srcdir)/src/utils/spindle_mkdir.c $(top_srcdir)/src/utils/parseloc.c -libspindleslurm_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/include -I$(top_srcdir)/src/logging -DUSE_PLUGIN_DEBUG -DDEBUG -DCUSTOM_GETENV -DCUSTOM_GETENV_FREE -DSCONTROL_BIN="$(SCONTROL_ABSPATH)" -DSINFO_BIN="$(SINFO_ABSPATH)" -DSPINDLE_DO_EXPORT +libspindleslurm_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/utils -I$(top_srcdir)/src/include -I$(top_srcdir)/src/logging -DUSE_PLUGIN_DEBUG -DDEBUG -DCUSTOM_GETENV -DCUSTOM_GETENV_FREE -DSCONTROL_BIN="$(SCONTROL_ABSPATH)" -DSINFO_BIN="$(SINFO_ABSPATH)" -DSPINDLE_DO_EXPORT libspindleslurm_la_LDFLAGS = $(AM_LDFLAGS) -ldl -version-info $(libver) libspindleslurm_la_LIBADD = $(top_builddir)/src/server/startup/libspindlebe.la $(top_builddir)/src/fe/startup/libspindlefe.la libspindleslurm_la_CFLAGS = $(CFLAGS) -fvisibility=hidden diff --git a/src/slurm_plugin/plugin_utils.c b/src/slurm_plugin/plugin_utils.c index f77531e4..0a8f0eaf 100644 --- a/src/slurm_plugin/plugin_utils.c +++ b/src/slurm_plugin/plugin_utils.c @@ -12,6 +12,7 @@ #include #include #include "plugin_utils.h" +#include "parseloc.h" #if !defined(STR) #define XSTR(X) #X @@ -297,33 +298,6 @@ int isFEHost(char **hostlist, unsigned int num_hosts) (void) error; return feresult; } - -static char* locSpecificDir(spindle_args_t *params) -{ - char *dir = NULL, *expanded_dir = NULL, *realized_dir = NULL; - - dir = params->commpath; - if (!dir) { - sdprintf(1, "ERROR: Location not filled in\n"); - goto done; - } - expanded_dir = parse_location(dir, params->number); - if (!expanded_dir) { - sdprintf(1, "ERROR: Could not expand file-system dir %s\n", dir); - goto done; - } - realized_dir = realize(expanded_dir); - if (!realized_dir) { - sdprintf(1, "ERROR: Could not turn dir to a real path\n"); - } - - done: - if (expanded_dir) - free(expanded_dir); - - return realized_dir; -} - #define SOCKET_PREFIX "spFE" static char* exitSocketPath(spindle_args_t *params) @@ -332,11 +306,11 @@ static char* exitSocketPath(spindle_args_t *params) char session_id_str[32]; size_t socket_path_len; - realized_dir = locSpecificDir(params); - if (!realized_dir) { + if( -1 == getFirstValidPath( params->commpaths, &( params->commpath ), params->number )){ err_printf("Could not get real path for FE exit socket\n"); goto done; } + realized_dir = strdup( params->commpath ); snprintf(session_id_str, sizeof(session_id_str), "%lu", (unsigned long) params->number); @@ -525,7 +499,10 @@ int isBEProc(spindle_args_t *params, unsigned int exit_phase) int beproc_result = -1; int fd = -1, error; - realized_dir = locSpecificDir(params); + if( -1 == getFirstValidPath( params->commpaths, &( params->commpath ), params->number ) ){ + return -1; + } + realized_dir = strdup( params->commpath ); gethostname(hostname, sizeof(hostname)); hostname[sizeof(hostname)-1] = '\0'; diff --git a/src/slurm_plugin/slurm_plugin.c b/src/slurm_plugin/slurm_plugin.c index 0989299c..7a2099a1 100644 --- a/src/slurm_plugin/slurm_plugin.c +++ b/src/slurm_plugin/slurm_plugin.c @@ -31,7 +31,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "spindle_launch.h" #include "plugin_utils.h" - +#include "parseloc.h" #include "config.h" #define SPINDLE_USE_SESSION "SPINDLE_USE_SESSION" @@ -704,7 +704,6 @@ static unique_id_t getUniqueID(spank_t spank, int session_enabled) static int fillInArgs(spank_t spank, spindle_args_t *args, int argc, char **argv, unique_id_t unique_id, int session_enabled) { int result; - char *symbolic_commpath, *orig_commpath; char *err_string; args->unique_id = unique_id; @@ -732,12 +731,9 @@ static int fillInArgs(spank_t spank, spindle_args_t *args, int argc, char **argv } - symbolic_commpath = args->commpath; - orig_commpath = parse_location(symbolic_commpath, args->number); - if( !orig_commpath ){ + if( -1 == getFirstValidPath( args->commpaths, &( args->commpath ), args->number ) ){ return -1; } - args->commpath = realize(orig_commpath); current_spank = spank; diff --git a/src/utils/parseloc.c b/src/utils/parseloc.c index bdf5ca1f..01183d01 100644 --- a/src/utils/parseloc.c +++ b/src/utils/parseloc.c @@ -370,6 +370,24 @@ void determineValidCachePaths( uint64_t *validBitIdx, char *origPathList, number free( pathList ); } +int getFirstValidPath( char *origPathList, char **firstValidPath, number_t number ){ + char *saveptr, *candidatePath, *pathList = strdup( origPathList ); + int rc = 0; + + candidatePath = strtok_r( pathList, ":", &saveptr ); + while( NULL != candidatePath ){ + rc = validateCandidatePath( candidatePath, firstValidPath, NULL, NULL, number ); + if( 1 == rc ){ // success + break; + } + *firstValidPath = NULL; + candidatePath = strtok_r( NULL, ":", &saveptr ); + } + free(pathList); + + return (1 == rc) ? 0 : -1; +} + void getValidCachePathByIndex( uint64_t validBitIdx, char **realizedCachePath, char **parsedCachePath, char **symbolicCachePath ){ uint64_t bitoffset = 0; if (!validBitIdx){ diff --git a/src/utils/parseloc.h b/src/utils/parseloc.h index a99409c3..5100364e 100644 --- a/src/utils/parseloc.h +++ b/src/utils/parseloc.h @@ -31,7 +31,7 @@ int is_local_prefix(const char *path, char **local_prefixes); int validateCandidatePath( char *candidatePath, char **realizedPath, char **parsedPath, char **symbolicPath, number_t number ); void determineValidCachePaths( uint64_t *validBitIdx, char *origPathList, number_t number ); void getValidCachePathByIndex( uint64_t validBitIdx, char **realizedCachePath, char **parsedCachePath, char **symbolicCachePath ); - +int getFirstValidPath( char *origPathList, char **firstValidPath, number_t number ); #if defined(__cplusplus) } #endif