diff --git a/.github/scripts/publish-releases.sh b/.github/scripts/publish-releases.sh index 3c6165b..5ce4271 100755 --- a/.github/scripts/publish-releases.sh +++ b/.github/scripts/publish-releases.sh @@ -2,8 +2,9 @@ # # Publish built artifacts as pre-releases and update the latest/ pointer # files. Expects the build artifacts in the directory given as the first -# argument (default: artifacts), one sub-directory per build, each -# containing a meta.json describing the build. +# argument (default: artifacts), one sub-directory per build (or a single +# build's files directly in the directory), each build described by its +# meta.json. set -euo pipefail shopt -s nullglob @@ -13,9 +14,12 @@ ARTIFACTS_DIR=${1:-artifacts} mkdir -p latest -for dir in "$ARTIFACTS_DIR"/*/; do - meta="$dir/meta.json" +# Each build's staged assets are marked by a meta.json. Multiple artifacts +# unpack to one sub-directory per build, but actions/download-artifact +# unpacks a single artifact into the artifacts dir itself, so check both. +for meta in "$ARTIFACTS_DIR"/meta.json "$ARTIFACTS_DIR"/*/meta.json; do [ -f "$meta" ] || continue + dir=$(dirname "$meta") version=$(jq -r .version "$meta") channel=$(jq -r .channel "$meta") sha=$(jq -r .sha "$meta") diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d2c975a..0eb4f26 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -97,7 +97,9 @@ jobs: - name: Build distribution run: | ./gradlew buildInfo releaseInfo - BUILD_PACK=1 ./gradlew pack + # installPlugin unpacks the core plugins into NXF_PLUGINS_DIR, + # the same way Nextflow's own e2e build does (test-e2e/run.sh) + BUILD_PACK=1 NXF_PLUGINS_DIR="$PWD/build/plugins" ./gradlew pack installPlugin - name: Stage release assets run: | @@ -106,6 +108,19 @@ jobs: cp "build/releases/nextflow-$MATRIX_VERSION-one.jar" stage/ cp "build/releases/nextflow-$MATRIX_VERSION-dist" stage/ cp nextflow stage/nextflow + # core plugins bundle: top-level entries are the unpacked plugin + # directories (-/), the layout the launcher + # expects for NXF_PLUGINS_DIR (see nextflow-io/nextflow#7219) + bundle="stage/nextflow-$MATRIX_VERSION-plugins.tar.gz" + listing=$(tar -czvf "$bundle" -C build/plugins .) + if grep -Ev '^\./(nf-[^/]+/|$)' <<< "$listing"; then + echo "::error::plugins bundle contains unexpected top-level entries" + exit 1 + fi + if ! grep -Eq '^\./nf-[^/]+/classes/META-INF/MANIFEST\.MF$' <<< "$listing"; then + echo "::error::plugins bundle contains no plugin manifests" + exit 1 + fi (cd stage && sha256sum * > checksums.sha256) jq -n \ --arg kind "$MATRIX_KIND" --arg channel "$MATRIX_CHANNEL" \ diff --git a/README.md b/README.md index ab170eb..bfe0f7f 100644 --- a/README.md +++ b/README.md @@ -81,10 +81,19 @@ Every release's notes contain ready-made copy-paste commands for both methods. |-------|-------------| | `nextflow--one.jar` | Fat jar downloaded by the Nextflow launcher (`NXF_BASE` method) | | `nextflow--dist` | Self-contained executable (launcher + jar in one file) | +| `nextflow--plugins.tar.gz` | Core plugins compiled from the same source (see [below](#core-plugins-bundle)) | | `nextflow` | The launcher script from that commit, with its default version set to the build | | `checksums.sha256` | SHA-256 checksums of the assets | | `meta.json` | Build metadata (source commit, branch or PR, version) | +### Core plugins bundle + +`nextflow--plugins.tar.gz` contains the core plugins (`nf-amazon`, `nf-google`, `nf-tower`, ...) compiled from the same source as the rest of the build. Its top-level entries are the unpacked plugin directories, named `-/` — the same layout Nextflow keeps in its plugins directory (`$NXF_HOME/plugins`). Note that the directory names carry each plugin's own version, which is unrelated to the build version in the asset name. + +The `NXF_DEV` launcher mode (added by [nextflow-io/nextflow#7219](https://github.com/nextflow-io/nextflow/pull/7219)) downloads this bundle automatically and uses it as `NXF_PLUGINS_DIR`, so a PR that changes a core plugin is tested against the plugin code from that PR. The plain `NXF_BASE`+`NXF_VER` method does **not** use the bundle: plugins are resolved from the plugin registry at runtime, the same way regular releases do. + +Releases published before this asset was introduced simply lack it; the `NXF_DEV` launcher prints a warning and falls back to the released plugins. + ## Triggering a build manually Builds for active PRs and tracked branches happen automatically within about 30 minutes of a push. To build something immediately, or to build a PR or branch outside the automatic rules, use the [build workflow](../../actions/workflows/build.yml) with "Run workflow", or the GitHub CLI: @@ -96,6 +105,6 @@ gh workflow run build.yml -R nextflow-io/nextflow-dev-builds -f branch=master ## Caveats -- Dev builds resolve Nextflow plugins (nf-amazon, nf-tower, etc.) from the plugin registry at runtime, the same way regular releases do. A PR that changes a plugin and bumps its version cannot be fully tested this way, since the new plugin version is not published anywhere. Core changes, which are the vast majority, work fine. +- When used via `NXF_BASE`+`NXF_VER`, dev builds resolve Nextflow plugins (nf-amazon, nf-tower, etc.) from the plugin registry at runtime, the same way regular releases do — a PR that changes a core plugin is then tested against the *released* plugin code. To test core plugin changes, use the `NXF_DEV` launcher mode, which picks up the [core plugins bundle](#core-plugins-bundle) built from the PR source. - Builds run only for code hosted in `nextflow-io/nextflow` (branches and PRs targeting it). - Old builds are deleted by the retention policy, so do not depend on a dev build URL staying around. Pin a regular release for anything that matters.