Manage installation and release of the exec harness and codspeed-memtrack

.github/workflows/release.yml

@@ -128,10 +128,6 @@ jobs:
             curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
             echo "$HOME/.cargo/bin" >> $GITHUB_PATH
           fi
-      - uses: swatinem/rust-cache@v2
-        with:
-          key: ${{ join(matrix.targets, '-') }}
-          cache-provider: ${{ matrix.cache_provider }}
       - name: Install dist
         run: ${{ matrix.install_dist.run }}
       # Get the dist-manifest

CONTRIBUTING.md

@@ -0,0 +1,85 @@
+# Contributing to CodSpeed Runner
+
+## Release Process
+
+This repository is a Cargo workspace containing multiple crates. The release process differs depending on which crate you're releasing.
+
+### Workspace Structure
+
+- **`codspeed-runner`**: The main CLI binary (`codspeed`)
+- **`memtrack`**: Memory tracking binary (`codspeed-memtrack`)
+- **`exec-harness`**: Execution harness binary
+- **`runner-shared`**: Shared library used by other crates
+
+### Releasing Support Crates (memtrack, exec-harness, runner-shared)
+
+For any crate other than the main runner:
+
+```bash
+cargo release -p <PACKAGE_NAME> --execute <VERSION_BUMP>
+```
+
+Where `<VERSION_BUMP>` is one of: `alpha`, `beta`, `patch`, `minor`, or `major`.
+
+**Examples:**
+
+```bash
+# Release a new patch version of memtrack
+cargo release -p memtrack --execute patch
+
+# Release a beta version of exec-harness
+cargo release -p exec-harness --execute beta
+```
+
+#### Post-Release: Update Version References
+
+After releasing `memtrack` or `exec-harness`, you **must** update the version references in the runner code:
+
+1. **For memtrack**: Update `MEMTRACK_CODSPEED_VERSION` in `src/executor/memory/executor.rs`:
+
+   ```rust
+   const MEMTRACK_CODSPEED_VERSION: &str = "X.Y.Z"; // Update to new version
+   ```
+
+2. **For exec-harness**: Update `EXEC_HARNESS_VERSION` in `src/exec/mod.rs`:
+   ```rust
+   const EXEC_HARNESS_VERSION: &str = "X.Y.Z"; // Update to new version
+   ```
+
+These constants are used by the runner to download and install the correct versions of the binaries from GitHub releases.
+
+### Releasing the Main Runner
+
+The main runner (`codspeed-runner`) should be released after ensuring all dependency versions are correct.
+
+#### Pre-Release Check
+
+**Verify binary version references**: Check that version constants in the runner code match the released versions:
+
+- `MEMTRACK_CODSPEED_VERSION` in `src/executor/memory/executor.rs`
+- `EXEC_HARNESS_VERSION` in `src/exec/mod.rs`
+
+#### Release Command
+
+```bash
+cargo release --execute <VERSION_BUMP>
+```
+
+Where `<VERSION_BUMP>` is one of: `alpha`, `beta`, `patch`, `minor`, or `major`.
+
+**Examples:**
+
+```bash
+# Release a new minor version
+cargo release --execute minor
+
+# Release a patch version
+cargo release --execute patch
+
+# Release a beta version
+cargo release --execute beta
+```
+
+## Known issue
+
+If one of the crates is currenlty in beta version, for example the runner is in beta version 4.4.2-beta.1, any alpha release will fail for the any crate, saying that only minor, major or patch releases is supported.

Cargo.toml

@@ -96,9 +96,12 @@ env_logger = "0.11.8"
 [workspace.metadata.release]
 sign-tag = true
 sign-commit = true
-shared-version = true
+shared-version = false
+consolidate-commits = false
+pre-release-commit-message = "chore: Release {{crate_name}} version {{version}} 🎉"
+
+[package.metadata.release]
 pre-release-hook = ["./scripts/pre-release.sh", "v{{version}}"]
-pre-release-commit-message = "chore: Release runner version {{version}} 🎉"
 
 [profile.dist]
 inherits = "release"
@@ -108,7 +111,7 @@ strip = true
 [package.metadata.dist]
 targets = ["aarch64-unknown-linux-musl", "x86_64-unknown-linux-musl"]
 
-# Config for 'dist'
+# Config for 'dist' for all crates in the workspace
 [workspace.metadata.dist]
 # Whether to consider the binaries in a package for distribution (defaults true)
 dist = true
@@ -121,13 +124,14 @@ installers = ["shell"]
 # The archive format to use for non-windows builds (defaults .tar.xz)
 unix-archive = ".tar.gz"
 # Which actions to run on pull requests
-pr-run-mode = "upload"
+pr-run-mode = "plan"
 # Post-announce jobs to run in CI
 post-announce-jobs = ["./bump-action"]
 # Path that installers should place binaries in
 install-path = "CARGO_HOME"
 # Whether to install an updater program
 install-updater = false
+# Build only the required packages, and individually
 precise-builds = true
 
 [workspace.metadata.dist.github-custom-runners]

crates/exec-harness/Cargo.toml

@@ -1,11 +1,16 @@
 [package]
 name = "exec-harness"
-version = "4.4.2-beta.1"
+version = "1.0.0"
 edition = "2024"
+repository = "https://github.com/CodSpeedHQ/runner"
+publish = false
 
 [dependencies]
 anyhow = { workspace = true }
 codspeed = "4.1.0"
 clap = { workspace = true }
 serde_json = { workspace = true }
 serde = { workspace = true }
+
+[package.metadata.dist]
+targets = ["aarch64-unknown-linux-musl", "x86_64-unknown-linux-musl"]

crates/exec-harness/src/main.rs

@@ -12,7 +12,10 @@ mod walltime;
 
 #[derive(Parser, Debug)]
 #[command(name = "exec-harness")]
-#[command(about = "CodSpeed exec harness - wraps commands with performance instrumentation")]
+#[command(
+    version,
+    about = "CodSpeed exec harness - wraps commands with performance instrumentation"
+)]
 struct Args {
     /// Optional benchmark name (defaults to command filename)
     #[arg(long)]

crates/memtrack/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "memtrack"
-version = "4.4.2-beta.1"
+version = "1.0.0"
 edition = "2024"
 repository = "https://github.com/CodSpeedHQ/runner"
 publish = false

src/binary_installer/mod.rs

@@ -0,0 +1,102 @@
+use crate::prelude::*;
+use crate::run::helpers::download_file;
+use semver::Version;
+use std::process::Command;
+use tempfile::NamedTempFile;
+use url::Url;
+
+mod versions;
+
+/// Ensure a binary is installed, or install it from a runner's GitHub release using the installer script.
+///
+/// This function checks if the binary is already installed with the correct version.
+/// If not, it downloads and executes an installer script from the CodSpeed runner repository.
+///
+/// # Arguments
+/// * `binary_name` - The binary command name (e.g., "codspeed-memtrack", "codspeed-exec-harness")
+/// * `version` - The version to install (e.g., "4.4.2-alpha.2")
+/// * `get_installer_url` - A closure that returns the URL to download the installer script.
+pub async fn ensure_binary_installed<F>(
+    binary_name: &str,
+    version: &str,
+    get_installer_url: F,
+) -> Result<()>
+where
+    F: FnOnce() -> String,
+{
+    if is_command_installed(
+        binary_name,
+        Version::parse(version).context("Invalid version format")?,
+    ) {
+        debug!("{binary_name} version {version} is already installed");
+        return Ok(());
+    }
+
+    let installer_url = Url::parse(&get_installer_url()).context("Invalid installer URL")?;
+
+    debug!("Downloading installer from: {installer_url}");
+
+    // Download the installer script to a temporary file
+    let temp_file = NamedTempFile::new().context("Failed to create temporary file")?;
+    download_file(&installer_url, temp_file.path()).await?;
+
+    // Execute the installer script
+    let output = Command::new("sh")
+        .arg(temp_file.path())
+        .output()
+        .context("Failed to execute installer command")?;
+
+    if !output.status.success() {
+        bail!(
+            "Failed to install {binary_name} version {version}. Installer exited with output: {output:?}",
+        );
+    }
+
+    if !is_command_installed(
+        binary_name,
+        Version::parse(version).context("Invalid version format")?,
+    ) {
+        bail!(
+            "Could not veryfy installation of {binary_name} version {version} after running installer"
+        );
+    }
+
+    info!("Successfully installed {binary_name} version {version}");
+    Ok(())
+}
+
+/// Check if the given command is installed and its version matches the expected version.
+///
+/// Expects the command to support the `--version` flag and return a version string.
+fn is_command_installed(command: &str, expected_version: Version) -> bool {
+    let is_command_installed = Command::new("which")
+        .arg(command)
+        .output()
+        .is_ok_and(|output| output.status.success());
+
+    if !is_command_installed {
+        debug!("{command} is not installed");
+        return false;
+    }
+
+    let Ok(version_output) = Command::new(command).arg("--version").output() else {
+        return false;
+    };
+
+    if !version_output.status.success() {
+        debug!(
+            "Failed to get command version. stderr: {}",
+            String::from_utf8_lossy(&version_output.stderr)
+        );
+        return false;
+    }
+
+    let version_string = String::from_utf8_lossy(&version_output.stdout);
+    let Ok(version) = versions::parse_from_output(&version_string) else {
+        return false;
+    };
+
+    debug!("Found {command} version: {version}");
+
+    versions::is_compatible(command, &version, &expected_version)
+}