diff --git a/lading/src/bin/lading.rs b/lading/src/bin/lading.rs index 9fd987507..b6d7cb1c7 100644 --- a/lading/src/bin/lading.rs +++ b/lading/src/bin/lading.rs @@ -662,6 +662,12 @@ async fn inner_main( }, () = &mut timer_watcher_wait => { info!("shutdown signal received."); + // First of four graceful-shutdown breadcrumbs. If a regression + // stalls the drain, the later three stop being reached and triage + // sees where progress died. + lading_antithesis::reachable!( + "lading began graceful shutdown when its experiment timer elapsed" + ); break Ok(()); } Some(res) = osrv_joinset.join_next() => { @@ -708,11 +714,22 @@ async fn inner_main( }; shutdown_broadcast.signal_and_wait().await; + // Second breadcrumb. The barrier waits only on registered peers, today just + // the capture manager. If the load-bearing watchers are ever registered, a + // generator stuck in its connect loop would hang here and this stops firing. + lading_antithesis::reachable!( + "lading's shutdown drain barrier returned without hanging" + ); // Await the capture manager task, if it exists, to ensure all data is // flushed. if let Some(handle) = capture_manager_handle { let _ = handle.await; + // Third breadcrumb. The capture manager task joined, so every matured + // capture record reached disk. + lading_antithesis::reachable!( + "lading flushed all matured capture data to disk during graceful shutdown" + ); } res @@ -813,6 +830,11 @@ fn main() -> Result<(), Error> { ); runtime.shutdown_timeout(max_shutdown_delay); info!("Bye. :)"); + // Fourth breadcrumb. The runtime drained within the shutdown-delay backstop + // and lading is about to return its exit status. + lading_antithesis::reachable!( + "lading drained its runtime within the shutdown-delay backstop and exited cleanly" + ); res } diff --git a/lading/src/generator/tcp.rs b/lading/src/generator/tcp.rs index 73027310b..87c82264a 100644 --- a/lading/src/generator/tcp.rs +++ b/lading/src/generator/tcp.rs @@ -278,6 +278,13 @@ impl TcpWorker { } () = &mut shutdown_wait => { info!("shutdown signal received"); + // A connected worker honors shutdown here via select. The + // connect-loop branch above does not select on shutdown, so a + // worker stuck reconnecting to an unreachable destination + // never reaches this point. The contrast localizes that gap. + lading_antithesis::reachable!( + "a connected tcp generator worker honored the shutdown signal and stopped promptly" + ); return Ok(()); }, } diff --git a/test/antithesis/CLAUDE.md b/test/antithesis/CLAUDE.md index f3a502fef..8fa0544d7 100644 --- a/test/antithesis/CLAUDE.md +++ b/test/antithesis/CLAUDE.md @@ -38,9 +38,17 @@ ready; the workload container runs it from its entrypoint. It receives lading's load and owns the "load arrived" assertion. SDK-linked but built without coverage instrumentation. Built, linted, and tested from the repo root like any other crate. -- `scenarios/general/` — the MVP scenario: `Dockerfile` (three targets: the - instrumented lading system under test, the sink, the workload), a - `docker-compose.yaml` snouty consumes as `--config`, `launch.env`, the - hard-wired `lading.yaml`, and the `workload/` build inputs. Config variation is - deferred to the workload seam. +- `harness/` — the shared harness crate. Holds the per-timeline config sampler + and the workload test commands baked into scenarios: `first_sample_config`, + `anytime_capture_consistent`, and `anytime_lading_drained_bounded`. +- `scenarios/general/` — the MVP scenario. lading pushes TCP load at the sink, + the workload samples a lading config per timeline and validates capture + crash-consistency across node faults. See `scenarios/general/README.md`. +- `scenarios/shutdown-safety/` — graceful-shutdown scenario. lading runs under a + `timeout` watchdog with its generator aimed at an unreachable destination, and + the workload asserts lading drains cleanly within a bound. No sink, no node + faults. See `scenarios/shutdown-safety/README.md`. +- Each scenario directory carries a `Dockerfile`, a `docker-compose.yaml` snouty + consumes as `--config`, `launch.env`, a `lading.yaml` when the config is fixed, + the `workload/` build inputs, and a `README.md`. - `bin/launch.sh` — the generic launcher shared by every scenario. diff --git a/test/antithesis/harness/Cargo.toml b/test/antithesis/harness/Cargo.toml index 56711f508..658bd10a0 100644 --- a/test/antithesis/harness/Cargo.toml +++ b/test/antithesis/harness/Cargo.toml @@ -17,6 +17,10 @@ path = "src/bin/first_sample_config.rs" name = "anytime_capture_consistent" path = "src/bin/anytime_capture_consistent.rs" +[[bin]] +name = "anytime_lading_drained_bounded" +path = "src/bin/anytime_lading_drained_bounded.rs" + [lints] workspace = true diff --git a/test/antithesis/harness/src/bin/anytime_lading_drained_bounded.rs b/test/antithesis/harness/src/bin/anytime_lading_drained_bounded.rs new file mode 100644 index 000000000..9c48afe9a --- /dev/null +++ b/test/antithesis/harness/src/bin/anytime_lading_drained_bounded.rs @@ -0,0 +1,66 @@ +//! Antithesis `anytime_` command: assert lading drains promptly on graceful +//! (experiment-timer) shutdown even when its generator points at an unreachable +//! destination. +//! +//! The lading (system under test) container runs lading under a wall-clock +//! `timeout` watchdog and writes lading's exit code to a sentinel file on the +//! shared volume. This checker runs in the (never-faulted) workload container +//! and fires whenever Antithesis chooses. It reads that exit code: +//! * rc == 0 -> lading drained within the watchdog bound (the good outcome). +//! * rc == 124 -> the `timeout` watchdog killed lading before it drained. That +//! would indicate the shutdown drain hung rather than completing promptly, +//! for example if a generator stuck against the unreachable destination +//! blocked the drain instead of being abandoned at runtime teardown. +//! * any other rc -> some other failure, e.g. a lading crash. +//! +//! It always exits 0; findings are reported as named assertions. + +use std::fs; + +/// File the lading entrypoint writes lading's exit code into. Overridable via +/// `LADING_EXIT_PATH`. +const DEFAULT_EXIT_PATH: &str = "/shared/lading_exit"; + +fn main() { + lading_antithesis::init(); + + let path = + std::env::var("LADING_EXIT_PATH").unwrap_or_else(|_| DEFAULT_EXIT_PATH.to_string()); + + let Ok(content) = fs::read_to_string(&path) else { + // The exit-code sentinel is not present yet this tick: lading has not + // finished (or been killed) yet. Nothing to check. + return; + }; + + let trimmed = content.trim(); + let Ok(rc) = trimmed.parse::() else { + // A non-integer sentinel means the entrypoint has not written a + // well-formed exit code yet; wait for a later tick. + return; + }; + + // Non-vacuity floor: prove the checker actually observed a recorded exit at + // least once across the run, so the always-invariant below is not passing + // vacuously on an absent file. + lading_antithesis::sometimes!( + !trimmed.is_empty(), + "lading recorded a bounded exit", + { "rc": rc } + ); + + // The invariant under test: on graceful (experiment-timer) shutdown with an + // unreachable destination, lading drains within the watchdog bound + // (rc == 0). A watchdog kill (rc == 124) would mean the shutdown drain hung + // rather than completing promptly. + lading_antithesis::always!( + rc == 0, + "lading drains within bound on graceful shutdown with an unreachable destination", + { "rc": rc } + ); + + lading_antithesis::reachable!( + "bounded-drain checker validated a recorded lading exit", + { "rc": rc } + ); +} diff --git a/test/antithesis/scenarios/general/README.md b/test/antithesis/scenarios/general/README.md new file mode 100644 index 000000000..43d8189f3 --- /dev/null +++ b/test/antithesis/scenarios/general/README.md @@ -0,0 +1,56 @@ +# General scenario + +This is the MVP scenario. lading generates TCP load into an instrumented sink, +and the run establishes two things across faults: that load actually arrives, and +that lading's capture files stay crash-consistent when lading is hard-killed. + +## How it works + +This scenario comprises the following components: + +* sink (oracle) +* lading, instrumented (system under test) +* workload (driver) + +The sink is a TCP server that counts received bytes and owns the "load arrived" +assertion. It is healthchecked and never faulted. lading is the system under +test, faulted at the node level. The workload writes the timeline's config and +validates lading's capture. + +The workload's `first_sample_config` command samples a lading config per timeline +from Antithesis randomness (payload variant, bytes per second, parallel +connections, and seed) and writes it, plus a `ready` sentinel, to the shared +volume. lading's entrypoint blocks on the sentinel, then boots under the sampled +config and pushes TCP load at the sink. lading writes its capture to a named +`capture` volume that outlives a `node_termination` of the lading container. + +Antithesis injects node faults on lading: a `node_termination` is a SIGKILL of +the whole container, so only artifacts on the named volume survive. The +workload's `anytime_capture_consistent` checker validates every capture file by +its format: + +* JSONL: a valid parseable prefix. A torn final line, the interrupted write, is + tolerated. A broken earlier line or a violated invariant is not. +* Parquet: footer-terminated, so a hard kill leaves it unreadable. That is + expected, not corruption. A readable Parquet must be internally consistent. +* Multi: both of the above, on the `.jsonl` and `.parquet` files. + +The checker asserts no mid-file corruption and monotonic `fetch_index` and +per-series time. The sink asserts `sometimes(total_bytes > 0)`. + +# Caveats + +1. JSONL survives a kill as a valid prefix because of the 60-second in-memory + maturity window. Matured intervals are on disk, the unmatured tail is lost on + abrupt death by design. This scenario checks consistency, not completeness. +2. Parquet and Multi are footer-terminated. An abrupt kill leaves the Parquet + file unreadable, which the checker treats as expected rather than a violation. +3. There is deliberately no blackhole. The sink is the only receiver, so a config + that expected a blackhole would not apply here. + +# Assumptions + +1. The `capture` named volume outlives a `node_termination` of lading. +2. The sink is never faulted and is a healthchecked dependency of lading. +3. Config sampling draws from Antithesis randomness so timelines branch across + the payload-variant menu. diff --git a/test/antithesis/scenarios/shutdown-safety/Dockerfile b/test/antithesis/scenarios/shutdown-safety/Dockerfile new file mode 100644 index 000000000..162c59184 --- /dev/null +++ b/test/antithesis/scenarios/shutdown-safety/Dockerfile @@ -0,0 +1,115 @@ +# syntax=docker/dockerfile:1 +# +# Antithesis harness image for the lading "shutdown-safety" scenario. +# +# Build context is the repository root. Named runtime targets: +# - lading : the instrumented lading binary (the system under test), sancov on +# - workload : bakes the bounded-drain checker, emits setup_complete, then idles +# +# There is deliberately NO sink: the point of this scenario is a generator aimed +# at an UNREACHABLE address (connection refused -> busy reconnect), exercising +# whether lading drains promptly on graceful (experiment-timer) shutdown. +# +# lading is built native x86_64-unknown-linux-gnu (glibc). + +# --------------------------------------------------------------------------- +# Shared build environment: Rust toolchain (pinned by rust-toolchain.toml) plus +# native build dependencies (protoc for prost/tonic, openssl for reqwest). +# --------------------------------------------------------------------------- +FROM debian:bookworm AS build-base +ENV DEBIAN_FRONTEND=noninteractive \ + NO_COLOR=1 \ + CARGO_TERM_COLOR=never +RUN apt-get update && \ + apt-get install --no-install-recommends -y \ + build-essential ca-certificates curl pkg-config libssl-dev cmake protobuf-compiler && \ + rm -rf /var/lib/apt/lists/* +# Install rustup with no default toolchain; the pinned toolchain auto-installs on +# first cargo invocation inside the repo (rust-toolchain.toml). +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ + sh -s -- -y --profile minimal --default-toolchain none +ENV PATH="/root/.cargo/bin:${PATH}" + +# --------------------------------------------------------------------------- +# Build the instrumented lading (the system under test). +# +# Coverage instrumentation uses the modern Antithesis Rust flow: the +# `antithesis-instrumentation` crate (referenced once in the binary behind the +# `antithesis` feature) provides the runtime shim, and these RUSTFLAGS enable +# LLVM sancov coverage. `--build-id` is required for symbolization. LTO is forced +# off for predictable sancov instrumentation, and split-debuginfo is forced off +# so the binary keeps embedded DWARF. The release profile already sets +# `panic = "abort"`, so any lading panic becomes SIGABRT, caught as a hard crash. +# +# The sancov RUSTFLAGS are passed via `--config target..rustflags` with an +# explicit `--target`, NOT via the RUSTFLAGS env var. With an explicit target, +# Cargo builds host artifacts (build scripts, proc-macros) for the host and does +# NOT apply the target rustflags to them, so they are not instrumented and link +# cleanly. +# --------------------------------------------------------------------------- +FROM build-base AS lading-builder +ENV CARGO_PROFILE_RELEASE_LTO=off \ + CARGO_PROFILE_RELEASE_SPLIT_DEBUGINFO=off +WORKDIR /lading +COPY . /lading +RUN --mount=type=cache,target=/lading/target,id=antithesis-lading-target \ + --mount=type=cache,target=/root/.cargo/registry,id=cargo-registry \ + --mount=type=cache,target=/root/.cargo/git,id=cargo-git \ + cargo build --release --package lading --bin lading --features antithesis \ + --target x86_64-unknown-linux-gnu \ + --config 'target.x86_64-unknown-linux-gnu.rustflags=["-Ccodegen-units=1","-Cpasses=sancov-module","-Cllvm-args=-sanitizer-coverage-level=3","-Cllvm-args=-sanitizer-coverage-trace-pc-guard","-Clink-args=-Wl,--build-id"]' && \ + cp /lading/target/x86_64-unknown-linux-gnu/release/lading /usr/local/bin/lading && \ + echo "Validating Antithesis instrumentation symbols..." && \ + nm /usr/local/bin/lading | grep -q "antithesis_load_libvoidstar" && \ + nm /usr/local/bin/lading | grep -q "sanitizer_cov_trace_pc_guard" && \ + echo "Instrumentation symbols present." + +# --------------------------------------------------------------------------- +# Build the bounded-drain checker, uninstrumented. It is a supporting harness +# component (a workload test command), not the system under test, so it needs no +# coverage instrumentation. There is no sink in this scenario. +# --------------------------------------------------------------------------- +FROM build-base AS tools-builder +WORKDIR /tools +COPY . /tools +RUN --mount=type=cache,target=/tools/target,id=antithesis-tools-target \ + --mount=type=cache,target=/root/.cargo/registry,id=cargo-registry \ + --mount=type=cache,target=/root/.cargo/git,id=cargo-git \ + cargo build --release --package harness --bin anytime_lading_drained_bounded && \ + cp /tools/target/release/anytime_lading_drained_bounded /usr/local/bin/anytime_lading_drained_bounded + +# --------------------------------------------------------------------------- +# Runtime: instrumented lading (system under test). +# --------------------------------------------------------------------------- +FROM debian:bookworm-slim AS lading +ENV NO_COLOR=1 +RUN apt-get update && \ + apt-get install --no-install-recommends -y ca-certificates libssl3 && \ + rm -rf /var/lib/apt/lists/* +COPY --from=lading-builder /usr/local/bin/lading /usr/local/bin/lading +COPY --chmod=755 test/antithesis/scenarios/shutdown-safety/lading-entrypoint.sh /usr/local/bin/lading-entrypoint.sh +# Fixed config, baked in: this scenario does NOT sample configs, so the generator +# is hard-wired to an unreachable address. +COPY test/antithesis/scenarios/shutdown-safety/lading.yaml /etc/lading/lading.yaml +# Antithesis scans /symbols in every pushed image and matches by build-id. +RUN mkdir -p /symbols && ln -s /usr/local/bin/lading /symbols/lading +ENTRYPOINT ["/usr/local/bin/lading-entrypoint.sh"] + +# --------------------------------------------------------------------------- +# Runtime: workload driver. Bakes the bounded-drain checker, emits +# setup_complete, then idles. +# +# The checker links only lading_antithesis + std (no lading, no openssl), so the +# slim base already carries everything it needs (libgcc_s, libc). No extra +# packages are installed. +# --------------------------------------------------------------------------- +FROM debian:bookworm-slim AS workload +ENV NO_COLOR=1 +COPY --chmod=755 test/antithesis/scenarios/shutdown-safety/workload/setup-complete.sh /opt/antithesis/setup-complete.sh +COPY --chmod=755 test/antithesis/scenarios/shutdown-safety/workload/entrypoint.sh /entrypoint.sh +# Antithesis test commands live here; the platform discovers and runs them from +# /opt/antithesis/test/v1/