diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6f44514eb7..0d3d7d9e14 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -133,6 +133,15 @@ create_key: paths: - pubkeys +# Unit tests for shared helpers in .gitlab/scripts/includes.sh (e.g. the +# snapshot-staleness check deploy.sh relies on to poll Sonatype). +shell-unit-tests: + stage: prepare + needs: [] + image: ${PREPARE_IMAGE} + script: + - bash .gitlab/scripts/tests/includes_test.sh + # Shared version detection used by benchmarks and reliability pipelines get-versions: extends: .get-versions diff --git a/.gitlab/benchmarks/.gitlab-ci.yml b/.gitlab/benchmarks/.gitlab-ci.yml index b6f7f1cef5..eee8d533a1 100644 --- a/.gitlab/benchmarks/.gitlab-ci.yml +++ b/.gitlab/benchmarks/.gitlab-ci.yml @@ -17,6 +17,7 @@ benchmarks-trigger: rules: - if: '$CANCELLED == "true"' when: never + - !reference [.skip-on-release, rules] - if: '$JDK_VERSION != null || $DEBUG_LEVEL != null || $HASH != null || $DOWNSTREAM != null' when: never - if: '$CI_PIPELINE_SOURCE == "schedule"' diff --git a/.gitlab/build-deploy/.gitlab-ci.yml b/.gitlab/build-deploy/.gitlab-ci.yml index 1d96d52390..ed27bff0cd 100644 --- a/.gitlab/build-deploy/.gitlab-ci.yml +++ b/.gitlab/build-deploy/.gitlab-ci.yml @@ -218,8 +218,7 @@ build-artifact: artifacts: false optional: true rules: - - if: '$CI_COMMIT_BRANCH =~ /^release\//' - when: never + - !reference [.skip-on-release, rules] - when: on_success tags: [ "arch:amd64" ] image: ${BUILD_IMAGE_X64} @@ -254,8 +253,7 @@ deploy-artifact: - job: build:arm64-musl artifacts: true rules: - - if: '$CI_COMMIT_BRANCH =~ /^release\//' - when: never + - !reference [.skip-on-release, rules] - when: on_success tags: [ "arch:amd64" ] image: ${BUILD_IMAGE_X64} @@ -345,7 +343,9 @@ notify-slack-on-success: artifacts: true - job: deploy-artifact artifacts: false - when: on_success + rules: + - !reference [.skip-on-release, rules] + - when: on_success image: registry.ddbuild.io/slack-notifier:v91289620-4ec922a-latest@sha256:06b24f392ccc383d371c72001520a254edef523bc0bfdc445f487106107b4202 tags: ["arch:amd64"] script: @@ -359,7 +359,9 @@ notify-slack-on-failure: artifacts: true - job: deploy-artifact artifacts: true - when: on_failure + rules: + - !reference [.skip-on-release, rules] + - when: on_failure image: registry.ddbuild.io/slack-notifier:v91289620-4ec922a-latest@sha256:06b24f392ccc383d371c72001520a254edef523bc0bfdc445f487106107b4202 tags: ["arch:amd64"] script: diff --git a/.gitlab/common.yml b/.gitlab/common.yml index 292a1462d1..b70151ee64 100644 --- a/.gitlab/common.yml +++ b/.gitlab/common.yml @@ -41,6 +41,17 @@ variables: KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: java-profiler +# Skip a job on release/* branches. build-artifact/deploy-artifact don't run +# there, so anything that needs: them must also skip there or pipeline +# creation fails. Splice into a job's own rules with: +# rules: +# - !reference [.skip-on-release, rules] +# - ... +.skip-on-release: + rules: + - if: '$CI_COMMIT_BRANCH =~ /^release\//' + when: never + # Install gh and crane when not already present in the image. # Extend this in before_script for jobs that need GitHub CLI or crane. .bootstrap-gh-tools: diff --git a/.gitlab/dd-trace-integration/.gitlab-ci.yml b/.gitlab/dd-trace-integration/.gitlab-ci.yml index 22aa991b88..3d7b0ec52c 100644 --- a/.gitlab/dd-trace-integration/.gitlab-ci.yml +++ b/.gitlab/dd-trace-integration/.gitlab-ci.yml @@ -17,6 +17,7 @@ prepare-patched-agent: - job: build-artifact artifacts: true rules: + - !reference [.skip-on-release, rules] - if: '$JDK_VERSION != null || $DEBUG_LEVEL != null || $HASH != null || $DOWNSTREAM != null' when: never - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' @@ -87,6 +88,7 @@ prepare-patched-agent: - job: prepare-patched-agent artifacts: true rules: + - !reference [.skip-on-release, rules] - if: '$JDK_VERSION != null || $DEBUG_LEVEL != null || $HASH != null || $DOWNSTREAM != null' when: never - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' @@ -268,6 +270,7 @@ report-dd-trace-results: - job: integration-test-arm64-musl artifacts: true rules: + - !reference [.skip-on-release, rules] - if: '$JDK_VERSION != null || $DEBUG_LEVEL != null || $HASH != null || $DOWNSTREAM != null' when: never - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' @@ -296,6 +299,7 @@ post-pr-comment: - job: integration-test-arm64-musl artifacts: true rules: + - !reference [.skip-on-release, rules] - if: '$JDK_VERSION != null || $DEBUG_LEVEL != null || $HASH != null || $DOWNSTREAM != null' when: never - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' diff --git a/.gitlab/scripts/deploy.sh b/.gitlab/scripts/deploy.sh index c0e74f676d..689f51bb05 100755 --- a/.gitlab/scripts/deploy.sh +++ b/.gitlab/scripts/deploy.sh @@ -40,6 +40,22 @@ if [ "$MODE" = "assemble" ] || [ "$MODE" = "all" ]; then ./gradlew -Pskip-native -Pskip-tests -Pddprof_version="${LIB_VERSION}" -PbuildInfo.build.number=$CI_JOB_ID -Pwith-libs="$(pwd)/libs" :ddprof-lib:jar assembleAll --exclude-task compileFuzzer --exclude-task sign --max-workers=1 --no-build-cache --stacktrace --info --no-watch-fs --no-daemon fi +# Resolve the debug/jar classifier's timestamped version from a snapshot's +# maven-metadata.xml. Only the last matching block is +# honored so a metadata file listing the same classifier/extension pair more +# than once can't produce multiple version lines from a single lookup. +CURL_TIMEOUT_OPTS=(--connect-timeout 10 --max-time 30) +get_debug_jar_snapshot_version() { + curl -fsSL "${CURL_TIMEOUT_OPTS[@]}" "$1" 2>/dev/null | awk ' + // { classifier=""; extension=""; value="" } + // { gsub(/<\/?classifier>/,""); gsub(/^[ \t]+|[ \t]+$/,""); classifier=$0 } + // { gsub(/<\/?extension>/,""); gsub(/^[ \t]+|[ \t]+$/,""); extension=$0 } + // { gsub(/<\/?value>/,""); gsub(/^[ \t]+|[ \t]+$/,""); value=$0 } + /<\/snapshotVersion>/ { if (classifier=="debug" && extension=="jar") last=value } + END { print last } + ' || true +} + # Publish task (only when publishing to Maven Central) if [ "$MODE" = "publish" ] || [ "$MODE" = "all" ]; then echo "=== Publishing to Sonatype ===" @@ -47,5 +63,43 @@ if [ "$MODE" = "publish" ] || [ "$MODE" = "all" ]; then echo "ERROR: GPG_PRIVATE_KEY is not set — run the create_key CI job first to provision the signing key in SSM (ci.java-profiler.signing.gpg_private_key)" exit 1 fi + META_URL="https://central.sonatype.com/repository/maven-snapshots/com/datadoghq/ddprof/${LIB_VERSION}/maven-metadata.xml" + # Record whatever debug/jar version is resolvable *before* publishing so the + # post-publish wait below can tell the freshly uploaded build apart from a + # preceding one that Sonatype hasn't finished replacing in the metadata yet. + PRE_PUBLISH_SNAPSHOT_VER="" + if [[ "${LIB_VERSION}" == *-SNAPSHOT ]]; then + PRE_PUBLISH_SNAPSHOT_VER=$(get_debug_jar_snapshot_version "${META_URL}") + fi + ./gradlew -Pskip-native -Pskip-tests -Pddprof_version="${LIB_VERSION}" -PbuildInfo.build.number=$CI_JOB_ID -Pwith-libs="$(pwd)/libs" publishToSonatype closeAndReleaseSonatypeStagingRepository --exclude-task compileFuzzer --max-workers=1 --no-build-cache --stacktrace --info --no-watch-fs --no-daemon + + # Downstream consumers (e.g. benchmarking-platform's run-benchmarks.sh) + # resolve the real timestamped filename via maven-metadata.xml right after + # this job finishes. Sonatype needs a little time to index a freshly + # published snapshot, so wait until the metadata (and the jar it points to) + # are actually resolvable before this job reports success. A version that + # matches PRE_PUBLISH_SNAPSHOT_VER is rejected: that's the preceding build's + # entry still lingering in the metadata, not the one just uploaded. + if [[ "${LIB_VERSION}" == *-SNAPSHOT ]]; then + echo "=== Waiting for snapshot artifact to become resolvable on Sonatype ===" + RESOLVED=0 + for attempt in $(seq 1 20); do + SNAPSHOT_VER=$(get_debug_jar_snapshot_version "${META_URL}") + if is_new_snapshot_version "${SNAPSHOT_VER}" "${PRE_PUBLISH_SNAPSHOT_VER}"; then + JAR_URL="https://central.sonatype.com/repository/maven-snapshots/com/datadoghq/ddprof/${LIB_VERSION}/ddprof-${SNAPSHOT_VER}-debug.jar" + if curl -fsSL "${CURL_TIMEOUT_OPTS[@]}" -o /dev/null "${JAR_URL}"; then + echo "Snapshot artifact resolvable: ${JAR_URL}" + RESOLVED=1 + break + fi + fi + echo "Attempt ${attempt}/20: newly published snapshot not yet resolvable, retrying in 15s..." + sleep 15 + done + if [ "${RESOLVED}" -ne 1 ]; then + echo "ERROR: newly published snapshot artifact still not resolvable after 20 attempts" >&2 + exit 1 + fi + fi fi diff --git a/.gitlab/scripts/includes.sh b/.gitlab/scripts/includes.sh index 676b4f29e4..dc40094d72 100755 --- a/.gitlab/scripts/includes.sh +++ b/.gitlab/scripts/includes.sh @@ -53,6 +53,12 @@ function setup_java_home() { echo "Using Java @ ${JAVA_HOME}" } +function is_new_snapshot_version() { + local candidate="$1" + local baseline="$2" + [ -n "${candidate}" ] && [ "${candidate}" != "${baseline}" ] +} + function collect_artifacts() { local target=$1 local artifact_type=$2 # "test" or "stresstest" diff --git a/.gitlab/scripts/tests/includes_test.sh b/.gitlab/scripts/tests/includes_test.sh new file mode 100644 index 0000000000..93a08a3d87 --- /dev/null +++ b/.gitlab/scripts/tests/includes_test.sh @@ -0,0 +1,54 @@ +#! /bin/bash +# Minimal, dependency-free unit tests for .gitlab/scripts/includes.sh helpers. +# Run with: bash .gitlab/scripts/tests/includes_test.sh + +set -eo pipefail + +HERE=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${HERE}/../includes.sh" + +FAILED=0 + +assert_true() { + local desc="$1" + shift + if "$@"; then + echo "PASS: ${desc}" + else + echo "FAIL: ${desc} — expected success" + FAILED=1 + fi +} + +assert_false() { + local desc="$1" + shift + if "$@"; then + echo "FAIL: ${desc} — expected failure" + FAILED=1 + else + echo "PASS: ${desc}" + fi +} + +# A freshly published snapshot version differs from the pre-publish baseline +# and must be accepted. +assert_true "differing candidate is accepted as new" \ + is_new_snapshot_version "20260101.120000-5" "20260101.100000-3" + +# The metadata still pointing at the pre-publish baseline is stale and must +# be rejected — this is the guard the retry loop in deploy.sh relies on. +assert_false "candidate matching baseline is rejected as stale" \ + is_new_snapshot_version "20260101.100000-3" "20260101.100000-3" + +# An empty candidate (metadata not resolvable yet) must be rejected +# regardless of the baseline. +assert_false "empty candidate is rejected" \ + is_new_snapshot_version "" "20260101.100000-3" + +# An empty baseline (no prior publish) plus any non-empty candidate must be +# accepted. +assert_true "non-empty candidate with empty baseline is accepted" \ + is_new_snapshot_version "20260101.120000-5" "" + +exit "${FAILED}"