diff --git a/.github/workflows/downstream-conformance.yml b/.github/workflows/downstream-conformance.yml new file mode 100644 index 0000000..671c6cc --- /dev/null +++ b/.github/workflows/downstream-conformance.yml @@ -0,0 +1,212 @@ +name: Regex universality evidence + +on: + pull_request: + paths: + - .github/workflows/downstream-conformance.yml + - ci/run-regex-conformance.sh + - evaluation-cases/** + - regex-conformance/** + - ufc-config.json + +permissions: + contents: read + +concurrency: + group: regex-universality-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + downstream: + name: ${{ matrix.slug }} regex evidence + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - slug: dd-trace-go + repository: DataDog/dd-trace-go + runtime: go + submodule: openfeature/ffe-system-test-data + setup: ":" + contract: go test ./openfeature -run '^TestTargetingRegexConformance$' -count=1 + enforce_contract: true + - slug: dd-trace-java + repository: DataDog/dd-trace-java + runtime: java + submodule: dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data + setup: ":" + contract: >- + ./gradlew + :products:feature-flagging:feature-flagging-api:cleanTest + :products:feature-flagging:feature-flagging-api:test + --tests datadog.trace.api.openfeature.DDEvaluatorTest.testRegexConformance + --no-daemon + enforce_contract: false + - slug: dd-trace-js + repository: DataDog/dd-trace-js + runtime: node + submodule: packages/dd-trace/test/openfeature/ffe-system-test-data + setup: >- + npm install -g bun@1.3.1 --prefer-offline --no-audit --no-fund && + bun install --linker=hoisted --trust --network-concurrency 8 + contract: npm run test:openfeature -- --grep 'targeting regex conformance' + # dd-trace-js uses native ECMAScript RegExp, not the fixture's RE2JS engine. + enforce_contract: false + - slug: dd-trace-py + repository: DataDog/dd-trace-py + runtime: python + submodule: tests/openfeature/ffe-system-test-data + # ddtest runs as uid 1000 in Docker while the Actions checkout is uid 1001. + setup: >- + mkdir -p .cache/cython .cache/pip .cache/sccache .riot && + chmod -R a+rwX . + contract: >- + scripts/run-tests --venv 14fc413 -- -- -vv + -k targeting_regex_conformance + enforce_contract: true + - slug: dd-trace-rb + repository: DataDog/dd-trace-rb + runtime: ruby + submodule: spec/datadog/open_feature/ffe-system-test-data + setup: bundle install --jobs 4 && bundle exec rake compile + contract: >- + bundle exec rspec spec/datadog/open_feature/native_evaluator_spec.rb + -e 'targeting regex conformance' + enforce_contract: true + - slug: dd-trace-dotnet + repository: DataDog/dd-trace-dotnet + runtime: dotnet + submodule: tracer/test/Datadog.Trace.Tests/FeatureFlags/ffe-system-test-data + setup: >- + dotnet restore + tracer/test/Datadog.Trace.Tests/Datadog.Trace.Tests.csproj + contract: >- + dotnet test tracer/test/Datadog.Trace.Tests/Datadog.Trace.Tests.csproj + --framework net10.0 --no-restore + --filter 'FullyQualifiedName~Datadog.Trace.Tests.FeatureFlags.FeatureFlagsEvaluatorTests.RegexConformance' + --verbosity minimal + enforce_contract: false + - slug: dd-trace-php + repository: DataDog/dd-trace-php + runtime: php + submodule: tests/FeatureFlags/ffe-system-test-data + setup: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev + contract: >- + make test_c TESTS=tests/ext/ffe/regex_conformance.phpt + MAX_TEST_PARALLELISM=1 + enforce_contract: true + - slug: libdatadog + repository: DataDog/libdatadog + runtime: rust + submodule: libdd-ffe-test-suite/ffe-system-test-data + setup: rustup toolchain install 1.87.0 --profile minimal + contract: >- + cargo +1.87.0 test -p libdd-ffe-test-suite + --test targeting_regex_conformance + evaluates_targeting_regex_conformance_fixture -- --exact --nocapture + enforce_contract: true + + steps: + - name: Check out proposed fixtures + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + path: fixtures + persist-credentials: false + + - name: Check out ${{ matrix.slug }} conformance branch + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: ${{ matrix.repository }} + ref: leo.romanovsky/ffe-regex-conformance-pr21 + path: downstream + persist-credentials: false + submodules: recursive + + - name: Set up Go + if: matrix.runtime == 'go' + uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 + with: + go-version: stable + cache: false + + - name: Set up Java + if: matrix.runtime == 'java' + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + with: + distribution: temurin + java-version: "21" + cache: gradle + cache-dependency-path: downstream/**/*.gradle* + + - name: Set up Node.js + if: matrix.runtime == 'node' + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "24" + + - name: Set up Python + if: matrix.runtime == 'python' + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.13" + + - name: Set up Ruby + if: matrix.runtime == 'ruby' + uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0 + with: + ruby-version: "3.4" + + - name: Set up .NET + if: matrix.runtime == 'dotnet' + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 + with: + dotnet-version: "10.0.x" + + - name: Set up PHP + if: matrix.runtime == 'php' + uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # v2.37.2 + with: + php-version: "8.3" + coverage: none + + - name: Install ${{ matrix.slug }} dependencies + env: + SETUP_COMMAND: ${{ matrix.setup }} + working-directory: downstream + run: bash -lc "$SETUP_COMMAND" + + - name: Run candidate regex contract + id: contract + env: + TEST_COMMAND: ${{ matrix.contract }} + run: | + fixtures/ci/run-regex-conformance.sh \ + "$GITHUB_WORKSPACE/fixtures" \ + "$GITHUB_WORKSPACE/downstream" \ + "${{ matrix.submodule }}" \ + "${{ github.event.pull_request.head.sha }}" \ + "$RUNNER_TEMP/downstream-conformance/${{ matrix.slug }}/contract" \ + -- \ + bash -lc "$TEST_COMMAND" + + - name: Report candidate contract failure + if: steps.contract.outputs.classification == 'candidate-failure' + run: | + echo "::warning title=${{ matrix.slug }} regex contract failure::The proposed fixture does not match the consumer branch. See the job summary and logs." + + - name: Upload regex evidence logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: downstream-conformance-${{ matrix.slug }} + path: ${{ runner.temp }}/downstream-conformance/${{ matrix.slug }}/**/*.log + if-no-files-found: warn + + - name: Enforce claimed engine observations + if: matrix.enforce_contract && steps.contract.outputs.classification == 'candidate-failure' + run: | + echo "${{ matrix.slug }} is an engine explicitly represented by the fixture, so this mismatch is blocking." + exit 1 diff --git a/ci/run-regex-conformance.sh b/ci/run-regex-conformance.sh new file mode 100755 index 0000000..7bfc05d --- /dev/null +++ b/ci/run-regex-conformance.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash + +set -euo pipefail + +usage() { + cat <<'EOF' >&2 +usage: run-regex-conformance.sh \ + \ + -- [args...] +EOF +} + +if [[ $# -lt 7 ]]; then + usage + exit 2 +fi + +fixture_repository=$1 +downstream_repository=$2 +fixture_submodule_path=$3 +candidate_fixture_sha=$4 +log_directory=$5 +shift 5 + +if [[ $1 != "--" ]]; then + usage + exit 2 +fi +shift + +fixture_repository=$(cd "$fixture_repository" && pwd) +downstream_repository=$(cd "$downstream_repository" && pwd) +fixture_checkout="$downstream_repository/$fixture_submodule_path" +log_file="$log_directory/candidate.log" + +# pull_request workflows check out GitHub's synthetic merge commit. Its first +# parent is the live stacked base (PR #21) and its second parent is this PR's +# head. Resolve the exact fixture base without hard-coding Blake's branch SHA. +if [[ -n ${GITHUB_BASE_REF:-} ]] && + git -C "$fixture_repository" cat-file -e HEAD^2 2>/dev/null && + [[ $(git -C "$fixture_repository" rev-parse HEAD^2) == "$candidate_fixture_sha" ]]; then + candidate_fixture_sha=$(git -C "$fixture_repository" rev-parse HEAD^1) +fi + +git -C "$fixture_repository" cat-file -e "$candidate_fixture_sha^{commit}" +downstream_sha=$(git -C "$downstream_repository" rev-parse HEAD) +declared_fixture_sha=$( + git -C "$downstream_repository" ls-tree HEAD -- "$fixture_submodule_path" | + awk '{print $3}' +) +if [[ $declared_fixture_sha != "$candidate_fixture_sha" ]]; then + printf 'downstream branch %s pins fixture %s, expected PR #21 base %s\n' \ + "$downstream_sha" "$declared_fixture_sha" "$candidate_fixture_sha" >&2 + exit 1 +fi +mkdir -p "$log_directory" + +git -C "$downstream_repository" submodule sync -- "$fixture_submodule_path" +git -C "$downstream_repository" submodule update \ + --init \ + --depth 1 \ + -- "$fixture_submodule_path" +git -c protocol.file.allow=always \ + -C "$fixture_checkout" \ + fetch --no-tags "$fixture_repository" "$candidate_fixture_sha" +git -C "$fixture_checkout" checkout --detach FETCH_HEAD + +status=0 +set +e +( + cd "$downstream_repository" + "$@" +) >"$log_file" 2>&1 +status=$? +set -e + +printf '\n===== candidate fixture (%s) =====\n' "$candidate_fixture_sha" +printf '===== downstream branch (%s) =====\n' "$downstream_sha" +if [[ $status -ne 0 ]]; then + printf '%s\n' '----- first reported failure -----' + grep -m 1 -A 3 ' FAILED' "$log_file" || true +fi +printf '%s\n' '----- last 200 log lines -----' +tail -n 200 "$log_file" +printf '===== candidate exit code: %s =====\n' "$status" + +if [[ $status -eq 0 ]]; then + classification=candidate-pass + summary="The proposed regex fixture passes this downstream conformance test." +else + classification=candidate-failure + summary="The proposed regex fixture disagrees with this downstream conformance test." +fi + +printf '\nclassification=%s\n' "$classification" +printf 'candidate_exit_code=%s\n' "$status" + +if [[ -n ${GITHUB_OUTPUT:-} ]]; then + { + printf 'classification=%s\n' "$classification" + printf 'candidate_exit_code=%s\n' "$status" + } >>"$GITHUB_OUTPUT" +fi + +if [[ -n ${GITHUB_STEP_SUMMARY:-} ]]; then + { + printf '### Regex universality evidence\n\n' + printf -- '- Downstream branch: `%s`\n' "$downstream_sha" + printf -- '- PR #21 fixture base: `%s`\n\n' "$candidate_fixture_sha" + printf '| Candidate exit code | Classification |\n' + printf '| ---: | --- |\n' + # shellcheck disable=SC2016 # Backticks are Markdown, not shell syntax. + printf '| `%s` | **%s** |\n\n' "$status" "$classification" + printf '%s\n\n' "$summary" + printf '%s\n' 'This evidence-only PR is not intended for merge. Inspect the attached consumer log before deciding whether the fixture or implementation is wrong.' + } >>"$GITHUB_STEP_SUMMARY" +fi + +# The workflow's artifact step uses if: always(), so preserve the actual unit +# test exit code while still retaining complete logs for the evidence report. +exit "$status"