ci: exercise downstream regex conformance

.github/workflows/downstream-conformance.yml

@@ -0,0 +1,212 @@
+name: Regex universality evidence
+
+on:
+  pull_request:
+    paths:
+      - .github/workflows/downstream-conformance.yml
+      - ci/run-regex-conformance.sh
+      - evaluation-cases/**
+      - regex-conformance/**
+      - ufc-config.json
+
+permissions:
+  contents: read
+
+concurrency:
+  group: regex-universality-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  downstream:
+    name: ${{ matrix.slug }} regex evidence
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - slug: dd-trace-go
+            repository: DataDog/dd-trace-go
+            runtime: go
+            submodule: openfeature/ffe-system-test-data
+            setup: ":"
+            contract: go test ./openfeature -run '^TestTargetingRegexConformance$' -count=1
+            enforce_contract: true
+          - slug: dd-trace-java
+            repository: DataDog/dd-trace-java
+            runtime: java
+            submodule: dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data
+            setup: ":"
+            contract: >-
+              ./gradlew
+              :products:feature-flagging:feature-flagging-api:cleanTest
+              :products:feature-flagging:feature-flagging-api:test
+              --tests datadog.trace.api.openfeature.DDEvaluatorTest.testRegexConformance
+              --no-daemon
+            enforce_contract: false
+          - slug: dd-trace-js
+            repository: DataDog/dd-trace-js
+            runtime: node
+            submodule: packages/dd-trace/test/openfeature/ffe-system-test-data
+            setup: >-
+              npm install -g bun@1.3.1 --prefer-offline --no-audit --no-fund &&
+              bun install --linker=hoisted --trust --network-concurrency 8
+            contract: npm run test:openfeature -- --grep 'targeting regex conformance'
+            # dd-trace-js uses native ECMAScript RegExp, not the fixture's RE2JS engine.
+            enforce_contract: false
+          - slug: dd-trace-py
+            repository: DataDog/dd-trace-py
+            runtime: python
+            submodule: tests/openfeature/ffe-system-test-data
+            # ddtest runs as uid 1000 in Docker while the Actions checkout is uid 1001.
+            setup: >-
+              mkdir -p .cache/cython .cache/pip .cache/sccache .riot &&
+              chmod -R a+rwX .
+            contract: >-
+              scripts/run-tests --venv 14fc413 -- -- -vv
+              -k targeting_regex_conformance
+            enforce_contract: true
+          - slug: dd-trace-rb
+            repository: DataDog/dd-trace-rb
+            runtime: ruby
+            submodule: spec/datadog/open_feature/ffe-system-test-data
+            setup: bundle install --jobs 4 && bundle exec rake compile
+            contract: >-
+              bundle exec rspec spec/datadog/open_feature/native_evaluator_spec.rb
+              -e 'targeting regex conformance'
+            enforce_contract: true
+          - slug: dd-trace-dotnet
+            repository: DataDog/dd-trace-dotnet
+            runtime: dotnet
+            submodule: tracer/test/Datadog.Trace.Tests/FeatureFlags/ffe-system-test-data
+            setup: >-
+              dotnet restore
+              tracer/test/Datadog.Trace.Tests/Datadog.Trace.Tests.csproj
+            contract: >-
+              dotnet test tracer/test/Datadog.Trace.Tests/Datadog.Trace.Tests.csproj
+              --framework net10.0 --no-restore
+              --filter 'FullyQualifiedName~Datadog.Trace.Tests.FeatureFlags.FeatureFlagsEvaluatorTests.RegexConformance'
+              --verbosity minimal
+            enforce_contract: false
+          - slug: dd-trace-php
+            repository: DataDog/dd-trace-php
+            runtime: php
+            submodule: tests/FeatureFlags/ffe-system-test-data
+            setup: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
+            contract: >-
+              make test_c TESTS=tests/ext/ffe/regex_conformance.phpt
+              MAX_TEST_PARALLELISM=1
+            enforce_contract: true
+          - slug: libdatadog
+            repository: DataDog/libdatadog
+            runtime: rust
+            submodule: libdd-ffe-test-suite/ffe-system-test-data
+            setup: rustup toolchain install 1.87.0 --profile minimal
+            contract: >-
+              cargo +1.87.0 test -p libdd-ffe-test-suite
+              --test targeting_regex_conformance
+              evaluates_targeting_regex_conformance_fixture -- --exact --nocapture
+            enforce_contract: true
+
+    steps:
+      - name: Check out proposed fixtures
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 0
+          path: fixtures
+          persist-credentials: false
+
+      - name: Check out ${{ matrix.slug }} conformance branch
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          repository: ${{ matrix.repository }}
+          ref: leo.romanovsky/ffe-regex-conformance-pr21
+          path: downstream
+          persist-credentials: false
+          submodules: recursive
+
+      - name: Set up Go
+        if: matrix.runtime == 'go'
+        uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
+        with:
+          go-version: stable
+          cache: false
+
+      - name: Set up Java
+        if: matrix.runtime == 'java'
+        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
+        with:
+          distribution: temurin
+          java-version: "21"
+          cache: gradle
+          cache-dependency-path: downstream/**/*.gradle*
+
+      - name: Set up Node.js
+        if: matrix.runtime == 'node'
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: "24"
+
+      - name: Set up Python
+        if: matrix.runtime == 'python'
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: "3.13"
+
+      - name: Set up Ruby
+        if: matrix.runtime == 'ruby'
+        uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
+        with:
+          ruby-version: "3.4"
+
+      - name: Set up .NET
+        if: matrix.runtime == 'dotnet'
+        uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
+        with:
+          dotnet-version: "10.0.x"
+
+      - name: Set up PHP
+        if: matrix.runtime == 'php'
+        uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # v2.37.2
+        with:
+          php-version: "8.3"
+          coverage: none
+
+      - name: Install ${{ matrix.slug }} dependencies
+        env:
+          SETUP_COMMAND: ${{ matrix.setup }}
+        working-directory: downstream
+        run: bash -lc "$SETUP_COMMAND"
+
+      - name: Run candidate regex contract
+        id: contract
+        env:
+          TEST_COMMAND: ${{ matrix.contract }}
+        run: |
+          fixtures/ci/run-regex-conformance.sh \
+            "$GITHUB_WORKSPACE/fixtures" \
+            "$GITHUB_WORKSPACE/downstream" \
+            "${{ matrix.submodule }}" \
+            "${{ github.event.pull_request.head.sha }}" \
+            "$RUNNER_TEMP/downstream-conformance/${{ matrix.slug }}/contract" \
+            -- \
+            bash -lc "$TEST_COMMAND"
+
+      - name: Report candidate contract failure
+        if: steps.contract.outputs.classification == 'candidate-failure'
+        run: |
+          echo "::warning title=${{ matrix.slug }} regex contract failure::The proposed fixture does not match the consumer branch. See the job summary and logs."
+
+      - name: Upload regex evidence logs
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: downstream-conformance-${{ matrix.slug }}
+          path: ${{ runner.temp }}/downstream-conformance/${{ matrix.slug }}/**/*.log
+          if-no-files-found: warn
+
+      - name: Enforce claimed engine observations
+        if: matrix.enforce_contract && steps.contract.outputs.classification == 'candidate-failure'
+        run: |
+          echo "${{ matrix.slug }} is an engine explicitly represented by the fixture, so this mismatch is blocking."
+          exit 1

ci/run-regex-conformance.sh

@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+usage() {
+  cat <<'EOF' >&2
+usage: run-regex-conformance.sh \
+  <fixture-repository> <downstream-repository> <fixture-submodule-path> \
+  <candidate-fixture-sha> <log-directory> -- <test-command> [args...]
+EOF
+}
+
+if [[ $# -lt 7 ]]; then
+  usage
+  exit 2
+fi
+
+fixture_repository=$1
+downstream_repository=$2
+fixture_submodule_path=$3
+candidate_fixture_sha=$4
+log_directory=$5
+shift 5
+
+if [[ $1 != "--" ]]; then
+  usage
+  exit 2
+fi
+shift
+
+fixture_repository=$(cd "$fixture_repository" && pwd)
+downstream_repository=$(cd "$downstream_repository" && pwd)
+fixture_checkout="$downstream_repository/$fixture_submodule_path"
+log_file="$log_directory/candidate.log"
+
+# pull_request workflows check out GitHub's synthetic merge commit. Its first
+# parent is the live stacked base (PR #21) and its second parent is this PR's
+# head. Resolve the exact fixture base without hard-coding Blake's branch SHA.
+if [[ -n ${GITHUB_BASE_REF:-} ]] &&
+  git -C "$fixture_repository" cat-file -e HEAD^2 2>/dev/null &&
+  [[ $(git -C "$fixture_repository" rev-parse HEAD^2) == "$candidate_fixture_sha" ]]; then
+  candidate_fixture_sha=$(git -C "$fixture_repository" rev-parse HEAD^1)
+fi
+
+git -C "$fixture_repository" cat-file -e "$candidate_fixture_sha^{commit}"
+downstream_sha=$(git -C "$downstream_repository" rev-parse HEAD)
+declared_fixture_sha=$(
+  git -C "$downstream_repository" ls-tree HEAD -- "$fixture_submodule_path" |
+    awk '{print $3}'
+)
+if [[ $declared_fixture_sha != "$candidate_fixture_sha" ]]; then
+  printf 'downstream branch %s pins fixture %s, expected PR #21 base %s\n' \
+    "$downstream_sha" "$declared_fixture_sha" "$candidate_fixture_sha" >&2
+  exit 1
+fi
+mkdir -p "$log_directory"
+
+git -C "$downstream_repository" submodule sync -- "$fixture_submodule_path"
+git -C "$downstream_repository" submodule update \
+  --init \
+  --depth 1 \
+  -- "$fixture_submodule_path"
+git -c protocol.file.allow=always \
+  -C "$fixture_checkout" \
+  fetch --no-tags "$fixture_repository" "$candidate_fixture_sha"
+git -C "$fixture_checkout" checkout --detach FETCH_HEAD
+
+status=0
+set +e
+(
+  cd "$downstream_repository"
+  "$@"
+) >"$log_file" 2>&1
+status=$?
+set -e
+
+printf '\n===== candidate fixture (%s) =====\n' "$candidate_fixture_sha"
+printf '===== downstream branch (%s) =====\n' "$downstream_sha"
+if [[ $status -ne 0 ]]; then
+  printf '%s\n' '----- first reported failure -----'
+  grep -m 1 -A 3 ' FAILED' "$log_file" || true
+fi
+printf '%s\n' '----- last 200 log lines -----'
+tail -n 200 "$log_file"
+printf '===== candidate exit code: %s =====\n' "$status"
+
+if [[ $status -eq 0 ]]; then
+  classification=candidate-pass
+  summary="The proposed regex fixture passes this downstream conformance test."
+else
+  classification=candidate-failure
+  summary="The proposed regex fixture disagrees with this downstream conformance test."
+fi
+
+printf '\nclassification=%s\n' "$classification"
+printf 'candidate_exit_code=%s\n' "$status"
+
+if [[ -n ${GITHUB_OUTPUT:-} ]]; then
+  {
+    printf 'classification=%s\n' "$classification"
+    printf 'candidate_exit_code=%s\n' "$status"
+  } >>"$GITHUB_OUTPUT"
+fi
+
+if [[ -n ${GITHUB_STEP_SUMMARY:-} ]]; then
+  {
+    printf '### Regex universality evidence\n\n'
+    printf -- '- Downstream branch: `%s`\n' "$downstream_sha"
+    printf -- '- PR #21 fixture base: `%s`\n\n' "$candidate_fixture_sha"
+    printf '| Candidate exit code | Classification |\n'
+    printf '| ---: | --- |\n'
+    # shellcheck disable=SC2016 # Backticks are Markdown, not shell syntax.
+    printf '| `%s` | **%s** |\n\n' "$status" "$classification"
+    printf '%s\n\n' "$summary"
+    printf '%s\n' 'This evidence-only PR is not intended for merge. Inspect the attached consumer log before deciding whether the fixture or implementation is wrong.'
+  } >>"$GITHUB_STEP_SUMMARY"
+fi
+
+# The workflow's artifact step uses if: always(), so preserve the actual unit
+# test exit code while still retaining complete logs for the evidence report.
+exit "$status"