Use canonical FFE fixtures

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "gitsubmodule"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/ci.yml

@@ -29,6 +29,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98
+        with:
+          submodules: recursive
       - name: Install runner
         uses: ./.github/actions/install_runner
       # force /bin/bash in order to test against bash 3.2 on macOS

.github/workflows/compute_libraries_and_scenarios.yml

@@ -47,12 +47,14 @@ jobs:
         uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98
         with:
           repository: 'DataDog/system-tests'
+          submodules: recursive
       - name: Checkout main branch
         uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98
         with:
           repository: 'DataDog/system-tests'
           ref: 'main'
           path: 'original'
+          submodules: recursive
       - name: Install runner
         uses: ./.github/actions/install_runner
       - name: Get scenario map

.github/workflows/lint.yml

@@ -9,6 +9,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98
+        with:
+          submodules: recursive
       - name: Install runner
         uses: ./.github/actions/install_runner
       - run: source venv/bin/activate

.github/workflows/run-exotics.yml

@@ -4,14 +4,14 @@ on:
   workflow_call:
 
 jobs:
-
   fuzzer:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout
       uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98
       with:
         repository: 'DataDog/system-tests'
+        submodules: recursive
     - name: Install runner
       uses: ./.github/actions/install_runner
     - name: Build

.github/workflows/run-parametric.yml

@@ -107,6 +107,7 @@ jobs:
         with:
           repository: DataDog/system-tests
           ref: ${{ steps.compute_ref.outputs.ref }}
+          submodules: recursive
       - name: Install runner
         uses: ./.github/actions/install_runner
       - name: Get binaries artifact

.github/workflows/system-tests.yml

@@ -204,6 +204,7 @@ jobs:
         with:
           repository: 'DataDog/system-tests'
           ref: ${{ steps.compute_ref.outputs.ref }}
+          submodules: recursive
       - name: Get binaries artifact
         if : ${{ needs.compute_parameters.outputs.binaries_artifact != '' }}
         uses: ./.github/actions/download_artifact

.gitlab-ci.yml

@@ -20,6 +20,7 @@ stages:
 
 variables:
     TEST: 1
+    GIT_SUBMODULE_STRATEGY: recursive
     # dd-repo-tools mirror_images.py (pinned). Used by the mirror_images_* jobs.
     MIRROR_IMAGES_URL: "https://binaries.ddbuild.io/dd-repo-tools/default/ca/385a670d8fafd57f8c3c9872823aa8dcd65b53cc/mirror_images.py"
     # Destination registry for mirrored CI images.
@@ -60,7 +61,8 @@ compute_pipeline:
           else
             echo "⚠️  SYSTEM_TESTS_REF variable is not set, skipping git checkout"
           fi
-           SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner
+          git submodule update --init --recursive
+          SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner
           source venv/bin/activate
       fi
     - python utils/scripts/compute-workflow-parameters.py nodejs -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "tests/parametric/test_ffe/ffe-system-test-data"]
+	path = tests/parametric/test_ffe/ffe-system-test-data
+	url = https://github.com/DataDog/ffe-system-test-data.git

manifests/dotnet.yml

@@ -847,6 +847,9 @@ manifest:
   tests/parametric/test_ffe/test_configuration_sources.py: missing_feature (FFL-2703 tracks .NET agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract)
   tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: '>=3.36.0'  # Modified by easy win activation script
   tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_flag_evaluation: missing_feature  # Created by easy win activation script
+  tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_null_targeting_key: missing_feature (FFE provider evaluation is not implemented in the .NET parametric test app)
+  ? tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_numeric_flag_with_integer_default
+  : missing_feature (FFE provider evaluation is not implemented in the .NET parametric test app)
   tests/parametric/test_ffe/test_span_enrichment.py: missing_feature
   tests/parametric/test_headers_b3.py::Test_Headers_B3::test_headers_b3_migrated_extract_valid: missing_feature (Need to remove b3=b3multi alias)
   tests/parametric/test_headers_b3.py::Test_Headers_B3::test_headers_b3_migrated_inject_valid: missing_feature (Need to remove b3=b3multi alias)

manifests/ruby.yml

@@ -2217,6 +2217,9 @@ manifest:
   tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: incomplete_test_app (The parametric test app does not emit restart span links or preserve baggage)
   tests/parametric/test_ffe/test_configuration_sources.py: missing_feature (FFL-2701 tracks Ruby agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract)
   tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v2.23.0
+  tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_null_targeting_key: missing_feature (Ruby OpenFeature does not support explicit null targeting keys)
+  ? tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_numeric_flag_with_integer_default
+  : missing_feature (Ruby OpenFeature treats integer-shaped numeric defaults as integer evaluations)
   tests/parametric/test_ffe/test_span_enrichment.py: missing_feature
   tests/parametric/test_headers_b3.py::Test_Headers_B3::test_headers_b3_extract_invalid:  # Easy win for all weblogs and version 2.27.0
     - declaration: missing_feature (Missing for 2.x)

tests/ffe/README.md

@@ -17,6 +17,31 @@ This directory contains system tests for the Feature Flags & Experimentation (FF
 ./run.sh FEATURE_FLAGGING_AND_EXPERIMENTATION --library <language>
 ```
 
+## Canonical parametric fixtures
+
+The parametric evaluation suite consumes the SDK-neutral fixtures from
+[`DataDog/ffe-system-test-data`](https://github.com/DataDog/ffe-system-test-data) through the
+`tests/parametric/test_ffe/ffe-system-test-data` git submodule. Initialize it after cloning or
+switching to a revision that changes the fixture pin:
+
+```bash
+git submodule update --init --recursive
+```
+
+Make shared evaluator fixture changes in `ffe-system-test-data`, not by adding copied JSON files to
+`system-tests`. After those changes merge, Dependabot proposes the weekly submodule update. To test
+an update manually before that PR is created:
+
+```bash
+git submodule update --remote tests/parametric/test_ffe/ffe-system-test-data
+TEST_LIBRARY=<language> ./run.sh PARAMETRIC \
+  tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation
+```
+
+GitHub and GitLab parametric CI initialize the submodule recursively. If a selected FFE test cannot
+find the canonical corpus, collection leaves an explicit failing case with the initialization
+command instead of silently dropping evaluator coverage.
+
 ---
 
 # Eval Metrics Implementation Guide
@@ -182,7 +207,7 @@ The tests use `interfaces.agent.get_metrics()` to retrieve metrics from the agen
 
 ### 1. Test returns `parse_error` instead of expected reason
 - Check UFC fixture format (especially `totalShards` placement)
-- Verify the fixture matches `flags-v1.json` format in dd-trace-py
+- Verify the fixture matches `ffe-system-test-data/ufc-config.json`
 
 ### 2. Test returns `static` instead of `split`
 - Need multiple variations with different shard ranges