diff --git a/.github/workflows/update-ffe-fixtures.yml b/.github/workflows/update-ffe-fixtures.yml new file mode 100644 index 00000000000..360caba4b57 --- /dev/null +++ b/.github/workflows/update-ffe-fixtures.yml @@ -0,0 +1,73 @@ +name: Update FFE fixtures + +on: + workflow_dispatch: + inputs: + fixture_ref: + description: Branch, tag, or commit from DataDog/ffe-system-test-data + required: true + default: main + type: string + schedule: + - cron: "0 0 * * 0" + +concurrency: + group: update-ffe-fixtures + cancel-in-progress: false + +jobs: + update-ffe-fixtures: + runs-on: ubuntu-latest + permissions: + id-token: write + + steps: + - uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 + with: + persist-credentials: false + + - name: Update fixture snapshot + id: fixtures + env: + FFE_FIXTURE_REF: ${{ inputs.fixture_ref || 'main' }} + run: python3 utils/scripts/update_ffe_fixtures.py --ref "$FFE_FIXTURE_REF" + + - uses: DataDog/dd-octo-sts-action@96a25462dbcb10ebf0bfd6e2ccc917d2ab235b9a # v1.0.4 + if: steps.fixtures.outputs.changed == 'true' + id: octo-sts + with: + scope: DataDog/system-tests + policy: self.update-ffe-fixtures.create-pr + + - name: Create Pull Request + if: steps.fixtures.outputs.changed == 'true' + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + with: + token: ${{ steps.octo-sts.outputs.token }} + branch: actions/update-ffe-fixtures + base: main + commit-message: "test(ffe): update canonical fixtures" + title: "test(ffe): update canonical fixtures" + draft: true + sign-commits: true + add-paths: tests/parametric/test_ffe/ffe-system-test-data + body: | + ## Motivation + + Keep system-tests synchronized with the canonical FFE evaluator fixtures. New fixtures may intentionally make tests fail when they expose downstream evaluator bugs that should be addressed before merging the update. + + Canonical source: https://github.com/DataDog/ffe-system-test-data + Source commit: ${{ steps.fixtures.outputs.source_commit }} + + ## Changes + + - Refresh the checked-in FFE fixture snapshot. + - Load ${{ steps.fixtures.outputs.fixture_count }} canonical evaluation cases. + + ## Decisions + + The snapshot remains checked in so local and CI tests work from a normal clone. The update script applies a client-owned disallow list, and the workflow opens a signed draft PR only when fixture contents change. + + ## Verification + + The existing FFE parametric tests run against the refreshed snapshot. Treat new failures as useful downstream conformance signals and fix or classify them in this update PR before merging it. diff --git a/tests/ffe/README.md b/tests/ffe/README.md index 9aca2fb8dba..127c9e10ba4 100644 --- a/tests/ffe/README.md +++ b/tests/ffe/README.md @@ -17,6 +17,23 @@ This directory contains system tests for the Feature Flags & Experimentation (FF ./run.sh FEATURE_FLAGGING_AND_EXPERIMENTATION --library ``` +## Canonical parametric fixtures + +The parametric evaluation suite reads the checked-in snapshot under +`tests/parametric/test_ffe/ffe-system-test-data`. The canonical source is +[`DataDog/ffe-system-test-data`](https://github.com/DataDog/ffe-system-test-data). + +Make shared evaluator fixture changes in the canonical repository first. A weekly workflow applies +a client-owned disallow list, copies the fixture contents into `system-tests`, and opens a signed +draft PR when the snapshot changes. Refresh a branch manually with: + +```bash +python3 utils/scripts/update_ffe_fixtures.py --ref +``` + +The files are checked in intentionally, so local and CI tests work from an ordinary clone without +submodule initialization or network access. + --- # Eval Metrics Implementation Guide diff --git a/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md b/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md new file mode 100644 index 00000000000..83dbd780b41 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md @@ -0,0 +1,10 @@ +# FFE Fixture Snapshot + +These files are copied from the canonical FFE fixture repository. + +Canonical source: https://github.com/DataDog/ffe-system-test-data +Source commit: ea8b5cc5ce335109f11f3efbc5fd608f98a3ca54 + +Do not edit these fixtures directly in system-tests. Add or update shared FFE behavior in ffe-system-test-data first, then refresh this snapshot. + +The weekly update workflow runs `python3 utils/scripts/update_ffe_fixtures.py` and opens a signed draft test PR only when the allowed fixture contents change. diff --git a/tests/parametric/test_ffe/test-case-boolean-false-assignment.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-false-assignment.json similarity index 74% rename from tests/parametric/test_ffe/test-case-boolean-false-assignment.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-false-assignment.json index cdd2b0a513d..ab718f5bee1 100644 --- a/tests/parametric/test_ffe/test-case-boolean-false-assignment.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-false-assignment.json @@ -1,38 +1,41 @@ [ { - "flag": "boolean-false-assignment", - "variationType": "BOOLEAN", - "defaultValue": true, - "targetingKey": "alice", "attributes": { "should_disable_feature": true }, + "defaultValue": true, + "flag": "boolean-false-assignment", "result": { + "reason": "TARGETING_MATCH", "value": false - } + }, + "targetingKey": "alice", + "variationType": "BOOLEAN" }, { - "flag": "boolean-false-assignment", - "variationType": "BOOLEAN", - "defaultValue": true, - "targetingKey": "bob", "attributes": { "should_disable_feature": false }, + "defaultValue": true, + "flag": "boolean-false-assignment", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "bob", + "variationType": "BOOLEAN" }, { - "flag": "boolean-false-assignment", - "variationType": "BOOLEAN", - "defaultValue": true, - "targetingKey": "charlie", "attributes": { "unknown_attribute": "value" }, + "defaultValue": true, + "flag": "boolean-false-assignment", "result": { + "reason": "DEFAULT", "value": true - } + }, + "targetingKey": "charlie", + "variationType": "BOOLEAN" } -] \ No newline at end of file +] diff --git a/tests/parametric/test_ffe/test-case-boolean-one-of-matches.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-one-of-matches.json similarity index 72% rename from tests/parametric/test_ffe/test-case-boolean-one-of-matches.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-one-of-matches.json index f616614d936..392ec97ec0f 100644 --- a/tests/parametric/test_ffe/test-case-boolean-one-of-matches.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-one-of-matches.json @@ -1,192 +1,208 @@ [ { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "alice", "attributes": { "one_of_flag": true }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 1 - } + }, + "targetingKey": "alice", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "bob", "attributes": { "one_of_flag": false }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "bob", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "charlie", "attributes": { "one_of_flag": "True" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "charlie", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "derek", "attributes": { "matches_flag": true }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 2 - } + }, + "targetingKey": "derek", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "erica", "attributes": { "matches_flag": false }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "erica", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "frank", "attributes": { "not_matches_flag": false }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "frank", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "george", "attributes": { "not_matches_flag": true }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 4 - } + }, + "targetingKey": "george", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "haley", "attributes": { "not_matches_flag": "False" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 4 - } + }, + "targetingKey": "haley", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "ivy", "attributes": { "not_one_of_flag": true }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "ivy", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "julia", "attributes": { "not_one_of_flag": false }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "julia", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "kim", "attributes": { "not_one_of_flag": "False" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "kim", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "lucas", "attributes": { "not_one_of_flag": "true" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "lucas", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "mike", "attributes": { "not_one_of_flag": "false" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "mike", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "nicole", "attributes": { "null_flag": "null" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 5 - } + }, + "targetingKey": "nicole", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "owen", "attributes": { "null_flag": null }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "owen", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "pete", "attributes": {}, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "pete", + "variationType": "INTEGER" } ] diff --git a/tests/parametric/test_ffe/test-case-comparator-operator-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-comparator-operator-flag.json similarity index 68% rename from tests/parametric/test_ffe/test-case-comparator-operator-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-comparator-operator-flag.json index 2d94f30eb30..e4daa2de7c2 100644 --- a/tests/parametric/test_ffe/test-case-comparator-operator-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-comparator-operator-flag.json @@ -1,64 +1,69 @@ [ { - "flag": "comparator-operator-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "alice", "attributes": { - "size": 5, - "country": "US" + "country": "US", + "size": 5 }, + "defaultValue": "unknown", + "flag": "comparator-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "small" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "comparator-operator-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "bob", "attributes": { - "size": 10, - "country": "Canada" + "country": "Canada", + "size": 10 }, + "defaultValue": "unknown", + "flag": "comparator-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "medium" - } + }, + "targetingKey": "bob", + "variationType": "STRING" }, { - "flag": "comparator-operator-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "charlie", "attributes": { "size": 25 }, + "defaultValue": "unknown", + "flag": "comparator-operator-test", "result": { + "reason": "DEFAULT", "value": "unknown" - } + }, + "targetingKey": "charlie", + "variationType": "STRING" }, { - "flag": "comparator-operator-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "david", "attributes": { "size": 26 }, + "defaultValue": "unknown", + "flag": "comparator-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "large" - } + }, + "targetingKey": "david", + "variationType": "STRING" }, { - "flag": "comparator-operator-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "elize", "attributes": { "country": "UK" }, + "defaultValue": "unknown", + "flag": "comparator-operator-test", "result": { + "reason": "DEFAULT", "value": "unknown" - } + }, + "targetingKey": "elize", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/test-case-disabled-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-disabled-flag.json similarity index 59% rename from tests/parametric/test_ffe/test-case-disabled-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-disabled-flag.json index 0da79189ade..208e5c96943 100644 --- a/tests/parametric/test_ffe/test-case-disabled-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-disabled-flag.json @@ -1,40 +1,43 @@ [ { - "flag": "disabled_flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "alice", "attributes": { - "email": "alice@mycompany.com", - "country": "US" + "country": "US", + "email": "alice@mycompany.com" }, + "defaultValue": 0, + "flag": "disabled_flag", "result": { + "reason": "DISABLED", "value": 0 - } + }, + "targetingKey": "alice", + "variationType": "INTEGER" }, { - "flag": "disabled_flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "bob", "attributes": { - "email": "bob@example.com", - "country": "Canada" + "country": "Canada", + "email": "bob@example.com" }, + "defaultValue": 0, + "flag": "disabled_flag", "result": { + "reason": "DISABLED", "value": 0 - } + }, + "targetingKey": "bob", + "variationType": "INTEGER" }, { - "flag": "disabled_flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": 0, + "flag": "disabled_flag", "result": { + "reason": "DISABLED", "value": 0 - } + }, + "targetingKey": "charlie", + "variationType": "INTEGER" } ] diff --git a/tests/parametric/test_ffe/test-case-empty-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-flag.json similarity index 63% rename from tests/parametric/test_ffe/test-case-empty-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-flag.json index 52100b1fe47..18ee5381fdc 100644 --- a/tests/parametric/test_ffe/test-case-empty-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-flag.json @@ -1,40 +1,43 @@ [ { - "flag": "empty_flag", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "alice", "attributes": { - "email": "alice@mycompany.com", - "country": "US" + "country": "US", + "email": "alice@mycompany.com" }, + "defaultValue": "default_value", + "flag": "empty_flag", "result": { + "reason": "DEFAULT", "value": "default_value" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "empty_flag", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "bob", "attributes": { - "email": "bob@example.com", - "country": "Canada" + "country": "Canada", + "email": "bob@example.com" }, + "defaultValue": "default_value", + "flag": "empty_flag", "result": { + "reason": "DEFAULT", "value": "default_value" - } + }, + "targetingKey": "bob", + "variationType": "STRING" }, { - "flag": "empty_flag", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": "default_value", + "flag": "empty_flag", "result": { + "reason": "DEFAULT", "value": "default_value" - } + }, + "targetingKey": "charlie", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/test-case-empty-string-variation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-string-variation.json similarity index 75% rename from tests/parametric/test_ffe/test-case-empty-string-variation.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-string-variation.json index 1af431f3db4..f1fa3994af2 100644 --- a/tests/parametric/test_ffe/test-case-empty-string-variation.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-string-variation.json @@ -1,38 +1,41 @@ [ { - "flag": "empty-string-variation", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "empty_user", "attributes": { "content_type": "minimal" }, + "defaultValue": "default_value", + "flag": "empty-string-variation", "result": { + "reason": "TARGETING_MATCH", "value": "" - } + }, + "targetingKey": "empty_user", + "variationType": "STRING" }, { - "flag": "empty-string-variation", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "full_user", "attributes": { "content_type": "full" }, + "defaultValue": "default_value", + "flag": "empty-string-variation", "result": { + "reason": "TARGETING_MATCH", "value": "detailed_content" - } + }, + "targetingKey": "full_user", + "variationType": "STRING" }, { - "flag": "empty-string-variation", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "default_user", "attributes": { "content_type": "unknown" }, + "defaultValue": "default_value", + "flag": "empty-string-variation", "result": { + "reason": "DEFAULT", "value": "default_value" - } + }, + "targetingKey": "default_user", + "variationType": "STRING" } -] \ No newline at end of file +] diff --git a/tests/parametric/test_ffe/test-case-falsy-value-assignments.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-falsy-value-assignments.json similarity index 73% rename from tests/parametric/test_ffe/test-case-falsy-value-assignments.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-falsy-value-assignments.json index 4ad259620d8..6a8cee51bcc 100644 --- a/tests/parametric/test_ffe/test-case-falsy-value-assignments.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-falsy-value-assignments.json @@ -1,38 +1,41 @@ [ { - "flag": "falsy-value-assignments", - "variationType": "INTEGER", - "defaultValue": 999, - "targetingKey": "zero_user", "attributes": { "plan_tier": "free" }, + "defaultValue": 999, + "flag": "falsy-value-assignments", "result": { + "reason": "TARGETING_MATCH", "value": 0 - } + }, + "targetingKey": "zero_user", + "variationType": "INTEGER" }, { - "flag": "falsy-value-assignments", - "variationType": "INTEGER", - "defaultValue": 999, - "targetingKey": "premium_user", "attributes": { "plan_tier": "premium" }, + "defaultValue": 999, + "flag": "falsy-value-assignments", "result": { + "reason": "TARGETING_MATCH", "value": 100 - } + }, + "targetingKey": "premium_user", + "variationType": "INTEGER" }, { - "flag": "falsy-value-assignments", - "variationType": "INTEGER", - "defaultValue": 999, - "targetingKey": "unknown_user", "attributes": { "plan_tier": "trial" }, + "defaultValue": 999, + "flag": "falsy-value-assignments", "result": { + "reason": "DEFAULT", "value": 999 - } + }, + "targetingKey": "unknown_user", + "variationType": "INTEGER" } -] \ No newline at end of file +] diff --git a/tests/parametric/test_ffe/test-case-flag-with-empty-string.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-flag-with-empty-string.json similarity index 72% rename from tests/parametric/test_ffe/test-case-flag-with-empty-string.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-flag-with-empty-string.json index 0bf562efa35..6a5d9cd62fd 100644 --- a/tests/parametric/test_ffe/test-case-flag-with-empty-string.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-flag-with-empty-string.json @@ -1,24 +1,26 @@ [ { - "flag": "empty_string_flag", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "alice", "attributes": { "country": "US" }, + "defaultValue": "default", + "flag": "empty_string_flag", "result": { + "reason": "TARGETING_MATCH", "value": "" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "empty_string_flag", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "bob", "attributes": {}, + "defaultValue": "default", + "flag": "empty_string_flag", "result": { + "reason": "SPLIT", "value": "non_empty" - } + }, + "targetingKey": "bob", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/test-case-integer-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-integer-flag.json similarity index 66% rename from tests/parametric/test_ffe/test-case-integer-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-integer-flag.json index 5e930885b44..8ff11aec1fc 100644 --- a/tests/parametric/test_ffe/test-case-integer-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-integer-flag.json @@ -1,244 +1,267 @@ [ { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "alice", "attributes": { - "email": "alice@mycompany.com", - "country": "US" + "country": "US", + "email": "alice@mycompany.com" }, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "alice", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "bob", "attributes": { - "email": "bob@example.com", - "country": "Canada" + "country": "Canada", + "email": "bob@example.com" }, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "bob", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "charlie", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "debra", "attributes": { - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com" }, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "debra", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "1", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "1", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "2", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "2", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "3", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "3", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "4", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "4", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "5", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "5", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "6", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "6", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "7", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "7", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "8", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "8", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "9", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "9", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "10", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "10", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "11", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "11", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "12", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "12", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "13", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "13", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "14", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "14", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "15", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "15", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "16", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "16", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "17", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "17", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "18", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "18", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "19", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "19", + "variationType": "INTEGER" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-condition-operands.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-condition-operands.json new file mode 100644 index 00000000000..be226381455 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-condition-operands.json @@ -0,0 +1,59 @@ +[ + { + "description": "GT requires a numeric configured operand. A flag with a string operand must return a parse error rather than falling through to the catch-all allocation.", + "attributes": { + "age": 42 + }, + "defaultValue": "default", + "flag": "invalid-gt-condition-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "invalid-gt", + "variationType": "STRING" + }, + { + "description": "ONE_OF requires an array configured operand. A flag with a scalar operand must return a parse error rather than falling through to the catch-all allocation.", + "attributes": { + "role": "admin" + }, + "defaultValue": "default", + "flag": "invalid-one-of-condition-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "invalid-one-of", + "variationType": "STRING" + }, + { + "description": "IS_NULL requires a boolean configured operand. A flag with a string operand must return a parse error rather than falling through to the catch-all allocation.", + "attributes": { + "deleted_at": null + }, + "defaultValue": "default", + "flag": "invalid-is-null-condition-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "invalid-is-null", + "variationType": "STRING" + }, + { + "description": "A valid flag in the same configuration must still evaluate after malformed condition operands are isolated.", + "attributes": {}, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "valid-neighbor", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-regex-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-regex-isolation.json new file mode 100644 index 00000000000..3ad596cbf06 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-regex-isolation.json @@ -0,0 +1,17 @@ +[ + { + "description": "The malformed flag uses the invalid regex *@datadoghq.com before a catch-all allocation. SDKs must return a parse error without poisoning the full configuration.", + "attributes": { + "email": "harish@datadoghq.com" + }, + "defaultValue": "default", + "flag": "invalid-regex-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "harish", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-shard-bounds-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-shard-bounds-isolation.json new file mode 100644 index 00000000000..ca8553f099c --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-shard-bounds-isolation.json @@ -0,0 +1,40 @@ +[ + { + "description": "The malformed flag has totalShards above the unsigned 32-bit range. SDKs must return a parse error without poisoning the full configuration.", + "attributes": {}, + "defaultValue": "default", + "flag": "overflow-total-shards-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "description": "The malformed flag has a negative shard range start. SDKs must return a parse error without poisoning the full configuration.", + "attributes": {}, + "defaultValue": "default", + "flag": "negative-shard-range-start-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "description": "A valid flag in the same config must still evaluate after invalid shard-bound flags are skipped.", + "attributes": {}, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/test-case-kill-switch-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-kill-switch-flag.json similarity index 67% rename from tests/parametric/test_ffe/test-case-kill-switch-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-kill-switch-flag.json index 8f34a1bc3af..3af700dc711 100644 --- a/tests/parametric/test_ffe/test-case-kill-switch-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-kill-switch-flag.json @@ -1,290 +1,314 @@ [ { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "alice", "attributes": { - "email": "alice@mycompany.com", - "country": "US" + "country": "US", + "email": "alice@mycompany.com" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "alice", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "bob", "attributes": { - "email": "bob@example.com", - "country": "Canada" + "country": "Canada", + "email": "bob@example.com" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "bob", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "barbara", "attributes": { - "email": "barbara@example.com", - "country": "canada" + "country": "canada", + "email": "barbara@example.com" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "barbara", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "charlie", "attributes": { "age": 40 }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "charlie", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "debra", "attributes": { - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "debra", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "1", "attributes": {}, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "1", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "2", "attributes": { "country": "Mexico" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "2", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "3", "attributes": { - "country": "UK", - "age": 50 + "age": 50, + "country": "UK" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "3", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "4", "attributes": { "country": "Germany" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "4", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "5", "attributes": { "country": "Germany" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "5", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "6", "attributes": { "country": "Germany" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "6", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "7", "attributes": { - "country": "US", - "age": 12 + "age": 12, + "country": "US" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "7", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "8", "attributes": { - "country": "Italy", - "age": 60 + "age": 60, + "country": "Italy" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "8", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "9", "attributes": { "email": "email@email.com" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "9", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "10", "attributes": {}, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "10", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "11", "attributes": {}, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "11", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "12", "attributes": { "country": "US" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "12", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "13", "attributes": { "country": "Canada" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "13", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "14", "attributes": {}, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "14", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "15", "attributes": { "country": "Denmark" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "15", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "16", "attributes": { "country": "Norway" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "16", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "17", "attributes": { "country": "UK" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "17", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "18", "attributes": { "country": "UK" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "18", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "19", "attributes": { "country": "UK" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "19", + "variationType": "BOOLEAN" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-malformed-flag-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-malformed-flag-isolation.json new file mode 100644 index 00000000000..41672357c23 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-malformed-flag-isolation.json @@ -0,0 +1,27 @@ +[ + { + "description": "The malformed flag has allocations encoded as a string instead of an array. SDKs must return a parse error without poisoning the full configuration.", + "attributes": {}, + "defaultValue": "default", + "flag": "malformed-allocations-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "description": "A valid flag in the same config must still evaluate after the malformed flag is skipped.", + "attributes": {}, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-microsecond-date-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-microsecond-date-flag.json new file mode 100644 index 00000000000..3dba337ad02 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-microsecond-date-flag.json @@ -0,0 +1,39 @@ +[ + { + "attributes": {}, + "defaultValue": "unknown", + "flag": "microsecond-date-test", + "result": { + "reason": "DEFAULT", + "value": "active" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "attributes": { + "country": "US" + }, + "defaultValue": "unknown", + "flag": "microsecond-date-test", + "result": { + "reason": "DEFAULT", + "value": "active" + }, + "targetingKey": "bob", + "variationType": "STRING" + }, + { + "attributes": { + "version": "1.0.0" + }, + "defaultValue": "unknown", + "flag": "microsecond-date-test", + "result": { + "reason": "DEFAULT", + "value": "active" + }, + "targetingKey": "charlie", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-missing-split-shards-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-missing-split-shards-isolation.json new file mode 100644 index 00000000000..d6dfeb180ea --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-missing-split-shards-isolation.json @@ -0,0 +1,27 @@ +[ + { + "description": "The malformed flag has a split that omits required shards. SDKs must return a parse error without poisoning the full configuration.", + "attributes": {}, + "defaultValue": "default", + "flag": "missing-split-shards-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "description": "A valid flag in the same config must still evaluate after the missing-shards flag is skipped.", + "attributes": {}, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/test-case-new-user-onboarding-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-new-user-onboarding-flag.json similarity index 69% rename from tests/parametric/test_ffe/test-case-new-user-onboarding-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-new-user-onboarding-flag.json index 8ed3e2a024f..bb08523de4e 100644 --- a/tests/parametric/test_ffe/test-case-new-user-onboarding-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-new-user-onboarding-flag.json @@ -1,318 +1,344 @@ [ { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "alice", "attributes": { - "email": "alice@mycompany.com", - "country": "US" + "country": "US", + "email": "alice@mycompany.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "green" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "bob", "attributes": { - "email": "bob@example.com", - "country": "Canada" + "country": "Canada", + "email": "bob@example.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "bob", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "charlie", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "debra", "attributes": { - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "debra", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "zach", "attributes": { - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "purple" - } + }, + "targetingKey": "zach", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "zach", "attributes": { - "id": "override-id", - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com", + "id": "override-id" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "zach", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "Zach", "attributes": { - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "Zach", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "1", "attributes": {}, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "1", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "2", "attributes": { "country": "Mexico" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "2", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "3", "attributes": { - "country": "UK", - "age": 33 + "age": 33, + "country": "UK" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "control" - } + }, + "targetingKey": "3", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "4", "attributes": { "country": "Germany" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "red" - } + }, + "targetingKey": "4", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "5", "attributes": { "country": "Germany" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "yellow" - } + }, + "targetingKey": "5", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "6", "attributes": { "country": "Germany" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "yellow" - } + }, + "targetingKey": "6", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "7", "attributes": { "country": "US" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "7", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "8", "attributes": { "country": "Italy" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "red" - } + }, + "targetingKey": "8", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "9", "attributes": { "email": "email@email.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "9", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "10", "attributes": {}, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "10", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "11", "attributes": {}, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "11", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "12", "attributes": { "country": "US" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "12", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "13", "attributes": { "country": "Canada" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "13", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "14", "attributes": {}, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "14", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "15", "attributes": { "country": "Denmark" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "yellow" - } + }, + "targetingKey": "15", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "16", "attributes": { "country": "Norway" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "control" - } + }, + "targetingKey": "16", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "17", "attributes": { "country": "UK" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "control" - } + }, + "targetingKey": "17", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "18", "attributes": { "country": "UK" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "18", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "19", "attributes": { "country": "UK" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "red" - } + }, + "targetingKey": "19", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/test-case-no-allocations-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-no-allocations-flag.json similarity index 68% rename from tests/parametric/test_ffe/test-case-no-allocations-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-no-allocations-flag.json index 132c39db32a..7d985b8efd4 100644 --- a/tests/parametric/test_ffe/test-case-no-allocations-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-no-allocations-flag.json @@ -1,52 +1,55 @@ [ { - "flag": "no_allocations_flag", - "variationType": "JSON", + "attributes": { + "country": "US", + "email": "alice@mycompany.com" + }, "defaultValue": { "hello": "world" }, - "targetingKey": "alice", - "attributes": { - "email": "alice@mycompany.com", - "country": "US" - }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "hello": "world" } - } + }, + "targetingKey": "alice", + "variationType": "JSON" }, { - "flag": "no_allocations_flag", - "variationType": "JSON", + "attributes": { + "country": "Canada", + "email": "bob@example.com" + }, "defaultValue": { "hello": "world" }, - "targetingKey": "bob", - "attributes": { - "email": "bob@example.com", - "country": "Canada" - }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "hello": "world" } - } + }, + "targetingKey": "bob", + "variationType": "JSON" }, { - "flag": "no_allocations_flag", - "variationType": "JSON", - "defaultValue": { - "hello": "world" - }, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": { + "hello": "world" + }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "hello": "world" } - } + }, + "targetingKey": "charlie", + "variationType": "JSON" } ] diff --git a/tests/parametric/test_ffe/test-case-null-operator-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-operator-flag.json similarity index 67% rename from tests/parametric/test_ffe/test-case-null-operator-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-operator-flag.json index 09e5d78dacd..8063ff8fee3 100644 --- a/tests/parametric/test_ffe/test-case-null-operator-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-operator-flag.json @@ -1,64 +1,69 @@ [ { - "flag": "null-operator-test", - "variationType": "STRING", - "defaultValue": "default-null", - "targetingKey": "alice", "attributes": { - "size": 5, - "country": "US" + "country": "US", + "size": 5 }, + "defaultValue": "default-null", + "flag": "null-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "old" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "null-operator-test", - "variationType": "STRING", - "defaultValue": "default-null", - "targetingKey": "bob", "attributes": { - "size": 10, - "country": "Canada" + "country": "Canada", + "size": 10 }, + "defaultValue": "default-null", + "flag": "null-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "new" - } + }, + "targetingKey": "bob", + "variationType": "STRING" }, { - "flag": "null-operator-test", - "variationType": "STRING", - "defaultValue": "default-null", - "targetingKey": "charlie", "attributes": { "size": null }, + "defaultValue": "default-null", + "flag": "null-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "old" - } + }, + "targetingKey": "charlie", + "variationType": "STRING" }, { - "flag": "null-operator-test", - "variationType": "STRING", - "defaultValue": "default-null", - "targetingKey": "david", "attributes": { "size": 26 }, + "defaultValue": "default-null", + "flag": "null-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "new" - } + }, + "targetingKey": "david", + "variationType": "STRING" }, { - "flag": "null-operator-test", - "variationType": "STRING", - "defaultValue": "default-null", - "targetingKey": "elize", "attributes": { "country": "UK" }, + "defaultValue": "default-null", + "flag": "null-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "old" - } + }, + "targetingKey": "elize", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-shard-range-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-shard-range-isolation.json new file mode 100644 index 00000000000..15f4af5a77d --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-shard-range-isolation.json @@ -0,0 +1,15 @@ +[ + { + "description": "The malformed flag has a null shard range before a catch-all allocation. SDKs must return a parse error without poisoning the full configuration.", + "attributes": {}, + "defaultValue": "default", + "flag": "null-shard-range-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-targeting-key.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-targeting-key.json new file mode 100644 index 00000000000..a2226d1b21e --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-targeting-key.json @@ -0,0 +1,40 @@ +[ + { + "attributes": {}, + "defaultValue": 42, + "flag": "numeric_flag", + "result": { + "reason": "STATIC", + "value": 3.1415926 + }, + "targetingKey": null, + "variationType": "NUMERIC" + }, + { + "attributes": { + "country": "France" + }, + "defaultValue": "default", + "flag": "new-user-onboarding", + "result": { + "reason": "ERROR", + "value": "default" + }, + "targetingKey": null, + "variationType": "STRING" + }, + { + "attributes": { + "country": "France", + "email": "null@mycompany.com" + }, + "defaultValue": "default", + "flag": "new-user-onboarding", + "result": { + "reason": "TARGETING_MATCH", + "value": "green" + }, + "targetingKey": null, + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/test-case-numeric-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-flag.json similarity index 52% rename from tests/parametric/test_ffe/test-case-numeric-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-flag.json index 757f0f70e55..cc52dcdff4d 100644 --- a/tests/parametric/test_ffe/test-case-numeric-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-flag.json @@ -1,40 +1,43 @@ [ { - "flag": "numeric_flag", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "alice", "attributes": { - "email": "alice@mycompany.com", - "country": "US" + "country": "US", + "email": "alice@mycompany.com" }, + "defaultValue": 0, + "flag": "numeric_flag", "result": { + "reason": "STATIC", "value": 3.1415926 - } + }, + "targetingKey": "alice", + "variationType": "NUMERIC" }, { - "flag": "numeric_flag", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "bob", "attributes": { - "email": "bob@example.com", - "country": "Canada" + "country": "Canada", + "email": "bob@example.com" }, + "defaultValue": 0, + "flag": "numeric_flag", "result": { + "reason": "STATIC", "value": 3.1415926 - } + }, + "targetingKey": "bob", + "variationType": "NUMERIC" }, { - "flag": "numeric_flag", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": 0, + "flag": "numeric_flag", "result": { + "reason": "STATIC", "value": 3.1415926 - } + }, + "targetingKey": "charlie", + "variationType": "NUMERIC" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of-default.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of-default.json new file mode 100644 index 00000000000..d8a7e8ba092 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of-default.json @@ -0,0 +1,13 @@ +[ + { + "attributes": {}, + "defaultValue": 0, + "flag": "numeric-one-of", + "result": { + "reason": "DEFAULT", + "value": 0 + }, + "targetingKey": "race-default", + "variationType": "INTEGER" + } +] diff --git a/tests/parametric/test_ffe/test-case-numeric-one-of.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of.json similarity index 70% rename from tests/parametric/test_ffe/test-case-numeric-one-of.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of.json index 9eaccbc477c..bfa957335d3 100644 --- a/tests/parametric/test_ffe/test-case-numeric-one-of.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of.json @@ -1,86 +1,93 @@ [ { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "alice", "attributes": { "number": 1 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 1 - } + }, + "targetingKey": "alice", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "bob", "attributes": { "number": 2 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "bob", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "charlie", "attributes": { "number": 3 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "charlie", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "derek", "attributes": { "number": 4 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "derek", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "erica", "attributes": { "number": "1" }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 1 - } + }, + "targetingKey": "erica", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "frank", "attributes": { "number": 1 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 1 - } + }, + "targetingKey": "frank", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "george", "attributes": { "number": 123456789 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 2 - } + }, + "targetingKey": "george", + "variationType": "INTEGER" } ] diff --git a/tests/parametric/test_ffe/test-case-of-7-empty-targeting-key.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-of-7-empty-targeting-key.json similarity index 73% rename from tests/parametric/test_ffe/test-case-of-7-empty-targeting-key.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-of-7-empty-targeting-key.json index 03f03ab1ecb..e99a207b6da 100644 --- a/tests/parametric/test_ffe/test-case-of-7-empty-targeting-key.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-of-7-empty-targeting-key.json @@ -1,12 +1,13 @@ [ { - "flag": "empty-targeting-key-flag", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "", "attributes": {}, + "defaultValue": "default", + "flag": "empty-targeting-key-flag", "result": { + "reason": "STATIC", "value": "on-value" - } + }, + "targetingKey": "", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-regex-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-regex-flag.json new file mode 100644 index 00000000000..13ea0c3ce3b --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-regex-flag.json @@ -0,0 +1,83 @@ +[ + { + "attributes": { + "email": "user.name+tag@capture.example" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "capturing-groups" + }, + "targetingKey": "capture", + "variationType": "STRING" + }, + { + "attributes": { + "email": "llega mañana temprano" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "unicode-word-boundary" + }, + "targetingKey": "unicode", + "variationType": "STRING" + }, + { + "attributes": { + "email": "alice@example.com", + "version": "1.15.0" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "partial-example" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "attributes": { + "email": "bob@test.com", + "version": "0.20.1" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "test" + }, + "targetingKey": "bob", + "variationType": "STRING" + }, + { + "attributes": { + "version": "2.1.13" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "DEFAULT", + "value": "none" + }, + "targetingKey": "charlie", + "variationType": "STRING" + }, + { + "attributes": { + "email": "derek@gmail.com", + "version": "2.1.13" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "DEFAULT", + "value": "none" + }, + "targetingKey": "derek", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-comparison-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-comparison-flag.json new file mode 100644 index 00000000000..bc6fe8b92bc --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-comparison-flag.json @@ -0,0 +1,169 @@ +[ + { + "attributes": { + "app_version": "1.2.3-alpha.4" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "legacy" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "1.2.3-alpha.5" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "legacy" + }, + "targetingKey": "bob", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "2.3.4" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "stable" + }, + "targetingKey": "carol", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "2.9.1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "stable" + }, + "targetingKey": "dave", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "3.4.5-beta.1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "stable" + }, + "targetingKey": "eve", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "3.4.5-beta.2" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "beta" + }, + "targetingKey": "frank", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "4.5.6-rc.0" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "beta" + }, + "targetingKey": "grace", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "4.5.6-rc.1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "launch" + }, + "targetingKey": "henry", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "4.5.6-rc.1+build.42" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "launch" + }, + "targetingKey": "mia", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "4.5.6" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "future" + }, + "targetingKey": "iris", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "5.6.7" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "jane", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "not-a-semver" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "kyle", + "variationType": "STRING" + }, + { + "attributes": {}, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "liam", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-validation-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-validation-flag.json new file mode 100644 index 00000000000..9fa90622c0c --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-validation-flag.json @@ -0,0 +1,129 @@ +[ + { + "description": "The maximum safe integer value is valid in every core version component.", + "attributes": { + "app_version": "9007199254740991.9007199254740991.9007199254740991" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "maximum" + }, + "targetingKey": "max-core", + "variationType": "STRING" + }, + { + "description": "Semver requires all three core components.", + "attributes": { + "app_version": "1.2" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "short-version", + "variationType": "STRING" + }, + { + "description": "Semver does not accept a v prefix.", + "attributes": { + "app_version": "v1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "v-prefix", + "variationType": "STRING" + }, + { + "description": "Core numeric components cannot have leading zeros.", + "attributes": { + "app_version": "01.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "core-leading-zero", + "variationType": "STRING" + }, + { + "description": "Numeric prerelease identifiers cannot have leading zeros.", + "attributes": { + "app_version": "1.2.3-01" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "prerelease-leading-zero", + "variationType": "STRING" + }, + { + "description": "Prerelease identifiers cannot be empty.", + "attributes": { + "app_version": "1.2.3-alpha..1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "empty-prerelease-identifier", + "variationType": "STRING" + }, + { + "description": "Build metadata identifiers cannot be empty.", + "attributes": { + "app_version": "1.2.3+build..1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "empty-build-identifier", + "variationType": "STRING" + }, + { + "description": "Prerelease and build identifiers are ASCII-only.", + "attributes": { + "app_version": "1.2.3-α" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "non-ascii-prerelease", + "variationType": "STRING" + }, + { + "description": "A flag with an invalid configured SemVer comparand returns a parse error while the rest of the configuration remains usable.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-comparand", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/test-case-start-and-end-date-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-start-and-end-date-flag.json similarity index 65% rename from tests/parametric/test_ffe/test-case-start-and-end-date-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-start-and-end-date-flag.json index 7a48ec35886..53bf8859cdd 100644 --- a/tests/parametric/test_ffe/test-case-start-and-end-date-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-start-and-end-date-flag.json @@ -1,40 +1,43 @@ [ { - "flag": "start-and-end-date-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "alice", "attributes": { - "version": "1.15.0", - "country": "US" + "country": "US", + "version": "1.15.0" }, + "defaultValue": "unknown", + "flag": "start-and-end-date-test", "result": { + "reason": "DEFAULT", "value": "current" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "start-and-end-date-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "bob", "attributes": { - "version": "0.20.1", - "country": "Canada" + "country": "Canada", + "version": "0.20.1" }, + "defaultValue": "unknown", + "flag": "start-and-end-date-test", "result": { + "reason": "DEFAULT", "value": "current" - } + }, + "targetingKey": "bob", + "variationType": "STRING" }, { - "flag": "start-and-end-date-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "charlie", "attributes": { "version": "2.1.13" }, + "defaultValue": "unknown", + "flag": "start-and-end-date-test", "result": { + "reason": "DEFAULT", "value": "current" - } + }, + "targetingKey": "charlie", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-fields-tolerance.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-fields-tolerance.json new file mode 100644 index 00000000000..1bc2c914534 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-fields-tolerance.json @@ -0,0 +1,16 @@ +[ + { + "description": "The flag and surrounding UFC config include unknown object fields that SDKs must ignore while evaluating the known schema fields.", + "attributes": { + "country": "US" + }, + "defaultValue": "default", + "flag": "unknown-fields-tolerance-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "on" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-operator-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-operator-isolation.json new file mode 100644 index 00000000000..c37fde4d573 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-operator-isolation.json @@ -0,0 +1,31 @@ +[ + { + "description": "The first allocation uses an unknown operator and the second is a catch-all trap. SDKs must return a parse error without falling through to the catch-all allocation.", + "attributes": { + "country": "US" + }, + "defaultValue": "default", + "flag": "operator-grease-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "description": "A valid flag in the same config must still evaluate after the unknown-operator flag is skipped.", + "attributes": { + "country": "US" + }, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-variant-type-mismatch.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-variant-type-mismatch.json new file mode 100644 index 00000000000..ca3b775ad61 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-variant-type-mismatch.json @@ -0,0 +1,27 @@ +[ + { + "description": "A flag whose variation violates its declared type returns a parse error while the rest of the configuration remains usable.", + "attributes": {}, + "defaultValue": 0, + "flag": "variant-type-mismatch-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": 0 + }, + "targetingKey": "user-1", + "variationType": "INTEGER" + }, + { + "description": "A valid flag in the same configuration continues to evaluate when a malformed neighboring flag returns a parse error.", + "attributes": {}, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "valid-neighbor", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-flag-that-does-not-exist.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-flag-that-does-not-exist.json new file mode 100644 index 00000000000..0be74f1f7ac --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-flag-that-does-not-exist.json @@ -0,0 +1,46 @@ +[ + { + "attributes": { + "country": "US", + "email": "alice@mycompany.com" + }, + "defaultValue": 0, + "flag": "flag-that-does-not-exist", + "result": { + "errorCode": "FLAG_NOT_FOUND", + "reason": "ERROR", + "value": 0 + }, + "targetingKey": "alice", + "variationType": "NUMERIC" + }, + { + "attributes": { + "country": "Canada", + "email": "bob@example.com" + }, + "defaultValue": 0, + "flag": "flag-that-does-not-exist", + "result": { + "errorCode": "FLAG_NOT_FOUND", + "reason": "ERROR", + "value": 0 + }, + "targetingKey": "bob", + "variationType": "NUMERIC" + }, + { + "attributes": { + "age": 50 + }, + "defaultValue": 0, + "flag": "flag-that-does-not-exist", + "result": { + "errorCode": "FLAG_NOT_FOUND", + "reason": "ERROR", + "value": 0 + }, + "targetingKey": "charlie", + "variationType": "NUMERIC" + } +] diff --git a/tests/parametric/test_ffe/test-json-config-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-json-config-flag.json similarity index 62% rename from tests/parametric/test_ffe/test-json-config-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-json-config-flag.json index ecc799546b0..45a9c2ee973 100644 --- a/tests/parametric/test_ffe/test-json-config-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-json-config-flag.json @@ -1,72 +1,76 @@ [ { - "flag": "json-config-flag", - "variationType": "JSON", + "attributes": { + "country": "US", + "email": "alice@mycompany.com" + }, "defaultValue": { "foo": "bar" }, - "targetingKey": "alice", - "attributes": { - "email": "alice@mycompany.com", - "country": "US" - }, + "flag": "json-config-flag", "result": { + "reason": "SPLIT", "value": { + "float": 1.0, "integer": 1, - "string": "one", - "float": 1.0 + "string": "one" } - } + }, + "targetingKey": "alice", + "variationType": "JSON" }, { - "flag": "json-config-flag", - "variationType": "JSON", + "attributes": { + "country": "Canada", + "email": "bob@example.com" + }, "defaultValue": { "foo": "bar" }, - "targetingKey": "bob", - "attributes": { - "email": "bob@example.com", - "country": "Canada" - }, + "flag": "json-config-flag", "result": { + "reason": "SPLIT", "value": { + "float": 2.0, "integer": 2, - "string": "two", - "float": 2.0 + "string": "two" } - } + }, + "targetingKey": "bob", + "variationType": "JSON" }, { - "flag": "json-config-flag", - "variationType": "JSON", - "defaultValue": { - "foo": "bar" - }, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": { + "foo": "bar" + }, + "flag": "json-config-flag", "result": { + "reason": "SPLIT", "value": { + "float": 2.0, "integer": 2, - "string": "two", - "float": 2.0 + "string": "two" } - } + }, + "targetingKey": "charlie", + "variationType": "JSON" }, { - "flag": "json-config-flag", - "variationType": "JSON", - "defaultValue": { - "foo": "bar" - }, - "targetingKey": "diana", "attributes": { "Force Empty": true }, + "defaultValue": { + "foo": "bar" + }, + "flag": "json-config-flag", "result": { + "reason": "TARGETING_MATCH", "value": {} - } + }, + "targetingKey": "diana", + "variationType": "JSON" } ] diff --git a/tests/parametric/test_ffe/test-no-allocations-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-no-allocations-flag.json similarity index 70% rename from tests/parametric/test_ffe/test-no-allocations-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-no-allocations-flag.json index 45867e5897c..2b03b9a39a1 100644 --- a/tests/parametric/test_ffe/test-no-allocations-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-no-allocations-flag.json @@ -1,52 +1,55 @@ [ { - "flag": "no_allocations_flag", - "variationType": "JSON", + "attributes": { + "country": "US", + "email": "alice@mycompany.com" + }, "defaultValue": { "message": "Hello, world!" }, - "targetingKey": "alice", - "attributes": { - "email": "alice@mycompany.com", - "country": "US" - }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "message": "Hello, world!" } - } + }, + "targetingKey": "alice", + "variationType": "JSON" }, { - "flag": "no_allocations_flag", - "variationType": "JSON", + "attributes": { + "country": "Canada", + "email": "bob@example.com" + }, "defaultValue": { "message": "Hello, world!" }, - "targetingKey": "bob", - "attributes": { - "email": "bob@example.com", - "country": "Canada" - }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "message": "Hello, world!" } - } + }, + "targetingKey": "bob", + "variationType": "JSON" }, { - "flag": "no_allocations_flag", - "variationType": "JSON", - "defaultValue": { - "message": "Hello, world!" - }, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": { + "message": "Hello, world!" + }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "message": "Hello, world!" } - } + }, + "targetingKey": "charlie", + "variationType": "JSON" } ] diff --git a/tests/parametric/test_ffe/test-special-characters.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-special-characters.json similarity index 77% rename from tests/parametric/test_ffe/test-special-characters.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-special-characters.json index 120647ec3b5..1ed7795bb95 100644 --- a/tests/parametric/test_ffe/test-special-characters.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-special-characters.json @@ -1,54 +1,58 @@ [ { - "flag": "special-characters", - "variationType": "JSON", - "defaultValue": {}, - "targetingKey": "ash", "attributes": {}, + "defaultValue": {}, + "flag": "special-characters", "result": { + "reason": "SPLIT", "value": { "a": "kümmert", "b": "schön" } - } + }, + "targetingKey": "ash", + "variationType": "JSON" }, { - "flag": "special-characters", - "variationType": "JSON", - "defaultValue": {}, - "targetingKey": "ben", "attributes": {}, + "defaultValue": {}, + "flag": "special-characters", "result": { + "reason": "SPLIT", "value": { "a": "піклуватися", "b": "любов" } - } + }, + "targetingKey": "ben", + "variationType": "JSON" }, { - "flag": "special-characters", - "variationType": "JSON", - "defaultValue": {}, - "targetingKey": "cameron", "attributes": {}, + "defaultValue": {}, + "flag": "special-characters", "result": { + "reason": "SPLIT", "value": { "a": "照顾", "b": "漂亮" } - } + }, + "targetingKey": "cameron", + "variationType": "JSON" }, { - "flag": "special-characters", - "variationType": "JSON", - "defaultValue": {}, - "targetingKey": "darryl", "attributes": {}, + "defaultValue": {}, + "flag": "special-characters", "result": { + "reason": "SPLIT", "value": { "a": "🤗", "b": "🌸" } - } + }, + "targetingKey": "darryl", + "variationType": "JSON" } ] diff --git a/tests/parametric/test_ffe/test-string-with-special-characters.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-string-with-special-characters.json similarity index 77% rename from tests/parametric/test_ffe/test-string-with-special-characters.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-string-with-special-characters.json index e56322d44f0..32397a48398 100644 --- a/tests/parametric/test_ffe/test-string-with-special-characters.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-string-with-special-characters.json @@ -1,794 +1,860 @@ [ { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_spaces", "attributes": { "string_with_spaces": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": " a b c d e f " - } + }, + "targetingKey": "string_with_spaces", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_space", "attributes": { "string_with_only_one_space": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": " " - } + }, + "targetingKey": "string_with_only_one_space", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_spaces", "attributes": { "string_with_only_multiple_spaces": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": " " - } + }, + "targetingKey": "string_with_only_multiple_spaces", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_dots", "attributes": { "string_with_dots": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ".a.b.c.d.e.f." - } + }, + "targetingKey": "string_with_dots", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_dot", "attributes": { "string_with_only_one_dot": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "." - } + }, + "targetingKey": "string_with_only_one_dot", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_dots", "attributes": { "string_with_only_multiple_dots": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "......." - } + }, + "targetingKey": "string_with_only_multiple_dots", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_comas", "attributes": { "string_with_comas": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ",a,b,c,d,e,f," - } + }, + "targetingKey": "string_with_comas", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_coma", "attributes": { "string_with_only_one_coma": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "," - } + }, + "targetingKey": "string_with_only_one_coma", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_comas", "attributes": { "string_with_only_multiple_comas": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ",,,,,,," - } + }, + "targetingKey": "string_with_only_multiple_comas", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_colons", "attributes": { "string_with_colons": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ":a:b:c:d:e:f:" - } + }, + "targetingKey": "string_with_colons", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_colon", "attributes": { "string_with_only_one_colon": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ":" - } + }, + "targetingKey": "string_with_only_one_colon", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_colons", "attributes": { "string_with_only_multiple_colons": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ":::::::" - } + }, + "targetingKey": "string_with_only_multiple_colons", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_semicolons", "attributes": { "string_with_semicolons": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ";a;b;c;d;e;f;" - } + }, + "targetingKey": "string_with_semicolons", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_semicolon", "attributes": { "string_with_only_one_semicolon": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ";" - } + }, + "targetingKey": "string_with_only_one_semicolon", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_semicolons", "attributes": { "string_with_only_multiple_semicolons": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ";;;;;;;" - } + }, + "targetingKey": "string_with_only_multiple_semicolons", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_slashes", "attributes": { "string_with_slashes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "/a/b/c/d/e/f/" - } + }, + "targetingKey": "string_with_slashes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_slash", "attributes": { "string_with_only_one_slash": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "/" - } + }, + "targetingKey": "string_with_only_one_slash", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_slashes", "attributes": { "string_with_only_multiple_slashes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "///////" - } + }, + "targetingKey": "string_with_only_multiple_slashes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_dashes", "attributes": { "string_with_dashes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "-a-b-c-d-e-f-" - } + }, + "targetingKey": "string_with_dashes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_dash", "attributes": { "string_with_only_one_dash": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "-" - } + }, + "targetingKey": "string_with_only_one_dash", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_dashes", "attributes": { "string_with_only_multiple_dashes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "-------" - } + }, + "targetingKey": "string_with_only_multiple_dashes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_underscores", "attributes": { "string_with_underscores": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "_a_b_c_d_e_f_" - } + }, + "targetingKey": "string_with_underscores", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_underscore", "attributes": { "string_with_only_one_underscore": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "_" - } + }, + "targetingKey": "string_with_only_one_underscore", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_underscores", "attributes": { "string_with_only_multiple_underscores": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "_______" - } + }, + "targetingKey": "string_with_only_multiple_underscores", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_plus_signs", "attributes": { "string_with_plus_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "+a+b+c+d+e+f+" - } + }, + "targetingKey": "string_with_plus_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_plus_sign", "attributes": { "string_with_only_one_plus_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "+" - } + }, + "targetingKey": "string_with_only_one_plus_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_plus_signs", "attributes": { "string_with_only_multiple_plus_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "+++++++" - } + }, + "targetingKey": "string_with_only_multiple_plus_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_equal_signs", "attributes": { "string_with_equal_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "=a=b=c=d=e=f=" - } + }, + "targetingKey": "string_with_equal_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_equal_sign", "attributes": { "string_with_only_one_equal_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "=" - } + }, + "targetingKey": "string_with_only_one_equal_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_equal_signs", "attributes": { "string_with_only_multiple_equal_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "=======" - } + }, + "targetingKey": "string_with_only_multiple_equal_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_dollar_signs", "attributes": { "string_with_dollar_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "$a$b$c$d$e$f$" - } + }, + "targetingKey": "string_with_dollar_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_dollar_sign", "attributes": { "string_with_only_one_dollar_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "$" - } + }, + "targetingKey": "string_with_only_one_dollar_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_dollar_signs", "attributes": { "string_with_only_multiple_dollar_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "$$$$$$$" - } + }, + "targetingKey": "string_with_only_multiple_dollar_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_at_signs", "attributes": { "string_with_at_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "@a@b@c@d@e@f@" - } + }, + "targetingKey": "string_with_at_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_at_sign", "attributes": { "string_with_only_one_at_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "@" - } + }, + "targetingKey": "string_with_only_one_at_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_at_signs", "attributes": { "string_with_only_multiple_at_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "@@@@@@@" - } + }, + "targetingKey": "string_with_only_multiple_at_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_amp_signs", "attributes": { "string_with_amp_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { - "value": "&a&b&c&d&e&f&" - } + "reason": "TARGETING_MATCH", + "value": "\u0026a\u0026b\u0026c\u0026d\u0026e\u0026f\u0026" + }, + "targetingKey": "string_with_amp_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_amp_sign", "attributes": { "string_with_only_one_amp_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { - "value": "&" - } + "reason": "TARGETING_MATCH", + "value": "\u0026" + }, + "targetingKey": "string_with_only_one_amp_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_amp_signs", "attributes": { "string_with_only_multiple_amp_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { - "value": "&&&&&&&" - } + "reason": "TARGETING_MATCH", + "value": "\u0026\u0026\u0026\u0026\u0026\u0026\u0026" + }, + "targetingKey": "string_with_only_multiple_amp_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_hash_signs", "attributes": { "string_with_hash_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "#a#b#c#d#e#f#" - } + }, + "targetingKey": "string_with_hash_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_hash_sign", "attributes": { "string_with_only_one_hash_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "#" - } + }, + "targetingKey": "string_with_only_one_hash_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_hash_signs", "attributes": { "string_with_only_multiple_hash_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "#######" - } + }, + "targetingKey": "string_with_only_multiple_hash_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_percentage_signs", "attributes": { "string_with_percentage_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "%a%b%c%d%e%f%" - } + }, + "targetingKey": "string_with_percentage_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_percentage_sign", "attributes": { "string_with_only_one_percentage_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "%" - } + }, + "targetingKey": "string_with_only_one_percentage_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_percentage_signs", "attributes": { "string_with_only_multiple_percentage_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "%%%%%%%" - } + }, + "targetingKey": "string_with_only_multiple_percentage_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_tilde_signs", "attributes": { "string_with_tilde_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "~a~b~c~d~e~f~" - } + }, + "targetingKey": "string_with_tilde_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_tilde_sign", "attributes": { "string_with_only_one_tilde_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "~" - } + }, + "targetingKey": "string_with_only_one_tilde_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_tilde_signs", "attributes": { "string_with_only_multiple_tilde_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "~~~~~~~" - } + }, + "targetingKey": "string_with_only_multiple_tilde_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_asterix_signs", "attributes": { "string_with_asterix_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "*a*b*c*d*e*f*" - } + }, + "targetingKey": "string_with_asterix_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_asterix_sign", "attributes": { "string_with_only_one_asterix_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "*" - } + }, + "targetingKey": "string_with_only_one_asterix_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_asterix_signs", "attributes": { "string_with_only_multiple_asterix_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "*******" - } + }, + "targetingKey": "string_with_only_multiple_asterix_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_single_quotes", "attributes": { "string_with_single_quotes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "'a'b'c'd'e'f'" - } + }, + "targetingKey": "string_with_single_quotes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_single_quote", "attributes": { "string_with_only_one_single_quote": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "'" - } + }, + "targetingKey": "string_with_only_one_single_quote", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_single_quotes", "attributes": { "string_with_only_multiple_single_quotes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "'''''''" - } + }, + "targetingKey": "string_with_only_multiple_single_quotes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_question_marks", "attributes": { "string_with_question_marks": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "?a?b?c?d?e?f?" - } + }, + "targetingKey": "string_with_question_marks", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_question_mark", "attributes": { "string_with_only_one_question_mark": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "?" - } + }, + "targetingKey": "string_with_only_one_question_mark", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_question_marks", "attributes": { "string_with_only_multiple_question_marks": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "???????" - } + }, + "targetingKey": "string_with_only_multiple_question_marks", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_exclamation_marks", "attributes": { "string_with_exclamation_marks": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "!a!b!c!d!e!f!" - } + }, + "targetingKey": "string_with_exclamation_marks", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_exclamation_mark", "attributes": { "string_with_only_one_exclamation_mark": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "!" - } + }, + "targetingKey": "string_with_only_one_exclamation_mark", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_exclamation_marks", "attributes": { "string_with_only_multiple_exclamation_marks": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "!!!!!!!" - } + }, + "targetingKey": "string_with_only_multiple_exclamation_marks", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_opening_parentheses", "attributes": { "string_with_opening_parentheses": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "(a(b(c(d(e(f(" - } + }, + "targetingKey": "string_with_opening_parentheses", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_opening_parenthese", "attributes": { "string_with_only_one_opening_parenthese": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "(" - } + }, + "targetingKey": "string_with_only_one_opening_parenthese", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_opening_parentheses", "attributes": { "string_with_only_multiple_opening_parentheses": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "(((((((" - } + }, + "targetingKey": "string_with_only_multiple_opening_parentheses", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_closing_parentheses", "attributes": { "string_with_closing_parentheses": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ")a)b)c)d)e)f)" - } + }, + "targetingKey": "string_with_closing_parentheses", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_closing_parenthese", "attributes": { "string_with_only_one_closing_parenthese": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ")" - } + }, + "targetingKey": "string_with_only_one_closing_parenthese", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_closing_parentheses", "attributes": { "string_with_only_multiple_closing_parentheses": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ")))))))" - } + }, + "targetingKey": "string_with_only_multiple_closing_parentheses", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/flags-v1.json b/tests/parametric/test_ffe/ffe-system-test-data/ufc-config.json similarity index 78% rename from tests/parametric/test_ffe/flags-v1.json rename to tests/parametric/test_ffe/ffe-system-test-data/ufc-config.json index f01bfe2ef86..52fd31d5ff1 100644 --- a/tests/parametric/test_ffe/flags-v1.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/ufc-config.json @@ -2,8 +2,11 @@ "createdAt": "2024-04-17T19:40:53.716Z", "format": "SERVER", "environment": { - "name": "Test" + "name": "Test", + "unknownEnvironmentField": "ignored" }, + "unknownTopLevelField": "ignored", + "unknownTopLevelNull": null, "flags": { "empty_flag": { "key": "empty_flag", @@ -78,9 +81,59 @@ "test": { "key": "test", "value": "test" + }, + "capturing-groups": { + "key": "capturing-groups", + "value": "capturing-groups" + }, + "unicode-word-boundary": { + "key": "unicode-word-boundary", + "value": "unicode-word-boundary" } }, "allocations": [ + { + "key": "capturing-groups", + "rules": [ + { + "conditions": [ + { + "attribute": "email", + "operator": "MATCHES", + "value": "^([[:alnum:]._%+-]+)@capture\\.example$" + } + ] + } + ], + "splits": [ + { + "variationKey": "capturing-groups", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "unicode-word-boundary", + "rules": [ + { + "conditions": [ + { + "attribute": "email", + "operator": "MATCHES", + "value": "(?u)\\bmañana\\b" + } + ] + } + ], + "splits": [ + { + "variationKey": "unicode-word-boundary", + "shards": [] + } + ], + "doLog": true + }, { "key": "partial-example", "rules": [ @@ -612,6 +665,229 @@ } ] }, + "semver-comparison-test": { + "key": "semver-comparison-test", + "enabled": true, + "variationType": "STRING", + "variations": { + "legacy": { + "key": "legacy", + "value": "legacy" + }, + "stable": { + "key": "stable", + "value": "stable" + }, + "beta": { + "key": "beta", + "value": "beta" + }, + "launch": { + "key": "launch", + "value": "launch" + }, + "future": { + "key": "future", + "value": "future" + }, + "maximum": { + "key": "maximum", + "value": "maximum" + } + }, + "allocations": [ + { + "key": "maximum-version", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "9007199254740991.9007199254740991.9007199254740991" + } + ] + } + ], + "splits": [ + { + "variationKey": "maximum", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "legacy-versions", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_LTE", + "value": "1.2.3-alpha.4" + } + ] + }, + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "1.2.3-alpha.4" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "2.3.4" + } + ] + } + ], + "splits": [ + { + "variationKey": "legacy", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "stable-versions", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_GTE", + "value": "2.3.4" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "3.4.5-beta.2" + } + ] + } + ], + "splits": [ + { + "variationKey": "stable", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "beta-versions", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_GTE", + "value": "3.4.5-beta.2" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "4.5.6-rc.1" + } + ] + } + ], + "splits": [ + { + "variationKey": "beta", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "launch-version", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "4.5.6-rc.1" + } + ] + } + ], + "splits": [ + { + "variationKey": "launch", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "future-versions", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "4.5.6-rc.1" + }, + { + "attribute": "app_version", + "operator": "SEMVER_NEQ", + "value": "5.6.7" + } + ] + } + ], + "splits": [ + { + "variationKey": "future", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-test": { + "key": "semver-invalid-comparand-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses an invalid configured SemVer comparand. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "18446744073709551616.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, "start-and-end-date-test": { "key": "start-and-end-date-test", "enabled": true, @@ -3293,6 +3569,578 @@ "doLog": true } ] + }, + "microsecond-date-test": { + "key": "microsecond-date-test", + "enabled": true, + "variationType": "STRING", + "variations": { + "expired": { + "key": "expired", + "value": "expired" + }, + "active": { + "key": "active", + "value": "active" + }, + "future": { + "key": "future", + "value": "future" + } + }, + "allocations": [ + { + "key": "expired-allocation", + "splits": [ + { + "variationKey": "expired", + "shards": [] + } + ], + "endAt": "2002-10-31T09:00:00.594321Z", + "doLog": true + }, + { + "key": "future-allocation", + "splits": [ + { + "variationKey": "future", + "shards": [] + } + ], + "startAt": "2052-10-31T09:00:00.123456Z", + "doLog": true + }, + { + "key": "active-allocation", + "splits": [ + { + "variationKey": "active", + "shards": [] + } + ], + "startAt": "2022-10-31T09:00:00.235982Z", + "endAt": "2050-10-31T09:00:00.987654Z", + "doLog": true + } + ] + }, + "unknown-fields-tolerance-flag": { + "key": "unknown-fields-tolerance-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately contains unknown fields at every fixed-schema UFC level. SDKs must ignore those fields and evaluate the flag normally.", + "unknownFlagField": "ignored", + "variations": { + "on": { + "key": "on", + "value": "on", + "unknownVariationField": "ignored" + } + }, + "allocations": [ + { + "key": "unknown-fields-allocation", + "unknownAllocationField": "ignored", + "rules": [ + { + "unknownRuleField": "ignored", + "conditions": [ + { + "attribute": "country", + "operator": "ONE_OF", + "value": [ + "US" + ], + "unknownConditionField": "ignored" + } + ] + } + ], + "splits": [ + { + "variationKey": "on", + "unknownSplitField": "ignored", + "shards": [ + { + "salt": "unknown-fields-tolerance-flag", + "totalShards": 10000, + "unknownShardField": "ignored", + "ranges": [ + { + "start": 0, + "end": 10000, + "unknownRangeField": "ignored" + } + ] + } + ] + } + ], + "doLog": true + } + ] + }, + "operator-grease-flag": { + "key": "operator-grease-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses an unknown condition operator. SDKs must remove this flag rather than falling through to the catch-all allocation or rejecting the whole config.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + }, + "on": { + "key": "on", + "value": "on" + } + }, + "allocations": [ + { + "key": "grease-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "country", + "operator": "NOT_A_REAL_OPERATOR", + "value": "anything" + } + ] + } + ], + "splits": [ + { + "variationKey": "trap", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "default-allocation", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "invalid-regex-flag": { + "key": "invalid-regex-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses the invalid regex *@datadoghq.com. SDKs must remove this flag rather than falling through to a later allocation or rejecting the whole config.", + "variations": { + "invalid-regex-match": { + "key": "invalid-regex-match", + "value": "invalid-regex-match" + }, + "catch-all": { + "key": "catch-all", + "value": "catch-all" + } + }, + "allocations": [ + { + "key": "invalid-regex-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "email", + "operator": "MATCHES", + "value": "*@datadoghq.com" + } + ] + } + ], + "splits": [ + { + "variationKey": "invalid-regex-match", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "invalid-regex-catch-all", + "rules": [], + "splits": [ + { + "variationKey": "catch-all", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "variant-type-mismatch-flag": { + "key": "variant-type-mismatch-flag", + "enabled": true, + "variationType": "INTEGER", + "comment": "This flag is intentionally invalid: one variation contains a string even though the flag declares INTEGER values. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "invalid": { + "key": "invalid", + "value": "not-an-integer" + }, + "valid": { + "key": "valid", + "value": 0 + } + }, + "allocations": [ + { + "key": "default-allocation", + "rules": [], + "splits": [ + { + "variationKey": "invalid", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "invalid-gt-condition-flag": { + "key": "invalid-gt-condition-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately gives GT a string operand. SDKs must remove this flag rather than falling through to the catch-all allocation.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + }, + "catch-all": { + "key": "catch-all", + "value": "catch-all" + } + }, + "allocations": [ + { + "key": "invalid-gt-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "age", + "operator": "GT", + "value": "not-a-number" + } + ] + } + ], + "splits": [ + { + "variationKey": "trap", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "invalid-gt-catch-all", + "rules": [], + "splits": [ + { + "variationKey": "catch-all", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "invalid-one-of-condition-flag": { + "key": "invalid-one-of-condition-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately gives ONE_OF a scalar operand. SDKs must remove this flag rather than falling through to the catch-all allocation.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + }, + "catch-all": { + "key": "catch-all", + "value": "catch-all" + } + }, + "allocations": [ + { + "key": "invalid-one-of-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "role", + "operator": "ONE_OF", + "value": "admin" + } + ] + } + ], + "splits": [ + { + "variationKey": "trap", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "invalid-one-of-catch-all", + "rules": [], + "splits": [ + { + "variationKey": "catch-all", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "invalid-is-null-condition-flag": { + "key": "invalid-is-null-condition-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately gives IS_NULL a non-boolean operand. SDKs must remove this flag rather than falling through to the catch-all allocation.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + }, + "catch-all": { + "key": "catch-all", + "value": "catch-all" + } + }, + "allocations": [ + { + "key": "invalid-is-null-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "deleted_at", + "operator": "IS_NULL", + "value": "not-a-boolean" + } + ] + } + ], + "splits": [ + { + "variationKey": "trap", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "invalid-is-null-catch-all", + "rules": [], + "splits": [ + { + "variationKey": "catch-all", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "malformed-allocations-flag": { + "key": "malformed-allocations-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag is intentionally UFC-schema-invalid: allocations must be an array. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "on": { + "key": "on", + "value": "on" + } + }, + "allocations": "this-is-not-a-list" + }, + "missing-split-shards-flag": { + "key": "missing-split-shards-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag is intentionally UFC-schema-invalid: split.shards is required even when the intended split is unsharded. SDKs must treat omitted shards differently from an explicit empty shards array.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + } + }, + "allocations": [ + { + "key": "malformed-split-allocation", + "rules": [], + "splits": [ + { + "variationKey": "trap" + } + ], + "doLog": true + } + ] + }, + "overflow-total-shards-flag": { + "key": "overflow-total-shards-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag is intentionally UFC-schema-invalid: split.shards[].totalShards must fit in an unsigned 32-bit integer. SDKs must remove this flag rather than overflowing or poisoning the whole config.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + } + }, + "allocations": [ + { + "key": "malformed-shard-allocation", + "rules": [], + "splits": [ + { + "variationKey": "trap", + "shards": [ + { + "salt": "overflow-total-shards-flag", + "totalShards": 4294967296, + "ranges": [ + { + "start": 0, + "end": 1 + } + ] + } + ] + } + ], + "doLog": true + } + ] + }, + "negative-shard-range-start-flag": { + "key": "negative-shard-range-start-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag is intentionally UFC-schema-invalid: split.shards[].ranges[].start must be an unsigned integer. SDKs must remove this flag rather than treating a negative range as matching traffic.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + } + }, + "allocations": [ + { + "key": "malformed-shard-allocation", + "rules": [], + "splits": [ + { + "variationKey": "trap", + "shards": [ + { + "salt": "negative-shard-range-start-flag", + "totalShards": 100, + "ranges": [ + { + "start": -1, + "end": 1 + } + ] + } + ] + } + ], + "doLog": true + } + ] + }, + "null-shard-range-flag": { + "key": "null-shard-range-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag is intentionally UFC-schema-invalid: split.shards[].ranges[] contains null. SDKs must remove this flag rather than panicking or poisoning the whole config.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + }, + "catch-all": { + "key": "catch-all", + "value": "catch-all" + } + }, + "allocations": [ + { + "key": "null-shard-range-allocation", + "rules": [], + "splits": [ + { + "variationKey": "trap", + "shards": [ + { + "salt": "null-shard-range-flag", + "totalShards": 8192, + "ranges": [ + null + ] + } + ] + } + ], + "doLog": true + }, + { + "key": "null-shard-range-catch-all", + "rules": [], + "splits": [ + { + "variationKey": "catch-all", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "valid-flag-after-invalid-config": { + "key": "valid-flag-after-invalid-config", + "enabled": true, + "variationType": "STRING", + "comment": "This valid neighbor proves malformed or unsupported flags do not poison the rest of the remote config.", + "variations": { + "expected": { + "key": "expected", + "value": "expected" + } + }, + "allocations": [ + { + "key": "default-allocation", + "rules": [], + "splits": [ + { + "variationKey": "expected", + "shards": [] + } + ], + "doLog": true + } + ] } } -} \ No newline at end of file +} diff --git a/tests/parametric/test_ffe/test-case-regex-flag.json b/tests/parametric/test_ffe/test-case-regex-flag.json deleted file mode 100644 index 94aa87f23a9..00000000000 --- a/tests/parametric/test_ffe/test-case-regex-flag.json +++ /dev/null @@ -1,53 +0,0 @@ -[ - { - "flag": "regex-flag", - "variationType": "STRING", - "defaultValue": "none", - "targetingKey": "alice", - "attributes": { - "version": "1.15.0", - "email": "alice@example.com" - }, - "result": { - "value": "partial-example" - } - }, - { - "flag": "regex-flag", - "variationType": "STRING", - "defaultValue": "none", - "targetingKey": "bob", - "attributes": { - "version": "0.20.1", - "email": "bob@test.com" - }, - "result": { - "value": "test" - } - }, - { - "flag": "regex-flag", - "variationType": "STRING", - "defaultValue": "none", - "targetingKey": "charlie", - "attributes": { - "version": "2.1.13" - }, - "result": { - "value": "none" - } - }, - { - "flag": "regex-flag", - "variationType": "STRING", - "defaultValue": "none", - "targetingKey": "derek", - "attributes": { - "version": "2.1.13", - "email": "derek@gmail.com" - }, - "result": { - "value": "none" - } - } -] diff --git a/tests/parametric/test_ffe/test-flag-that-does-not-exist.json b/tests/parametric/test_ffe/test-flag-that-does-not-exist.json deleted file mode 100644 index 7499bba1c50..00000000000 --- a/tests/parametric/test_ffe/test-flag-that-does-not-exist.json +++ /dev/null @@ -1,40 +0,0 @@ -[ - { - "flag": "flag-that-does-not-exist", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "alice", - "attributes": { - "email": "alice@mycompany.com", - "country": "US" - }, - "result": { - "value": 0.0 - } - }, - { - "flag": "flag-that-does-not-exist", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "bob", - "attributes": { - "email": "bob@example.com", - "country": "Canada" - }, - "result": { - "value": 0.0 - } - }, - { - "flag": "flag-that-does-not-exist", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "charlie", - "attributes": { - "age": 50 - }, - "result": { - "value": 0.0 - } - } -] diff --git a/tests/parametric/test_ffe/test_configuration_sources.py b/tests/parametric/test_ffe/test_configuration_sources.py index 195edddd1ad..83b7a7ea19d 100644 --- a/tests/parametric/test_ffe/test_configuration_sources.py +++ b/tests/parametric/test_ffe/test_configuration_sources.py @@ -34,7 +34,7 @@ parametrize = pytest.mark.parametrize pytest_plugins = ["utils.mocked_backend.ffe"] -UFC_VALID_FIXTURE = Path(__file__).parent / "flags-v1.json" +UFC_VALID_FIXTURE = Path(__file__).parent / "ffe-system-test-data" / "ufc-config.json" RC_PRODUCT = "FFE_FLAGS" TEST_API_KEY = "system-tests-mock-api-key" MOCK_STATUS_ATTEMPTS = 25 diff --git a/tests/parametric/test_ffe/test_dynamic_evaluation.py b/tests/parametric/test_ffe/test_dynamic_evaluation.py index 7d48a1e0b7c..e19dc7132d9 100644 --- a/tests/parametric/test_ffe/test_dynamic_evaluation.py +++ b/tests/parametric/test_ffe/test_dynamic_evaluation.py @@ -18,6 +18,8 @@ RC_PATH = f"datadog/2/{RC_PRODUCT}" FFE_READY_RETRY_ATTEMPTS = 10 FFE_READY_RETRY_INTERVAL_SECONDS = 0.2 +FFE_SYSTEM_TEST_DATA_DIR = Path(__file__).parent / "ffe-system-test-data" +FFE_EVALUATION_CASES_DIR = FFE_SYSTEM_TEST_DATA_DIR / "evaluation-cases" parametrize = pytest.mark.parametrize @@ -25,10 +27,12 @@ # Load the UFC fixture file at module level def _load_ufc_fixture() -> dict[str, Any]: """Load the UFC fixture file.""" - fixture_path = Path(__file__).parent / "flags-v1.json" + fixture_path = FFE_SYSTEM_TEST_DATA_DIR / "ufc-config.json" if not fixture_path.exists(): - pytest.skip(f"Fixture file not found: {fixture_path}") + raise FileNotFoundError( + f"Fixture file not found: {fixture_path}. Run `python3 utils/scripts/update_ffe_fixtures.py`." + ) with fixture_path.open() as f: return json.load(f) @@ -36,13 +40,17 @@ def _load_ufc_fixture() -> dict[str, Any]: def _get_test_case_files() -> list[str]: """Get all test case files from the fixtures directory.""" - test_data_dir = Path(__file__).parent - if not test_data_dir.exists(): - return [] + if not FFE_EVALUATION_CASES_DIR.exists(): + raise FileNotFoundError( + f"Fixture directory not found: {FFE_EVALUATION_CASES_DIR}. " + "Run `python3 utils/scripts/update_ffe_fixtures.py`." + ) - # Exclude base fixtures that aren't test cases - excluded = {"flags-v1.json", "span-enrichment-flags.json"} - return [f.name for f in test_data_dir.iterdir() if f.suffix == ".json" and f.name not in excluded] + test_case_files = sorted(f.name for f in FFE_EVALUATION_CASES_DIR.iterdir() if f.suffix == ".json") + if not test_case_files: + raise AssertionError(f"No FFE JSON fixtures found in {FFE_EVALUATION_CASES_DIR}") + + return test_case_files # Load fixture at module level for reuse across tests @@ -153,7 +161,7 @@ def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI """ # Load the test case file - test_case_path = Path(__file__).parent / test_case_file + test_case_path = FFE_EVALUATION_CASES_DIR / test_case_file if not test_case_path.exists(): pytest.skip(f"Test case file not found: {test_case_path}") @@ -197,3 +205,20 @@ def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI f"flag='{flag}', targetingKey='{targeting_key}', " f"expected={expected_result}, actual={actual_value}" ) + + expected_reason = test_case["result"]["reason"] + actual_reason = result.get("reason") + assert actual_reason == expected_reason, ( + f"Test case {i} in {test_case_file} failed: " + f"flag='{flag}', targetingKey='{targeting_key}', " + f"expected reason={expected_reason}, actual reason={actual_reason}" + ) + + expected_error_code = test_case["result"].get("errorCode") + if expected_error_code is not None: + actual_error_code = result.get("errorCode") + assert actual_error_code == expected_error_code, ( + f"Test case {i} in {test_case_file} failed: " + f"flag='{flag}', targetingKey='{targeting_key}', " + f"expected errorCode={expected_error_code}, actual errorCode={actual_error_code}" + ) diff --git a/tests/test_the_test/test_update_ffe_fixtures.py b/tests/test_the_test/test_update_ffe_fixtures.py new file mode 100644 index 00000000000..1a7dd3fd041 --- /dev/null +++ b/tests/test_the_test/test_update_ffe_fixtures.py @@ -0,0 +1,79 @@ +# ruff: noqa: SLF001 + +from pathlib import Path + +import pytest + +from utils import scenarios +from utils.scripts import update_ffe_fixtures + + +@scenarios.test_the_test +@pytest.mark.parametrize("fixture_ref", ["main", "feature/fixture-update", "v1.2.3", "ea8b5cc5"]) +def test_validate_fixture_ref_accepts_git_refs(fixture_ref: str) -> None: + update_ffe_fixtures._validate_fixture_ref(fixture_ref) + + +@scenarios.test_the_test +@pytest.mark.parametrize("fixture_ref", ["", "-main", "../main", "main^", "main ref"]) +def test_validate_fixture_ref_rejects_unsafe_refs(fixture_ref: str) -> None: + with pytest.raises(ValueError, match="Invalid FFE fixture ref"): + update_ffe_fixtures._validate_fixture_ref(fixture_ref) + + +@scenarios.test_the_test +def test_copy_fixture_snapshot_uses_client_disallow_list(tmp_path: Path) -> None: + source = tmp_path / "source" + snapshot = tmp_path / "snapshot" + (source / "evaluation-cases").mkdir(parents=True) + snapshot.mkdir() + (source / "evaluation-cases" / "case.json").write_text("[]", encoding="utf-8") + (source / "ufc-config.json").write_text("{}", encoding="utf-8") + (source / "README.md").write_text("upstream documentation", encoding="utf-8") + + update_ffe_fixtures._copy_fixture_snapshot(source, snapshot) + + assert (snapshot / "ufc-config.json").is_file() + assert (snapshot / "evaluation-cases" / "case.json").is_file() + assert not (snapshot / "README.md").exists() + + +@scenarios.test_the_test +def test_copy_fixture_snapshot_rejects_symbolic_links(tmp_path: Path) -> None: + source = tmp_path / "source" + snapshot = tmp_path / "snapshot" + source.mkdir() + snapshot.mkdir() + target = source / "target.json" + target.write_text("{}", encoding="utf-8") + (source / "linked.json").symlink_to(target) + + with pytest.raises(ValueError, match="Refusing to copy symbolic link"): + update_ffe_fixtures._copy_fixture_snapshot(source, snapshot) + + +@scenarios.test_the_test +def test_validate_fixture_snapshot_counts_evaluations(tmp_path: Path) -> None: + cases_directory = tmp_path / "evaluation-cases" + cases_directory.mkdir() + (tmp_path / "ufc-config.json").write_text("{}", encoding="utf-8") + (cases_directory / "first.json").write_text('[{"flag": "one"}]', encoding="utf-8") + (cases_directory / "second.json").write_text('[{"flag": "two"}, {"flag": "three"}]', encoding="utf-8") + + assert update_ffe_fixtures._validate_fixture_snapshot(tmp_path) == 3 + + +@scenarios.test_the_test +def test_have_same_contents_ignores_generated_source_metadata(tmp_path: Path) -> None: + snapshot = tmp_path / "snapshot" + destination = tmp_path / "destination" + snapshot.mkdir() + destination.mkdir() + (snapshot / "fixture.json").write_text("{}", encoding="utf-8") + (destination / "fixture.json").write_text("{}", encoding="utf-8") + (destination / "SOURCE.md").write_text("generated", encoding="utf-8") + + assert update_ffe_fixtures._have_same_contents(snapshot, destination) + + (destination / "fixture.json").write_text('{"changed": true}', encoding="utf-8") + assert not update_ffe_fixtures._have_same_contents(snapshot, destination) diff --git a/utils/mocked_backend/ffe.py b/utils/mocked_backend/ffe.py index 9aed63a9724..4bbb5684f97 100644 --- a/utils/mocked_backend/ffe.py +++ b/utils/mocked_backend/ffe.py @@ -48,7 +48,7 @@ EXPECTED_DD_ENV = "test" CONFIG_QUERY = urlencode({"dd_env": EXPECTED_DD_ENV}) REPO_ROOT = Path(__file__).parents[2] -UFC_FIXTURE_PATH = REPO_ROOT / "tests" / "parametric" / "test_ffe" / "flags-v1.json" +UFC_FIXTURE_PATH = REPO_ROOT / "tests" / "parametric" / "test_ffe" / "ffe-system-test-data" / "ufc-config.json" MALFORMED_UFC_BYTES = b'{"flags": [' UFC_RESPONSE_TYPE = "universal-flag-configuration" diff --git a/utils/scripts/update_ffe_fixtures.py b/utils/scripts/update_ffe_fixtures.py new file mode 100644 index 00000000000..9b1c5406506 --- /dev/null +++ b/utils/scripts/update_ffe_fixtures.py @@ -0,0 +1,211 @@ +"""Refresh the checked-in FFE fixture snapshot from its canonical repository.""" + +import argparse +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +FIXTURE_DESTINATION = REPOSITORY_ROOT / "tests" / "parametric" / "test_ffe" / "ffe-system-test-data" +FIXTURE_REPOSITORY = "https://github.com/DataDog/ffe-system-test-data.git" +FIXTURE_COPY_DISALLOW_LIST = frozenset( + { + ".git", + ".github", + ".gitignore", + "ci", + "CONTRIBUTING.md", + "LICENSE", + "LICENSE-3rdparty.csv", + "NOTICE", + "README.md", + "SOURCE.md", + } +) +FIXTURE_REF_PATTERN = re.compile(r"[A-Za-z0-9._/-]+") + + +def _validate_fixture_ref(fixture_ref: str) -> None: + if ( + not fixture_ref + or fixture_ref.startswith("-") + or ".." in fixture_ref + or FIXTURE_REF_PATTERN.fullmatch(fixture_ref) is None + ): + msg = f"Invalid FFE fixture ref: {fixture_ref!r}" + raise ValueError(msg) + + +def _run_git(working_directory: Path, *arguments: str) -> str: + environment = { + **os.environ, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": os.devnull, + } + result = subprocess.run( + ["git", *arguments], + cwd=working_directory, + env=environment, + check=True, + capture_output=True, + text=True, + ) + return result.stdout.strip() + + +def _copy_entry(source: Path, destination: Path) -> None: + if source.is_symlink(): + msg = f"Refusing to copy symbolic link from FFE fixture repository: {source}" + raise ValueError(msg) + + if source.is_file(): + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + return + + if not source.is_dir(): + msg = f"Refusing to copy unsupported filesystem entry from FFE fixture repository: {source}" + raise ValueError(msg) + + destination.mkdir(parents=True, exist_ok=True) + for child in sorted(source.iterdir()): + _copy_entry(child, destination / child.name) + + +def _copy_fixture_snapshot(source: Path, snapshot: Path) -> None: + for entry in sorted(source.iterdir()): + if entry.name in FIXTURE_COPY_DISALLOW_LIST: + continue + _copy_entry(entry, snapshot / entry.name) + + +def _validate_fixture_snapshot(snapshot: Path) -> int: + config_path = snapshot / "ufc-config.json" + cases_directory = snapshot / "evaluation-cases" + if not config_path.is_file() or not cases_directory.is_dir(): + msg = "FFE fixture repository does not contain the expected fixture layout" + raise ValueError(msg) + + with config_path.open(encoding="utf-8") as config_file: + json.load(config_file) + + case_files = sorted(cases_directory.glob("*.json")) + if not case_files: + msg = "No FFE JSON fixture files found" + raise ValueError(msg) + + fixture_count = 0 + for case_file in case_files: + with case_file.open(encoding="utf-8") as fixture_file: + test_cases = json.load(fixture_file) + if not isinstance(test_cases, list): + msg = f"{case_file} must contain a JSON array of test cases" + raise TypeError(msg) + fixture_count += len(test_cases) + + if fixture_count == 0: + msg = "No FFE fixture test cases found" + raise ValueError(msg) + + return fixture_count + + +def _relative_files(directory: Path, excluded_files: frozenset[str] = frozenset()) -> list[Path]: + return sorted( + path.relative_to(directory) + for path in directory.rglob("*") + if path.is_file() and path.relative_to(directory).as_posix() not in excluded_files + ) + + +def _have_same_contents(snapshot: Path, destination: Path) -> bool: + if not destination.is_dir(): + return False + + snapshot_files = _relative_files(snapshot) + destination_files = _relative_files(destination, frozenset({"SOURCE.md"})) + if snapshot_files != destination_files: + return False + + return all( + (snapshot / relative_path).read_bytes() == (destination / relative_path).read_bytes() + for relative_path in snapshot_files + ) + + +def _write_source_metadata(snapshot: Path, source_commit: str) -> None: + (snapshot / "SOURCE.md").write_text( + f"""# FFE Fixture Snapshot + +These files are copied from the canonical FFE fixture repository. + +Canonical source: https://github.com/DataDog/ffe-system-test-data +Source commit: {source_commit} + +Do not edit these fixtures directly in system-tests. Add or update shared FFE behavior in +ffe-system-test-data first, then refresh this snapshot. + +The weekly update workflow runs `python3 utils/scripts/update_ffe_fixtures.py` and opens a signed +draft test PR only when the allowed fixture contents change. +""", + encoding="utf-8", + ) + + +def _set_github_outputs(source_commit: str, fixture_count: int, *, changed: bool) -> None: + github_output = os.environ.get("GITHUB_OUTPUT") + if github_output: + with Path(github_output).open("a", encoding="utf-8") as output_file: + output_file.write(f"source_commit={source_commit}\n") + output_file.write(f"fixture_count={fixture_count}\n") + output_file.write(f"changed={str(changed).lower()}\n") + + +def update_fixtures(fixture_ref: str) -> bool: + """Update the snapshot and return whether its allowed contents changed.""" + _validate_fixture_ref(fixture_ref) + + with tempfile.TemporaryDirectory(prefix="ffe-system-test-data-") as temporary_directory: + temporary_path = Path(temporary_directory) + source = temporary_path / "source" + snapshot = temporary_path / "snapshot" + source.mkdir() + snapshot.mkdir() + + _run_git(source, "init", "--quiet") + _run_git(source, "remote", "add", "origin", FIXTURE_REPOSITORY) + _run_git(source, "fetch", "--quiet", "--depth", "1", "origin", fixture_ref) + _run_git(source, "checkout", "--quiet", "--detach", "FETCH_HEAD") + source_commit = _run_git(source, "rev-parse", "HEAD") + + _copy_fixture_snapshot(source, snapshot) + fixture_count = _validate_fixture_snapshot(snapshot) + changed = not _have_same_contents(snapshot, FIXTURE_DESTINATION) + + if changed: + _write_source_metadata(snapshot, source_commit) + if FIXTURE_DESTINATION.exists(): + shutil.rmtree(FIXTURE_DESTINATION) + shutil.copytree(snapshot, FIXTURE_DESTINATION) + + _set_github_outputs(source_commit, fixture_count, changed=changed) + print(f"Checked FFE fixtures from DataDog/ffe-system-test-data@{source_commit}") + print(f"Loaded {fixture_count} canonical fixture cases") + print(f"Fixture snapshot changed: {str(changed).lower()}") + return changed + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--ref", default="main", help="Branch, tag, or commit from DataDog/ffe-system-test-data") + arguments = parser.parse_args() + update_fixtures(arguments.ref) + + +if __name__ == "__main__": + main()