test: additional semver coverage and error assertions

README.md

@@ -160,7 +160,10 @@ python3 ci/validate-fixtures.py
 | `test-case-numeric-one-of-default.json` | Numeric ONE_OF flag returning the default value when no rule matches |
 | `test-case-regex-flag.json` | Flag using regex matching operator |
 | `test-case-semver-comparison-flag.json` | Flag using semver comparison operators (SEMVER_EQ, SEMVER_NEQ, SEMVER_LT, SEMVER_LTE, SEMVER_GT, SEMVER_GTE), including prerelease ordering and invalid/missing attribute handling |
-| `test-case-semver-validation-flag.json` | Rust-compatible SemVer parsing boundaries, invalid syntax, and invalid configured comparands |
+| `test-case-semver-precedence-flag.json` | SemVer precedence edge cases that are easy to get wrong: multi-digit numeric prerelease ordering (beta.2 < beta.11), build-metadata-ignored across every operator (EQ/NEQ/LT/LTE/GT/GTE), and alphanumeric prerelease ordering (alpha < alpha.1 < alpha.beta < beta < release) |
+| `test-case-semver-validation-flag.json` | Rust-compatible SemVer parsing boundaries, invalid syntax, non-string attributes, and invalid configured comparands |
+| `test-case-semver-invalid-comparand-waterfall.json` | An invalid configured SemVer comparand in a non-last allocation of a waterfall must reject the whole flag at parse time rather than skipping it and falling through to a later valid default |
+| `test-case-semver-invalid-comparand-categories.json` | Configured SemVer comparand boundaries: a valid hyphen-heavy prerelease matches, while invalid syntax (including consecutive dots and a Ruby-style `.DEV` suffix), short, v-prefixed, leading-zero, and non-string comparands reject their flag at parse time with PARSE_ERROR without poisoning the rest of the configuration |
 | `test-case-start-and-end-date-flag.json` | Flag with start/end date time bounds |
 | `test-case-unknown-fields-tolerance.json` | Unknown UFC object fields are ignored |
 | `test-case-unknown-operator-isolation.json` | A flag with an unknown operator is removed without poisoning valid flags |

evaluation-cases/test-case-semver-invalid-comparand-categories.json

@@ -0,0 +1,135 @@
+[
+  {
+    "description": "A non-parseable SemVer comparand (\"not-a-version\") rejects the flag at parse time with a PARSE_ERROR rather than matching or coercing.",
+    "attributes": {
+      "app_version": "1.2.3"
+    },
+    "defaultValue": "unknown",
+    "flag": "semver-invalid-comparand-syntax-test",
+    "result": {
+      "errorCode": "PARSE_ERROR",
+      "reason": "ERROR",
+      "value": "unknown"
+    },
+    "targetingKey": "invalid-syntax",
+    "variationType": "STRING"
+  },
+  {
+    "description": "A SemVer comparand with repeated hyphens inside its prerelease identifiers (\"1.2.3----R-S.12.9.1--.12+meta\") is valid and matches normally.",
+    "attributes": {
+      "app_version": "1.2.3----R-S.12.9.1--.12+meta"
+    },
+    "defaultValue": "unknown",
+    "flag": "semver-valid-comparand-hyphenated-prerelease-test",
+    "result": {
+      "reason": "TARGETING_MATCH",
+      "value": "matched"
+    },
+    "targetingKey": "valid-hyphenated-prerelease",
+    "variationType": "STRING"
+  },
+  {
+    "description": "A SemVer comparand with consecutive dots in its prerelease identifiers (\"1.0.0-alpha..1\") rejects the flag at parse time with a PARSE_ERROR.",
+    "attributes": {
+      "app_version": "1.0.0-alpha.1"
+    },
+    "defaultValue": "unknown",
+    "flag": "semver-invalid-comparand-consecutive-dots-test",
+    "result": {
+      "errorCode": "PARSE_ERROR",
+      "reason": "ERROR",
+      "value": "unknown"
+    },
+    "targetingKey": "invalid-consecutive-dots",
+    "variationType": "STRING"
+  },
+  {
+    "description": "A Ruby-style dot-suffixed SemVer comparand (\"1.2.3.DEV\") rejects the flag at parse time with a PARSE_ERROR.",
+    "attributes": {
+      "app_version": "1.2.3"
+    },
+    "defaultValue": "unknown",
+    "flag": "semver-invalid-comparand-dot-suffix-test",
+    "result": {
+      "errorCode": "PARSE_ERROR",
+      "reason": "ERROR",
+      "value": "unknown"
+    },
+    "targetingKey": "invalid-dot-suffix",
+    "variationType": "STRING"
+  },
+  {
+    "description": "A short (two-component) SemVer comparand (\"1.2\") rejects the flag at parse time with a PARSE_ERROR.",
+    "attributes": {
+      "app_version": "1.2.3"
+    },
+    "defaultValue": "unknown",
+    "flag": "semver-invalid-comparand-short-test",
+    "result": {
+      "errorCode": "PARSE_ERROR",
+      "reason": "ERROR",
+      "value": "unknown"
+    },
+    "targetingKey": "invalid-short",
+    "variationType": "STRING"
+  },
+  {
+    "description": "A v-prefixed SemVer comparand (\"v1.2.3\") rejects the flag at parse time with a PARSE_ERROR.",
+    "attributes": {
+      "app_version": "1.2.3"
+    },
+    "defaultValue": "unknown",
+    "flag": "semver-invalid-comparand-vprefix-test",
+    "result": {
+      "errorCode": "PARSE_ERROR",
+      "reason": "ERROR",
+      "value": "unknown"
+    },
+    "targetingKey": "invalid-vprefix",
+    "variationType": "STRING"
+  },
+  {
+    "description": "A leading-zero core component in the SemVer comparand (\"01.2.3\") rejects the flag at parse time with a PARSE_ERROR.",
+    "attributes": {
+      "app_version": "1.2.3"
+    },
+    "defaultValue": "unknown",
+    "flag": "semver-invalid-comparand-leading-zero-test",
+    "result": {
+      "errorCode": "PARSE_ERROR",
+      "reason": "ERROR",
+      "value": "unknown"
+    },
+    "targetingKey": "invalid-leading-zero",
+    "variationType": "STRING"
+  },
+  {
+    "description": "A non-string (JSON number) SemVer comparand rejects the flag at parse time with a PARSE_ERROR rather than coercing the number to a string.",
+    "attributes": {
+      "app_version": "1.2.3"
+    },
+    "defaultValue": "unknown",
+    "flag": "semver-invalid-comparand-non-string-test",
+    "result": {
+      "errorCode": "PARSE_ERROR",
+      "reason": "ERROR",
+      "value": "unknown"
+    },
+    "targetingKey": "invalid-non-string",
+    "variationType": "STRING"
+  },
+  {
+    "description": "The rejected invalid-comparand flags must not poison the rest of the configuration: a neighboring valid SemVer flag still evaluates normally.",
+    "attributes": {
+      "app_version": "2.3.4"
+    },
+    "defaultValue": "unknown",
+    "flag": "semver-comparison-test",
+    "result": {
+      "reason": "TARGETING_MATCH",
+      "value": "stable"
+    },
+    "targetingKey": "after-invalid-categories",
+    "variationType": "STRING"
+  }
+]

evaluation-cases/test-case-semver-invalid-comparand-waterfall.json

@@ -0,0 +1,17 @@
+[
+  {
+    "description": "An invalid configured SemVer comparand in a non-last allocation must reject the whole flag at parse time, even though a valid default allocation follows it in the waterfall. An implementation that skips the invalid allocation and falls through to the default would return \"fallback\" with a STATIC reason instead of an error.",
+    "attributes": {
+      "app_version": "1.2.3"
+    },
+    "defaultValue": "unknown",
+    "flag": "semver-invalid-comparand-waterfall-test",
+    "result": {
+      "errorCode": "PARSE_ERROR",
+      "reason": "ERROR",
+      "value": "unknown"
+    },
+    "targetingKey": "waterfall",
+    "variationType": "STRING"
+  }
+]

evaluation-cases/test-case-semver-precedence-flag.json

@@ -0,0 +1,170 @@
+[
+  {
+    "description": "Multi-digit numeric prerelease identifiers compare by numeric value, not lexicographically: 1.0.0-beta.2 must precede 1.0.0-beta.11. An implementation that string-compares identifiers would treat \"2\" as greater than \"11\" and fail to match.",
+    "attributes": {
+      "app_version": "10.0.0-beta.2"
+    },
+    "defaultValue": "unknown",
+    "flag": "semver-precedence-test",
+    "result": {
+      "reason": "TARGETING_MATCH",
+      "value": "numeric-prerelease"
+    },
+    "targetingKey": "numeric-prerelease",
+    "variationType": "STRING"
+  },
+  {
+    "description": "Build metadata is ignored for equality: 1.0.0+linux and 1.0.0+darwin have equal precedence, so SEMVER_EQ matches.",
+    "attributes": {
+      "app_version": "20.0.0+linux"
+    },
+    "defaultValue": "unknown",
+    "flag": "semver-precedence-test",
+    "result": {
+      "reason": "TARGETING_MATCH",
+      "value": "build-equal-precedence"
+    },
+    "targetingKey": "build-equal-precedence",
+    "variationType": "STRING"
+  },
+  {
+    "description": "Dotted/multi-segment build metadata is ignored for equality: 4.0.0+exp.sha.5114f85 equals 4.0.0.",
+    "attributes": {
+      "app_version": "30.0.0+exp.sha.5114f85"
+    },
+    "defaultValue": "unknown",
+    "flag": "semver-precedence-test",
+    "result": {
+      "reason": "TARGETING_MATCH",
+      "value": "build-dotted-eq"
+    },
+    "targetingKey": "build-dotted-eq",
+    "variationType": "STRING"
+  },
+  {
+    "description": "Build metadata is ignored for SEMVER_LTE: 4.0.0+build.42 <= 4.0.0 is true (match).",
+    "attributes": {
+      "app_version": "40.0.0+build.42"
+    },
+    "defaultValue": "unknown",
+    "flag": "semver-precedence-test",
+    "result": {
+      "reason": "TARGETING_MATCH",
+      "value": "build-lte"
+    },
+    "targetingKey": "build-lte",
+    "variationType": "STRING"
+  },
+  {
+    "description": "Build metadata is ignored for SEMVER_GTE: 4.0.0+build.42 >= 4.0.0 is true (match).",
+    "attributes": {
+      "app_version": "50.0.0+build.42"
+    },
+    "defaultValue": "unknown",
+    "flag": "semver-precedence-test",
+    "result": {
+      "reason": "TARGETING_MATCH",
+      "value": "build-gte"
+    },
+    "targetingKey": "build-gte",
+    "variationType": "STRING"
+  },
+  {
+    "description": "Build metadata is ignored for SEMVER_NEQ: 4.0.0+build.42 != 4.0.0 is false (no match), so the evaluation falls through to the default.",
+    "attributes": {
+      "app_version": "60.0.0+build.42"
+    },
+    "defaultValue": "unknown",
+    "flag": "semver-precedence-test",
+    "result": {
+      "reason": "DEFAULT",
+      "value": "unknown"
+    },
+    "targetingKey": "build-neq",
+    "variationType": "STRING"
+  },
+  {
+    "description": "Build metadata is ignored for SEMVER_LT: 4.0.0+build.42 < 4.0.0 is false (no match), so the evaluation falls through to the default.",
+    "attributes": {
+      "app_version": "70.0.0+build.42"
+    },
+    "defaultValue": "unknown",
+    "flag": "semver-precedence-test",
+    "result": {
+      "reason": "DEFAULT",
+      "value": "unknown"
+    },
+    "targetingKey": "build-lt",
+    "variationType": "STRING"
+  },
+  {
+    "description": "Build metadata is ignored for SEMVER_GT: 4.0.0+build.42 > 4.0.0 is false (no match), so the evaluation falls through to the default.",
+    "attributes": {
+      "app_version": "80.0.0+build.42"
+    },
+    "defaultValue": "unknown",
+    "flag": "semver-precedence-test",
+    "result": {
+      "reason": "DEFAULT",
+      "value": "unknown"
+    },
+    "targetingKey": "build-gt",
+    "variationType": "STRING"
+  },
+  {
+    "description": "Alphanumeric prerelease ordering: 1.0.0-alpha precedes 1.0.0-alpha.1 (fewer identifiers first), so SEMVER_LT matches.",
+    "attributes": {
+      "app_version": "90.0.0-alpha"
+    },
+    "defaultValue": "unknown",
+    "flag": "semver-precedence-test",
+    "result": {
+      "reason": "TARGETING_MATCH",
+      "value": "alpha-before-alpha-1"
+    },
+    "targetingKey": "alpha-before-alpha-1",
+    "variationType": "STRING"
+  },
+  {
+    "description": "Alphanumeric prerelease ordering: numeric identifiers have lower precedence than alphanumeric (1.0.0-alpha.1 precedes 1.0.0-alpha.beta), so SEMVER_LT matches.",
+    "attributes": {
+      "app_version": "91.0.0-alpha.1"
+    },
+    "defaultValue": "unknown",
+    "flag": "semver-precedence-test",
+    "result": {
+      "reason": "TARGETING_MATCH",
+      "value": "alpha-1-before-alpha-beta"
+    },
+    "targetingKey": "alpha-1-before-alpha-beta",
+    "variationType": "STRING"
+  },
+  {
+    "description": "Alphanumeric prerelease ordering: identifiers are compared lexicographically (1.0.0-alpha.beta precedes 1.0.0-beta), so SEMVER_LT matches.",
+    "attributes": {
+      "app_version": "92.0.0-alpha.beta"
+    },
+    "defaultValue": "unknown",
+    "flag": "semver-precedence-test",
+    "result": {
+      "reason": "TARGETING_MATCH",
+      "value": "alpha-beta-before-beta"
+    },
+    "targetingKey": "alpha-beta-before-beta",
+    "variationType": "STRING"
+  },
+  {
+    "description": "Alphanumeric prerelease ordering: a prerelease version precedes the corresponding release (1.0.0-beta precedes 1.0.0), so SEMVER_LT matches.",
+    "attributes": {
+      "app_version": "93.0.0-beta"
+    },
+    "defaultValue": "unknown",
+    "flag": "semver-precedence-test",
+    "result": {
+      "reason": "TARGETING_MATCH",
+      "value": "beta-before-release"
+    },
+    "targetingKey": "beta-before-release",
+    "variationType": "STRING"
+  }
+]