From a4cddcfba2729506ca3bf651e3da4e292b4d31d6 Mon Sep 17 00:00:00 2001 From: Maxim Svistunov Date: Tue, 4 Aug 2026 10:35:04 +0200 Subject: [PATCH 01/14] LCORE-2343: add unified-mode e2e configuration fixtures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Create tests/e2e/configuration/unified-mode/ with library-mode/server-mode variants (same two-subdir layout configure_service resolves), covering the five unified-mode feature files: - unified-providers: minimal unified config driven only by top-level inference.providers over the default baseline (R1/S5); openai-specific. - unified-config-only / unified-relative-profile: profile: run.yaml — the CI-materialized repo-root run.yaml as baseline, provider-agnostic (R1/R8; two files because the features pin the intents separately). - unified-absolute-profile: container-absolute profile paths, differing per mode subdir (/app-root vs /opt/app-root mounts). - unified-native-override-{scalar,list}: R5 replacement semantics fixtures, synthesis-only, never booted. - invalid-{providers,config}-and-legacy: mutual-exclusion validation fixtures (R3); invalid-version-legacy-unified-body: R11 marker mismatch (needs LCORE-2872's cross-validation to fail for the right reason). - legacy-for-migration: legacy half of the migration fixture pair, kept free of enrichment sections so migrate-then-synthesize round-trips losslessly (LCORE-3370). Every fixture is validated against the real Configuration model: bootable ones load, invalid ones fail with the intended error. The test-generated lightspeed-stack-unified-migrated.yaml is gitignored, and the directory README documents each fixture's purpose. --- .gitignore | 3 ++ .../e2e/configuration/unified-mode/README.md | 24 ++++++++++++++ ...speed-stack-invalid-config-and-legacy.yaml | 25 ++++++++++++++ ...ed-stack-invalid-providers-and-legacy.yaml | 31 +++++++++++++++++ ...k-invalid-version-legacy-unified-body.yaml | 31 +++++++++++++++++ ...lightspeed-stack-legacy-for-migration.yaml | 23 +++++++++++++ ...tspeed-stack-unified-absolute-profile.yaml | 24 ++++++++++++++ .../lightspeed-stack-unified-config-only.yaml | 24 ++++++++++++++ ...ed-stack-unified-native-override-list.yaml | 29 ++++++++++++++++ ...-stack-unified-native-override-scalar.yaml | 28 ++++++++++++++++ .../lightspeed-stack-unified-providers.yaml | 29 ++++++++++++++++ ...tspeed-stack-unified-relative-profile.yaml | 24 ++++++++++++++ ...speed-stack-invalid-config-and-legacy.yaml | 27 +++++++++++++++ ...ed-stack-invalid-providers-and-legacy.yaml | 33 +++++++++++++++++++ ...k-invalid-version-legacy-unified-body.yaml | 33 +++++++++++++++++++ ...lightspeed-stack-legacy-for-migration.yaml | 23 +++++++++++++ ...tspeed-stack-unified-absolute-profile.yaml | 26 +++++++++++++++ .../lightspeed-stack-unified-config-only.yaml | 26 +++++++++++++++ ...ed-stack-unified-native-override-list.yaml | 31 +++++++++++++++++ ...-stack-unified-native-override-scalar.yaml | 30 +++++++++++++++++ .../lightspeed-stack-unified-providers.yaml | 31 +++++++++++++++++ ...tspeed-stack-unified-relative-profile.yaml | 26 +++++++++++++++ 22 files changed, 581 insertions(+) create mode 100644 tests/e2e/configuration/unified-mode/README.md create mode 100644 tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-invalid-config-and-legacy.yaml create mode 100644 tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-invalid-providers-and-legacy.yaml create mode 100644 tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-invalid-version-legacy-unified-body.yaml create mode 100644 tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-legacy-for-migration.yaml create mode 100644 tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-absolute-profile.yaml create mode 100644 tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-config-only.yaml create mode 100644 tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-native-override-list.yaml create mode 100644 tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-native-override-scalar.yaml create mode 100644 tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-providers.yaml create mode 100644 tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-relative-profile.yaml create mode 100644 tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-invalid-config-and-legacy.yaml create mode 100644 tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-invalid-providers-and-legacy.yaml create mode 100644 tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-invalid-version-legacy-unified-body.yaml create mode 100644 tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-legacy-for-migration.yaml create mode 100644 tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-absolute-profile.yaml create mode 100644 tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-config-only.yaml create mode 100644 tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-native-override-list.yaml create mode 100644 tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-native-override-scalar.yaml create mode 100644 tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-providers.yaml create mode 100644 tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-relative-profile.yaml diff --git a/.gitignore b/.gitignore index 15f06684a..35c352250 100644 --- a/.gitignore +++ b/.gitignore @@ -202,3 +202,6 @@ local-run.yaml .sisyphus/ # Per-developer feature design overrides (see docs/contributing/feature-design.config) .feature-design.config.local + +# Generated at e2e test time by the unified-mode --migrate-config step +tests/e2e/configuration/unified-mode/**/lightspeed-stack-unified-migrated.yaml diff --git a/tests/e2e/configuration/unified-mode/README.md b/tests/e2e/configuration/unified-mode/README.md new file mode 100644 index 000000000..79eaaabe0 --- /dev/null +++ b/tests/e2e/configuration/unified-mode/README.md @@ -0,0 +1,24 @@ +# Unified-mode e2e configuration fixtures + +Fixtures for the five `unified-mode-*.feature` files (LCORE-2341/LCORE-2343). +Same layout as the parent directory: `library-mode/` and `server-mode/` +variants differing only in the `llama_stack` block; the harness resolves +`//` via the standard `configure_service` logic. + +All profile-based fixtures reference `run.yaml` — the repo-root copy the CI +harness materializes from `tests/e2e/configs/run-.yaml` — so they stay +provider-agnostic across the providers matrix. + +| Fixture | Purpose | +|---|---| +| `lightspeed-stack-unified-providers.yaml` | Minimal unified config driven only by top-level `inference.providers` (default baseline, R1/S5). openai-specific — used by `@openai-only` scenarios. | +| `lightspeed-stack-unified-config-only.yaml` | Unified config driven only by `llama_stack.config` (`profile: run.yaml`, R1). | +| `lightspeed-stack-unified-relative-profile.yaml` | Same shape as config-only; exists to pin R8 (relative `profile:` resolves against the config file's directory) as a distinct intent. | +| `lightspeed-stack-unified-absolute-profile.yaml` | `profile:` as a container-absolute path (differs per mode subdir). | +| `lightspeed-stack-unified-native-override-scalar.yaml` | `native_override` replaces an overlapping scalar key (R5). Synthesis-only; never booted. | +| `lightspeed-stack-unified-native-override-list.yaml` | `native_override` replaces an overlapping list wholesale (R5). Synthesis-only; never booted. | +| `lightspeed-stack-invalid-providers-and-legacy.yaml` | INVALID: `inference.providers` + `library_client_config_path` (mutual exclusion, R3). Validation-only. | +| `lightspeed-stack-invalid-config-and-legacy.yaml` | INVALID: `llama_stack.config` + `library_client_config_path` (R3). Validation-only. | +| `lightspeed-stack-invalid-version-legacy-unified-body.yaml` | INVALID: `config_format_version: legacy` on a unified-shaped body (R11, LCORE-2872). Validation-only. | +| `lightspeed-stack-legacy-for-migration.yaml` | Legacy half of "the legacy migration fixture pair"; paired with the repo-root `run.yaml`. Deliberately free of enrichment sections so migrate→synthesize round-trips losslessly (see LCORE-3370). | +| `lightspeed-stack-unified-migrated.yaml` | Generated at test time by the `--migrate-config` step; gitignored and cleaned up after each scenario. | diff --git a/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-invalid-config-and-legacy.yaml b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-invalid-config-and-legacy.yaml new file mode 100644 index 000000000..bdb8af911 --- /dev/null +++ b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-invalid-config-and-legacy.yaml @@ -0,0 +1,25 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds the stack in-process + use_as_library_client: true + config: + profile: run.yaml + # INVALID: config block plus the legacy path (mutual exclusion, R3) + library_client_config_path: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-invalid-providers-and-legacy.yaml b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-invalid-providers-and-legacy.yaml new file mode 100644 index 000000000..124fd453c --- /dev/null +++ b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-invalid-providers-and-legacy.yaml @@ -0,0 +1,31 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds the stack in-process + use_as_library_client: true + # INVALID: synthesis input plus the legacy path (mutual exclusion, R3) + library_client_config_path: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini + # Unified synthesis input (Decision S5): the high-level provider entry + # replaces the default baseline's openai provider by id at synthesis time. + providers: + - type: openai + id: openai + api_key_env: OPENAI_API_KEY + allowed_models: + - ${env.E2E_OPENAI_MODEL:=gpt-4o-mini} diff --git a/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-invalid-version-legacy-unified-body.yaml b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-invalid-version-legacy-unified-body.yaml new file mode 100644 index 000000000..9ae7b389d --- /dev/null +++ b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-invalid-version-legacy-unified-body.yaml @@ -0,0 +1,31 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds the stack in-process + use_as_library_client: true +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini + # Unified synthesis input (Decision S5): the high-level provider entry + # replaces the default baseline's openai provider by id at synthesis time. + providers: + - type: openai + id: openai + api_key_env: OPENAI_API_KEY + allowed_models: + - ${env.E2E_OPENAI_MODEL:=gpt-4o-mini} +# INVALID: explicit legacy marker on a unified-shaped body (R11, LCORE-2872) +config_format_version: legacy diff --git a/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-legacy-for-migration.yaml b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-legacy-for-migration.yaml new file mode 100644 index 000000000..6393142b5 --- /dev/null +++ b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-legacy-for-migration.yaml @@ -0,0 +1,23 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds the stack in-process + use_as_library_client: true + # Legacy two-file shape: external run.yaml, no synthesis input + library_client_config_path: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-absolute-profile.yaml b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-absolute-profile.yaml new file mode 100644 index 000000000..089fb0afb --- /dev/null +++ b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-absolute-profile.yaml @@ -0,0 +1,24 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds the stack in-process + use_as_library_client: true + config: + # Absolute path as mounted in the library-mode container + profile: /app-root/run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-config-only.yaml b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-config-only.yaml new file mode 100644 index 000000000..b3df26828 --- /dev/null +++ b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-config-only.yaml @@ -0,0 +1,24 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds the stack in-process + use_as_library_client: true + config: + # Synthesis baseline: the CI-materialized run.yaml (provider-agnostic) + profile: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-native-override-list.yaml b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-native-override-list.yaml new file mode 100644 index 000000000..403cc7a00 --- /dev/null +++ b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-native-override-list.yaml @@ -0,0 +1,29 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds the stack in-process + use_as_library_client: true + config: + profile: run.yaml + # R5: lists replace wholesale - the synthesized apis must equal exactly + # this list, not a merge with the baseline's (never booted - synthesis only) + native_override: + apis: + - inference + - tool_runtime +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-native-override-scalar.yaml b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-native-override-scalar.yaml new file mode 100644 index 000000000..02f67f028 --- /dev/null +++ b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-native-override-scalar.yaml @@ -0,0 +1,28 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds the stack in-process + use_as_library_client: true + config: + profile: run.yaml + # R5: the raw escape hatch wins; this scalar replaces the baseline's + # safety.excluded_categories value wholesale (never booted - synthesis only) + native_override: + safety: + excluded_categories: unified-override-marker +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-providers.yaml b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-providers.yaml new file mode 100644 index 000000000..731c39b5d --- /dev/null +++ b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-providers.yaml @@ -0,0 +1,29 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds the stack in-process + use_as_library_client: true +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini + # Unified synthesis input (Decision S5): the high-level provider entry + # replaces the default baseline's openai provider by id at synthesis time. + providers: + - type: openai + id: openai + api_key_env: OPENAI_API_KEY + allowed_models: + - ${env.E2E_OPENAI_MODEL:=gpt-4o-mini} diff --git a/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-relative-profile.yaml b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-relative-profile.yaml new file mode 100644 index 000000000..228b2d40b --- /dev/null +++ b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-relative-profile.yaml @@ -0,0 +1,24 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds the stack in-process + use_as_library_client: true + config: + # R8: relative profile resolves against this file's loaded location + profile: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-invalid-config-and-legacy.yaml b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-invalid-config-and-legacy.yaml new file mode 100644 index 000000000..34dbfb06f --- /dev/null +++ b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-invalid-config-and-legacy.yaml @@ -0,0 +1,27 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to the separate llama-stack service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy + config: + profile: run.yaml + # INVALID: config block plus the legacy path (mutual exclusion, R3) + library_client_config_path: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-invalid-providers-and-legacy.yaml b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-invalid-providers-and-legacy.yaml new file mode 100644 index 000000000..6cbd5f50a --- /dev/null +++ b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-invalid-providers-and-legacy.yaml @@ -0,0 +1,33 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to the separate llama-stack service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy + # INVALID: synthesis input plus the legacy path (mutual exclusion, R3) + library_client_config_path: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini + # Unified synthesis input (Decision S5): the high-level provider entry + # replaces the default baseline's openai provider by id at synthesis time. + providers: + - type: openai + id: openai + api_key_env: OPENAI_API_KEY + allowed_models: + - ${env.E2E_OPENAI_MODEL:=gpt-4o-mini} diff --git a/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-invalid-version-legacy-unified-body.yaml b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-invalid-version-legacy-unified-body.yaml new file mode 100644 index 000000000..f3f9b9cdc --- /dev/null +++ b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-invalid-version-legacy-unified-body.yaml @@ -0,0 +1,33 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to the separate llama-stack service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini + # Unified synthesis input (Decision S5): the high-level provider entry + # replaces the default baseline's openai provider by id at synthesis time. + providers: + - type: openai + id: openai + api_key_env: OPENAI_API_KEY + allowed_models: + - ${env.E2E_OPENAI_MODEL:=gpt-4o-mini} +# INVALID: explicit legacy marker on a unified-shaped body (R11, LCORE-2872) +config_format_version: legacy diff --git a/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-legacy-for-migration.yaml b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-legacy-for-migration.yaml new file mode 100644 index 000000000..76b2ac36d --- /dev/null +++ b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-legacy-for-migration.yaml @@ -0,0 +1,23 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to the separate llama-stack service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-absolute-profile.yaml b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-absolute-profile.yaml new file mode 100644 index 000000000..1a1c4d8a1 --- /dev/null +++ b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-absolute-profile.yaml @@ -0,0 +1,26 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to the separate llama-stack service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy + config: + # Absolute path as mounted in the llama-stack container + profile: /opt/app-root/run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-config-only.yaml b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-config-only.yaml new file mode 100644 index 000000000..3881fdf4e --- /dev/null +++ b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-config-only.yaml @@ -0,0 +1,26 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to the separate llama-stack service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy + config: + # Synthesis baseline: the CI-materialized run.yaml (provider-agnostic) + profile: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-native-override-list.yaml b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-native-override-list.yaml new file mode 100644 index 000000000..7454bab43 --- /dev/null +++ b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-native-override-list.yaml @@ -0,0 +1,31 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to the separate llama-stack service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy + config: + profile: run.yaml + # R5: lists replace wholesale - the synthesized apis must equal exactly + # this list, not a merge with the baseline's (never booted - synthesis only) + native_override: + apis: + - inference + - tool_runtime +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-native-override-scalar.yaml b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-native-override-scalar.yaml new file mode 100644 index 000000000..3451fb526 --- /dev/null +++ b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-native-override-scalar.yaml @@ -0,0 +1,30 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to the separate llama-stack service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy + config: + profile: run.yaml + # R5: the raw escape hatch wins; this scalar replaces the baseline's + # safety.excluded_categories value wholesale (never booted - synthesis only) + native_override: + safety: + excluded_categories: unified-override-marker +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-providers.yaml b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-providers.yaml new file mode 100644 index 000000000..4ca585947 --- /dev/null +++ b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-providers.yaml @@ -0,0 +1,31 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to the separate llama-stack service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini + # Unified synthesis input (Decision S5): the high-level provider entry + # replaces the default baseline's openai provider by id at synthesis time. + providers: + - type: openai + id: openai + api_key_env: OPENAI_API_KEY + allowed_models: + - ${env.E2E_OPENAI_MODEL:=gpt-4o-mini} diff --git a/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-relative-profile.yaml b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-relative-profile.yaml new file mode 100644 index 000000000..3c6c8512a --- /dev/null +++ b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-relative-profile.yaml @@ -0,0 +1,26 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to the separate llama-stack service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy + config: + # R8: relative profile resolves against this file's loaded location + profile: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini From c2c39ce1b8b1f87a8bbb9535a34b8448f0cb1f0d Mon Sep 17 00:00:00 2001 From: Maxim Svistunov Date: Tue, 4 Aug 2026 10:35:04 +0200 Subject: [PATCH 02/14] LCORE-2343: restore legacy library-mode boot coverage with a dedicated fixture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LCORE-2342 migrated the standard library-mode baseline to unified mode (config.profile: run.yaml), which silently changed what unified-mode-legacy.feature's library scenario exercises: it now boots the unified baseline, not the deprecated two-file path, so R2's library-mode legacy coverage was gone. Add lightspeed-stack-legacy.yaml — identical to the baseline except its llama_stack block uses the true legacy shape (use_as_library_client + library_client_config_path: run.yaml, no synthesis input) — and point the library scenario's Given at it. This is the one deliberate Gherkin edit in LCORE-2343, agreed with Maxim in planning; the server-mode scenario is untouched since container-side enrichment there is genuinely legacy. --- .../library-mode/lightspeed-stack-legacy.yaml | 46 +++++++++++++++++++ .../e2e/features/unified-mode-legacy.feature | 5 +- 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 tests/e2e/configuration/library-mode/lightspeed-stack-legacy.yaml diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-legacy.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-legacy.yaml new file mode 100644 index 000000000..993f0d812 --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-legacy.yaml @@ -0,0 +1,46 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Legacy two-file shape (R2 deprecation window): external run.yaml consumed + # via library_client_config_path; no unified synthesis input. Kept as a + # dedicated fixture because the standard library-mode baseline migrated to + # unified mode in LCORE-2342, which silently removed legacy boot coverage. + use_as_library_client: true + library_client_config_path: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +byok_rag: + - rag_id: e2e-test-docs + rag_type: inline::faiss + embedding_model: sentence-transformers/all-mpnet-base-v2 + embedding_dimension: 768 + vector_db_id: ${env.FAISS_VECTOR_STORE_ID} + db_path: ${env.KV_RAG_PATH:=~/.llama/storage/rag/kv_store.db} + score_multiplier: 1.0 + +rag: + tool: + - e2e-test-docs + +shields: + - name: pii-redaction + provider_id: redaction + config: + rules: + - pattern: '\d+' + replacement: '[NUM]' + diff --git a/tests/e2e/features/unified-mode-legacy.feature b/tests/e2e/features/unified-mode-legacy.feature index 407c43e6a..d4b9c1402 100644 --- a/tests/e2e/features/unified-mode-legacy.feature +++ b/tests/e2e/features/unified-mode-legacy.feature @@ -12,7 +12,10 @@ Feature: Legacy two-file configuration during deprecation window @skip-in-server-mode Scenario: Legacy two-file configuration still boots and serves requests in library mode - Given The service uses the lightspeed-stack.yaml configuration + # lightspeed-stack-legacy.yaml (not the standard baseline): LCORE-2342 + # migrated the library-mode baseline to unified mode, so only a dedicated + # legacy-shaped fixture still exercises the deprecated two-file path (R2). + Given The service uses the lightspeed-stack-legacy.yaml configuration And The service is restarted When I access endpoint "readiness" using HTTP GET method Then The status code of the response is 200 From 82260afbeca711c5f74dde9898e9dcda14564c4d Mon Sep 17 00:00:00 2001 From: Maxim Svistunov Date: Tue, 4 Aug 2026 11:13:02 +0200 Subject: [PATCH 03/14] LCORE-2343: implement unified-mode step definitions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add tests/e2e/features/steps/unified_mode.py — the 16 step patterns the validation, migration, and synthesis features need (boot and legacy resolve entirely through existing generic steps). Per the planning decisions: - All artifact steps operate on the ON-DISK configuration (the repo-root lightspeed-stack.yaml copy configure_service applied), never the live service. - Validation runs the service CLI (--dump-configuration) as a black-box subprocess from the repo root and asserts a non-zero exit, so the error-contains assertions can never pass against a healthy load. - Migration runs the real --migrate-config CLI; the output lands in the active mode subdir under the gitignored name later Gherkin references, and is cleaned up per scenario. - Synthesis runs the config CLI exactly as the server entrypoint does (unified auto-detection -> synthesize_to_file, giving the 0600 mode the permissions scenario asserts). Round-trip and override assertions parse YAML and compare data, never bytes; override assertions are self-referential against the fixture's native_override and additionally assert the baseline differed, so replacements can't pass vacuously. - The --synthesized-config-output scenario launches a short-lived local service from the library-mode fixture variant on a rewritten port and polls for the custom output file (the flag is library-mode-only by design; running containers cannot be restarted with new CLI args). - The startup-log step is mode-aware: in server mode the synthesis evidence is emitted by the llama-stack container (entrypoint + CLI), not the lightspeed-stack container the Gherkin names — asserted against the synthesizing container with the rationale documented in the step. behave --dry-run over the five features: 24 scenarios, 200 steps, zero undefined. --- tests/e2e/features/steps/unified_mode.py | 477 +++++++++++++++++++++++ 1 file changed, 477 insertions(+) create mode 100644 tests/e2e/features/steps/unified_mode.py diff --git a/tests/e2e/features/steps/unified_mode.py b/tests/e2e/features/steps/unified_mode.py new file mode 100644 index 000000000..880db4171 --- /dev/null +++ b/tests/e2e/features/steps/unified_mode.py @@ -0,0 +1,477 @@ +"""Step definitions for the unified-mode e2e features (LCORE-2343). + +Covers configuration validation, legacy-to-unified migration, and run.yaml +synthesis for the five ``unified-mode-*.feature`` files. + +Design rules (from the LCORE-2343 planning notes): + +- Validation, migration, and synthesis steps operate on the **on-disk** + configuration artifacts — never the live service. "The active + configuration" is the repo-root ``lightspeed-stack.yaml`` copy that + ``configure_service`` applied. +- Migration and synthesis run the real CLIs as subprocesses — exactly the + surface the server entrypoint and operators use — and assertions parse + the produced YAML (data equality, never byte comparison). +- The synthesis-log step is mode-aware: in server mode the synthesis + evidence is emitted by the llama-stack container (entrypoint + CLI), not + the lightspeed-stack container the Gherkin names; the scenario's intent + (R10: the synthesized path is logged at startup) is asserted against the + container that actually synthesizes. +""" + +import difflib +import os +import re +import shutil +import stat +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import Any, Optional + +import yaml +from behave import given, step, then, when # pyright: ignore +from behave.runner import Context + +# Generated by the --migrate-config step; matches the .gitignore entry. +MIGRATED_CONFIG_BASENAME = "lightspeed-stack-unified-migrated.yaml" +# Legacy half of "the legacy migration fixture pair"; its run.yaml half is +# the repo-root run.yaml the CI harness materializes. +MIGRATION_PAIR_LCS_BASENAME = "lightspeed-stack-legacy-for-migration.yaml" + +CLI_TIMEOUT_SECONDS = 120 +CUSTOM_OUTPUT_POLL_SECONDS = 60 + + +def _mode_subdir(context: Context) -> str: + """Return the mode fixture subdirectory name for the current harness mode.""" + return "library-mode" if context.is_library_mode else "server-mode" + + +def _config_dir(context: Context) -> Path: + """Resolve the active fixture directory, mode subdir included when present. + + Mirrors ``configure_service``'s resolution so files referenced by name in + Gherkin (fixture pairs, migrated output) land where that step finds them. + """ + base = Path( + getattr(context, "lightspeed_stack_config_directory", "") + or "tests/e2e/configuration" + ) + mode_base = base / _mode_subdir(context) + return mode_base if mode_base.is_dir() else base + + +def _active_config_path() -> Path: + """Return the on-disk active configuration (the applied repo-root copy).""" + return Path("lightspeed-stack.yaml") + + +def _run_cli( + args: list[str], cwd: Optional[Path] = None +) -> subprocess.CompletedProcess: + """Run a repo CLI as a subprocess, capturing output, never raising.""" + return subprocess.run( + [sys.executable, *args], + cwd=str(cwd) if cwd else None, + capture_output=True, + text=True, + timeout=CLI_TIMEOUT_SECONDS, + check=False, + ) + + +def _load_yaml(path: Path) -> Any: + """Parse a YAML file.""" + with open(path, "r", encoding="utf-8") as f: + return yaml.safe_load(f) + + +def _native_override(config_path: Path) -> dict[str, Any]: + """Extract llama_stack.config.native_override from a config file.""" + config = _load_yaml(config_path) + override = ((config.get("llama_stack") or {}).get("config") or {}).get( + "native_override" + ) + assert override, f"{config_path} carries no llama_stack.config.native_override" + return override + + +def _synthesized(context: Context) -> Path: + """Return the synthesized run.yaml path recorded by an earlier step.""" + path = getattr(context, "synthesized_run_yaml_path", None) + assert path, "no synthesis step ran before this assertion" + return Path(path) + + +# --------------------------------------------------------------------------- +# Validation (unified-mode-validation.feature) +# --------------------------------------------------------------------------- + + +@when("configuration validation is attempted for the active configuration") +def attempt_configuration_validation(context: Context) -> None: + """Validate the on-disk active configuration via the service CLI. + + Runs ``lightspeed_stack.py --dump-configuration -c lightspeed-stack.yaml`` + as a black-box subprocess: ``main()`` loads (and thereby validates) the + configuration before any dump handling, so a Pydantic validation failure + surfaces on stderr with a non-zero exit code. cwd is the repo root so the + invalid fixtures' ``library_client_config_path: run.yaml`` resolves to the + harness-materialized run.yaml and the captured failure is the intended + cross-field error, not a file-not-found. + """ + result = _run_cli( + [ + "src/lightspeed_stack.py", + "--dump-configuration", + "-c", + str(_active_config_path()), + ] + ) + context.validation_returncode = result.returncode + context.validation_output = result.stdout + result.stderr + assert result.returncode != 0, ( + "expected the active configuration to fail validation, but the load " + f"succeeded (rc=0). Output:\n{context.validation_output}" + ) + + +@then("the validation error contains {text}") +def validation_error_contains(context: Context, text: str) -> None: + """Assert the captured validation failure mentions the given text.""" + output = getattr(context, "validation_output", None) + assert output is not None, "no validation attempt ran before this assertion" + assert ( + text.strip() in output + ), f"validation error does not contain {text!r}. Full output:\n{output}" + + +# --------------------------------------------------------------------------- +# Migration (unified-mode-migration.feature) +# --------------------------------------------------------------------------- + + +@step("lightspeed-stack --migrate-config is run for the legacy migration fixture pair") +def run_migrate_config(context: Context) -> None: + """Migrate the legacy fixture pair into the active fixture directory. + + The pair is ``lightspeed-stack-legacy-for-migration.yaml`` (mode subdir) + plus the repo-root ``run.yaml`` the harness materializes. The output lands + in the same mode subdir under the name later Gherkin steps reference, so + ``configure_service`` can boot it; it is gitignored and cleaned up after + the scenario. + """ + pair_lcs = _config_dir(context) / MIGRATION_PAIR_LCS_BASENAME + pair_run = Path("run.yaml") + output = _config_dir(context) / MIGRATED_CONFIG_BASENAME + assert pair_lcs.is_file(), f"missing migration fixture {pair_lcs}" + assert pair_run.is_file(), "repo-root run.yaml (harness-materialized) missing" + + result = _run_cli( + [ + "src/lightspeed_stack.py", + "--migrate-config", + "--run-yaml", + str(pair_run), + "-c", + str(pair_lcs), + "--migrate-output", + str(output), + ] + ) + assert result.returncode == 0 and output.is_file(), ( + f"--migrate-config failed (rc={result.returncode}).\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + context.migrated_config_path = output + context.migration_pair_run_yaml = pair_run + context.add_cleanup(lambda: output.unlink(missing_ok=True)) + + +@then("the file {filename} contains {text}") +def file_contains(context: Context, filename: str, text: str) -> None: + """Assert a file in the active fixture directory contains a substring.""" + path = _config_dir(context) / filename.strip() + content = path.read_text(encoding="utf-8") + assert text.strip() in content, f"{path} does not contain {text!r}" + + +@then("the file {filename} does not contain {text}") +def file_does_not_contain(context: Context, filename: str, text: str) -> None: + """Assert a file in the active fixture directory lacks a substring.""" + path = _config_dir(context) / filename.strip() + content = path.read_text(encoding="utf-8") + assert text.strip() not in content, f"{path} unexpectedly contains {text!r}" + + +# --------------------------------------------------------------------------- +# Synthesis (unified-mode-synthesis.feature + migration round-trip) +# --------------------------------------------------------------------------- + + +@step("the active unified configuration is synthesized to run.yaml") +def synthesize_active_configuration(context: Context) -> None: + """Synthesize a run.yaml from the on-disk unified configuration. + + Runs the config CLI exactly as the server entrypoint does (unified + auto-detection dispatches to ``synthesize_to_file``, which also gives the + 0600 output mode). Source precedence: the migrated config when the + migration step ran in this scenario, else the active on-disk config. + When the custom-output service step ran instead, this step is a + pass-through — the service subprocess performs the synthesis. + """ + if getattr(context, "custom_output_path", None): + return + + source = getattr(context, "migrated_config_path", None) or _active_config_path() + scratch = Path(tempfile.mkdtemp(prefix="lcore-e2e-synthesis-")) + context.add_cleanup(lambda: shutil.rmtree(scratch, ignore_errors=True)) + output = scratch / "run.yaml" + + result = _run_cli( + ["src/llama_stack_configuration.py", "-c", str(source), "-o", str(output)] + ) + assert result.returncode == 0 and output.is_file(), ( + f"synthesis CLI failed (rc={result.returncode}) for {source}.\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + context.synthesized_run_yaml_path = output + + +@then( + "the synthesized run.yaml parses to the same data as the legacy migration fixture run.yaml" +) +def synthesized_round_trips(context: Context) -> None: + """Assert migrate-then-synthesize reproduces the pair's run.yaml (data equality).""" + synthesized = _load_yaml(_synthesized(context)) + original = _load_yaml( + getattr(context, "migration_pair_run_yaml", None) or Path("run.yaml") + ) + if synthesized != original: + diff = "\n".join( + difflib.unified_diff( + yaml.dump(original, sort_keys=True).splitlines(), + yaml.dump(synthesized, sort_keys=True).splitlines(), + fromfile="pair run.yaml", + tofile="synthesized", + lineterm="", + ) + ) + raise AssertionError(f"round-trip data mismatch:\n{diff}") + + +@then( + "the synthesized run.yaml contains the native_override scalar value for safety.excluded_categories" +) +def synthesized_scalar_override(context: Context) -> None: + """Assert the override's scalar replaced the baseline value at that key (R5).""" + override_value = _native_override(_active_config_path())["safety"][ + "excluded_categories" + ] + synthesized = _load_yaml(_synthesized(context)) + actual = (synthesized.get("safety") or {}).get("excluded_categories") + assert actual == override_value, ( + f"safety.excluded_categories is {actual!r}, expected the " + f"native_override value {override_value!r}" + ) + baseline = _load_yaml(Path("run.yaml")) + baseline_value = (baseline.get("safety") or {}).get("excluded_categories") + assert baseline_value != override_value, ( + "fixture and baseline agree on safety.excluded_categories — the " + "replacement assertion would be vacuous" + ) + + +@then("the synthesized run.yaml contains exactly the native_override list for apis") +def synthesized_list_override(context: Context) -> None: + """Assert the override list replaced the baseline's apis wholesale (R5).""" + override_list = _native_override(_active_config_path())["apis"] + synthesized = _load_yaml(_synthesized(context)) + assert ( + synthesized.get("apis") == override_list + ), f"apis is {synthesized.get('apis')!r}, expected exactly {override_list!r}" + baseline = _load_yaml(Path("run.yaml")) + assert ( + baseline.get("apis") != override_list + ), "fixture and baseline agree on apis — wholesale replacement would be vacuous" + + +@then("the synthesized run.yaml contains ${{env.OPENAI_API_KEY}}") +def synthesized_keeps_env_reference(context: Context) -> None: + """Assert the emitted secret stays an environment reference on disk (R6).""" + content = _synthesized(context).read_text(encoding="utf-8") + assert ( + "${env.OPENAI_API_KEY}" in content + ), "synthesized run.yaml does not carry the ${env.OPENAI_API_KEY} reference" + + +@then("the synthesized run.yaml does not contain the resolved OPENAI_API_KEY value") +def synthesized_no_literal_secret(context: Context) -> None: + """Assert the literal secret value never lands on disk (R6).""" + secret = os.environ.get("OPENAI_API_KEY", "") + assert secret, ( + "OPENAI_API_KEY is not set in the harness environment — the " + "no-literal-secret assertion would be vacuous" + ) + content = _synthesized(context).read_text(encoding="utf-8") + assert ( + secret not in content + ), "synthesized run.yaml contains the resolved OPENAI_API_KEY value" + + +@then("the synthesized run.yaml file permissions are 0600") +def synthesized_permissions(context: Context) -> None: + """Assert the synthesized file is owner-read/write only (R10).""" + mode = stat.S_IMODE(os.stat(_synthesized(context)).st_mode) + assert mode == 0o600, f"synthesized run.yaml mode is {oct(mode)}, expected 0o600" + + +# --------------------------------------------------------------------------- +# --synthesized-config-output (unified-mode-synthesis.feature) +# --------------------------------------------------------------------------- + + +@given( + "lightspeed-stack is started with --synthesized-config-output set to a custom path" +) +def start_with_custom_synthesis_output(context: Context) -> None: + """Launch a short-lived local service with a custom synthesis output path. + + The flag only affects library-mode in-process synthesis, and the running + containers cannot be restarted with different CLI args — so this step + always uses the library-mode variant of the active fixture, copied to a + scratch directory with ``service.port`` rewritten to avoid clashing with + the running stack. The subprocess synthesizes during app startup; the + following Then steps poll for the file, and the process is killed on + scenario cleanup. + """ + active_basename = _active_config_path().name + fixture_basename = Path( + getattr(context, "feature_config", "") + or "lightspeed-stack-unified-providers.yaml" + ).name + base = Path( + getattr(context, "lightspeed_stack_config_directory", "") + or "tests/e2e/configuration" + ) + library_fixture = base / "library-mode" / fixture_basename + if not library_fixture.is_file(): + library_fixture = _config_dir(context) / fixture_basename + assert library_fixture.is_file(), f"no library-mode fixture for {active_basename}" + + scratch = Path(tempfile.mkdtemp(prefix="lcore-e2e-synthout-")) + context.add_cleanup(lambda: shutil.rmtree(scratch, ignore_errors=True)) + + config = _load_yaml(library_fixture) + config.setdefault("service", {})["port"] = 8099 + scratch_config = scratch / "lightspeed-stack.yaml" + with open(scratch_config, "w", encoding="utf-8") as f: + yaml.safe_dump(config, f) + + custom_output = scratch / "custom-run.yaml" + process = subprocess.Popen( # pylint: disable=consider-using-with + [ + sys.executable, + str(Path("src/lightspeed_stack.py").resolve()), + "-c", + str(scratch_config), + "--synthesized-config-output", + str(custom_output), + ], + cwd=str(scratch), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + context.custom_output_path = custom_output + context.custom_output_scratch = scratch + context.custom_output_process = process + + def _kill() -> None: + if process.poll() is None: + process.kill() + process.wait(timeout=10) + + context.add_cleanup(_kill) + + +@then("the synthesized run.yaml is written to the custom output path") +def custom_output_written(context: Context) -> None: + """Poll for the custom-path synthesis output and validate it parses.""" + custom_output = Path(context.custom_output_path) + process = context.custom_output_process + deadline = time.monotonic() + CUSTOM_OUTPUT_POLL_SECONDS + while time.monotonic() < deadline: + if custom_output.is_file() and custom_output.stat().st_size > 0: + break + if process.poll() is not None and not custom_output.is_file(): + out = process.stdout.read() if process.stdout else "" + raise AssertionError( + f"service exited (rc={process.returncode}) before writing the " + f"custom synthesis output.\n{out[-2000:]}" + ) + time.sleep(0.5) + assert custom_output.is_file(), ( + f"custom synthesis output {custom_output} did not appear within " + f"{CUSTOM_OUTPUT_POLL_SECONDS}s" + ) + assert isinstance(_load_yaml(custom_output), dict) + context.synthesized_run_yaml_path = custom_output + + +@then("the default synthesized run.yaml path does not exist") +def default_output_absent(context: Context) -> None: + """Assert the default synthesis location was not used (override took effect).""" + scratch = Path(context.custom_output_scratch) + default_path = scratch / ".generated" / "run.yaml" + assert ( + not default_path.exists() + ), f"default synthesis path {default_path} exists despite the override" + + +# --------------------------------------------------------------------------- +# Startup logging (unified-mode-synthesis.feature) — mode-aware, see module +# docstring and LCORE-2343 planning decision Q2. +# --------------------------------------------------------------------------- + + +@then("the lightspeed-stack container logs contain synthesized run.yaml") +def container_logs_show_synthesis(context: Context) -> None: + """Assert the container that synthesizes logged the synthesized-config path. + + Library mode: the lightspeed-stack container itself synthesizes in-process + and logs "Using synthesized Llama Stack config at ". Server mode: + synthesis happens in the llama-stack container (entrypoint + config CLI), + which logs the generated-config path — the Gherkin names lightspeed-stack, + but the scenario's intent (R10: the path is logged at startup) can only be + observed on the synthesizing container. Deviation agreed in planning (Q2). + """ + if context.is_library_mode: + container = "lightspeed-stack" + pattern = r"synthesized.*run\.yaml|Using synthesized Llama Stack config" + else: + container = "llama-stack" + pattern = ( + r"Wrote synthesized Llama Stack configuration" + r"|Using generated config:.*run\.yaml" + r"|mode auto-detected" + ) + + result = subprocess.run( + ["docker", "logs", container], + capture_output=True, + text=True, + timeout=60, + check=False, + ) + assert ( + result.returncode == 0 + ), f"docker logs {container} failed: {result.stderr[-500:]}" + logs = result.stdout + result.stderr + assert re.search(pattern, logs), ( + f"{container} logs carry no synthesis-path evidence " + f"(pattern {pattern!r} not found)" + ) From 9e5ed0fbad4d473c94e716e44c045acb572f36db Mon Sep 17 00:00:00 2001 From: Maxim Svistunov Date: Tue, 4 Aug 2026 11:13:02 +0200 Subject: [PATCH 04/14] LCORE-2343: gate unified-mode features for Prow and the providers matrix Tag all five unified-mode features @skip-in-prow: the new steps rely on Docker containers and local subprocesses, neither of which exists in the Prow environment (existing convention, handled in before_scenario). Add an @openai-only tag on the two inference.providers boot scenarios and a matching before_scenario skip keyed on E2E_DEFAULT_PROVIDER_OVERRIDE: the providers workflow runs the full unsharded test list against azure/watsonx/bedrock matrices, and the unified-providers fixture hardcodes an openai provider that cannot serve those models' queries. Profile-based fixtures stay provider-agnostic (they consume the CI-materialized run.yaml) and need no gating. --- tests/e2e/features/environment.py | 15 +++++++++++++++ tests/e2e/features/unified-mode-boot.feature | 6 +++--- tests/e2e/features/unified-mode-legacy.feature | 2 +- tests/e2e/features/unified-mode-migration.feature | 2 +- tests/e2e/features/unified-mode-synthesis.feature | 2 +- .../e2e/features/unified-mode-validation.feature | 2 +- 6 files changed, 22 insertions(+), 7 deletions(-) diff --git a/tests/e2e/features/environment.py b/tests/e2e/features/environment.py index 273d80ec9..152ed2b73 100644 --- a/tests/e2e/features/environment.py +++ b/tests/e2e/features/environment.py @@ -242,6 +242,21 @@ def before_scenario(context: Context, scenario: Scenario) -> None: scenario.skip("Skipped in Prow (requires Docker Compose services)") return + # Skip openai-specific scenarios on non-openai provider matrices: the + # providers workflow runs the full test list with E2E_DEFAULT_PROVIDER_OVERRIDE + # set (azure/watsonx/...), and fixtures that hardcode an openai provider + # (e.g. the unified-mode inference.providers fixture) cannot serve queries + # for those models. + provider_override = os.getenv("E2E_DEFAULT_PROVIDER_OVERRIDE", "") + if "openai-only" in scenario.effective_tags and provider_override not in ( + "", + "openai", + ): + scenario.skip( + f"Skipped on provider matrix '{provider_override}' (openai-only fixture)" + ) + return + # In Prow, verify the lightspeed port-forward is alive before each scenario. # Port-forwards can silently die between scenarios (e.g. pod restart, TCP reset). if is_prow_environment(): diff --git a/tests/e2e/features/unified-mode-boot.feature b/tests/e2e/features/unified-mode-boot.feature index 7c1413818..fc91713de 100644 --- a/tests/e2e/features/unified-mode-boot.feature +++ b/tests/e2e/features/unified-mode-boot.feature @@ -1,4 +1,4 @@ -@cfg_unified @skip +@cfg_unified @skip @skip-in-prow Feature: Unified mode configuration boot Background: @@ -10,7 +10,7 @@ Feature: Unified mode configuration boot # --- library mode (@skip-in-server-mode) --- - @skip-in-server-mode + @skip-in-server-mode @openai-only Scenario: Unified config with inference.providers boots and serves requests in library mode Given The service uses the lightspeed-stack-unified-providers.yaml configuration And The service is restarted @@ -62,7 +62,7 @@ Feature: Unified mode configuration boot # --- server mode (@skip-in-library-mode) --- - @skip-in-library-mode + @skip-in-library-mode @openai-only Scenario: Unified config with inference.providers boots and serves requests in server mode Given The service uses the lightspeed-stack-unified-providers.yaml configuration And Llama Stack is restarted diff --git a/tests/e2e/features/unified-mode-legacy.feature b/tests/e2e/features/unified-mode-legacy.feature index d4b9c1402..9404e09a8 100644 --- a/tests/e2e/features/unified-mode-legacy.feature +++ b/tests/e2e/features/unified-mode-legacy.feature @@ -1,4 +1,4 @@ -@cfg_unified @skip +@cfg_unified @skip @skip-in-prow Feature: Legacy two-file configuration during deprecation window Background: diff --git a/tests/e2e/features/unified-mode-migration.feature b/tests/e2e/features/unified-mode-migration.feature index b5a2d06eb..f6773327e 100644 --- a/tests/e2e/features/unified-mode-migration.feature +++ b/tests/e2e/features/unified-mode-migration.feature @@ -1,4 +1,4 @@ -@cfg_unified @skip +@cfg_unified @skip @skip-in-prow Feature: Legacy to unified configuration migration Background: diff --git a/tests/e2e/features/unified-mode-synthesis.feature b/tests/e2e/features/unified-mode-synthesis.feature index 52b4254fa..83a4f9366 100644 --- a/tests/e2e/features/unified-mode-synthesis.feature +++ b/tests/e2e/features/unified-mode-synthesis.feature @@ -1,4 +1,4 @@ -@cfg_unified @skip +@cfg_unified @skip @skip-in-prow Feature: Unified mode configuration synthesis Background: diff --git a/tests/e2e/features/unified-mode-validation.feature b/tests/e2e/features/unified-mode-validation.feature index 5a1688335..41563a4c5 100644 --- a/tests/e2e/features/unified-mode-validation.feature +++ b/tests/e2e/features/unified-mode-validation.feature @@ -1,4 +1,4 @@ -@cfg_unified @skip +@cfg_unified @skip @skip-in-prow Feature: Unified mode configuration validation Background: From 568ef6007e42ca65f6ae9a7723127af87662a299 Mon Sep 17 00:00:00 2001 From: Maxim Svistunov Date: Tue, 4 Aug 2026 11:20:09 +0200 Subject: [PATCH 05/14] LCORE-2343: unskip the unified-mode feature files Remove the @skip placeholder tag from the five unified-mode features: the step definitions and fixtures they need now exist. The features keep @skip-in-prow (Docker/subprocess dependencies) and stay in test_list.txt under @e2e_group_2, so CI shards pick them up via 'not @skip and @e2e_group_2' and local runs via --tags=-skip. --- tests/e2e/features/unified-mode-boot.feature | 2 +- tests/e2e/features/unified-mode-legacy.feature | 2 +- tests/e2e/features/unified-mode-migration.feature | 2 +- tests/e2e/features/unified-mode-synthesis.feature | 2 +- tests/e2e/features/unified-mode-validation.feature | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/e2e/features/unified-mode-boot.feature b/tests/e2e/features/unified-mode-boot.feature index fc91713de..a78928200 100644 --- a/tests/e2e/features/unified-mode-boot.feature +++ b/tests/e2e/features/unified-mode-boot.feature @@ -1,4 +1,4 @@ -@cfg_unified @skip @skip-in-prow +@cfg_unified @skip-in-prow Feature: Unified mode configuration boot Background: diff --git a/tests/e2e/features/unified-mode-legacy.feature b/tests/e2e/features/unified-mode-legacy.feature index 9404e09a8..e54a64950 100644 --- a/tests/e2e/features/unified-mode-legacy.feature +++ b/tests/e2e/features/unified-mode-legacy.feature @@ -1,4 +1,4 @@ -@cfg_unified @skip @skip-in-prow +@cfg_unified @skip-in-prow Feature: Legacy two-file configuration during deprecation window Background: diff --git a/tests/e2e/features/unified-mode-migration.feature b/tests/e2e/features/unified-mode-migration.feature index f6773327e..68cd2aaeb 100644 --- a/tests/e2e/features/unified-mode-migration.feature +++ b/tests/e2e/features/unified-mode-migration.feature @@ -1,4 +1,4 @@ -@cfg_unified @skip @skip-in-prow +@cfg_unified @skip-in-prow Feature: Legacy to unified configuration migration Background: diff --git a/tests/e2e/features/unified-mode-synthesis.feature b/tests/e2e/features/unified-mode-synthesis.feature index 83a4f9366..3bfba3a65 100644 --- a/tests/e2e/features/unified-mode-synthesis.feature +++ b/tests/e2e/features/unified-mode-synthesis.feature @@ -1,4 +1,4 @@ -@cfg_unified @skip @skip-in-prow +@cfg_unified @skip-in-prow Feature: Unified mode configuration synthesis Background: diff --git a/tests/e2e/features/unified-mode-validation.feature b/tests/e2e/features/unified-mode-validation.feature index 41563a4c5..0a03bb4f3 100644 --- a/tests/e2e/features/unified-mode-validation.feature +++ b/tests/e2e/features/unified-mode-validation.feature @@ -1,4 +1,4 @@ -@cfg_unified @skip @skip-in-prow +@cfg_unified @skip-in-prow Feature: Unified mode configuration validation Background: From d60105f37bae955dcc4ad8c5d99dd22848542a65 Mon Sep 17 00:00:00 2001 From: Maxim Svistunov Date: Tue, 4 Aug 2026 11:34:05 +0200 Subject: [PATCH 06/14] LCORE-2343: close the health-vs-listen race in lightspeed restarts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit restart_container waits on docker health, but docker can report healthy before uvicorn binds the published port — the exact race wait_for_lightspeed_stack_http_ready documents and was, until now, only closed in the proxy steps. The unified-mode boot scenarios are the slowest restarts in the suite (first unified/default-baseline boots) and hit that window reliably: the restart step passed while the following readiness GET got connection-refused. Call the existing HTTP-ready wait from restart_container for the lightspeed-stack container, closing the race for every restarting scenario; when the port is already accepting, the first poll returns immediately. --- tests/e2e/utils/utils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/e2e/utils/utils.py b/tests/e2e/utils/utils.py index b9cb03806..5e0774a3f 100644 --- a/tests/e2e/utils/utils.py +++ b/tests/e2e/utils/utils.py @@ -507,6 +507,14 @@ def restart_container(container_name: str) -> None: # that restart the container don't time out. wait_for_container_health(container_name) + # Docker health can report healthy before uvicorn binds the published + # port (the documented race wait_for_lightspeed_stack_http_ready exists + # for). Unified-mode first boots are the slowest restarts in the suite + # and hit that window reliably, so close it here for every restart + # rather than only in the proxy steps. + if container_name == "lightspeed-stack": + wait_for_lightspeed_stack_http_ready() + if container_name == "llama-stack": from tests.e2e.features.steps.health import ( reset_llama_stack_disrupt_once_tracking, From 9a081ac4e2184d78936d7cefafd32408068c7aa1 Mon Sep 17 00:00:00 2001 From: Maxim Svistunov Date: Tue, 4 Aug 2026 12:09:05 +0200 Subject: [PATCH 07/14] LCORE-2343: make the migrated e2e config readable by the container user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --migrate-config writes its output 0600 (R10: migrated files may carry lifted secrets), but the boot scenarios copy that file to the repo root for the container to consume, and the container user cannot read a host-owned 0600 file — the migrated-config boot scenario died on config read. Relax the harness copy to 0644 after a successful migration; the fixture pair is env-reference-only by design, so no secret can leak. --- tests/e2e/features/steps/unified_mode.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/e2e/features/steps/unified_mode.py b/tests/e2e/features/steps/unified_mode.py index 880db4171..f865f913a 100644 --- a/tests/e2e/features/steps/unified_mode.py +++ b/tests/e2e/features/steps/unified_mode.py @@ -186,6 +186,11 @@ def run_migrate_config(context: Context) -> None: f"--migrate-config failed (rc={result.returncode}).\n" f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" ) + # The CLI writes 0600 (R10: migrated files may carry lifted secrets), but + # the container user must be able to read the copy configure_service puts + # at the repo root to boot it. The fixture pair is env-reference-only by + # design, so relaxing the harness copy is safe. + os.chmod(output, 0o644) context.migrated_config_path = output context.migration_pair_run_yaml = pair_run context.add_cleanup(lambda: output.unlink(missing_ok=True)) From 9783e358cb1b06a3e267967d9b0cd449c5185c4d Mon Sep 17 00:00:00 2001 From: Maxim Svistunov Date: Wed, 26 Aug 2026 12:43:58 +0200 Subject: [PATCH 08/14] LCORE-2343: migrate the legacy e2e fixture to the unified rag schema The library-mode legacy fixture declared its BYOK store under a top-level byok_rag key with rag_type, and its tool retrieval sources under rag.tool. LCORE-1426 (commit c1de7f91) refactored RAG configuration into a single rag section: stores moved to rag.byok.stores, retrieval sources to rag.retrieval.tool.sources, and RagStore.rag_type was replaced by RagStore.backend, whose validator accepts only the values in SUPPORTED_RAG_BACKENDS (faiss, pgvector). Configuration models inherit ConfigurationBase with extra=forbid, so after rebasing onto main this fixture raised two extra_forbidden validation errors (rag.tool and byok_rag) and the config could not be loaded at all, failing every legacy library-mode scenario that consumes it. Move the store under rag.byok.stores, replace rag_type: inline::faiss with backend: faiss, and nest the retrieval source list under rag.retrieval.tool.sources. score_multiplier, db_path, embedding_model, embedding_dimension and vector_db_id are unchanged and remain valid RagStore fields. The source id stays e2e-test-docs so the validate_retrieval_sources model validator still resolves it against the declared store. --- .../library-mode/lightspeed-stack-legacy.yaml | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-legacy.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-legacy.yaml index 993f0d812..aedf2fc72 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-legacy.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-legacy.yaml @@ -23,18 +23,20 @@ authentication: inference: default_provider: openai default_model: gpt-4o-mini -byok_rag: - - rag_id: e2e-test-docs - rag_type: inline::faiss - embedding_model: sentence-transformers/all-mpnet-base-v2 - embedding_dimension: 768 - vector_db_id: ${env.FAISS_VECTOR_STORE_ID} - db_path: ${env.KV_RAG_PATH:=~/.llama/storage/rag/kv_store.db} - score_multiplier: 1.0 - rag: - tool: - - e2e-test-docs + byok: + stores: + - rag_id: e2e-test-docs + backend: faiss + embedding_model: sentence-transformers/all-mpnet-base-v2 + embedding_dimension: 768 + vector_db_id: ${env.FAISS_VECTOR_STORE_ID} + db_path: ${env.KV_RAG_PATH:=~/.llama/storage/rag/kv_store.db} + score_multiplier: 1.0 + retrieval: + tool: + sources: + - e2e-test-docs shields: - name: pii-redaction From db64eed12b0b17352f2ee06629d22710e2688690 Mon Sep 17 00:00:00 2001 From: Maxim Svistunov Date: Thu, 27 Aug 2026 16:21:06 +0200 Subject: [PATCH 09/14] LCORE-2343: assert the migrate-config 0600 mode instead of relaxing it The migration step ran --migrate-config straight into the fixture directory and then chmod'ed that file to 0644 so the container user could read the copy configure_service places at the repo root. That widened the CLI's own output, which is the artifact R10 governs: migrated configurations may carry secrets lifted out of the legacy run.yaml, and the CLI deliberately writes them owner-only (it logs "mode 0600" when it does). The harness therefore destroyed the property it exists to protect, and no scenario noticed because the 0600 assertion only covers the synthesized run.yaml, not the migrated config. A fixture that ever carries a real secret would have had it published world-readable with nothing failing. Migrate into a scratch directory instead, assert the CLI wrote 0600 there, and publish a separate deliberate 0644 copy under the name the Gherkin steps reference for configure_service to boot. The mode relaxation now applies to a copy that exists only for the harness, the CLI artifact keeps its mode, and the R10 guarantee gains the direct assertion it previously lacked. --- tests/e2e/features/steps/unified_mode.py | 30 +++++++++++++++++++----- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/tests/e2e/features/steps/unified_mode.py b/tests/e2e/features/steps/unified_mode.py index f865f913a..d7699ac84 100644 --- a/tests/e2e/features/steps/unified_mode.py +++ b/tests/e2e/features/steps/unified_mode.py @@ -170,6 +170,13 @@ def run_migrate_config(context: Context) -> None: assert pair_lcs.is_file(), f"missing migration fixture {pair_lcs}" assert pair_run.is_file(), "repo-root run.yaml (harness-materialized) missing" + # Migrate into a scratch directory so the CLI's own artifact keeps the + # mode it was written with and can be asserted on (R10) instead of being + # relaxed in place. + scratch = Path(tempfile.mkdtemp(prefix="lcore-e2e-migrate-")) + context.add_cleanup(lambda: shutil.rmtree(scratch, ignore_errors=True)) + cli_output = scratch / MIGRATED_CONFIG_BASENAME + result = _run_cli( [ "src/lightspeed_stack.py", @@ -179,18 +186,29 @@ def run_migrate_config(context: Context) -> None: "-c", str(pair_lcs), "--migrate-output", - str(output), + str(cli_output), ] ) - assert result.returncode == 0 and output.is_file(), ( + assert result.returncode == 0 and cli_output.is_file(), ( f"--migrate-config failed (rc={result.returncode}).\n" f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" ) - # The CLI writes 0600 (R10: migrated files may carry lifted secrets), but - # the container user must be able to read the copy configure_service puts - # at the repo root to boot it. The fixture pair is env-reference-only by - # design, so relaxing the harness copy is safe. + # R10: migrated files may carry lifted secrets, so the CLI must write them + # owner-only. Assert it here rather than silently relaxing the artifact. + cli_mode = stat.S_IMODE(os.stat(cli_output).st_mode) + assert cli_mode == 0o600, ( + f"--migrate-config wrote {cli_output} with mode {oct(cli_mode)}, " + "expected 0o600 (R10)" + ) + + # configure_service boots a repo-root copy of this file, and the container + # user cannot read a host-owned 0600 file. Publish a deliberate 0644 *copy* + # into the fixture directory for the harness to boot; the CLI artifact + # above keeps its 0600 mode. The fixture pair is env-reference-only by + # design, so nothing secret is widened. + shutil.copyfile(cli_output, output) os.chmod(output, 0o644) + context.migrated_cli_output_path = cli_output context.migrated_config_path = output context.migration_pair_run_yaml = pair_run context.add_cleanup(lambda: output.unlink(missing_ok=True)) From 676006b75467b13c98e3f69ccfc1996f50ccb772 Mon Sep 17 00:00:00 2001 From: Maxim Svistunov Date: Thu, 27 Aug 2026 16:21:50 +0200 Subject: [PATCH 10/14] LCORE-2343: match the synthesis log assertions to the OGX-renamed messages The startup-logging step matched "Using synthesized Llama Stack config" in library mode and "Wrote synthesized Llama Stack configuration" in server mode. The OGX rename (PRs #2516 and #2547) replaced both: client.py now logs "Using synthesized OGX config at %s" and llama_stack_configuration.py logs "Wrote synthesized OGX configuration to %s (mode 0600)". Neither scenario started failing, which is the reason to fix it now rather than after a real breakage. Library mode kept passing only because the sibling alternative "synthesized.*run\.yaml" incidentally matches the OGX line, the synthesized file being named run.yaml; server mode kept passing on the entrypoint's own "Using generated config:" and "mode auto-detected" echoes. Both scenarios were therefore asserting something other than the message they name, and would have gone silently unprotected the moment the output filename or the entrypoint echoes changed. Point the patterns at the current messages and record in the docstring where each one is emitted, so the next rename has an obvious place to look. --- tests/e2e/features/steps/unified_mode.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/e2e/features/steps/unified_mode.py b/tests/e2e/features/steps/unified_mode.py index d7699ac84..418d5af19 100644 --- a/tests/e2e/features/steps/unified_mode.py +++ b/tests/e2e/features/steps/unified_mode.py @@ -466,19 +466,23 @@ def container_logs_show_synthesis(context: Context) -> None: """Assert the container that synthesizes logged the synthesized-config path. Library mode: the lightspeed-stack container itself synthesizes in-process - and logs "Using synthesized Llama Stack config at ". Server mode: - synthesis happens in the llama-stack container (entrypoint + config CLI), - which logs the generated-config path — the Gherkin names lightspeed-stack, - but the scenario's intent (R10: the path is logged at startup) can only be - observed on the synthesizing container. Deviation agreed in planning (Q2). + and logs "Using synthesized OGX config at ". Server mode: synthesis + happens in the llama-stack container (entrypoint + config CLI), which logs + the generated-config path — the Gherkin names lightspeed-stack, but the + scenario's intent (R10: the path is logged at startup) can only be observed + on the synthesizing container. Deviation agreed in planning (Q2). + + The message text follows the OGX rename (PRs #2516/#2547): client.py logs + "Using synthesized OGX config at %s" and llama_stack_configuration.py logs + "Wrote synthesized OGX configuration to %s (mode 0600)". """ if context.is_library_mode: container = "lightspeed-stack" - pattern = r"synthesized.*run\.yaml|Using synthesized Llama Stack config" + pattern = r"Using synthesized OGX config|synthesized.*run\.yaml" else: container = "llama-stack" pattern = ( - r"Wrote synthesized Llama Stack configuration" + r"Wrote synthesized OGX configuration" r"|Using generated config:.*run\.yaml" r"|mode auto-detected" ) From 924c044edaf9c1e1fe129343e68336984617ada2 Mon Sep 17 00:00:00 2001 From: Maxim Svistunov Date: Thu, 27 Aug 2026 16:22:25 +0200 Subject: [PATCH 11/14] LCORE-2343: name the migration boot scenarios for what they assert Both scenarios were titled "drives byte-identical Llama Stack behavior", but neither compares anything byte for byte: they boot the migrated configuration, assert readiness returns 200, and assert a query returns 200. The byte-level claim belongs to "migrate then synthesize round-trips to the original run.yaml" earlier in the same file, which does compare parsed data. Feature files are read as specification, so a title that overstates its scenario misleads anyone auditing what unified-mode migration is actually covered by. Rename both to "boots and serves queries", which is what the steps verify. No step definition, CI tag filter or test_list entry references either title, so this is a documentation-only change. --- tests/e2e/features/unified-mode-migration.feature | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/features/unified-mode-migration.feature b/tests/e2e/features/unified-mode-migration.feature index 68cd2aaeb..82ccfe7f1 100644 --- a/tests/e2e/features/unified-mode-migration.feature +++ b/tests/e2e/features/unified-mode-migration.feature @@ -23,7 +23,7 @@ Feature: Legacy to unified configuration migration # --- library mode (@skip-in-server-mode) --- @skip-in-server-mode - Scenario: Migrated unified configuration drives byte-identical Llama Stack behavior in library mode + Scenario: Migrated unified configuration boots and serves queries in library mode Given lightspeed-stack --migrate-config is run for the legacy migration fixture pair And The service uses the lightspeed-stack-unified-migrated.yaml configuration And The service is restarted @@ -39,7 +39,7 @@ Feature: Legacy to unified configuration migration # --- server mode (@skip-in-library-mode) --- @skip-in-library-mode - Scenario: Migrated unified configuration drives byte-identical Llama Stack behavior in server mode + Scenario: Migrated unified configuration boots and serves queries in server mode Given lightspeed-stack --migrate-config is run for the legacy migration fixture pair And The service uses the lightspeed-stack-unified-migrated.yaml configuration And Llama Stack is restarted From aced8d55026295abace5ce67c5b1d14fd703be1c Mon Sep 17 00:00:00 2001 From: Maxim Svistunov Date: Thu, 27 Aug 2026 16:55:06 +0200 Subject: [PATCH 12/14] LCORE-2343: drop the trailing blank line from the legacy e2e fixture YAMLlint reports "too many blank lines (1 > 0)" at the end of lightspeed-stack-legacy.yaml, which fails the configured formatting check. Strip the trailing newline so the file ends immediately after its last mapping entry. --- .../e2e/configuration/library-mode/lightspeed-stack-legacy.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-legacy.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-legacy.yaml index aedf2fc72..6b4ea3c09 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-legacy.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-legacy.yaml @@ -45,4 +45,3 @@ shields: rules: - pattern: '\d+' replacement: '[NUM]' - From 4df7f21b6174fd3de4bd228afc24a2989d1145be Mon Sep 17 00:00:00 2001 From: Maxim Svistunov Date: Thu, 27 Aug 2026 16:55:06 +0200 Subject: [PATCH 13/14] LCORE-2343: require a synthesized path in every accepted startup-log message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The startup-log step accepted "mode auto-detected" in server mode and a bare "Using synthesized OGX config" in library mode, neither of which carries a path. scripts/llama-stack-entrypoint.sh echoes "(mode auto-detected)" unconditionally and *before* it runs the config CLI, so a scenario asserting R10 ("the synthesized path is logged at startup") passed even when synthesis had failed outright — the precise failure the assertion exists to catch. Require a non-empty path in every alternative: "Using synthesized OGX config at ", "Wrote synthesized OGX configuration to ", and the entrypoint's "Using generated config: ", which unlike the auto-detect echo is only emitted after a successful generation. Verified against the three real messages, and that the pre-synthesis echo is now rejected. --- tests/e2e/features/steps/unified_mode.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/e2e/features/steps/unified_mode.py b/tests/e2e/features/steps/unified_mode.py index 418d5af19..69bd5b3f7 100644 --- a/tests/e2e/features/steps/unified_mode.py +++ b/tests/e2e/features/steps/unified_mode.py @@ -475,16 +475,20 @@ def container_logs_show_synthesis(context: Context) -> None: The message text follows the OGX rename (PRs #2516/#2547): client.py logs "Using synthesized OGX config at %s" and llama_stack_configuration.py logs "Wrote synthesized OGX configuration to %s (mode 0600)". + + Every accepted pattern must carry a path. The entrypoint echoes "(mode + auto-detected)" *before* synthesis runs and unconditionally, so matching it + would let the scenario pass on a failed synthesis — the opposite of what + R10 asks. "Using generated config: " is only echoed on success. """ if context.is_library_mode: container = "lightspeed-stack" - pattern = r"Using synthesized OGX config|synthesized.*run\.yaml" + pattern = r"Using synthesized OGX config at \S+" else: container = "llama-stack" pattern = ( - r"Wrote synthesized OGX configuration" - r"|Using generated config:.*run\.yaml" - r"|mode auto-detected" + r"Wrote synthesized OGX configuration to \S+" + r"|Using generated config:\s*\S+" ) result = subprocess.run( From 160db3e9205b93a0580acc4bd11166349dd9a2bb Mon Sep 17 00:00:00 2001 From: Maxim Svistunov Date: Thu, 27 Aug 2026 16:55:06 +0200 Subject: [PATCH 14/14] LCORE-2343: bound the HTTP readiness wait with a single monotonic deadline wait_for_lightspeed_stack_http_ready counted attempts rather than tracking wall-clock time, so its real ceiling was the per-request timeout plus the sleeps: 80 * 5s + 79 * 1.5s = 518.5s. The AssertionError reported only the backoff total, "~120s", understating the worst case by a factor of four. That gap matters more since this branch wires the wait into every lightspeed-stack restart rather than only the proxy steps: a container that never binds its port could stall a run for over eight minutes per restart, across the eight call sites of restart_container, while the failure text claimed two. Replace the attempt counter with one monotonic deadline covering both the requests and the sleeps, clamp each request timeout to the time remaining, skip a final sleep that would overrun the budget, and report the attempts and elapsed time actually spent. The default budget is 120s, which is what the old message always claimed the bound was. --- tests/e2e/utils/utils.py | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/tests/e2e/utils/utils.py b/tests/e2e/utils/utils.py index 5e0774a3f..365cb2ecf 100644 --- a/tests/e2e/utils/utils.py +++ b/tests/e2e/utils/utils.py @@ -555,8 +555,9 @@ def restart_lightspeed_stack_service( def wait_for_lightspeed_stack_http_ready( - max_attempts: int = 80, + timeout_s: float = 120.0, delay_s: float = 1.5, + request_timeout_s: float = 5.0, ) -> None: """Block until Lightspeed Stack accepts HTTP on the host-mapped port. @@ -567,10 +568,18 @@ def wait_for_lightspeed_stack_http_ready( Treats HTTP 200 and 401 as success: the process is listening. Auth-enabled configs (e.g. RBAC jwk-token) return 401 on probes without a Bearer token. + Bounded by a single monotonic deadline covering both the requests and the + sleeps, and each request is additionally capped at the time remaining, so + the total wait cannot exceed ``timeout_s``. An attempt-counted loop cannot + give that guarantee: with a per-request timeout the worst case is + ``attempts * request_timeout + (attempts - 1) * delay``, which for the + previous defaults was 518.5s while the failure message reported 120s. + Parameters: ---------- - max_attempts: Maximum GET attempts. + timeout_s: Total wall-clock budget for becoming reachable. delay_s: Sleep between attempts. + request_timeout_s: Per-request timeout, clamped to the time remaining. Raises: ------ AssertionError: If ``/liveness`` does not return an accepted status in time. @@ -580,26 +589,35 @@ def wait_for_lightspeed_stack_http_ready( host = os.getenv("E2E_LSC_HOSTNAME", "localhost") port = os.getenv("E2E_LSC_PORT", "8080") url = f"http://{host}:{port}/liveness" - for attempt in range(max_attempts): + started = time.monotonic() + deadline = started + timeout_s + attempt = 0 + while True: + attempt += 1 + remaining = deadline - time.monotonic() + if remaining <= 0: + break try: - response = requests.get(url, timeout=5) + response = requests.get(url, timeout=min(request_timeout_s, remaining)) if response.status_code in (200, 401): return detail = response.text[:200].replace("\n", " ") print( - f"⏱ HTTP wait LSC {attempt + 1}/{max_attempts} " + f"⏱ HTTP wait LSC attempt {attempt} " f"({url} -> {response.status_code}: {detail})..." ) except requests.RequestException as exc: print( - f"⏱ HTTP wait LSC {attempt + 1}/{max_attempts} " + f"⏱ HTTP wait LSC attempt {attempt} " f"({url} -> {exc.__class__.__name__}: {exc})..." ) - if attempt < max_attempts - 1: - time.sleep(delay_s) + if time.monotonic() + delay_s >= deadline: + break + time.sleep(delay_s) + elapsed = time.monotonic() - started raise AssertionError( f"Lightspeed Stack did not become reachable at {url!r} " - f"after {max_attempts} attempts (~{max_attempts * delay_s:.0f}s)" + f"after {attempt} attempts / {elapsed:.0f}s (budget {timeout_s:.0f}s)" )