LCORE-3048 : snuryyeva/okp rag implementation (DO NOT MERGE)

.konflux/artifacts.lock.yaml

@@ -3,6 +3,6 @@ metadata:
   version: "1.0"
 
 artifacts:
-  - download_url: https://github.com/lightspeed-core/lightspeed-providers/archive/8cd1b3d3bdd841ea99d31b334ae00a275581661c.zip
-    checksum: sha256:6e660a30babcd1c00eef1686b251f9368410e03e08a7c0d4a680db00693ac448
+  - download_url: https://github.com/lightspeed-core/lightspeed-providers/archive/faf6a89a3ad7856e2e7a934324f31d146108acdb.zip
+    checksum: sha256:c5ef9158146de2193daa16b1c3a0b338b20d136b6d3bb69da32ad592ad626f36
     filename: lightspeed-providers.zip

.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml

@@ -212,6 +212,10 @@ spec:
           - name: quay-aipcc-password
             secret:
               secretName: quay-aipcc-password
+          # Red Hat registry pull secret (docker-registry type) - will be copied to test namespace
+          - name: redhat-registry-pull-secret-source
+            secret:
+              secretName: redhat-registry-io-pull-secret
           - name: credentials
             emptyDir: {}
         steps:
@@ -251,6 +255,9 @@ spec:
                 mountPath: /var/run/quay-aipcc-name
               - name: quay-aipcc-password
                 mountPath: /var/run/quay-aipcc-password
+              # Red Hat registry pull secret (docker-registry type)
+              - name: redhat-registry-pull-secret-source
+                mountPath: /var/run/redhat-registry-pull-secret
               - name: credentials
                 mountPath: /credentials
             env:
@@ -274,6 +281,11 @@ spec:
                 value: "true"
               - name: OTEL_ANONYMIZATION_SECRET
                 value: "lightspeed-stack-otel-anonymization-dev-default"
+              # PipelineRun metadata for ownerReference in secret creation
+              - name: TEKTON_PIPELINERUN_NAME
+                value: "$(context.pipelineRun.name)"
+              - name: TEKTON_PIPELINERUN_UID
+                value: "$(context.pipelineRun.uid)"
             image: registry.access.redhat.com/ubi9/ubi-minimal
             script: |
               set +e

Makefile

@@ -1,6 +1,9 @@
 SHELL := /bin/bash
 #TODO: We need to rename all those python and config files as well
 
+# Define comma for use in $(if) expressions (where commas are argument separators)
+COMMA := ,
+
 ARTIFACT_DIR := $(if $(ARTIFACT_DIR),$(ARTIFACT_DIR),tests/test_results)
 PATH_TO_PLANTUML := ~/bin
 
@@ -16,6 +19,7 @@ OGX_CONFIG ?= run.yaml
 OGX_CONTAINER_NAME ?= lightspeed-ogx
 OGX_IMAGE ?= lightspeed-ogx:local
 OGX_PORT ?= 8321
+LIGHTSPEED_PROVIDERS_DIR ?= $(shell [ -d ../lightspeed-providers ] && cd ../lightspeed-providers && pwd)
 CONTAINER_RUNTIME ?= $(shell command -v podman 2>/dev/null || command -v docker 2>/dev/null)
 
 .PHONY: run \
@@ -86,6 +90,9 @@ start-ogx-container: build-ogx-image ## Start OGX container
 		-v $(PWD)/$(CONFIG):/opt/app-root/lightspeed-stack.yaml:ro,z \
 		-v $(PWD)/scripts/ogx-entrypoint.sh:/opt/app-root/enrich-entrypoint.sh:ro,z \
 		-v $(PWD)/src/ogx_configuration.py:/opt/app-root/ogx_configuration.py:ro,z \
+		$(if $(LIGHTSPEED_PROVIDERS_DIR),-v $(LIGHTSPEED_PROVIDERS_DIR)/lightspeed_stack_providers:/opt/app-root/providers/lightspeed_stack_providers:ro$(COMMA)z) \
+		$(if $(LIGHTSPEED_PROVIDERS_DIR),-v $(LIGHTSPEED_PROVIDERS_DIR)/resources/external_providers:/opt/app-root/src/.llama/providers.d:ro$(COMMA)z) \
+		$(if $(LIGHTSPEED_PROVIDERS_DIR),-e EXTERNAL_PROVIDERS_DIR=/opt/app-root/src/.llama/providers.d) \
 		-e OPENAI_API_KEY \
 		-e BRAVE_SEARCH_API_KEY \
 		-e TAVILY_SEARCH_API_KEY \
@@ -164,7 +171,7 @@ test-e2e-local: ## Run end to end tests for the service (no script wrapper)
 
 # Tag-based subsets (@cfg_* on features/scenarios). Default runs all config groups; override for one shard, e.g.
 #   E2E_BEHAVE_TAG_EXPR='not @skip and @cfg_authorized' make test-e2e-tagged-local
-E2E_BEHAVE_TAG_EXPR ?= not @skip and (@cfg_default or @cfg_authorized or @cfg_mcp or @cfg_mcp_invalid or @cfg_mcp_api_auth or @cfg_rbac or @cfg_rh_identity or @cfg_negative or @cfg_skills or @cfg_skills_directory or @cfg_shields or @cfg_byok_pdf or @cfg_tls or @cfg_degraded or @cfg_unified)
+E2E_BEHAVE_TAG_EXPR ?= not @skip and (@cfg_default or @cfg_authorized or @cfg_mcp or @cfg_mcp_invalid or @cfg_mcp_api_auth or @cfg_rbac or @cfg_rh_identity or @cfg_negative or @cfg_skills or @cfg_skills_directory or @cfg_shields or @cfg_byok_pdf or @cfg_tls or @cfg_degraded or @cfg_unified or @cfg_okp)
 
 test-e2e-tagged: ## Run e2e tests with E2E_BEHAVE_TAG_EXPR (default: all @cfg_*)
 	script -q -e -c "uv run behave --color --format pretty --tags=\"$(E2E_BEHAVE_TAG_EXPR)\" -D dump_errors=true @tests/e2e/test_list.txt"

docs/user_doc/okp_guide.md

@@ -51,12 +51,20 @@ validating that queries return referenced chunks.
 
 ## Step 1: Launch OKP
 
-Start the OKP RAG service with Podman:
+Start the OKP RAG service with Podman or Docker:
 
+**Using Podman:**
 ```bash
+podman login registry.redhat.io
 podman run --rm -d -p 8081:8080 registry.redhat.io/offline-knowledge-portal/rhokp-rhel9:latest
 ```
 
+**Using Docker:**
+```bash
+docker login registry.redhat.io
+docker run --rm -d -p 8081:8080 registry.redhat.io/offline-knowledge-portal/rhokp-rhel9:latest
+```
+
 > **Note:** Remove `-d` to run in the foreground.
 
 * The service listens on **port 8081** on the host (mapped from 8080 in the container).  Lightspeed Stack itself listens on `8080`, so this avoids port conflicts.
@@ -105,41 +113,61 @@ Edit your Lightspeed Stack config file (e.g. `lightspeed-stack.yaml`) and add
 the following top-level sections so that OKP is used for either inline or tool
 RAG:
 
-Inline RAG:
+**Inline RAG:**
 
 ```yaml
 # RAG configuration
 rag:
-  inline:
-  - okp
-okp:
-  rhokp_url: ${env.RH_SERVER_OKP}
-  offline: true
+  retrieval:
+    inline:
+      sources:
+        - okp
+      max_chunks: 3      # Final tight cap on chunks returned to user
+  okp:
+    rhokp_url: ${env.RH_SERVER_OKP}
+    offline: true
+    max_chunks: 5        # Initial generous fetch from OKP
 ```
 
-Tool RAG:
+**Tool RAG:**
 
 ```yaml
 # RAG configuration
 rag:
-  tool:
-  - okp
-okp:
-  rhokp_url: ${env.RH_SERVER_OKP}
-  offline: true
+  retrieval:
+    tool:
+      sources:
+        - okp
+      max_chunks: 10     # Tool RAG limit
+  okp:
+    rhokp_url: ${env.RH_SERVER_OKP}
+    offline: true
+    max_chunks: 5
 ```
 
-* **`rag.inline`** and **`rag.tool`**: Enable OKP as the RAG source for inline context injection and for the RAG tool.  Tool rag means the LLM will be provided a search tool it can choose to invoke to find relevant content and augment the user prompt.  The tool may or may not be invoked.  Inline means a rag search and prompt augmentation will always occur.
+* **`rag.retrieval.inline`** and **`rag.retrieval.tool`**: Enable OKP as the RAG source for inline context injection and for the RAG tool. Tool RAG means the LLM will be provided a search tool it can choose to invoke to find relevant content and augment the user prompt. The tool may or may not be invoked. Inline RAG means a RAG search and prompt augmentation will always occur.
 * **`okp.offline`**: When `true`, source URLs use `parent_id` (offline/Mimir-style). When `false`, use `reference_url` (online).
+* **`okp.search_mode`**: Default Solr search strategy for OKP queries. Controls *how* Solr ranks chunks, not *which* documents are eligible (that is `chunk_filter_query` and per-request `solr.filters`). Allowed values:
+  - **`hybrid`** (default when the field is omitted): combines vector similarity with keyword (BM25) text search. Requires an embedding model. This is the global default.
+  - **`semantic`**: pure vector search against Solr embeddings. Requires an embedding model. Best when the query is conceptual rather than matching document wording.
+  - **`keyword`**: BM25 text search only. No embedding model is needed, so it is the right choice for air-gapped or embedding-free deployments, and it is typically faster than hybrid.
+* **`max_chunks` behavior**:
+  - **Inline RAG**: `rag.retrieval.inline.max_chunks` is the final tight cap on chunks returned in the response, while `rag.okp.max_chunks` is the initial generous fetch from OKP before filtering and ranking.
+  - **Tool RAG**: `rag.retrieval.tool.max_chunks` sets the limit on chunks available to the tool.
 
 If you want to filter the docs to a specific product, you can include a static query filter such as:
 
 ```yaml
 okp:
   offline: true
+  search_mode: keyword
   chunk_filter_query: "product:*openshift* AND product_version:4.21"
 ```
 
+A per-request `solr.mode` on `/v1/query` (or streaming/responses) overrides this YAML default for that call. Per-request `lexical` is treated as `keyword`.
+
+> **Important:** When running OGX in **Server mode**, changes to static OKP settings such as `chunk_filter_query` and `search_mode` require an OGX restart to take effect. They are applied when Lightspeed Stack enriches OGX configuration at startup and will not propagate automatically during runtime.
+
 When you launch Lightspeed Stack it will augment the OGX configuration (the synthesized run.yaml in unified mode, or your external run.yaml in the deprecated legacy mode) with
 configuration for OKP.
 
@@ -258,7 +286,7 @@ curl -sX POST http://localhost:8080/v1/query \
 - **Static filters preserved:** The configured `chunk_filter_query` (e.g., `"product:*openshift*"`) is always applied
 - **Dynamic filters added:** Request filters are combined with static filters using AND logic
 - **String escaping:** Special Solr characters in filter values are automatically escaped
-- **Works with all search modes:** Filters apply to `semantic`, `hybrid`, and `lexical` search modes
+- **Works with all search modes:** Filters apply whether the search mode is `semantic`, `hybrid`, or `keyword` (per-request `lexical` is an alias of `keyword`).
 
 ---
 
@@ -312,9 +340,9 @@ Example response excerpt:
 ],
 ```
 
-> **Note:** The first time you query the system the response may take
-> additional time because it must first download the necessary embedding model
-> to perform the vector search.
+> **Note:** The first time you query the system, `hybrid` or `semantic` search may take
+> additional time because OGX must download the embedding model used for vector
+> search. `keyword` search skips that download.
 
 If you see no RAG context, verify:
 

src/ogx_configuration.py

@@ -819,7 +819,7 @@ def enrich_vector_store(
 # =============================================================================
 
 
-def enrich_solr(  # pylint: disable=too-many-locals,too-many-statements
+def enrich_solr(  # pylint: disable=too-many-locals,too-many-statements,too-many-branches
     ls_config: dict[str, Any],
     rag_config: dict[str, Any],
     okp_config: dict[str, Any],
@@ -860,6 +860,15 @@ def enrich_solr(  # pylint: disable=too-many-locals,too-many-statements
 
     logger.info("Enriching OGX config with OKP")
 
+    # run-ci.yaml comments this out; Solr is a remote provider and needs providers.d.
+    if "external_providers_dir" not in ls_config:
+        ls_config["external_providers_dir"] = (
+            "${env.EXTERNAL_PROVIDERS_DIR:=/opt/app-root/providers.d}"
+        )
+        logger.info(
+            "Added external_providers_dir to OGX config for remote provider resolution"
+        )
+
     # Add vector_io provider for Solr
     if "providers" not in ls_config:
         ls_config["providers"] = {}

tests/e2e-prow/rhoai/manifests/lightspeed/ogx-openai.yaml

@@ -43,12 +43,22 @@ spec:
           # Fast-path: PVC already has a valid venv from a previous pod creation in this pipeline run.
           # TLS scenarios delete+recreate this pod up to 16 times; skipping the expensive install
           # reduces per-restart time from ~6-15 min to ~30-90 s (just RAG seed refresh + chown).
+          # Check if cached revision matches requested revision
+          CACHED_REVISION=""
+          if [[ -f /opt/app-root/.cached-revision ]]; then
+            CACHED_REVISION=$(cat /opt/app-root/.cached-revision)
+          fi
+          REPO_REVISION="${REPO_REVISION:-main}"
+
           if [[ -d /opt/app-root/.venv ]] \
               && /opt/app-root/.venv/bin/python --version >/dev/null 2>&1 \
               && [[ -d /opt/app-root/src ]] \
               && [[ -f /opt/app-root/scripts/ogx-entrypoint.sh ]] \
-              && [[ -f /opt/app-root/src/ogx_configuration.py ]]; then
-            echo "PVC cache hit: app-root already provisioned — skipping full install"
+              && [[ -f /opt/app-root/src/ogx_configuration.py ]] \
+              && [[ -d /opt/app-root/lightspeed_stack_providers ]] \
+              && [[ -d /opt/app-root/providers.d ]] \
+              && [[ "$CACHED_REVISION" == "$REPO_REVISION" ]]; then
+            echo "PVC cache hit: app-root already provisioned at $REPO_REVISION — skipping full install"
             mkdir -p /opt/app-root/.e2e-rag-seed /opt/app-root/.e2e-rag-work /opt/app-root/src/.ogx /opt/app-root/src/.llama/storage/files
             if [[ -f /rag-seed/kv_store.db.gz ]]; then
               gzip -dc /rag-seed/kv_store.db.gz > /opt/app-root/.e2e-rag-seed/kv_store.db
@@ -81,6 +91,19 @@ spec:
           (cd /opt/app-root/repo && tar cf - .) | (cd /opt/app-root && tar xf -)
           rm -rf /opt/app-root/repo
           sed -i 's|/opt/app-root/repo/.venv|/opt/app-root/.venv|g' /opt/app-root/.venv/bin/* 2>/dev/null || true
+          # Download lightspeed-providers (external provider code + specs) — matches Containerfile
+          # Default matches deploy/lightspeed-stack/Containerfile ARG LIGHTSPEED_PROVIDERS_COMMIT
+          PROVIDERS_COMMIT="${PROVIDERS_COMMIT:-faf6a89a3ad7856e2e7a934324f31d146108acdb}"
+          ZIP_PATH="/tmp/lightspeed-providers.zip"
+          EXTRACT_DIR="/tmp/providers"
+          curl -fL --retry 2 --retry-delay 5 "https://github.com/lightspeed-core/lightspeed-providers/archive/${PROVIDERS_COMMIT}.zip" -o "${ZIP_PATH}"
+          mkdir -p "${EXTRACT_DIR}"
+          export ZIP_PATH EXTRACT_DIR
+          ROOT_DIR=$(python3 -c 'import os, zipfile; z=zipfile.ZipFile(os.environ["ZIP_PATH"]); print(z.namelist()[0].split("/")[0])')
+          python3 -c 'import os, zipfile; zipfile.ZipFile(os.environ["ZIP_PATH"]).extractall(os.environ["EXTRACT_DIR"])'
+          mv "${EXTRACT_DIR}/${ROOT_DIR}/lightspeed_stack_providers" /opt/app-root/
+          mv "${EXTRACT_DIR}/${ROOT_DIR}/resources/external_providers" /opt/app-root/providers.d
+          rm -rf "${ZIP_PATH}" "${EXTRACT_DIR}"
           mkdir -p /opt/app-root/.e2e-rag-seed /opt/app-root/.e2e-rag-work /opt/app-root/src/.ogx /opt/app-root/src/.llama/storage/files
           if [[ ! -f /rag-seed/kv_store.db.gz ]]; then
             echo "FATAL: missing /rag-seed/kv_store.db.gz (ConfigMap rag-data key kv_store.db.gz)"
@@ -96,6 +119,8 @@ spec:
           cp -f /opt/app-root/.e2e-rag-seed/kv_store.db /opt/app-root/.e2e-rag-work/kv_store.db
           cp -f /opt/app-root/scripts/ogx-entrypoint.sh /opt/app-root/enrich-entrypoint.sh
           cp -f /opt/app-root/src/ogx_configuration.py /opt/app-root/ogx_configuration.py
+          # Save the provisioned revision for cache validation
+          echo "${REPO_REVISION:-main}" > /opt/app-root/.cached-revision
           chmod 755 /opt/app-root/enrich-entrypoint.sh
           chmod -R 775 /opt/app-root && chown -R 1001:0 /opt/app-root
       volumeMounts:
@@ -138,7 +163,9 @@ spec:
         - name: PATH
           value: "/opt/app-root/.venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
         - name: PYTHONPATH
-          value: "/opt/app-root/src"
+          value: "/opt/app-root:/opt/app-root/src"
+        - name: EXTERNAL_PROVIDERS_DIR
+          value: "/opt/app-root/providers.d"
         - name: HOME
           value: "/opt/app-root/src"
         # Match GitHub Actions docker-compose + ogx-entrypoint.sh:
@@ -188,6 +215,10 @@ spec:
             secretKeyRef:
               name: llama-stack-ip-secret
               key: key
+        # In-cluster OKP Solr Service. Used when LCS YAML lists okp as a RAG source
+        # and OGX enriches run.yaml. Offline doc_url rewriting stays on LCS (localhost:8081 default).
+        - name: RH_SERVER_OKP
+          value: "http://okp-solr-service-svc:8080"
       command:
         - /bin/bash
         - -c

tests/e2e-prow/rhoai/manifests/lightspeed/ogx-prow.yaml

@@ -84,7 +84,9 @@ spec:
       workingDir: /opt/app-root
       env:
         - name: PYTHONPATH
-          value: "/opt/app-root/src"
+          value: "/opt/app-root:/opt/app-root/src"
+        - name: EXTERNAL_PROVIDERS_DIR
+          value: "/opt/app-root/providers.d"
         - name: HOME
           value: "/opt/app-root/src"
         # Prevent OGX from shutil.move(~/.llama → ~/.ogx) which would steal the fixture.

tests/e2e-prow/rhoai/manifests/lightspeed/okp-solr.yaml

@@ -0,0 +1,62 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: okp-solr-service
+  labels:
+    app: okp-solr
+spec:
+  securityContext:
+    runAsNonRoot: true
+    seccompProfile:
+      type: RuntimeDefault
+  imagePullSecrets:
+    - name: redhat-registry-pull-secret
+  containers:
+    - name: okp-solr
+      image: registry.redhat.io/offline-knowledge-portal/rhokp-rhel9:latest
+      imagePullPolicy: IfNotPresent
+      securityContext:
+        allowPrivilegeEscalation: false
+        capabilities:
+          drop: ["ALL"]
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
+      ports:
+        - containerPort: 8080
+          name: http
+      readinessProbe:
+        httpGet:
+          path: /solr
+          port: 8080
+        initialDelaySeconds: 30
+        periodSeconds: 10
+        timeoutSeconds: 5
+        failureThreshold: 3
+      livenessProbe:
+        httpGet:
+          path: /solr
+          port: 8080
+        initialDelaySeconds: 60
+        periodSeconds: 30
+        timeoutSeconds: 5
+        failureThreshold: 3
+      resources:
+        requests:
+          memory: "512Mi"
+          cpu: "250m"
+        limits:
+          memory: "2Gi"
+          cpu: "1000m"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: okp-solr-service-svc
+spec:
+  selector:
+    app: okp-solr
+  ports:
+    - port: 8080
+      targetPort: 8080
+      name: http