From 40efe50a348e72628b318bfae9a514d0b32b4768 Mon Sep 17 00:00:00 2001 From: Thomas Flament Date: Wed, 18 Mar 2026 09:05:41 +0100 Subject: [PATCH 1/8] bootstrap-kind: auto-detect rootless podman and use delegated systemd scope When running kind with rootless podman from a graphical terminal, cgroup controllers may not be delegated to the process's cgroup. Detect this and automatically wrap kind create in a systemd scope with Delegate=yes. --- .github/scripts/end2end/bootstrap-kind.sh | 26 ++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/.github/scripts/end2end/bootstrap-kind.sh b/.github/scripts/end2end/bootstrap-kind.sh index 3be0dbe46..4e7e7961b 100755 --- a/.github/scripts/end2end/bootstrap-kind.sh +++ b/.github/scripts/end2end/bootstrap-kind.sh @@ -82,13 +82,37 @@ $(add_workers) EOF } +needs_delegated_scope() { + # When running rootless podman from a graphical terminal (e.g. GNOME/VTE), + # cgroup controllers may not be delegated to the process's cgroup, causing + # kind to fail. Detect this by checking what podman sees. + # See https://kind.sigs.k8s.io/docs/user/rootless/#creating-a-kind-cluster-with-rootless-podman + if [ "$(docker info --format '{{.Host.Security.Rootless}}' 2>/dev/null)" != "true" ]; then + return 1 + fi + controllers=$(docker info --format '{{.Host.CgroupControllers}}' 2>/dev/null) || return 1 + for c in cpu memory pids; do + case "$controllers" in + *"$c"*) ;; + *) return 0 ;; + esac + done + return 1 +} + create_cluster() { if kind get clusters | grep -q "^${CLUSTER_NAME}$"; then echo "Kind cluster ${CLUSTER_NAME} already exists. Skipping creation." return fi - kind create cluster --name=${CLUSTER_NAME} --config=config.yaml + DELEGATE="" + if needs_delegated_scope; then + echo "cgroup controllers not fully available, running kind under a delegated systemd scope" + DELEGATE="systemd-run --user --scope --property=Delegate=yes" + fi + + $DELEGATE kind create cluster --name=${CLUSTER_NAME} --config=config.yaml } create_registry From cece6419d80103dfe2547976b8273410f839feb7 Mon Sep 17 00:00:00 2001 From: Thomas Flament Date: Wed, 18 Mar 2026 09:06:06 +0100 Subject: [PATCH 2/8] bootstrap-kind: remove unused local registry setup The kind local registry was only partially implemented (missing containerd certs.d configuration on nodes) and was never used by any script. All images are loaded via kind load docker-image. Removing it also fixes a podman networking incompatibility. --- .github/scripts/end2end/bootstrap-kind.sh | 29 ----------------------- 1 file changed, 29 deletions(-) diff --git a/.github/scripts/end2end/bootstrap-kind.sh b/.github/scripts/end2end/bootstrap-kind.sh index 4e7e7961b..db97a81f5 100755 --- a/.github/scripts/end2end/bootstrap-kind.sh +++ b/.github/scripts/end2end/bootstrap-kind.sh @@ -6,29 +6,6 @@ NODE_IMAGE=${1:-kindest/node:kindest/node:v1.32.8@sha256:abd489f042d2b644e2d033f VOLUME_ROOT=${2:-/artifacts} WORKER_NODE_COUNT=${3:-0} CLUSTER_NAME=${CLUSTER_NAME:-kind} -REG_NAME='kind-registry' -REG_PORT='5000' - -create_registry() { - echo "Creating local image registry on localhost:${REG_PORT}" - - if [ "$(docker inspect -f '{{.State.Running}}' "${REG_NAME}" 2>/dev/null)" != 'true' ]; then - docker run \ - -d --restart=always -p "${REG_PORT}:5000" --name "${REG_NAME}" \ - registry:2 - fi -} - -connect_registry() { - local inspect_filter="{{range .Containers}}{{if eq .Name \"${REG_NAME}\"}}true{{end}}{{end}}" - if [ "$(docker network inspect -f "${inspect_filter}" kind 2>/dev/null)" != 'true' ]; then - docker network connect kind "${REG_NAME}" - fi - - for node in $(kind get nodes --name ${CLUSTER_NAME}); do - kubectl annotate --overwrite node "${node}" "kind.x-k8s.io/registry=localhost:${REG_PORT}"; - done -} add_workers() { local count=0 @@ -48,10 +25,6 @@ bootstrap_kind() { cat > config.yaml << EOF kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 -containerdConfigPatches: -- |- - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${REG_PORT}"] - endpoint = ["http://${REG_NAME}:${REG_PORT}"] nodes: - role: control-plane image: ${NODE_IMAGE} @@ -115,7 +88,5 @@ create_cluster() { $DELEGATE kind create cluster --name=${CLUSTER_NAME} --config=config.yaml } -create_registry bootstrap_kind create_cluster -connect_registry From c00e02302e0df480ae41324ffcc620be28821294 Mon Sep 17 00:00:00 2001 From: Thomas Flament Date: Wed, 18 Mar 2026 09:06:19 +0100 Subject: [PATCH 3/8] deploy-zkop: use local zenko-operator checkout if available When ../zenko-operator exists, symlink it instead of cloning from GitHub. This avoids creating a nested git repo and allows using a local working copy for development. Also clean up the operator image after loading it into kind. --- .github/scripts/end2end/deploy-zkop.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/scripts/end2end/deploy-zkop.sh b/.github/scripts/end2end/deploy-zkop.sh index 697f2f04b..77f659d69 100755 --- a/.github/scripts/end2end/deploy-zkop.sh +++ b/.github/scripts/end2end/deploy-zkop.sh @@ -7,11 +7,18 @@ set -ex docker pull "${OPERATOR_IMAGE_NAME}:${OPERATOR_IMAGE_TAG}" kind load docker-image "${OPERATOR_IMAGE_NAME}:${OPERATOR_IMAGE_TAG}" +docker rmi "${OPERATOR_IMAGE_NAME}:${OPERATOR_IMAGE_TAG}" OPERATOR_PATH=./.github/scripts/end2end/operator -git init $OPERATOR_PATH -cd $OPERATOR_PATH -git fetch --depth 1 --no-tags https://git:${GIT_ACCESS_TOKEN}@github.com/scality/zenko-operator.git ${OPERATOR_IMAGE_TAG} -git checkout FETCH_HEAD +LOCAL_OPERATOR_PATH=../zenko-operator +if [ -d "$LOCAL_OPERATOR_PATH" ]; then + echo "Using local zenko-operator checkout at $LOCAL_OPERATOR_PATH" + ln -sfn "$(readlink -f "$LOCAL_OPERATOR_PATH")" "$OPERATOR_PATH" +else + git init $OPERATOR_PATH + git -C $OPERATOR_PATH fetch --depth 1 --no-tags https://git:${GIT_ACCESS_TOKEN}@github.com/scality/zenko-operator.git ${OPERATOR_IMAGE_TAG} + git -C $OPERATOR_PATH checkout FETCH_HEAD +fi +cd $OPERATOR_PATH tilt ci \ No newline at end of file From aa71ccbc16cb44d3675d3bd661f63ab9979cce72 Mon Sep 17 00:00:00 2001 From: Thomas Flament Date: Wed, 18 Mar 2026 09:09:13 +0100 Subject: [PATCH 4/8] setup: override VOLUME_ROOT for local development Use $PWD/artifacts instead of /artifacts for kind volume mounts so the setup works outside CI where /artifacts does not exist. --- .devcontainer/setup.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh index 915d00c66..ec63d0680 100755 --- a/.devcontainer/setup.sh +++ b/.devcontainer/setup.sh @@ -7,6 +7,9 @@ ZENKO_ENV_FILE="$HOME/.zenko.env" yq eval '.env | to_entries | .[] | "export " + .key + "=" + (.value | tostring | @sh)' .github/workflows/end2end.yaml \ | sed 's/\${{[^}]*}}//g' > "$ZENKO_ENV_FILE" echo 'export GIT_ACCESS_TOKEN="${GITHUB_TOKEN}"' >> "$ZENKO_ENV_FILE" + +echo 'export VOLUME_ROOT=$PWD/artifacts' >> "$ZENKO_ENV_FILE" +mkdir -p "$PWD/artifacts/data" # Disable GCP tests as we don't have credentials setup in devcontainer echo 'export GCP_BACKEND_DESTINATION_LOCATION=' >> "$ZENKO_ENV_FILE" From ba7af6464f2ef99391c5cdf4e6056ebf59172db4 Mon Sep 17 00:00:00 2001 From: Thomas Flament Date: Wed, 18 Mar 2026 15:12:27 +0100 Subject: [PATCH 5/8] patch-coredns: support HOST_DNS override for Podman environments Podman's aardvark-dns (used as the default DNS inside KinD nodes) fails to forward external DNS queries reliably. CoreDNS forwards to /etc/resolv.conf which points to aardvark-dns, causing SERVFAIL for external domains like ghcr.io. When HOST_DNS is set, patch-coredns.sh uses it as the CoreDNS forward target instead of /etc/resolv.conf. The devcontainer setup.sh detects the host nameserver and exports HOST_DNS. --- .devcontainer/setup.sh | 1 + .github/scripts/end2end/patch-coredns.sh | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh index ec63d0680..85d997451 100755 --- a/.devcontainer/setup.sh +++ b/.devcontainer/setup.sh @@ -9,6 +9,7 @@ yq eval '.env | to_entries | .[] | "export " + .key + "=" + (.value | tostring | echo 'export GIT_ACCESS_TOKEN="${GITHUB_TOKEN}"' >> "$ZENKO_ENV_FILE" echo 'export VOLUME_ROOT=$PWD/artifacts' >> "$ZENKO_ENV_FILE" +echo "export HOST_DNS=$(awk '/^nameserver/{print \$2; exit}' /etc/resolv.conf)" >> "$ZENKO_ENV_FILE" mkdir -p "$PWD/artifacts/data" # Disable GCP tests as we don't have credentials setup in devcontainer echo 'export GCP_BACKEND_DESTINATION_LOCATION=' >> "$ZENKO_ENV_FILE" diff --git a/.github/scripts/end2end/patch-coredns.sh b/.github/scripts/end2end/patch-coredns.sh index b9087d271..cc12d3830 100755 --- a/.github/scripts/end2end/patch-coredns.sh +++ b/.github/scripts/end2end/patch-coredns.sh @@ -4,6 +4,12 @@ set -exu export ZENKO_NAME=${1:-end2end} +if [ -n "${HOST_DNS:-}" ]; then + COREDNS_FORWARD_TARGET="$HOST_DNS" +else + COREDNS_FORWARD_TARGET="/etc/resolv.conf" +fi + corefile=" .:53 { errors @@ -41,7 +47,7 @@ corefile=" ttl 30 } prometheus :9153 - forward . /etc/resolv.conf + forward . ${COREDNS_FORWARD_TARGET} cache 30 loop reload From ec4e25dbaa17f633953449150e00299d63a21412 Mon Sep 17 00:00:00 2001 From: Thomas Flament Date: Wed, 18 Mar 2026 16:33:37 +0100 Subject: [PATCH 6/8] common: use scheme variable and 127.0.0.1 for get_token Use a scheme variable based on ENABLE_KEYCLOAK_HTTPS instead of hardcoding https. Use 127.0.0.1 instead of localhost to avoid curl resolving to IPv6 ::1, which kind does not bind on. --- .github/scripts/end2end/common.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/scripts/end2end/common.sh b/.github/scripts/end2end/common.sh index eb490d702..85ff0f921 100644 --- a/.github/scripts/end2end/common.sh +++ b/.github/scripts/end2end/common.sh @@ -1,11 +1,16 @@ get_token() { + if [[ "${ENABLE_KEYCLOAK_HTTPS}" == "true" ]]; then + local scheme=https + else + local scheme=http + fi curl -k -H "Host: keycloak.zenko.local" \ -d "client_id=${OIDC_CLIENT_ID}" \ -d "username=${OIDC_USERNAME}" \ -d "password=${OIDC_PASSWORD}" \ -d "grant_type=password" \ -d "scope=openid" \ - https://localhost/auth/realms/${OIDC_REALM}/protocol/openid-connect/token | \ + ${scheme}://127.0.0.1/auth/realms/${OIDC_REALM}/protocol/openid-connect/token | \ jq -cr '.id_token' } From 3a24af3a50be61b6dfe6d1c8d5edb4baf5ef0501 Mon Sep 17 00:00:00 2001 From: Thomas Flament Date: Fri, 20 Mar 2026 14:14:36 +0100 Subject: [PATCH 7/8] docs: add /etc/hosts pre-setup for local environments --- .devcontainer/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.devcontainer/README.md b/.devcontainer/README.md index d8301a2d6..15b3f8164 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -3,6 +3,16 @@ A [VS Code extension](https://marketplace.visualstudio.com/items?itemName=GitHub.codespaces) is available for Codespaces. +## Pre-setup: /etc/hosts (local environments only) + +When running locally (not in a Codespace/devcontainer where you are root), +the CTST configuration script needs Zenko hostnames to resolve to localhost. +Add them before running setup to avoid a `sudo` prompt mid-run: + +```bash +echo "127.0.0.1 iam.zenko.local s3-local-file.zenko.local keycloak.zenko.local sts.zenko.local management.zenko.local s3.zenko.local website.mywebsite.com utilization.zenko.local aws-mock.zenko.local azure-mock.zenko.local blob.azure-mock.zenko.local queue.azure-mock.zenko.local devstoreaccount1.blob.azure-mock.zenko.local devstoreaccount1.queue.azure-mock.zenko.local dr.zenko.local" | sudo tee -a /etc/hosts +``` + ## Running CTST tests in the codespace ```bash From 305067fb4f3a7e2c5564efa72b8b30ac37d99b00 Mon Sep 17 00:00:00 2001 From: Thomas Flament Date: Mon, 23 Mar 2026 14:59:45 +0100 Subject: [PATCH 8/8] bootstrap-kind: clear DNS search domains in kind config Podman injects "dns.podman" into node resolv.conf, which leaks into pod DNS config. CoreDNS cannot resolve this domain, causing 8s timeouts per lookup and killing pods that rely on fast DNS during init (e.g. zookeeper). --- .github/scripts/end2end/bootstrap-kind.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/scripts/end2end/bootstrap-kind.sh b/.github/scripts/end2end/bootstrap-kind.sh index db97a81f5..4717f8e73 100755 --- a/.github/scripts/end2end/bootstrap-kind.sh +++ b/.github/scripts/end2end/bootstrap-kind.sh @@ -25,6 +25,8 @@ bootstrap_kind() { cat > config.yaml << EOF kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 +networking: + dnsSearch: [] nodes: - role: control-plane image: ${NODE_IMAGE}