From 0a63707a866aa8223058a0b638ef0759a6515392 Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Tue, 14 Jul 2026 12:05:24 -0500 Subject: [PATCH 1/4] Reconfigure how retries are done to have a lower timeout but more retries --- README.md | 3 ++- templates/agof-config-bootstrap-job.yaml | 1 + values.yaml | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 902574c..01a82e8 100644 --- a/README.md +++ b/README.md @@ -295,7 +295,8 @@ secrets: | agof.iac_repo | string | `""` | | | agof.iac_revision | string | `""` | | | agof.vaultFileKey | string | `""` | | -| configJob.activeDeadlineSeconds | int | `3600` | | +| configJob.activeDeadlineSeconds | int | `600` | | +| configJob.backoffLimit | int | `20` | | | configJob.configTimeout | int | `1800` | | | configJob.image | string | `"quay.io/hybridcloudpatterns/imperative-container:v1"` | | | configJob.imagePullPolicy | string | `"Always"` | | diff --git a/templates/agof-config-bootstrap-job.yaml b/templates/agof-config-bootstrap-job.yaml index e417a85..d4dd394 100644 --- a/templates/agof-config-bootstrap-job.yaml +++ b/templates/agof-config-bootstrap-job.yaml @@ -12,6 +12,7 @@ spec: backoffLimit: 99 template: spec: + backoffLimit: {{ $.Values.configJob.backoffLimit }} activeDeadlineSeconds: {{ $.Values.configJob.activeDeadlineSeconds }} {{- include "aap-config.app.configjobspec" $ | nindent 6 }} {{- end }}{{/* if not $.Values.configJob.disabled */}} diff --git a/values.yaml b/values.yaml index 330562c..766dbd3 100644 --- a/values.yaml +++ b/values.yaml @@ -46,7 +46,8 @@ agof: configJob: image: quay.io/hybridcloudpatterns/imperative-container:v1 schedule: "10 */2 * * *" - activeDeadlineSeconds: 3600 + activeDeadlineSeconds: 600 + backoffLimit: 20 imagePullPolicy: Always configTimeout: 1800 From d242f8547bddda6fb3fe9d086093d3cdaec6acb1 Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Tue, 14 Jul 2026 12:07:25 -0500 Subject: [PATCH 2/4] Add release note doc --- Chart.yaml | 2 +- README.md | 5 ++++- README.md.gotmpl | 3 +++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Chart.yaml b/Chart.yaml index 14eab62..02c28bd 100644 --- a/Chart.yaml +++ b/Chart.yaml @@ -5,7 +5,7 @@ home: https://github.com/validatedpatterns/aap-config-chart.git keywords: - pattern name: aap-config -version: 0.2.6 +version: 0.2.7 dependencies: - name: vp-rbac version: '0.1.*' diff --git a/README.md b/README.md index 01a82e8..f72dcfa 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # aap-config -![Version: 0.2.6](https://img.shields.io/badge/Version-0.2.6-informational?style=flat-square) +![Version: 0.2.7](https://img.shields.io/badge/Version-0.2.7-informational?style=flat-square) A Helm chart to build and deploy secrets using external-secrets for ansible-edge-gitops @@ -53,6 +53,9 @@ is unset or left at chart defaults; when `cac_*` is explicitly set, it wins over * v0.2.6: Track v3 of AGOF by default +* v0.2.7: Reconfigure how retries are done. Include configurable backoffLimit (default +20) and reduce activeDeadline seconds to 600 (10 min). + ### Git authentication secret (`agof.gitAuthSecret`) When `agof_repo` or the config-as-code repo (`agof.cac_repo` / `agof.iac_repo`) is diff --git a/README.md.gotmpl b/README.md.gotmpl index 2988600..6a26467 100644 --- a/README.md.gotmpl +++ b/README.md.gotmpl @@ -54,6 +54,9 @@ is unset or left at chart defaults; when `cac_*` is explicitly set, it wins over * v0.2.6: Track v3 of AGOF by default +* v0.2.7: Reconfigure how retries are done. Include configurable backoffLimit (default +20) and reduce activeDeadline seconds to 600 (10 min). + ### Git authentication secret (`agof.gitAuthSecret`) When `agof_repo` or the config-as-code repo (`agof.cac_repo` / `agof.iac_repo`) is From 3406827ab3070f910acb863a0b844b38fee665b6 Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Tue, 14 Jul 2026 12:13:11 -0500 Subject: [PATCH 3/4] Move retry tuning to validation job for slow ESO sync. Revert configJob changes and apply shorter deadlines with a higher backoffLimit on validationJob so external secrets get more frequent retries. Co-authored-by: Cursor --- README.md | 11 ++++++----- README.md.gotmpl | 5 +++-- templates/agof-config-bootstrap-job.yaml | 1 - templates/external-secrets-validation-job.yaml | 2 +- values.yaml | 6 +++--- 5 files changed, 13 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index f72dcfa..b6c7cfb 100644 --- a/README.md +++ b/README.md @@ -53,8 +53,9 @@ is unset or left at chart defaults; when `cac_*` is explicitly set, it wins over * v0.2.6: Track v3 of AGOF by default -* v0.2.7: Reconfigure how retries are done. Include configurable backoffLimit (default -20) and reduce activeDeadline seconds to 600 (10 min). +* v0.2.7: Reconfigure external secrets validation retries. Add configurable +`validationJob.backoffLimit` (default 20) and reduce `validationJob.activeDeadlineSeconds` +to 600 (10 min) so slow ESO sync gets more frequent job-level retries. ### Git authentication secret (`agof.gitAuthSecret`) @@ -298,8 +299,7 @@ secrets: | agof.iac_repo | string | `""` | | | agof.iac_revision | string | `""` | | | agof.vaultFileKey | string | `""` | | -| configJob.activeDeadlineSeconds | int | `600` | | -| configJob.backoffLimit | int | `20` | | +| configJob.activeDeadlineSeconds | int | `3600` | | | configJob.configTimeout | int | `1800` | | | configJob.image | string | `"quay.io/hybridcloudpatterns/imperative-container:v1"` | | | configJob.imagePullPolicy | string | `"Always"` | | @@ -308,7 +308,8 @@ secrets: | secretStore.name | string | `"vault-backend"` | | | serviceAccountName | string | `"aap-config-sa"` | | | serviceAccountNamespace | string | `"aap-config"` | | -| validationJob.activeDeadlineSeconds | int | `3600` | | +| validationJob.activeDeadlineSeconds | int | `600` | | +| validationJob.backoffLimit | int | `20` | | | validationJob.disabled | bool | `false` | | | vp-rbac.clusterRoles.view-routes.rules[0].apiGroups[0] | string | `"route.openshift.io"` | | | vp-rbac.clusterRoles.view-routes.rules[0].resources[0] | string | `"routes"` | | diff --git a/README.md.gotmpl b/README.md.gotmpl index 6a26467..429d292 100644 --- a/README.md.gotmpl +++ b/README.md.gotmpl @@ -54,8 +54,9 @@ is unset or left at chart defaults; when `cac_*` is explicitly set, it wins over * v0.2.6: Track v3 of AGOF by default -* v0.2.7: Reconfigure how retries are done. Include configurable backoffLimit (default -20) and reduce activeDeadline seconds to 600 (10 min). +* v0.2.7: Reconfigure external secrets validation retries. Add configurable +`validationJob.backoffLimit` (default 20) and reduce `validationJob.activeDeadlineSeconds` +to 600 (10 min) so slow ESO sync gets more frequent job-level retries. ### Git authentication secret (`agof.gitAuthSecret`) diff --git a/templates/agof-config-bootstrap-job.yaml b/templates/agof-config-bootstrap-job.yaml index d4dd394..e417a85 100644 --- a/templates/agof-config-bootstrap-job.yaml +++ b/templates/agof-config-bootstrap-job.yaml @@ -12,7 +12,6 @@ spec: backoffLimit: 99 template: spec: - backoffLimit: {{ $.Values.configJob.backoffLimit }} activeDeadlineSeconds: {{ $.Values.configJob.activeDeadlineSeconds }} {{- include "aap-config.app.configjobspec" $ | nindent 6 }} {{- end }}{{/* if not $.Values.configJob.disabled */}} diff --git a/templates/external-secrets-validation-job.yaml b/templates/external-secrets-validation-job.yaml index 874530d..4815e0a 100644 --- a/templates/external-secrets-validation-job.yaml +++ b/templates/external-secrets-validation-job.yaml @@ -9,7 +9,7 @@ metadata: spec: parallelism: 1 completions: 1 - backoffLimit: 3 + backoffLimit: {{ $.Values.validationJob.backoffLimit }} activeDeadlineSeconds: {{ $.Values.validationJob.activeDeadlineSeconds }} template: spec: diff --git a/values.yaml b/values.yaml index 766dbd3..4900d2d 100644 --- a/values.yaml +++ b/values.yaml @@ -46,8 +46,7 @@ agof: configJob: image: quay.io/hybridcloudpatterns/imperative-container:v1 schedule: "10 */2 * * *" - activeDeadlineSeconds: 600 - backoffLimit: 20 + activeDeadlineSeconds: 3600 imagePullPolicy: Always configTimeout: 1800 @@ -57,7 +56,8 @@ serviceAccountNamespace: aap-config # Validation job configuration validationJob: disabled: false - activeDeadlineSeconds: 3600 + activeDeadlineSeconds: 600 + backoffLimit: 20 # RBAC configuration using vp-rbac subchart vp-rbac: From 4b0213f90199a3e3d5c7f7c215ac82cde994e26f Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Tue, 14 Jul 2026 12:31:30 -0500 Subject: [PATCH 4/4] Re-order to prevent races --- README.md | 4 +++- README.md.gotmpl | 4 +++- templates/agof-config-bootstrap-job.yaml | 2 +- templates/external-secrets-validation-job.yaml | 4 +++- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b6c7cfb..71841a8 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,9 @@ is unset or left at chart defaults; when `cac_*` is explicitly set, it wins over * v0.2.7: Reconfigure external secrets validation retries. Add configurable `validationJob.backoffLimit` (default 20) and reduce `validationJob.activeDeadlineSeconds` -to 600 (10 min) so slow ESO sync gets more frequent job-level retries. +to 600 (10 min). Run the validation job as an Argo Sync hook at sync-wave 2 with +`HookSucceeded,BeforeHookCreation` so ExternalSecrets apply in wave 1 before +validation runs and each sync retry gets a fresh job without OutOfSync drift. ### Git authentication secret (`agof.gitAuthSecret`) diff --git a/README.md.gotmpl b/README.md.gotmpl index 429d292..de07850 100644 --- a/README.md.gotmpl +++ b/README.md.gotmpl @@ -56,7 +56,9 @@ is unset or left at chart defaults; when `cac_*` is explicitly set, it wins over * v0.2.7: Reconfigure external secrets validation retries. Add configurable `validationJob.backoffLimit` (default 20) and reduce `validationJob.activeDeadlineSeconds` -to 600 (10 min) so slow ESO sync gets more frequent job-level retries. +to 600 (10 min). Run the validation job as an Argo Sync hook at sync-wave 2 with +`HookSucceeded,BeforeHookCreation` so ExternalSecrets apply in wave 1 before +validation runs and each sync retry gets a fresh job without OutOfSync drift. ### Git authentication secret (`agof.gitAuthSecret`) diff --git a/templates/agof-config-bootstrap-job.yaml b/templates/agof-config-bootstrap-job.yaml index e417a85..277aa5b 100644 --- a/templates/agof-config-bootstrap-job.yaml +++ b/templates/agof-config-bootstrap-job.yaml @@ -5,7 +5,7 @@ kind: Job metadata: name: agof-config-bootstrap-job annotations: - argocd.argoproj.io/sync-wave: "2" + argocd.argoproj.io/sync-wave: "3" spec: parallelism: 1 completions: 1 diff --git a/templates/external-secrets-validation-job.yaml b/templates/external-secrets-validation-job.yaml index 4815e0a..58bc48e 100644 --- a/templates/external-secrets-validation-job.yaml +++ b/templates/external-secrets-validation-job.yaml @@ -5,7 +5,9 @@ kind: Job metadata: name: external-secrets-validation-job annotations: - argocd.argoproj.io/sync-wave: "1" + argocd.argoproj.io/sync-wave: "2" + argocd.argoproj.io/hook: Sync + argocd.argoproj.io/hook-delete-policy: HookSucceeded,BeforeHookCreation spec: parallelism: 1 completions: 1