diff --git a/api/v1alpha1/temporalworkerownedresource_types.go b/api/v1alpha1/temporalworkerownedresource_types.go new file mode 100644 index 00000000..f7fae5f7 --- /dev/null +++ b/api/v1alpha1/temporalworkerownedresource_types.go @@ -0,0 +1,103 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2024 Datadog, Inc. + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// WorkerDeploymentReference references a TemporalWorkerDeployment in the same namespace. +type WorkerDeploymentReference struct { + // Name of the TemporalWorkerDeployment resource in the same namespace. + // +kubebuilder:validation:Required + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` + Name string `json:"name"` +} + +// TemporalWorkerOwnedResourceSpec defines the desired state of TemporalWorkerOwnedResource. +type TemporalWorkerOwnedResourceSpec struct { + // WorkerRef references the TemporalWorkerDeployment to attach this resource to. + // +kubebuilder:validation:Required + WorkerRef WorkerDeploymentReference `json:"workerRef"` + + // Object is the Kubernetes resource template to attach to each versioned Deployment. + // One copy of this resource is created per active Build ID, owned by the corresponding + // versioned Deployment (so it is garbage collected when the Deployment is deleted). + // + // The object must include apiVersion, kind, and spec. The metadata.name and + // metadata.namespace are generated by the controller and must not be set by the user. + // + // String values in the spec may contain Go template expressions: + // {{ .DeploymentName }} - the controller-generated versioned Deployment name + // {{ .Namespace }} - the namespace of the resource + // {{ .BuildID }} - the Build ID for this version + // + // The controller also auto-injects two well-known fields if they are absent: + // scaleTargetRef - set to point at the versioned Deployment (for HPA, KEDA, WPA, etc.) + // matchLabels - set to the versioned Deployment's selector labels (for PDB, WPA, etc.) + // +kubebuilder:validation:Required + // +kubebuilder:pruning:PreserveUnknownFields + Object runtime.RawExtension `json:"object"` +} + +// OwnedResourceVersionStatus describes the status of an owned resource for a single Build ID. +type OwnedResourceVersionStatus struct { + // BuildID is the Build ID of the versioned Deployment this status entry refers to. + BuildID string `json:"buildID"` + + // Applied is true if the resource was successfully applied for this Build ID. + Applied bool `json:"applied"` + + // ResourceName is the name of the applied Kubernetes resource. + // +optional + ResourceName string `json:"resourceName,omitempty"` + + // Message describes any error if Applied is false. + // +optional + Message string `json:"message,omitempty"` + + // LastTransitionTime is the last time this status entry was updated. + LastTransitionTime metav1.Time `json:"lastTransitionTime"` +} + +// TemporalWorkerOwnedResourceStatus defines the observed state of TemporalWorkerOwnedResource. +type TemporalWorkerOwnedResourceStatus struct { + // Versions describes the per-Build-ID status of owned resources. + // +optional + Versions []OwnedResourceVersionStatus `json:"versions,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:resource:shortName=twor +//+kubebuilder:printcolumn:name="Worker",type="string",JSONPath=".spec.workerRef.name",description="Referenced TemporalWorkerDeployment" +//+kubebuilder:printcolumn:name="Kind",type="string",JSONPath=".spec.object.kind",description="Kind of owned resource" +//+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Age" + +// TemporalWorkerOwnedResource attaches an arbitrary namespaced Kubernetes resource +// (HPA, PDB, WPA, custom CRDs, etc.) to each per-Build-ID versioned Deployment +// managed by a TemporalWorkerDeployment. One copy of the resource is created per +// active Build ID and is owned by the corresponding versioned Deployment. +type TemporalWorkerOwnedResource struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec TemporalWorkerOwnedResourceSpec `json:"spec,omitempty"` + Status TemporalWorkerOwnedResourceStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// TemporalWorkerOwnedResourceList contains a list of TemporalWorkerOwnedResource. +type TemporalWorkerOwnedResourceList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []TemporalWorkerOwnedResource `json:"items"` +} + +func init() { + SchemeBuilder.Register(&TemporalWorkerOwnedResource{}, &TemporalWorkerOwnedResourceList{}) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 1decf431..8241c712 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -519,3 +519,132 @@ func (in *WorkflowExecution) DeepCopy() *WorkflowExecution { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkerDeploymentReference) DeepCopyInto(out *WorkerDeploymentReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkerDeploymentReference. +func (in *WorkerDeploymentReference) DeepCopy() *WorkerDeploymentReference { + if in == nil { + return nil + } + out := new(WorkerDeploymentReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OwnedResourceVersionStatus) DeepCopyInto(out *OwnedResourceVersionStatus) { + *out = *in + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OwnedResourceVersionStatus. +func (in *OwnedResourceVersionStatus) DeepCopy() *OwnedResourceVersionStatus { + if in == nil { + return nil + } + out := new(OwnedResourceVersionStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TemporalWorkerOwnedResourceSpec) DeepCopyInto(out *TemporalWorkerOwnedResourceSpec) { + *out = *in + out.WorkerRef = in.WorkerRef + in.Object.DeepCopyInto(&out.Object) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TemporalWorkerOwnedResourceSpec. +func (in *TemporalWorkerOwnedResourceSpec) DeepCopy() *TemporalWorkerOwnedResourceSpec { + if in == nil { + return nil + } + out := new(TemporalWorkerOwnedResourceSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TemporalWorkerOwnedResourceStatus) DeepCopyInto(out *TemporalWorkerOwnedResourceStatus) { + *out = *in + if in.Versions != nil { + in, out := &in.Versions, &out.Versions + *out = make([]OwnedResourceVersionStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TemporalWorkerOwnedResourceStatus. +func (in *TemporalWorkerOwnedResourceStatus) DeepCopy() *TemporalWorkerOwnedResourceStatus { + if in == nil { + return nil + } + out := new(TemporalWorkerOwnedResourceStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TemporalWorkerOwnedResource) DeepCopyInto(out *TemporalWorkerOwnedResource) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TemporalWorkerOwnedResource. +func (in *TemporalWorkerOwnedResource) DeepCopy() *TemporalWorkerOwnedResource { + if in == nil { + return nil + } + out := new(TemporalWorkerOwnedResource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TemporalWorkerOwnedResource) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TemporalWorkerOwnedResourceList) DeepCopyInto(out *TemporalWorkerOwnedResourceList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]TemporalWorkerOwnedResource, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TemporalWorkerOwnedResourceList. +func (in *TemporalWorkerOwnedResourceList) DeepCopy() *TemporalWorkerOwnedResourceList { + if in == nil { + return nil + } + out := new(TemporalWorkerOwnedResourceList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TemporalWorkerOwnedResourceList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} diff --git a/internal/controller/execplan.go b/internal/controller/execplan.go index df3c6769..e55cbeaf 100644 --- a/internal/controller/execplan.go +++ b/internal/controller/execplan.go @@ -190,5 +190,39 @@ func (r *TemporalWorkerDeploymentReconciler) executePlan(ctx context.Context, l } } + // Apply owned resources via Server-Side Apply. + // Partial failure isolation: all resources are attempted even if some fail; + // errors are collected and returned together. + var ownedResourceErrors []error + for _, apply := range p.ApplyOwnedResources { + l.Info("applying owned resource", + "name", apply.Resource.GetName(), + "kind", apply.Resource.GetKind(), + "fieldManager", apply.FieldManager, + ) + // client.Apply uses Server-Side Apply, which is a create-or-update operation: + // if the resource does not yet exist the API server creates it; if it already + // exists the API server merges only the fields owned by this field manager, + // leaving fields owned by other managers (e.g. the HPA controller) untouched. + // client.ForceOwnership allows this field manager to claim any fields that were + // previously owned by a different manager (e.g. after a field manager rename). + if err := r.Client.Patch( + ctx, + apply.Resource, + client.Apply, + client.ForceOwnership, + client.FieldOwner(apply.FieldManager), + ); err != nil { + l.Error(err, "unable to apply owned resource", + "name", apply.Resource.GetName(), + "kind", apply.Resource.GetKind(), + ) + ownedResourceErrors = append(ownedResourceErrors, err) + } + } + if len(ownedResourceErrors) > 0 { + return fmt.Errorf("errors applying owned resources (%d failures): %v", len(ownedResourceErrors), ownedResourceErrors[0]) + } + return nil } diff --git a/internal/controller/genplan.go b/internal/controller/genplan.go index 050158cc..4c777b33 100644 --- a/internal/controller/genplan.go +++ b/internal/controller/genplan.go @@ -16,6 +16,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" ) // plan holds the actions to execute during reconciliation @@ -38,6 +39,9 @@ type plan struct { // Build IDs of versions from which the controller should // remove IgnoreLastModifierKey from the version metadata RemoveIgnoreLastModifierBuilds []string + + // OwnedResources to apply via Server-Side Apply, one per (TWOR × Build ID) pair. + ApplyOwnedResources []planner.OwnedResourceApply } // startWorkflowConfig defines a workflow to be started @@ -128,6 +132,16 @@ func (r *TemporalWorkerDeploymentReconciler) generatePlan( RolloutStrategy: rolloutStrategy, } + // Fetch all TemporalWorkerOwnedResources that reference this TWD so that the planner + // can render one apply action per (TWOR × active Build ID) pair. + var tworList temporaliov1alpha1.TemporalWorkerOwnedResourceList + if err := r.List(ctx, &tworList, + client.InNamespace(w.Namespace), + client.MatchingFields{tworWorkerRefKey: w.Name}, + ); err != nil { + return nil, fmt.Errorf("unable to list TemporalWorkerOwnedResources: %w", err) + } + planResult, err := planner.GeneratePlan( l, k8sState, @@ -140,6 +154,7 @@ func (r *TemporalWorkerDeploymentReconciler) generatePlan( r.MaxDeploymentVersionsIneligibleForDeletion, gateInput, isGateInputSecret, + tworList.Items, ) if err != nil { return nil, fmt.Errorf("error generating plan: %w", err) @@ -154,6 +169,7 @@ func (r *TemporalWorkerDeploymentReconciler) generatePlan( plan.UpdateVersionConfig = planResult.VersionConfig plan.RemoveIgnoreLastModifierBuilds = planResult.RemoveIgnoreLastModifierBuilds + plan.ApplyOwnedResources = planResult.ApplyOwnedResources // Convert test workflows for _, wf := range planResult.TestWorkflows { diff --git a/internal/controller/worker_controller.go b/internal/controller/worker_controller.go index d6471890..b54b77b2 100644 --- a/internal/controller/worker_controller.go +++ b/internal/controller/worker_controller.go @@ -36,6 +36,9 @@ const ( // TODO(jlegrone): add this everywhere deployOwnerKey = ".metadata.controller" buildIDLabel = "temporal.io/build-id" + + // tworWorkerRefKey is the field index key for TemporalWorkerOwnedResource by workerRef.name. + tworWorkerRefKey = ".spec.workerRef.name" ) // getAPIKeySecretName extracts the secret name from a SecretKeySelector @@ -274,11 +277,24 @@ func (r *TemporalWorkerDeploymentReconciler) SetupWithManager(mgr ctrl.Manager) return err } + // Index TemporalWorkerOwnedResource by spec.workerRef.name for efficient listing. + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &temporaliov1alpha1.TemporalWorkerOwnedResource{}, tworWorkerRefKey, func(rawObj client.Object) []string { + twor, ok := rawObj.(*temporaliov1alpha1.TemporalWorkerOwnedResource) + if !ok { + mgr.GetLogger().Error(fmt.Errorf("error indexing TemporalWorkerOwnedResources"), "could not convert raw object", rawObj) + return nil + } + return []string{twor.Spec.WorkerRef.Name} + }); err != nil { + return err + } + recoverPanic := !r.DisableRecoverPanic return ctrl.NewControllerManagedBy(mgr). For(&temporaliov1alpha1.TemporalWorkerDeployment{}). Owns(&appsv1.Deployment{}). Watches(&temporaliov1alpha1.TemporalConnection{}, handler.EnqueueRequestsFromMapFunc(r.findTWDsUsingConnection)). + Watches(&temporaliov1alpha1.TemporalWorkerOwnedResource{}, handler.EnqueueRequestsFromMapFunc(r.findTWDsForOwnedResource)). WithOptions(controller.Options{ MaxConcurrentReconciles: 100, RecoverPanic: &recoverPanic, @@ -286,6 +302,22 @@ func (r *TemporalWorkerDeploymentReconciler) SetupWithManager(mgr ctrl.Manager) Complete(r) } +// findTWDsForOwnedResource maps a TemporalWorkerOwnedResource to the TWD reconcile request. +func (r *TemporalWorkerDeploymentReconciler) findTWDsForOwnedResource(ctx context.Context, twor client.Object) []reconcile.Request { + tworObj, ok := twor.(*temporaliov1alpha1.TemporalWorkerOwnedResource) + if !ok { + return nil + } + return []reconcile.Request{ + { + NamespacedName: types.NamespacedName{ + Name: tworObj.Spec.WorkerRef.Name, + Namespace: twor.GetNamespace(), + }, + }, + } +} + func (r *TemporalWorkerDeploymentReconciler) findTWDsUsingConnection(ctx context.Context, tc client.Object) []reconcile.Request { var requests []reconcile.Request diff --git a/internal/k8s/deployments.go b/internal/k8s/deployments.go index 3fd30c54..22af5526 100644 --- a/internal/k8s/deployments.go +++ b/internal/k8s/deployments.go @@ -206,10 +206,7 @@ func NewDeploymentWithOwnerRef( buildID string, connection temporaliov1alpha1.TemporalConnectionSpec, ) *appsv1.Deployment { - selectorLabels := map[string]string{ - twdNameLabel: TruncateString(CleanStringForDNS(objectMeta.GetName()), 63), - BuildIDLabel: TruncateString(buildID, 63), - } + selectorLabels := ComputeSelectorLabels(objectMeta.GetName(), buildID) // Set pod labels podLabels := make(map[string]string) diff --git a/internal/k8s/ownedresources.go b/internal/k8s/ownedresources.go new file mode 100644 index 00000000..d9536cb6 --- /dev/null +++ b/internal/k8s/ownedresources.go @@ -0,0 +1,292 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2024 Datadog, Inc. + +package k8s + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "strings" + "text/template" + + temporaliov1alpha1 "github.com/temporalio/temporal-worker-controller/api/v1alpha1" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// OwnedResourceFieldManager returns the SSA field manager string for a given TWOR. +// +// The field manager identity has two requirements: +// 1. Stable across reconcile loops — the API server uses it to track which fields +// this controller "owns". Changing it abandons the old ownership records and +// causes spurious field conflicts until the old entries expire. +// 2. Unique per TWOR instance — if two different TWORs both render resources into +// the same namespace, their field managers must differ so each can own its own +// set of fields without conflicting with the other. +// +// Using "twc/{namespace}/{name}" satisfies both: it never changes for a given TWOR +// and is globally unique within the cluster (namespace+name is a unique identifier). +// The string is capped at 128 characters to stay within the Kubernetes API limit. +func OwnedResourceFieldManager(twor *temporaliov1alpha1.TemporalWorkerOwnedResource) string { + fm := "twc/" + twor.Namespace + "/" + twor.Name + return TruncateString(fm, 128) +} + +const ( + // ownedResourceMaxNameLen is the maximum length of a generated owned resource name. + // 253 is the RFC 1123 DNS subdomain limit used by most Kubernetes resource types + // (HPA, PDB, and most CRDs). Resources with stricter limits (e.g. 63-char DNS + // labels) are rare for the autoscaler/policy use cases TWOR targets. + ownedResourceMaxNameLen = 253 + + // ownedResourceHashLen is the number of hex characters used for the uniqueness suffix. + // 8 hex chars = 4 bytes = 32 bits, giving negligible collision probability + // (< 1 in 10^7 even across thousands of resources in a cluster). + ownedResourceHashLen = 8 +) + +// ComputeOwnedResourceName generates a deterministic, DNS-safe name for the owned resource +// instance corresponding to a given (twdName, tworName, buildID) triple. +// +// The name has the form: +// +// {human-readable-prefix}-{8-char-hash} +// +// The 8-character hash is computed from the full untruncated triple BEFORE any length +// capping occurs. This guarantees that two different triples — including triples that +// differ only in the buildID — always produce different names, even if the human-readable +// prefix is truncated. The buildID is therefore always uniquely represented via the hash, +// regardless of how long twdName or tworName are. +func ComputeOwnedResourceName(twdName, tworName, buildID string) string { + // Hash the full triple first, before any truncation. + h := sha256.Sum256([]byte(twdName + tworName + buildID)) + hashSuffix := hex.EncodeToString(h[:ownedResourceHashLen/2]) // 4 bytes → 8 hex chars + + // Build the human-readable prefix and truncate so the total fits in maxLen. + // suffixLen = len("-") + ownedResourceHashLen + const suffixLen = 1 + ownedResourceHashLen + raw := CleanStringForDNS(twdName + ResourceNameSeparator + tworName + ResourceNameSeparator + buildID) + prefix := TruncateString(raw, ownedResourceMaxNameLen-suffixLen) + // Trim any trailing separator that results from truncating mid-segment. + prefix = strings.TrimRight(prefix, ResourceNameSeparator) + + return prefix + ResourceNameSeparator + hashSuffix +} + +// ComputeSelectorLabels returns the selector labels used by a versioned Deployment. +// These are the same labels set on the Deployment.Spec.Selector.MatchLabels. +func ComputeSelectorLabels(twdName, buildID string) map[string]string { + return map[string]string{ + twdNameLabel: TruncateString(CleanStringForDNS(twdName), 63), + BuildIDLabel: TruncateString(buildID, 63), + } +} + +// TemplateData holds the variables available in Go template expressions within spec.object. +type TemplateData struct { + // DeploymentName is the controller-generated versioned Deployment name. + DeploymentName string + // TemporalNamespace is the Temporal namespace the worker connects to. + TemporalNamespace string + // BuildID is the Build ID for this version. + BuildID string +} + +// RenderOwnedResource produces the Unstructured object to apply via SSA for a given +// TemporalWorkerOwnedResource and versioned Deployment. +// +// Processing order: +// 1. Unmarshal spec.object into an Unstructured +// 2. Auto-inject scaleTargetRef and matchLabels (Layer 1) +// 3. Render Go templates in all string values (Layer 2) +// 4. Set metadata (name, namespace, labels, owner reference) +func RenderOwnedResource( + twor *temporaliov1alpha1.TemporalWorkerOwnedResource, + deployment *appsv1.Deployment, + buildID string, + temporalNamespace string, +) (*unstructured.Unstructured, error) { + // Step 1: unmarshal the raw object + var raw map[string]interface{} + if err := json.Unmarshal(twor.Spec.Object.Raw, &raw); err != nil { + return nil, fmt.Errorf("failed to unmarshal spec.object: %w", err) + } + + data := TemplateData{ + DeploymentName: deployment.Name, + TemporalNamespace: temporalNamespace, + BuildID: buildID, + } + + selectorLabels := ComputeSelectorLabels(twor.Spec.WorkerRef.Name, buildID) + + // Step 2: auto-inject scaleTargetRef and matchLabels into spec subtree + if spec, ok := raw["spec"].(map[string]interface{}); ok { + autoInjectFields(spec, data.DeploymentName, selectorLabels) + } + + // Step 3: render Go templates in all string values + rendered, err := renderTemplateValues(raw, data) + if err != nil { + return nil, fmt.Errorf("failed to render templates: %w", err) + } + raw = rendered.(map[string]interface{}) + + // Step 4: set metadata + resourceName := ComputeOwnedResourceName(twor.Spec.WorkerRef.Name, twor.Name, buildID) + + meta, _ := raw["metadata"].(map[string]interface{}) + if meta == nil { + meta = make(map[string]interface{}) + } + meta["name"] = resourceName + meta["namespace"] = twor.Namespace + + // Merge labels + existingLabels, _ := meta["labels"].(map[string]interface{}) + if existingLabels == nil { + existingLabels = make(map[string]interface{}) + } + for k, v := range selectorLabels { + existingLabels[k] = v + } + meta["labels"] = existingLabels + + // Set owner reference pointing to the versioned Deployment so k8s GC cleans up + // the owned resource when the Deployment is deleted. + blockOwnerDeletion := true + isController := true + meta["ownerReferences"] = []interface{}{ + map[string]interface{}{ + "apiVersion": appsv1.SchemeGroupVersion.String(), + "kind": "Deployment", + "name": deployment.Name, + "uid": string(deployment.UID), + "blockOwnerDeletion": blockOwnerDeletion, + "controller": isController, + }, + } + + raw["metadata"] = meta + + obj := &unstructured.Unstructured{Object: raw} + return obj, nil +} + +// autoInjectFields recursively traverses obj and injects scaleTargetRef and matchLabels +// wherever the key is present with a null value. Users signal intent by writing +// `scaleTargetRef: null` or `matchLabels: null` in their spec.object. This covers Layer 1 +// auto-injection without the risk of adding unknown fields to resource types that don't use them. +func autoInjectFields(obj map[string]interface{}, deploymentName string, selectorLabels map[string]string) { + for k, v := range obj { + switch k { + case "scaleTargetRef": + // Inject only when the key is present but null (user opted in) + if v == nil { + obj[k] = map[string]interface{}{ + "apiVersion": appsv1.SchemeGroupVersion.String(), + "kind": "Deployment", + "name": deploymentName, + } + } + case "matchLabels": + // Inject only when the key is present but null (user opted in) + if v == nil { + labels := make(map[string]interface{}, len(selectorLabels)) + for lk, lv := range selectorLabels { + labels[lk] = lv + } + obj[k] = labels + } + default: + // Recurse into nested objects + if nested, ok := v.(map[string]interface{}); ok { + autoInjectFields(nested, deploymentName, selectorLabels) + } else if arr, ok := v.([]interface{}); ok { + for _, item := range arr { + if nestedItem, ok := item.(map[string]interface{}); ok { + autoInjectFields(nestedItem, deploymentName, selectorLabels) + } + } + } + } + } +} + +// renderTemplateValues recursively traverses a JSON-decoded value tree and renders +// Go template expressions in all string values. Returns the modified value. +func renderTemplateValues(v interface{}, data TemplateData) (interface{}, error) { + switch typed := v.(type) { + case string: + rendered, err := renderString(typed, data) + if err != nil { + return nil, err + } + return rendered, nil + case map[string]interface{}: + for k, val := range typed { + rendered, err := renderTemplateValues(val, data) + if err != nil { + return nil, fmt.Errorf("field %q: %w", k, err) + } + typed[k] = rendered + } + return typed, nil + case []interface{}: + for i, item := range typed { + rendered, err := renderTemplateValues(item, data) + if err != nil { + return nil, fmt.Errorf("index %d: %w", i, err) + } + typed[i] = rendered + } + return typed, nil + default: + // numbers, booleans, nil — pass through unchanged + return v, nil + } +} + +// renderString renders a single string as a Go template with the given data. +// Strings without template expressions are returned unchanged. +func renderString(s string, data TemplateData) (string, error) { + // Fast path: skip parsing if no template markers + if !containsTemplateMarker(s) { + return s, nil + } + tmpl, err := template.New("").Option("missingkey=error").Parse(s) + if err != nil { + return "", fmt.Errorf("invalid template %q: %w", s, err) + } + var buf bytes.Buffer + if err := tmpl.Execute(&buf, data); err != nil { + return "", fmt.Errorf("template execution failed for %q: %w", s, err) + } + return buf.String(), nil +} + +// containsTemplateMarker returns true if s contains "{{" indicating a Go template expression. +func containsTemplateMarker(s string) bool { + for i := 0; i < len(s)-1; i++ { + if s[i] == '{' && s[i+1] == '{' { + return true + } + } + return false +} + +// OwnedResourceVersionStatusForBuildID is a helper to build a status entry. +func OwnedResourceVersionStatusForBuildID(buildID, resourceName string, applied bool, message string) temporaliov1alpha1.OwnedResourceVersionStatus { + return temporaliov1alpha1.OwnedResourceVersionStatus{ + BuildID: buildID, + Applied: applied, + ResourceName: resourceName, + Message: message, + LastTransitionTime: metav1.Now(), + } +} diff --git a/internal/k8s/ownedresources_test.go b/internal/k8s/ownedresources_test.go new file mode 100644 index 00000000..4000224d --- /dev/null +++ b/internal/k8s/ownedresources_test.go @@ -0,0 +1,305 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2024 Datadog, Inc. + +package k8s + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + temporaliov1alpha1 "github.com/temporalio/temporal-worker-controller/api/v1alpha1" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" +) + +// expectedOwnedResourceName replicates the naming logic for use in tests. +func expectedOwnedResourceName(twdName, tworName, buildID string) string { + h := sha256.Sum256([]byte(twdName + tworName + buildID)) + hashSuffix := hex.EncodeToString(h[:4]) + raw := CleanStringForDNS(twdName + "-" + tworName + "-" + buildID) + prefix := strings.TrimRight(TruncateString(raw, 253-9), "-") + return prefix + "-" + hashSuffix +} + +func TestComputeOwnedResourceName(t *testing.T) { + t.Run("short names produce human-readable result with hash suffix", func(t *testing.T) { + got := ComputeOwnedResourceName("my-worker", "my-hpa", "image-abc123") + // Should start with the human-readable prefix + assert.True(t, strings.HasPrefix(got, "my-worker-my-hpa-image-abc123-"), "got: %q", got) + // Should be ≤ 253 chars + assert.LessOrEqual(t, len(got), 253) + }) + + t.Run("special chars are cleaned for DNS", func(t *testing.T) { + got := ComputeOwnedResourceName("my_worker", "my/hpa", "image:latest") + assert.True(t, strings.HasPrefix(got, "my-worker-my-hpa-image-latest-"), "got: %q", got) + assert.LessOrEqual(t, len(got), 253) + }) + + t.Run("deterministic — same inputs always produce same name", func(t *testing.T) { + a := ComputeOwnedResourceName("w", "r", "b1") + b := ComputeOwnedResourceName("w", "r", "b1") + assert.Equal(t, a, b) + }) + + t.Run("different buildIDs always produce different names (hash suffix)", func(t *testing.T) { + // Even if the prefix would be identical after truncation, the hash must differ. + name1 := ComputeOwnedResourceName("my-worker", "my-hpa", "build-aaa") + name2 := ComputeOwnedResourceName("my-worker", "my-hpa", "build-bbb") + assert.NotEqual(t, name1, name2) + }) + + t.Run("very long names are still ≤ 253 chars and distinct per buildID", func(t *testing.T) { + longTWD := strings.Repeat("w", 63) + longTWOR := strings.Repeat("r", 253) // maximum k8s object name + buildID1 := "build-" + strings.Repeat("a", 57) + buildID2 := "build-" + strings.Repeat("b", 57) + + n1 := ComputeOwnedResourceName(longTWD, longTWOR, buildID1) + n2 := ComputeOwnedResourceName(longTWD, longTWOR, buildID2) + + assert.LessOrEqual(t, len(n1), 253, "name1 length: %d", len(n1)) + assert.LessOrEqual(t, len(n2), 253, "name2 length: %d", len(n2)) + assert.NotEqual(t, n1, n2, "names must differ even when prefix is fully truncated") + }) + + t.Run("name matches expected formula", func(t *testing.T) { + got := ComputeOwnedResourceName("my-worker", "my-hpa", "abc123") + assert.Equal(t, expectedOwnedResourceName("my-worker", "my-hpa", "abc123"), got) + }) +} + +func TestComputeSelectorLabels(t *testing.T) { + labels := ComputeSelectorLabels("my-worker", "abc-123") + assert.Equal(t, "my-worker", labels[twdNameLabel]) + assert.Equal(t, "abc-123", labels[BuildIDLabel]) +} + +func TestContainsTemplateMarker(t *testing.T) { + assert.True(t, containsTemplateMarker("hello {{ .DeploymentName }}")) + assert.False(t, containsTemplateMarker("hello world")) + assert.False(t, containsTemplateMarker("{ single brace }")) +} + +func TestRenderString(t *testing.T) { + data := TemplateData{ + DeploymentName: "my-worker-abc123", + TemporalNamespace: "my-temporal-ns", + BuildID: "abc123", + } + + tests := []struct { + input string + want string + }{ + {"plain string", "plain string"}, + {"{{ .DeploymentName }}", "my-worker-abc123"}, + {"{{ .TemporalNamespace }}", "my-temporal-ns"}, + {"{{ .BuildID }}", "abc123"}, + {"Monitor for build {{ .BuildID }}", "Monitor for build abc123"}, + {"{{ .DeploymentName }}.{{ .TemporalNamespace }}", "my-worker-abc123.my-temporal-ns"}, + } + for _, tc := range tests { + got, err := renderString(tc.input, data) + require.NoError(t, err) + assert.Equal(t, tc.want, got) + } +} + +func TestAutoInjectFields_ScaleTargetRef(t *testing.T) { + selectorLabels := map[string]string{ + BuildIDLabel: "abc123", + twdNameLabel: "my-worker", + } + + t.Run("does not inject scaleTargetRef when key is entirely absent", func(t *testing.T) { + spec := map[string]interface{}{ + "minReplicas": 1, + "maxReplicas": 5, + } + autoInjectFields(spec, "my-worker-abc123", selectorLabels) + _, hasKey := spec["scaleTargetRef"] + assert.False(t, hasKey, "scaleTargetRef should not be injected when absent (user must opt in with null)") + }) + + t.Run("injects scaleTargetRef when explicitly null (user opt-in)", func(t *testing.T) { + spec := map[string]interface{}{ + "scaleTargetRef": nil, + } + autoInjectFields(spec, "my-worker-abc123", selectorLabels) + ref, ok := spec["scaleTargetRef"].(map[string]interface{}) + require.True(t, ok) + assert.Equal(t, "my-worker-abc123", ref["name"]) + assert.Equal(t, "Deployment", ref["kind"]) + assert.Equal(t, appsv1.SchemeGroupVersion.String(), ref["apiVersion"]) + }) + + t.Run("does not overwrite existing scaleTargetRef", func(t *testing.T) { + spec := map[string]interface{}{ + "scaleTargetRef": map[string]interface{}{ + "name": "custom-deployment", + "kind": "Deployment", + }, + } + autoInjectFields(spec, "my-worker-abc123", selectorLabels) + ref := spec["scaleTargetRef"].(map[string]interface{}) + assert.Equal(t, "custom-deployment", ref["name"], "should not overwrite user-provided ref") + }) +} + +func TestAutoInjectFields_MatchLabels(t *testing.T) { + selectorLabels := map[string]string{ + BuildIDLabel: "abc123", + twdNameLabel: "my-worker", + } + + t.Run("does not inject matchLabels when key is absent", func(t *testing.T) { + spec := map[string]interface{}{ + "selector": map[string]interface{}{}, + } + autoInjectFields(spec, "my-worker-abc123", selectorLabels) + selector := spec["selector"].(map[string]interface{}) + _, hasKey := selector["matchLabels"] + assert.False(t, hasKey, "matchLabels should not be injected when absent (user must opt in with null)") + }) + + t.Run("injects matchLabels when explicitly null (user opt-in)", func(t *testing.T) { + spec := map[string]interface{}{ + "selector": map[string]interface{}{ + "matchLabels": nil, + }, + } + autoInjectFields(spec, "my-worker-abc123", selectorLabels) + selector := spec["selector"].(map[string]interface{}) + labels, ok := selector["matchLabels"].(map[string]interface{}) + require.True(t, ok) + assert.Equal(t, "abc123", labels[BuildIDLabel]) + assert.Equal(t, "my-worker", labels[twdNameLabel]) + }) + + t.Run("does not overwrite existing matchLabels", func(t *testing.T) { + spec := map[string]interface{}{ + "selector": map[string]interface{}{ + "matchLabels": map[string]interface{}{ + "custom": "label", + }, + }, + } + autoInjectFields(spec, "my-worker-abc123", selectorLabels) + selector := spec["selector"].(map[string]interface{}) + labels := selector["matchLabels"].(map[string]interface{}) + assert.Equal(t, "label", labels["custom"], "should not overwrite user-provided labels") + }) +} + +func TestRenderOwnedResource(t *testing.T) { + hpaSpec := map[string]interface{}{ + "apiVersion": "autoscaling/v2", + "kind": "HorizontalPodAutoscaler", + "spec": map[string]interface{}{ + "scaleTargetRef": nil, // opt in to auto-injection + "minReplicas": float64(2), + "maxReplicas": float64(10), + }, + } + rawBytes, err := json.Marshal(hpaSpec) + require.NoError(t, err) + + twor := &temporaliov1alpha1.TemporalWorkerOwnedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-hpa", + Namespace: "default", + }, + Spec: temporaliov1alpha1.TemporalWorkerOwnedResourceSpec{ + WorkerRef: temporaliov1alpha1.WorkerDeploymentReference{ + Name: "my-worker", + }, + Object: runtime.RawExtension{Raw: rawBytes}, + }, + } + + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-worker-abc123", + Namespace: "default", + UID: types.UID("test-uid-123"), + }, + } + buildID := "abc123" + + obj, err := RenderOwnedResource(twor, deployment, buildID, "my-temporal-ns") + require.NoError(t, err) + + // Check metadata — name follows the hash-suffix formula + assert.Equal(t, expectedOwnedResourceName("my-worker", "my-hpa", "abc123"), obj.GetName()) + assert.Equal(t, "default", obj.GetNamespace()) + + // Check selector labels were added + labels := obj.GetLabels() + assert.Equal(t, "abc123", labels[BuildIDLabel]) + assert.Equal(t, "my-worker", labels[twdNameLabel]) + + // Check owner reference points to the Deployment + ownerRefs := obj.GetOwnerReferences() + require.Len(t, ownerRefs, 1) + assert.Equal(t, "my-worker-abc123", ownerRefs[0].Name) + assert.Equal(t, "Deployment", ownerRefs[0].Kind) + assert.Equal(t, types.UID("test-uid-123"), ownerRefs[0].UID) + + // Check scaleTargetRef was auto-injected + spec, ok := obj.Object["spec"].(map[string]interface{}) + require.True(t, ok) + ref, ok := spec["scaleTargetRef"].(map[string]interface{}) + require.True(t, ok, "scaleTargetRef should have been auto-injected") + assert.Equal(t, "my-worker-abc123", ref["name"]) +} + +func TestRenderOwnedResource_WithTemplates(t *testing.T) { + objSpec := map[string]interface{}{ + "apiVersion": "monitoring.example.com/v1", + "kind": "WorkloadMonitor", + "spec": map[string]interface{}{ + "targetWorkload": "{{ .DeploymentName }}", + "description": "Monitor for build {{ .BuildID }} in {{ .TemporalNamespace }}", + }, + } + rawBytes, err := json.Marshal(objSpec) + require.NoError(t, err) + + twor := &temporaliov1alpha1.TemporalWorkerOwnedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-monitor", + Namespace: "production", + }, + Spec: temporaliov1alpha1.TemporalWorkerOwnedResourceSpec{ + WorkerRef: temporaliov1alpha1.WorkerDeploymentReference{ + Name: "my-worker", + }, + Object: runtime.RawExtension{Raw: rawBytes}, + }, + } + + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-worker-abc123", + Namespace: "production", + UID: types.UID("uid-abc"), + }, + } + + obj, err := RenderOwnedResource(twor, deployment, "abc123", "my-temporal-ns") + require.NoError(t, err) + + spec, ok := obj.Object["spec"].(map[string]interface{}) + require.True(t, ok) + assert.Equal(t, "my-worker-abc123", spec["targetWorkload"]) + assert.Equal(t, "Monitor for build abc123 in my-temporal-ns", spec["description"]) +} diff --git a/internal/planner/planner.go b/internal/planner/planner.go index 5410d359..c757d739 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -16,8 +16,15 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) +// OwnedResourceApply holds a rendered owned resource to apply via Server-Side Apply. +type OwnedResourceApply struct { + Resource *unstructured.Unstructured + FieldManager string +} + // Plan holds the actions to execute during reconciliation type Plan struct { // Which actions to take @@ -30,6 +37,8 @@ type Plan struct { // Build IDs of versions from which the controller should // remove IgnoreLastModifierKey from the version metadata RemoveIgnoreLastModifierBuilds []string + // ApplyOwnedResources holds resources to apply via SSA, one per (TWOR × Build ID) pair. + ApplyOwnedResources []OwnedResourceApply } // VersionConfig defines version configuration for Temporal @@ -78,6 +87,7 @@ func GeneratePlan( maxVersionsIneligibleForDeletion int32, gateInput []byte, isGateInputSecret bool, + twors []temporaliov1alpha1.TemporalWorkerOwnedResource, ) (*Plan, error) { plan := &Plan{ ScaleDeployments: make(map[*corev1.ObjectReference]uint32), @@ -114,9 +124,44 @@ func GeneratePlan( // TODO(jlegrone): generate warnings/events on the TemporalWorkerDeployment resource when buildIDs are reachable // but have no corresponding Deployment. + plan.ApplyOwnedResources = getOwnedResourceApplies(l, twors, k8sState, spec.WorkerOptions.TemporalNamespace) + return plan, nil } +// getOwnedResourceApplies renders one OwnedResourceApply for each (TWOR × active Build ID) pair. +// Pairs that fail to render are logged and skipped; they do not block the rest. +func getOwnedResourceApplies( + l logr.Logger, + twors []temporaliov1alpha1.TemporalWorkerOwnedResource, + k8sState *k8s.DeploymentState, + temporalNamespace string, +) []OwnedResourceApply { + var applies []OwnedResourceApply + for i := range twors { + twor := &twors[i] + if twor.Spec.Object.Raw == nil { + l.Info("skipping TemporalWorkerOwnedResource with empty spec.object", "name", twor.Name) + continue + } + for buildID, deployment := range k8sState.Deployments { + rendered, err := k8s.RenderOwnedResource(twor, deployment, buildID, temporalNamespace) + if err != nil { + l.Error(err, "failed to render TemporalWorkerOwnedResource", + "twor", twor.Name, + "buildID", buildID, + ) + continue + } + applies = append(applies, OwnedResourceApply{ + Resource: rendered, + FieldManager: k8s.OwnedResourceFieldManager(twor), + }) + } + } + return applies +} + // checkAndUpdateDeploymentConnectionSpec determines whether the Deployment for the given buildID is // out-of-date with respect to the provided TemporalConnectionSpec. If an update is required, it mutates // the existing Deployment in-place and returns a pointer to that Deployment. If no update is needed or diff --git a/internal/planner/planner_test.go b/internal/planner/planner_test.go index 9873c2b5..5f9d4ea4 100644 --- a/internal/planner/planner_test.go +++ b/internal/planner/planner_test.go @@ -6,6 +6,7 @@ package planner import ( "context" + "encoding/json" "slices" "testing" "time" @@ -22,6 +23,8 @@ import ( corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" ) func TestGeneratePlan(t *testing.T) { @@ -41,6 +44,8 @@ func TestGeneratePlan(t *testing.T) { expectConfigSetCurrent *bool // pointer so we can test nil expectConfigRampPercent *int32 // pointer so we can test nil, in percentage (0-100) maxVersionsIneligibleForDeletion *int32 // set by env if non-nil, else default 75 + twors []temporaliov1alpha1.TemporalWorkerOwnedResource + expectOwnedResourceApplies int }{ { name: "empty state creates new deployment", @@ -390,6 +395,65 @@ func TestGeneratePlan(t *testing.T) { }, expectUpdate: 1, }, + { + name: "one TWOR with two deployments produces two owned resource applies", + k8sState: &k8s.DeploymentState{ + Deployments: map[string]*appsv1.Deployment{ + "build-a": createDeploymentWithUID("worker-build-a", "uid-a"), + "build-b": createDeploymentWithUID("worker-build-b", "uid-b"), + }, + DeploymentsByTime: []*appsv1.Deployment{}, + DeploymentRefs: map[string]*corev1.ObjectReference{}, + }, + status: &temporaliov1alpha1.TemporalWorkerDeploymentStatus{ + TargetVersion: temporaliov1alpha1.TargetWorkerDeploymentVersion{ + BaseWorkerDeploymentVersion: temporaliov1alpha1.BaseWorkerDeploymentVersion{ + BuildID: "build-a", + Status: temporaliov1alpha1.VersionStatusCurrent, + Deployment: &corev1.ObjectReference{Name: "worker-build-a"}, + }, + }, + }, + spec: &temporaliov1alpha1.TemporalWorkerDeploymentSpec{ + Replicas: func() *int32 { r := int32(1); return &r }(), + }, + state: &temporal.TemporalWorkerState{}, + config: &Config{ + RolloutStrategy: temporaliov1alpha1.RolloutStrategy{}, + }, + twors: []temporaliov1alpha1.TemporalWorkerOwnedResource{ + createTestTWOR("my-hpa", "my-worker"), + }, + expectScale: 1, + expectOwnedResourceApplies: 2, + }, + { + name: "no TWORs produces no owned resource applies", + k8sState: &k8s.DeploymentState{ + Deployments: map[string]*appsv1.Deployment{ + "build-a": createDeploymentWithUID("worker-build-a", "uid-a"), + }, + DeploymentsByTime: []*appsv1.Deployment{}, + DeploymentRefs: map[string]*corev1.ObjectReference{}, + }, + status: &temporaliov1alpha1.TemporalWorkerDeploymentStatus{ + TargetVersion: temporaliov1alpha1.TargetWorkerDeploymentVersion{ + BaseWorkerDeploymentVersion: temporaliov1alpha1.BaseWorkerDeploymentVersion{ + BuildID: "build-a", + Status: temporaliov1alpha1.VersionStatusCurrent, + Deployment: &corev1.ObjectReference{Name: "worker-build-a"}, + }, + }, + }, + spec: &temporaliov1alpha1.TemporalWorkerDeploymentSpec{ + Replicas: func() *int32 { r := int32(1); return &r }(), + }, + state: &temporal.TemporalWorkerState{}, + config: &Config{RolloutStrategy: temporaliov1alpha1.RolloutStrategy{}}, + twors: nil, + expectScale: 1, + expectOwnedResourceApplies: 0, + }, } for _, tc := range testCases { @@ -402,7 +466,7 @@ func TestGeneratePlan(t *testing.T) { maxV = *tc.maxVersionsIneligibleForDeletion } - plan, err := GeneratePlan(logr.Discard(), tc.k8sState, tc.status, tc.spec, tc.state, createDefaultConnectionSpec(), tc.config, "test/namespace", maxV, nil, false) + plan, err := GeneratePlan(logr.Discard(), tc.k8sState, tc.status, tc.spec, tc.state, createDefaultConnectionSpec(), tc.config, "test/namespace", maxV, nil, false, tc.twors) require.NoError(t, err) assert.Equal(t, tc.expectDelete, len(plan.DeleteDeployments), "unexpected number of deletions") @@ -411,6 +475,7 @@ func TestGeneratePlan(t *testing.T) { assert.Equal(t, tc.expectUpdate, len(plan.UpdateDeployments), "unexpected number of updates") assert.Equal(t, tc.expectWorkflow, len(plan.TestWorkflows), "unexpected number of test workflows") assert.Equal(t, tc.expectConfig, plan.VersionConfig != nil, "unexpected version config presence") + assert.Equal(t, tc.expectOwnedResourceApplies, len(plan.ApplyOwnedResources), "unexpected number of owned resource applies") if tc.expectConfig { assert.NotNil(t, plan.VersionConfig, "expected version config") @@ -1928,7 +1993,7 @@ func TestComplexVersionStateScenarios(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - plan, err := GeneratePlan(logr.Discard(), tc.k8sState, tc.status, tc.spec, tc.state, createDefaultConnectionSpec(), tc.config, "test/namespace", defaults.MaxVersionsIneligibleForDeletion, nil, false) + plan, err := GeneratePlan(logr.Discard(), tc.k8sState, tc.status, tc.spec, tc.state, createDefaultConnectionSpec(), tc.config, "test/namespace", defaults.MaxVersionsIneligibleForDeletion, nil, false, nil) require.NoError(t, err) assert.Equal(t, tc.expectDeletes, len(plan.DeleteDeployments), "unexpected number of deletes") @@ -2825,3 +2890,323 @@ func TestResolveGateInput(t *testing.T) { }) } } + +func TestGetOwnedResourceApplies(t *testing.T) { + testCases := []struct { + name string + twors []temporaliov1alpha1.TemporalWorkerOwnedResource + k8sState *k8s.DeploymentState + expectCount int + }{ + { + name: "no TWORs produces no applies", + twors: nil, + k8sState: &k8s.DeploymentState{ + Deployments: map[string]*appsv1.Deployment{ + "build-a": createDeploymentWithUID("worker-build-a", "uid-a"), + }, + }, + expectCount: 0, + }, + { + name: "no deployments produces no applies", + twors: []temporaliov1alpha1.TemporalWorkerOwnedResource{ + createTestTWOR("my-hpa", "my-worker"), + }, + k8sState: &k8s.DeploymentState{ + Deployments: map[string]*appsv1.Deployment{}, + }, + expectCount: 0, + }, + { + name: "1 TWOR × 1 deployment produces 1 apply", + twors: []temporaliov1alpha1.TemporalWorkerOwnedResource{ + createTestTWOR("my-hpa", "my-worker"), + }, + k8sState: &k8s.DeploymentState{ + Deployments: map[string]*appsv1.Deployment{ + "build-a": createDeploymentWithUID("worker-build-a", "uid-a"), + }, + }, + expectCount: 1, + }, + { + name: "1 TWOR × 2 deployments produces 2 applies", + twors: []temporaliov1alpha1.TemporalWorkerOwnedResource{ + createTestTWOR("my-hpa", "my-worker"), + }, + k8sState: &k8s.DeploymentState{ + Deployments: map[string]*appsv1.Deployment{ + "build-a": createDeploymentWithUID("worker-build-a", "uid-a"), + "build-b": createDeploymentWithUID("worker-build-b", "uid-b"), + }, + }, + expectCount: 2, + }, + { + name: "2 TWORs × 1 deployment produces 2 applies", + twors: []temporaliov1alpha1.TemporalWorkerOwnedResource{ + createTestTWOR("my-hpa", "my-worker"), + createTestTWOR("my-pdb", "my-worker"), + }, + k8sState: &k8s.DeploymentState{ + Deployments: map[string]*appsv1.Deployment{ + "build-a": createDeploymentWithUID("worker-build-a", "uid-a"), + }, + }, + expectCount: 2, + }, + { + name: "2 TWORs × 2 deployments produces 4 applies", + twors: []temporaliov1alpha1.TemporalWorkerOwnedResource{ + createTestTWOR("my-hpa", "my-worker"), + createTestTWOR("my-pdb", "my-worker"), + }, + k8sState: &k8s.DeploymentState{ + Deployments: map[string]*appsv1.Deployment{ + "build-a": createDeploymentWithUID("worker-build-a", "uid-a"), + "build-b": createDeploymentWithUID("worker-build-b", "uid-b"), + }, + }, + expectCount: 4, + }, + { + name: "TWOR with nil Raw is skipped without blocking others", + twors: []temporaliov1alpha1.TemporalWorkerOwnedResource{ + { + ObjectMeta: metav1.ObjectMeta{Name: "nil-raw", Namespace: "default"}, + Spec: temporaliov1alpha1.TemporalWorkerOwnedResourceSpec{ + WorkerRef: temporaliov1alpha1.WorkerDeploymentReference{Name: "my-worker"}, + Object: runtime.RawExtension{Raw: nil}, + }, + }, + createTestTWOR("my-hpa", "my-worker"), + }, + k8sState: &k8s.DeploymentState{ + Deployments: map[string]*appsv1.Deployment{ + "build-a": createDeploymentWithUID("worker-build-a", "uid-a"), + }, + }, + expectCount: 1, // only the valid TWOR + }, + { + name: "TWOR with invalid template is skipped without blocking others", + twors: []temporaliov1alpha1.TemporalWorkerOwnedResource{ + createTestTWORWithInvalidTemplate("bad-template", "my-worker"), + createTestTWOR("my-hpa", "my-worker"), + }, + k8sState: &k8s.DeploymentState{ + Deployments: map[string]*appsv1.Deployment{ + "build-a": createDeploymentWithUID("worker-build-a", "uid-a"), + }, + }, + expectCount: 1, // only the valid TWOR + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + applies := getOwnedResourceApplies(logr.Discard(), tc.twors, tc.k8sState, "test-temporal-ns") + assert.Equal(t, tc.expectCount, len(applies), "unexpected number of owned resource applies") + }) + } +} + +func TestGetOwnedResourceApplies_ApplyContents(t *testing.T) { + twor := createTestTWOR("my-hpa", "my-worker") + deployment := createDeploymentWithUID("my-worker-build-abc", "uid-abc") + k8sState := &k8s.DeploymentState{ + Deployments: map[string]*appsv1.Deployment{ + "build-abc": deployment, + }, + } + + applies := getOwnedResourceApplies(logr.Discard(), []temporaliov1alpha1.TemporalWorkerOwnedResource{twor}, k8sState, "test-temporal-ns") + require.Len(t, applies, 1) + + apply := applies[0] + + // Field manager must be the stable TWOR-scoped identifier + assert.Equal(t, k8s.OwnedResourceFieldManager(&twor), apply.FieldManager) + + // Resource kind and apiVersion must come from the template + assert.Equal(t, "HorizontalPodAutoscaler", apply.Resource.GetKind()) + assert.Equal(t, "autoscaling/v2", apply.Resource.GetAPIVersion()) + + // Resource must be owned by the versioned Deployment + ownerRefs := apply.Resource.GetOwnerReferences() + require.Len(t, ownerRefs, 1) + assert.Equal(t, deployment.Name, ownerRefs[0].Name) + assert.Equal(t, "Deployment", ownerRefs[0].Kind) + assert.Equal(t, types.UID("uid-abc"), ownerRefs[0].UID) + + // Resource name must be deterministic + assert.Equal(t, k8s.ComputeOwnedResourceName("my-worker", "my-hpa", "build-abc"), apply.Resource.GetName()) +} + +func TestGetOwnedResourceApplies_FieldManagerDistinctPerTWOR(t *testing.T) { + twor1 := createTestTWOR("my-hpa", "my-worker") + twor2 := createTestTWOR("my-pdb", "my-worker") + k8sState := &k8s.DeploymentState{ + Deployments: map[string]*appsv1.Deployment{ + "build-a": createDeploymentWithUID("worker-build-a", "uid-a"), + }, + } + + applies := getOwnedResourceApplies(logr.Discard(), []temporaliov1alpha1.TemporalWorkerOwnedResource{twor1, twor2}, k8sState, "test-temporal-ns") + require.Len(t, applies, 2) + + fms := make(map[string]bool) + for _, a := range applies { + fms[a.FieldManager] = true + } + assert.Len(t, fms, 2, "each TWOR must produce a distinct field manager") +} + +// createTestTWOR builds a minimal valid TemporalWorkerOwnedResource for use in tests. +// The embedded object is a stub HPA with scaleTargetRef opted in for auto-injection. +func createTestTWOR(name, workerRefName string) temporaliov1alpha1.TemporalWorkerOwnedResource { + hpaSpec := map[string]interface{}{ + "apiVersion": "autoscaling/v2", + "kind": "HorizontalPodAutoscaler", + "spec": map[string]interface{}{ + "scaleTargetRef": nil, // opt in to auto-injection + "minReplicas": float64(1), + "maxReplicas": float64(5), + }, + } + raw, _ := json.Marshal(hpaSpec) + return temporaliov1alpha1.TemporalWorkerOwnedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + }, + Spec: temporaliov1alpha1.TemporalWorkerOwnedResourceSpec{ + WorkerRef: temporaliov1alpha1.WorkerDeploymentReference{Name: workerRefName}, + Object: runtime.RawExtension{Raw: raw}, + }, + } +} + +// createDeploymentWithUID builds a Deployment with the given name and UID, with the default +// connection spec hash annotation pre-set so it does not trigger an update during plan generation. +func createDeploymentWithUID(name, uid string) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + UID: types.UID(uid), + }, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + k8s.ConnectionSpecHashAnnotation: k8s.ComputeConnectionSpecHash(createDefaultConnectionSpec()), + }, + }, + }, + }, + } +} + +func TestGetOwnedResourceApplies_MatchLabelsInjection(t *testing.T) { + // PDB with matchLabels opted in for auto-injection via null sentinel. + pdbSpec := map[string]interface{}{ + "apiVersion": "policy/v1", + "kind": "PodDisruptionBudget", + "spec": map[string]interface{}{ + "selector": map[string]interface{}{ + "matchLabels": nil, // null = opt in to auto-injection + }, + "minAvailable": float64(1), + }, + } + raw, _ := json.Marshal(pdbSpec) + twor := temporaliov1alpha1.TemporalWorkerOwnedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "my-pdb", Namespace: "default"}, + Spec: temporaliov1alpha1.TemporalWorkerOwnedResourceSpec{ + WorkerRef: temporaliov1alpha1.WorkerDeploymentReference{Name: "my-worker"}, + Object: runtime.RawExtension{Raw: raw}, + }, + } + + deployment := createDeploymentWithUID("my-worker-build-abc", "uid-abc") + k8sState := &k8s.DeploymentState{ + Deployments: map[string]*appsv1.Deployment{"build-abc": deployment}, + } + + applies := getOwnedResourceApplies(logr.Discard(), []temporaliov1alpha1.TemporalWorkerOwnedResource{twor}, k8sState, "test-temporal-ns") + require.Len(t, applies, 1) + + spec, ok := applies[0].Resource.Object["spec"].(map[string]interface{}) + require.True(t, ok) + selector, ok := spec["selector"].(map[string]interface{}) + require.True(t, ok) + matchLabels, ok := selector["matchLabels"].(map[string]interface{}) + require.True(t, ok, "matchLabels should have been auto-injected") + + // The injected labels must equal ComputeSelectorLabels(workerRef, buildID). + expected := k8s.ComputeSelectorLabels("my-worker", "build-abc") + for k, v := range expected { + assert.Equal(t, v, matchLabels[k], "injected matchLabels[%q]", k) + } + assert.Len(t, matchLabels, len(expected), "no extra keys should be injected") +} + +func TestGetOwnedResourceApplies_GoTemplateRendering(t *testing.T) { + // Arbitrary CRD that uses all three template variables. + objSpec := map[string]interface{}{ + "apiVersion": "monitoring.example.com/v1", + "kind": "WorkloadMonitor", + "spec": map[string]interface{}{ + "targetWorkload": "{{ .DeploymentName }}", + "versionLabel": "build-{{ .BuildID }}", + "temporalNamespace": "{{ .TemporalNamespace }}", + }, + } + raw, _ := json.Marshal(objSpec) + twor := temporaliov1alpha1.TemporalWorkerOwnedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "my-monitor", Namespace: "k8s-production"}, + Spec: temporaliov1alpha1.TemporalWorkerOwnedResourceSpec{ + WorkerRef: temporaliov1alpha1.WorkerDeploymentReference{Name: "my-worker"}, + Object: runtime.RawExtension{Raw: raw}, + }, + } + + deployment := createDeploymentWithUID("my-worker-build-abc", "uid-abc") + k8sState := &k8s.DeploymentState{ + Deployments: map[string]*appsv1.Deployment{"build-abc": deployment}, + } + + applies := getOwnedResourceApplies(logr.Discard(), []temporaliov1alpha1.TemporalWorkerOwnedResource{twor}, k8sState, "temporal-production") + require.Len(t, applies, 1) + + spec, ok := applies[0].Resource.Object["spec"].(map[string]interface{}) + require.True(t, ok) + assert.Equal(t, "my-worker-build-abc", spec["targetWorkload"], ".DeploymentName not rendered") + assert.Equal(t, "build-build-abc", spec["versionLabel"], ".BuildID not rendered") + assert.Equal(t, "temporal-production", spec["temporalNamespace"], ".TemporalNamespace not rendered") +} + +// createTestTWORWithInvalidTemplate builds a TWOR whose spec.object contains a broken Go +// template expression, causing RenderOwnedResource to return an error. +func createTestTWORWithInvalidTemplate(name, workerRefName string) temporaliov1alpha1.TemporalWorkerOwnedResource { + badSpec := map[string]interface{}{ + "apiVersion": "autoscaling/v2", + "kind": "HorizontalPodAutoscaler", + "spec": map[string]interface{}{ + "description": "{{ .NonExistentField }}", // will fail with missingkey=error + }, + } + raw, _ := json.Marshal(badSpec) + return temporaliov1alpha1.TemporalWorkerOwnedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + }, + Spec: temporaliov1alpha1.TemporalWorkerOwnedResourceSpec{ + WorkerRef: temporaliov1alpha1.WorkerDeploymentReference{Name: workerRefName}, + Object: runtime.RawExtension{Raw: raw}, + }, + } +}