diff --git a/api/v1/dpunetwork_types.go b/api/v1/dpunetwork_types.go new file mode 100644 index 000000000..6fbb8b22d --- /dev/null +++ b/api/v1/dpunetwork_types.go @@ -0,0 +1,85 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// DpuNetworkSpec defines the desired state of DpuNetwork. +type DpuNetworkSpec struct { + // NodeSelector specifies which nodes this DpuNetwork should apply to. + // If empty, the DpuNetwork will apply to all nodes. + // +optional + NodeSelector *metav1.LabelSelector `json:"nodeSelector,omitempty"` + + // DpuSelector specifies which DPUs (and their VFs) this DpuNetwork targets. + // + // Note: Today this is treated as an opaque selector definition; the controller + // parses vfId ranges from matchExpressions (if present) but does not yet + // validate against a per-VF inventory. + // +optional + DpuSelector *metav1.LabelSelector `json:"dpuSelector,omitempty"` + + // IsDisruptive indicates whether the network should be treated as disruptive + // by downstream components. + // +optional + IsDisruptive bool `json:"isDisruptive,omitempty"` +} + +// DpuNetworkStatus defines the observed state of DpuNetwork. +type DpuNetworkStatus struct { + // Conditions is the status of the DpuNetwork. + // +optional + Conditions []metav1.Condition `json:"conditions,omitempty"` + + // ResourceName is the Kubernetes extended resource name generated for this network. + // +optional + ResourceName string `json:"resourceName,omitempty"` + + // SelectedVFs is the list of VF IDs parsed from vfId ranges. + // +optional + SelectedVFs []int32 `json:"selectedVFs,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:resource:scope=Cluster,shortName=dpunet +//+kubebuilder:printcolumn:name="Resource",type="string",JSONPath=".status.resourceName" +//+kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type=='Ready')].status" + +// DpuNetwork is the Schema for the dpunetworks API. +type DpuNetwork struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec DpuNetworkSpec `json:"spec,omitempty"` + Status DpuNetworkStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// DpuNetworkList contains a list of DpuNetwork. +type DpuNetworkList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []DpuNetwork `json:"items"` +} + +func init() { + SchemeBuilder.Register(&DpuNetwork{}, &DpuNetworkList{}) +} diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 17645862a..0623e9e78 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -215,6 +215,117 @@ func (in *DataProcessingUnitStatus) DeepCopy() *DataProcessingUnitStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DpuNetwork) DeepCopyInto(out *DpuNetwork) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DpuNetwork. +func (in *DpuNetwork) DeepCopy() *DpuNetwork { + if in == nil { + return nil + } + out := new(DpuNetwork) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DpuNetwork) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DpuNetworkList) DeepCopyInto(out *DpuNetworkList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]DpuNetwork, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DpuNetworkList. +func (in *DpuNetworkList) DeepCopy() *DpuNetworkList { + if in == nil { + return nil + } + out := new(DpuNetworkList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DpuNetworkList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DpuNetworkSpec) DeepCopyInto(out *DpuNetworkSpec) { + *out = *in + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = new(metav1.LabelSelector) + (*in).DeepCopyInto(*out) + } + if in.DpuSelector != nil { + in, out := &in.DpuSelector, &out.DpuSelector + *out = new(metav1.LabelSelector) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DpuNetworkSpec. +func (in *DpuNetworkSpec) DeepCopy() *DpuNetworkSpec { + if in == nil { + return nil + } + out := new(DpuNetworkSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DpuNetworkStatus) DeepCopyInto(out *DpuNetworkStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.SelectedVFs != nil { + in, out := &in.SelectedVFs, &out.SelectedVFs + *out = make([]int32, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DpuNetworkStatus. +func (in *DpuNetworkStatus) DeepCopy() *DpuNetworkStatus { + if in == nil { + return nil + } + out := new(DpuNetworkStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DpuOperatorConfig) DeepCopyInto(out *DpuOperatorConfig) { *out = *in diff --git a/cmd/main.go b/cmd/main.go index 8211d16b6..9261eb5d7 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -133,6 +133,13 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "DataProcessingUnit") os.Exit(1) } + if err := (&controller.DpuNetworkReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "DpuNetwork") + os.Exit(1) + } if os.Getenv("ENABLE_WEBHOOKS") != "false" { if err = (&configv1.DpuOperatorConfig{}).SetupWebhookWithManager(mgr); err != nil { setupLog.Error(err, "unable to create webhook", "webhook", "DpuOperatorConfig") diff --git a/config/crd/bases/config.openshift.io_dpunetworks.yaml b/config/crd/bases/config.openshift.io_dpunetworks.yaml new file mode 100644 index 000000000..ba16867da --- /dev/null +++ b/config/crd/bases/config.openshift.io_dpunetworks.yaml @@ -0,0 +1,231 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + name: dpunetworks.config.openshift.io +spec: + group: config.openshift.io + names: + kind: DpuNetwork + listKind: DpuNetworkList + plural: dpunetworks + shortNames: + - dpunet + singular: dpunetwork + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.resourceName + name: Resource + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + name: v1 + schema: + openAPIV3Schema: + description: DpuNetwork is the Schema for the dpunetworks API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: DpuNetworkSpec defines the desired state of DpuNetwork. + properties: + dpuSelector: + description: |- + DpuSelector specifies which DPUs (and their VFs) this DpuNetwork targets. + + Note: Today this is treated as an opaque selector definition; the controller + parses vfId ranges from matchExpressions (if present) but does not yet + validate against a per-VF inventory. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + isDisruptive: + description: |- + IsDisruptive indicates whether the network should be treated as disruptive + by downstream components. + type: boolean + nodeSelector: + description: |- + NodeSelector specifies which nodes this DpuNetwork should apply to. + If empty, the DpuNetwork will apply to all nodes. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: object + status: + description: DpuNetworkStatus defines the observed state of DpuNetwork. + properties: + conditions: + description: Conditions is the status of the DpuNetwork. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + resourceName: + description: ResourceName is the Kubernetes extended resource name + generated for this network. + type: string + selectedVFs: + description: SelectedVFs is the list of VF IDs parsed from vfId ranges. + items: + format: int32 + type: integer + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 8f8026c39..f039203de 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -6,6 +6,7 @@ resources: - bases/config.openshift.io_servicefunctionchains.yaml - bases/config.openshift.io_dataprocessingunits.yaml - bases/config.openshift.io_dataprocessingunitconfigs.yaml +- bases/config.openshift.io_dpunetworks.yaml #+kubebuilder:scaffold:crdkustomizeresource patches: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 42ef81004..ab4a2d83c 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -8,6 +8,7 @@ rules: - "" resources: - '*' + - configmaps - serviceaccounts verbs: - create @@ -60,6 +61,7 @@ rules: resources: - dataprocessingunitconfigs - dataprocessingunits + - dpunetworks - dpuoperatorconfigs - servicefunctionchains - servicefunctionchains/finalizers @@ -76,6 +78,7 @@ rules: resources: - dataprocessingunitconfigs/finalizers - dataprocessingunits/finalizers + - dpunetworks/finalizers - dpuoperatorconfigs/finalizers verbs: - update @@ -84,6 +87,7 @@ rules: resources: - dataprocessingunitconfigs/status - dataprocessingunits/status + - dpunetworks/status - dpuoperatorconfigs/status - servicefunctionchains/status verbs: diff --git a/examples/dpunetwork-net1.yaml b/examples/dpunetwork-net1.yaml new file mode 100644 index 000000000..f49a483f4 --- /dev/null +++ b/examples/dpunetwork-net1.yaml @@ -0,0 +1,16 @@ +apiVersion: config.openshift.io/v1 +kind: DpuNetwork +metadata: + name: net1 +spec: + dpuSelector: + matchExpressions: + - key: vfId + operator: In + values: + - "0-3" + nodeSelector: + matchLabels: + node-role.kubernetes.io/worker: "" + isDisruptive: true + diff --git a/examples/dpunetwork-net2.yaml b/examples/dpunetwork-net2.yaml new file mode 100644 index 000000000..9935c79ef --- /dev/null +++ b/examples/dpunetwork-net2.yaml @@ -0,0 +1,15 @@ +apiVersion: config.openshift.io/v1 +kind: DpuNetwork +metadata: + name: net2 +spec: + dpuSelector: + matchExpressions: + - key: vfId + operator: In + values: + - "4-6" + nodeSelector: + matchLabels: + node-role.kubernetes.io/worker: "" + isDisruptive: true diff --git a/internal/controller/dpunetwork_controller.go b/internal/controller/dpunetwork_controller.go new file mode 100644 index 000000000..3ead0ac5c --- /dev/null +++ b/internal/controller/dpunetwork_controller.go @@ -0,0 +1,358 @@ +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "sort" + "strconv" + "strings" + + netattdefv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + configv1 "github.com/openshift/dpu-operator/api/v1" + "github.com/openshift/dpu-operator/pkgs/vars" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +const ( + dpuDevicePluginConfigMapName = "dpu-device-plugin-config" + defaultDpuNetworkNADNamespace = "default" + dpuNetworkFinalizer = "config.openshift.io/dpunetwork-cleanup" +) + +type dpuDevicePluginConfig struct { + Resources []dpuDevicePluginResource `json:"resources"` +} + +type dpuDevicePluginResource struct { + ResourceName string `json:"resourceName"` + DpuNetworkName string `json:"dpuNetworkName"` + NodeSelector *metav1.LabelSelector `json:"nodeSelector,omitempty"` + VfRanges []string `json:"vfRanges,omitempty"` + IsDisruptive bool `json:"isDisruptive,omitempty"` +} + +// DpuNetworkReconciler reconciles a DpuNetwork object +type DpuNetworkReconciler struct { + client.Client + Scheme *runtime.Scheme +} + +//+kubebuilder:rbac:groups=config.openshift.io,resources=dpunetworks,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=config.openshift.io,resources=dpunetworks/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=config.openshift.io,resources=dpunetworks/finalizers,verbs=update +//+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch;create;update;patch;delete + +func (r *DpuNetworkReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + net := &configv1.DpuNetwork{} + if err := r.Get(ctx, req.NamespacedName, net); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + // Handle deletion: remove this network's entry from the shared ConfigMap, + // then remove the finalizer so Kubernetes can delete the CR. + if !net.DeletionTimestamp.IsZero() { + if controllerutil.ContainsFinalizer(net, dpuNetworkFinalizer) { + if err := r.removeFromDevicePluginConfigMap(ctx, net.Name); err != nil { + logger.Error(err, "Failed to clean up ConfigMap entry on deletion") + return ctrl.Result{}, err + } + controllerutil.RemoveFinalizer(net, dpuNetworkFinalizer) + if err := r.Update(ctx, net); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{}, nil + } + + // Ensure finalizer is present for cleanup on deletion. + if !controllerutil.ContainsFinalizer(net, dpuNetworkFinalizer) { + controllerutil.AddFinalizer(net, dpuNetworkFinalizer) + if err := r.Update(ctx, net); err != nil { + return ctrl.Result{}, err + } + } + + resourceName := fmt.Sprintf("openshift.io/dpunetwork-%s", net.Name) + selectedVFs, vfRanges := parseVfRangesFromSelector(net.Spec.DpuSelector) + + if meta.FindStatusCondition(net.Status.Conditions, "Ready") == nil { + meta.SetStatusCondition(&net.Status.Conditions, metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionFalse, + Reason: "Reconciling", + Message: "Reconciling DpuNetwork", + }) + } + + // Ensure ConfigMap used by per-node daemons/device plugins. + if err := r.ensureDevicePluginConfigMap(ctx, net, resourceName, vfRanges); err != nil { + meta.SetStatusCondition(&net.Status.Conditions, metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionFalse, + Reason: "ConfigMapError", + Message: err.Error(), + }) + _ = r.Status().Update(ctx, net) + return ctrl.Result{}, err + } + + // Ensure NetworkAttachmentDefinition for this network. + if err := r.ensureNAD(ctx, net, resourceName); err != nil { + meta.SetStatusCondition(&net.Status.Conditions, metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionFalse, + Reason: "NADError", + Message: err.Error(), + }) + _ = r.Status().Update(ctx, net) + return ctrl.Result{}, err + } + + net.Status.ResourceName = resourceName + net.Status.SelectedVFs = selectedVFs + meta.SetStatusCondition(&net.Status.Conditions, metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionTrue, + Reason: "ComponentsReady", + Message: "ConfigMap and NAD ensured", + }) + + if err := r.Status().Update(ctx, net); err != nil { + logger.Error(err, "Failed to update DpuNetwork status") + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil +} + +func (r *DpuNetworkReconciler) ensureDevicePluginConfigMap(ctx context.Context, net *configv1.DpuNetwork, resourceName string, vfRanges []string) error { + cm := &corev1.ConfigMap{} + key := types.NamespacedName{Name: dpuDevicePluginConfigMapName, Namespace: vars.Namespace} + err := r.Get(ctx, key, cm) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + if apierrors.IsNotFound(err) { + cm = &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: key.Name, Namespace: key.Namespace}} + } + + // Build config.json payload. For now, we append one resource entry per DpuNetwork. + cfg := dpuDevicePluginConfig{} + if cm.Data != nil { + if raw := cm.Data["config.json"]; raw != "" { + _ = json.Unmarshal([]byte(raw), &cfg) + } + } + + // Upsert entry for this DpuNetwork. + newEntry := dpuDevicePluginResource{ + ResourceName: resourceName, + DpuNetworkName: net.Name, + NodeSelector: net.Spec.NodeSelector, + VfRanges: vfRanges, + IsDisruptive: net.Spec.IsDisruptive, + } + var out []dpuDevicePluginResource + for _, e := range cfg.Resources { + if e.DpuNetworkName == net.Name { + continue + } + out = append(out, e) + } + out = append(out, newEntry) + // stable ordering + sort.Slice(out, func(i, j int) bool { return out[i].DpuNetworkName < out[j].DpuNetworkName }) + cfg.Resources = out + + payload, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return err + } + if cm.Data == nil { + cm.Data = map[string]string{} + } + cm.Data["config.json"] = string(payload) + + // Owner reference: the ConfigMap is shared across all DpuNetwork CRs, so we do NOT set controller reference. + // (Multiple controller references would be invalid.) + + if cm.CreationTimestamp.IsZero() { + return r.Create(ctx, cm) + } + return r.Update(ctx, cm) +} + +func (r *DpuNetworkReconciler) removeFromDevicePluginConfigMap(ctx context.Context, networkName string) error { + cm := &corev1.ConfigMap{} + key := types.NamespacedName{Name: dpuDevicePluginConfigMapName, Namespace: vars.Namespace} + if err := r.Get(ctx, key, cm); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return err + } + + cfg := dpuDevicePluginConfig{} + if cm.Data != nil { + if raw := cm.Data["config.json"]; raw != "" { + _ = json.Unmarshal([]byte(raw), &cfg) + } + } + + var out []dpuDevicePluginResource + for _, e := range cfg.Resources { + if e.DpuNetworkName == networkName { + continue + } + out = append(out, e) + } + + if len(out) == len(cfg.Resources) { + return nil + } + + cfg.Resources = out + if len(out) == 0 { + // All networks removed; delete the ConfigMap so the daemon + // transitions back to the default device plugin. + return r.Delete(ctx, cm) + } + + payload, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return err + } + cm.Data["config.json"] = string(payload) + return r.Update(ctx, cm) +} + +func (r *DpuNetworkReconciler) ensureNAD(ctx context.Context, net *configv1.DpuNetwork, resourceName string) error { + bridgeID := stableBridgeID(net.Name) + nadName := fmt.Sprintf("%s-nad", net.Name) + key := types.NamespacedName{Name: nadName, Namespace: defaultDpuNetworkNADNamespace} + + nad := &netattdefv1.NetworkAttachmentDefinition{} + err := r.Get(ctx, key, nad) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + if apierrors.IsNotFound(err) { + nad = &netattdefv1.NetworkAttachmentDefinition{ObjectMeta: metav1.ObjectMeta{Name: key.Name, Namespace: key.Namespace}} + } + + // Make this NAD owned by the DpuNetwork for cleanup. + if err := controllerutil.SetControllerReference(net, nad, r.Scheme); err != nil { + return err + } + + if nad.Annotations == nil { + nad.Annotations = map[string]string{} + } + nad.Annotations["dpu.config.openshift.io/dpu-network"] = net.Name + nad.Annotations["k8s.v1.cni.cncf.io/resourceName"] = resourceName + + // Keep this JSON minimal for now; the dpu-cni and NRI integration can evolve. + nad.Spec.Config = fmt.Sprintf(`{"type":"dpu-cni","cniVersion":"0.4.0","name":"dpu-cni","BridgeID":"%s","IsDisruptive":"%t"}`, bridgeID, net.Spec.IsDisruptive) + + if nad.CreationTimestamp.IsZero() { + return r.Create(ctx, nad) + } + return r.Update(ctx, nad) +} + +func stableBridgeID(name string) string { + sum := sha256.Sum256([]byte(name)) + // keep short but collision-resistant enough for demo purposes + return hex.EncodeToString(sum[:])[:8] +} + +func parseVfRangesFromSelector(sel *metav1.LabelSelector) ([]int32, []string) { + if sel == nil { + return nil, nil + } + + var ranges []string + for _, expr := range sel.MatchExpressions { + if expr.Key != "vfId" { + continue + } + if strings.ToLower(string(expr.Operator)) != "in" { + // only handle In for now + continue + } + for _, v := range expr.Values { + v = strings.TrimSpace(v) + if v == "" { + continue + } + ranges = append(ranges, v) + } + } + + vfSet := map[int32]struct{}{} + for _, r := range ranges { + for _, id := range expandRange(r) { + vfSet[id] = struct{}{} + } + } + + var vfs []int32 + for id := range vfSet { + vfs = append(vfs, id) + } + sort.Slice(vfs, func(i, j int) bool { return vfs[i] < vfs[j] }) + return vfs, ranges +} + +func expandRange(s string) []int32 { + // supports "N" or "A-B" + if !strings.Contains(s, "-") { + v, err := strconv.Atoi(s) + if err != nil { + return nil + } + return []int32{int32(v)} + } + parts := strings.SplitN(s, "-", 2) + if len(parts) != 2 { + return nil + } + start, err1 := strconv.Atoi(strings.TrimSpace(parts[0])) + end, err2 := strconv.Atoi(strings.TrimSpace(parts[1])) + if err1 != nil || err2 != nil { + return nil + } + if end < start { + start, end = end, start + } + out := make([]int32, 0, end-start+1) + for i := start; i <= end; i++ { + out = append(out, int32(i)) + } + return out +} + +// SetupWithManager sets up the controller with the Manager. +func (r *DpuNetworkReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&configv1.DpuNetwork{}). + Complete(r) +} diff --git a/internal/controller/dpunetwork_controller_test.go b/internal/controller/dpunetwork_controller_test.go new file mode 100644 index 000000000..306da2cfc --- /dev/null +++ b/internal/controller/dpunetwork_controller_test.go @@ -0,0 +1,178 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "encoding/json" + "os" + "sync" + + netattdefv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + configv1 "github.com/openshift/dpu-operator/api/v1" + "github.com/openshift/dpu-operator/internal/scheme" + "github.com/openshift/dpu-operator/internal/testutils" + "github.com/openshift/dpu-operator/pkgs/vars" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +type testDevicePluginConfig struct { + Resources []testDevicePluginResource `json:"resources"` +} + +type testDevicePluginResource struct { + ResourceName string `json:"resourceName"` + DpuNetworkName string `json:"dpuNetworkName"` +} + +func startDpuNetworkControllerManager(ctx context.Context, client *rest.Config, wg *sync.WaitGroup) ctrl.Manager { + mgr, err := ctrl.NewManager(client, ctrl.Options{ + Scheme: scheme.Scheme, + Metrics: server.Options{ + BindAddress: ":18002", + }, + WebhookServer: webhook.NewServer(webhook.Options{Port: 9444}), + LeaderElectionID: "dpunetwork-controller-test.openshift.io", + }) + Expect(err).NotTo(HaveOccurred()) + + reconciler := &DpuNetworkReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme()} + err = reconciler.SetupWithManager(mgr) + Expect(err).NotTo(HaveOccurred()) + + wg.Add(1) + go func() { + defer GinkgoRecover() + err := mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred()) + wg.Done() + }() + + <-mgr.Elected() + return mgr +} + +var _ = Describe("DpuNetwork Controller", Ordered, func() { + var ( + cancel context.CancelFunc + ctx context.Context + wg sync.WaitGroup + restConfig *rest.Config + mgr ctrl.Manager + testCluster testutils.KindCluster + ) + + BeforeAll(func() { + opts := zap.Options{Development: true} + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + + testCluster = testutils.KindCluster{Name: "dpu-operator-dpunetwork-test-cluster"} + restConfig = testCluster.EnsureExists() + ctx, cancel = context.WithCancel(context.Background()) + mgr = startDpuNetworkControllerManager(ctx, restConfig, &wg) + + // DpuNetwork controller writes into vars.Namespace, so make sure it exists. + ns := testutils.DpuOperatorNamespace() + testutils.CreateNamespace(mgr.GetClient(), ns) + }) + + AfterAll(func() { + cancel() + wg.Wait() + if os.Getenv("FAST_TEST") == "false" { + testCluster.EnsureDeleted() + } + }) + + It("should create/update ConfigMap and NAD and set status", func() { + net := &configv1.DpuNetwork{ + ObjectMeta: metav1.ObjectMeta{Name: "net1"}, + Spec: configv1.DpuNetworkSpec{ + DpuSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{{ + Key: "vfId", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"1-3", "5"}, + }}, + }, + }, + } + + Expect(mgr.GetClient().Create(context.Background(), net)).To(Succeed()) + + expectedResource := "openshift.io/dpunetwork-net1" + + By("Ensuring ConfigMap is written") + Eventually(func() (string, error) { + cm := &corev1.ConfigMap{} + err := mgr.GetClient().Get(context.Background(), types.NamespacedName{Name: "dpu-device-plugin-config", Namespace: vars.Namespace}, cm) + if err != nil { + return "", err + } + return cm.Data["config.json"], nil + }, testutils.TestAPITimeout*5, testutils.TestRetryInterval).ShouldNot(BeEmpty()) + + cm := &corev1.ConfigMap{} + Expect(mgr.GetClient().Get(context.Background(), types.NamespacedName{Name: "dpu-device-plugin-config", Namespace: vars.Namespace}, cm)).To(Succeed()) + cfg := testDevicePluginConfig{} + Expect(json.Unmarshal([]byte(cm.Data["config.json"]), &cfg)).To(Succeed()) + + found := false + for _, r := range cfg.Resources { + if r.DpuNetworkName == "net1" { + Expect(r.ResourceName).To(Equal(expectedResource)) + found = true + break + } + } + Expect(found).To(BeTrue()) + + By("Ensuring NAD is created") + Eventually(func() (map[string]string, error) { + nad := &netattdefv1.NetworkAttachmentDefinition{} + err := mgr.GetClient().Get(context.Background(), types.NamespacedName{Name: "net1-nad", Namespace: "default"}, nad) + if err != nil { + return nil, err + } + return nad.Annotations, nil + }, testutils.TestAPITimeout*5, testutils.TestRetryInterval).Should(HaveKeyWithValue("k8s.v1.cni.cncf.io/resourceName", expectedResource)) + + By("Ensuring status is set") + Eventually(func() string { + latest := &configv1.DpuNetwork{} + if err := mgr.GetClient().Get(context.Background(), types.NamespacedName{Name: "net1"}, latest); err != nil { + return "" + } + if !meta.IsStatusConditionTrue(latest.Status.Conditions, "Ready") { + return "" + } + return latest.Status.ResourceName + }, testutils.TestAPITimeout*5, testutils.TestRetryInterval).Should(Equal(expectedResource)) + }) +}) diff --git a/vendor/github.com/openshift/dpu-operator/api/v1/dpunetwork_types.go b/vendor/github.com/openshift/dpu-operator/api/v1/dpunetwork_types.go new file mode 100644 index 000000000..6fbb8b22d --- /dev/null +++ b/vendor/github.com/openshift/dpu-operator/api/v1/dpunetwork_types.go @@ -0,0 +1,85 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// DpuNetworkSpec defines the desired state of DpuNetwork. +type DpuNetworkSpec struct { + // NodeSelector specifies which nodes this DpuNetwork should apply to. + // If empty, the DpuNetwork will apply to all nodes. + // +optional + NodeSelector *metav1.LabelSelector `json:"nodeSelector,omitempty"` + + // DpuSelector specifies which DPUs (and their VFs) this DpuNetwork targets. + // + // Note: Today this is treated as an opaque selector definition; the controller + // parses vfId ranges from matchExpressions (if present) but does not yet + // validate against a per-VF inventory. + // +optional + DpuSelector *metav1.LabelSelector `json:"dpuSelector,omitempty"` + + // IsDisruptive indicates whether the network should be treated as disruptive + // by downstream components. + // +optional + IsDisruptive bool `json:"isDisruptive,omitempty"` +} + +// DpuNetworkStatus defines the observed state of DpuNetwork. +type DpuNetworkStatus struct { + // Conditions is the status of the DpuNetwork. + // +optional + Conditions []metav1.Condition `json:"conditions,omitempty"` + + // ResourceName is the Kubernetes extended resource name generated for this network. + // +optional + ResourceName string `json:"resourceName,omitempty"` + + // SelectedVFs is the list of VF IDs parsed from vfId ranges. + // +optional + SelectedVFs []int32 `json:"selectedVFs,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:resource:scope=Cluster,shortName=dpunet +//+kubebuilder:printcolumn:name="Resource",type="string",JSONPath=".status.resourceName" +//+kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type=='Ready')].status" + +// DpuNetwork is the Schema for the dpunetworks API. +type DpuNetwork struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec DpuNetworkSpec `json:"spec,omitempty"` + Status DpuNetworkStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// DpuNetworkList contains a list of DpuNetwork. +type DpuNetworkList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []DpuNetwork `json:"items"` +} + +func init() { + SchemeBuilder.Register(&DpuNetwork{}, &DpuNetworkList{}) +} diff --git a/vendor/github.com/openshift/dpu-operator/api/v1/zz_generated.deepcopy.go b/vendor/github.com/openshift/dpu-operator/api/v1/zz_generated.deepcopy.go index 17645862a..0623e9e78 100644 --- a/vendor/github.com/openshift/dpu-operator/api/v1/zz_generated.deepcopy.go +++ b/vendor/github.com/openshift/dpu-operator/api/v1/zz_generated.deepcopy.go @@ -215,6 +215,117 @@ func (in *DataProcessingUnitStatus) DeepCopy() *DataProcessingUnitStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DpuNetwork) DeepCopyInto(out *DpuNetwork) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DpuNetwork. +func (in *DpuNetwork) DeepCopy() *DpuNetwork { + if in == nil { + return nil + } + out := new(DpuNetwork) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DpuNetwork) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DpuNetworkList) DeepCopyInto(out *DpuNetworkList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]DpuNetwork, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DpuNetworkList. +func (in *DpuNetworkList) DeepCopy() *DpuNetworkList { + if in == nil { + return nil + } + out := new(DpuNetworkList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DpuNetworkList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DpuNetworkSpec) DeepCopyInto(out *DpuNetworkSpec) { + *out = *in + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = new(metav1.LabelSelector) + (*in).DeepCopyInto(*out) + } + if in.DpuSelector != nil { + in, out := &in.DpuSelector, &out.DpuSelector + *out = new(metav1.LabelSelector) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DpuNetworkSpec. +func (in *DpuNetworkSpec) DeepCopy() *DpuNetworkSpec { + if in == nil { + return nil + } + out := new(DpuNetworkSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DpuNetworkStatus) DeepCopyInto(out *DpuNetworkStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.SelectedVFs != nil { + in, out := &in.SelectedVFs, &out.SelectedVFs + *out = make([]int32, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DpuNetworkStatus. +func (in *DpuNetworkStatus) DeepCopy() *DpuNetworkStatus { + if in == nil { + return nil + } + out := new(DpuNetworkStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DpuOperatorConfig) DeepCopyInto(out *DpuOperatorConfig) { *out = *in