added subnet uuid and lb byoip support

cloud-controller-manager/do/lb_annotations.go

@@ -190,4 +190,12 @@ const (
 	// are permitted. If load balancer type is REGIONAL then it will default to DUALSTACK. If load balancer type
 	// is REGIONAL_NETWORK then it will default to IPV4.
 	annDONetworkStack = annDOLoadBalancerBase + "network-stack"
+
+	// annDOLoadBalancerIP is the annotation used to assign a BYOIP address when creating the load balancer.
+	// The IP must be an unassigned BYOIP address on the account in the same region. Create-only; ignored on update.
+	annDOLoadBalancerIP = annDOLoadBalancerBase + "ip"
+
+	// annDOSubnetUUID is the annotation used to place the load balancer in a specific VPC subnet.
+	// Must be a valid subnet UUID in the cluster VPC. Specifying a subnet UUID is in private preview.
+	annDOSubnetUUID = annDOLoadBalancerBase + "subnet-uuid"
 )

cloud-controller-manager/do/loadbalancers.go

@@ -232,7 +232,10 @@
 		}
 
 	case errLBNotFound:
-		// LB missing
+		// LB missing. IP is create-only (BYOIP); never send it on update.
+		if ip := getLoadBalancerIP(service); ip != "" {
+			lbRequest.IP = ip
+		}
 		lb, _, err = l.resources.gclient.LoadBalancers.Create(ctx, lbRequest)
 		if err != nil {
 			logLBInfo("CREATE", lbRequest, 2)
@@ -990,6 +993,7 @@
 
 	req.Region = l.region
 	req.VPCUUID = l.resources.clusterVPCID
+	req.VPCSubnetUUID = getSubnetUUID(service)
 	return req, nil
 }
 
@@ -1324,6 +1328,14 @@
 	return strings.ToLower(service.Annotations[annDOHostname])
 }
 
+func getLoadBalancerIP(service *v1.Service) string {
+	return strings.TrimSpace(service.Annotations[annDOLoadBalancerIP])
+}
+
+func getSubnetUUID(service *v1.Service) string {
+	return strings.TrimSpace(service.Annotations[annDOSubnetUUID])
+}
+
 // healthCheckPort returns the health check port specified, defaulting
 // to the first port in the service otherwise.
 func healthCheckPort(service *v1.Service) (int, error) {

cloud-controller-manager/do/loadbalancers_test.go

@@ -4715,6 +4715,186 @@ func Test_buildLoadBalancerRequestWithClusterID(t *testing.T) {
 	}
 }
 
+func Test_buildLoadBalancerRequest_SubnetUUID(t *testing.T) {
+	const subnetUUID = "3d6f6fdc-8b7e-49fd-a3c2-0d73ec4e40b4"
+
+	service := &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test",
+			Annotations: map[string]string{
+				annDOSubnetUUID: subnetUUID,
+			},
+		},
+		Spec: v1.ServiceSpec{
+			Ports: []v1.ServicePort{
+				{
+					Name:     "test",
+					Protocol: "TCP",
+					Port:     int32(80),
+					NodePort: int32(30000),
+				},
+			},
+		},
+	}
+	nodes := []*v1.Node{
+		newNodeWithIPs("node-1", "10.0.0.1", "2001:db8::1", true),
+	}
+	fakeClient := newFakeDropletClient(
+		&fakeDropletService{
+			listFunc: func(context.Context, *godo.ListOptions) ([]godo.Droplet, *godo.Response, error) {
+				return []godo.Droplet{{ID: 100, Name: "node-1"}}, newFakeOKResponse(), nil
+			},
+		},
+	)
+	fakeResources := newResources(clusterID, "vpc_uuid", publicAccessFirewall{}, fakeClient)
+	fakeResources.clusterVPCID = "vpc_uuid"
+
+	lb := &loadBalancers{
+		resources:     fakeResources,
+		region:        "nyc3",
+		clusterID:     clusterID,
+		defaultLBType: godo.LoadBalancerTypeRegionalNetwork,
+	}
+
+	lbr, err := lb.buildLoadBalancerRequest(context.Background(), service, nodes)
+	if err != nil {
+		t.Fatalf("got error: %s", err)
+	}
+	if want, got := subnetUUID, lbr.VPCSubnetUUID; want != got {
+		t.Errorf("incorrect subnet uuid\nwant: %#v\n got: %#v", want, got)
+	}
+	if lbr.IP != "" {
+		t.Errorf("expected IP to be empty on build (create-only), got %q", lbr.IP)
+	}
+}
+
+func Test_EnsureLoadBalancer_BYOIPCreateOnly(t *testing.T) {
+	const byoip = "10.41.29.5"
+	const subnetUUID = "3d6f6fdc-8b7e-49fd-a3c2-0d73ec4e40b4"
+
+	nodes := []*v1.Node{
+		newNodeWithIPs("node-1", "10.0.0.1", "2001:db8::1", true),
+	}
+	servicePorts := []v1.ServicePort{
+		{
+			Name:     "test",
+			Protocol: "TCP",
+			Port:     int32(80),
+			NodePort: int32(30000),
+		},
+	}
+
+	newLB := func(t *testing.T, fakeLB *fakeLBService, svc *v1.Service) *loadBalancers {
+		t.Helper()
+		fakeDroplet := &fakeDropletService{
+			listFunc: func(context.Context, *godo.ListOptions) ([]godo.Droplet, *godo.Response, error) {
+				return []godo.Droplet{{ID: 100, Name: "node-1"}}, newFakeOKResponse(), nil
+			},
+		}
+		certStore := make(map[string]*godo.Certificate)
+		fakeCert := newKVCertService(certStore, true)
+		client := newFakeClient(fakeDroplet, fakeLB, &fakeCert)
+		resources := newResources("", "", publicAccessFirewall{}, client)
+		resources.kclient = fake.NewSimpleClientset()
+		if _, err := resources.kclient.CoreV1().Services(svc.Namespace).Create(context.Background(), svc, metav1.CreateOptions{}); err != nil {
+			t.Fatalf("failed to add service to fake client: %s", err)
+		}
+		return &loadBalancers{
+			resources:     resources,
+			region:        "nyc3",
+			defaultLBType: godo.LoadBalancerTypeRegionalNetwork,
+		}
+	}
+
+	t.Run("create sends IP and subnet UUID", func(t *testing.T) {
+		var gotReq *godo.LoadBalancerRequest
+		fakeLB := &fakeLBService{
+			listFn: func(context.Context, *godo.ListOptions) ([]godo.LoadBalancer, *godo.Response, error) {
+				return []godo.LoadBalancer{}, newFakeOKResponse(), nil
+			},
+			createFn: func(_ context.Context, lbr *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) {
+				gotReq = lbr
+				lb := createLB()
+				lb.IP = byoip
+				return lb, newFakeOKResponse(), nil
+			},
+			updateFn: func(context.Context, string, *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) {
+				t.Fatal("update should not have been invoked")
+				return nil, nil, nil
+			},
+		}
+		svc := &v1.Service{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test",
+				UID:  "foobar123",
+				Annotations: map[string]string{
+					annDOLoadBalancerIP: byoip,
+					annDOSubnetUUID:     subnetUUID,
+				},
+			},
+			Spec: v1.ServiceSpec{Ports: servicePorts},
+		}
+		lb := newLB(t, fakeLB, svc)
+
+		if _, err := lb.EnsureLoadBalancer(context.Background(), "cluster", svc, nodes); err != nil {
+			t.Fatalf("EnsureLoadBalancer: %v", err)
+		}
+		if gotReq == nil {
+			t.Fatal("expected create to be called")
+		}
+		if want, got := byoip, gotReq.IP; want != got {
+			t.Errorf("create IP: want %q, got %q", want, got)
+		}
+		if want, got := subnetUUID, gotReq.VPCSubnetUUID; want != got {
+			t.Errorf("create subnet UUID: want %q, got %q", want, got)
+		}
+	})
+
+	t.Run("update does not send IP but does send subnet UUID", func(t *testing.T) {
+		var gotReq *godo.LoadBalancerRequest
+		existing := createLB()
+		fakeLB := &fakeLBService{
+			getFn: func(context.Context, string) (*godo.LoadBalancer, *godo.Response, error) {
+				return existing, newFakeOKResponse(), nil
+			},
+			createFn: func(context.Context, *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) {
+				t.Fatal("create should not have been invoked")
+				return nil, nil, nil
+			},
+			updateFn: func(_ context.Context, _ string, lbr *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) {
+				gotReq = lbr
+				return existing, newFakeOKResponse(), nil
+			},
+		}
+		svc := &v1.Service{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test",
+				UID:  "foobar123",
+				Annotations: map[string]string{
+					annDOLoadBalancerID: existing.ID,
+					annDOLoadBalancerIP: byoip,
+					annDOSubnetUUID:     subnetUUID,
+				},
+			},
+			Spec: v1.ServiceSpec{Ports: servicePorts},
+		}
+		lb := newLB(t, fakeLB, svc)
+
+		if _, err := lb.EnsureLoadBalancer(context.Background(), "cluster", svc, nodes); err != nil {
+			t.Fatalf("EnsureLoadBalancer: %v", err)
+		}
+		if gotReq == nil {
+			t.Fatal("expected update to be called")
+		}
+		if gotReq.IP != "" {
+			t.Errorf("update must not send IP, got %q", gotReq.IP)
+		}
+		if want, got := subnetUUID, gotReq.VPCSubnetUUID; want != got {
+			t.Errorf("update subnet UUID: want %q, got %q", want, got)
+		}
+	})
+}
+
 func Test_nodeToDropletIDs(t *testing.T) {
 	testcases := []struct {
 		name         string

docs/controllers/services/annotations.md

@@ -236,6 +236,22 @@ Specifies the type of the load balancer. Options are "REGIONAL" or "REGIONAL_NET
 Specifies the network availability of the load balancer. Options are "EXTERNAL" and "INTERNAL". Defaults to "EXTERNAL"
 External load balancer will be accessible via the public internet. Internal load balancer will only be accessible via a members in the vpc.
 
+## service.beta.kubernetes.io/do-loadbalancer-ip
+
+Specifies a BYOIP address to assign when creating the Load Balancer. The address must be an unassigned BYOIP address on the account in the same region as the cluster. When omitted, DigitalOcean provisions a system-allocated floating IP.
+
+**Note**
+
+This annotation is create-only. Changing or removing it after the Load Balancer exists has no effect; the assigned IP cannot be changed via update.
+
+## service.beta.kubernetes.io/do-loadbalancer-subnet-uuid
+
+Specifies the UUID of the VPC subnet to place the Load Balancer in. Must be a valid subnet in the cluster VPC (`DO_CLUSTER_VPC_ID`). When omitted, DigitalOcean chooses a subnet automatically.
+
+**Note**
+
+Specifying a subnet UUID is in private preview. Contact DigitalOcean support to review its public availability.
+
 ## service.beta.kubernetes.io/do-loadbalancer-network-stack
 
 Specifies what network addressing will be supported by the load balancer. Options are "IPV4" and "DUALSTACK". "DUALSTACK" is the option to support both IPv4 and IPv6 networking on the load balancer. 

docs/controllers/services/examples/http-with-byoip-and-subnet.yml

@@ -0,0 +1,40 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: http-with-byoip
+  annotations:
+    service.beta.kubernetes.io/do-loadbalancer-protocol: "http"
+    # Optional: assign an unassigned BYOIP address on create (create-only).
+    service.beta.kubernetes.io/do-loadbalancer-ip: "203.0.113.10"
+    # Optional: place the LB in a specific VPC subnet (private preview).
+    service.beta.kubernetes.io/do-loadbalancer-subnet-uuid: "3d6f6fdc-8b7e-49fd-a3c2-0d73ec4e40b4"
+spec:
+  type: LoadBalancer
+  selector:
+    app: nginx-example
+  ports:
+    - name: http
+      protocol: TCP
+      port: 80
+      targetPort: 80
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-example
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: nginx-example
+  template:
+    metadata:
+      labels:
+        app: nginx-example
+    spec:
+      containers:
+        - name: nginx
+          image: nginx
+          ports:
+            - containerPort: 80
+              protocol: TCP