diff --git a/api/v1/coroot_types.go b/api/v1/coroot_types.go index 5df5ddf..0538055 100644 --- a/api/v1/coroot_types.go +++ b/api/v1/coroot_types.go @@ -4,6 +4,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -149,6 +150,34 @@ type ClickhouseSpec struct { LogLevel string `json:"logLevel,omitempty"` Keeper ClickhouseKeeperSpec `json:"keeper,omitempty"` + + // S3 storage configuration for ClickHouse (optional). + S3 *ClickhouseS3Spec `json:"s3,omitempty"` +} + +type ClickhouseS3Spec struct { + // S3 endpoint URL. E.g., https://s3.amazonaws.com/my-bucket/clickhouse/ + Endpoint string `json:"endpoint"` + // S3 region (optional). + Region string `json:"region,omitempty"` + // S3 credentials (optional — omit for IAM/IRSA/workload identity). + Credentials *S3Credentials `json:"credentials,omitempty"` + // Local cache size for S3 reads (default: 10Gi). Must be less than clickhouse storage size. + CacheSize resource.Quantity `json:"cacheSize,omitempty"` + // Storage mode: "tiered" keeps recent data on local disk and moves older data to S3; + // "s3only" stores all data on S3 using local disk only for cache. + // +kubebuilder:validation:Enum="tiered";"s3only" + // +kubebuilder:default="tiered" + Mode string `json:"mode,omitempty"` + // Fraction of local disk free space that triggers moving data to S3 (default: 0.1). Only used in tiered mode. + MoveFactor string `json:"moveFactor,omitempty"` +} + +type S3Credentials struct { + // Secret reference for the access key ID. + AccessKeyID *corev1.SecretKeySelector `json:"accessKeyId,omitempty"` + // Secret reference for the secret access key. + SecretAccessKey *corev1.SecretKeySelector `json:"secretAccessKey,omitempty"` } type ClickhouseKeeperSpec struct { diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 66e7b47..45ae62b 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -533,6 +533,27 @@ func (in *ClickhouseKeeperSpec) DeepCopy() *ClickhouseKeeperSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClickhouseS3Spec) DeepCopyInto(out *ClickhouseS3Spec) { + *out = *in + if in.Credentials != nil { + in, out := &in.Credentials, &out.Credentials + *out = new(S3Credentials) + (*in).DeepCopyInto(*out) + } + out.CacheSize = in.CacheSize.DeepCopy() +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClickhouseS3Spec. +func (in *ClickhouseS3Spec) DeepCopy() *ClickhouseS3Spec { + if in == nil { + return nil + } + out := new(ClickhouseS3Spec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClickhouseSpec) DeepCopyInto(out *ClickhouseSpec) { *out = *in @@ -566,6 +587,11 @@ func (in *ClickhouseSpec) DeepCopyInto(out *ClickhouseSpec) { } in.Image.DeepCopyInto(&out.Image) in.Keeper.DeepCopyInto(&out.Keeper) + if in.S3 != nil { + in, out := &in.S3, &out.S3 + *out = new(ClickhouseS3Spec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClickhouseSpec. @@ -1667,6 +1693,31 @@ func (in *RemoteCorootSpec) DeepCopy() *RemoteCorootSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *S3Credentials) DeepCopyInto(out *S3Credentials) { + *out = *in + if in.AccessKeyID != nil { + in, out := &in.AccessKeyID, &out.AccessKeyID + *out = new(corev1.SecretKeySelector) + (*in).DeepCopyInto(*out) + } + if in.SecretAccessKey != nil { + in, out := &in.SecretAccessKey, &out.SecretAccessKey + *out = new(corev1.SecretKeySelector) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new S3Credentials. +func (in *S3Credentials) DeepCopy() *S3Credentials { + if in == nil { + return nil + } + out := new(S3Credentials) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SLOAvailabilityOverride) DeepCopyInto(out *SLOAvailabilityOverride) { *out = *in diff --git a/config/crd/coroot.com_coroots.yaml b/config/crd/coroot.com_coroots.yaml index 7cbe355..9f2faef 100644 --- a/config/crd/coroot.com_coroots.yaml +++ b/config/crd/coroot.com_coroots.yaml @@ -3328,6 +3328,92 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + s3: + description: S3 storage configuration for ClickHouse (optional). + properties: + cacheSize: + anyOf: + - type: integer + - type: string + description: 'Local cache size for S3 reads (default: 10Gi). + Must be less than clickhouse storage size.' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + credentials: + description: S3 credentials (optional — omit for IAM/IRSA/workload + identity). + properties: + accessKeyId: + description: Secret reference for the access key ID. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secretAccessKey: + description: Secret reference for the secret access key. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + endpoint: + description: S3 endpoint URL. E.g., https://s3.amazonaws.com/my-bucket/clickhouse/ + type: string + mode: + default: tiered + description: |- + Storage mode: "tiered" keeps recent data on local disk and moves older data to S3; + "s3only" stores all data on S3 using local disk only for cache. + enum: + - tiered + - s3only + type: string + moveFactor: + description: 'Fraction of local disk free space that triggers + moving data to S3 (default: 0.1). Only used in tiered mode.' + type: string + region: + description: S3 region (optional). + type: string + required: + - endpoint + type: object shards: type: integer storage: diff --git a/config/crd/coroot.com_coroots_legacy.yaml b/config/crd/coroot.com_coroots_legacy.yaml index d1f1196..f672f5c 100644 --- a/config/crd/coroot.com_coroots_legacy.yaml +++ b/config/crd/coroot.com_coroots_legacy.yaml @@ -2999,6 +2999,83 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + s3: + description: S3 storage configuration for ClickHouse (optional). + properties: + cacheSize: + anyOf: + - type: integer + - type: string + description: 'Local cache size for S3 reads (default: 10Gi). Must be less than clickhouse storage size.' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + type: string + credentials: + description: S3 credentials (optional — omit for IAM/IRSA/workload identity). + properties: + accessKeyId: + description: Secret reference for the access key ID. + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + secretAccessKey: + description: Secret reference for the secret access key. + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + type: object + endpoint: + description: S3 endpoint URL. E.g., https://s3.amazonaws.com/my-bucket/clickhouse/ + type: string + mode: + default: tiered + description: |- + Storage mode: "tiered" keeps recent data on local disk and moves older data to S3; + "s3only" stores all data on S3 using local disk only for cache. + enum: + - tiered + - s3only + type: string + moveFactor: + description: 'Fraction of local disk free space that triggers moving data to S3 (default: 0.1). Only used in tiered mode.' + type: string + region: + description: S3 region (optional). + type: string + required: + - endpoint + type: object shards: type: integer storage: diff --git a/controller/clickhouse.go b/controller/clickhouse.go index 59d7c8f..5432622 100644 --- a/controller/clickhouse.go +++ b/controller/clickhouse.go @@ -238,11 +238,36 @@ func (r *CorootReconciler) clickhouseStatefulSets(cr *corootv1.Coroot) []*appsv1 }, } + if s3 := cr.Spec.Clickhouse.S3; s3 != nil && s3.Credentials != nil { + s3Envs := []corev1.EnvVar{} + if s3.Credentials.AccessKeyID != nil { + s3Envs = append(s3Envs, corev1.EnvVar{ + Name: "AWS_ACCESS_KEY_ID", + ValueFrom: &corev1.EnvVarSource{SecretKeyRef: s3.Credentials.AccessKeyID}, + }) + } + if s3.Credentials.SecretAccessKey != nil { + s3Envs = append(s3Envs, corev1.EnvVar{ + Name: "AWS_SECRET_ACCESS_KEY", + ValueFrom: &corev1.EnvVarSource{SecretKeyRef: s3.Credentials.SecretAccessKey}, + }) + } + ss.Spec.Template.Spec.Containers[0].Env = append(ss.Spec.Template.Spec.Containers[0].Env, s3Envs...) + } + res = append(res, ss) } return res } +type s3ConfigParams struct { + Endpoint string + Region string + CacheSize string + Mode string + MoveFactor string +} + func clickhouseConfigCmd(filename string, cr *corootv1.Coroot, shards, replicas, keepers int) string { params := struct { Namespace string @@ -251,6 +276,7 @@ func clickhouseConfigCmd(filename string, cr *corootv1.Coroot, shards, replicas, Replicas []int Keepers []int LogLevel string + S3 *s3ConfigParams }{ Namespace: cr.Namespace, Name: cr.Name, @@ -268,6 +294,27 @@ func clickhouseConfigCmd(filename string, cr *corootv1.Coroot, shards, replicas, if params.LogLevel == "" { params.LogLevel = "warning" } + if s3 := cr.Spec.Clickhouse.S3; s3 != nil { + cacheSize := s3.CacheSize + if cacheSize.IsZero() { + cacheSize, _ = resource.ParseQuantity("10Gi") + } + mode := s3.Mode + if mode == "" { + mode = "tiered" + } + moveFactor := s3.MoveFactor + if moveFactor == "" { + moveFactor = "0.1" + } + params.S3 = &s3ConfigParams{ + Endpoint: s3.Endpoint, + Region: s3.Region, + CacheSize: cacheSize.String(), + Mode: mode, + MoveFactor: moveFactor, + } + } var out bytes.Buffer _ = clickhouseConfigTemplate.Execute(&out, params) return "cat < " + filename + out.String() + "EOF" @@ -345,6 +392,58 @@ var clickhouseConfigTemplate = template.Must(template.New("").Parse(` /clickhouse/task_queue/ddl +{{- if .S3 }} + + + false + s3_{{ .S3.Mode }} + + + + + + s3 + {{ .S3.Endpoint }}{shard}/{replica}/ + {{- if .S3.Region }} + {{ .S3.Region }} + {{- end }} + true + + + cache + s3_disk + /var/lib/clickhouse/s3_cache/ + {{ .S3.CacheSize }} + 1 + + + + {{- if eq .S3.Mode "tiered" }} + + + + default + + + s3_cache + false + false + + + {{ .S3.MoveFactor }} + + {{- else }} + + +
+ s3_cache +
+ + + {{- end }} + + +{{- end }} `)) diff --git a/controller/coroot.go b/controller/coroot.go index a68a05e..fe4ed1e 100644 --- a/controller/coroot.go +++ b/controller/coroot.go @@ -43,6 +43,23 @@ func (r *CorootReconciler) validateCoroot(ctx context.Context, cr *corootv1.Coro cr.Spec.Service.GRPCPort = 4317 } + if s3 := cr.Spec.Clickhouse.S3; s3 != nil { + storageSize := cr.Spec.Clickhouse.Storage.Size + if storageSize.IsZero() { + storageSize, _ = resource.ParseQuantity("100Gi") + } + cacheSize := s3.CacheSize + if cacheSize.IsZero() { + cacheSize, _ = resource.ParseQuantity("10Gi") + } + if cacheSize.Cmp(storageSize) >= 0 { + logErr("ClickHouse S3 cache size (%s) must be less than storage size (%s).", cacheSize.String(), storageSize.String()) + } + if !strings.HasSuffix(s3.Endpoint, "/") { + s3.Endpoint += "/" + } + } + var err error if tls := cr.Spec.TLS; tls != nil {