diff --git a/Dockerfile b/Dockerfile index e6a07c9d..c19b4318 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ # under the License. # Build the manager binary -FROM golang:1.21.12 as builder +FROM golang:1.21.12 AS builder ARG TARGETOS ARG TARGETARCH diff --git a/api/disaggregated/cluster/v1/types.go b/api/disaggregated/cluster/v1/types.go index a6726861..f96b6eb6 100644 --- a/api/disaggregated/cluster/v1/types.go +++ b/api/disaggregated/cluster/v1/types.go @@ -64,7 +64,7 @@ type ComputeCluster struct { //Name is the identifier of computeCluster, name can be used specify what computeCluster to run sql. if not set, will use `computeCluster` and the index in array to set.ep: computeCluster-1. Name string `json:"name,omitempty"` - //ClusterId is the identifier of computeCluster, this will distinguish all com puteCluster in meta. + //ClusterId is the identifier of computeCluster, this will distinguish all computeCluster in meta. ClusterId string `json:"clusterId,omitempty"` CommonSpec `json:",inline"` @@ -294,8 +294,14 @@ const ( //Failed represents service failed to start, can't be accessed. Failed Phase = "Failed" //Creating represents service in creating stage. - Reconciling Phase = "Reconciling" - ReconclingDropFailed Phase = "ReconclingDropFailed" + Reconciling Phase = "Reconciling" + + //Scaling represents service in Scaling. + Scaling Phase = "Scaling" + ScaleDownFailed Phase = "ScaleDownFailed" + ResumeFailed Phase = "ResumeFailed" + SuspendFailed Phase = "SuspendFailed" + Suspended Phase = "Suspended" ) type AvailableStatus string @@ -308,6 +314,8 @@ const ( UnAvailable AvailableStatus = "UnAvailable" ) +// AvailableStatus StatefulsetName ServiceName + type ComputeClusterStatus struct { //Phase represent the stage of reconciling. Phase Phase `json:"phase,omitempty"` @@ -321,6 +329,8 @@ type ComputeClusterStatus struct { AvailableStatus AvailableStatus `json:"availableStatus,omitempty"` //ClusterId display the clusterId of compute cluster in meta. ClusterId string `json:"clusterId,omitempty"` + //suspend replicas display the replicas of compute cluster before resume. + SuspendReplicas int32 `json:"suspendReplicas,omitempty"` // replicas is the number of Pods created by the StatefulSet controller. Replicas int32 `json:"replicas,omitempty"` diff --git a/api/disaggregated/cluster/v1/unique_id.go b/api/disaggregated/cluster/v1/unique_id.go index 480f80bd..2fe034ce 100644 --- a/api/disaggregated/cluster/v1/unique_id.go +++ b/api/disaggregated/cluster/v1/unique_id.go @@ -64,6 +64,7 @@ func (ddc *DorisDisaggregatedCluster) GetInstanceId() string { // need config in vaultConfigMap. return "" } + func (ddc *DorisDisaggregatedCluster) GetCCId(cc *ComputeCluster) string { if cc == nil || ddc == nil { return "" diff --git a/api/disaggregated/cluster/v1/utils.go b/api/disaggregated/cluster/v1/utils.go index 13b375af..6ce72ff6 100644 --- a/api/disaggregated/cluster/v1/utils.go +++ b/api/disaggregated/cluster/v1/utils.go @@ -37,3 +37,7 @@ var ( DisaggregatedBE DisaggregatedComponentType = "BE" DisaggregatedMS DisaggregatedComponentType = "MS" ) + +const ( + DefaultMetaserviceNumber int32 = 2 +) diff --git a/api/disaggregated/cluster/v1/zz_generated.deepcopy.go b/api/disaggregated/cluster/v1/zz_generated.deepcopy.go index abc658cc..1b036b19 100644 --- a/api/disaggregated/cluster/v1/zz_generated.deepcopy.go +++ b/api/disaggregated/cluster/v1/zz_generated.deepcopy.go @@ -190,21 +190,6 @@ func (in *ConfigMap) DeepCopy() *ConfigMap { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *DisMS) DeepCopyInto(out *DisMS) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DisMS. -func (in *DisMS) DeepCopy() *DisMS { - if in == nil { - return nil - } - out := new(DisMS) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DorisDisaggregatedCluster) DeepCopyInto(out *DorisDisaggregatedCluster) { *out = *in @@ -267,7 +252,7 @@ func (in *DorisDisaggregatedClusterList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DorisDisaggregatedClusterSpec) DeepCopyInto(out *DorisDisaggregatedClusterSpec) { *out = *in - out.DisMS = in.DisMS + in.MetaService.DeepCopyInto(&out.MetaService) in.FeSpec.DeepCopyInto(&out.FeSpec) if in.ComputeClusters != nil { in, out := &in.ComputeClusters, &out.ComputeClusters @@ -291,6 +276,7 @@ func (in *DorisDisaggregatedClusterSpec) DeepCopy() *DorisDisaggregatedClusterSp // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DorisDisaggregatedClusterStatus) DeepCopyInto(out *DorisDisaggregatedClusterStatus) { *out = *in + out.MetaServiceStatus = in.MetaServiceStatus out.FEStatus = in.FEStatus out.ClusterHealth = in.ClusterHealth if in.ComputeClusterStatuses != nil { @@ -337,6 +323,22 @@ func (in *ExportService) DeepCopy() *ExportService { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FDB) DeepCopyInto(out *FDB) { + *out = *in + out.ConfigMapNamespaceName = in.ConfigMapNamespaceName +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FDB. +func (in *FDB) DeepCopy() *FDB { + if in == nil { + return nil + } + out := new(FDB) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FEStatus) DeepCopyInto(out *FEStatus) { *out = *in @@ -368,6 +370,53 @@ func (in *FeSpec) DeepCopy() *FeSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MetaService) DeepCopyInto(out *MetaService) { + *out = *in + in.CommonSpec.DeepCopyInto(&out.CommonSpec) + out.FDB = in.FDB +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetaService. +func (in *MetaService) DeepCopy() *MetaService { + if in == nil { + return nil + } + out := new(MetaService) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MetaServiceStatus) DeepCopyInto(out *MetaServiceStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetaServiceStatus. +func (in *MetaServiceStatus) DeepCopy() *MetaServiceStatus { + if in == nil { + return nil + } + out := new(MetaServiceStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NamespaceName) DeepCopyInto(out *NamespaceName) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NamespaceName. +func (in *NamespaceName) DeepCopy() *NamespaceName { + if in == nil { + return nil + } + out := new(NamespaceName) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PersistentVolume) DeepCopyInto(out *PersistentVolume) { *out = *in diff --git a/config/crd/bases/crds.yaml b/config/crd/bases/crds.yaml index 414c7821..9a0d5b0d 100644 --- a/config/crd/bases/crds.yaml +++ b/config/crd/bases/crds.yaml @@ -8467,6 +8467,9 @@ spec: - jsonPath: .status.clusterHealth.health name: ClusterHealth type: string + - jsonPath: .status.metaServiceStatus.phase + name: MSPhase + type: string - jsonPath: .status.feStatus.phase name: FEPhase type: string @@ -12188,100 +12191,1938 @@ spec: description: MetaService describe the metaservice that cluster want to storage metadata. properties: - name: - description: Name specify the name of metaservice resource. - type: string - namespace: - description: Namespace specify the namespace of metaservice deployed. - type: string - type: object - type: object - status: - properties: - clusterHealth: - properties: - ccAvailableCount: - description: the available numbers of compute cluster. - format: int32 - type: integer - ccCount: - description: the number of compute cluster. - format: int32 - type: integer - ccFullAvailableCount: - description: the full available numbers of compute cluster, represents - all pod in compute cluster are ready. - format: int32 - type: integer - feAvailable: - description: represents the fe available or not. - type: boolean - health: - description: represents the cluster overall status. - type: string - type: object - computeClusterStatuses: - description: ComputeClusterStatuses reflect a list of computeCluster - status. - items: - properties: - ComputeClusterName: - description: represents the compute cluster. - type: string - availableReplicas: - description: Total number of available pods (ready for at least - minReadySeconds) targeted by this statefulset. - format: int32 - type: integer - availableStatus: - description: AvailableStatus represents the compute cluster - available or not. - type: string - clusterId: - description: ClusterId display the clusterId of compute cluster - in meta. - type: string - phase: - description: Phase represent the stage of reconciling. - type: string - replicas: - description: replicas is the number of Pods created by the StatefulSet - controller. - format: int32 - type: integer - serviceName: - description: the service that can access the compute cluster - pods. - type: string - statefulsetName: - description: the statefulset of control this compute cluster - pods. - type: string - type: object - type: array - feStatus: - description: FEStatus describe the fe status. - properties: - availableStatus: - description: AvailableStatus represents the fe available or not. - type: string - clusterId: - description: ClusterId display the clusterId of fe in meta. - type: string - phase: - description: Phase represent the stage of reconciling. - type: string - type: object - instanceId: - description: ClusterId display the clusterId of DorisDisaggregatedCluster - in meta. - type: string - msEndpoint: - description: the ms address for store meta of disaggregated cluster. - type: string - msToken: - description: the token for access ms service. - type: string + affinity: + description: Affinity is a group of affinity scheduling rules. + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for + the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods + to nodes that satisfy the affinity expressions specified + by this field, but it may choose a node that violates + one or more of the expressions. The node that is most + preferred is the one with the greatest sum of weights, + i.e. for each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum by iterating + through the elements of this field and adding "weight" + to the sum if the node matches the corresponding matchExpressions; + the node(s) with the highest sum are the most preferred. + items: + description: An empty preferred scheduling term matches + all objects with implicit weight 0 (i.e. it's a no-op). + A null preferred scheduling term matches no objects + (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with + the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement is + a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. If the + operator is Gt or Lt, the values array + must have a single element, which will + be interpreted as an integer. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement is + a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. If the + operator is Gt or Lt, the values array + must have a single element, which will + be interpreted as an integer. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the + corresponding nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified by + this field are not met at scheduling time, the pod will + not be scheduled onto the node. If the affinity requirements + specified by this field cease to be met at some point + during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from + its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. + The terms are ORed. + items: + description: A null or empty node selector term + matches no objects. The requirements of them are + ANDed. The TopologySelectorTerm type implements + a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement is + a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. If the + operator is Gt or Lt, the values array + must have a single element, which will + be interpreted as an integer. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement is + a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. If the + operator is Gt or Lt, the values array + must have a single element, which will + be interpreted as an integer. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. + co-locate this pod in the same node, zone, etc. as some + other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods + to nodes that satisfy the affinity expressions specified + by this field, but it may choose a node that violates + one or more of the expressions. The node that is most + preferred is the one with the greatest sum of weights, + i.e. for each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum by iterating + through the elements of this field and adding "weight" + to the sum if the node has pods which matches the corresponding + podAffinityTerm; the node(s) with the highest sum are + the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by + this field and the ones listed in the namespaces + field. null selector and null or empty namespaces + list means "this pod's namespace". An empty + selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list + of namespace names that the term applies to. + The term is applied to the union of the namespaces + listed in this field and the ones selected + by namespaceSelector. null or empty namespaces + list and null namespaceSelector means "this + pod's namespace". + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the + pods matching the labelSelector in the specified + namespaces, where co-located is defined as + running on a node whose value of the label + with key topologyKey matches that of any node + on which any of the selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching the + corresponding podAffinityTerm, in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified by + this field are not met at scheduling time, the pod will + not be scheduled onto the node. If the affinity requirements + specified by this field cease to be met at some point + during pod execution (e.g. due to a pod label update), + the system may or may not try to eventually evict the + pod from its node. When there are multiple elements, + the lists of nodes corresponding to each podAffinityTerm + are intersected, i.e. all terms must be satisfied. + items: + description: Defines a set of pods (namely those matching + the labelSelector relative to the given namespace(s)) + that this pod should be co-located (affinity) or not + co-located (anti-affinity) with, where co-located + is defined as running on a node whose value of the + label with key matches that of any node + on which a pod of the set of pods is running + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by this + field and the ones listed in the namespaces field. + null selector and null or empty namespaces list + means "this pod's namespace". An empty selector + ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list + of namespace names that the term applies to. The + term is applied to the union of the namespaces + listed in this field and the ones selected by + namespaceSelector. null or empty namespaces list + and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching the labelSelector in the specified namespaces, + where co-located is defined as running on a node + whose value of the label with key topologyKey + matches that of any node on which any of the selected + pods is running. Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules + (e.g. avoid putting this pod in the same node, zone, etc. + as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods + to nodes that satisfy the anti-affinity expressions + specified by this field, but it may choose a node that + violates one or more of the expressions. The node that + is most preferred is the one with the greatest sum of + weights, i.e. for each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + anti-affinity expressions, etc.), compute a sum by iterating + through the elements of this field and adding "weight" + to the sum if the node has pods which matches the corresponding + podAffinityTerm; the node(s) with the highest sum are + the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by + this field and the ones listed in the namespaces + field. null selector and null or empty namespaces + list means "this pod's namespace". An empty + selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list + of namespace names that the term applies to. + The term is applied to the union of the namespaces + listed in this field and the ones selected + by namespaceSelector. null or empty namespaces + list and null namespaceSelector means "this + pod's namespace". + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the + pods matching the labelSelector in the specified + namespaces, where co-located is defined as + running on a node whose value of the label + with key topologyKey matches that of any node + on which any of the selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching the + corresponding podAffinityTerm, in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the anti-affinity requirements specified + by this field are not met at scheduling time, the pod + will not be scheduled onto the node. If the anti-affinity + requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod + label update), the system may or may not try to eventually + evict the pod from its node. When there are multiple + elements, the lists of nodes corresponding to each podAffinityTerm + are intersected, i.e. all terms must be satisfied. + items: + description: Defines a set of pods (namely those matching + the labelSelector relative to the given namespace(s)) + that this pod should be co-located (affinity) or not + co-located (anti-affinity) with, where co-located + is defined as running on a node whose value of the + label with key matches that of any node + on which a pod of the set of pods is running + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by this + field and the ones listed in the namespaces field. + null selector and null or empty namespaces list + means "this pod's namespace". An empty selector + ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list + of namespace names that the term applies to. The + term is applied to the union of the namespaces + listed in this field and the ones selected by + namespaceSelector. null or empty namespaces list + and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching the labelSelector in the specified namespaces, + where co-located is defined as running on a node + whose value of the label with key topologyKey + matches that of any node on which any of the selected + pods is running. Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + annotations: + additionalProperties: + type: string + description: Annotations is an unstructured key value map stored + with a resource that may be set by external tools to store and + retrieve arbitrary metadata. + type: object + claims: + description: "Claims lists the names of resources, defined in + spec.resourceClaims, that are used by this container. \n This + is an alpha field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. It can only be set + for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry in pod.spec.resourceClaims + of the Pod where this field is used. It makes that resource + available inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + configMaps: + description: ConfigMaps describe all configmap that need to be + mounted. + items: + properties: + mountPath: + description: 'display the path of configMap be mounted in + pod. the component start conf please mount to /etc/doris, + ep: fe-configmap contains ''fe.conf'', mountPath must + be ''/etc/doris''. key in configMap''s data is file name.' + type: string + name: + description: Name specify the configmap need to be mounted + in pod in deployed namespace. + type: string + type: object + type: array + containerSecurityContext: + description: Security context for all containers running in the + pod (unless they override it). + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls whether a + process can gain more privileges than its parent process. + This bool directly controls if the no_new_privs flag will + be set on the container process. AllowPrivilegeEscalation + is true always when the container is: 1) run as Privileged + 2) has CAP_SYS_ADMIN Note that this field cannot be set + when spec.os.name is windows.' + type: boolean + capabilities: + description: The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the + container runtime. Note that this field cannot be set when + spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + type: object + privileged: + description: Run container in privileged mode. Processes in + privileged containers are essentially equivalent to root + on the host. Defaults to false. Note that this field cannot + be set when spec.os.name is windows. + type: boolean + procMount: + description: procMount denotes the type of proc mount to use + for the containers. The default is DefaultProcMount which + uses the container runtime defaults for readonly paths and + masked paths. This requires the ProcMountType feature flag + to be enabled. Note that this field cannot be set when spec.os.name + is windows. + type: string + readOnlyRootFilesystem: + description: Whether this container has a read-only root filesystem. + Default is false. Note that this field cannot be set when + spec.os.name is windows. + type: boolean + runAsGroup: + description: The GID to run the entrypoint of the container + process. Uses runtime default if unset. May also be set + in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence. Note that this field cannot be set when + spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run as a non-root + user. If true, the Kubelet will validate the image at runtime + to ensure that it does not run as UID 0 (root) and fail + to start the container if it does. If unset or false, no + such validation will be performed. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container + process. Defaults to user specified in image metadata if + unspecified. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. Note + that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random + SELinux context for each container. May also be set in + PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence. Note that this field cannot be set when + spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: The seccomp options to use by this container. + If seccomp options are provided at both the pod & container + level, the container options override the pod options. Note + that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: localhostProfile indicates a profile defined + in a file on the node should be used. The profile must + be preconfigured on the node to work. Must be a descending + path, relative to the kubelet's configured seccomp profile + location. Must only be set if type is "Localhost". + type: string + type: + description: "type indicates which kind of seccomp profile + will be applied. Valid options are: \n Localhost - a + profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile + should be used. Unconfined - no profile should be applied." + type: string + required: + - type + type: object + windowsOptions: + description: The Windows specific settings applied to all + containers. If unspecified, the options from the PodSecurityContext + will be used. If set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is + linux. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA admission + webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential spec named + by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. + type: string + hostProcess: + description: HostProcess determines if a container should + be run as a 'Host Process' container. This field is + alpha-level and will only be honored by components that + enable the WindowsHostProcessContainers feature flag. + Setting this field without the feature flag will result + in errors when validating the Pod. All of a Pod's containers + must have the same effective HostProcess value (it is + not allowed to have a mix of HostProcess containers + and non-HostProcess containers). In addition, if HostProcess + is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: The UserName in Windows to run the entrypoint + of the container process. Defaults to the user specified + in image metadata if unspecified. May also be set in + PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence. + type: string + type: object + type: object + envVars: + description: EnvVars is a slice of environment variables that + are added to the pods, the default is empty. + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must be a + C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in + the container and any service environment variables. If + a variable cannot be resolved, the reference in the input + string will be unchanged. Double $$ are reduced to a single + $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, + metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, requests.memory + and requests.ephemeral-storage) are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + fdb: + description: specify the address of fdb that used by doris Compute-storage + decoupled cluster. + properties: + address: + description: if fdb directly deployed in machine, please add + fdb access address which generated in 'etc/foundationdb/fdb.cluster' + default. + type: string + configMapNamespaceName: + description: if fdb deployed in kubernetes by fdb-kubernetes-operator, + please specify the namespace and configmap's name generated + by `fdb-kubernetes-operator` in deployed fdbcluster namespace. + properties: + name: + type: string + namespace: + type: string + type: object + type: object + hostAliases: + description: HostAliases is an optional list of hosts and IPs + that will be injected into the pod's hosts file if specified. + This is only valid for non-hostNetwork pods. + items: + description: HostAlias holds the mapping between IP and hostnames + that will be injected as an entry in the pod's hosts file. + properties: + hostnames: + description: Hostnames for the above IP address. + items: + type: string + type: array + ip: + description: IP address of the host file entry. + type: string + type: object + type: array + image: + description: Image is the Disaggregated docker image to deploy. + please reference the selectdb repository to find. + type: string + imagePullSecrets: + description: 'ImagePullSecrets is an optional list of references + to secrets in the same namespace to use for pulling any of the + images used by this PodSpec. If specified, these secrets will + be passed to individual puller implementations for them to use. + More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod' + items: + description: LocalObjectReference contains enough information + to let you locate the referenced object inside the same namespace. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + type: array + labels: + additionalProperties: + type: string + description: Labels for organize and categorize objects + type: object + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute resources + allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + noStoreLog: + description: when set true, the log will store in disk that created + by volumeClaimTemplate + type: boolean + nodeSelector: + additionalProperties: + type: string + description: specify what's node to deploy compute cluster pod. + type: object + persistentVolume: + description: VolumeClaimTemplate allows customizing the persistent + volume claim for the pod. + properties: + annotations: + additionalProperties: + type: string + description: Annotation for PVC pods. Users can adapt the + storage authentication and pv binding of the cloud platform + through configuration. It only takes effect in the first + configuration and cannot be added or modified later. + type: object + logNotStore: + description: if config true, the log will mount a pvc to store + logs. the pvc size is definitely 200Gi, as the log recycling + system will regular recycling. + type: boolean + mountPaths: + description: specify mountPaths, if not config, operator will + refer from be.conf `cache_file_path`. when mountPaths=[]{"/opt/path1", + "/opt/path2"}, will create two pvc mount the two paths. + also, operator will mount the cache_file_path config in + be.conf . if mountPaths have duplicated path in cache_file_path, + operator will only create one pvc. + items: + type: string + type: array + persistentVolumeClaimSpec: + description: PersistentVolumeClaimSpec is a list of claim + spec about storage that pods are required. + properties: + accessModes: + description: 'accessModes contains the desired access + modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'dataSource field can be used to specify + either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) If the provisioner + or an external controller can support the specified + data source, it will create a new volume based on the + contents of the specified data source. When the AnyVolumeDataSource + feature gate is enabled, dataSource contents will be + copied to dataSourceRef, and dataSourceRef contents + will be copied to dataSource when dataSourceRef.namespace + is not specified. If the namespace is specified, then + dataSourceRef will not be copied to dataSource.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: 'dataSourceRef specifies the object from + which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty + API group (non core object) or a PersistentVolumeClaim + object. When this field is specified, volume binding + will only succeed if the type of the specified object + matches some installed volume populator or dynamic provisioner. + This field will replace the functionality of the dataSource + field and as such if both fields are non-empty, they + must have the same value. For backwards compatibility, + when namespace isn''t specified in dataSourceRef, both + fields (dataSource and dataSourceRef) will be set to + the same value automatically if one of them is empty + and the other is non-empty. When namespace is specified + in dataSourceRef, dataSource isn''t set to the same + value and must be empty. There are three important differences + between dataSource and dataSourceRef: * While dataSource + only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim + objects. * While dataSource ignores disallowed values + (dropping them), dataSourceRef preserves all values, + and generates an error if a disallowed value is specified. + * While dataSource only allows local objects, dataSourceRef + allows objects in any namespaces. (Beta) Using this + field requires the AnyVolumeDataSource feature gate + to be enabled. (Alpha) Using the namespace field of + dataSourceRef requires the CrossNamespaceVolumeDataSource + feature gate to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + namespace: + description: Namespace is the namespace of resource + being referenced Note that when a namespace is specified, + a gateway.networking.k8s.io/ReferenceGrant object + is required in the referent namespace to allow that + namespace's owner to accept the reference. See the + ReferenceGrant documentation for details. (Alpha) + This field requires the CrossNamespaceVolumeDataSource + feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: 'resources represents the minimum resources + the volume should have. If RecoverVolumeExpansionFailure + feature is enabled users are allowed to specify resource + requirements that are lower than previous value but + must still be higher than capacity recorded in the status + field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + claims: + description: "Claims lists the names of resources, + defined in spec.resourceClaims, that are used by + this container. \n This is an alpha field and requires + enabling the DynamicResourceAllocation feature gate. + \n This field is immutable. It can only be set for + containers." + items: + description: ResourceClaim references one entry + in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one + entry in pod.spec.resourceClaims of the Pod + where this field is used. It makes that resource + available inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount + of compute resources required. If Requests is omitted + for a container, it defaults to Limits if that is + explicitly specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: selector is a label query over volumes to + consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values + array must be non-empty. If the operator is + Exists or DoesNotExist, the values array must + be empty. This array is replaced during a + strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: 'storageClassName is the name of the StorageClass + required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume is + required by the claim. Value of Filesystem is implied + when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference to the + PersistentVolume backing this claim. + type: string + type: object + type: object + replicas: + description: Replicas represent the number of desired Pod. fe + default is 2. fe is master-slave architecture only one is master. + format: int32 + type: integer + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + securityContext: + description: Security context for pod. + properties: + fsGroup: + description: "A special supplemental group that applies to + all containers in a pod. Some volume types allow the Kubelet + to change the ownership of that volume to be owned by the + pod: \n 1. The owning GID will be the FSGroup 2. The setgid + bit is set (new files created in the volume will be owned + by FSGroup) 3. The permission bits are OR'd with rw-rw---- + \n If unset, the Kubelet will not modify the ownership and + permissions of any volume. Note that this field cannot be + set when spec.os.name is windows." + format: int64 + type: integer + fsGroupChangePolicy: + description: 'fsGroupChangePolicy defines behavior of changing + ownership and permission of the volume before being exposed + inside Pod. This field will only apply to volume types which + support fsGroup based ownership(and permissions). It will + have no effect on ephemeral volume types such as: secret, + configmaps and emptydir. Valid values are "OnRootMismatch" + and "Always". If not specified, "Always" is used. Note that + this field cannot be set when spec.os.name is windows.' + type: string + runAsGroup: + description: The GID to run the entrypoint of the container + process. Uses runtime default if unset. May also be set + in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. Note that this field + cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run as a non-root + user. If true, the Kubelet will validate the image at runtime + to ensure that it does not run as UID 0 (root) and fail + to start the container if it does. If unset or false, no + such validation will be performed. May also be set in SecurityContext. If + set in both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container + process. Defaults to user specified in image metadata if + unspecified. May also be set in SecurityContext. If set + in both SecurityContext and PodSecurityContext, the value + specified in SecurityContext takes precedence for that container. + Note that this field cannot be set when spec.os.name is + windows. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random + SELinux context for each container. May also be set in + SecurityContext. If set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence + for that container. Note that this field cannot be set when + spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: The seccomp options to use by the containers + in this pod. Note that this field cannot be set when spec.os.name + is windows. + properties: + localhostProfile: + description: localhostProfile indicates a profile defined + in a file on the node should be used. The profile must + be preconfigured on the node to work. Must be a descending + path, relative to the kubelet's configured seccomp profile + location. Must only be set if type is "Localhost". + type: string + type: + description: "type indicates which kind of seccomp profile + will be applied. Valid options are: \n Localhost - a + profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile + should be used. Unconfined - no profile should be applied." + type: string + required: + - type + type: object + supplementalGroups: + description: A list of groups applied to the first process + run in each container, in addition to the container's primary + GID, the fsGroup (if specified), and group memberships defined + in the container image for the uid of the container process. + If unspecified, no additional groups are added to any container. + Note that group memberships defined in the container image + for the uid of the container process are still effective, + even if they are not included in this list. Note that this + field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + sysctls: + description: Sysctls hold a list of namespaced sysctls used + for the pod. Pods with unsupported sysctls (by the container + runtime) might fail to launch. Note that this field cannot + be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + windowsOptions: + description: The Windows specific settings applied to all + containers. If unspecified, the options within a container's + SecurityContext will be used. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence. Note that this field cannot be set when + spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA admission + webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential spec named + by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. + type: string + hostProcess: + description: HostProcess determines if a container should + be run as a 'Host Process' container. This field is + alpha-level and will only be honored by components that + enable the WindowsHostProcessContainers feature flag. + Setting this field without the feature flag will result + in errors when validating the Pod. All of a Pod's containers + must have the same effective HostProcess value (it is + not allowed to have a mix of HostProcess containers + and non-HostProcess containers). In addition, if HostProcess + is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: The UserName in Windows to run the entrypoint + of the container process. Defaults to the user specified + in image metadata if unspecified. May also be set in + PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence. + type: string + type: object + type: object + service: + description: export metaservice for accessing from outside k8s. + properties: + annotations: + additionalProperties: + type: string + description: Annotations for using function on different cloud + platform. + type: object + portMaps: + description: PortMaps specify node port for target port in + pod, when the service type=NodePort. + items: + description: PortMap for ServiceType=NodePort situation. + properties: + nodePort: + description: 'The port on each node on which this service + is exposed when type is NodePort or LoadBalancer. Usually + assigned by the system. If a value is specified, in-range, + and not in use it will be used, otherwise the operation + will fail. If not specified, a port will be allocated + if this Service requires one. If this field is specified + when creating a Service which does not need it, creation + will fail. This field will be wiped when updating + a Service to no longer need it (e.g. changing type + from NodePort to ClusterIP). More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + need in 30000-32767' + format: int32 + type: integer + targetPort: + description: 'Number or name of the port to access on + the pods targeted by the service. Number must be in + the range 1 to 65535. Name must be an IANA_SVC_NAME. + If this is a string, it will be looked up as a named + port in the target Pod''s container ports. If this + is not specified, the value of the ''port'' field + is used (an identity map). This field is ignored for + services with clusterIP=None, and should be omitted + or set equal to the ''port'' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service' + format: int32 + type: integer + type: object + type: array + type: + description: 'type of service,the possible value for the service + type are : ClusterIP, NodePort, LoadBalancer,ExternalName. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' + type: string + type: object + serviceAccount: + description: serviceAccount for compute node access cloud service. + type: string + startTimeout: + description: pod start timeout, unit is second + format: int32 + type: integer + systemInitialization: + description: SystemInitialization for fe, be setting system parameters. + properties: + args: + description: Arguments to the entrypoint. + items: + type: string + type: array + command: + description: Entrypoint array. Not executed within a shell. + items: + type: string + type: array + initImage: + description: Image for doris initialization, default is selectdb/alpine:latest. + type: string + type: object + tolerations: + description: (Optional) Tolerations for scheduling pods onto some + dedicated nodes + items: + description: The pod this Toleration is attached to tolerates + any taint that matches the triple using + the matching operator . + properties: + effect: + description: Effect indicates the taint effect to match. + Empty means match all taint effects. When specified, allowed + values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Key is the taint key that the toleration applies + to. Empty means match all taint keys. If the key is empty, + operator must be Exists; this combination means to match + all values and all keys. + type: string + operator: + description: Operator represents a key's relationship to + the value. Valid operators are Exists and Equal. Defaults + to Equal. Exists is equivalent to wildcard for value, + so that a pod can tolerate all taints of a particular + category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period of + time the toleration (which must be of effect NoExecute, + otherwise this field is ignored) tolerates the taint. + By default, it is not set, which means tolerate the taint + forever (do not evict). Zero and negative values will + be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration matches + to. If the operator is Exists, the value should be empty, + otherwise just a regular string. + type: string + type: object + type: array + type: object + type: object + status: + properties: + clusterHealth: + properties: + ccAvailableCount: + description: the available numbers of compute cluster. + format: int32 + type: integer + ccCount: + description: the number of compute cluster. + format: int32 + type: integer + ccFullAvailableCount: + description: the full available numbers of compute cluster, represents + all pod in compute cluster are ready. + format: int32 + type: integer + feAvailable: + description: represents the fe available or not. + type: boolean + health: + description: represents the cluster overall status. + type: string + type: object + computeClusterStatuses: + description: ComputeClusterStatuses reflect a list of computeCluster + status. + items: + properties: + ComputeClusterName: + description: represents the compute cluster. + type: string + availableReplicas: + description: Total number of available pods (ready for at least + minReadySeconds) targeted by this statefulset. + format: int32 + type: integer + availableStatus: + description: AvailableStatus represents the compute cluster + available or not. + type: string + clusterId: + description: ClusterId display the clusterId of compute cluster + in meta. + type: string + phase: + description: Phase represent the stage of reconciling. + type: string + replicas: + description: replicas is the number of Pods created by the StatefulSet + controller. + format: int32 + type: integer + serviceName: + description: the service that can access the compute cluster + pods. + type: string + statefulsetName: + description: the statefulset of control this compute cluster + pods. + type: string + suspendReplicas: + description: suspend replicas display the replicas of compute + cluster before resume. + format: int32 + type: integer + type: object + type: array + feStatus: + description: FEStatus describe the fe status. + properties: + availableStatus: + description: AvailableStatus represents the fe available or not. + type: string + clusterId: + description: ClusterId display the clusterId of fe in meta. + type: string + phase: + description: Phase represent the stage of reconciling. + type: string + type: object + instanceId: + description: ClusterId display the clusterId of DorisDisaggregatedCluster + in meta. + type: string + metaServiceStatus: + description: describe the metaservice status now. + properties: + availableStatus: + description: AvailableStatus represents the metaservice available + or not. + type: string + metaServiceEndpoint: + description: the meta service address for store meta of disaggregated + cluster. + type: string + msToken: + description: the token for access ms service. + type: string + phase: + description: Phase represent the stage of reconciling. + type: string + type: object type: object type: object served: true diff --git a/config/crd/bases/disaggregated.cluster.doris.com_dorisdisaggregatedclusters.yaml b/config/crd/bases/disaggregated.cluster.doris.com_dorisdisaggregatedclusters.yaml index cfcc3ac3..688f76f3 100644 --- a/config/crd/bases/disaggregated.cluster.doris.com_dorisdisaggregatedclusters.yaml +++ b/config/crd/bases/disaggregated.cluster.doris.com_dorisdisaggregatedclusters.yaml @@ -21,6 +21,9 @@ spec: - jsonPath: .status.clusterHealth.health name: ClusterHealth type: string + - jsonPath: .status.metaServiceStatus.phase + name: MSPhase + type: string - jsonPath: .status.feStatus.phase name: FEPhase type: string @@ -1921,17 +1924,6 @@ spec: type: array type: object type: array - disMS: - description: MetaService describe the metaservice that cluster want - to storage metadata. - properties: - name: - description: Name specify the name of metaservice resource. - type: string - namespace: - description: Namespace specify the namespace of metaservice deployed. - type: string - type: object feSpec: description: FeSpec describe the fe specification of doris disaggregated cluster. @@ -3749,93 +3741,1942 @@ spec: of file object information. example S3. configmap have to config, please reference the doc. type: string - type: object - status: - properties: - clusterHealth: - properties: - ccAvailableCount: - description: the available numbers of compute cluster. - format: int32 - type: integer - ccCount: - description: the number of compute cluster. - format: int32 - type: integer - ccFullAvailableCount: - description: the full available numbers of compute cluster, represents - all pod in compute cluster are ready. - format: int32 - type: integer - feAvailable: - description: represents the fe available or not. - type: boolean - health: - description: represents the cluster overall status. - type: string - type: object - computeClusterStatuses: - description: ComputeClusterStatuses reflect a list of computeCluster - status. - items: - properties: - ComputeClusterName: - description: represents the compute cluster. - type: string - availableReplicas: - description: Total number of available pods (ready for at least - minReadySeconds) targeted by this statefulset. - format: int32 - type: integer - availableStatus: - description: AvailableStatus represents the compute cluster - available or not. - type: string - clusterId: - description: ClusterId display the clusterId of compute cluster - in meta. - type: string - phase: - description: Phase represent the stage of reconciling. - type: string - replicas: - description: replicas is the number of Pods created by the StatefulSet - controller. - format: int32 - type: integer - serviceName: - description: the service that can access the compute cluster - pods. - type: string - statefulsetName: - description: the statefulset of control this compute cluster - pods. - type: string - type: object - type: array - feStatus: - description: FEStatus describe the fe status. + metaService: + description: MetaService describe the metaservice that cluster want + to storage metadata. properties: - availableStatus: - description: AvailableStatus represents the fe available or not. - type: string - clusterId: - description: ClusterId display the clusterId of fe in meta. - type: string - phase: - description: Phase represent the stage of reconciling. - type: string - type: object - instanceId: - description: ClusterId display the clusterId of DorisDisaggregatedCluster - in meta. - type: string - msEndpoint: - description: the ms address for store meta of disaggregated cluster. - type: string - msToken: - description: the token for access ms service. - type: string + affinity: + description: Affinity is a group of affinity scheduling rules. + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for + the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods + to nodes that satisfy the affinity expressions specified + by this field, but it may choose a node that violates + one or more of the expressions. The node that is most + preferred is the one with the greatest sum of weights, + i.e. for each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum by iterating + through the elements of this field and adding "weight" + to the sum if the node matches the corresponding matchExpressions; + the node(s) with the highest sum are the most preferred. + items: + description: An empty preferred scheduling term matches + all objects with implicit weight 0 (i.e. it's a no-op). + A null preferred scheduling term matches no objects + (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with + the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement is + a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. If the + operator is Gt or Lt, the values array + must have a single element, which will + be interpreted as an integer. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement is + a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. If the + operator is Gt or Lt, the values array + must have a single element, which will + be interpreted as an integer. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the + corresponding nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified by + this field are not met at scheduling time, the pod will + not be scheduled onto the node. If the affinity requirements + specified by this field cease to be met at some point + during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from + its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. + The terms are ORed. + items: + description: A null or empty node selector term + matches no objects. The requirements of them are + ANDed. The TopologySelectorTerm type implements + a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement is + a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. If the + operator is Gt or Lt, the values array + must have a single element, which will + be interpreted as an integer. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement is + a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. If the + operator is Gt or Lt, the values array + must have a single element, which will + be interpreted as an integer. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. + co-locate this pod in the same node, zone, etc. as some + other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods + to nodes that satisfy the affinity expressions specified + by this field, but it may choose a node that violates + one or more of the expressions. The node that is most + preferred is the one with the greatest sum of weights, + i.e. for each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum by iterating + through the elements of this field and adding "weight" + to the sum if the node has pods which matches the corresponding + podAffinityTerm; the node(s) with the highest sum are + the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by + this field and the ones listed in the namespaces + field. null selector and null or empty namespaces + list means "this pod's namespace". An empty + selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list + of namespace names that the term applies to. + The term is applied to the union of the namespaces + listed in this field and the ones selected + by namespaceSelector. null or empty namespaces + list and null namespaceSelector means "this + pod's namespace". + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the + pods matching the labelSelector in the specified + namespaces, where co-located is defined as + running on a node whose value of the label + with key topologyKey matches that of any node + on which any of the selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching the + corresponding podAffinityTerm, in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified by + this field are not met at scheduling time, the pod will + not be scheduled onto the node. If the affinity requirements + specified by this field cease to be met at some point + during pod execution (e.g. due to a pod label update), + the system may or may not try to eventually evict the + pod from its node. When there are multiple elements, + the lists of nodes corresponding to each podAffinityTerm + are intersected, i.e. all terms must be satisfied. + items: + description: Defines a set of pods (namely those matching + the labelSelector relative to the given namespace(s)) + that this pod should be co-located (affinity) or not + co-located (anti-affinity) with, where co-located + is defined as running on a node whose value of the + label with key matches that of any node + on which a pod of the set of pods is running + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by this + field and the ones listed in the namespaces field. + null selector and null or empty namespaces list + means "this pod's namespace". An empty selector + ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list + of namespace names that the term applies to. The + term is applied to the union of the namespaces + listed in this field and the ones selected by + namespaceSelector. null or empty namespaces list + and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching the labelSelector in the specified namespaces, + where co-located is defined as running on a node + whose value of the label with key topologyKey + matches that of any node on which any of the selected + pods is running. Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules + (e.g. avoid putting this pod in the same node, zone, etc. + as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods + to nodes that satisfy the anti-affinity expressions + specified by this field, but it may choose a node that + violates one or more of the expressions. The node that + is most preferred is the one with the greatest sum of + weights, i.e. for each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + anti-affinity expressions, etc.), compute a sum by iterating + through the elements of this field and adding "weight" + to the sum if the node has pods which matches the corresponding + podAffinityTerm; the node(s) with the highest sum are + the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by + this field and the ones listed in the namespaces + field. null selector and null or empty namespaces + list means "this pod's namespace". An empty + selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list + of namespace names that the term applies to. + The term is applied to the union of the namespaces + listed in this field and the ones selected + by namespaceSelector. null or empty namespaces + list and null namespaceSelector means "this + pod's namespace". + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the + pods matching the labelSelector in the specified + namespaces, where co-located is defined as + running on a node whose value of the label + with key topologyKey matches that of any node + on which any of the selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching the + corresponding podAffinityTerm, in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the anti-affinity requirements specified + by this field are not met at scheduling time, the pod + will not be scheduled onto the node. If the anti-affinity + requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod + label update), the system may or may not try to eventually + evict the pod from its node. When there are multiple + elements, the lists of nodes corresponding to each podAffinityTerm + are intersected, i.e. all terms must be satisfied. + items: + description: Defines a set of pods (namely those matching + the labelSelector relative to the given namespace(s)) + that this pod should be co-located (affinity) or not + co-located (anti-affinity) with, where co-located + is defined as running on a node whose value of the + label with key matches that of any node + on which a pod of the set of pods is running + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by this + field and the ones listed in the namespaces field. + null selector and null or empty namespaces list + means "this pod's namespace". An empty selector + ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list + of namespace names that the term applies to. The + term is applied to the union of the namespaces + listed in this field and the ones selected by + namespaceSelector. null or empty namespaces list + and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching the labelSelector in the specified namespaces, + where co-located is defined as running on a node + whose value of the label with key topologyKey + matches that of any node on which any of the selected + pods is running. Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + annotations: + additionalProperties: + type: string + description: Annotations is an unstructured key value map stored + with a resource that may be set by external tools to store and + retrieve arbitrary metadata. + type: object + claims: + description: "Claims lists the names of resources, defined in + spec.resourceClaims, that are used by this container. \n This + is an alpha field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. It can only be set + for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry in pod.spec.resourceClaims + of the Pod where this field is used. It makes that resource + available inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + configMaps: + description: ConfigMaps describe all configmap that need to be + mounted. + items: + properties: + mountPath: + description: 'display the path of configMap be mounted in + pod. the component start conf please mount to /etc/doris, + ep: fe-configmap contains ''fe.conf'', mountPath must + be ''/etc/doris''. key in configMap''s data is file name.' + type: string + name: + description: Name specify the configmap need to be mounted + in pod in deployed namespace. + type: string + type: object + type: array + containerSecurityContext: + description: Security context for all containers running in the + pod (unless they override it). + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls whether a + process can gain more privileges than its parent process. + This bool directly controls if the no_new_privs flag will + be set on the container process. AllowPrivilegeEscalation + is true always when the container is: 1) run as Privileged + 2) has CAP_SYS_ADMIN Note that this field cannot be set + when spec.os.name is windows.' + type: boolean + capabilities: + description: The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the + container runtime. Note that this field cannot be set when + spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + type: object + privileged: + description: Run container in privileged mode. Processes in + privileged containers are essentially equivalent to root + on the host. Defaults to false. Note that this field cannot + be set when spec.os.name is windows. + type: boolean + procMount: + description: procMount denotes the type of proc mount to use + for the containers. The default is DefaultProcMount which + uses the container runtime defaults for readonly paths and + masked paths. This requires the ProcMountType feature flag + to be enabled. Note that this field cannot be set when spec.os.name + is windows. + type: string + readOnlyRootFilesystem: + description: Whether this container has a read-only root filesystem. + Default is false. Note that this field cannot be set when + spec.os.name is windows. + type: boolean + runAsGroup: + description: The GID to run the entrypoint of the container + process. Uses runtime default if unset. May also be set + in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence. Note that this field cannot be set when + spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run as a non-root + user. If true, the Kubelet will validate the image at runtime + to ensure that it does not run as UID 0 (root) and fail + to start the container if it does. If unset or false, no + such validation will be performed. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container + process. Defaults to user specified in image metadata if + unspecified. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. Note + that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random + SELinux context for each container. May also be set in + PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence. Note that this field cannot be set when + spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: The seccomp options to use by this container. + If seccomp options are provided at both the pod & container + level, the container options override the pod options. Note + that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: localhostProfile indicates a profile defined + in a file on the node should be used. The profile must + be preconfigured on the node to work. Must be a descending + path, relative to the kubelet's configured seccomp profile + location. Must only be set if type is "Localhost". + type: string + type: + description: "type indicates which kind of seccomp profile + will be applied. Valid options are: \n Localhost - a + profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile + should be used. Unconfined - no profile should be applied." + type: string + required: + - type + type: object + windowsOptions: + description: The Windows specific settings applied to all + containers. If unspecified, the options from the PodSecurityContext + will be used. If set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is + linux. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA admission + webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential spec named + by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. + type: string + hostProcess: + description: HostProcess determines if a container should + be run as a 'Host Process' container. This field is + alpha-level and will only be honored by components that + enable the WindowsHostProcessContainers feature flag. + Setting this field without the feature flag will result + in errors when validating the Pod. All of a Pod's containers + must have the same effective HostProcess value (it is + not allowed to have a mix of HostProcess containers + and non-HostProcess containers). In addition, if HostProcess + is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: The UserName in Windows to run the entrypoint + of the container process. Defaults to the user specified + in image metadata if unspecified. May also be set in + PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence. + type: string + type: object + type: object + envVars: + description: EnvVars is a slice of environment variables that + are added to the pods, the default is empty. + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must be a + C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in + the container and any service environment variables. If + a variable cannot be resolved, the reference in the input + string will be unchanged. Double $$ are reduced to a single + $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, + metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, requests.memory + and requests.ephemeral-storage) are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + fdb: + description: specify the address of fdb that used by doris Compute-storage + decoupled cluster. + properties: + address: + description: if fdb directly deployed in machine, please add + fdb access address which generated in 'etc/foundationdb/fdb.cluster' + default. + type: string + configMapNamespaceName: + description: if fdb deployed in kubernetes by fdb-kubernetes-operator, + please specify the namespace and configmap's name generated + by `fdb-kubernetes-operator` in deployed fdbcluster namespace. + properties: + name: + type: string + namespace: + type: string + type: object + type: object + hostAliases: + description: HostAliases is an optional list of hosts and IPs + that will be injected into the pod's hosts file if specified. + This is only valid for non-hostNetwork pods. + items: + description: HostAlias holds the mapping between IP and hostnames + that will be injected as an entry in the pod's hosts file. + properties: + hostnames: + description: Hostnames for the above IP address. + items: + type: string + type: array + ip: + description: IP address of the host file entry. + type: string + type: object + type: array + image: + description: Image is the Disaggregated docker image to deploy. + please reference the selectdb repository to find. + type: string + imagePullSecrets: + description: 'ImagePullSecrets is an optional list of references + to secrets in the same namespace to use for pulling any of the + images used by this PodSpec. If specified, these secrets will + be passed to individual puller implementations for them to use. + More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod' + items: + description: LocalObjectReference contains enough information + to let you locate the referenced object inside the same namespace. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + type: array + labels: + additionalProperties: + type: string + description: Labels for organize and categorize objects + type: object + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute resources + allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + noStoreLog: + description: when set true, the log will store in disk that created + by volumeClaimTemplate + type: boolean + nodeSelector: + additionalProperties: + type: string + description: specify what's node to deploy compute cluster pod. + type: object + persistentVolume: + description: VolumeClaimTemplate allows customizing the persistent + volume claim for the pod. + properties: + annotations: + additionalProperties: + type: string + description: Annotation for PVC pods. Users can adapt the + storage authentication and pv binding of the cloud platform + through configuration. It only takes effect in the first + configuration and cannot be added or modified later. + type: object + logNotStore: + description: if config true, the log will mount a pvc to store + logs. the pvc size is definitely 200Gi, as the log recycling + system will regular recycling. + type: boolean + mountPaths: + description: specify mountPaths, if not config, operator will + refer from be.conf `cache_file_path`. when mountPaths=[]{"/opt/path1", + "/opt/path2"}, will create two pvc mount the two paths. + also, operator will mount the cache_file_path config in + be.conf . if mountPaths have duplicated path in cache_file_path, + operator will only create one pvc. + items: + type: string + type: array + persistentVolumeClaimSpec: + description: PersistentVolumeClaimSpec is a list of claim + spec about storage that pods are required. + properties: + accessModes: + description: 'accessModes contains the desired access + modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'dataSource field can be used to specify + either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) If the provisioner + or an external controller can support the specified + data source, it will create a new volume based on the + contents of the specified data source. When the AnyVolumeDataSource + feature gate is enabled, dataSource contents will be + copied to dataSourceRef, and dataSourceRef contents + will be copied to dataSource when dataSourceRef.namespace + is not specified. If the namespace is specified, then + dataSourceRef will not be copied to dataSource.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: 'dataSourceRef specifies the object from + which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty + API group (non core object) or a PersistentVolumeClaim + object. When this field is specified, volume binding + will only succeed if the type of the specified object + matches some installed volume populator or dynamic provisioner. + This field will replace the functionality of the dataSource + field and as such if both fields are non-empty, they + must have the same value. For backwards compatibility, + when namespace isn''t specified in dataSourceRef, both + fields (dataSource and dataSourceRef) will be set to + the same value automatically if one of them is empty + and the other is non-empty. When namespace is specified + in dataSourceRef, dataSource isn''t set to the same + value and must be empty. There are three important differences + between dataSource and dataSourceRef: * While dataSource + only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim + objects. * While dataSource ignores disallowed values + (dropping them), dataSourceRef preserves all values, + and generates an error if a disallowed value is specified. + * While dataSource only allows local objects, dataSourceRef + allows objects in any namespaces. (Beta) Using this + field requires the AnyVolumeDataSource feature gate + to be enabled. (Alpha) Using the namespace field of + dataSourceRef requires the CrossNamespaceVolumeDataSource + feature gate to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + namespace: + description: Namespace is the namespace of resource + being referenced Note that when a namespace is specified, + a gateway.networking.k8s.io/ReferenceGrant object + is required in the referent namespace to allow that + namespace's owner to accept the reference. See the + ReferenceGrant documentation for details. (Alpha) + This field requires the CrossNamespaceVolumeDataSource + feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: 'resources represents the minimum resources + the volume should have. If RecoverVolumeExpansionFailure + feature is enabled users are allowed to specify resource + requirements that are lower than previous value but + must still be higher than capacity recorded in the status + field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + claims: + description: "Claims lists the names of resources, + defined in spec.resourceClaims, that are used by + this container. \n This is an alpha field and requires + enabling the DynamicResourceAllocation feature gate. + \n This field is immutable. It can only be set for + containers." + items: + description: ResourceClaim references one entry + in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one + entry in pod.spec.resourceClaims of the Pod + where this field is used. It makes that resource + available inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount + of compute resources required. If Requests is omitted + for a container, it defaults to Limits if that is + explicitly specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: selector is a label query over volumes to + consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values + array must be non-empty. If the operator is + Exists or DoesNotExist, the values array must + be empty. This array is replaced during a + strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: 'storageClassName is the name of the StorageClass + required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume is + required by the claim. Value of Filesystem is implied + when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference to the + PersistentVolume backing this claim. + type: string + type: object + type: object + replicas: + description: Replicas represent the number of desired Pod. fe + default is 2. fe is master-slave architecture only one is master. + format: int32 + type: integer + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + securityContext: + description: Security context for pod. + properties: + fsGroup: + description: "A special supplemental group that applies to + all containers in a pod. Some volume types allow the Kubelet + to change the ownership of that volume to be owned by the + pod: \n 1. The owning GID will be the FSGroup 2. The setgid + bit is set (new files created in the volume will be owned + by FSGroup) 3. The permission bits are OR'd with rw-rw---- + \n If unset, the Kubelet will not modify the ownership and + permissions of any volume. Note that this field cannot be + set when spec.os.name is windows." + format: int64 + type: integer + fsGroupChangePolicy: + description: 'fsGroupChangePolicy defines behavior of changing + ownership and permission of the volume before being exposed + inside Pod. This field will only apply to volume types which + support fsGroup based ownership(and permissions). It will + have no effect on ephemeral volume types such as: secret, + configmaps and emptydir. Valid values are "OnRootMismatch" + and "Always". If not specified, "Always" is used. Note that + this field cannot be set when spec.os.name is windows.' + type: string + runAsGroup: + description: The GID to run the entrypoint of the container + process. Uses runtime default if unset. May also be set + in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. Note that this field + cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run as a non-root + user. If true, the Kubelet will validate the image at runtime + to ensure that it does not run as UID 0 (root) and fail + to start the container if it does. If unset or false, no + such validation will be performed. May also be set in SecurityContext. If + set in both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container + process. Defaults to user specified in image metadata if + unspecified. May also be set in SecurityContext. If set + in both SecurityContext and PodSecurityContext, the value + specified in SecurityContext takes precedence for that container. + Note that this field cannot be set when spec.os.name is + windows. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random + SELinux context for each container. May also be set in + SecurityContext. If set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence + for that container. Note that this field cannot be set when + spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: The seccomp options to use by the containers + in this pod. Note that this field cannot be set when spec.os.name + is windows. + properties: + localhostProfile: + description: localhostProfile indicates a profile defined + in a file on the node should be used. The profile must + be preconfigured on the node to work. Must be a descending + path, relative to the kubelet's configured seccomp profile + location. Must only be set if type is "Localhost". + type: string + type: + description: "type indicates which kind of seccomp profile + will be applied. Valid options are: \n Localhost - a + profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile + should be used. Unconfined - no profile should be applied." + type: string + required: + - type + type: object + supplementalGroups: + description: A list of groups applied to the first process + run in each container, in addition to the container's primary + GID, the fsGroup (if specified), and group memberships defined + in the container image for the uid of the container process. + If unspecified, no additional groups are added to any container. + Note that group memberships defined in the container image + for the uid of the container process are still effective, + even if they are not included in this list. Note that this + field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + sysctls: + description: Sysctls hold a list of namespaced sysctls used + for the pod. Pods with unsupported sysctls (by the container + runtime) might fail to launch. Note that this field cannot + be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + windowsOptions: + description: The Windows specific settings applied to all + containers. If unspecified, the options within a container's + SecurityContext will be used. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence. Note that this field cannot be set when + spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA admission + webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential spec named + by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. + type: string + hostProcess: + description: HostProcess determines if a container should + be run as a 'Host Process' container. This field is + alpha-level and will only be honored by components that + enable the WindowsHostProcessContainers feature flag. + Setting this field without the feature flag will result + in errors when validating the Pod. All of a Pod's containers + must have the same effective HostProcess value (it is + not allowed to have a mix of HostProcess containers + and non-HostProcess containers). In addition, if HostProcess + is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: The UserName in Windows to run the entrypoint + of the container process. Defaults to the user specified + in image metadata if unspecified. May also be set in + PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence. + type: string + type: object + type: object + service: + description: export metaservice for accessing from outside k8s. + properties: + annotations: + additionalProperties: + type: string + description: Annotations for using function on different cloud + platform. + type: object + portMaps: + description: PortMaps specify node port for target port in + pod, when the service type=NodePort. + items: + description: PortMap for ServiceType=NodePort situation. + properties: + nodePort: + description: 'The port on each node on which this service + is exposed when type is NodePort or LoadBalancer. Usually + assigned by the system. If a value is specified, in-range, + and not in use it will be used, otherwise the operation + will fail. If not specified, a port will be allocated + if this Service requires one. If this field is specified + when creating a Service which does not need it, creation + will fail. This field will be wiped when updating + a Service to no longer need it (e.g. changing type + from NodePort to ClusterIP). More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + need in 30000-32767' + format: int32 + type: integer + targetPort: + description: 'Number or name of the port to access on + the pods targeted by the service. Number must be in + the range 1 to 65535. Name must be an IANA_SVC_NAME. + If this is a string, it will be looked up as a named + port in the target Pod''s container ports. If this + is not specified, the value of the ''port'' field + is used (an identity map). This field is ignored for + services with clusterIP=None, and should be omitted + or set equal to the ''port'' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service' + format: int32 + type: integer + type: object + type: array + type: + description: 'type of service,the possible value for the service + type are : ClusterIP, NodePort, LoadBalancer,ExternalName. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' + type: string + type: object + serviceAccount: + description: serviceAccount for compute node access cloud service. + type: string + startTimeout: + description: pod start timeout, unit is second + format: int32 + type: integer + systemInitialization: + description: SystemInitialization for fe, be setting system parameters. + properties: + args: + description: Arguments to the entrypoint. + items: + type: string + type: array + command: + description: Entrypoint array. Not executed within a shell. + items: + type: string + type: array + initImage: + description: Image for doris initialization, default is selectdb/alpine:latest. + type: string + type: object + tolerations: + description: (Optional) Tolerations for scheduling pods onto some + dedicated nodes + items: + description: The pod this Toleration is attached to tolerates + any taint that matches the triple using + the matching operator . + properties: + effect: + description: Effect indicates the taint effect to match. + Empty means match all taint effects. When specified, allowed + values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Key is the taint key that the toleration applies + to. Empty means match all taint keys. If the key is empty, + operator must be Exists; this combination means to match + all values and all keys. + type: string + operator: + description: Operator represents a key's relationship to + the value. Valid operators are Exists and Equal. Defaults + to Equal. Exists is equivalent to wildcard for value, + so that a pod can tolerate all taints of a particular + category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period of + time the toleration (which must be of effect NoExecute, + otherwise this field is ignored) tolerates the taint. + By default, it is not set, which means tolerate the taint + forever (do not evict). Zero and negative values will + be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration matches + to. If the operator is Exists, the value should be empty, + otherwise just a regular string. + type: string + type: object + type: array + type: object + type: object + status: + properties: + clusterHealth: + properties: + ccAvailableCount: + description: the available numbers of compute cluster. + format: int32 + type: integer + ccCount: + description: the number of compute cluster. + format: int32 + type: integer + ccFullAvailableCount: + description: the full available numbers of compute cluster, represents + all pod in compute cluster are ready. + format: int32 + type: integer + feAvailable: + description: represents the fe available or not. + type: boolean + health: + description: represents the cluster overall status. + type: string + type: object + computeClusterStatuses: + description: ComputeClusterStatuses reflect a list of computeCluster + status. + items: + properties: + ComputeClusterName: + description: represents the compute cluster. + type: string + availableReplicas: + description: Total number of available pods (ready for at least + minReadySeconds) targeted by this statefulset. + format: int32 + type: integer + availableStatus: + description: AvailableStatus represents the compute cluster + available or not. + type: string + clusterId: + description: ClusterId display the clusterId of compute cluster + in meta. + type: string + phase: + description: Phase represent the stage of reconciling. + type: string + replicas: + description: replicas is the number of Pods created by the StatefulSet + controller. + format: int32 + type: integer + serviceName: + description: the service that can access the compute cluster + pods. + type: string + statefulsetName: + description: the statefulset of control this compute cluster + pods. + type: string + suspendReplicas: + description: suspend replicas display the replicas of compute + cluster before resume. + format: int32 + type: integer + type: object + type: array + feStatus: + description: FEStatus describe the fe status. + properties: + availableStatus: + description: AvailableStatus represents the fe available or not. + type: string + clusterId: + description: ClusterId display the clusterId of fe in meta. + type: string + phase: + description: Phase represent the stage of reconciling. + type: string + type: object + instanceId: + description: ClusterId display the clusterId of DorisDisaggregatedCluster + in meta. + type: string + metaServiceStatus: + description: describe the metaservice status now. + properties: + availableStatus: + description: AvailableStatus represents the metaservice available + or not. + type: string + metaServiceEndpoint: + description: the meta service address for store meta of disaggregated + cluster. + type: string + msToken: + description: the token for access ms service. + type: string + phase: + description: Phase represent the stage of reconciling. + type: string + type: object type: object type: object served: true diff --git a/doc/api.md b/doc/api.md index d1d62edc..89e564bc 100644 --- a/doc/api.md +++ b/doc/api.md @@ -2593,5 +2593,5 @@ string

Generated with gen-crd-api-reference-docs -on git commit 7d8fdc8. +on git commit 711e5a4.

diff --git a/doc/disaggregated_cluster_api.md b/doc/disaggregated_cluster_api.md index 568c5b96..c47a7eb1 100644 --- a/doc/disaggregated_cluster_api.md +++ b/doc/disaggregated_cluster_api.md @@ -13,7 +13,7 @@ Resource Types:

AvailableStatus (string alias)

-(Appears on:ComputeClusterStatus, FEStatus) +(Appears on:ComputeClusterStatus, FEStatus, MetaServiceStatus)

@@ -109,7 +109,7 @@ int32

CommonSpec

-(Appears on:ComputeCluster, FeSpec) +(Appears on:ComputeCluster, FeSpec, MetaService)

@@ -524,6 +524,17 @@ string +suspendReplicas
+ +int32 + + + +

suspend replicas display the replicas of compute cluster before resume.

+ + + + replicas
int32 @@ -587,45 +598,6 @@ key in configMap’s data is file name.

-

DisMS -

-

-(Appears on:DorisDisaggregatedClusterSpec) -

-
-
- - - - - - - - - - - - - - - - - -
FieldDescription
-namespace
- -string - -		 -

Namespace specify the namespace of metaservice deployed.

-
-name
- -string - -		 -

Name specify the name of metaservice resource.

-

DisaggregatedComponentType (string alias)

@@ -684,10 +656,10 @@ configmap have to config, please reference the doc.

-disMS
+metaService
- -DisMS + +MetaService @@ -767,10 +739,10 @@ configmap have to config, please reference the doc.

-disMS
+metaService
- -DisMS + +MetaService @@ -834,24 +806,15 @@ string -msEndpoint
- -string - - - -

the ms address for store meta of disaggregated cluster.

- - - - -msToken
+metaServiceStatus
-string + +MetaServiceStatus + -

the token for access ms service.

+

describe the metaservice status now.

@@ -950,6 +913,47 @@ map[string]string +

FDB +

+

+(Appears on:MetaService) +

+
+
+ + + + + + + + + + + + + + + + + +
FieldDescription
+address
+ +string + +		 +

if fdb directly deployed in machine, please add fdb access address which generated in ‘etc/foundationdb/fdb.cluster’ default.

+
+configMapNamespaceName
+ + +NamespaceName + + +		 +

if fdb deployed in kubernetes by fdb-kubernetes-operator, please specify the namespace and configmap’s name generated by fdb-kubernetes-operator in deployed fdbcluster namespace.

+

FEStatus

@@ -1043,6 +1047,153 @@ CommonSpec

+

MetaService +

+

+(Appears on:DorisDisaggregatedClusterSpec) +

+
+
+ + + + + + + + + + + + + + + + + +
FieldDescription
+CommonSpec
+ + +CommonSpec + + +		 +

+(Members of CommonSpec are embedded into this type.) +

+
+fdb
+ + +FDB + + +		 +

specify the address of fdb that used by doris Compute-storage decoupled cluster.

+
+

MetaServiceStatus +

+

+(Appears on:DorisDisaggregatedClusterStatus) +

+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+phase
+ + +Phase + + +		 +

Phase represent the stage of reconciling.

+
+availableStatus
+ + +AvailableStatus + + +		 +

AvailableStatus represents the metaservice available or not.

+
+metaServiceEndpoint
+ +string + +		 +

the meta service address for store meta of disaggregated cluster.

+
+msToken
+ +string + +		 +

the token for access ms service.

+
+

NamespaceName +

+

+(Appears on:FDB) +

+
+
+ + + + + + + + + + + + + + + + + +
FieldDescription
+namespace
+ +string + +		 +
+name
+ +string + +		 +

PersistentVolume

@@ -1113,7 +1264,7 @@ It only takes effect in the first configuration and cannot be added or modified

Phase (string alias)

-(Appears on:ComputeClusterStatus, FEStatus) +(Appears on:ComputeClusterStatus, FEStatus, MetaServiceStatus)

@@ -1132,6 +1283,17 @@ It only takes effect in the first configuration and cannot be added or modified

"Reconciling"

Creating represents service in creating stage.

+

"ResumeFailed"

+ +

"ScaleDownFailed"

+ +

"Scaling"

+

Scaling represents service in Scaling.

+ +

"SuspendFailed"

+ +

"Suspended"

+

"Upgrading"

Upgrading represents the spec of the service changed, service in smoothing upgrade.

@@ -1284,5 +1446,5 @@ string

Generated with gen-crd-api-reference-docs -on git commit 7d8fdc8. +on git commit 711e5a4.

diff --git a/pkg/common/utils/disaggregated_ms/ms_http/http.go b/pkg/common/utils/disaggregated_ms/ms_http/http.go index b4c3e38d..dba150c1 100644 --- a/pkg/common/utils/disaggregated_ms/ms_http/http.go +++ b/pkg/common/utils/disaggregated_ms/ms_http/http.go @@ -20,18 +20,22 @@ package ms_http import ( "bytes" "fmt" + dv1 "github.com/selectdb/doris-operator/api/disaggregated/cluster/v1" "github.com/selectdb/doris-operator/pkg/common/utils/disaggregated_ms/ms_meta" "io" "k8s.io/apimachinery/pkg/util/json" "net/http" + "strings" ) const ( - CREATE_INSTANCE_PREFIX_TEMPLATE = `http://%s/MetaService/http/create_instance?token=%s` - DELETE_INSTANCE_PREFIX_TEMPLATE = `http://%s/MetaService/http/drop_instance?token=%s` - GET_INSTANCE_PREFIX_TEMPLATE = `http://%s/MetaService/http/get_instance?token=%s&instance_id=%s` - DROP_NODE_PREFIX_TEMPLATE = `http://%s/MetaService/http/drop_node?token=%s` - GET_CLUSTER_PREFIX_TEMPLATE = `http://%s/MetaService/http/get_cluster?token=%s` + CREATE_INSTANCE_PREFIX_TEMPLATE = `http://%s/MetaService/http/create_instance?token=%s` + DELETE_INSTANCE_PREFIX_TEMPLATE = `http://%s/MetaService/http/drop_instance?token=%s` + GET_INSTANCE_PREFIX_TEMPLATE = `http://%s/MetaService/http/get_instance?token=%s&instance_id=%s` + DROP_NODE_PREFIX_TEMPLATE = `http://%s/MetaService/http/drop_node?token=%s` + GET_CLUSTER_PREFIX_TEMPLATE = `http://%s/MetaService/http/get_cluster?token=%s` + SET_CLUSTER_STATUS_PREFIX_TEMPLATE = `http://%s/MetaService/http/set_cluster_status?token=%s` + DROP_CLUSTER_STATUS_PREFIX_TEMPLATE = `http://%s/MetaService/http/drop_cluster?token=%s` ) //realize the metaservice interface @@ -142,6 +146,22 @@ func GetFECluster(endpoint, token, instanceId, cloudUniqueId string) ([]*NodeInf return mr.MSResponseResultNodesToNodeInfos() } +func GetBECluster(endpoint, token, cloudUniqueId, clusterId string) ([]*NodeInfo, error) { + param := map[string]interface{}{ + "cloud_unique_id": cloudUniqueId, + "cluster_id": clusterId, + } + str, _ := json.Marshal(param) + addr := fmt.Sprintf(GET_CLUSTER_PREFIX_TEMPLATE, endpoint, token) + + mr, err := putRequest(addr, str) + if err != nil { + return nil, fmt.Errorf("GetBECluster putRequest failed: %w", err) + } + + return mr.MSResponseResultNodesToNodeInfos() +} + func DropFENodes(endpoint, token, instanceID string, nodes []*NodeInfo) (*MSResponse, error) { cluster := Cluster{ ClusterName: FeClusterName, @@ -176,13 +196,71 @@ func dropNodesFromSpecifyCluster(endpoint, token, instanceID string, cluster Clu } +// DropBENodes dropNodesFromSpecifyCluster drop all nodes of specify cluster from ms +func DropBENodes(endpoint, token, instanceID string, cluster Cluster) (*MSResponse, error) { + mr, err := dropNodesFromSpecifyCluster(endpoint, token, instanceID, cluster) + if err != nil { + return mr, fmt.Errorf("DropBENodes dropNodesFromSpecifyCluster failed: %w", err) + } + return mr, nil +} + // suspend cluster -func SuspendComputeCluster() (*MSResponse, error) { - //TODO: suspend compute cluster - return nil, nil +func SuspendComputeCluster(endpoint, token, instanceID, clusterID string) error { + response, err := SetClusterStatus(endpoint, token, instanceID, clusterID, "SUSPENDED") + if err != nil { + return fmt.Errorf("SuspendComputeCluster SetClusterStatus failed: %w", err) + } + if response.Code != SuccessCode && !strings.Contains(response.Msg, "original cluster is SUSPENDED") { + return fmt.Errorf("SuspendComputeCluster SetClusterStatus failed: %s", response.Msg) + } + return nil } -func DropComputeCluster() (*MSResponse, error) { - //TODO: drop compute cluster - return nil, nil +func ResumeComputeCluster(endpoint, token, instanceID, clusterID string) error { + response, err := SetClusterStatus(endpoint, token, instanceID, clusterID, "NORMAL") + if err != nil { + return fmt.Errorf("ResumeComputeCluster SetClusterStatus failed: %w", err) + } + if response.Code != SuccessCode && !strings.Contains(response.Msg, "original cluster is NORMAL") { + return fmt.Errorf("ResumeComputeCluster SetClusterStatus failed: %s", response.Msg) + } + return nil +} + +// SetClusterStatus resume cluster +func SetClusterStatus(endpoint, token, instanceID, clusterID, status string) (*MSResponse, error) { + param := map[string]interface{}{ + "instance_id": instanceID, + "cluster": map[string]interface{}{ + "cluster_id": clusterID, + "cluster_status": status, + }, + } + str, _ := json.Marshal(param) + addr := fmt.Sprintf(SET_CLUSTER_STATUS_PREFIX_TEMPLATE, endpoint, token) + + mr, err := putRequest(addr, str) + if err != nil { + return nil, fmt.Errorf("SetClusterStatus putRequest failed: %w", err) + } + return mr, nil +} + +func DropComputeCluster(endpoint, token, instanceID string, ccs *dv1.ComputeClusterStatus) (*MSResponse, error) { + param := map[string]interface{}{ + "instance_id": instanceID, + "cluster": map[string]interface{}{ + "cluster_id": ccs.ClusterId, + "cluster_name": ccs.ComputeClusterName, + }, + } + str, _ := json.Marshal(param) + addr := fmt.Sprintf(DROP_CLUSTER_STATUS_PREFIX_TEMPLATE, endpoint, token) + + mr, err := putRequest(addr, str) + if err != nil { + return nil, fmt.Errorf("DropComputeCluster putRequest failed: %w", err) + } + return mr, nil } diff --git a/pkg/common/utils/disaggregated_ms/ms_http/metaservice.go b/pkg/common/utils/disaggregated_ms/ms_http/metaservice.go index b16dd2a4..1e0584a3 100644 --- a/pkg/common/utils/disaggregated_ms/ms_http/metaservice.go +++ b/pkg/common/utils/disaggregated_ms/ms_http/metaservice.go @@ -36,6 +36,7 @@ const ( FeClusterId = "RESERVED_CLUSTER_ID_FOR_SQL_SERVER" FeClusterName = "RESERVED_CLUSTER_NAME_FOR_SQL_SERVER" FeNodeType = "SQL" + BeNodeType = "COMPUTE" ) type NodeInfo struct { @@ -46,7 +47,7 @@ type NodeInfo struct { Status string `json:"-"` NodeType string `json:"node_type,omitempty"` EditLogPort int `json:"edit_log_port,omitempty"` - HeartbeatPort string `json:"heartbeat_port,omitempty"` + HeartbeatPort int `json:"heartbeat_port,omitempty"` Host string `json:"-"` } @@ -64,9 +65,13 @@ type MSRequest struct { func (mr *MSResponse) MSResponseResultNodesToNodeInfos() ([]*NodeInfo, error) { + if mr.Code != SuccessCode { + return nil, errors.New("MSResponseResultNodesToNodeInfos response code is not OKļ¼Œcode is: " + mr.Code + ", msg: " + mr.Msg + "") + } + nodes, ok := mr.Result["nodes"] if !ok { - return nil, errors.New("MSResponseResultNodes is not exist") + return nil, errors.New("MSResponseResultNodesToNodeInfos get nodes failed") } jsonStr, err := json.Marshal(nodes) diff --git a/pkg/common/utils/k8s/client.go b/pkg/common/utils/k8s/client.go index a1f084cb..43d8f7e9 100644 --- a/pkg/common/utils/k8s/client.go +++ b/pkg/common/utils/k8s/client.go @@ -22,7 +22,6 @@ import ( "errors" "fmt" "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2" - dv1 "github.com/selectdb/doris-operator/api/disaggregated/cluster/v1" mv1 "github.com/selectdb/doris-operator/api/disaggregated/metaservice/v1" dorisv1 "github.com/selectdb/doris-operator/api/doris/v1" "github.com/selectdb/doris-operator/pkg/common/utils" @@ -325,46 +324,6 @@ func SetDorisClusterPhase( return k8sclient.Status().Update(ctx, &edcr) } -func SetClusterPhase( - ctx context.Context, - k8sclient client.Client, - ddcName, namespace string, - phase dv1.Phase, - componentType dv1.DisaggregatedComponentType, - ccStsNames []string, -) error { - var edcr dv1.DorisDisaggregatedCluster - if err := k8sclient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: ddcName}, &edcr); err != nil { - return err - } - isStatusEqual := true - switch componentType { - case dv1.DisaggregatedFE: - isStatusEqual = edcr.Status.FEStatus.Phase == phase - edcr.Status.FEStatus.Phase = phase - case dv1.DisaggregatedBE: - for i, ccs := range edcr.Status.ComputeClusterStatuses { - name := ccs.StatefulsetName - for _, ccStsName := range ccStsNames { - if ccStsName == name { - if ccs.Phase != phase { - isStatusEqual = false - } - edcr.Status.ComputeClusterStatuses[i].Phase = phase - } - } - } - default: - klog.Infof("SetClusterPhase not support type=%s", componentType) - return nil - } - if isStatusEqual { - klog.Infof("UpdateDDCPhase will not change cluster %s Phase, it is already %s ,DDC name: %s, namespace: %s,", componentType, phase, ddcName, namespace) - return nil - } - return k8sclient.Status().Update(ctx, &edcr) -} - func GetDisaggregatedConfigMaps(ctx context.Context, k8scient client.Client, namespace string, cms []mv1.ConfigMap) ([]*corev1.ConfigMap, error) { var configMaps []*corev1.ConfigMap errMessage := "" diff --git a/pkg/controller/disaggregated_cluster_controller.go b/pkg/controller/disaggregated_cluster_controller.go index 950fd5f2..ae0096d0 100644 --- a/pkg/controller/disaggregated_cluster_controller.go +++ b/pkg/controller/disaggregated_cluster_controller.go @@ -299,6 +299,7 @@ func (dc *DisaggregatedClusterReconciler) reconcileSub(ctx context.Context, ddc // when spec revert by operator should update cr or directly update status. func (dc *DisaggregatedClusterReconciler) updateObjectORStatus(ctx context.Context, ddc *dv1.DorisDisaggregatedCluster, preHv string) (ctrl.Result, error) { postHv := hash.HashObject(ddc.Spec) + deepCopyDDC := ddc.DeepCopy() if preHv != postHv { var eddc dv1.DorisDisaggregatedCluster if err := dc.Get(ctx, types.NamespacedName{Namespace: ddc.Namespace, Name: ddc.Name}, &eddc); err == nil || !apierrors.IsNotFound(err) { @@ -308,24 +309,19 @@ func (dc *DisaggregatedClusterReconciler) updateObjectORStatus(ctx context.Conte } if err := dc.Update(ctx, ddc); err != nil { klog.Errorf("disaggreatedClusterReconciler update DorisDisaggregatedCluster namespace %s name %s failed, err=%s", ddc.Namespace, ddc.Name, err.Error()) - return ctrl.Result{}, err + //return ctrl.Result{}, err } - - //if cr updated, update cr. or update status. - return ctrl.Result{}, nil } - - return dc.updateDorisDisaggregatedClusterStatus(ctx, ddc) + return dc.updateDorisDisaggregatedClusterStatus(ctx, deepCopyDDC) } func (dc *DisaggregatedClusterReconciler) updateDorisDisaggregatedClusterStatus(ctx context.Context, ddc *dv1.DorisDisaggregatedCluster) (ctrl.Result, error) { var eddc dv1.DorisDisaggregatedCluster - if err := dc.Get(ctx, types.NamespacedName{Namespace: ddc.Namespace, Name: ddc.Name}, &eddc); err != nil { - return ctrl.Result{}, err - } - - ddc.Status.DeepCopyInto(&eddc.Status) if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + if err := dc.Get(ctx, types.NamespacedName{Namespace: ddc.Namespace, Name: ddc.Name}, &eddc); err != nil { + return err + } + ddc.Status.DeepCopyInto(&eddc.Status) return dc.Status().Update(ctx, &eddc) }); err != nil { klog.Errorf("updateDorisDisaggregatedClusterStatus update status failed err: %s", err.Error()) diff --git a/pkg/controller/sub_controller/disaggregated_cluster/computeclusters/controller.go b/pkg/controller/sub_controller/disaggregated_cluster/computeclusters/controller.go index 556368cc..1b68c1ab 100644 --- a/pkg/controller/sub_controller/disaggregated_cluster/computeclusters/controller.go +++ b/pkg/controller/sub_controller/disaggregated_cluster/computeclusters/controller.go @@ -19,8 +19,11 @@ package computeclusters import ( "context" + "encoding/json" "errors" + "fmt" dv1 "github.com/selectdb/doris-operator/api/disaggregated/cluster/v1" + "github.com/selectdb/doris-operator/pkg/common/utils" "github.com/selectdb/doris-operator/pkg/common/utils/disaggregated_ms/ms_http" "github.com/selectdb/doris-operator/pkg/common/utils/k8s" "github.com/selectdb/doris-operator/pkg/common/utils/resource" @@ -34,6 +37,8 @@ import ( "regexp" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "strconv" + "strings" "sync" ) @@ -125,14 +130,9 @@ func (dccs *DisaggregatedComputeClustersController) feAvailable(ddc *dv1.DorisDi } func (dccs *DisaggregatedComputeClustersController) computeClusterSync(ctx context.Context, ddc *dv1.DorisDisaggregatedCluster, cc *dv1.ComputeCluster) (*sc.Event, error) { - //1. generate resources. - //2. initial compute cluster status. - //3. sync resources. - //TODO: 3. judge suspend - if cc.Replicas != nil && *cc.Replicas == 0 { - ms_http.SuspendComputeCluster() + if cc.Replicas == nil { + cc.Replicas = resource.GetInt32Pointer(1) } - cvs := dccs.GetConfigValuesFromConfigMaps(ddc.Namespace, resource.BE_RESOLVEKEY, cc.CommonSpec.ConfigMaps) st := dccs.NewStatefulset(ddc, cc, cvs) svc := dccs.newService(ddc, cc, cvs) @@ -143,7 +143,7 @@ func (dccs *DisaggregatedComputeClustersController) computeClusterSync(ctx conte klog.Errorf("disaggregatedComputeClustersController reconcile service namespace %s name %s failed, err=%s", svc.Namespace, svc.Name, err.Error()) return event, err } - event, err = dccs.reconcileStatefulset(ctx, st) + event, err = dccs.reconcileStatefulset(ctx, st, ddc, cc) if err != nil { klog.Errorf("disaggregatedComputeClustersController reconcile statefulset namespace %s name %s failed, err=%s", st.Namespace, st.Name, err.Error()) } @@ -151,7 +151,7 @@ func (dccs *DisaggregatedComputeClustersController) computeClusterSync(ctx conte return event, err } -func (dccs *DisaggregatedComputeClustersController) reconcileStatefulset(ctx context.Context, st *appv1.StatefulSet) (*sc.Event, error) { +func (dccs *DisaggregatedComputeClustersController) reconcileStatefulset(ctx context.Context, st *appv1.StatefulSet, cluster *dv1.DorisDisaggregatedCluster, cc *dv1.ComputeCluster) (*sc.Event, error) { var est appv1.StatefulSet if err := dccs.K8sclient.Get(ctx, types.NamespacedName{Namespace: st.Namespace, Name: st.Name}, &est); apierrors.IsNotFound(err) { if err = k8s.CreateClientObject(ctx, dccs.K8sclient, st); err != nil { @@ -165,6 +165,27 @@ func (dccs *DisaggregatedComputeClustersController) reconcileStatefulset(ctx con return nil, err } + var ccStatus *dv1.ComputeClusterStatus + + for i := range cluster.Status.ComputeClusterStatuses { + if cluster.Status.ComputeClusterStatuses[i].ClusterId == cc.ClusterId { + ccStatus = &cluster.Status.ComputeClusterStatuses[i] + break + } + } + scaleType := getScaleType(st, &est, ccStatus.Phase) + + if scaleType == "resume" { + if ccStatus.SuspendReplicas != *(st.Spec.Replicas) { + errMessage := fmt.Sprintf("ResumeComputeCluster configuration is abnormal. The replicas of resumes(%d) is not equal to the replicas of suspends(%d).", *st.Spec.Replicas, ccStatus.SuspendReplicas) + return &sc.Event{ + Type: sc.EventNormal, + Reason: sc.CCResumeReplicasInconsistency, + Message: errMessage, + }, errors.New(errMessage) + } + } + if err := k8s.ApplyStatefulSet(ctx, dccs.K8sclient, st, func(st, est *appv1.StatefulSet) bool { return resource.StatefulsetDeepEqualWithOmitKey(st, est, dv1.DisaggregatedSpecHashValueAnnotation, true, false) }); err != nil { @@ -172,30 +193,88 @@ func (dccs *DisaggregatedComputeClustersController) reconcileStatefulset(ctx con return &sc.Event{Type: sc.EventWarning, Reason: sc.CCApplyResourceFailed, Message: err.Error()}, err } + switch scaleType { + case "resume": + err := ms_http.ResumeComputeCluster(cluster.Status.MetaServiceStatus.MetaServiceEndpoint, cluster.Status.MetaServiceStatus.MsToken, cluster.Status.InstanceId, cc.ClusterId) + ccStatus.SuspendReplicas = 0 + if err != nil { + ccStatus.Phase = dv1.ResumeFailed + klog.Errorf("computeClusterSync ResumeComputeCluster response failed , err: %s", err.Error()) + return &sc.Event{ + Type: sc.EventNormal, + Reason: sc.CCResumeStatusRequestFailed, + Message: "ResumeComputeCluster request of disaggregated BE failed: " + err.Error(), + }, err + } + ccStatus.Phase = dv1.Scaling + case "scaleDown": + if err := dccs.dropCCFromHttpClient(cluster, cc); err != nil { + ccStatus.Phase = dv1.ScaleDownFailed + klog.Errorf("ScaleDownBE failed, err:%s ", err.Error()) + return &sc.Event{Type: sc.EventWarning, Reason: sc.CCHTTPFailed, Message: err.Error()}, + err + } + ccStatus.Phase = dv1.Scaling + case "suspend": + err := ms_http.SuspendComputeCluster(cluster.Status.MetaServiceStatus.MetaServiceEndpoint, cluster.Status.MetaServiceStatus.MsToken, cluster.Status.InstanceId, cc.ClusterId) + if err != nil { + ccStatus.Phase = dv1.SuspendFailed + klog.Errorf("computeClusterSync SuspendComputeCluster response failed , err: %s", err.Error()) + return &sc.Event{ + Type: sc.EventNormal, + Reason: sc.CCSuspendStatusRequestFailed, + Message: "SuspendComputeCluster request of disaggregated BE failed: " + err.Error(), + }, err + } + ccStatus.SuspendReplicas = *est.Spec.Replicas + ccStatus.Phase = dv1.Suspended + } + return nil, nil } +func getScaleType(st, est *appv1.StatefulSet, phase dv1.Phase) string { + if (*(st.Spec.Replicas) > *(est.Spec.Replicas) && *(est.Spec.Replicas) == 0) || phase == dv1.ResumeFailed { + return "resume" + } + + if (*(st.Spec.Replicas) < *(est.Spec.Replicas) && *(st.Spec.Replicas) > 0) || phase == dv1.ScaleDownFailed { + return "scaleDown" + } + + if (*(st.Spec.Replicas) < *(est.Spec.Replicas) && *(st.Spec.Replicas) == 0) || phase == dv1.SuspendFailed { + return "suspend" + } + return "" +} + // initial compute cluster status before sync resources. status changing with sync steps, and generate the last status by classify pods. func (dccs *DisaggregatedComputeClustersController) initialCCStatus(ddc *dv1.DorisDisaggregatedCluster, cc *dv1.ComputeCluster) { ccss := ddc.Status.ComputeClusterStatuses - for i, _ := range ccss { - if ccss[i].ComputeClusterName == cc.Name || ccss[i].ClusterId == cc.ClusterId { - ccss[i].Phase = dv1.Reconciling - return - } - } - - ccs := dv1.ComputeClusterStatus{ + defaultStatus := dv1.ComputeClusterStatus{ Phase: dv1.Reconciling, ComputeClusterName: cc.Name, ClusterId: cc.ClusterId, + StatefulsetName: ddc.GetCCStatefulsetName(cc), + ServiceName: ddc.GetCCServiceName(cc), //set for status updated. Replicas: *cc.Replicas, } - if ddc.Status.ComputeClusterStatuses == nil { - ddc.Status.ComputeClusterStatuses = []dv1.ComputeClusterStatus{} + + for i := range ccss { + if ccss[i].ClusterId == cc.ClusterId { + if ccss[i].Phase == dv1.ScaleDownFailed || ccss[i].Phase == dv1.Suspended || + ccss[i].Phase == dv1.SuspendFailed || ccss[i].Phase == dv1.ResumeFailed || + ccss[i].Phase == dv1.Scaling { + defaultStatus.Phase = ccss[i].Phase + } + defaultStatus.SuspendReplicas = ccss[i].SuspendReplicas + ccss[i] = defaultStatus + return + } } - ddc.Status.ComputeClusterStatuses = append(ddc.Status.ComputeClusterStatuses, ccs) + + ddc.Status.ComputeClusterStatuses = append(ddc.Status.ComputeClusterStatuses, defaultStatus) } // clusterId and cloudUniqueId is not allowed update, when be mistakenly modified on these fields, operator should revert it by status fields. @@ -275,9 +354,10 @@ func (dccs *DisaggregatedComputeClustersController) ClearResources(ctx context.C ddc := obj.(*dv1.DorisDisaggregatedCluster) var clearCCs []dv1.ComputeClusterStatus var eCCs []dv1.ComputeClusterStatus + for i, ccs := range ddc.Status.ComputeClusterStatuses { for _, cc := range ddc.Spec.ComputeClusters { - if ccs.ComputeClusterName == cc.Name || ccs.ClusterId == cc.ClusterId { + if ccs.ClusterId == cc.ClusterId { eCCs = append(eCCs, ddc.Status.ComputeClusterStatuses[i]) goto NoNeedAppend } @@ -288,7 +368,8 @@ func (dccs *DisaggregatedComputeClustersController) ClearResources(ctx context.C NoNeedAppend: } - for i, ccs := range clearCCs { + for i := range clearCCs { + ccs := clearCCs[i] cleared := true if err := k8s.DeleteStatefulset(ctx, dccs.K8sclient, ddc.Namespace, ccs.StatefulsetName); err != nil { cleared = false @@ -304,18 +385,33 @@ func (dccs *DisaggregatedComputeClustersController) ClearResources(ctx context.C if !cleared { eCCs = append(eCCs, clearCCs[i]) } else { - //TODO: 12. drop compute cluster from meta - ms_http.DropComputeCluster() + // drop compute cluster from meta + response, err := ms_http.DropComputeCluster(ddc.Status.MetaServiceStatus.MetaServiceEndpoint, ddc.Status.MetaServiceStatus.MsToken, ddc.Status.InstanceId, &ccs) + if err != nil { + klog.Errorf("computeClusterSync ClearResources DropComputeCluster response failed , response: %s", err.Error()) + dccs.K8srecorder.Event(ddc, string(sc.EventWarning), string(sc.CCHTTPFailed), "DropComputeCluster request failed: "+err.Error()) + } + if response.Code != ms_http.SuccessCode { + jsonData, _ := json.Marshal(response) + klog.Errorf("computeClusterSync ClearResources DropComputeCluster response failed , response: %s", jsonData) + dccs.K8srecorder.Event(ddc, string(sc.EventWarning), string(sc.CCHTTPFailed), "DropComputeCluster request failed: "+response.Msg) + } + } + } - //TODO:13. drop pvcs - for _, cc := range eCCs { - dccs.ClearStatefulsetUnusedPVCs(cc) + for i := range eCCs { + err := dccs.ClearStatefulsetUnusedPVCs(ctx, ddc, eCCs[i]) + if err != nil { + klog.Errorf("disaggregatedComputeClustersController ClearStatefulsetUnusedPVCs clear whole ComputeCluster PVC failed, err=%s", err.Error()) + } } - //TODO:13. drop pvcs - for _, cc := range clearCCs { - dccs.ClearStatefulsetUnusedPVCs(cc) + for i := range clearCCs { + err := dccs.ClearStatefulsetUnusedPVCs(ctx, ddc, clearCCs[i]) + if err != nil { + klog.Errorf("disaggregatedComputeClustersController ClearStatefulsetUnusedPVCs clear part ComputeCluster PVC failed, err=%s", err.Error()) + } } ddc.Status.ComputeClusterStatuses = eCCs @@ -323,8 +419,69 @@ func (dccs *DisaggregatedComputeClustersController) ClearResources(ctx context.C return true, nil } -func (dccs *DisaggregatedComputeClustersController) ClearStatefulsetUnusedPVCs(cc dv1.ComputeClusterStatus) { +// ClearStatefulsetUnusedPVCs +// 1.delete unused pvc skip cluster is Suspend +// 2.delete unused pvc for statefulset +// 3.delete pvc if not used by any statefulset +func (dccs *DisaggregatedComputeClustersController) ClearStatefulsetUnusedPVCs(ctx context.Context, ddc *dv1.DorisDisaggregatedCluster, ccs dv1.ComputeClusterStatus) error { + var cc *dv1.ComputeCluster + for i := range ddc.Spec.ComputeClusters { + if ddc.Spec.ComputeClusters[i].ClusterId == ccs.ClusterId { + cc = &ddc.Spec.ComputeClusters[i] + } + } + + currentPVCs := corev1.PersistentVolumeClaimList{} + pvcMap := make(map[string]*corev1.PersistentVolumeClaim) + pvcLabels := dccs.newCCPodsSelector(ddc.Name, ccs.ClusterId) + + if err := dccs.K8sclient.List(ctx, ¤tPVCs, client.InNamespace(ddc.Namespace), client.MatchingLabels(pvcLabels)); err != nil { + dccs.K8srecorder.Event(ddc, string(sc.EventWarning), sc.PVCListFailed, fmt.Sprintf("DisaggregatedComputeClustersController ClearStatefulsetUnusedPVCs list pvc failed:%s!", err.Error())) + return err + } + + for i := range currentPVCs.Items { + pvcMap[currentPVCs.Items[i].Name] = ¤tPVCs.Items[i] + } + + if cc != nil { + replicas := int(*cc.Replicas) + stsName := ddc.GetCCStatefulsetName(cc) + cvs := dccs.GetConfigValuesFromConfigMaps(ddc.Namespace, resource.BE_RESOLVEKEY, cc.CommonSpec.ConfigMaps) + paths, _ := dccs.getCacheMaxSizeAndPaths(cvs) + + if ccs.Phase == dv1.Suspended || ccs.Phase == dv1.SuspendFailed || replicas == 0 { + return nil + } + + var reservePVCNameList []string + + for i := 0; i < replicas; i++ { + iStr := strconv.Itoa(i) + reservePVCNameList = append(reservePVCNameList, resource.BuildPVCName(stsName, iStr, LogStoreName)) + for j := 0; j < len(paths); j++ { + jStr := strconv.Itoa(j) + reservePVCNameList = append(reservePVCNameList, resource.BuildPVCName(stsName, iStr, StorageStorePreName+jStr)) + } + } + + for _, pvcName := range reservePVCNameList { + if _, ok := pvcMap[pvcName]; ok { + delete(pvcMap, pvcName) + } + } + } + + var mergeError error + for _, claim := range pvcMap { + if err := k8s.DeletePVC(ctx, dccs.K8sclient, claim.Namespace, claim.Name, pvcLabels); err != nil { + dccs.K8srecorder.Event(ddc, string(sc.EventWarning), sc.PVCDeleteFailed, err.Error()) + klog.Errorf("ClearStatefulsetUnusedPVCs deletePVCs failed: namespace %s, name %s delete pvc %s, err: %s .", claim.Namespace, claim.Name, claim.Name, err.Error()) + mergeError = utils.MergeError(mergeError, err) + } + } + return mergeError } func (dccs *DisaggregatedComputeClustersController) GetControllerName() string { @@ -405,3 +562,53 @@ func (dccs *DisaggregatedComputeClustersController) updateCCStatus(ddc *dv1.Dori } return nil } + +func (dfc *DisaggregatedComputeClustersController) dropCCFromHttpClient(cluster *dv1.DorisDisaggregatedCluster, cc *dv1.ComputeCluster) error { + ccReplica := cc.Replicas + + // drop be can also use the unique id of fe + unionId := "1:" + cluster.GetInstanceId() + ":" + cluster.GetFEStatefulsetName() + "-0" + + ccNodes, err := ms_http.GetBECluster(cluster.Status.MetaServiceStatus.MetaServiceEndpoint, cluster.Status.MetaServiceStatus.MsToken, unionId, cc.ClusterId) + if err != nil { + klog.Errorf("dropCCFromHttpClient GetBECluster failed, err:%s ", err.Error()) + return err + } + + var dropNodes []*ms_http.NodeInfo + for _, node := range ccNodes { + splitCloudUniqueIDArr := strings.Split(node.CloudUniqueID, "-") + podNum, err := strconv.Atoi(splitCloudUniqueIDArr[len(splitCloudUniqueIDArr)-1]) + if err != nil { + klog.Errorf("splitCloudUniqueIDArr can not split CloudUniqueID : %s,err:%s", node.CloudUniqueID, err.Error()) + return err + } + if podNum >= int(*ccReplica) { + dropNodes = append(dropNodes, node) + } + } + if len(dropNodes) == 0 { + return nil + } + + reqCluster := ms_http.Cluster{ + ClusterName: cc.Name, + ClusterID: cc.ClusterId, + Type: ms_http.BeNodeType, + Nodes: dropNodes, + } + + specifyCluster, err := ms_http.DropBENodes(cluster.Status.MetaServiceStatus.MetaServiceEndpoint, cluster.Status.MetaServiceStatus.MsToken, cluster.GetInstanceId(), reqCluster) + if err != nil { + klog.Errorf("dropCCFromHttpClient DropBENodes failed, err:%s ", err.Error()) + return err + } + + if specifyCluster.Code != ms_http.SuccessCode { + jsonData, _ := json.Marshal(specifyCluster) + klog.Errorf("dropCCFromHttpClient DropBENodes response failed , response: %s", jsonData) + return err + } + + return nil +} diff --git a/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/controller.go b/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/controller.go index 7fde4ebd..ba4da111 100644 --- a/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/controller.go +++ b/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/controller.go @@ -99,7 +99,7 @@ func (dfc *DisaggregatedFEController) Sync(ctx context.Context, obj client.Objec return err } - if *(ddc.Spec.FeSpec.Replicas) < DefaultFeReplicaNumber { + if ddc.Spec.FeSpec.Replicas == nil || *(ddc.Spec.FeSpec.Replicas) < DefaultFeReplicaNumber { klog.Errorf("disaggregatedFEController sync disaggregatedDorisCluster namespace=%s,name=%s ,The number of disaggregated fe replicas is illegal and has been corrected to the default value %d", ddc.Namespace, ddc.Name, DefaultFeReplicaNumber) dfc.K8srecorder.Event(ddc, string(sc.EventNormal), string(sc.FESpecSetError), "The number of disaggregated fe replicas is illegal and has been corrected to the default value 2") ddc.Spec.FeSpec.Replicas = &DefaultFeReplicaNumber @@ -243,7 +243,7 @@ func (dfc *DisaggregatedFEController) UpdateComponentStatus(obj client.Object) e // initial fe status before sync resources. status changing with sync steps, and generate the last status by classify pods. func (dfc *DisaggregatedFEController) initialFEStatus(ddc *dv1.DorisDisaggregatedCluster) { - if ddc.Status.FEStatus.Phase == dv1.Reconciling { + if ddc.Status.FEStatus.Phase == dv1.Reconciling || ddc.Status.FEStatus.Phase == dv1.ScaleDownFailed || ddc.Status.FEStatus.Phase == dv1.Scaling { return } feStatus := dv1.FEStatus{ @@ -268,20 +268,7 @@ func (dfc *DisaggregatedFEController) reconcileStatefulset(ctx context.Context, } // fe scale check and set FEStatus phase - if cluster.Spec.FeSpec.Replicas == nil { - cluster.Spec.FeSpec.Replicas = resource.GetInt32Pointer(0) - } scaleNumber := *(cluster.Spec.FeSpec.Replicas) - *(est.Spec.Replicas) - if scaleNumber != 0 { // set fe Phase as Reconciling - cluster.Status.FEStatus.Phase = dv1.Reconciling - // In Reconcile, it is possible that the status cannot be updated in time, - // resulting in an error in the status judgment based on the last status, - // so the status will be forced to modify here - if err := k8s.SetClusterPhase(ctx, dfc.K8sclient, cluster.Name, cluster.Namespace, dv1.Reconciling, dv1.DisaggregatedFE, nil); err != nil { - klog.Errorf("SetDDCPhase 'Reconciling' failed err:%s ", err.Error()) - return &sc.Event{Type: sc.EventWarning, Reason: sc.FEStatusUpdateFailed, Message: err.Error()}, err - } - } // apply fe StatefulSet if err := k8s.ApplyStatefulSet(ctx, dfc.K8sclient, st, func(st, est *appv1.StatefulSet) bool { @@ -292,12 +279,14 @@ func (dfc *DisaggregatedFEController) reconcileStatefulset(ctx context.Context, } // if fe scale, drop fe node by http - if scaleNumber < 0 || cluster.Status.FEStatus.Phase == dv1.Reconciling { + if scaleNumber < 0 || cluster.Status.FEStatus.Phase == dv1.ScaleDownFailed { if err := dfc.dropFEFromHttpClient(cluster); err != nil { + cluster.Status.FEStatus.Phase = dv1.ScaleDownFailed klog.Errorf("ScaleDownFE failed, err:%s ", err.Error()) return &sc.Event{Type: sc.EventWarning, Reason: sc.FEHTTPFailed, Message: err.Error()}, err } + cluster.Status.FEStatus.Phase = dv1.Scaling } //dropped @@ -308,7 +297,7 @@ func (dfc *DisaggregatedFEController) reconcileStatefulset(ctx context.Context, func (dfc *DisaggregatedFEController) dropFEFromHttpClient(cluster *dv1.DorisDisaggregatedCluster) error { feReplica := cluster.Spec.FeSpec.Replicas - unionId := "1:" + cluster.GetInstanceId() + cluster.GetFEStatefulsetName() + "-0" + unionId := "1:" + cluster.GetInstanceId() + ":" + cluster.GetFEStatefulsetName() + "-0" feCluster, err := ms_http.GetFECluster(cluster.Status.MetaServiceStatus.MetaServiceEndpoint, cluster.Status.MetaServiceStatus.MsToken, cluster.GetInstanceId(), unionId) if err != nil { klog.Errorf("dropFEFromHttpClient GetFECluster failed, err:%s ", err.Error()) diff --git a/pkg/controller/sub_controller/disaggregated_cluster/metaservice/statefulset.go b/pkg/controller/sub_controller/disaggregated_cluster/metaservice/statefulset.go index ea51784a..f2e3c4ff 100644 --- a/pkg/controller/sub_controller/disaggregated_cluster/metaservice/statefulset.go +++ b/pkg/controller/sub_controller/disaggregated_cluster/metaservice/statefulset.go @@ -4,6 +4,7 @@ import ( "context" dv1 "github.com/selectdb/doris-operator/api/disaggregated/cluster/v1" "github.com/selectdb/doris-operator/pkg/common/utils/k8s" + "github.com/selectdb/doris-operator/pkg/common/utils/metadata" "github.com/selectdb/doris-operator/pkg/common/utils/resource" sc "github.com/selectdb/doris-operator/pkg/controller/sub_controller" appv1 "k8s.io/api/apps/v1" @@ -58,8 +59,10 @@ func (dms *DisaggregatedMSController) newStatefulset(ddc *dv1.DorisDisaggregated volumeClaimTemplates = append(volumeClaimTemplates, pvc) } + replicas := metadata.GetInt32Pointer(dv1.DefaultMetaserviceNumber) + func() { - st.Spec.Replicas = msSpec.Replicas + st.Spec.Replicas = replicas st.Spec.Selector = &metav1.LabelSelector{ MatchLabels: matchLabels, } diff --git a/pkg/controller/sub_controller/events.go b/pkg/controller/sub_controller/events.go index fc987066..61f5de4c 100644 --- a/pkg/controller/sub_controller/events.go +++ b/pkg/controller/sub_controller/events.go @@ -52,13 +52,15 @@ var ( FEStatefulsetDeleteFailed EventReason = "FEStatefulsetDeleteFailed" FEHTTPFailed EventReason = "FEHTTPResponseFailed" FEServiceDeleteFailed EventReason = "FEServiceDeleteFailed" - FEStatusUpdateFailed EventReason = "FEStatusUpdatedFailed" ComputeClustersEmpty EventReason = "CCsEmpty" + CCHTTPFailed EventReason = "CCHTTPResponseFailed" + CCSuspendStatusRequestFailed EventReason = "CCSuspendStatusRequestFailed" + CCResumeReplicasInconsistency EventReason = "CCSuspendReplicasInconsistency" + CCResumeStatusRequestFailed EventReason = "CCResumeStatusRequestFailed" CCUniqueIdentifierDuplicate EventReason = "CCUniqueIdentifierDuplicate" CCUniqueIdentifierNotMatchRegex EventReason = "CCUniqueIdentifierNotMatchRegex" CCCreateResourceFailed EventReason = "CCCreateResourceFailed" CCApplyResourceFailed EventReason = "CCApplyResourceFailed" - CCStatusUpdateFailed EventReason = "CCStatusUpdatedFailed" CCStatefulsetDeleteFailed EventReason = "CCStatefulsetDeleteFailed" CCServiceDeleteFailed EventReason = "CCServiceDeleteFailed" DisaggregatedMetaServiceGetFailed EventReason = "DisaggregatedMetaServiceGetFailed"