From 48126ffd3545fd39f35aa5606899cea779b3f5cf Mon Sep 17 00:00:00 2001 From: LavenderQAQ Date: Tue, 8 Aug 2023 11:44:02 +0000 Subject: [PATCH] refactor: define vertically updated interfaces for different implementations Signed-off-by: LavenderQAQ --- apis/apps/pub/inplace_update.go | 3 +- pkg/features/kruise_features.go | 5 +- .../inplaceupdate/inplace_update_defaults.go | 22 +++--- .../inplaceupdate/inplace_update_vertical.go | 76 +++++++++++++++++++ 4 files changed, 92 insertions(+), 14 deletions(-) create mode 100644 pkg/util/inplaceupdate/inplace_update_vertical.go diff --git a/apis/apps/pub/inplace_update.go b/apis/apps/pub/inplace_update.go index 01122b66fc..a26cf1a886 100644 --- a/apis/apps/pub/inplace_update.go +++ b/apis/apps/pub/inplace_update.go @@ -94,7 +94,8 @@ type InPlaceUpdateContainerBatch struct { // InPlaceUpdateContainerStatus records the statuses of the container that are mainly used // to determine whether the InPlaceUpdate is completed. type InPlaceUpdateContainerStatus struct { - ImageID string `json:"imageID,omitempty"` + ImageID string `json:"imageID,omitempty"` + Resource v1.ResourceRequirements `json:"resource,omitempty"` } // InPlaceUpdateStrategy defines the strategies for in-place update. diff --git a/pkg/features/kruise_features.go b/pkg/features/kruise_features.go index b8e2ffd66b..69bb60d026 100644 --- a/pkg/features/kruise_features.go +++ b/pkg/features/kruise_features.go @@ -108,7 +108,7 @@ const ( // ImagePullJobGate enable imagepulljob-controller execute ImagePullJob. ImagePullJobGate featuregate.Feature = "ImagePullJobGate" - + // InPlaceWorkloadVerticalScaling enable CloneSet/Advanced StatefulSet controller to support vertical scaling // of managed Pods. InPlaceWorkloadVerticalScaling featuregate.Feature = "InPlaceWorkloadVerticalScaling" @@ -136,11 +136,8 @@ var defaultFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ PreDownloadImageForDaemonSetUpdate: {Default: false, PreRelease: featuregate.Alpha}, CloneSetEventHandlerOptimization: {Default: false, PreRelease: featuregate.Alpha}, PreparingUpdateAsUpdate: {Default: false, PreRelease: featuregate.Alpha}, -<<<<<<< HEAD ImagePullJobGate: {Default: false, PreRelease: featuregate.Alpha}, -======= InPlaceWorkloadVerticalScaling: {Default: false, PreRelease: featuregate.Alpha}, ->>>>>>> feat: enhanced in-place update module to support vertical scaling } func init() { diff --git a/pkg/util/inplaceupdate/inplace_update_defaults.go b/pkg/util/inplaceupdate/inplace_update_defaults.go index 9cff5205fd..8fde6756f5 100644 --- a/pkg/util/inplaceupdate/inplace_update_defaults.go +++ b/pkg/util/inplaceupdate/inplace_update_defaults.go @@ -45,6 +45,8 @@ func SetOptionsDefaults(opts *UpdateOptions) *UpdateOptions { } if utilfeature.DefaultFeatureGate.Enabled(features.InPlaceWorkloadVerticalScaling) { + registerVerticalUpdate() + if opts.CalculateSpec == nil { opts.CalculateSpec = defaultCalculateInPlaceUpdateSpecWithVerticalUpdate } @@ -60,7 +62,6 @@ func SetOptionsDefaults(opts *UpdateOptions) *UpdateOptions { if opts.CheckContainersUpdateCompleted == nil { opts.CheckContainersUpdateCompleted = defaultCheckContainersInPlaceUpdateCompletedWithVerticalUpdate } - } else { if opts.CalculateSpec == nil { opts.CalculateSpec = defaultCalculateInPlaceUpdateSpec @@ -77,7 +78,6 @@ func SetOptionsDefaults(opts *UpdateOptions) *UpdateOptions { if opts.CheckContainersUpdateCompleted == nil { opts.CheckContainersUpdateCompleted = defaultCheckContainersInPlaceUpdateCompleted } - } return opts @@ -498,6 +498,8 @@ func checkAllContainersHashConsistent(pod *v1.Pod, runtimeContainerMetaSet *apps func defaultPatchUpdateSpecToPodWithVerticalUpdate(pod *v1.Pod, spec *UpdateSpec, state *appspub.InPlaceUpdateState) (*v1.Pod, error) { klog.V(5).Infof("Begin to in-place update pod %s/%s with update spec %v, state %v", pod.Namespace, pod.Name, util.DumpJSON(spec), util.DumpJSON(state)) + registerVerticalUpdate() + state.NextContainerImages = make(map[string]string) state.NextContainerRefMetadata = make(map[string]metav1.ObjectMeta) state.NextContainerResources = make(map[string]v1.ResourceRequirements) @@ -566,12 +568,7 @@ func defaultPatchUpdateSpecToPodWithVerticalUpdate(pod *v1.Pod, spec *UpdateSpec containersImageChanged.Insert(c.Name) } if resourceExists { - for key, quantity := range newResource.Limits { - c.Resources.Limits[key] = quantity - } - for key, quantity := range newResource.Requests { - c.Resources.Requests[key] = quantity - } + verticalUpdateOperator.UpdateResource(pod, c, &newResource) containersResourceChanged.Insert(c.Name) } } else { @@ -780,6 +777,8 @@ func defaultCalculateInPlaceUpdateSpecWithVerticalUpdate(oldRevision, newRevisio // If the imageID in containerStatuses has not been changed, we assume that kubelet has not updated // containers in Pod. func DefaultCheckInPlaceUpdateCompletedWithVerticalUpdate(pod *v1.Pod) error { + registerVerticalUpdate() + if _, isInGraceState := appspub.GetInPlaceUpdateGrace(pod); isInGraceState { return fmt.Errorf("still in grace period of in-place update") } @@ -798,6 +797,8 @@ func DefaultCheckInPlaceUpdateCompletedWithVerticalUpdate(pod *v1.Pod) error { } func defaultCheckContainersInPlaceUpdateCompletedWithVerticalUpdate(pod *v1.Pod, inPlaceUpdateState *appspub.InPlaceUpdateState) error { + registerVerticalUpdate() + runtimeContainerMetaSet, err := appspub.GetRuntimeContainerMetaSet(pod) if err != nil { return err @@ -840,7 +841,10 @@ func defaultCheckContainersInPlaceUpdateCompletedWithVerticalUpdate(pod *v1.Pod, return fmt.Errorf("container %s imageID not changed", cs.Name) } } - // TODO(LavenderQAQ): Check the vertical updating status of the container + // Determine whether the vertical update was successful by the resource values in the pod's spec and status + if !verticalUpdateOperator.IsUpdateCompleted(pod, &containerResources, &inPlaceUpdateState.LastContainerStatuses) { + return fmt.Errorf("container %s resources not changed", cs.Name) + } delete(inPlaceUpdateState.LastContainerStatuses, cs.Name) } } diff --git a/pkg/util/inplaceupdate/inplace_update_vertical.go b/pkg/util/inplaceupdate/inplace_update_vertical.go new file mode 100644 index 0000000000..d019d2b91a --- /dev/null +++ b/pkg/util/inplaceupdate/inplace_update_vertical.go @@ -0,0 +1,76 @@ +/* +Copyright 2023 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package inplaceupdate + +import ( + appspub "github.com/openkruise/kruise/apis/apps/pub" + v1 "k8s.io/api/core/v1" +) + +// For In-place workload vertical scaling +type VerticalUpdateInterface interface { + // Pass in the container to be modified and the expected resource values, + // as well as the information for the entire pod for compatibility with some internal vertical scaling implementations + UpdateResource(pod *v1.Pod, container *v1.Container, resource *v1.ResourceRequirements) + // To determine whether the container has been successfully vertical updated, + // pass in the expected resources of the container and its current status, + // as well as the information for the entire pod for compatibility with some internal vertical scaling implementations + IsUpdateCompleted(pod *v1.Pod, containerResources *map[string]v1.ResourceRequirements, containerStatuses *map[string]appspub.InPlaceUpdateContainerStatus) bool +} + +var verticalUpdateOperator VerticalUpdateInterface = nil + +// To register vertical update operations, +// you can register different vertical update implementations here +func registerVerticalUpdate() { + if verticalUpdateOperator == nil { + verticalUpdateOperator = &VerticalUpdate{} + } +} + +// VerticalUpdate represents the vertical scaling of k8s standard +type VerticalUpdate struct{} + +// UpdateResource implements vertical updates by directly modifying the container's resources, +// conforming to the k8s community standard +func (v *VerticalUpdate) UpdateResource(pod *v1.Pod, container *v1.Container, newResource *v1.ResourceRequirements) { + for key, quantity := range newResource.Limits { + container.Resources.Limits[key] = quantity + } + for key, quantity := range newResource.Requests { + container.Resources.Requests[key] = quantity + } +} + +// IsUpdateCompleted directly determines whether the current container is vertically updated by the spec and status of the container, +// which conforms to the k8s community standard +func (v *VerticalUpdate) IsUpdateCompleted(pod *v1.Pod, containerResources *map[string]v1.ResourceRequirements, containerStatuses *map[string]appspub.InPlaceUpdateContainerStatus) bool { + return true +} + +// Can access different pod vertical scaling implementations with the community +// type VerticalUpdateInternal struct{} + +// func (v *VerticalUpdateInternal) GetResourceInStatus(pod *v1.Pod) *v1.ResourceRequirements { +// return nil +// } + +// func (v *VerticalUpdateInternal) GetResourceInSpec(pod *v1.Pod) *v1.ResourceRequirements { +// return nil +// } + +// func (v *VerticalUpdateInternal) SetResourceInSpec(pod *v1.Pod, res *v1.ResourceRequirements) {}