From d2a98b7c741ebbf5f59f7dbfc41edad850845d4f Mon Sep 17 00:00:00 2001 From: "liheng.zms" Date: Thu, 27 Jul 2023 15:36:22 +0800 Subject: [PATCH] add prometheus for pub and deletion protection Signed-off-by: liheng.zms --- main.go | 2 + pkg/control/pubcontrol/api.go | 17 +++++-- pkg/control/pubcontrol/pub_control.go | 6 +++ pkg/control/pubcontrol/pub_control_metrics.go | 38 ++++++++++++++ pkg/control/pubcontrol/pub_control_utils.go | 29 ++++++++--- .../pubcontrol/pub_control_utils_test.go | 15 +++--- pkg/controller/cloneset/sync/api.go | 3 -- .../cloneset/sync/cloneset_update.go | 2 +- .../cloneset/sync/cloneset_update_test.go | 2 - .../podunavailablebudget_controller.go | 10 ++-- .../pub_controller_test.go | 4 +- .../pub_pod_event_handler.go | 11 ++-- .../validating/builtin_handlers.go | 2 + .../cloneset_create_update_handler.go | 2 + .../validating/crd_handler.go | 1 + .../namespace/validating/namespace_handler.go | 3 +- .../validating/pod_create_update_handler.go | 5 +- .../pod/validating/pod_unavailable_budget.go | 4 +- .../validating/pod_unavailable_budget_test.go | 26 ++++++---- .../statefulset_create_update_handler.go | 1 + .../uniteddeployment_create_update_handler.go | 2 + .../util/deletionprotection/prometheus.go | 51 +++++++++++++++++++ 22 files changed, 179 insertions(+), 57 deletions(-) create mode 100644 pkg/control/pubcontrol/pub_control_metrics.go create mode 100644 pkg/webhook/util/deletionprotection/prometheus.go diff --git a/main.go b/main.go index 368a6df399..6108c7d598 100644 --- a/main.go +++ b/main.go @@ -43,6 +43,7 @@ import ( appsv1beta1 "github.com/openkruise/kruise/apis/apps/v1beta1" policyv1alpha1 "github.com/openkruise/kruise/apis/policy/v1alpha1" extclient "github.com/openkruise/kruise/pkg/client" + "github.com/openkruise/kruise/pkg/control/pubcontrol" "github.com/openkruise/kruise/pkg/controller" "github.com/openkruise/kruise/pkg/features" "github.com/openkruise/kruise/pkg/util" @@ -188,6 +189,7 @@ func main() { setupLog.Error(err, "unable to start ControllerFinder") os.Exit(1) } + pubcontrol.InitPubControl(mgr.GetClient(), controllerfinder.Finder, mgr.GetEventRecorderFor("pub-controller")) setupLog.Info("register field index") if err := fieldindex.RegisterFieldIndexes(mgr.GetCache()); err != nil { diff --git a/pkg/control/pubcontrol/api.go b/pkg/control/pubcontrol/api.go index d8019b473e..fc4e4e8eeb 100644 --- a/pkg/control/pubcontrol/api.go +++ b/pkg/control/pubcontrol/api.go @@ -20,10 +20,16 @@ import ( policyv1alpha1 "github.com/openkruise/kruise/apis/policy/v1alpha1" "github.com/openkruise/kruise/pkg/util/controllerfinder" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" ) -type PubControl interface { +var PubControl pubControl +var recorder record.EventRecorder +var kclient client.Client + +type pubControl interface { // IsPodReady indicates whether pod is fully ready // 1. pod.Status.Phase == v1.PodRunning // 2. pod.condition PodReady == true @@ -41,9 +47,12 @@ type PubControl interface { IsPodUnavailableChanged(oldPod, newPod *corev1.Pod) bool // get pub for pod GetPubForPod(pod *corev1.Pod) (*policyv1alpha1.PodUnavailableBudget, error) + // get pod controller of + GetPodControllerOf(pod *corev1.Pod) *metav1.OwnerReference } -func NewPubControl(client client.Client) PubControl { - controllerFinder := controllerfinder.Finder - return &commonControl{controllerFinder: controllerFinder, Client: client} +func InitPubControl(cli client.Client, finder *controllerfinder.ControllerFinder, rec record.EventRecorder) { + recorder = rec + kclient = cli + PubControl = &commonControl{controllerFinder: finder, Client: cli} } diff --git a/pkg/control/pubcontrol/pub_control.go b/pkg/control/pubcontrol/pub_control.go index 5230e174c0..ec9823ded9 100644 --- a/pkg/control/pubcontrol/pub_control.go +++ b/pkg/control/pubcontrol/pub_control.go @@ -31,6 +31,7 @@ import ( "github.com/openkruise/kruise/pkg/util/inplaceupdate" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" kubecontroller "k8s.io/kubernetes/pkg/controller" @@ -43,6 +44,7 @@ type commonControl struct { } func (c *commonControl) IsPodReady(pod *corev1.Pod) bool { + klog.Infof("IsPodReady pod(%s)", util.DumpJSON(pod)) // 1. pod.Status.Phase == v1.PodRunning // 2. pod.condition PodReady == true if !util.IsRunningAndReady(pod) { @@ -168,6 +170,10 @@ func (c *commonControl) GetPubForPod(pod *corev1.Pod) (*policyv1alpha1.PodUnavai return pub, nil } +func (c *commonControl) GetPodControllerOf(pod *corev1.Pod) *metav1.OwnerReference { + return metav1.GetControllerOf(pod) +} + func getSidecarSetsInPod(pod *corev1.Pod) (sidecarSets, containers sets.String) { containers = sets.NewString() sidecarSets = sets.NewString() diff --git a/pkg/control/pubcontrol/pub_control_metrics.go b/pkg/control/pubcontrol/pub_control_metrics.go new file mode 100644 index 0000000000..13f02cd663 --- /dev/null +++ b/pkg/control/pubcontrol/pub_control_metrics.go @@ -0,0 +1,38 @@ +/* +Copyright 2023 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pubcontrol + +import ( + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +var ( + PodUnavailableBudgetMetrics = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "pod_unavailable_budget", + Help: "Pod Unavailable Budget Metrics", + // kind = CloneSet, Deployment, StatefulSet, etc. + // name = workload.name, if pod don't have workload, then name is pod.name + // username = client useragent + }, []string{"kind_namespace_name", "username"}, + ) +) + +func init() { + metrics.Registry.MustRegister(PodUnavailableBudgetMetrics) +} diff --git a/pkg/control/pubcontrol/pub_control_utils.go b/pkg/control/pubcontrol/pub_control_utils.go index 2736de2fab..7249f00315 100644 --- a/pkg/control/pubcontrol/pub_control_utils.go +++ b/pkg/control/pubcontrol/pub_control_utils.go @@ -35,7 +35,6 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/retry" "k8s.io/klog/v2" - "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -58,7 +57,7 @@ const ( // parameters: // 1. allowed(bool) indicates whether to allow this update operation // 2. err(error) -func PodUnavailableBudgetValidatePod(client client.Client, control PubControl, pod *corev1.Pod, operation policyv1alpha1.PubOperation, dryRun bool) (allowed bool, reason string, err error) { +func PodUnavailableBudgetValidatePod(pod *corev1.Pod, operation policyv1alpha1.PubOperation, username string, dryRun bool) (allowed bool, reason string, err error) { klog.V(3).Infof("validating pod(%s/%s) operation(%s) for PodUnavailableBudget", pod.Namespace, pod.Name, operation) // pods that contain annotations[pod.kruise.io/pub-no-protect]="true" will be ignore // and will no longer check the pub quota @@ -66,13 +65,13 @@ func PodUnavailableBudgetValidatePod(client client.Client, control PubControl, p klog.V(3).Infof("pod(%s/%s) contains annotations[%s]=true, then don't need check pub", pod.Namespace, pod.Name, policyv1alpha1.PodPubNoProtectionAnnotation) return true, "", nil // If the pod is not ready, it doesn't count towards healthy and we should not decrement - } else if !control.IsPodReady(pod) { + } else if !PubControl.IsPodReady(pod) { klog.V(3).Infof("pod(%s/%s) is not ready, then don't need check pub", pod.Namespace, pod.Name) return true, "", nil } // pub for pod - pub, err := control.GetPubForPod(pod) + pub, err := PubControl.GetPubForPod(pod) if err != nil { return false, "", err // if there is no matching PodUnavailableBudget, just return true @@ -119,7 +118,7 @@ func PodUnavailableBudgetValidatePod(client client.Client, control PubControl, p } informerCached := &policyv1alpha1.PodUnavailableBudget{} - if err := client.Get(context.TODO(), types.NamespacedName{Namespace: pub.Namespace, + if err := kclient.Get(context.TODO(), types.NamespacedName{Namespace: pub.Namespace, Name: pub.Name}, informerCached); err == nil { var localRV, informerRV int64 _ = runtime.Convert_string_To_int64(&pubClone.ResourceVersion, &localRV, nil) @@ -133,8 +132,22 @@ func PodUnavailableBudgetValidatePod(client client.Client, control PubControl, p // Try to verify-and-decrement // If it was false already, or if it becomes false during the course of our retries, - err := checkAndDecrement(pod.Name, pubClone, operation) + err = checkAndDecrement(pod.Name, pubClone, operation) if err != nil { + var kind, namespace, name string + if ref := PubControl.GetPodControllerOf(pod); ref != nil { + kind = ref.Kind + name = ref.Name + } else { + kind = "unknown" + name = pod.Name + } + namespace = pod.Namespace + if namespace == "" { + namespace = "default" + } + PodUnavailableBudgetMetrics.WithLabelValues(fmt.Sprintf("%s_%s_%s", kind, namespace, name), username).Add(1) + recorder.Eventf(pod, corev1.EventTypeWarning, "PubPreventPodDeletion", "openkruise pub prevents pod deletion") return err } @@ -147,7 +160,7 @@ func PodUnavailableBudgetValidatePod(client client.Client, control PubControl, p pubClone.Namespace, pubClone.Name, len(pubClone.Status.DisruptedPods), len(pubClone.Status.UnavailablePods), pubClone.Status.TotalReplicas, pubClone.Status.DesiredAvailable, pubClone.Status.CurrentAvailable, pubClone.Status.UnavailableAllowed) start = time.Now() - err = client.Status().Update(context.TODO(), pubClone) + err = kclient.Status().Update(context.TODO(), pubClone) costOfUpdate += time.Since(start) if err == nil { if err = util.GlobalCache.Add(pubClone); err != nil { @@ -155,7 +168,7 @@ func PodUnavailableBudgetValidatePod(client client.Client, control PubControl, p } return nil } - // if conflict, then retry + // if conflicts, then retry conflictTimes++ refresh = true return err diff --git a/pkg/control/pubcontrol/pub_control_utils_test.go b/pkg/control/pubcontrol/pub_control_utils_test.go index 13049d98a4..9a8da83759 100644 --- a/pkg/control/pubcontrol/pub_control_utils_test.go +++ b/pkg/control/pubcontrol/pub_control_utils_test.go @@ -21,16 +21,17 @@ import ( "testing" "time" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - appspub "github.com/openkruise/kruise/apis/apps/pub" policyv1alpha1 "github.com/openkruise/kruise/apis/policy/v1alpha1" + "github.com/openkruise/kruise/pkg/util/controllerfinder" apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/tools/record" podutil "k8s.io/kubernetes/pkg/api/v1/pod" utilpointer "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -278,8 +279,9 @@ func TestPodUnavailableBudgetValidatePod(t *testing.T) { for _, cs := range cases { t.Run(cs.name, func(t *testing.T) { fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cs.getPub()).Build() - control := NewPubControl(fakeClient) - allow, _, err := PodUnavailableBudgetValidatePod(fakeClient, control, cs.getPod(), cs.operation, false) + finder := &controllerfinder.ControllerFinder{Client: fakeClient} + InitPubControl(fakeClient, finder, record.NewFakeRecorder(10)) + allow, _, err := PodUnavailableBudgetValidatePod(cs.getPod(), cs.operation, "fake-user", false) if err != nil { t.Fatalf("PodUnavailableBudgetValidatePod failed: %s", err.Error()) } @@ -385,9 +387,10 @@ func TestGetPodUnavailableBudgetForPod(t *testing.T) { for _, cs := range cases { t.Run(cs.name, func(t *testing.T) { fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cs.getDeployment(), cs.getReplicaSet(), cs.getPub()).Build() - control := NewPubControl(fakeClient) + finder := &controllerfinder.ControllerFinder{Client: fakeClient} + InitPubControl(fakeClient, finder, record.NewFakeRecorder(10)) pod := cs.getPod() - pub, err := control.GetPubForPod(pod) + pub, err := PubControl.GetPubForPod(pod) if err != nil { t.Fatalf("GetPubForPod failed: %s", err.Error()) } diff --git a/pkg/controller/cloneset/sync/api.go b/pkg/controller/cloneset/sync/api.go index 54d88ea29c..0bb438de75 100644 --- a/pkg/controller/cloneset/sync/api.go +++ b/pkg/controller/cloneset/sync/api.go @@ -18,7 +18,6 @@ package sync import ( appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" - "github.com/openkruise/kruise/pkg/control/pubcontrol" clonesetutils "github.com/openkruise/kruise/pkg/controller/cloneset/utils" "github.com/openkruise/kruise/pkg/util/controllerfinder" "github.com/openkruise/kruise/pkg/util/inplaceupdate" @@ -49,7 +48,6 @@ type realControl struct { inplaceControl inplaceupdate.Interface recorder record.EventRecorder controllerFinder *controllerfinder.ControllerFinder - pubControl pubcontrol.PubControl } func New(c client.Client, recorder record.EventRecorder) Interface { @@ -59,6 +57,5 @@ func New(c client.Client, recorder record.EventRecorder) Interface { lifecycleControl: lifecycle.New(c), recorder: recorder, controllerFinder: controllerfinder.Finder, - pubControl: pubcontrol.NewPubControl(c), } } diff --git a/pkg/controller/cloneset/sync/cloneset_update.go b/pkg/controller/cloneset/sync/cloneset_update.go index 5c1862e058..40089214ec 100644 --- a/pkg/controller/cloneset/sync/cloneset_update.go +++ b/pkg/controller/cloneset/sync/cloneset_update.go @@ -134,7 +134,7 @@ func (c *realControl) Update(cs *appsv1alpha1.CloneSet, pod := pods[idx] // Determine the pub before updating the pod if utilfeature.DefaultFeatureGate.Enabled(features.PodUnavailableBudgetUpdateGate) { - allowed, _, err := pubcontrol.PodUnavailableBudgetValidatePod(c.Client, c.pubControl, pod, policyv1alpha1.PubUpdateOperation, false) + allowed, _, err := pubcontrol.PodUnavailableBudgetValidatePod(pod, policyv1alpha1.PubUpdateOperation, "kruise-manager", false) if err != nil { return err // pub check does not pass, try again in seconds diff --git a/pkg/controller/cloneset/sync/cloneset_update_test.go b/pkg/controller/cloneset/sync/cloneset_update_test.go index 3db0c4109e..17add8342a 100644 --- a/pkg/controller/cloneset/sync/cloneset_update_test.go +++ b/pkg/controller/cloneset/sync/cloneset_update_test.go @@ -28,7 +28,6 @@ import ( "github.com/openkruise/kruise/apis" appspub "github.com/openkruise/kruise/apis/apps/pub" appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" - "github.com/openkruise/kruise/pkg/control/pubcontrol" clonesetcore "github.com/openkruise/kruise/pkg/controller/cloneset/core" clonesetutils "github.com/openkruise/kruise/pkg/controller/cloneset/utils" "github.com/openkruise/kruise/pkg/features" @@ -959,7 +958,6 @@ func TestUpdate(t *testing.T) { inplaceupdate.New(fakeClient, clonesetutils.RevisionAdapterImpl), record.NewFakeRecorder(10), &controllerfinder.ControllerFinder{Client: fakeClient}, - pubcontrol.NewPubControl(fakeClient), } currentRevision := mc.updateRevision if len(mc.revisions) > 0 { diff --git a/pkg/controller/podunavailablebudget/podunavailablebudget_controller.go b/pkg/controller/podunavailablebudget/podunavailablebudget_controller.go index e6db298534..68a602c1c7 100644 --- a/pkg/controller/podunavailablebudget/podunavailablebudget_controller.go +++ b/pkg/controller/podunavailablebudget/podunavailablebudget_controller.go @@ -104,7 +104,6 @@ func newReconciler(mgr manager.Manager) reconcile.Reconciler { Scheme: mgr.GetScheme(), recorder: mgr.GetEventRecorderFor("podunavailablebudget-controller"), controllerFinder: controllerfinder.Finder, - pubControl: pubcontrol.NewPubControl(mgr.GetClient()), } } @@ -203,7 +202,6 @@ type ReconcilePodUnavailableBudget struct { Scheme *runtime.Scheme recorder record.EventRecorder controllerFinder *controllerfinder.ControllerFinder - pubControl pubcontrol.PubControl } // +kubebuilder:rbac:groups=policy.kruise.io,resources=podunavailablebudgets,verbs=get;list;watch;create;update;patch;delete @@ -251,7 +249,7 @@ func (r *ReconcilePodUnavailableBudget) Reconcile(_ context.Context, req ctrl.Re func (r *ReconcilePodUnavailableBudget) syncPodUnavailableBudget(pub *policyv1alpha1.PodUnavailableBudget) (*time.Time, error) { currentTime := time.Now() - pods, expectedCount, err := r.pubControl.GetPodsForPub(pub) + pods, expectedCount, err := pubcontrol.PubControl.GetPodsForPub(pub) if err != nil { return nil, err } @@ -319,7 +317,7 @@ func (r *ReconcilePodUnavailableBudget) syncPodUnavailableBudget(pub *policyv1al // unavailablePods contains information about pods whose specification changed(in-place update), in case of informer cache latency, after 5 seconds to remove it. var disruptedPods, unavailablePods map[string]metav1.Time disruptedPods, unavailablePods, recheckTime = r.buildDisruptedAndUnavailablePods(pods, pubClone, currentTime) - currentAvailable := countAvailablePods(pods, disruptedPods, unavailablePods, r.pubControl) + currentAvailable := countAvailablePods(pods, disruptedPods, unavailablePods) start = time.Now() updateErr := r.updatePubStatus(pubClone, currentAvailable, desiredAvailable, expectedCount, disruptedPods, unavailablePods) @@ -361,7 +359,7 @@ func (r *ReconcilePodUnavailableBudget) patchRelatedPubAnnotationInPod(pub *poli return nil } -func countAvailablePods(pods []*corev1.Pod, disruptedPods, unavailablePods map[string]metav1.Time, control pubcontrol.PubControl) (currentAvailable int32) { +func countAvailablePods(pods []*corev1.Pod, disruptedPods, unavailablePods map[string]metav1.Time) (currentAvailable int32) { recordPods := sets.String{} for pName := range disruptedPods { recordPods.Insert(pName) @@ -379,7 +377,7 @@ func countAvailablePods(pods []*corev1.Pod, disruptedPods, unavailablePods map[s continue } // pod consistent and ready - if control.IsPodStateConsistent(pod) && control.IsPodReady(pod) { + if pubcontrol.PubControl.IsPodStateConsistent(pod) && pubcontrol.PubControl.IsPodReady(pod) { currentAvailable++ } } diff --git a/pkg/controller/podunavailablebudget/pub_controller_test.go b/pkg/controller/podunavailablebudget/pub_controller_test.go index 2a1f7be073..6d9f932abc 100644 --- a/pkg/controller/podunavailablebudget/pub_controller_test.go +++ b/pkg/controller/podunavailablebudget/pub_controller_test.go @@ -1150,13 +1150,13 @@ func TestPubReconcile(t *testing.T) { for _, obj := range cs.getReplicaSet() { _ = fakeClient.Create(context.TODO(), obj) } - + finder := &controllerfinder.ControllerFinder{Client: fakeClient} + pubcontrol.InitPubControl(fakeClient, finder, record.NewFakeRecorder(10)) controllerfinder.Finder = &controllerfinder.ControllerFinder{Client: fakeClient} reconciler := ReconcilePodUnavailableBudget{ Client: fakeClient, recorder: record.NewFakeRecorder(10), controllerFinder: &controllerfinder.ControllerFinder{Client: fakeClient}, - pubControl: pubcontrol.NewPubControl(fakeClient), } _, err := reconciler.syncPodUnavailableBudget(pub) if err != nil { diff --git a/pkg/controller/podunavailablebudget/pub_pod_event_handler.go b/pkg/controller/podunavailablebudget/pub_pod_event_handler.go index c4b4855a4f..5f75ba38bf 100644 --- a/pkg/controller/podunavailablebudget/pub_pod_event_handler.go +++ b/pkg/controller/podunavailablebudget/pub_pod_event_handler.go @@ -48,14 +48,12 @@ var _ handler.EventHandler = &enqueueRequestForPod{} func newEnqueueRequestForPod(c client.Client) handler.EventHandler { e := &enqueueRequestForPod{client: c} e.controllerFinder = controllerfinder.Finder - e.pubControl = pubcontrol.NewPubControl(c) return e } type enqueueRequestForPod struct { client client.Client controllerFinder *controllerfinder.ControllerFinder - pubControl pubcontrol.PubControl } func (p *enqueueRequestForPod) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) { @@ -78,7 +76,7 @@ func (p *enqueueRequestForPod) addPod(q workqueue.RateLimitingInterface, obj run } var pub *policyv1alpha1.PodUnavailableBudget if pod.Annotations[pubcontrol.PodRelatedPubAnnotation] != "" { - pub, _ = p.pubControl.GetPubForPod(pod) + pub, _ = pubcontrol.PubControl.GetPubForPod(pod) } if pub == nil { return @@ -142,11 +140,11 @@ func (p *enqueueRequestForPod) updatePod(q workqueue.RateLimitingInterface, old, return } - pub, _ := p.pubControl.GetPubForPod(newPod) + pub, _ := pubcontrol.PubControl.GetPubForPod(newPod) if pub == nil { return } - if isReconcile, enqueueDelayTime := isPodAvailableChanged(oldPod, newPod, pub, p.pubControl); isReconcile { + if isReconcile, enqueueDelayTime := isPodAvailableChanged(oldPod, newPod, pub); isReconcile { q.AddAfter(reconcile.Request{ NamespacedName: types.NamespacedName{ Name: pub.Name, @@ -157,7 +155,7 @@ func (p *enqueueRequestForPod) updatePod(q workqueue.RateLimitingInterface, old, } -func isPodAvailableChanged(oldPod, newPod *corev1.Pod, pub *policyv1alpha1.PodUnavailableBudget, control pubcontrol.PubControl) (bool, time.Duration) { +func isPodAvailableChanged(oldPod, newPod *corev1.Pod, pub *policyv1alpha1.PodUnavailableBudget) (bool, time.Duration) { var enqueueDelayTime time.Duration // If the pod's deletion timestamp is set, remove endpoint from ready address. if oldPod.DeletionTimestamp.IsZero() && !newPod.DeletionTimestamp.IsZero() { @@ -169,6 +167,7 @@ func isPodAvailableChanged(oldPod, newPod *corev1.Pod, pub *policyv1alpha1.PodUn return false, enqueueDelayTime } + control := pubcontrol.PubControl // If the pod's readiness has changed, the associated endpoint address // will move from the unready endpoints set to the ready endpoints. // So for the purposes of an endpoint, a readiness change on a pod diff --git a/pkg/webhook/builtinworkloads/validating/builtin_handlers.go b/pkg/webhook/builtinworkloads/validating/builtin_handlers.go index f0d9df324b..71f69693ce 100644 --- a/pkg/webhook/builtinworkloads/validating/builtin_handlers.go +++ b/pkg/webhook/builtinworkloads/validating/builtin_handlers.go @@ -18,6 +18,7 @@ package validating import ( "context" + "fmt" "net/http" "github.com/openkruise/kruise/pkg/webhook/util/deletionprotection" @@ -79,6 +80,7 @@ func (h *WorkloadHandler) Handle(ctx context.Context, req admission.Request) adm } if err := deletionprotection.ValidateWorkloadDeletion(metaObj, replicas); err != nil { + deletionprotection.WorkloadDeletionProtectionMetrics.WithLabelValues(fmt.Sprintf("%s_%s_%s", req.Kind.Kind, metaObj.GetNamespace(), metaObj.GetName()), req.UserInfo.Username).Add(1) return admission.Errored(http.StatusForbidden, err) } return admission.ValidationResponse(true, "") diff --git a/pkg/webhook/cloneset/validating/cloneset_create_update_handler.go b/pkg/webhook/cloneset/validating/cloneset_create_update_handler.go index 7e1c93cc3d..84209898f8 100644 --- a/pkg/webhook/cloneset/validating/cloneset_create_update_handler.go +++ b/pkg/webhook/cloneset/validating/cloneset_create_update_handler.go @@ -18,6 +18,7 @@ package validating import ( "context" + "fmt" "net/http" appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" @@ -74,6 +75,7 @@ func (h *CloneSetCreateUpdateHandler) Handle(ctx context.Context, req admission. return admission.Errored(http.StatusBadRequest, err) } if err := deletionprotection.ValidateWorkloadDeletion(oldObj, oldObj.Spec.Replicas); err != nil { + deletionprotection.WorkloadDeletionProtectionMetrics.WithLabelValues(fmt.Sprintf("%s_%s_%s", req.Kind.Kind, oldObj.GetNamespace(), oldObj.GetName()), req.UserInfo.Username).Add(1) return admission.Errored(http.StatusForbidden, err) } } diff --git a/pkg/webhook/customresourcedefinition/validating/crd_handler.go b/pkg/webhook/customresourcedefinition/validating/crd_handler.go index 684f7fbe4a..196c7ab0ff 100644 --- a/pkg/webhook/customresourcedefinition/validating/crd_handler.go +++ b/pkg/webhook/customresourcedefinition/validating/crd_handler.go @@ -85,6 +85,7 @@ func (h *CRDHandler) Handle(ctx context.Context, req admission.Request) admissio } if err := deletionprotection.ValidateCRDDeletion(h.Client, metaObj, gvk); err != nil { + deletionprotection.CRDDeletionProtectionMetrics.WithLabelValues(metaObj.GetName(), req.UserInfo.Username).Add(1) return admission.Errored(http.StatusForbidden, err) } return admission.ValidationResponse(true, "") diff --git a/pkg/webhook/namespace/validating/namespace_handler.go b/pkg/webhook/namespace/validating/namespace_handler.go index 7064410261..d2299fc04e 100644 --- a/pkg/webhook/namespace/validating/namespace_handler.go +++ b/pkg/webhook/namespace/validating/namespace_handler.go @@ -49,13 +49,12 @@ func (h *NamespaceHandler) Handle(ctx context.Context, req admission.Request) ad klog.Warningf("Skip to validate namespace %s deletion for no old object, maybe because of Kubernetes version < 1.16", req.Name) return admission.ValidationResponse(true, "") } - obj := &v1.Namespace{} if err := h.Decoder.DecodeRaw(req.AdmissionRequest.OldObject, obj); err != nil { return admission.Errored(http.StatusBadRequest, err) } - if err := deletionprotection.ValidateNamespaceDeletion(h.Client, obj); err != nil { + deletionprotection.NamespaceDeletionProtectionMetrics.WithLabelValues(obj.Name, req.UserInfo.Username).Add(1) return admission.Errored(http.StatusForbidden, err) } return admission.ValidationResponse(true, "") diff --git a/pkg/webhook/pod/validating/pod_create_update_handler.go b/pkg/webhook/pod/validating/pod_create_update_handler.go index cfb1fa7b16..e164db40a6 100644 --- a/pkg/webhook/pod/validating/pod_create_update_handler.go +++ b/pkg/webhook/pod/validating/pod_create_update_handler.go @@ -20,7 +20,6 @@ import ( "context" "net/http" - "github.com/openkruise/kruise/pkg/control/pubcontrol" "github.com/openkruise/kruise/pkg/features" "github.com/openkruise/kruise/pkg/util/controllerfinder" utilfeature "github.com/openkruise/kruise/pkg/util/feature" @@ -42,8 +41,7 @@ type PodCreateHandler struct { // Decoder decodes objects Decoder *admission.Decoder - finders *controllerfinder.ControllerFinder - pubControl pubcontrol.PubControl + finders *controllerfinder.ControllerFinder } func (h *PodCreateHandler) validatingPodFn(ctx context.Context, req admission.Request) (allowed bool, reason string, err error) { @@ -91,7 +89,6 @@ var _ inject.Client = &PodCreateHandler{} func (h *PodCreateHandler) InjectClient(c client.Client) error { h.Client = c h.finders = controllerfinder.Finder - h.pubControl = pubcontrol.NewPubControl(c) return nil } diff --git a/pkg/webhook/pod/validating/pod_unavailable_budget.go b/pkg/webhook/pod/validating/pod_unavailable_budget.go index 36063f54db..3d48f5f808 100644 --- a/pkg/webhook/pod/validating/pod_unavailable_budget.go +++ b/pkg/webhook/pod/validating/pod_unavailable_budget.go @@ -58,7 +58,7 @@ func (p *PodCreateHandler) podUnavailableBudgetValidatingPod(ctx context.Context return false, "", err } // the change will not cause pod unavailability, then pass - if !p.pubControl.IsPodUnavailableChanged(oldPod, newPod) { + if !pubcontrol.PubControl.IsPodUnavailableChanged(oldPod, newPod) { klog.V(6).Infof("validate pod(%s/%s) changed can not cause unavailability, then don't need check pub", newPod.Namespace, newPod.Name) return true, "", nil } @@ -122,5 +122,5 @@ func (p *PodCreateHandler) podUnavailableBudgetValidatingPod(ctx context.Context if checkPod.Annotations[pubcontrol.PodRelatedPubAnnotation] == "" { return true, "", nil } - return pubcontrol.PodUnavailableBudgetValidatePod(p.Client, p.pubControl, checkPod, operation, dryRun) + return pubcontrol.PodUnavailableBudgetValidatePod(checkPod, operation, req.UserInfo.Username, dryRun) } diff --git a/pkg/webhook/pod/validating/pod_unavailable_budget_test.go b/pkg/webhook/pod/validating/pod_unavailable_budget_test.go index ac285dbede..57bdb9e64a 100644 --- a/pkg/webhook/pod/validating/pod_unavailable_budget_test.go +++ b/pkg/webhook/pod/validating/pod_unavailable_budget_test.go @@ -22,13 +22,12 @@ import ( "testing" "time" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - appspub "github.com/openkruise/kruise/apis/apps/pub" policyv1alpha1 "github.com/openkruise/kruise/apis/policy/v1alpha1" "github.com/openkruise/kruise/pkg/control/pubcontrol" "github.com/openkruise/kruise/pkg/control/sidecarcontrol" "github.com/openkruise/kruise/pkg/util" + "github.com/openkruise/kruise/pkg/util/controllerfinder" admissionv1 "k8s.io/api/admission/v1" apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -36,6 +35,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/tools/record" podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/pkg/apis/policy" "sigs.k8s.io/controller-runtime/pkg/client" @@ -475,10 +476,11 @@ func TestValidateUpdatePodForPub(t *testing.T) { decoder, _ := admission.NewDecoder(scheme) fClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cs.pub()).Build() podHandler := PodCreateHandler{ - Client: fClient, - Decoder: decoder, - pubControl: pubcontrol.NewPubControl(fClient), + Client: fClient, + Decoder: decoder, } + finder := &controllerfinder.ControllerFinder{Client: fClient} + pubcontrol.InitPubControl(fClient, finder, record.NewFakeRecorder(10)) oldPodRaw := runtime.RawExtension{ Raw: []byte(util.DumpJSON(cs.oldPod())), } @@ -670,10 +672,11 @@ func TestValidateEvictPodForPub(t *testing.T) { decoder, _ := admission.NewDecoder(scheme) fClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cs.pub(), cs.newPod()).Build() podHandler := PodCreateHandler{ - Client: fClient, - Decoder: decoder, - pubControl: pubcontrol.NewPubControl(fClient), + Client: fClient, + Decoder: decoder, } + finder := &controllerfinder.ControllerFinder{Client: fClient} + pubcontrol.InitPubControl(fClient, finder, record.NewFakeRecorder(10)) evictionRaw := runtime.RawExtension{ Raw: []byte(util.DumpJSON(cs.eviction())), } @@ -826,10 +829,11 @@ func TestValidateDeletePodForPub(t *testing.T) { decoder, _ := admission.NewDecoder(scheme) fClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cs.pub(), cs.newPod()).Build() podHandler := PodCreateHandler{ - Client: fClient, - Decoder: decoder, - pubControl: pubcontrol.NewPubControl(fClient), + Client: fClient, + Decoder: decoder, } + finder := &controllerfinder.ControllerFinder{Client: fClient} + pubcontrol.InitPubControl(fClient, finder, record.NewFakeRecorder(10)) deletionRaw := runtime.RawExtension{ Raw: []byte(util.DumpJSON(cs.deletion())), } diff --git a/pkg/webhook/statefulset/validating/statefulset_create_update_handler.go b/pkg/webhook/statefulset/validating/statefulset_create_update_handler.go index 329cb5e517..747001acf3 100644 --- a/pkg/webhook/statefulset/validating/statefulset_create_update_handler.go +++ b/pkg/webhook/statefulset/validating/statefulset_create_update_handler.go @@ -86,6 +86,7 @@ func (h *StatefulSetCreateUpdateHandler) Handle(ctx context.Context, req admissi return admission.Errored(http.StatusBadRequest, err) } if err := deletionprotection.ValidateWorkloadDeletion(oldObj, oldObj.Spec.Replicas); err != nil { + deletionprotection.WorkloadDeletionProtectionMetrics.WithLabelValues(fmt.Sprintf("%s_%s_%s", req.Kind.Kind, oldObj.GetNamespace(), oldObj.GetName()), req.UserInfo.Username).Add(1) return admission.Errored(http.StatusForbidden, err) } } diff --git a/pkg/webhook/uniteddeployment/validating/uniteddeployment_create_update_handler.go b/pkg/webhook/uniteddeployment/validating/uniteddeployment_create_update_handler.go index 049df4a320..4f1883a9ab 100644 --- a/pkg/webhook/uniteddeployment/validating/uniteddeployment_create_update_handler.go +++ b/pkg/webhook/uniteddeployment/validating/uniteddeployment_create_update_handler.go @@ -18,6 +18,7 @@ package validating import ( "context" + "fmt" "net/http" appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" @@ -76,6 +77,7 @@ func (h *UnitedDeploymentCreateUpdateHandler) Handle(ctx context.Context, req ad return admission.Errored(http.StatusBadRequest, err) } if err := deletionprotection.ValidateWorkloadDeletion(oldObj, oldObj.Spec.Replicas); err != nil { + deletionprotection.WorkloadDeletionProtectionMetrics.WithLabelValues(fmt.Sprintf("%s_%s_%s", req.Kind.Kind, oldObj.GetNamespace(), oldObj.GetName()), req.UserInfo.Username).Add(1) return admission.Errored(http.StatusForbidden, err) } } diff --git a/pkg/webhook/util/deletionprotection/prometheus.go b/pkg/webhook/util/deletionprotection/prometheus.go new file mode 100644 index 0000000000..247cc4cc56 --- /dev/null +++ b/pkg/webhook/util/deletionprotection/prometheus.go @@ -0,0 +1,51 @@ +/* +Copyright 2023 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package deletionprotection + +import ( + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +var ( + NamespaceDeletionProtectionMetrics = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "namespace_deletion_protection", + Help: "Namespace Deletion Protection", + }, []string{"name", "username"}, + ) + + CRDDeletionProtectionMetrics = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "crd_deletion_protection", + Help: "CustomResourceDefinition Deletion Protection", + }, []string{"name", "username"}, + ) + + WorkloadDeletionProtectionMetrics = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "workload_deletion_protection", + Help: "Workload Deletion Protection", + }, []string{"kind_namespace_name", "username"}, + ) +) + +func init() { + metrics.Registry.MustRegister(NamespaceDeletionProtectionMetrics) + metrics.Registry.MustRegister(CRDDeletionProtectionMetrics) + metrics.Registry.MustRegister(WorkloadDeletionProtectionMetrics) +}