From 9661049efcd65562a754cc4f3ccc15abf902be49 Mon Sep 17 00:00:00 2001 From: "mingzhou.swx" Date: Fri, 4 Aug 2023 15:39:25 +0800 Subject: [PATCH] update daemonset and statefulset to 1.24.16 Signed-off-by: Kuromesi compatible with v1 and v1alpha2 cri api version Signed-off-by: mingzhou.swx update cloneset ut for go 1.19 Signed-off-by: mingzhou.swx --- .github/workflows/ci.yaml | 4 +- .github/workflows/e2e-1.16.yaml | 2 +- ...e-1.20.yaml => e2e-1.20-EphemeralJob.yaml} | 4 +- .../{e2e-1.24.yaml => e2e-1.26.yaml} | 6 +- .../cloneset/utils/pod_sorter_test.go | 22 +-- .../statefulset/stateful_set_control.go | 27 +-- pkg/daemon/criruntime/factory.go | 12 +- .../criruntime/imageruntime/containerd.go | 2 + pkg/daemon/criruntime/imageruntime/cri.go | 165 ++++++++++++++++-- pkg/daemon/criruntime/imageruntime/helpers.go | 27 +++ 10 files changed, 222 insertions(+), 49 deletions(-) rename .github/workflows/{e2e-1.20.yaml => e2e-1.20-EphemeralJob.yaml} (98%) rename .github/workflows/{e2e-1.24.yaml => e2e-1.26.yaml} (99%) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b029b4a198..fa8eec7d20 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -10,7 +10,7 @@ on: env: # Common versions - GO_VERSION: '1.18' + GO_VERSION: '1.19' GOLANGCI_VERSION: 'v1.47' DOCKER_BUILDX_VERSION: 'v0.4.2' @@ -53,7 +53,7 @@ jobs: run: | make generate - name: Lint golang code - uses: golangci/golangci-lint-action@v3.2.0 + uses: golangci/golangci-lint-action@v3.5.0 with: version: ${{ env.GOLANGCI_VERSION }} args: --verbose diff --git a/.github/workflows/e2e-1.16.yaml b/.github/workflows/e2e-1.16.yaml index 1ffdfcb906..a8ae45224d 100644 --- a/.github/workflows/e2e-1.16.yaml +++ b/.github/workflows/e2e-1.16.yaml @@ -10,7 +10,7 @@ on: env: # Common versions - GO_VERSION: '1.18' + GO_VERSION: '1.19' KIND_VERSION: 'v0.14.0' KIND_IMAGE: 'kindest/node:v1.16.15' KIND_CLUSTER_NAME: 'ci-testing' diff --git a/.github/workflows/e2e-1.20.yaml b/.github/workflows/e2e-1.20-EphemeralJob.yaml similarity index 98% rename from .github/workflows/e2e-1.20.yaml rename to .github/workflows/e2e-1.20-EphemeralJob.yaml index a9b83c1011..f969a81d77 100644 --- a/.github/workflows/e2e-1.20.yaml +++ b/.github/workflows/e2e-1.20-EphemeralJob.yaml @@ -1,4 +1,4 @@ -name: E2E-1.20 +name: E2E-1.20-EphemeralJob on: push: @@ -10,7 +10,7 @@ on: env: # Common versions - GO_VERSION: '1.18' + GO_VERSION: '1.19' KIND_VERSION: 'v0.14.0' KIND_IMAGE: 'kindest/node:v1.20.7' KIND_CLUSTER_NAME: 'ci-testing' diff --git a/.github/workflows/e2e-1.24.yaml b/.github/workflows/e2e-1.26.yaml similarity index 99% rename from .github/workflows/e2e-1.24.yaml rename to .github/workflows/e2e-1.26.yaml index 3d6c432a8d..9daf217837 100644 --- a/.github/workflows/e2e-1.24.yaml +++ b/.github/workflows/e2e-1.26.yaml @@ -1,4 +1,4 @@ -name: E2E-1.24 +name: E2E-1.26 on: push: @@ -10,10 +10,10 @@ on: env: # Common versions - GO_VERSION: '1.18' + GO_VERSION: '1.19' KIND_ACTION_VERSION: 'v1.3.0' KIND_VERSION: 'v0.14.0' - KIND_IMAGE: 'kindest/node:v1.24.2' + KIND_IMAGE: 'kindest/node:v1.26.4' KIND_CLUSTER_NAME: 'ci-testing' jobs: diff --git a/pkg/controller/cloneset/utils/pod_sorter_test.go b/pkg/controller/cloneset/utils/pod_sorter_test.go index e83040eac0..d46f92ecc4 100644 --- a/pkg/controller/cloneset/utils/pod_sorter_test.go +++ b/pkg/controller/cloneset/utils/pod_sorter_test.go @@ -194,13 +194,13 @@ func TestSpreadConstraintsRanker(t *testing.T) { expectedPodsSorted: []types.UID{ "d-0", "d-1", - "d-2", "e-0", - "c-0", "e-1", - "c-1", "e-2", - "b-0", "f-0", "i-0", + "c-0", "e-0", + "d-2", "e-1", + "b-0", "e-2", + "c-1", "f-0", "i-0", "d-3", "e-3", "i-1", - "c-2", "f-1", "i-2", - "b-1", "e-4", "i-3", + "b-1", "f-1", "i-2", + "c-2", "e-4", "i-3", "d-4", "f-2", "i-4", }, }, @@ -229,14 +229,14 @@ func TestSpreadConstraintsRanker(t *testing.T) { expectedPodsSorted: []types.UID{ "d-0", "e-0", "d-1", "e-1", - "d-2", "i-0", - "c-0", "e-2", + "c-0", "i-0", + "d-2", "e-2", "i-1", "f-0", - "c-1", "b-0", + "b-0", "c-1", "i-2", "d-3", "e-3", "f-1", - "i-3", "c-2", - "b-1", "e-4", + "i-3", "b-1", + "c-2", "e-4", "d-4", "f-2", "i-4", }, }, diff --git a/pkg/controller/statefulset/stateful_set_control.go b/pkg/controller/statefulset/stateful_set_control.go index 68d86e3eeb..603cdcf5bb 100644 --- a/pkg/controller/statefulset/stateful_set_control.go +++ b/pkg/controller/statefulset/stateful_set_control.go @@ -565,19 +565,20 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( } // If we find a Pod that is currently terminating, we must wait until graceful deletion // completes before we continue to make progress. - if isTerminating(replicas[i]) { - if monotonic { - klog.V(4).InfoS("StatefulSet is waiting for Pod to Terminate", - "statefulSet", klog.KObj(set), "pod", klog.KObj(replicas[i])) - return &status, nil - } else if isTerminating(replicas[i]) && decreaseAndCheckMaxUnavailable(scaleMaxUnavailable) { - klog.V(4).Infof( - "StatefulSet %s/%s Pod %s is Terminating, and break pods scale", - set.Namespace, - set.Name, - replicas[i].Name) - break - } + if isTerminating(replicas[i]) && monotonic { + klog.V(4).Infof( + "StatefulSet %s/%s is waiting for Pod %s to Terminate", + set.Namespace, + set.Name, + replicas[i].Name) + return &status, nil + } else if isTerminating(replicas[i]) && decreaseAndCheckMaxUnavailable(scaleMaxUnavailable) { + klog.V(4).Infof( + "StatefulSet %s/%s Pod %s is Terminating, and break pods scale", + set.Namespace, + set.Name, + replicas[i].Name) + break } // Update InPlaceUpdateReady condition for pod if res := ssc.inplaceControl.Refresh(replicas[i], nil); res.RefreshErr != nil { diff --git a/pkg/daemon/criruntime/factory.go b/pkg/daemon/criruntime/factory.go index 8279b1cd6d..bebef92198 100644 --- a/pkg/daemon/criruntime/factory.go +++ b/pkg/daemon/criruntime/factory.go @@ -90,12 +90,6 @@ func NewFactory(varRunPath string, accountManager daemonutil.ImagePullAccountMan var typedVersion *runtimeapi.VersionResponse switch cfg.runtimeType { - case ContainerRuntimeDocker: - imageService, err = runtimeimage.NewDockerImageService(cfg.runtimeURI, accountManager) - if err != nil { - klog.Warningf("Failed to new image service for %v (%s, %s): %v", cfg.runtimeType, cfg.runtimeURI, cfg.runtimeRemoteURI, err) - continue - } case ContainerRuntimeContainerd, ContainerRuntimeCommonCRI, ContainerRuntimePouch: addr, _, err := kubeletutil.GetAddressAndDialer(cfg.runtimeRemoteURI) if err != nil { @@ -107,6 +101,12 @@ func NewFactory(varRunPath string, accountManager daemonutil.ImagePullAccountMan klog.Warningf("Failed to new image service for %v (%s, %s): %v", cfg.runtimeType, cfg.runtimeURI, cfg.runtimeRemoteURI, err) continue } + case ContainerRuntimeDocker: + imageService, err = runtimeimage.NewDockerImageService(cfg.runtimeURI, accountManager) + if err != nil { + klog.Warningf("Failed to new image service for %v (%s, %s): %v", cfg.runtimeType, cfg.runtimeURI, cfg.runtimeRemoteURI, err) + continue + } } if _, err = imageService.ListImages(context.TODO()); err != nil { diff --git a/pkg/daemon/criruntime/imageruntime/containerd.go b/pkg/daemon/criruntime/imageruntime/containerd.go index e4706708c5..1b9921e406 100644 --- a/pkg/daemon/criruntime/imageruntime/containerd.go +++ b/pkg/daemon/criruntime/imageruntime/containerd.go @@ -73,6 +73,7 @@ func NewContainerdImageService( accountManager: accountManager, snapshotter: snapshotter, client: client, + // TODO: compatible with v1alpha2 cri api criImageClient: runtimeapi.NewImageServiceClient(conn), httpProxy: httpProxy, }, nil @@ -325,6 +326,7 @@ func getDefaultValuesFromCRIStatus(conn *grpc.ClientConn) (snapshotter string, h ctx, cancel := context.WithTimeout(context.TODO(), 10*time.Second) defer cancel() + // TODO: compatible with v1alpha2 cri api rclient := runtimeapi.NewRuntimeServiceClient(conn) resp, err := rclient.Status(ctx, &runtimeapi.StatusRequest{Verbose: true}) if err != nil { diff --git a/pkg/daemon/criruntime/imageruntime/cri.go b/pkg/daemon/criruntime/imageruntime/cri.go index 7190e662e1..a4072985c8 100644 --- a/pkg/daemon/criruntime/imageruntime/cri.go +++ b/pkg/daemon/criruntime/imageruntime/cri.go @@ -16,6 +16,7 @@ package imageruntime import ( "context" "io" + "reflect" "time" appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" @@ -26,6 +27,7 @@ import ( v1 "k8s.io/api/core/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + runtimeapiv1alpha2 "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/kubelet/cri/remote/util" "k8s.io/kubernetes/pkg/util/parsers" @@ -53,26 +55,47 @@ func NewCRIImageService(runtimeURI string, accountManager daemonutil.ImagePullAc return nil, err } - klog.V(4).InfoS("Finding the CRI API image version") - imageClient := runtimeapi.NewImageServiceClient(conn) - - if _, err := imageClient.ImageFsInfo(ctx, &runtimeapi.ImageFsInfoRequest{}); err == nil { - klog.V(2).InfoS("Using CRI v1 image API") + imageClientV1, imageClientV1alpha2, err := determineImageClientAPIVersion(conn) + if err != nil { + klog.ErrorS(err, "Failed to determine CRI image API version") + return nil, err } return &commonCRIImageService{ - accountManager: accountManager, - criImageClient: imageClient, + accountManager: accountManager, + criImageClient: imageClientV1, + criImageClientV1alpha2: imageClientV1alpha2, }, nil } type commonCRIImageService struct { - accountManager daemonutil.ImagePullAccountManager - criImageClient runtimeapi.ImageServiceClient + accountManager daemonutil.ImagePullAccountManager + criImageClient runtimeapi.ImageServiceClient + criImageClientV1alpha2 runtimeapiv1alpha2.ImageServiceClient +} + +func (c *commonCRIImageService) useV1API() bool { + return c.criImageClientV1alpha2 == nil || reflect.ValueOf(c.criImageClientV1alpha2).IsNil() } // PullImage implements ImageService.PullImage. func (c *commonCRIImageService) PullImage(ctx context.Context, imageName, tag string, pullSecrets []v1.Secret, sandboxConfig *appsv1alpha1.SandboxConfig) (ImagePullStatusReader, error) { + if c.useV1API() { + return c.pullImageV1(ctx, imageName, tag, pullSecrets, sandboxConfig) + } + return c.pullImageV1alpha2(ctx, imageName, tag, pullSecrets, sandboxConfig) +} + +// ListImages implements ImageService.ListImages. +func (c *commonCRIImageService) ListImages(ctx context.Context) ([]ImageInfo, error) { + if c.useV1API() { + return c.listImagesV1(ctx) + } + return c.listImagesV1alpha2(ctx) +} + +// PullImage implements ImageService.PullImage using v1 CRI client. +func (c *commonCRIImageService) pullImageV1(ctx context.Context, imageName, tag string, pullSecrets []v1.Secret, sandboxConfig *appsv1alpha1.SandboxConfig) (ImagePullStatusReader, error) { registry := daemonutil.ParseRegistry(imageName) fullImageName := imageName + ":" + tag repoToPull, _, _, err := parsers.ParseImageName(fullImageName) @@ -172,8 +195,8 @@ func (c *commonCRIImageService) PullImage(ctx context.Context, imageName, tag st return newImagePullStatusReader(pipeR), nil } -// ListImages implements ImageService.ListImages. -func (c *commonCRIImageService) ListImages(ctx context.Context) ([]ImageInfo, error) { +// ListImages implements ImageService.ListImages using V1 CRI client. +func (c *commonCRIImageService) listImagesV1(ctx context.Context) ([]ImageInfo, error) { listImagesReq := &runtimeapi.ListImagesRequest{} listImagesResp, err := c.criImageClient.ListImages(ctx, listImagesReq) if err != nil { @@ -190,3 +213,123 @@ func (c *commonCRIImageService) ListImages(ctx context.Context) ([]ImageInfo, er } return collection, nil } + +// PullImage implements ImageService.PullImage using v1alpha2 CRI client. +func (c *commonCRIImageService) pullImageV1alpha2(ctx context.Context, imageName, tag string, pullSecrets []v1.Secret, sandboxConfig *appsv1alpha1.SandboxConfig) (ImagePullStatusReader, error) { + registry := daemonutil.ParseRegistry(imageName) + fullImageName := imageName + ":" + tag + repoToPull, _, _, err := parsers.ParseImageName(fullImageName) + if err != nil { + return nil, err + } + // Reader + pipeR, pipeW := io.Pipe() + defer pipeW.Close() + + var auth *runtimeapiv1alpha2.AuthConfig + pullImageReq := &runtimeapiv1alpha2.PullImageRequest{ + Image: &runtimeapiv1alpha2.ImageSpec{ + Image: fullImageName, + Annotations: make(map[string]string), + }, + Auth: auth, //default is nil + } + if sandboxConfig != nil { + pullImageReq.SandboxConfig = &runtimeapiv1alpha2.PodSandboxConfig{ + Annotations: sandboxConfig.Annotations, + Labels: sandboxConfig.Labels, + } + if pullImageReq.SandboxConfig.Annotations == nil { + pullImageReq.SandboxConfig.Annotations = map[string]string{} + } + } else { + pullImageReq.SandboxConfig = &runtimeapiv1alpha2.PodSandboxConfig{ + Annotations: map[string]string{}, + } + } + // Add this default annotation to avoid unexpected panic caused by sandboxConfig is nil + // for some runtime implementations. + pullImageReq.SandboxConfig.Annotations[pullingImageSandboxConfigAnno] = "kruise-daemon" + + if len(pullSecrets) > 0 { + var authInfos []daemonutil.AuthInfo + authInfos, err = convertToRegistryAuths(pullSecrets, repoToPull) + if err == nil { + var pullErrs []error + for _, authInfo := range authInfos { + var pullErr error + klog.V(5).Infof("Pull image %v:%v with user %v", imageName, tag, authInfo.Username) + pullImageReq.Auth = &runtimeapiv1alpha2.AuthConfig{ + Username: authInfo.Username, + Password: authInfo.Password, + } + _, pullErr = c.criImageClientV1alpha2.PullImage(ctx, pullImageReq) + if pullErr == nil { + pipeW.CloseWithError(io.EOF) + return newImagePullStatusReader(pipeR), nil + } + klog.Warningf("Failed to pull image %v:%v with user %v, err %v", imageName, tag, authInfo.Username, pullErr) + pullErrs = append(pullErrs, pullErr) + + } + if len(pullErrs) > 0 { + err = utilerrors.NewAggregate(pullErrs) + } + } + } + + // Try the default secret + if c.accountManager != nil { + var authInfo *daemonutil.AuthInfo + var defaultErr error + authInfo, defaultErr = c.accountManager.GetAccountInfo(registry) + if defaultErr != nil { + klog.Warningf("Failed to get account for registry %v, err %v", registry, defaultErr) + // When the default account acquisition fails, try to pull anonymously + } else if authInfo != nil { + klog.V(5).Infof("Pull image %v:%v with user %v", imageName, tag, authInfo.Username) + pullImageReq.Auth = &runtimeapiv1alpha2.AuthConfig{ + Username: authInfo.Username, + Password: authInfo.Password, + } + _, err = c.criImageClientV1alpha2.PullImage(ctx, pullImageReq) + if err == nil { + pipeW.CloseWithError(io.EOF) + return newImagePullStatusReader(pipeR), nil + } + klog.Warningf("Failed to pull image %v:%v, err %v", imageName, tag, err) + return nil, err + } + } + + if err != nil { + return nil, err + } + + // Anonymous pull + _, err = c.criImageClientV1alpha2.PullImage(ctx, pullImageReq) + if err != nil { + return nil, errors.Wrapf(err, "Failed to pull image reference %q", fullImageName) + } + pipeW.CloseWithError(io.EOF) + return newImagePullStatusReader(pipeR), nil +} + +// ListImages implements ImageService.ListImages using V1alpha2 CRI client. +func (c *commonCRIImageService) listImagesV1alpha2(ctx context.Context) ([]ImageInfo, error) { + listImagesReq := &runtimeapiv1alpha2.ListImagesRequest{} + listImagesResp, err := c.criImageClientV1alpha2.ListImages(ctx, listImagesReq) + if err != nil { + return nil, err + } + collection := make([]ImageInfo, 0, len(listImagesResp.GetImages())) + for _, img := range listImagesResp.GetImages() { + collection = append(collection, ImageInfo{ + ID: img.GetId(), + RepoTags: img.GetRepoTags(), + RepoDigests: img.GetRepoDigests(), + Size: int64(img.GetSize_()), + }) + } + return collection, nil +} diff --git a/pkg/daemon/criruntime/imageruntime/helpers.go b/pkg/daemon/criruntime/imageruntime/helpers.go index cfc3804010..cd1e18dbc7 100644 --- a/pkg/daemon/criruntime/imageruntime/helpers.go +++ b/pkg/daemon/criruntime/imageruntime/helpers.go @@ -17,14 +17,21 @@ limitations under the License. package imageruntime import ( + "context" "encoding/json" "fmt" "io" + "time" dockermessage "github.com/docker/docker/pkg/jsonmessage" daemonutil "github.com/openkruise/kruise/pkg/daemon/util" "github.com/openkruise/kruise/pkg/util" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" v1 "k8s.io/api/core/v1" + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + runtimeapiv1alpha2 "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/credentialprovider" credentialprovidersecrets "k8s.io/kubernetes/pkg/credentialprovider/secrets" @@ -239,3 +246,23 @@ func (c ImageInfo) ContainsImage(name string, tag string) bool { } return false } + +func determineImageClientAPIVersion(conn *grpc.ClientConn) (runtimeapi.ImageServiceClient, runtimeapiv1alpha2.ImageServiceClient, error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + klog.V(4).InfoS("Finding the CRI API image version") + imageClientV1 := runtimeapi.NewImageServiceClient(conn) + + _, err := imageClientV1.ImageFsInfo(ctx, &runtimeapi.ImageFsInfoRequest{}) + if err == nil { + klog.V(2).InfoS("Using CRI v1 image API") + return imageClientV1, nil, nil + + } else if status.Code(err) == codes.Unimplemented { + klog.V(2).InfoS("Falling back to CRI v1alpha2 image API (deprecated in k8s 1.24)") + return nil, runtimeapiv1alpha2.NewImageServiceClient(conn), nil + } + + return nil, nil, fmt.Errorf("unable to determine image API version: %w", err) +}