feat: support image policy

Signed-off-by: HIHIA <283304489@qq.com>

cmd/sealer/cmd/cluster/run.go

@@ -392,7 +392,7 @@ func installApplication(appImageName string, envs []string, app *v2.Application,
  return loadToRegistry(infraDriver, distributor)
  }
 
- installer := clusterruntime.NewAppInstaller(infraDriver, distributor, extension)
+ installer := clusterruntime.NewAppInstaller(infraDriver, distributor, extension, imageEngine)
  err = installer.Install(infraDriver.GetHostIPListByRole(common.MASTER)[0], v2App.GetImageLaunchCmds())
  if err != nil {
  return err

common/common.go

@@ -127,6 +127,72 @@ const (
  WINDOWS = "windows"
 )
 
+const (
+ ImagePolicyLabelKey = "app.alpha.sealer.io/image-policy-plugin"
+ ImagePolicyPluginKyverno = "kyverno"
+ ImagePolicyTemplateYamlName = "image-policy-template.yaml"
+ ImagePolicyTemplate = `
+apiVersion : kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+ name: [[.name]]-redirect-registry
+ namespace: kube-system
+spec:
+ background: false
+ rules:
+ - name: prepend-registry-containers
+ match:
+ resources:
+ kinds:
+ - Pod
+ preconditions:
+ all:
+ - key: "{{request.operation}}"
+ operator: In
+ value:
+ - CREATE
+ - UPDATE
+ mutate:
+ foreach:
+ - list: "request.object.spec.containers"
+ preconditions:
+ all:
+ - key: "{{element.image}}"
+ operator: AnyIn
+ value: [[.imageList]]
+ patchStrategicMerge:
+ spec:
+ containers:
+ - name: "{{ element.name }}"
+ image: "[[.registry]]/{{ images.containers.{{element.name}}.path}}:{{images.containers.{{element.name}}.tag}}"
+ - name: prepend-registry-initcontainers
+ match:
+ resources:
+ kinds:
+ - Pod
+ preconditions:
+ all:
+ - key: "{{request.operation}}"
+ operator: In
+ value:
+ - CREATE
+ - UPDATE
+ mutate:
+ foreach:
+ - list: "request.object.spec.initContainers"
+ preconditions:
+ all:
+ - key: "{{element.image}}"
+ operator: AnyIn
+ value: [[.imageList]]
+ patchStrategicMerge:
+ spec:
+ initContainers:
+ - name: "{{ element.name }}"
+ image: "[[.registry]]/{{ images.initContainers.{{element.name}}.path}}:{{images.initContainers.{{element.name}}.tag}}"
+`
+)
+
 func GetSealerWorkDir() string {
  return filepath.Join(GetHomeDir(), ".sealer")
 }

pkg/cluster-runtime/apps.go

@@ -26,7 +26,10 @@ import (
  "github.com/sealerio/sealer/common"
  containerruntime "github.com/sealerio/sealer/pkg/container-runtime"
  v12 "github.com/sealerio/sealer/pkg/define/image/v1"
+ "github.com/sealerio/sealer/pkg/define/options"
  "github.com/sealerio/sealer/pkg/imagedistributor"
+ "github.com/sealerio/sealer/pkg/imageengine"
+ "github.com/sealerio/sealer/pkg/imagepolicy/kyverno"
  "github.com/sealerio/sealer/pkg/infradriver"
  "github.com/sealerio/sealer/pkg/registry"
  "github.com/sirupsen/logrus"
@@ -36,13 +39,15 @@ type AppInstaller struct {
  infraDriver infradriver.InfraDriver
  distributor imagedistributor.Distributor
  extension v12.ImageExtension
+ imageEngine imageengine.Interface
 }
 
-func NewAppInstaller(infraDriver infradriver.InfraDriver, distributor imagedistributor.Distributor, extension v12.ImageExtension) AppInstaller {
+func NewAppInstaller(infraDriver infradriver.InfraDriver, distributor imagedistributor.Distributor, extension v12.ImageExtension, imageEngine imageengine.Interface) AppInstaller {
  return AppInstaller{
  infraDriver: infraDriver,
  distributor: distributor,
  extension: extension,
+ imageEngine: imageEngine,
  }
 }
 
@@ -90,6 +95,20 @@ func (i AppInstaller) Launch(master0 net.IP, launchCmds []string) error {
  ex = shell.NewLex('\\')
  )
 
+ var imagePolicyEngine *kyverno.Engine
+ clusterImageName := i.infraDriver.GetClusterImageName()
+ extension, err := i.imageEngine.GetSealerImageExtension(&options.GetImageAnnoOptions{ImageNameOrID: clusterImageName})
+ if err != nil {
+ return err
+ }
+ val, ok := extension.Labels[common.ImagePolicyLabelKey]
+ if ok && val == common.ImagePolicyPluginKyverno {
+ imagePolicyEngine, err = kyverno.NewKyvernoImagePolicyEngine()
+ if err != nil {
+ return err
+ }
+ }
+
  for _, value := range launchCmds {
  if value == "" {
  continue
@@ -102,6 +121,16 @@ func (i AppInstaller) Launch(master0 net.IP, launchCmds []string) error {
  if err = i.infraDriver.CmdAsync(master0, fmt.Sprintf(common.CdAndExecCmd, rootfsPath, cmdline)); err != nil {
  return err
  }
+
+ if imagePolicyEngine != nil {
+ ok, err := imagePolicyEngine.IsImagePolicyApp(cmdline)
+ if err != nil {
+ return err
+ }
+ if ok {
+ imagePolicyEngine.CreateImagePolicyRule(i.infraDriver, i.imageEngine, cmdline)
+ }
+ }
  }
 
  return i.save(common.GetDefaultApplicationFile())

pkg/cluster-runtime/installer.go

@@ -202,7 +202,7 @@ func (i *Installer) Install() error {
  return err
  }
 
- appInstaller := NewAppInstaller(i.infraDriver, i.Distributor, extension)
+ appInstaller := NewAppInstaller(i.infraDriver, i.Distributor, extension, i.ImageEngine)
 
  v2App, err := application.NewV2Application(v2.ConstructApplication(i.Application, cmds, appNames), extension)
  if err != nil {

pkg/imageengine/buildah/sealer_image_extension.go

@@ -51,6 +51,16 @@ func GetImageExtensionFromAnnotations(annotations map[string]string) (image_v1.I
  return extension, nil
 }
 
+func (engine *Engine) GetSealerContainerImageList(opts *options.GetImageAnnoOptions) ([]*image_v1.ContainerImage, error) {
+ annotation, err := engine.GetImageAnnotation(opts)
+ result, err := GetContainerImagesFromAnnotations(annotation)
+ if err != nil {
+ return []*image_v1.ContainerImage{}, errors.Wrapf(err, "failed to get %s in image %s", image_v1.SealerImageContainerImageList, opts.ImageNameOrID)
+ }
+
+ return result, nil
+}
+
 func GetContainerImagesFromAnnotations(annotations map[string]string) ([]*image_v1.ContainerImage, error) {
  var containerImageList []*image_v1.ContainerImage
  annotationStr := annotations[image_v1.SealerImageContainerImageList]

pkg/imageengine/interface.go

@@ -61,6 +61,8 @@ type Interface interface {
 
  GetSealerImageExtension(opts *options.GetImageAnnoOptions) (v1.ImageExtension, error)
 
+ GetSealerContainerImageList(opts *options.GetImageAnnoOptions) ([]*v1.ContainerImage, error)
+
  LookupManifest(name string) (*libimage.ManifestList, error)
 
  CreateManifest(name string, opts *options.ManifestCreateOpts) error

pkg/imagepolicy/common.go

@@ -0,0 +1,21 @@
+// Copyright © 2023 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package imagepolicy
+
+import "github.com/sealerio/sealer/pkg/imagepolicy/kyverno"
+
+func NewImagePolicyEngine() (Interface, error) {
+ return kyverno.NewKyvernoImagePolicyEngine()
+}

pkg/imagepolicy/interface.go

@@ -0,0 +1,26 @@
+// Copyright © 2023 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package imagepolicy
+
+import (
+ "github.com/sealerio/sealer/pkg/imageengine"
+ "github.com/sealerio/sealer/pkg/infradriver"
+)
+
+type Interface interface {
+ IsImagePolicyApp(appName string) (bool, error)
+
+ CreateImagePolicyRule(infraDriver infradriver.InfraDriver, imageEngine imageengine.Interface, appName string) error
+}

pkg/imagepolicy/kyverno/engine.go

@@ -0,0 +1,113 @@
+// Copyright © 2023 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kyverno
+
+import (
+ "context"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "strings"
+
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
+ "k8s.io/apimachinery/pkg/runtime/serializer"
+ runtimeClient "sigs.k8s.io/controller-runtime/pkg/client"
+
+ kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
+ "github.com/sealerio/sealer/common"
+ imagecommon "github.com/sealerio/sealer/pkg/define/options"
+ "github.com/sealerio/sealer/pkg/imageengine"
+ "github.com/sealerio/sealer/pkg/infradriver"
+ "github.com/sealerio/sealer/pkg/rootfs"
+ "github.com/sealerio/sealer/pkg/runtime"
+ k "github.com/sealerio/sealer/pkg/runtime/kubernetes"
+ apimachineryRuntime "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/client-go/kubernetes/scheme"
+)
+
+type Engine struct {
+ driver runtime.Driver
+}
+
+func NewKyvernoImagePolicyEngine() (*Engine, error) {
+ driver, err := k.NewKubeDriver(k.AdminKubeConfPath)
+ if err != nil {
+ return nil, err
+ }
+
+ return &Engine{
+ driver: driver,
+ }, nil
+}
+
+// TODO: make it configurable
+func (engine *Engine) IsImagePolicyApp(appName string) (bool, error) {
+ return appName == "kyverno", nil
+}
+
+func (engine *Engine) CreateImagePolicyRule(infraDriver infradriver.InfraDriver, imageEngine imageengine.Interface, appName string) error {
+ imagePolicyTemplateStr := common.ImagePolicyTemplate
+ templatePath := filepath.Clean(filepath.Join(infraDriver.GetClusterRootfsPath(), rootfs.GlobalManager.App().Root(), appName, common.ImagePolicyTemplateYamlName))
+ if _, err := os.Stat(templatePath); err != nil {
+ if !os.IsNotExist(err) {
+ return err
+ }
+ } else {
+ yamlFile, err := ioutil.ReadFile(templatePath)
+ if err != nil {
+ return err
+ }
+ imagePolicyTemplateStr = string(yamlFile)
+ }
+
+ clusterImageName := infraDriver.GetClusterImageName()
+ imageList, err := imageEngine.GetSealerContainerImageList(&imagecommon.GetImageAnnoOptions{ImageNameOrID: clusterImageName})
+ clusterPolicy := &kyvernov1.ClusterPolicy{}
+
+ var imageListArray []string
+ for _, containerImage := range imageList {
+ imageListArray = append(imageListArray, "\""+containerImage.Image+"\"")
+ }
+
+ ctx := map[string]string{
+ "name": clusterImageName,
+ "imageList": "[" + strings.Join(imageListArray, ",") + "]",
+ "registry": infraDriver.GetClusterRegistry().LocalRegistry.Domain,
+ }
+ imagePolicyTemplate, err := SubsituteTemplate(imagePolicyTemplateStr, ctx)
+ if err != nil {
+ return err
+ }
+
+ if err := kyvernov1.AddToScheme(scheme.Scheme); err != nil {
+ return err
+ }
+ decoder := serializer.NewCodecFactory(scheme.Scheme).UniversalDecoder()
+ if err := apimachineryRuntime.DecodeInto(decoder, []byte(imagePolicyTemplate), clusterPolicy); err != nil {
+ return err
+ }
+
+ if err := engine.driver.Create(context.Background(), clusterPolicy, &runtimeClient.CreateOptions{}); err != nil {
+ if !apierrors.IsAlreadyExists(err) {
+ return fmt.Errorf("unable to create image policy: %v", err)
+ }
+
+ if err := engine.driver.Update(context.Background(), clusterPolicy, &runtimeClient.UpdateOptions{}); err != nil {
+ return fmt.Errorf("unable to update image policy: %v", err)
+ }
+ }
+ return nil
+}

pkg/imagepolicy/kyverno/util.go

@@ -0,0 +1,35 @@
+// Copyright © 2023 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kyverno
+
+import (
+ "bytes"
+ "text/template"
+)
+
+// SubsituteTemplate fills out the templates based on the context
+func SubsituteTemplate(tmpl string, context interface{}) (string, error) {
+ // Use [[]] as identifier to avoid conflicts
+ t, tmplPrsErr := template.New("test").Delims("[[", "]]").Option("missingkey=zero").Parse(tmpl)
+ if tmplPrsErr != nil {
+ return "", tmplPrsErr
+ }
+ writer := bytes.NewBuffer([]byte{})
+ if err := t.Execute(writer, context); nil != err {
+ return "", err
+ }
+
+ return writer.String(), nil
+}