From 91e6fd3648f36a38243ff60767a2129bd6e07eb6 Mon Sep 17 00:00:00 2001 From: Danil Grigorev Date: Sun, 3 Mar 2024 22:47:39 +0100 Subject: [PATCH] Add GCP support and separate out plugin implementations Signed-off-by: Danil Grigorev --- api/v1alpha1/veleroinstallation_operations.go | 2 + api/v1alpha1/veleroinstallation_types.go | 47 ++++ api/v1alpha1/zz_generated.deepcopy.go | 37 +++ ....cluster.x-k8s.io_veleroinstallations.yaml | 62 +++++ .../samples/gcp/_v1alpha1_velerobackup.yaml | 17 ++ .../gcp/_v1alpha1_veleroinstallation.yaml | 24 ++ .../samples/gcp/_v1alpha1_velerorestore.yaml | 18 ++ .../gcp/_v1alpha1_velerorestoreschedule.yaml | 19 ++ .../samples/gcp/_v1alpha1_veleroschedule.yaml | 19 ++ hack/setup-velero-bucket-gcp.sh | 46 ++++ .../controller/velerobackup_controller.go | 4 +- .../veleroinstallation_controller.go | 223 ++---------------- .../controller/velerorestore_controller.go | 7 +- .../controller/veleroschedule_controller.go | 4 +- internal/plugin/aws.go | 72 ++++++ internal/plugin/azure.go | 70 ++++++ internal/plugin/gcp.go | 73 ++++++ internal/plugin/generic.go | 183 ++++++++++++++ internal/plugin/interface.go | 34 +++ 19 files changed, 750 insertions(+), 211 deletions(-) create mode 100644 config/samples/gcp/_v1alpha1_velerobackup.yaml create mode 100644 config/samples/gcp/_v1alpha1_veleroinstallation.yaml create mode 100644 config/samples/gcp/_v1alpha1_velerorestore.yaml create mode 100644 config/samples/gcp/_v1alpha1_velerorestoreschedule.yaml create mode 100644 config/samples/gcp/_v1alpha1_veleroschedule.yaml create mode 100644 hack/setup-velero-bucket-gcp.sh create mode 100644 internal/plugin/aws.go create mode 100644 internal/plugin/azure.go create mode 100644 internal/plugin/gcp.go create mode 100644 internal/plugin/generic.go create mode 100644 internal/plugin/interface.go diff --git a/api/v1alpha1/veleroinstallation_operations.go b/api/v1alpha1/veleroinstallation_operations.go index 09f4008..f39c8e0 100644 --- a/api/v1alpha1/veleroinstallation_operations.go +++ b/api/v1alpha1/veleroinstallation_operations.go @@ -22,6 +22,8 @@ func (p Provider) Name() string { return "aws" case p.Azure != nil: return "azure" + case p.GCP != nil: + return "velero.io/gcp" default: panic("Unknown type of provider supplied") } diff --git a/api/v1alpha1/veleroinstallation_types.go b/api/v1alpha1/veleroinstallation_types.go index 7dab34d..c8e3a40 100644 --- a/api/v1alpha1/veleroinstallation_types.go +++ b/api/v1alpha1/veleroinstallation_types.go @@ -49,6 +49,7 @@ type VeleroInstallationSpec struct { type Provider struct { AWS *AWS `json:"aws,omitempty"` Azure *Azure `json:"azure,omitempty"` + GCP *GCP `json:"gcp,omitempty"` } type AWS struct { @@ -77,6 +78,18 @@ type Azure struct { Config AzureConfig `json:"config,omitempty"` } +type GCP struct { + // +optional + PluginURL string `json:"pluginURL"` + + // +optional + PluginTag string `json:"pluginTag"` + + CredentialMap CredentialMap `json:"credentialMap,omitempty"` + + Config GCPConfig `json:"config,omitempty"` +} + type AWSConfig struct { // +optional Region string `json:"region,omitempty"` @@ -100,6 +113,40 @@ type AzureConfig struct { // +optional SubscriptionId string `json:"subscriptionId"` } + +type GCPConfig struct { + // Name of the GCP service account to use for this backup storage location. Specify the + // service account here if you want to use workload identity instead of providing the key file. + // + // Optional (defaults to "false"). + // +optional + ServiceAccount string `json:"serviceAccount"` + + // Name of the Cloud KMS key to use to encrypt backups stored in this location, in the form + // "projects/P/locations/L/keyRings/R/cryptoKeys/K". See customer-managed Cloud KMS keys + // (https://cloud.google.com/storage/docs/encryption/using-customer-managed-keys) for details. + // +optional + KMSKeyName string `json:"kmsKeyName"` + + // The GCP location where snapshots should be stored. See the GCP documentation + // (https://cloud.google.com/storage/docs/locations#available_locations) for the + // full list. If not specified, snapshots are stored in the default location + // (https://cloud.google.com/compute/docs/disks/create-snapshots#default_location). + // + // Example: us-central1 + // +optional + SnapshotLocation string `json:"snapshotLocation,omitempty"` + + // The project ID where existing snapshots should be retrieved from during restores, if + // different than the project that your IAM account is in. This field has no effect on + // where new snapshots are created; it is only useful for restoring existing snapshots + // from a different project. + // + // Optional (defaults to the project that the GCP IAM account is in). + // Example: my-alternate-project + Project string `json:"project,omitempty"` +} + type VeleroHelmState struct { DeployNodeAgent bool `json:"deployNodeAgent"` CleanUpCRDs bool `json:"cleanUpCRDs"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index abab28a..12739bc 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -278,6 +278,38 @@ func (in *Credentials) DeepCopy() *Credentials { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GCP) DeepCopyInto(out *GCP) { + *out = *in + out.CredentialMap = in.CredentialMap + out.Config = in.Config +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GCP. +func (in *GCP) DeepCopy() *GCP { + if in == nil { + return nil + } + out := new(GCP) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GCPConfig) DeepCopyInto(out *GCPConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GCPConfig. +func (in *GCPConfig) DeepCopy() *GCPConfig { + if in == nil { + return nil + } + out := new(GCPConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Provider) DeepCopyInto(out *Provider) { *out = *in @@ -291,6 +323,11 @@ func (in *Provider) DeepCopyInto(out *Provider) { *out = new(Azure) **out = **in } + if in.GCP != nil { + in, out := &in.GCP, &out.GCP + *out = new(GCP) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Provider. diff --git a/config/crd/bases/addons.cluster.x-k8s.io_veleroinstallations.yaml b/config/crd/bases/addons.cluster.x-k8s.io_veleroinstallations.yaml index 03ad07b..1d2da12 100644 --- a/config/crd/bases/addons.cluster.x-k8s.io_veleroinstallations.yaml +++ b/config/crd/bases/addons.cluster.x-k8s.io_veleroinstallations.yaml @@ -394,6 +394,68 @@ spec: pluginURL: type: string type: object + gcp: + properties: + config: + properties: + kmsKeyName: + description: |- + Name of the Cloud KMS key to use to encrypt backups stored in this location, in the form + "projects/P/locations/L/keyRings/R/cryptoKeys/K". See customer-managed Cloud KMS keys + (https://cloud.google.com/storage/docs/encryption/using-customer-managed-keys) for details. + type: string + project: + description: |- + The project ID where existing snapshots should be retrieved from during restores, if + different than the project that your IAM account is in. This field has no effect on + where new snapshots are created; it is only useful for restoring existing snapshots + from a different project. + + + Optional (defaults to the project that the GCP IAM account is in). + Example: my-alternate-project + type: string + serviceAccount: + description: |- + Name of the GCP service account to use for this backup storage location. Specify the + service account here if you want to use workload identity instead of providing the key file. + + + Optional (defaults to "false"). + type: string + snapshotLocation: + description: |- + The GCP location where snapshots should be stored. See the GCP documentation + (https://cloud.google.com/storage/docs/locations#available_locations) for the + full list. If not specified, snapshots are stored in the default location + (https://cloud.google.com/compute/docs/disks/create-snapshots#default_location). + + + Example: us-central1 + type: string + type: object + credentialMap: + properties: + from: + type: string + namespaceName: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + to: + type: string + type: object + pluginTag: + type: string + pluginURL: + type: string + type: object type: object state: properties: diff --git a/config/samples/gcp/_v1alpha1_velerobackup.yaml b/config/samples/gcp/_v1alpha1_velerobackup.yaml new file mode 100644 index 0000000..d925623 --- /dev/null +++ b/config/samples/gcp/_v1alpha1_velerobackup.yaml @@ -0,0 +1,17 @@ +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: VeleroBackup +metadata: + labels: + app.kubernetes.io/name: velerobackup + app.kubernetes.io/instance: gcpbackup-sample + app.kubernetes.io/part-of: cluster-api-addon-provider-velero + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: cluster-api-addon-provider-velero + name: gcpbackup-sample + namespace: creategitops-me7ee7 +spec: + installation: + ref: + kind: VeleroInstallation + name: gcpinstallation-sample + namespace: creategitops-me7ee7 \ No newline at end of file diff --git a/config/samples/gcp/_v1alpha1_veleroinstallation.yaml b/config/samples/gcp/_v1alpha1_veleroinstallation.yaml new file mode 100644 index 0000000..59d56f6 --- /dev/null +++ b/config/samples/gcp/_v1alpha1_veleroinstallation.yaml @@ -0,0 +1,24 @@ +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: VeleroInstallation +metadata: + labels: + app.kubernetes.io/name: veleroinstallation + app.kubernetes.io/instance: gcpinstallation-sample + app.kubernetes.io/part-of: cluster-api-addon-provider-velero + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: cluster-api-addon-provider-velero + name: gcpinstallation-sample + namespace: creategitops-me7ee7 +spec: + bucket: dgrigore-bucket + namespace: velero-gcp + provider: + gcp: + credentialMap: + namespaceName: + name: gcp-credentials + namespace: default + state: + deployNodeAgent: true + cleanUpCRDs: true + credentials: {} diff --git a/config/samples/gcp/_v1alpha1_velerorestore.yaml b/config/samples/gcp/_v1alpha1_velerorestore.yaml new file mode 100644 index 0000000..35bf4cf --- /dev/null +++ b/config/samples/gcp/_v1alpha1_velerorestore.yaml @@ -0,0 +1,18 @@ +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: VeleroRestore +metadata: + labels: + app.kubernetes.io/name: velerorestore + app.kubernetes.io/instance: gcprestore-sample + app.kubernetes.io/part-of: cluster-api-addon-provider-velero + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: cluster-api-addon-provider-velero + name: gcprestore-sample + namespace: creategitops-me7ee7 +spec: + backupName: gcpbackup-sample + installation: + ref: + kind: VeleroInstallation + name: gcpinstallation-sample + namespace: creategitops-me7ee7 \ No newline at end of file diff --git a/config/samples/gcp/_v1alpha1_velerorestoreschedule.yaml b/config/samples/gcp/_v1alpha1_velerorestoreschedule.yaml new file mode 100644 index 0000000..3b976c5 --- /dev/null +++ b/config/samples/gcp/_v1alpha1_velerorestoreschedule.yaml @@ -0,0 +1,19 @@ +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: VeleroRestore +metadata: + labels: + app.kubernetes.io/name: velerorestore + app.kubernetes.io/instance: gcpschedule-sample + app.kubernetes.io/part-of: cluster-api-addon-provider-velero + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: cluster-api-addon-provider-velero + name: gcprestoreshedule-sample + namespace: creategitops-me7ee7 +spec: + backupName: "" + scheduleName: gcpschedule-sample + installation: + ref: + kind: VeleroInstallation + name: gcpinstallation-sample + namespace: creategitops-me7ee7 diff --git a/config/samples/gcp/_v1alpha1_veleroschedule.yaml b/config/samples/gcp/_v1alpha1_veleroschedule.yaml new file mode 100644 index 0000000..65ce1b3 --- /dev/null +++ b/config/samples/gcp/_v1alpha1_veleroschedule.yaml @@ -0,0 +1,19 @@ +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: VeleroSchedule +metadata: + labels: + app.kubernetes.io/name: veleroschedule + app.kubernetes.io/instance: gcpschedule-sample + app.kubernetes.io/part-of: cluster-api-addon-provider-velero + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: cluster-api-addon-provider-velero + name: gcpschedule-sample + namespace: creategitops-me7ee7 +spec: + template: {} + schedule: "* * * * *" + installation: + ref: + kind: VeleroInstallation + name: gcpinstallation-sample + namespace: creategitops-me7ee7 diff --git a/hack/setup-velero-bucket-gcp.sh b/hack/setup-velero-bucket-gcp.sh new file mode 100644 index 0000000..894dbc7 --- /dev/null +++ b/hack/setup-velero-bucket-gcp.sh @@ -0,0 +1,46 @@ +BUCKET=dgrigore-bucket +gsutil mb gs://$BUCKET/ + +PROJECT_ID=$(gcloud config get-value project) +GSA_NAME=velero-dgrigore +gcloud iam service-accounts create $GSA_NAME \ + --display-name "Velero service account dgrigore" + +SERVICE_ACCOUNT_EMAIL=$(gcloud iam service-accounts list \ + --filter="displayName:Velero service account" \ + --format 'value(email)') + +ROLE_PERMISSIONS=( + compute.disks.get + compute.disks.create + compute.disks.createSnapshot + compute.projects.get + compute.snapshots.get + compute.snapshots.create + compute.snapshots.useReadOnly + compute.snapshots.delete + compute.zones.get + storage.objects.create + storage.objects.delete + storage.objects.get + storage.objects.list + iam.serviceAccounts.signBlob +) + +gcloud iam roles create dgrigorevelero.server \ + --project $PROJECT_ID \ + --title "Velero Server" \ + --permissions "$(IFS=","; echo "${ROLE_PERMISSIONS[*]}")" + +gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member serviceAccount:$SERVICE_ACCOUNT_EMAIL \ + --role projects/$PROJECT_ID/roles/dgrigorevelero.server + +gsutil iam ch serviceAccount:$SERVICE_ACCOUNT_EMAIL:objectAdmin gs://${BUCKET} + +gcloud iam service-accounts keys create credentials-velero \ + --iam-account $SERVICE_ACCOUNT_EMAIL + +kubectl create secret generic -n default gcp-credentials --from-file=gcp=credentials-velero + +rm credentials-velero \ No newline at end of file diff --git a/internal/controller/velerobackup_controller.go b/internal/controller/velerobackup_controller.go index 7da1f69..2664910 100644 --- a/internal/controller/velerobackup_controller.go +++ b/internal/controller/velerobackup_controller.go @@ -59,12 +59,12 @@ func (r *VeleroBackupReconciler) SetupWithManager(ctx context.Context, mgr ctrl. // // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.17.0/pkg/reconcile -func (r *VeleroBackupReconciler) Reconcile(ctx context.Context, clusterRef client.ObjectKey, installation *veleroaddonv1.VeleroInstallation, backup *veleroaddonv1.VeleroBackup) (ctrl.Result, error) { +func (r *VeleroBackupReconciler) Reconcile(ctx context.Context, _ client.ObjectKey, installation *veleroaddonv1.VeleroInstallation, backup *veleroaddonv1.VeleroBackup) (ctrl.Result, error) { _ = log.FromContext(ctx) r.Backup = &velerov1.Backup{ ObjectMeta: metav1.ObjectMeta{ - Name: backup.Name + "-" + clusterRef.Name, + Name: backup.Name, Namespace: cmp.Or(installation.Spec.HelmSpec.ReleaseNamespace, installation.Spec.Namespace, "velero"), Annotations: map[string]string{ proxyKeyAnnotation: string(veleroaddonv1.ToNamespaceName(backup)), diff --git a/internal/controller/veleroinstallation_controller.go b/internal/controller/veleroinstallation_controller.go index 1ba1849..f31d1ef 100644 --- a/internal/controller/veleroinstallation_controller.go +++ b/internal/controller/veleroinstallation_controller.go @@ -17,25 +17,17 @@ limitations under the License. package controller import ( - "cmp" "context" - "slices" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - kerrors "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/utils/ptr" "sigs.k8s.io/cluster-api/controllers/remote" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/yaml" veleroaddonv1 "addons.cluster.x-k8s.io/cluster-api-addon-provider-velero/api/v1alpha1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "addons.cluster.x-k8s.io/cluster-api-addon-provider-velero/internal/plugin" helmv1 "sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" ) @@ -61,201 +53,28 @@ type VeleroInstallationReconciler struct { // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.17.0/pkg/reconcile func (r *VeleroInstallationReconciler) Reconcile(ctx context.Context, installation *veleroaddonv1.VeleroInstallation) (ctrl.Result, error) { _ = log.FromContext(ctx) - locations := installation.Spec.State.Configuration.BackupStorageLocations - snapshotLocations := installation.Spec.State.Configuration.VolumeSnapshotLocations - index, snapshotIndex := -1, -1 - from, fromNamespace := "", "" - provider := installation.Spec.Provider - location := veleroaddonv1.BackupStorageLocation{ - Provider: provider.Name(), - } - snapshotLocation := veleroaddonv1.VolumeSnapshotLocation{ - Provider: provider.Name(), - } - if index = slices.IndexFunc(locations, func(l veleroaddonv1.BackupStorageLocation) bool { - return l.Name == ptr.To(provider.Name()) - }); index > -1 { - location = locations[index] - } - - if snapshotIndex = slices.IndexFunc(snapshotLocations, func(l veleroaddonv1.VolumeSnapshotLocation) bool { - return l.Name == ptr.To(provider.Name()) - }); snapshotIndex > -1 { - snapshotLocation = snapshotLocations[snapshotIndex] - } switch { - case provider.AWS != nil: - location.Bucket = installation.Spec.Bucket - location.Config = map[string]string{ - "s3Url": provider.AWS.Config.S3Url, - "region": provider.AWS.Config.Region, - } - - snapshotLocation.Config = map[string]string{ - "region": provider.AWS.Config.Region, - } - - image := cmp.Or(installation.Spec.Provider.AWS.PluginURL, "velero/velero-plugin-for-aws") - tag := cmp.Or(installation.Spec.Provider.AWS.PluginTag, "latest") - installation.Spec.State.InitContainers = []corev1.Container{{ - Name: "velero-plugin-for-aws", - Image: image + ":" + tag, - ImagePullPolicy: corev1.PullIfNotPresent, - VolumeMounts: []corev1.VolumeMount{{ - Name: "plugins", - MountPath: "/target", - }}, - }} - - from = cmp.Or(provider.AWS.CredentialMap.NamespaceName.Name, provider.AWS.CredentialMap.From) - fromNamespace = cmp.Or(provider.AWS.CredentialMap.NamespaceName.Namespace, installation.Namespace) - location.CredentialKey = veleroaddonv1.CredentialKey{ - Name: cmp.Or(provider.AWS.CredentialMap.To, from), - Key: provider.Name(), - } - snapshotLocation.CredentialKey = veleroaddonv1.CredentialKey{ - Name: cmp.Or(provider.AWS.CredentialMap.To, from), - Key: provider.Name(), - } - - case provider.Azure != nil: - location.Bucket = installation.Spec.Bucket - location.Config = map[string]string{ - "resourceGroup": provider.Azure.Config.ResourceGroup, - "storageAccount": provider.Azure.Config.StorageAccount, - "storageAccountKeyEnvVar": cmp.Or(provider.Azure.Config.StorageAccountKeyEnvVar, "AZURE_STORAGE_ACCOUNT_ACCESS_KEY"), - } - - snapshotLocation.Config = map[string]string{} - - image := cmp.Or(installation.Spec.Provider.Azure.PluginURL, "velero/velero-plugin-for-microsoft-azure") - tag := cmp.Or(installation.Spec.Provider.Azure.PluginTag, "latest") - installation.Spec.State.InitContainers = []corev1.Container{{ - Name: "velero-plugin-for-microsoft-azure", - Image: image + ":" + tag, - ImagePullPolicy: corev1.PullIfNotPresent, - VolumeMounts: []corev1.VolumeMount{{ - Name: "plugins", - MountPath: "/target", - }}, - }} - - from = cmp.Or(provider.Azure.CredentialMap.NamespaceName.Name, provider.Azure.CredentialMap.From) - fromNamespace = cmp.Or(provider.Azure.CredentialMap.NamespaceName.Namespace, installation.Namespace) - location.CredentialKey = veleroaddonv1.CredentialKey{ - Name: cmp.Or(provider.Azure.CredentialMap.To, from), - Key: provider.Name(), - } - snapshotLocation.CredentialKey = veleroaddonv1.CredentialKey{ - Name: cmp.Or(provider.Azure.CredentialMap.To, from), - Key: provider.Name(), - } - } - - // Plugins / values - if index > -1 { - locations[index] = location - } else { - locations = append(locations, location) - } - - if snapshotIndex > -1 { - snapshotLocations[snapshotIndex] = snapshotLocation - } else { - snapshotLocations = append(snapshotLocations, snapshotLocation) - } - - installation.Spec.State.Configuration.BackupStorageLocations = locations - installation.Spec.State.Configuration.VolumeSnapshotLocations = snapshotLocations - - // Secret sync - secret := &corev1.Secret{} - if err := r.Client.Get(ctx, types.NamespacedName{ - Name: from, - Namespace: fromNamespace, - }, secret); err != nil { - return ctrl.Result{}, err - } - - var errs []error - for _, cluster := range installation.Status.MatchingClusters { - cl, err := r.Tracker.GetClient(ctx, veleroaddonv1.RefToNamespaceName(&cluster).ObjectKey()) - if err != nil { - errs = append(errs, client.IgnoreNotFound(err)) - - continue - } - - newSecret := &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Secret", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: location.CredentialKey.Name, - Namespace: cmp.Or(installation.Spec.HelmSpec.ReleaseNamespace, installation.Spec.Namespace, "velero"), - }, - Data: secret.Data, - } - - errs = append(errs, cl.Patch(ctx, newSecret, client.Apply, client.ForceOwnership, client.FieldOwner("velero-addon"))) - } - - if err := kerrors.NewAggregate(errs); err != nil { - return ctrl.Result{}, err - } - - spec, err := yaml.Marshal(installation.Spec.State) - if err != nil { - return ctrl.Result{}, err - } - - helmProxy := templateHelmChartProxy(installation, installation.Spec.HelmSpec, spec) - if err := controllerutil.SetOwnerReference(installation, helmProxy, r.Client.Scheme()); err != nil { - return ctrl.Result{}, err - } - - if err := r.Client.Patch( - ctx, helmProxy, - client.Apply, client.ForceOwnership, client.FieldOwner("velero-addon")); err != nil { - return ctrl.Result{}, err - } - - installation.Status.HelmChartProxyStatus = helmProxy.Status - - return ctrl.Result{}, r.Client.Status().Update(ctx, installation) -} - -func templateHelmChartProxy(installation *veleroaddonv1.VeleroInstallation, helmSpec helmv1.HelmChartProxySpec, values []byte) *helmv1.HelmChartProxy { - clusterSelector := helmSpec.ClusterSelector - if installation.Spec.ClusterSelector.MatchExpressions != nil || installation.Spec.ClusterSelector.MatchLabels != nil { - clusterSelector = installation.Spec.ClusterSelector - } - - options := cmp.Or(helmSpec.Options, helmv1.HelmOptions{ - Install: helmv1.HelmInstallOptions{ - CreateNamespace: true, - }, - }) - - return &helmv1.HelmChartProxy{ - TypeMeta: metav1.TypeMeta{ - APIVersion: helmv1.GroupVersion.String(), - Kind: "HelmChartProxy", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: installation.Name, - Namespace: installation.Namespace, - }, - Spec: helmv1.HelmChartProxySpec{ - ClusterSelector: clusterSelector, - ReleaseNamespace: cmp.Or(installation.Spec.Namespace, "velero"), - RepoURL: cmp.Or(helmSpec.RepoURL, "https://vmware-tanzu.github.io/helm-charts"), - ChartName: cmp.Or(helmSpec.ChartName, "velero"), - ValuesTemplate: cmp.Or(helmSpec.ValuesTemplate, string(values)), - Options: options, - }, + case installation.Spec.Provider.AWS != nil: + return (&plugin.PluginReconciler[*veleroaddonv1.AWS]{ + Client: r.Client, + Scheme: r.Scheme, + Tracker: r.Tracker, + }).Reconcile(ctx, installation, &plugin.AWSPlugin{}, installation.Spec.Provider.AWS) + case installation.Spec.Provider.Azure != nil: + return (&plugin.PluginReconciler[*veleroaddonv1.Azure]{ + Client: r.Client, + Scheme: r.Scheme, + Tracker: r.Tracker, + }).Reconcile(ctx, installation, &plugin.AzurePlugin{}, installation.Spec.Provider.Azure) + case installation.Spec.Provider.GCP != nil: + return (&plugin.PluginReconciler[*veleroaddonv1.GCP]{ + Client: r.Client, + Scheme: r.Scheme, + Tracker: r.Tracker, + }).Reconcile(ctx, installation, &plugin.GCPPlugin{}, installation.Spec.Provider.GCP) + default: + return ctrl.Result{}, nil } } diff --git a/internal/controller/velerorestore_controller.go b/internal/controller/velerorestore_controller.go index e1badb5..dbb2394 100644 --- a/internal/controller/velerorestore_controller.go +++ b/internal/controller/velerorestore_controller.go @@ -60,20 +60,17 @@ func (r *VeleroRestoreReconciler) SetupWithManager(ctx context.Context, mgr ctrl // // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.17.0/pkg/reconcile -func (r *VeleroRestoreReconciler) Reconcile(ctx context.Context, clusterRef client.ObjectKey, installation *veleroaddonv1.VeleroInstallation, restore *veleroaddonv1.VeleroRestore) (ctrl.Result, error) { +func (r *VeleroRestoreReconciler) Reconcile(ctx context.Context, _ client.ObjectKey, installation *veleroaddonv1.VeleroInstallation, restore *veleroaddonv1.VeleroRestore) (ctrl.Result, error) { _ = log.FromContext(ctx) spec := restore.Spec.RestoreSpec - - spec.BackupName = spec.BackupName + "-" + clusterRef.Name if spec.ScheduleName != "" { spec.BackupName = "" - spec.ScheduleName = spec.ScheduleName + "-" + clusterRef.Name } r.Restore = &velerov1.Restore{ ObjectMeta: metav1.ObjectMeta{ - Name: restore.Name + "-" + clusterRef.Name, + Name: restore.Name, Namespace: cmp.Or(installation.Spec.HelmSpec.ReleaseNamespace, installation.Spec.Namespace, "velero"), Annotations: map[string]string{ proxyKeyAnnotation: string(veleroaddonv1.ToNamespaceName(restore)), diff --git a/internal/controller/veleroschedule_controller.go b/internal/controller/veleroschedule_controller.go index a59c8d4..af9cf93 100644 --- a/internal/controller/veleroschedule_controller.go +++ b/internal/controller/veleroschedule_controller.go @@ -60,12 +60,12 @@ func (r *VeleroScheduleReconciler) SetupWithManager(ctx context.Context, mgr ctr // // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.17.0/pkg/reconcile -func (r *VeleroScheduleReconciler) Reconcile(ctx context.Context, clusterRef client.ObjectKey, installation *veleroaddonv1.VeleroInstallation, schedule *veleroaddonv1.VeleroSchedule) (ctrl.Result, error) { +func (r *VeleroScheduleReconciler) Reconcile(ctx context.Context, _ client.ObjectKey, installation *veleroaddonv1.VeleroInstallation, schedule *veleroaddonv1.VeleroSchedule) (ctrl.Result, error) { _ = log.FromContext(ctx) r.Schedule = &velerov1.Schedule{ ObjectMeta: metav1.ObjectMeta{ - Name: schedule.Name + "-" + clusterRef.Name, + Name: schedule.Name, Namespace: cmp.Or(installation.Spec.HelmSpec.ReleaseNamespace, installation.Spec.Namespace, "velero"), Annotations: map[string]string{ proxyKeyAnnotation: string(veleroaddonv1.ToNamespaceName(schedule)), diff --git a/internal/plugin/aws.go b/internal/plugin/aws.go new file mode 100644 index 0000000..b3f11f3 --- /dev/null +++ b/internal/plugin/aws.go @@ -0,0 +1,72 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "cmp" + + veleroaddonv1 "addons.cluster.x-k8s.io/cluster-api-addon-provider-velero/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type AWSPlugin struct{} + +func (p *AWSPlugin) Plugin(installation *veleroaddonv1.VeleroInstallation, provider *veleroaddonv1.AWS) { + installation.Spec.State.InitContainers = []corev1.Container{{ + Name: "velero-plugin-for-aws", + Image: cmp.Or(provider.PluginURL, "velero/velero-plugin-for-aws") + ":" + cmp.Or(provider.PluginTag, "latest"), + ImagePullPolicy: corev1.PullIfNotPresent, + VolumeMounts: []corev1.VolumeMount{{ + Name: "plugins", + MountPath: "/target", + }}, + }} +} + +func (p *AWSPlugin) BackupStorageLocation(location veleroaddonv1.BackupStorageLocation, provider *veleroaddonv1.AWS) veleroaddonv1.BackupStorageLocation { + location.Config = map[string]string{ + "s3Url": provider.Config.S3Url, + "region": provider.Config.Region, + } + location.CredentialKey = veleroaddonv1.CredentialKey{ + Name: cmp.Or(provider.CredentialMap.To, p.Secret(provider).Name), + Key: veleroaddonv1.Provider{AWS: provider}.Name(), + } + + return location +} + +func (p *AWSPlugin) VolumeSnapshotLocation(snapshotLocation veleroaddonv1.VolumeSnapshotLocation, provider *veleroaddonv1.AWS) veleroaddonv1.VolumeSnapshotLocation { + snapshotLocation.Config = map[string]string{ + "region": provider.Config.Region, + } + snapshotLocation.CredentialKey = veleroaddonv1.CredentialKey{ + Name: cmp.Or(provider.CredentialMap.To, p.Secret(provider).Name), + Key: veleroaddonv1.Provider{AWS: provider}.Name(), + } + + return snapshotLocation +} + +func (p *AWSPlugin) Secret(provider *veleroaddonv1.AWS) client.ObjectKey { + return types.NamespacedName{ + Name: cmp.Or(provider.CredentialMap.NamespaceName.Name, provider.CredentialMap.From), + Namespace: provider.CredentialMap.NamespaceName.Namespace, + } +} diff --git a/internal/plugin/azure.go b/internal/plugin/azure.go new file mode 100644 index 0000000..9dc5fa9 --- /dev/null +++ b/internal/plugin/azure.go @@ -0,0 +1,70 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "cmp" + + veleroaddonv1 "addons.cluster.x-k8s.io/cluster-api-addon-provider-velero/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type AzurePlugin struct{} + +func (p *AzurePlugin) Plugin(installation *veleroaddonv1.VeleroInstallation, provider *veleroaddonv1.Azure) { + installation.Spec.State.InitContainers = []corev1.Container{{ + Name: "velero-plugin-for-microsoft-azure", + Image: cmp.Or(provider.PluginURL, "velero/velero-plugin-for-microsoft-azure") + ":" + cmp.Or(provider.PluginTag, "latest"), + ImagePullPolicy: corev1.PullIfNotPresent, + VolumeMounts: []corev1.VolumeMount{{ + Name: "plugins", + MountPath: "/target", + }}, + }} +} + +func (p *AzurePlugin) BackupStorageLocation(location veleroaddonv1.BackupStorageLocation, provider *veleroaddonv1.Azure) veleroaddonv1.BackupStorageLocation { + location.CredentialKey = veleroaddonv1.CredentialKey{ + Name: cmp.Or(provider.CredentialMap.To, p.Secret(provider).Name), + Key: veleroaddonv1.Provider{Azure: provider}.Name(), + } + location.Config = map[string]string{ + "resourceGroup": provider.Config.ResourceGroup, + "storageAccount": provider.Config.StorageAccount, + "storageAccountKeyEnvVar": cmp.Or(provider.Config.StorageAccountKeyEnvVar, "AZURE_STORAGE_ACCOUNT_ACCESS_KEY"), + } + + return location +} + +func (p *AzurePlugin) VolumeSnapshotLocation(snapshotLocation veleroaddonv1.VolumeSnapshotLocation, provider *veleroaddonv1.Azure) veleroaddonv1.VolumeSnapshotLocation { + snapshotLocation.CredentialKey = veleroaddonv1.CredentialKey{ + Name: cmp.Or(provider.CredentialMap.To, p.Secret(provider).Name), + Key: veleroaddonv1.Provider{Azure: provider}.Name(), + } + + return snapshotLocation +} + +func (p *AzurePlugin) Secret(provider *veleroaddonv1.Azure) client.ObjectKey { + return types.NamespacedName{ + Name: cmp.Or(provider.CredentialMap.NamespaceName.Name, provider.CredentialMap.From), + Namespace: provider.CredentialMap.NamespaceName.Namespace, + } +} diff --git a/internal/plugin/gcp.go b/internal/plugin/gcp.go new file mode 100644 index 0000000..a2e0ca2 --- /dev/null +++ b/internal/plugin/gcp.go @@ -0,0 +1,73 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "cmp" + + veleroaddonv1 "addons.cluster.x-k8s.io/cluster-api-addon-provider-velero/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type GCPPlugin struct{} + +func (p *GCPPlugin) Plugin(installation *veleroaddonv1.VeleroInstallation, provider *veleroaddonv1.GCP) { + installation.Spec.State.InitContainers = []corev1.Container{{ + Name: "velero-plugin-for-gcp", + Image: cmp.Or(provider.PluginURL, "velero/velero-plugin-for-gcp") + ":" + cmp.Or(provider.PluginTag, "latest"), + ImagePullPolicy: corev1.PullIfNotPresent, + VolumeMounts: []corev1.VolumeMount{{ + Name: "plugins", + MountPath: "/target", + }}, + }} +} + +func (p *GCPPlugin) BackupStorageLocation(location veleroaddonv1.BackupStorageLocation, provider *veleroaddonv1.GCP) veleroaddonv1.BackupStorageLocation { + location.Config = map[string]string{ + "serviceAccount": provider.Config.ServiceAccount, + "kmsKeyName": provider.Config.KMSKeyName, + } + location.CredentialKey = veleroaddonv1.CredentialKey{ + Name: cmp.Or(provider.CredentialMap.To, p.Secret(provider).Name), + Key: "gcp", + } + + return location +} + +func (p *GCPPlugin) VolumeSnapshotLocation(snapshotLocation veleroaddonv1.VolumeSnapshotLocation, provider *veleroaddonv1.GCP) veleroaddonv1.VolumeSnapshotLocation { + snapshotLocation.Config = map[string]string{ + "snapshotLocation": provider.Config.SnapshotLocation, + "project": provider.Config.Project, + } + snapshotLocation.CredentialKey = veleroaddonv1.CredentialKey{ + Name: cmp.Or(provider.CredentialMap.To, p.Secret(provider).Name), + Key: "gcp", + } + + return snapshotLocation +} + +func (p *GCPPlugin) Secret(provider *veleroaddonv1.GCP) client.ObjectKey { + return types.NamespacedName{ + Name: cmp.Or(provider.CredentialMap.NamespaceName.Name, provider.CredentialMap.From), + Namespace: provider.CredentialMap.NamespaceName.Namespace, + } +} diff --git a/internal/plugin/generic.go b/internal/plugin/generic.go new file mode 100644 index 0000000..15ed942 --- /dev/null +++ b/internal/plugin/generic.go @@ -0,0 +1,183 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "cmp" + "context" + "slices" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + kerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/utils/ptr" + "sigs.k8s.io/cluster-api/controllers/remote" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/yaml" + + veleroaddonv1 "addons.cluster.x-k8s.io/cluster-api-addon-provider-velero/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + helmv1 "sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" +) + +type PluginReconciler[P Plugin] struct { + client.Client + Scheme *runtime.Scheme + Tracker *remote.ClusterCacheTracker +} + +func (r *PluginReconciler[P]) Reconcile(ctx context.Context, installation *veleroaddonv1.VeleroInstallation, veleroPlugin VeleroPlugin[P], provider P) (ctrl.Result, error) { + _ = log.FromContext(ctx) + locations := installation.Spec.State.Configuration.BackupStorageLocations + snapshotLocations := installation.Spec.State.Configuration.VolumeSnapshotLocations + index, snapshotIndex := -1, -1 + + prov := installation.Spec.Provider + location := veleroaddonv1.BackupStorageLocation{ + Provider: prov.Name(), + Bucket: installation.Spec.Bucket, + Prefix: ptr.To("{{ .Cluster.metadata.name }}"), + } + snapshotLocation := veleroaddonv1.VolumeSnapshotLocation{ + Provider: prov.Name(), + } + if index = slices.IndexFunc(locations, func(l veleroaddonv1.BackupStorageLocation) bool { + return l.Provider == prov.Name() + }); index > -1 { + location = locations[index] + } else { + index = len(locations) + locations = append(locations, location) + } + + if snapshotIndex = slices.IndexFunc(snapshotLocations, func(l veleroaddonv1.VolumeSnapshotLocation) bool { + return l.Provider == prov.Name() + }); snapshotIndex > -1 { + snapshotLocation = snapshotLocations[snapshotIndex] + } else { + snapshotIndex = len(snapshotLocations) + snapshotLocations = append(snapshotLocations, snapshotLocation) + } + + veleroPlugin.Plugin(installation, provider) + locations[index] = veleroPlugin.BackupStorageLocation(location, provider) + snapshotLocations[snapshotIndex] = veleroPlugin.VolumeSnapshotLocation(snapshotLocation, provider) + + installation.Spec.State.Configuration.VolumeSnapshotLocations = snapshotLocations + installation.Spec.State.Configuration.BackupStorageLocations = locations + + from := veleroPlugin.Secret(provider) + if from.Name != "" { + from.Namespace = cmp.Or(from.Namespace, installation.Namespace) + to := types.NamespacedName{ + Name: locations[index].CredentialKey.Name, + Namespace: cmp.Or(installation.Spec.HelmSpec.ReleaseNamespace, installation.Spec.Namespace, "velero"), + } + if err := r.syncSecret(ctx, installation, from, to); err != nil { + return ctrl.Result{}, err + } + } + + spec, err := yaml.Marshal(installation.Spec.State) + if err != nil { + return ctrl.Result{}, err + } + + helmProxy := templateHelmChartProxy(installation, installation.Spec.HelmSpec, spec) + if err := controllerutil.SetOwnerReference(installation, helmProxy, r.Client.Scheme()); err != nil { + return ctrl.Result{}, err + } + + if err := r.Client.Patch( + ctx, helmProxy, + client.Apply, client.ForceOwnership, client.FieldOwner("velero-addon")); err != nil { + return ctrl.Result{}, err + } + + installation.Status.HelmChartProxyStatus = helmProxy.Status + + return ctrl.Result{}, r.Client.Status().Update(ctx, installation) +} + +func (r *PluginReconciler[P]) syncSecret(ctx context.Context, installation *veleroaddonv1.VeleroInstallation, from, to client.ObjectKey) error { + secret := &corev1.Secret{} + if err := r.Client.Get(ctx, from, secret); err != nil { + return err + } + + var errs []error + for _, cluster := range installation.Status.MatchingClusters { + cl, err := r.Tracker.GetClient(ctx, veleroaddonv1.RefToNamespaceName(&cluster).ObjectKey()) + if err != nil { + errs = append(errs, client.IgnoreNotFound(err)) + + continue + } + + newSecret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: to.Name, + Namespace: to.Namespace, + }, + Data: secret.Data, + } + + errs = append(errs, cl.Patch(ctx, newSecret, client.Apply, client.ForceOwnership, client.FieldOwner("velero-addon"))) + } + + return kerrors.NewAggregate(errs) +} + +func templateHelmChartProxy(installation *veleroaddonv1.VeleroInstallation, helmSpec helmv1.HelmChartProxySpec, values []byte) *helmv1.HelmChartProxy { + clusterSelector := helmSpec.ClusterSelector + if installation.Spec.ClusterSelector.MatchExpressions != nil || installation.Spec.ClusterSelector.MatchLabels != nil { + clusterSelector = installation.Spec.ClusterSelector + } + + options := cmp.Or(helmSpec.Options, helmv1.HelmOptions{ + Install: helmv1.HelmInstallOptions{ + CreateNamespace: true, + }, + }) + + return &helmv1.HelmChartProxy{ + TypeMeta: metav1.TypeMeta{ + APIVersion: helmv1.GroupVersion.String(), + Kind: "HelmChartProxy", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: installation.Name, + Namespace: installation.Namespace, + }, + Spec: helmv1.HelmChartProxySpec{ + ClusterSelector: clusterSelector, + ReleaseNamespace: cmp.Or(installation.Spec.Namespace, "velero"), + RepoURL: cmp.Or(helmSpec.RepoURL, "https://vmware-tanzu.github.io/helm-charts"), + ChartName: cmp.Or(helmSpec.ChartName, "velero"), + ValuesTemplate: cmp.Or(helmSpec.ValuesTemplate, string(values)), + Options: options, + }, + } +} diff --git a/internal/plugin/interface.go b/internal/plugin/interface.go new file mode 100644 index 0000000..3dce97f --- /dev/null +++ b/internal/plugin/interface.go @@ -0,0 +1,34 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + veleroaddonv1 "addons.cluster.x-k8s.io/cluster-api-addon-provider-velero/api/v1alpha1" +) + +type Plugin interface { + *veleroaddonv1.AWS | *veleroaddonv1.Azure | *veleroaddonv1.GCP +} + +type VeleroPlugin[P Plugin] interface { + Plugin(installation *veleroaddonv1.VeleroInstallation, provider P) + BackupStorageLocation(location veleroaddonv1.BackupStorageLocation, provider P) veleroaddonv1.BackupStorageLocation + VolumeSnapshotLocation(snapshotLocation veleroaddonv1.VolumeSnapshotLocation, provider P) veleroaddonv1.VolumeSnapshotLocation + Secret(provider P) client.ObjectKey +}