From 35f3993a3dc67fdf0417d1461da883008a268496 Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Wed, 23 Oct 2024 16:11:51 -0500 Subject: [PATCH] fix itemcollector --- pkg/backup/backup.go | 1 - pkg/backup/backup_test.go | 1384 +++++++++++++++++----------------- pkg/backup/item_collector.go | 76 +- 3 files changed, 747 insertions(+), 714 deletions(-) diff --git a/pkg/backup/backup.go b/pkg/backup/backup.go index 0304e71ced..8e6e3c5a42 100644 --- a/pkg/backup/backup.go +++ b/pkg/backup/backup.go @@ -239,7 +239,6 @@ func (kb *kubernetesBackupper) BackupWithResolvers( if err := kb.writeBackupVersion(tw); err != nil { return errors.WithStack(err) } - backupRequest.NamespaceIncludesExcludes = getNamespaceIncludesExcludes(backupRequest.Backup) log.Infof("Including namespaces: %s", backupRequest.NamespaceIncludesExcludes.IncludesString()) log.Infof("Excluding namespaces: %s", backupRequest.NamespaceIncludesExcludes.ExcludesString()) diff --git a/pkg/backup/backup_test.go b/pkg/backup/backup_test.go index 77969cfe96..262c803a57 100644 --- a/pkg/backup/backup_test.go +++ b/pkg/backup/backup_test.go @@ -177,105 +177,159 @@ func TestBackupOldResourceFiltering(t *testing.T) { want []string actions []biav2.BackupItemAction }{ - // { - // name: "no filters backs up everything", - // backup: defaultBackup().Result(), - // apiResources: []*test.APIResource{ - // test.Pods( - // builder.ForPod("foo", "bar").Result(), - // builder.ForPod("zoo", "raz").Result(), - // ), - // test.Deployments( - // builder.ForDeployment("foo", "bar").Result(), - // builder.ForDeployment("zoo", "raz").Result(), - // ), - // }, - // want: []string{ - // "resources/pods/namespaces/foo/bar.json", - // "resources/pods/namespaces/zoo/raz.json", - // "resources/deployments.apps/namespaces/foo/bar.json", - // "resources/deployments.apps/namespaces/zoo/raz.json", - // "resources/pods/v1-preferredversion/namespaces/foo/bar.json", - // "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", - // "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", - // "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", - // }, - // }, - // { - // name: "included resources filter only backs up resources of those types", - // backup: defaultBackup(). - // IncludedResources("pods"). - // Result(), - // apiResources: []*test.APIResource{ - // test.Pods( - // builder.ForPod("foo", "bar").Result(), - // builder.ForPod("zoo", "raz").Result(), - // ), - // test.Deployments( - // builder.ForDeployment("foo", "bar").Result(), - // builder.ForDeployment("zoo", "raz").Result(), - // ), - // }, - // want: []string{ - // "resources/pods/namespaces/foo/bar.json", - // "resources/pods/namespaces/zoo/raz.json", - // "resources/pods/v1-preferredversion/namespaces/foo/bar.json", - // "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", - // }, - // }, - // { - // name: "excluded resources filter only backs up resources not of those types", - // backup: defaultBackup(). - // ExcludedResources("deployments"). - // Result(), - // apiResources: []*test.APIResource{ - // test.Pods( - // builder.ForPod("foo", "bar").Result(), - // builder.ForPod("zoo", "raz").Result(), - // ), - // test.Deployments( - // builder.ForDeployment("foo", "bar").Result(), - // builder.ForDeployment("zoo", "raz").Result(), - // ), - // }, - // want: []string{ - // "resources/pods/namespaces/foo/bar.json", - // "resources/pods/namespaces/zoo/raz.json", - // "resources/pods/v1-preferredversion/namespaces/foo/bar.json", - // "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", - // }, - // }, - // { - // name: "included namespaces filter only backs up resources in those namespaces", - // backup: defaultBackup(). - // IncludedNamespaces("foo"). - // Result(), - // apiResources: []*test.APIResource{ - // test.Pods( - // builder.ForPod("foo", "bar").Result(), - // builder.ForPod("zoo", "raz").Result(), - // ), - // test.Deployments( - // builder.ForDeployment("foo", "bar").Result(), - // builder.ForDeployment("zoo", "raz").Result(), - // ), - // }, - // want: []string{ - // "resources/pods/namespaces/foo/bar.json", - // "resources/deployments.apps/namespaces/foo/bar.json", - // "resources/pods/v1-preferredversion/namespaces/foo/bar.json", - // "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", - // }, - // }, - { - name: "included namespaces by label filter only backs up resources in those namespaces", + { + name: "no filters backs up everything", + backup: defaultBackup().Result(), + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("foo", "bar").Result(), + builder.ForPod("zoo", "raz").Result(), + ), + test.Deployments( + builder.ForDeployment("foo", "bar").Result(), + builder.ForDeployment("zoo", "raz").Result(), + ), + }, + want: []string{ + "resources/pods/namespaces/foo/bar.json", + "resources/pods/namespaces/zoo/raz.json", + "resources/deployments.apps/namespaces/foo/bar.json", + "resources/deployments.apps/namespaces/zoo/raz.json", + "resources/pods/v1-preferredversion/namespaces/foo/bar.json", + "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", + "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", + "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", + }, + }, + { + name: "included resources filter only backs up resources of those types", + backup: defaultBackup(). + IncludedResources("pods"). + Result(), + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("foo", "bar").Result(), + builder.ForPod("zoo", "raz").Result(), + ), + test.Deployments( + builder.ForDeployment("foo", "bar").Result(), + builder.ForDeployment("zoo", "raz").Result(), + ), + }, + want: []string{ + "resources/pods/namespaces/foo/bar.json", + "resources/pods/namespaces/zoo/raz.json", + "resources/pods/v1-preferredversion/namespaces/foo/bar.json", + "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", + }, + }, + { + name: "excluded resources filter only backs up resources not of those types", + backup: defaultBackup(). + ExcludedResources("deployments"). + Result(), + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("foo", "bar").Result(), + builder.ForPod("zoo", "raz").Result(), + ), + test.Deployments( + builder.ForDeployment("foo", "bar").Result(), + builder.ForDeployment("zoo", "raz").Result(), + ), + }, + want: []string{ + "resources/pods/namespaces/foo/bar.json", + "resources/pods/namespaces/zoo/raz.json", + "resources/pods/v1-preferredversion/namespaces/foo/bar.json", + "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", + }, + }, + { + name: "included namespaces filter only backs up resources in those namespaces", backup: defaultBackup(). + IncludedNamespaces("foo"). + Result(), + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("foo", "bar").Result(), + builder.ForPod("zoo", "raz").Result(), + ), + test.Deployments( + builder.ForDeployment("foo", "bar").Result(), + builder.ForDeployment("zoo", "raz").Result(), + ), + }, + want: []string{ + "resources/pods/namespaces/foo/bar.json", + "resources/deployments.apps/namespaces/foo/bar.json", + "resources/pods/v1-preferredversion/namespaces/foo/bar.json", + "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", + }, + }, + { + name: "included namespaces via label filter backs up all resources in those namespaces and other includedNamespaces resources containing label", + backup: defaultBackup(). + IncludedNamespaces("moo"). LabelSelector(&metav1.LabelSelector{MatchLabels: map[string]string{"backup-ns": "true"}}). Result(), apiResources: []*test.APIResource{ test.Namespaces( builder.ForNamespace("foo").ObjectMeta(builder.WithLabels("backup-ns", "true")).Result(), + builder.ForNamespace("moo").Result(), + ), + test.Pods( + builder.ForPod("foo", "bar").Result(), + builder.ForPod("moo", "mar").ObjectMeta(builder.WithLabels("backup-ns", "true")).Result(), + builder.ForPod("moo", "unlabled-not-included").Result(), + builder.ForPod("zoo", "raz").Result(), + ), + test.Deployments( + builder.ForDeployment("foo", "bar").Result(), + builder.ForDeployment("zoo", "raz").Result(), + ), + }, + want: []string{ + "resources/deployments.apps/namespaces/foo/bar.json", + "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", + "resources/pods/v1-preferredversion/namespaces/foo/bar.json", + "resources/pods/v1-preferredversion/namespaces/moo/mar.json", + "resources/pods/namespaces/foo/bar.json", + "resources/pods/namespaces/moo/mar.json", + "resources/namespaces/cluster/foo.json", + "resources/namespaces/cluster/moo.json", + "resources/namespaces/v1-preferredversion/cluster/foo.json", + "resources/namespaces/v1-preferredversion/cluster/moo.json", + }, + }, + { + name: "excluded namespaces filter only backs up resources not in those namespaces", + backup: defaultBackup(). + ExcludedNamespaces("zoo"). + Result(), + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("foo", "bar").Result(), + builder.ForPod("zoo", "raz").Result(), + ), + test.Deployments( + builder.ForDeployment("foo", "bar").Result(), + builder.ForDeployment("zoo", "raz").Result(), ), + }, + want: []string{ + "resources/pods/namespaces/foo/bar.json", + "resources/deployments.apps/namespaces/foo/bar.json", + "resources/pods/v1-preferredversion/namespaces/foo/bar.json", + "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", + }, + }, + { + name: "IncludeClusterResources=false only backs up namespaced resources", + backup: defaultBackup(). + IncludeClusterResources(false). + Result(), + apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), @@ -284,609 +338,565 @@ func TestBackupOldResourceFiltering(t *testing.T) { builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), + test.PVs( + builder.ForPersistentVolume("bar").Result(), + builder.ForPersistentVolume("baz").Result(), + ), + }, + want: []string{ + "resources/pods/namespaces/foo/bar.json", + "resources/pods/namespaces/zoo/raz.json", + "resources/deployments.apps/namespaces/foo/bar.json", + "resources/deployments.apps/namespaces/zoo/raz.json", + "resources/pods/v1-preferredversion/namespaces/foo/bar.json", + "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", + "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", + "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", + }, + }, + { + name: "label selector only backs up matching resources", + backup: defaultBackup(). + LabelSelector(&metav1.LabelSelector{MatchLabels: map[string]string{"a": "b"}}). + Result(), + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels("a", "b")).Result(), + builder.ForPod("zoo", "raz").Result(), + ), + test.Deployments( + builder.ForDeployment("foo", "bar").Result(), + builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels("a", "b")).Result(), + ), + test.PVs( + builder.ForPersistentVolume("bar").ObjectMeta(builder.WithLabels("a", "b")).Result(), + builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels("a", "c")).Result(), + ), + }, + want: []string{ + "resources/pods/namespaces/foo/bar.json", + "resources/deployments.apps/namespaces/zoo/raz.json", + "resources/persistentvolumes/cluster/bar.json", + "resources/pods/v1-preferredversion/namespaces/foo/bar.json", + "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", + "resources/persistentvolumes/v1-preferredversion/cluster/bar.json", + }, + }, + { + name: "OrLabelSelector only backs up matching resources", + backup: defaultBackup(). + OrLabelSelector([]*metav1.LabelSelector{{MatchLabels: map[string]string{"a1": "b1"}}, {MatchLabels: map[string]string{"a2": "b2"}}, + {MatchLabels: map[string]string{"a3": "b3"}}, {MatchLabels: map[string]string{"a4": "b4"}}}). + Result(), + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels("a1", "b1")).Result(), + builder.ForPod("zoo", "raz").Result(), + ), + test.Deployments( + builder.ForDeployment("foo", "bar").Result(), + builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels("a2", "b2")).Result(), + ), + test.PVs( + builder.ForPersistentVolume("bar").ObjectMeta(builder.WithLabels("a4", "b4")).Result(), + builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels("a5", "b5")).Result(), + ), + }, + want: []string{ + "resources/pods/namespaces/foo/bar.json", + "resources/deployments.apps/namespaces/zoo/raz.json", + "resources/persistentvolumes/cluster/bar.json", + "resources/pods/v1-preferredversion/namespaces/foo/bar.json", + "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", + "resources/persistentvolumes/v1-preferredversion/cluster/bar.json", + }, + }, + { + name: "resources with velero.io/exclude-from-backup=true label are not included", + backup: defaultBackup(). + Result(), + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true")).Result(), + builder.ForPod("zoo", "raz").Result(), + ), + test.Deployments( + builder.ForDeployment("foo", "bar").Result(), + builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true")).Result(), + ), + test.PVs( + builder.ForPersistentVolume("bar").ObjectMeta(builder.WithLabels("a", "b")).Result(), + builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true")).Result(), + ), + }, + want: []string{ + "resources/pods/namespaces/zoo/raz.json", + "resources/deployments.apps/namespaces/foo/bar.json", + "resources/persistentvolumes/cluster/bar.json", + "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", + "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", + "resources/persistentvolumes/v1-preferredversion/cluster/bar.json", + }, + }, + { + name: "resources with velero.io/exclude-from-backup=true label are not included even if matching label selector", + backup: defaultBackup(). + LabelSelector(&metav1.LabelSelector{MatchLabels: map[string]string{"a": "b"}}). + Result(), + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true", "a", "b")).Result(), + builder.ForPod("zoo", "raz").ObjectMeta(builder.WithLabels("a", "b")).Result(), + ), + test.Deployments( + builder.ForDeployment("foo", "bar").Result(), + builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true", "a", "b")).Result(), + ), + test.PVs( + builder.ForPersistentVolume("bar").ObjectMeta(builder.WithLabels("a", "b")).Result(), + builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels("a", "b", velerov1.ExcludeFromBackupLabel, "true")).Result(), + ), + }, + want: []string{ + "resources/pods/namespaces/zoo/raz.json", + "resources/persistentvolumes/cluster/bar.json", + "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", + "resources/persistentvolumes/v1-preferredversion/cluster/bar.json", + }, + }, + { + name: "resources with velero.io/exclude-from-backup label specified but not 'true' are included", + backup: defaultBackup(). + Result(), + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "false")).Result(), + builder.ForPod("zoo", "raz").Result(), + ), + test.Deployments( + builder.ForDeployment("foo", "bar").Result(), + builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "1")).Result(), + ), + test.PVs( + builder.ForPersistentVolume("bar").ObjectMeta(builder.WithLabels("a", "b")).Result(), + builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "")).Result(), + ), }, want: []string{ "resources/pods/namespaces/foo/bar.json", + "resources/pods/namespaces/zoo/raz.json", "resources/deployments.apps/namespaces/foo/bar.json", + "resources/deployments.apps/namespaces/zoo/raz.json", + "resources/persistentvolumes/cluster/bar.json", + "resources/persistentvolumes/cluster/baz.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", + "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", + "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", + "resources/persistentvolumes/v1-preferredversion/cluster/bar.json", + "resources/persistentvolumes/v1-preferredversion/cluster/baz.json", + }, + }, + { + name: "should include cluster-scoped resources if backing up subset of namespaces and IncludeClusterResources=true", + backup: defaultBackup(). + IncludedNamespaces("ns-1", "ns-2"). + IncludeClusterResources(true). + Result(), + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("ns-1", "pod-1").Result(), + builder.ForPod("ns-2", "pod-1").Result(), + builder.ForPod("ns-3", "pod-1").Result(), + ), + test.PVs( + builder.ForPersistentVolume("pv-1").Result(), + builder.ForPersistentVolume("pv-2").Result(), + ), + }, + want: []string{ + "resources/pods/namespaces/ns-1/pod-1.json", + "resources/pods/namespaces/ns-2/pod-1.json", + "resources/persistentvolumes/cluster/pv-1.json", + "resources/persistentvolumes/cluster/pv-2.json", + "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", + "resources/pods/v1-preferredversion/namespaces/ns-2/pod-1.json", + "resources/persistentvolumes/v1-preferredversion/cluster/pv-1.json", + "resources/persistentvolumes/v1-preferredversion/cluster/pv-2.json", + }, + }, + { + name: "should not include cluster-scoped resource if backing up subset of namespaces and IncludeClusterResources=false", + backup: defaultBackup(). + IncludedNamespaces("ns-1", "ns-2"). + IncludeClusterResources(false). + Result(), + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("ns-1", "pod-1").Result(), + builder.ForPod("ns-2", "pod-1").Result(), + builder.ForPod("ns-3", "pod-1").Result(), + ), + test.PVs( + builder.ForPersistentVolume("pv-1").Result(), + builder.ForPersistentVolume("pv-2").Result(), + ), + }, + want: []string{ + "resources/pods/namespaces/ns-1/pod-1.json", + "resources/pods/namespaces/ns-2/pod-1.json", + "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", + "resources/pods/v1-preferredversion/namespaces/ns-2/pod-1.json", + }, + }, + { + name: "should not include cluster-scoped resource if backing up subset of namespaces and IncludeClusterResources=nil", + backup: defaultBackup(). + IncludedNamespaces("ns-1", "ns-2"). + Result(), + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("ns-1", "pod-1").Result(), + builder.ForPod("ns-2", "pod-1").Result(), + builder.ForPod("ns-3", "pod-1").Result(), + ), + test.PVs( + builder.ForPersistentVolume("pv-1").Result(), + builder.ForPersistentVolume("pv-2").Result(), + ), + }, + want: []string{ + "resources/pods/namespaces/ns-1/pod-1.json", + "resources/pods/namespaces/ns-2/pod-1.json", + "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", + "resources/pods/v1-preferredversion/namespaces/ns-2/pod-1.json", + }, + }, + { + name: "should include cluster-scoped resources if backing up all namespaces and IncludeClusterResources=true", + backup: defaultBackup(). + IncludeClusterResources(true). + Result(), + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("ns-1", "pod-1").Result(), + builder.ForPod("ns-2", "pod-1").Result(), + builder.ForPod("ns-3", "pod-1").Result(), + ), + test.PVs( + builder.ForPersistentVolume("pv-1").Result(), + builder.ForPersistentVolume("pv-2").Result(), + ), + }, + want: []string{ + "resources/pods/namespaces/ns-1/pod-1.json", + "resources/pods/namespaces/ns-2/pod-1.json", + "resources/pods/namespaces/ns-3/pod-1.json", + "resources/persistentvolumes/cluster/pv-1.json", + "resources/persistentvolumes/cluster/pv-2.json", + "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", + "resources/pods/v1-preferredversion/namespaces/ns-2/pod-1.json", + "resources/pods/v1-preferredversion/namespaces/ns-3/pod-1.json", + "resources/persistentvolumes/v1-preferredversion/cluster/pv-1.json", + "resources/persistentvolumes/v1-preferredversion/cluster/pv-2.json", + }, + }, + { + name: "should not include cluster-scoped resources if backing up all namespaces and IncludeClusterResources=false", + backup: defaultBackup(). + IncludeClusterResources(false). + Result(), + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("ns-1", "pod-1").Result(), + builder.ForPod("ns-2", "pod-1").Result(), + builder.ForPod("ns-3", "pod-1").Result(), + ), + test.PVs( + builder.ForPersistentVolume("pv-1").Result(), + builder.ForPersistentVolume("pv-2").Result(), + ), + }, + want: []string{ + "resources/pods/namespaces/ns-1/pod-1.json", + "resources/pods/namespaces/ns-2/pod-1.json", + "resources/pods/namespaces/ns-3/pod-1.json", + "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", + "resources/pods/v1-preferredversion/namespaces/ns-2/pod-1.json", + "resources/pods/v1-preferredversion/namespaces/ns-3/pod-1.json", + }, + }, + { + name: "should include cluster-scoped resources if backing up all namespaces and IncludeClusterResources=nil", + backup: defaultBackup(). + Result(), + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("ns-1", "pod-1").Result(), + builder.ForPod("ns-2", "pod-1").Result(), + builder.ForPod("ns-3", "pod-1").Result(), + ), + test.PVs( + builder.ForPersistentVolume("pv-1").Result(), + builder.ForPersistentVolume("pv-2").Result(), + ), + }, + want: []string{ + "resources/pods/namespaces/ns-1/pod-1.json", + "resources/pods/namespaces/ns-2/pod-1.json", + "resources/pods/namespaces/ns-3/pod-1.json", + "resources/persistentvolumes/cluster/pv-1.json", + "resources/persistentvolumes/cluster/pv-2.json", + "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", + "resources/pods/v1-preferredversion/namespaces/ns-2/pod-1.json", + "resources/pods/v1-preferredversion/namespaces/ns-3/pod-1.json", + "resources/persistentvolumes/v1-preferredversion/cluster/pv-1.json", + "resources/persistentvolumes/v1-preferredversion/cluster/pv-2.json", + }, + }, + { + name: "when a wildcard and a specific resource are included, the wildcard takes precedence", + backup: defaultBackup(). + IncludedResources("*", "pods"). + Result(), + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("foo", "bar").Result(), + builder.ForPod("zoo", "raz").Result(), + ), + test.Deployments( + builder.ForDeployment("foo", "bar").Result(), + builder.ForDeployment("zoo", "raz").Result(), + ), + }, + want: []string{ + "resources/pods/namespaces/foo/bar.json", + "resources/pods/namespaces/zoo/raz.json", + "resources/deployments.apps/namespaces/foo/bar.json", + "resources/deployments.apps/namespaces/zoo/raz.json", + "resources/pods/v1-preferredversion/namespaces/foo/bar.json", + "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", + "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", + "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", + }, + }, + { + name: "wildcard excludes are ignored", + backup: defaultBackup(). + ExcludedResources("*"). + Result(), + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("foo", "bar").Result(), + builder.ForPod("zoo", "raz").Result(), + ), + test.Deployments( + builder.ForDeployment("foo", "bar").Result(), + builder.ForDeployment("zoo", "raz").Result(), + ), + }, + want: []string{ + "resources/pods/namespaces/foo/bar.json", + "resources/pods/namespaces/zoo/raz.json", + "resources/deployments.apps/namespaces/foo/bar.json", + "resources/deployments.apps/namespaces/zoo/raz.json", + "resources/pods/v1-preferredversion/namespaces/foo/bar.json", + "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", + "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", + "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", + }, + }, + { + name: "unresolvable included resources are ignored", + backup: defaultBackup(). + IncludedResources("pods", "unresolvable"). + Result(), + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("foo", "bar").Result(), + builder.ForPod("zoo", "raz").Result(), + ), + test.Deployments( + builder.ForDeployment("foo", "bar").Result(), + builder.ForDeployment("zoo", "raz").Result(), + ), + }, + want: []string{ + "resources/pods/namespaces/foo/bar.json", + "resources/pods/namespaces/zoo/raz.json", + "resources/pods/v1-preferredversion/namespaces/foo/bar.json", + "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", + }, + }, + { + name: "when all included resources are unresolvable, nothing is included", + backup: defaultBackup(). + IncludedResources("unresolvable-1", "unresolvable-2"). + Result(), + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("foo", "bar").Result(), + builder.ForPod("zoo", "raz").Result(), + ), + test.Deployments( + builder.ForDeployment("foo", "bar").Result(), + builder.ForDeployment("zoo", "raz").Result(), + ), + }, + want: []string{}, + }, + { + name: "unresolvable excluded resources are ignored", + backup: defaultBackup(). + ExcludedResources("deployments", "unresolvable"). + Result(), + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("foo", "bar").Result(), + builder.ForPod("zoo", "raz").Result(), + ), + test.Deployments( + builder.ForDeployment("foo", "bar").Result(), + builder.ForDeployment("zoo", "raz").Result(), + ), + }, + want: []string{ + "resources/pods/namespaces/foo/bar.json", + "resources/pods/namespaces/zoo/raz.json", + "resources/pods/v1-preferredversion/namespaces/foo/bar.json", + "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", + }, + }, + { + name: "when all excluded resources are unresolvable, nothing is excluded", + backup: defaultBackup(). + IncludedResources("*"). + ExcludedResources("unresolvable-1", "unresolvable-2"). + Result(), + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("foo", "bar").Result(), + builder.ForPod("zoo", "raz").Result(), + ), + test.Deployments( + builder.ForDeployment("foo", "bar").Result(), + builder.ForDeployment("zoo", "raz").Result(), + ), + }, + want: []string{ + "resources/pods/namespaces/foo/bar.json", + "resources/pods/namespaces/zoo/raz.json", + "resources/pods/v1-preferredversion/namespaces/foo/bar.json", + "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", + "resources/deployments.apps/namespaces/foo/bar.json", + "resources/deployments.apps/namespaces/zoo/raz.json", + "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", + "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", + }, + }, + { + name: "terminating resources are not backed up", + backup: defaultBackup().Result(), + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("ns-1", "pod-1").Result(), + builder.ForPod("ns-2", "pod-2").ObjectMeta(builder.WithDeletionTimestamp(time.Now())).Result(), + ), + }, + want: []string{ + "resources/pods/namespaces/ns-1/pod-1.json", + "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", + }, + }, + { + name: "new filters' default value should not impact the old filters' function", + backup: defaultBackup().IncludedNamespaces("foo").IncludeClusterResources(true).Result(), + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("foo", "bar").Volumes(builder.ForVolume("foo").PersistentVolumeClaimSource("test-1").Result()).Result(), + builder.ForPod("zoo", "raz").Result(), + ), + test.Deployments( + builder.ForDeployment("foo", "bar").Result(), + builder.ForDeployment("zoo", "raz").Result(), + ), + test.PVCs( + builder.ForPersistentVolumeClaim("foo", "test-1").VolumeName("test1").Result(), + ), + test.PVs( + builder.ForPersistentVolume("test1").Result(), + builder.ForPersistentVolume("test2").Result(), + ), + }, + want: []string{ + "resources/deployments.apps/namespaces/foo/bar.json", + "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", + "resources/persistentvolumeclaims/namespaces/foo/test-1.json", + "resources/persistentvolumeclaims/v1-preferredversion/namespaces/foo/test-1.json", + "resources/persistentvolumes/cluster/test1.json", + "resources/persistentvolumes/cluster/test2.json", + "resources/persistentvolumes/v1-preferredversion/cluster/test1.json", + "resources/persistentvolumes/v1-preferredversion/cluster/test2.json", + "resources/pods/namespaces/foo/bar.json", + "resources/pods/v1-preferredversion/namespaces/foo/bar.json", + }, + actions: []biav2.BackupItemAction{ + &pluggableAction{ + selector: velero.ResourceSelector{IncludedResources: []string{"persistentvolumeclaims"}}, + executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { + additionalItems := []velero.ResourceIdentifier{ + {GroupResource: kuberesource.PersistentVolumes, Name: "test1"}, + } + + return item, additionalItems, "", nil, nil + }, + }, + }, + }, + { + name: "Resource's CRD should be included", + backup: defaultBackup().IncludedNamespaces("foo").Result(), + apiResources: []*test.APIResource{ + test.CRDs( + builder.ForCustomResourceDefinitionV1Beta1("backups.velero.io").Result(), + builder.ForCustomResourceDefinitionV1Beta1("volumesnapshotlocations.velero.io").Result(), + builder.ForCustomResourceDefinitionV1Beta1("test.velero.io").Result(), + ), + test.VSLs( + builder.ForVolumeSnapshotLocation("foo", "bar").Result(), + ), + test.Backups( + builder.ForBackup("zoo", "raz").Result(), + ), + }, + want: []string{ + "resources/customresourcedefinitions.apiextensions.k8s.io/cluster/volumesnapshotlocations.velero.io.json", + "resources/customresourcedefinitions.apiextensions.k8s.io/v1beta1-preferredversion/cluster/volumesnapshotlocations.velero.io.json", + "resources/volumesnapshotlocations.velero.io/namespaces/foo/bar.json", + "resources/volumesnapshotlocations.velero.io/v1-preferredversion/namespaces/foo/bar.json", + }, + }, + { + name: "Resource's CRD is not included, when CRD is excluded.", + backup: defaultBackup().IncludedNamespaces("foo").ExcludedResources("customresourcedefinitions.apiextensions.k8s.io").Result(), + apiResources: []*test.APIResource{ + test.CRDs( + builder.ForCustomResourceDefinitionV1Beta1("backups.velero.io").Result(), + builder.ForCustomResourceDefinitionV1Beta1("volumesnapshotlocations.velero.io").Result(), + builder.ForCustomResourceDefinitionV1Beta1("test.velero.io").Result(), + ), + test.VSLs( + builder.ForVolumeSnapshotLocation("foo", "bar").Result(), + ), + test.Backups( + builder.ForBackup("zoo", "raz").Result(), + ), + }, + want: []string{ + "resources/volumesnapshotlocations.velero.io/namespaces/foo/bar.json", + "resources/volumesnapshotlocations.velero.io/v1-preferredversion/namespaces/foo/bar.json", }, }, - // { - // name: "excluded namespaces filter only backs up resources not in those namespaces", - // backup: defaultBackup(). - // ExcludedNamespaces("zoo"). - // Result(), - // apiResources: []*test.APIResource{ - // test.Pods( - // builder.ForPod("foo", "bar").Result(), - // builder.ForPod("zoo", "raz").Result(), - // ), - // test.Deployments( - // builder.ForDeployment("foo", "bar").Result(), - // builder.ForDeployment("zoo", "raz").Result(), - // ), - // }, - // want: []string{ - // "resources/pods/namespaces/foo/bar.json", - // "resources/deployments.apps/namespaces/foo/bar.json", - // "resources/pods/v1-preferredversion/namespaces/foo/bar.json", - // "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", - // }, - // }, - // { - // name: "IncludeClusterResources=false only backs up namespaced resources", - // backup: defaultBackup(). - // IncludeClusterResources(false). - // Result(), - // apiResources: []*test.APIResource{ - // test.Pods( - // builder.ForPod("foo", "bar").Result(), - // builder.ForPod("zoo", "raz").Result(), - // ), - // test.Deployments( - // builder.ForDeployment("foo", "bar").Result(), - // builder.ForDeployment("zoo", "raz").Result(), - // ), - // test.PVs( - // builder.ForPersistentVolume("bar").Result(), - // builder.ForPersistentVolume("baz").Result(), - // ), - // }, - // want: []string{ - // "resources/pods/namespaces/foo/bar.json", - // "resources/pods/namespaces/zoo/raz.json", - // "resources/deployments.apps/namespaces/foo/bar.json", - // "resources/deployments.apps/namespaces/zoo/raz.json", - // "resources/pods/v1-preferredversion/namespaces/foo/bar.json", - // "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", - // "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", - // "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", - // }, - // }, - // { - // name: "label selector only backs up matching resources", - // backup: defaultBackup(). - // LabelSelector(&metav1.LabelSelector{MatchLabels: map[string]string{"a": "b"}}). - // Result(), - // apiResources: []*test.APIResource{ - // test.Pods( - // builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels("a", "b")).Result(), - // builder.ForPod("zoo", "raz").Result(), - // ), - // test.Deployments( - // builder.ForDeployment("foo", "bar").Result(), - // builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels("a", "b")).Result(), - // ), - // test.PVs( - // builder.ForPersistentVolume("bar").ObjectMeta(builder.WithLabels("a", "b")).Result(), - // builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels("a", "c")).Result(), - // ), - // }, - // want: []string{ - // "resources/pods/namespaces/foo/bar.json", - // "resources/deployments.apps/namespaces/zoo/raz.json", - // "resources/persistentvolumes/cluster/bar.json", - // "resources/pods/v1-preferredversion/namespaces/foo/bar.json", - // "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", - // "resources/persistentvolumes/v1-preferredversion/cluster/bar.json", - // }, - // }, - // { - // name: "OrLabelSelector only backs up matching resources", - // backup: defaultBackup(). - // OrLabelSelector([]*metav1.LabelSelector{{MatchLabels: map[string]string{"a1": "b1"}}, {MatchLabels: map[string]string{"a2": "b2"}}, - // {MatchLabels: map[string]string{"a3": "b3"}}, {MatchLabels: map[string]string{"a4": "b4"}}}). - // Result(), - // apiResources: []*test.APIResource{ - // test.Pods( - // builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels("a1", "b1")).Result(), - // builder.ForPod("zoo", "raz").Result(), - // ), - // test.Deployments( - // builder.ForDeployment("foo", "bar").Result(), - // builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels("a2", "b2")).Result(), - // ), - // test.PVs( - // builder.ForPersistentVolume("bar").ObjectMeta(builder.WithLabels("a4", "b4")).Result(), - // builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels("a5", "b5")).Result(), - // ), - // }, - // want: []string{ - // "resources/pods/namespaces/foo/bar.json", - // "resources/deployments.apps/namespaces/zoo/raz.json", - // "resources/persistentvolumes/cluster/bar.json", - // "resources/pods/v1-preferredversion/namespaces/foo/bar.json", - // "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", - // "resources/persistentvolumes/v1-preferredversion/cluster/bar.json", - // }, - // }, - // { - // name: "resources with velero.io/exclude-from-backup=true label are not included", - // backup: defaultBackup(). - // Result(), - // apiResources: []*test.APIResource{ - // test.Pods( - // builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true")).Result(), - // builder.ForPod("zoo", "raz").Result(), - // ), - // test.Deployments( - // builder.ForDeployment("foo", "bar").Result(), - // builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true")).Result(), - // ), - // test.PVs( - // builder.ForPersistentVolume("bar").ObjectMeta(builder.WithLabels("a", "b")).Result(), - // builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true")).Result(), - // ), - // }, - // want: []string{ - // "resources/pods/namespaces/zoo/raz.json", - // "resources/deployments.apps/namespaces/foo/bar.json", - // "resources/persistentvolumes/cluster/bar.json", - // "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", - // "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", - // "resources/persistentvolumes/v1-preferredversion/cluster/bar.json", - // }, - // }, - // { - // name: "resources with velero.io/exclude-from-backup=true label are not included even if matching label selector", - // backup: defaultBackup(). - // LabelSelector(&metav1.LabelSelector{MatchLabels: map[string]string{"a": "b"}}). - // Result(), - // apiResources: []*test.APIResource{ - // test.Pods( - // builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true", "a", "b")).Result(), - // builder.ForPod("zoo", "raz").ObjectMeta(builder.WithLabels("a", "b")).Result(), - // ), - // test.Deployments( - // builder.ForDeployment("foo", "bar").Result(), - // builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true", "a", "b")).Result(), - // ), - // test.PVs( - // builder.ForPersistentVolume("bar").ObjectMeta(builder.WithLabels("a", "b")).Result(), - // builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels("a", "b", velerov1.ExcludeFromBackupLabel, "true")).Result(), - // ), - // }, - // want: []string{ - // "resources/pods/namespaces/zoo/raz.json", - // "resources/persistentvolumes/cluster/bar.json", - // "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", - // "resources/persistentvolumes/v1-preferredversion/cluster/bar.json", - // }, - // }, - // { - // name: "resources with velero.io/exclude-from-backup label specified but not 'true' are included", - // backup: defaultBackup(). - // Result(), - // apiResources: []*test.APIResource{ - // test.Pods( - // builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "false")).Result(), - // builder.ForPod("zoo", "raz").Result(), - // ), - // test.Deployments( - // builder.ForDeployment("foo", "bar").Result(), - // builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "1")).Result(), - // ), - // test.PVs( - // builder.ForPersistentVolume("bar").ObjectMeta(builder.WithLabels("a", "b")).Result(), - // builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "")).Result(), - // ), - // }, - // want: []string{ - // "resources/pods/namespaces/foo/bar.json", - // "resources/pods/namespaces/zoo/raz.json", - // "resources/deployments.apps/namespaces/foo/bar.json", - // "resources/deployments.apps/namespaces/zoo/raz.json", - // "resources/persistentvolumes/cluster/bar.json", - // "resources/persistentvolumes/cluster/baz.json", - // "resources/pods/v1-preferredversion/namespaces/foo/bar.json", - // "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", - // "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", - // "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", - // "resources/persistentvolumes/v1-preferredversion/cluster/bar.json", - // "resources/persistentvolumes/v1-preferredversion/cluster/baz.json", - // }, - // }, - // { - // name: "should include cluster-scoped resources if backing up subset of namespaces and IncludeClusterResources=true", - // backup: defaultBackup(). - // IncludedNamespaces("ns-1", "ns-2"). - // IncludeClusterResources(true). - // Result(), - // apiResources: []*test.APIResource{ - // test.Pods( - // builder.ForPod("ns-1", "pod-1").Result(), - // builder.ForPod("ns-2", "pod-1").Result(), - // builder.ForPod("ns-3", "pod-1").Result(), - // ), - // test.PVs( - // builder.ForPersistentVolume("pv-1").Result(), - // builder.ForPersistentVolume("pv-2").Result(), - // ), - // }, - // want: []string{ - // "resources/pods/namespaces/ns-1/pod-1.json", - // "resources/pods/namespaces/ns-2/pod-1.json", - // "resources/persistentvolumes/cluster/pv-1.json", - // "resources/persistentvolumes/cluster/pv-2.json", - // "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", - // "resources/pods/v1-preferredversion/namespaces/ns-2/pod-1.json", - // "resources/persistentvolumes/v1-preferredversion/cluster/pv-1.json", - // "resources/persistentvolumes/v1-preferredversion/cluster/pv-2.json", - // }, - // }, - // { - // name: "should not include cluster-scoped resource if backing up subset of namespaces and IncludeClusterResources=false", - // backup: defaultBackup(). - // IncludedNamespaces("ns-1", "ns-2"). - // IncludeClusterResources(false). - // Result(), - // apiResources: []*test.APIResource{ - // test.Pods( - // builder.ForPod("ns-1", "pod-1").Result(), - // builder.ForPod("ns-2", "pod-1").Result(), - // builder.ForPod("ns-3", "pod-1").Result(), - // ), - // test.PVs( - // builder.ForPersistentVolume("pv-1").Result(), - // builder.ForPersistentVolume("pv-2").Result(), - // ), - // }, - // want: []string{ - // "resources/pods/namespaces/ns-1/pod-1.json", - // "resources/pods/namespaces/ns-2/pod-1.json", - // "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", - // "resources/pods/v1-preferredversion/namespaces/ns-2/pod-1.json", - // }, - // }, - // { - // name: "should not include cluster-scoped resource if backing up subset of namespaces and IncludeClusterResources=nil", - // backup: defaultBackup(). - // IncludedNamespaces("ns-1", "ns-2"). - // Result(), - // apiResources: []*test.APIResource{ - // test.Pods( - // builder.ForPod("ns-1", "pod-1").Result(), - // builder.ForPod("ns-2", "pod-1").Result(), - // builder.ForPod("ns-3", "pod-1").Result(), - // ), - // test.PVs( - // builder.ForPersistentVolume("pv-1").Result(), - // builder.ForPersistentVolume("pv-2").Result(), - // ), - // }, - // want: []string{ - // "resources/pods/namespaces/ns-1/pod-1.json", - // "resources/pods/namespaces/ns-2/pod-1.json", - // "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", - // "resources/pods/v1-preferredversion/namespaces/ns-2/pod-1.json", - // }, - // }, - // { - // name: "should include cluster-scoped resources if backing up all namespaces and IncludeClusterResources=true", - // backup: defaultBackup(). - // IncludeClusterResources(true). - // Result(), - // apiResources: []*test.APIResource{ - // test.Pods( - // builder.ForPod("ns-1", "pod-1").Result(), - // builder.ForPod("ns-2", "pod-1").Result(), - // builder.ForPod("ns-3", "pod-1").Result(), - // ), - // test.PVs( - // builder.ForPersistentVolume("pv-1").Result(), - // builder.ForPersistentVolume("pv-2").Result(), - // ), - // }, - // want: []string{ - // "resources/pods/namespaces/ns-1/pod-1.json", - // "resources/pods/namespaces/ns-2/pod-1.json", - // "resources/pods/namespaces/ns-3/pod-1.json", - // "resources/persistentvolumes/cluster/pv-1.json", - // "resources/persistentvolumes/cluster/pv-2.json", - // "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", - // "resources/pods/v1-preferredversion/namespaces/ns-2/pod-1.json", - // "resources/pods/v1-preferredversion/namespaces/ns-3/pod-1.json", - // "resources/persistentvolumes/v1-preferredversion/cluster/pv-1.json", - // "resources/persistentvolumes/v1-preferredversion/cluster/pv-2.json", - // }, - // }, - // { - // name: "should not include cluster-scoped resources if backing up all namespaces and IncludeClusterResources=false", - // backup: defaultBackup(). - // IncludeClusterResources(false). - // Result(), - // apiResources: []*test.APIResource{ - // test.Pods( - // builder.ForPod("ns-1", "pod-1").Result(), - // builder.ForPod("ns-2", "pod-1").Result(), - // builder.ForPod("ns-3", "pod-1").Result(), - // ), - // test.PVs( - // builder.ForPersistentVolume("pv-1").Result(), - // builder.ForPersistentVolume("pv-2").Result(), - // ), - // }, - // want: []string{ - // "resources/pods/namespaces/ns-1/pod-1.json", - // "resources/pods/namespaces/ns-2/pod-1.json", - // "resources/pods/namespaces/ns-3/pod-1.json", - // "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", - // "resources/pods/v1-preferredversion/namespaces/ns-2/pod-1.json", - // "resources/pods/v1-preferredversion/namespaces/ns-3/pod-1.json", - // }, - // }, - // { - // name: "should include cluster-scoped resources if backing up all namespaces and IncludeClusterResources=nil", - // backup: defaultBackup(). - // Result(), - // apiResources: []*test.APIResource{ - // test.Pods( - // builder.ForPod("ns-1", "pod-1").Result(), - // builder.ForPod("ns-2", "pod-1").Result(), - // builder.ForPod("ns-3", "pod-1").Result(), - // ), - // test.PVs( - // builder.ForPersistentVolume("pv-1").Result(), - // builder.ForPersistentVolume("pv-2").Result(), - // ), - // }, - // want: []string{ - // "resources/pods/namespaces/ns-1/pod-1.json", - // "resources/pods/namespaces/ns-2/pod-1.json", - // "resources/pods/namespaces/ns-3/pod-1.json", - // "resources/persistentvolumes/cluster/pv-1.json", - // "resources/persistentvolumes/cluster/pv-2.json", - // "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", - // "resources/pods/v1-preferredversion/namespaces/ns-2/pod-1.json", - // "resources/pods/v1-preferredversion/namespaces/ns-3/pod-1.json", - // "resources/persistentvolumes/v1-preferredversion/cluster/pv-1.json", - // "resources/persistentvolumes/v1-preferredversion/cluster/pv-2.json", - // }, - // }, - // { - // name: "when a wildcard and a specific resource are included, the wildcard takes precedence", - // backup: defaultBackup(). - // IncludedResources("*", "pods"). - // Result(), - // apiResources: []*test.APIResource{ - // test.Pods( - // builder.ForPod("foo", "bar").Result(), - // builder.ForPod("zoo", "raz").Result(), - // ), - // test.Deployments( - // builder.ForDeployment("foo", "bar").Result(), - // builder.ForDeployment("zoo", "raz").Result(), - // ), - // }, - // want: []string{ - // "resources/pods/namespaces/foo/bar.json", - // "resources/pods/namespaces/zoo/raz.json", - // "resources/deployments.apps/namespaces/foo/bar.json", - // "resources/deployments.apps/namespaces/zoo/raz.json", - // "resources/pods/v1-preferredversion/namespaces/foo/bar.json", - // "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", - // "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", - // "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", - // }, - // }, - // { - // name: "wildcard excludes are ignored", - // backup: defaultBackup(). - // ExcludedResources("*"). - // Result(), - // apiResources: []*test.APIResource{ - // test.Pods( - // builder.ForPod("foo", "bar").Result(), - // builder.ForPod("zoo", "raz").Result(), - // ), - // test.Deployments( - // builder.ForDeployment("foo", "bar").Result(), - // builder.ForDeployment("zoo", "raz").Result(), - // ), - // }, - // want: []string{ - // "resources/pods/namespaces/foo/bar.json", - // "resources/pods/namespaces/zoo/raz.json", - // "resources/deployments.apps/namespaces/foo/bar.json", - // "resources/deployments.apps/namespaces/zoo/raz.json", - // "resources/pods/v1-preferredversion/namespaces/foo/bar.json", - // "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", - // "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", - // "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", - // }, - // }, - // { - // name: "unresolvable included resources are ignored", - // backup: defaultBackup(). - // IncludedResources("pods", "unresolvable"). - // Result(), - // apiResources: []*test.APIResource{ - // test.Pods( - // builder.ForPod("foo", "bar").Result(), - // builder.ForPod("zoo", "raz").Result(), - // ), - // test.Deployments( - // builder.ForDeployment("foo", "bar").Result(), - // builder.ForDeployment("zoo", "raz").Result(), - // ), - // }, - // want: []string{ - // "resources/pods/namespaces/foo/bar.json", - // "resources/pods/namespaces/zoo/raz.json", - // "resources/pods/v1-preferredversion/namespaces/foo/bar.json", - // "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", - // }, - // }, - // { - // name: "when all included resources are unresolvable, nothing is included", - // backup: defaultBackup(). - // IncludedResources("unresolvable-1", "unresolvable-2"). - // Result(), - // apiResources: []*test.APIResource{ - // test.Pods( - // builder.ForPod("foo", "bar").Result(), - // builder.ForPod("zoo", "raz").Result(), - // ), - // test.Deployments( - // builder.ForDeployment("foo", "bar").Result(), - // builder.ForDeployment("zoo", "raz").Result(), - // ), - // }, - // want: []string{}, - // }, - // { - // name: "unresolvable excluded resources are ignored", - // backup: defaultBackup(). - // ExcludedResources("deployments", "unresolvable"). - // Result(), - // apiResources: []*test.APIResource{ - // test.Pods( - // builder.ForPod("foo", "bar").Result(), - // builder.ForPod("zoo", "raz").Result(), - // ), - // test.Deployments( - // builder.ForDeployment("foo", "bar").Result(), - // builder.ForDeployment("zoo", "raz").Result(), - // ), - // }, - // want: []string{ - // "resources/pods/namespaces/foo/bar.json", - // "resources/pods/namespaces/zoo/raz.json", - // "resources/pods/v1-preferredversion/namespaces/foo/bar.json", - // "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", - // }, - // }, - // { - // name: "when all excluded resources are unresolvable, nothing is excluded", - // backup: defaultBackup(). - // IncludedResources("*"). - // ExcludedResources("unresolvable-1", "unresolvable-2"). - // Result(), - // apiResources: []*test.APIResource{ - // test.Pods( - // builder.ForPod("foo", "bar").Result(), - // builder.ForPod("zoo", "raz").Result(), - // ), - // test.Deployments( - // builder.ForDeployment("foo", "bar").Result(), - // builder.ForDeployment("zoo", "raz").Result(), - // ), - // }, - // want: []string{ - // "resources/pods/namespaces/foo/bar.json", - // "resources/pods/namespaces/zoo/raz.json", - // "resources/pods/v1-preferredversion/namespaces/foo/bar.json", - // "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", - // "resources/deployments.apps/namespaces/foo/bar.json", - // "resources/deployments.apps/namespaces/zoo/raz.json", - // "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", - // "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", - // }, - // }, - // { - // name: "terminating resources are not backed up", - // backup: defaultBackup().Result(), - // apiResources: []*test.APIResource{ - // test.Pods( - // builder.ForPod("ns-1", "pod-1").Result(), - // builder.ForPod("ns-2", "pod-2").ObjectMeta(builder.WithDeletionTimestamp(time.Now())).Result(), - // ), - // }, - // want: []string{ - // "resources/pods/namespaces/ns-1/pod-1.json", - // "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", - // }, - // }, - // { - // name: "new filters' default value should not impact the old filters' function", - // backup: defaultBackup().IncludedNamespaces("foo").IncludeClusterResources(true).Result(), - // apiResources: []*test.APIResource{ - // test.Pods( - // builder.ForPod("foo", "bar").Volumes(builder.ForVolume("foo").PersistentVolumeClaimSource("test-1").Result()).Result(), - // builder.ForPod("zoo", "raz").Result(), - // ), - // test.Deployments( - // builder.ForDeployment("foo", "bar").Result(), - // builder.ForDeployment("zoo", "raz").Result(), - // ), - // test.PVCs( - // builder.ForPersistentVolumeClaim("foo", "test-1").VolumeName("test1").Result(), - // ), - // test.PVs( - // builder.ForPersistentVolume("test1").Result(), - // builder.ForPersistentVolume("test2").Result(), - // ), - // }, - // want: []string{ - // "resources/deployments.apps/namespaces/foo/bar.json", - // "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", - // "resources/persistentvolumeclaims/namespaces/foo/test-1.json", - // "resources/persistentvolumeclaims/v1-preferredversion/namespaces/foo/test-1.json", - // "resources/persistentvolumes/cluster/test1.json", - // "resources/persistentvolumes/cluster/test2.json", - // "resources/persistentvolumes/v1-preferredversion/cluster/test1.json", - // "resources/persistentvolumes/v1-preferredversion/cluster/test2.json", - // "resources/pods/namespaces/foo/bar.json", - // "resources/pods/v1-preferredversion/namespaces/foo/bar.json", - // }, - // actions: []biav2.BackupItemAction{ - // &pluggableAction{ - // selector: velero.ResourceSelector{IncludedResources: []string{"persistentvolumeclaims"}}, - // executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { - // additionalItems := []velero.ResourceIdentifier{ - // {GroupResource: kuberesource.PersistentVolumes, Name: "test1"}, - // } - - // return item, additionalItems, "", nil, nil - // }, - // }, - // }, - // }, - // { - // name: "Resource's CRD should be included", - // backup: defaultBackup().IncludedNamespaces("foo").Result(), - // apiResources: []*test.APIResource{ - // test.CRDs( - // builder.ForCustomResourceDefinitionV1Beta1("backups.velero.io").Result(), - // builder.ForCustomResourceDefinitionV1Beta1("volumesnapshotlocations.velero.io").Result(), - // builder.ForCustomResourceDefinitionV1Beta1("test.velero.io").Result(), - // ), - // test.VSLs( - // builder.ForVolumeSnapshotLocation("foo", "bar").Result(), - // ), - // test.Backups( - // builder.ForBackup("zoo", "raz").Result(), - // ), - // }, - // want: []string{ - // "resources/customresourcedefinitions.apiextensions.k8s.io/cluster/volumesnapshotlocations.velero.io.json", - // "resources/customresourcedefinitions.apiextensions.k8s.io/v1beta1-preferredversion/cluster/volumesnapshotlocations.velero.io.json", - // "resources/volumesnapshotlocations.velero.io/namespaces/foo/bar.json", - // "resources/volumesnapshotlocations.velero.io/v1-preferredversion/namespaces/foo/bar.json", - // }, - // }, - // { - // name: "Resource's CRD is not included, when CRD is excluded.", - // backup: defaultBackup().IncludedNamespaces("foo").ExcludedResources("customresourcedefinitions.apiextensions.k8s.io").Result(), - // apiResources: []*test.APIResource{ - // test.CRDs( - // builder.ForCustomResourceDefinitionV1Beta1("backups.velero.io").Result(), - // builder.ForCustomResourceDefinitionV1Beta1("volumesnapshotlocations.velero.io").Result(), - // builder.ForCustomResourceDefinitionV1Beta1("test.velero.io").Result(), - // ), - // test.VSLs( - // builder.ForVolumeSnapshotLocation("foo", "bar").Result(), - // ), - // test.Backups( - // builder.ForBackup("zoo", "raz").Result(), - // ), - // }, - // want: []string{ - // "resources/volumesnapshotlocations.velero.io/namespaces/foo/bar.json", - // "resources/volumesnapshotlocations.velero.io/v1-preferredversion/namespaces/foo/bar.json", - // }, - // }, } for _, tc := range tests { diff --git a/pkg/backup/item_collector.go b/pkg/backup/item_collector.go index a1fce5ee74..83f76c199b 100644 --- a/pkg/backup/item_collector.go +++ b/pkg/backup/item_collector.go @@ -32,6 +32,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/pager" "github.com/vmware-tanzu/velero/pkg/client" @@ -46,6 +47,8 @@ import ( type itemCollector struct { log logrus.FieldLogger backupRequest *Request + // Namespaces that are included by the backup's labelSelector will backup all resources in that namespace even if resources are not labeled. + namespacesIncludedByLabelSelector sets.Set[string] discoveryHelper discovery.Helper dynamicFactory client.DynamicFactory cohabitatingResources map[string]*cohabitatingResource @@ -72,32 +75,26 @@ type nsTracker struct { orLabelSelector []labels.Selector namespaceFilter *collections.IncludesExcludes logger logrus.FieldLogger - - namespaceMap map[string]bool + namespaceMap sets.Set[string] // Set is a map[comparable]struct{} for minimal memory consumption. } // track add the namespace into the namespaceMap. func (nt *nsTracker) track(ns string) { if nt.namespaceMap == nil { - nt.namespaceMap = make(map[string]bool) - } - - if _, ok := nt.namespaceMap[ns]; !ok { - nt.namespaceMap[ns] = true + nt.namespaceMap = make(sets.Set[string]) } + nt.namespaceMap.Insert(ns) } // isTracked check whether the namespace's name exists in // namespaceMap. func (nt *nsTracker) isTracked(ns string) bool { - if nt.namespaceMap != nil { - return nt.namespaceMap[ns] - } - return false + return nt.namespaceMap.Has(ns) } -// init initialize the namespaceMap, and add elements according to +// init initialize the namespaceMap, and add (track) elements according to // namespace include/exclude filters and the backup label selectors. +// and return only namespaces that are added to tracker. func (nt *nsTracker) init( unstructuredNSs []unstructured.Unstructured, singleLabelSelector labels.Selector, @@ -105,9 +102,6 @@ func (nt *nsTracker) init( namespaceFilter *collections.IncludesExcludes, logger logrus.FieldLogger, ) { - if nt.namespaceMap == nil { - nt.namespaceMap = make(map[string]bool) - } nt.singleLabelSelector = singleLabelSelector nt.orLabelSelector = orLabelSelector nt.namespaceFilter = namespaceFilter @@ -194,6 +188,7 @@ func (r *itemCollector) getItemsFromResourceIdentifiers( // getAllItems gets all backup-relevant items from all API groups. func (r *itemCollector) getAllItems() []*kubernetesResource { + // getItems should return all namespaces and will be filtered by nsTracker.filterNamespaces resources := r.getItems(nil) return r.nsTracker.filterNamespaces(resources) @@ -436,13 +431,33 @@ func (r *itemCollector) getResourceItems( // Namespace are filtered by namespace include/exclude filters, // backup LabelSelectors and OrLabelSelectors are checked too. if gr == kuberesource.Namespaces { - return r.collectNamespaces( + namespaces, err := r.collectNamespaces( resource, gv, gr, preferredGVR, log, ) + if err != nil { + return nil, err + } + // The namespaces collected contains namespaces selected by namespace filters like include/exclude and label selector. + // Per https://github.com/vmware-tanzu/velero/issues/7492#issuecomment-1986146411 we want labelSelector included namespace to act as IncludedNamespaces so add to NamespaceIncludesExcludes string set. + for i := range namespaces { + if namespaces[i] != nil { + if !r.backupRequest.NamespaceIncludesExcludes.ShouldInclude(namespaces[i].name) { + // this namespace is included by the backup's label selector, not the namespace include/exclude filter + // this is a special case where we want to include this namespace in the backup and all resources in it regardless of the resource's label selector + // in other cases, if label selector is set, we only want to include resources that match the label selector + if r.namespacesIncludedByLabelSelector == nil { + r.namespacesIncludedByLabelSelector = make(sets.Set[string]) + } + r.namespacesIncludedByLabelSelector.Insert(namespaces[i].name) + r.backupRequest.NamespaceIncludesExcludes.Includes(namespaces[i].name) + } + } + } + return namespaces, err } clusterScoped := !resource.Namespaced @@ -509,9 +524,13 @@ func (r *itemCollector) listResourceByLabelsPerNamespace( logger.WithError(err).Error("Error getting dynamic client") return nil, err } - + labeledNsSoBackupUnlabled := false + if r.namespacesIncludedByLabelSelector.Has(namespace) { + logger.Infof("Listing all items in namespace %s because ns was selected by the backup's label selector", namespace) + labeledNsSoBackupUnlabled = true + } var orLabelSelectors []string - if r.backupRequest.Spec.OrLabelSelectors != nil { + if r.backupRequest.Spec.OrLabelSelectors != nil && !labeledNsSoBackupUnlabled { for _, s := range r.backupRequest.Spec.OrLabelSelectors { orLabelSelectors = append(orLabelSelectors, metav1.FormatLabelSelector(s)) } @@ -537,7 +556,7 @@ func (r *itemCollector) listResourceByLabelsPerNamespace( } var labelSelector string - if selector := r.backupRequest.Spec.LabelSelector; selector != nil { + if selector := r.backupRequest.Spec.LabelSelector; selector != nil && !labeledNsSoBackupUnlabled { labelSelector = metav1.FormatLabelSelector(selector) } @@ -565,20 +584,21 @@ func (r *itemCollector) writeToFile(item *unstructured.Unstructured) (string, er return "", errors.Wrap(err, "error creating temp file") } defer f.Close() - + // TODO: remove debug jsonBytes, err := json.Marshal(item) if err != nil { return "", errors.Wrap(err, "error converting item to JSON") } - + if _, err := f.Write(jsonBytes); err != nil { return "", errors.Wrap(err, "error writing JSON to file") } - + if err := f.Close(); err != nil { return "", errors.Wrap(err, "error closing file") } - + fmt.Printf("Wrote %v\nto item %s/%s to file %s\n",string(jsonBytes), item.GetNamespace(), item.GetName(), f.Name()) + return f.Name(), nil } @@ -594,7 +614,8 @@ func sortCoreGroup(group *metav1.APIResourceList) { // pod is backed up, we can perform a pre hook, then process pvcs and pvs (including taking a // snapshot), then perform a post hook on the pod. const ( - pod = iota + namespaces = iota + pod pvc pv other @@ -604,6 +625,8 @@ const ( // pods, pvcs, pvs, everything else. func coreGroupResourcePriority(resource string) int { switch strings.ToLower(resource) { + case "namespaces": + return namespaces case "pods": return pod case "persistentvolumeclaims": @@ -722,6 +745,7 @@ func (r *itemCollector) listItemsForLabel( } // collectNamespaces process namespace resource according to namespace filters. +// returns a list of ALL namespaces. Filtering will happen outside of this function. func (r *itemCollector) collectNamespaces( resource metav1.APIResource, gv schema.GroupVersion, @@ -780,7 +804,8 @@ func (r *itemCollector) collectNamespaces( orSelectors = append(orSelectors, orSelector) } } - + // init initialize the namespaceMap, and add elements according to + // namespace include/exclude filters and the backup label selectors. r.nsTracker.init( unstructuredList.Items, singleSelector, @@ -806,6 +831,5 @@ func (r *itemCollector) collectNamespaces( path: path, }) } - return items, nil }