Skip to content

Commit

Permalink
Merge pull request karmada-io#3801 from zhy76/e2e
Browse files Browse the repository at this point in the history 
add e2e test when cluster join, update, karmada need to reconcile search cache
  • Loading branch information
@karmada-bot
karmada-bot authored Jul 20, 2023
2 parents df1b791 + 4f20be1 commit 48e9a5b
Show file tree
Hide file tree
Showing 2 changed files with 205 additions and 9 deletions.
213 changes: 204 additions & 9 deletions test/e2e/search_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package e2e
import (
"context"
"fmt"
"os"
"sort"
"strings"
"time"
Expand All @@ -16,13 +17,18 @@ import (
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/rand"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/cli-runtime/pkg/genericclioptions"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"

clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1"
policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1"
searchv1alpha1 "github.com/karmada-io/karmada/pkg/apis/search/v1alpha1"
"github.com/karmada-io/karmada/pkg/karmadactl/join"
"github.com/karmada-io/karmada/pkg/karmadactl/options"
"github.com/karmada-io/karmada/pkg/karmadactl/unjoin"
cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util"
"github.com/karmada-io/karmada/test/e2e/framework"
testhelper "github.com/karmada-io/karmada/test/helper"
)
Expand Down Expand Up @@ -107,7 +113,7 @@ var _ = ginkgo.Describe("[karmada-search] karmada search testing", ginkgo.Ordere
m2Dm = testhelper.NewDeployment(testNamespace, m2DmName)
framework.CreateDeployment(member2Client, m2Dm)

rrName = "rr-" + rand.String(RandomStrLength)
rrName = resourceRegistryPrefix + rand.String(RandomStrLength)
rr = &searchv1alpha1.ResourceRegistry{
ObjectMeta: metav1.ObjectMeta{
Name: rrName,
Expand Down Expand Up @@ -160,7 +166,7 @@ var _ = ginkgo.Describe("[karmada-search] karmada search testing", ginkgo.Ordere
var rr *searchv1alpha1.ResourceRegistry

ginkgo.BeforeAll(func() {
rrName = "rr-" + rand.String(RandomStrLength)
rrName = resourceRegistryPrefix + rand.String(RandomStrLength)
rr = &searchv1alpha1.ResourceRegistry{
ObjectMeta: metav1.ObjectMeta{
Name: rrName,
Expand Down Expand Up @@ -201,7 +207,7 @@ var _ = ginkgo.Describe("[karmada-search] karmada search testing", ginkgo.Ordere
var rr2 *searchv1alpha1.ResourceRegistry

ginkgo.BeforeAll(func() {
rrName = "rr-" + rand.String(RandomStrLength)
rrName = resourceRegistryPrefix + rand.String(RandomStrLength)
rr = &searchv1alpha1.ResourceRegistry{
ObjectMeta: metav1.ObjectMeta{
Name: rrName,
Expand All @@ -220,7 +226,7 @@ var _ = ginkgo.Describe("[karmada-search] karmada search testing", ginkgo.Ordere
}
framework.CreateResourceRegistry(karmadaClient, rr)

rr2Name = "rr-" + rand.String(RandomStrLength)
rr2Name = resourceRegistryPrefix + rand.String(RandomStrLength)
rr2 = &searchv1alpha1.ResourceRegistry{
ObjectMeta: metav1.ObjectMeta{
Name: rr2Name,
Expand Down Expand Up @@ -261,7 +267,7 @@ var _ = ginkgo.Describe("[karmada-search] karmada search testing", ginkgo.Ordere
var rr *searchv1alpha1.ResourceRegistry

ginkgo.BeforeAll(func() {
rrName = "rr-" + rand.String(RandomStrLength)
rrName = resourceRegistryPrefix + rand.String(RandomStrLength)
rr = &searchv1alpha1.ResourceRegistry{
Spec: searchv1alpha1.ResourceRegistrySpec{
TargetCluster: policyv1alpha1.ClusterAffinity{
Expand Down Expand Up @@ -315,7 +321,7 @@ var _ = ginkgo.Describe("[karmada-search] karmada search testing", ginkgo.Ordere

ginkgo.It("[daemonset] should be searchable", func() {
ginkgo.By("create resourceRegistry, add daemonset")
rrName = "rr-" + rand.String(RandomStrLength)
rrName = resourceRegistryPrefix + rand.String(RandomStrLength)
rr = &searchv1alpha1.ResourceRegistry{
Spec: searchv1alpha1.ResourceRegistrySpec{
TargetCluster: policyv1alpha1.ClusterAffinity{
Expand Down Expand Up @@ -377,7 +383,7 @@ var _ = ginkgo.Describe("[karmada-search] karmada search testing", ginkgo.Ordere
ginkgo.BeforeAll(func() {
rr1 = &searchv1alpha1.ResourceRegistry{
ObjectMeta: metav1.ObjectMeta{
Name: "rr-" + rand.String(RandomStrLength),
Name: resourceRegistryPrefix + rand.String(RandomStrLength),
},
Spec: searchv1alpha1.ResourceRegistrySpec{
TargetCluster: policyv1alpha1.ClusterAffinity{
Expand All @@ -393,7 +399,7 @@ var _ = ginkgo.Describe("[karmada-search] karmada search testing", ginkgo.Ordere
}
rr2 = &searchv1alpha1.ResourceRegistry{
ObjectMeta: metav1.ObjectMeta{
Name: "rr-" + rand.String(RandomStrLength),
Name: resourceRegistryPrefix + rand.String(RandomStrLength),
},
Spec: searchv1alpha1.ResourceRegistrySpec{
TargetCluster: policyv1alpha1.ClusterAffinity{
Expand Down Expand Up @@ -529,7 +535,7 @@ var _ = ginkgo.Describe("[karmada-search] karmada search testing", ginkgo.Ordere
ginkgo.BeforeAll(func() {
rr = &searchv1alpha1.ResourceRegistry{
ObjectMeta: metav1.ObjectMeta{
Name: "rr-" + rand.String(RandomStrLength),
Name: resourceRegistryPrefix + rand.String(RandomStrLength),
},
Spec: searchv1alpha1.ResourceRegistrySpec{
TargetCluster: policyv1alpha1.ClusterAffinity{
Expand Down Expand Up @@ -711,4 +717,193 @@ var _ = ginkgo.Describe("[karmada-search] karmada search testing", ginkgo.Ordere
})
})
})

// test when cluster joined, updated, karmada will reconcile search cache
ginkgo.Describe("reconcile ResourceRegistry when clusters joined, updated", ginkgo.Ordered, func() {
ginkgo.Context("when cluster joined", func() {
var (
clusterName string
homeDir string
kubeConfigPath string
controlPlane string
clusterContext string
f cmdutil.Factory

rrName string
rr *searchv1alpha1.ResourceRegistry
resourceSelector searchv1alpha1.ResourceSelector
)

ginkgo.BeforeAll(func() {
clusterName = "member-e2e-" + rand.String(RandomStrLength)
homeDir = os.Getenv("HOME")
kubeConfigPath = fmt.Sprintf("%s/.kube/%s.config", homeDir, clusterName)
controlPlane = fmt.Sprintf("%s-control-plane", clusterName)
clusterContext = fmt.Sprintf("kind-%s", clusterName)
defaultConfigFlags := genericclioptions.NewConfigFlags(true).WithDeprecatedPasswordFlag().WithDiscoveryBurst(300).WithDiscoveryQPS(50.0)
defaultConfigFlags.Context = &karmadaContext
f = cmdutil.NewFactory(defaultConfigFlags)

// create a cluster
ginkgo.By(fmt.Sprintf("Creating cluster: %s", clusterName), func() {
err := createCluster(clusterName, kubeConfigPath, controlPlane, clusterContext)
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
})

// create a ResourceRegistry with a non joined cluster name
rrName = resourceRegistryPrefix + rand.String(RandomStrLength)
resourceSelector = searchv1alpha1.ResourceSelector{
APIVersion: "apps/v1",
Kind: "Deployment",
}
rr = &searchv1alpha1.ResourceRegistry{
ObjectMeta: metav1.ObjectMeta{
Name: rrName,
},
Spec: searchv1alpha1.ResourceRegistrySpec{
TargetCluster: policyv1alpha1.ClusterAffinity{
ClusterNames: []string{clusterName},
},
ResourceSelectors: []searchv1alpha1.ResourceSelector{
resourceSelector,
},
},
}
// create a ResourceRegistry with a non joined cluster name
ginkgo.By(fmt.Sprintf("create ResourceRegistry %s with a non joined cluster name %v and resource selector %v", rrName, clusterName, resourceSelector), func() {
framework.CreateResourceRegistry(karmadaClient, rr)
})
})

ginkgo.AfterAll(func() {
framework.RemoveResourceRegistry(karmadaClient, rrName)

ginkgo.By(fmt.Sprintf("Deleting clusters: %s", clusterName), func() {
err := deleteCluster(clusterName, kubeConfigPath)
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
_ = os.Remove(kubeConfigPath)
})
})

ginkgo.It("[member clusters joined] could reconcile ResourceRegistry", func() {
// search cache should not have the deployment
searchObject(pathAllDeployments, existsDeploymentName, false)
// join the cluster
ginkgo.By(fmt.Sprintf("Joinning cluster: %s", clusterName), func() {
opts := join.CommandJoinOption{
DryRun: false,
ClusterNamespace: "karmada-cluster",
ClusterName: clusterName,
ClusterContext: clusterContext,
ClusterKubeConfig: kubeConfigPath,
}
err := opts.Run(f)
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
})
// search cache should have the deployment
searchObject(pathAllDeployments, existsDeploymentName, true)

ginkgo.By(fmt.Sprintf("Unjoinning cluster: %s", clusterName), func() {
opts := unjoin.CommandUnjoinOption{
DryRun: false,
ClusterNamespace: "karmada-cluster",
ClusterName: clusterName,
ClusterContext: clusterContext,
ClusterKubeConfig: kubeConfigPath,
Wait: 5 * options.DefaultKarmadactlCommandDuration,
}
err := opts.Run(f)
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
})
})
})

ginkgo.Context("when cluster updated", func() {
var (
rrName string
rr *searchv1alpha1.ResourceRegistry
labelKey string
labelValue string
labelSelector *metav1.LabelSelector
resourceSelector searchv1alpha1.ResourceSelector
)

ginkgo.BeforeAll(func() {
// create a ResourceRegistry with label selector
rrName = resourceRegistryPrefix + rand.String(RandomStrLength)
labelKey = "karmada-cluster.k8s.some.com/managed"
labelValue = "true"
labelSelector = &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: labelKey,
Operator: metav1.LabelSelectorOpExists,
},
},
}
resourceSelector = searchv1alpha1.ResourceSelector{
APIVersion: "v1",
Kind: "Pod",
}
rr = &searchv1alpha1.ResourceRegistry{
ObjectMeta: metav1.ObjectMeta{
Name: rrName,
},
Spec: searchv1alpha1.ResourceRegistrySpec{
TargetCluster: policyv1alpha1.ClusterAffinity{
LabelSelector: labelSelector,
},
ResourceSelectors: []searchv1alpha1.ResourceSelector{
resourceSelector,
},
},
}
ginkgo.By(fmt.Sprintf("create ResourceRegistry %s with label selector %v and resource selector %v", rrName, labelSelector, resourceSelector), func() {
framework.CreateResourceRegistry(karmadaClient, rr)
})
})

ginkgo.AfterAll(func() {
framework.RemoveResourceRegistry(karmadaClient, rrName)
})

ginkgo.It("[member clusters updated, deleted label] could reconcile ResourceRegistry", func() {
// search the pod in the ResourceRegistry will fail
searchObject(pathAllPods, member1PodName, false)

// add the label to the member1 cluster
ginkgo.By(fmt.Sprintf("add label %s=%s to cluster %s", labelKey, labelValue, member1), func() {
framework.UpdateClusterLabels(karmadaClient, member1, map[string]string{labelKey: labelValue})
})

// search the pod in the ResourceRegistry will success
searchObject(pathAllPods, member1PodName, true)

// add the label to the member2 cluster
ginkgo.By(fmt.Sprintf("add label %s=%s to cluster %s", labelKey, labelValue, member2), func() {
framework.UpdateClusterLabels(karmadaClient, member2, map[string]string{labelKey: labelValue})
})

// search the pod in the ResourceRegistry will success
searchObject(pathAllPods, member2PodName, true)

// delete the label of the member2 cluster
ginkgo.By(fmt.Sprintf("delete label %s=%s to cluster %s", labelKey, labelValue, member2), func() {
framework.DeleteClusterLabels(karmadaClient, member2, map[string]string{labelKey: labelValue})
})

// search the pod in the ResourceRegistry will fail
searchObject(pathAllPods, member2PodName, false)

// delete the label of the member1 cluster
ginkgo.By(fmt.Sprintf("delete label %s=%s to cluster %s", labelKey, labelValue, member1), func() {
framework.DeleteClusterLabels(karmadaClient, member1, map[string]string{labelKey: labelValue})

})

// search the pod in the ResourceRegistry will fail
searchObject(pathAllPods, member1PodName, false)
})
})
})
})
1 change: 1 addition & 0 deletions test/e2e/suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ const (
podDisruptionBudgetNamePrefix = "poddisruptionbudget-"
federatedHPANamePrefix = "fhpa-"
cronFedratedHPANamePrefix = "cronfhpa-"
resourceRegistryPrefix = "rr-"

updateDeploymentReplicas = 2
updateStatefulSetReplicas = 2
Expand Down

0 comments on commit 48e9a5b

Please sign in to comment.