Skip to content

Commit

Permalink
tests: dump the kubeconfig for the testenv server
Browse files Browse the repository at this point in the history
This is going to be useful to inspect the state of the cluster after a
test fails.

Set the env variable SKIP_CLEANUP to "true" or "1" to skip the cleanup
of the environment to make use of the kubeconfig.

The kubeconfig is dumped in the testbin directory.

After debugging you will have to kill the testEnv. The two binaries that
are used by the testEnv are etcd and kube-apiserver. Check for the
running process with a command like

ps aux | grep -e kube-apiserver -e etcd

Signed-off-by: Raghavendra Talur <raghavendra.talur@gmail.com>
(cherry picked from commit 07a61b4)
  • Loading branch information
raghavendra-talur authored and rakeshgm committed Jul 20, 2024
1 parent 4664e7e commit 9157b6a
Show file tree
Hide file tree
Showing 3 changed files with 115 additions and 3 deletions.
46 changes: 44 additions & 2 deletions internal/controller/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package controllers_test

import (
"context"
"log"
"os"
"path/filepath"
"testing"
Expand Down Expand Up @@ -89,6 +90,8 @@ var (
objectStorers [2]ramencontrollers.ObjectStorer

ramenNamespace = "ns-envtest"

skipCleanup bool
)

func TestAPIs(t *testing.T) {
Expand Down Expand Up @@ -146,6 +149,11 @@ var _ = BeforeSuite(func() {
ramenNamespace = rNs
}

skipCleanupEnv, set := os.LookupEnv("SKIP_CLEANUP")
if set && (skipCleanupEnv == "true" || skipCleanupEnv == "1") {
skipCleanup = true
}

By("bootstrapping test environment")
testEnv = &envtest.Environment{
CRDDirectoryPaths: []string{
Expand All @@ -164,10 +172,25 @@ var _ = BeforeSuite(func() {
defer GinkgoRecover()
cfg, err = testEnv.Start()
DeferCleanup(func() error {
if skipCleanup {
By("skipping cleanup of the test environment")

return nil
}
By("tearing down the test environment")

return testEnv.Stop()
})
kubeConfigContent, err := ramencontrollers.ConvertRestConfigToKubeConfig(cfg)
if err != nil {
log.Fatalf("Failed to convert rest.Config to kubeconfig: %v", err)
}

filePath := "../../testbin/kubeconfig.yaml"
if err := ramencontrollers.WriteKubeConfigToFile(kubeConfigContent, filePath); err != nil {
log.Fatalf("Failed to write kubeconfig file: %v", err)
}

close(done)
}()
Eventually(done).WithTimeout(time.Minute).Should(BeClosed())
Expand Down Expand Up @@ -241,7 +264,15 @@ var _ = BeforeSuite(func() {
ramenConfig.DrClusterOperator.S3SecretDistributionEnabled = true
ramenConfig.MultiNamespace.FeatureEnabled = true
configMapCreate(ramenConfig)
DeferCleanup(configMapDelete)
DeferCleanup(func() error {
if skipCleanup {
By("skipping cleanup of the test environment")

return nil
}

return configMapDelete()
})

s3Secrets[0] = corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Namespace: ramenNamespace, Name: "s3secret0"},
Expand Down Expand Up @@ -382,7 +413,18 @@ var _ = BeforeSuite(func() {
Expect(err).ToNot(HaveOccurred())

ctx, cancel = context.WithCancel(context.TODO())
DeferCleanup(cancel)
DeferCleanup(func() error {
if skipCleanup {
By("skipping cleanup of the test environment")

return nil
}

cancel()

return nil
})

go func() {
err = k8sManager.Start(ctx)
Expect(err).ToNot(HaveOccurred())
Expand Down
44 changes: 44 additions & 0 deletions internal/controller/testutils.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package controllers

import (
"os"

"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/tools/clientcmd/api"
)

// WriteKubeConfigToFile writes the kubeconfig content to a specified file path.
func WriteKubeConfigToFile(kubeConfigContent, filePath string) error {
return os.WriteFile(filePath, []byte(kubeConfigContent), 0o600)
}

// ConvertRestConfigToKubeConfig converts a rest.Config to a kubeconfig string.
func ConvertRestConfigToKubeConfig(restConfig *rest.Config) (string, error) {
kubeConfig := api.NewConfig()

cluster := api.NewCluster()
cluster.Server = restConfig.Host
cluster.CertificateAuthorityData = restConfig.CAData

user := api.NewAuthInfo()
user.ClientCertificateData = restConfig.CertData
user.ClientKeyData = restConfig.KeyData
user.Token = restConfig.BearerToken

context := api.NewContext()
context.Cluster = "cluster"
context.AuthInfo = "user"

kubeConfig.Clusters["cluster"] = cluster
kubeConfig.AuthInfos["user"] = user
kubeConfig.Contexts["test"] = context
kubeConfig.CurrentContext = "test"

kubeConfigContent, err := clientcmd.Write(*kubeConfig)
if err != nil {
return "", err
}

return string(kubeConfigContent), nil
}
28 changes: 27 additions & 1 deletion internal/controller/volsync/volsync_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package volsync_test

import (
"context"
"log"
"os"
"path/filepath"
"testing"
Expand All @@ -19,6 +20,7 @@ import (
storagev1 "k8s.io/api/storage/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
cfgpolicyv1 "open-cluster-management.io/config-policy-controller/api/v1"
policyv1 "open-cluster-management.io/governance-policy-propagator/api/v1"
ctrl "sigs.k8s.io/controller-runtime"
Expand All @@ -29,6 +31,7 @@ import (
metrics "sigs.k8s.io/controller-runtime/pkg/metrics/server"

volsyncv1alpha1 "github.com/backube/volsync/api/v1alpha1"
controllers "github.com/ramendr/ramen/internal/controller"
"github.com/ramendr/ramen/internal/controller/util"
)

Expand Down Expand Up @@ -56,6 +59,9 @@ var (
volumeSnapshotClassB *snapv1.VolumeSnapshotClass

totalVolumeSnapshotClassCount = 0

skipCleanup bool
cfg *rest.Config
)

func TestVolsync(t *testing.T) {
Expand All @@ -64,6 +70,8 @@ func TestVolsync(t *testing.T) {
}

var _ = BeforeSuite(func() {
var err error

logger = zap.New(zap.UseFlagOptions(&zap.Options{
Development: true,
DestWriter: GinkgoWriter,
Expand All @@ -85,6 +93,11 @@ var _ = BeforeSuite(func() {
Expect(os.Setenv("KUBEBUILDER_ASSETS", string(content))).To(Succeed())
}

skipCleanupEnv, set := os.LookupEnv("SKIP_CLEANUP")
if set && (skipCleanupEnv == "true" || skipCleanupEnv == "1") {
skipCleanup = true
}

By("bootstrapping test environment")
testEnv = &envtest.Environment{
CRDDirectoryPaths: []string{
Expand All @@ -93,7 +106,7 @@ var _ = BeforeSuite(func() {
},
}

cfg, err := testEnv.Start()
cfg, err = testEnv.Start()
Expect(err).NotTo(HaveOccurred())
Expect(cfg).NotTo(BeNil())

Expand Down Expand Up @@ -217,6 +230,19 @@ var _ = BeforeSuite(func() {
})

var _ = AfterSuite(func() {
if skipCleanup {
kubeConfigContent, err := controllers.ConvertRestConfigToKubeConfig(cfg)
if err != nil {
log.Fatalf("Failed to convert rest.Config to kubeconfig: %v", err)
}

filePath := "../../../testbin/kubeconfig.yaml"
if err := controllers.WriteKubeConfigToFile(kubeConfigContent, filePath); err != nil {
log.Fatalf("Failed to write kubeconfig file: %v", err)
}

return
}
cancel()
By("tearing down the test environment")
err := testEnv.Stop()
Expand Down

0 comments on commit 9157b6a

Please sign in to comment.