Merge pull request #305 from zzxwill/force-deletion

Support force deletion of Configuration

api/v1beta2/configuration_types.go

@@ -73,6 +73,10 @@ type BaseConfigurationSpec struct {
 
  // Region is cloud provider's region. It will override the region in the region field of ProviderReference
  Region string `json:"customRegion,omitempty"`
+
+ // ForceDelete will force delete Configuration no matter which state it is or whether it has provisioned some resources
+ // It will help delete Configuration in unexpected cases.
+ ForceDelete *bool `json:"forceDelete,omitempty"`
 }
 
 // ConfigurationStatus defines the observed state of Configuration

chart/crds/terraform.core.oam.dev_configurations.yaml

@@ -212,6 +212,11 @@ spec:
  description: DeleteResource will determine whether provisioned cloud
  resources will be deleted when CR is deleted
  type: boolean
+ forceDelete:
+ description: ForceDelete will force delete Configuration no matter
+ which state it is or whether it has provisioned some resources It
+ will help delete Configuration in unexpected cases.
+ type: boolean
  hcl:
  description: HCL is the Terraform HCL type configuration
  type: string

controllers/configuration/configuration.go

@@ -111,8 +111,13 @@ func Get(ctx context.Context, k8sClient client.Client, namespacedName apitypes.N
 }
 
 // IsDeletable will check whether the Configuration can be deleted immediately
-// If deletable, it means no external cloud resources are provisioned
+// If deletable, it means
+// - no external cloud resources are provisioned
+//- it's in force-delete state
 func IsDeletable(ctx context.Context, k8sClient client.Client, configuration *v1beta2.Configuration) (bool, error) {
+ if configuration.Spec.ForceDelete != nil && *configuration.Spec.ForceDelete {
+ return true, nil
+ }
  if !configuration.Spec.InlineCredentials {
  providerRef := GetProviderNamespacedName(*configuration)
  providerObj, err := provider.GetProviderFromConfiguration(ctx, k8sClient, providerRef.Namespace, providerRef.Name)

controllers/configuration_controller.go

@@ -355,61 +355,75 @@ func (r *ConfigurationReconciler) terraformDestroy(ctx context.Context, namespac
  return err
  }
 
+ if configuration.Spec.ForceDelete != nil && *configuration.Spec.ForceDelete {
+ // Try to clean up more sub-resources as possible. Ignore the issues if it hit any.
+ if err := r.cleanUpSubResources(ctx, namespace, configuration, meta); err != nil {
+ klog.Warningf("Failed to clean up sub-resources, but it's ignored as the resources are being forced to delete: %s", err)
+ }
+ return nil
+ }
  // When the deletion Job process succeeded, clean up work is starting.
  if err := k8sClient.Get(ctx, client.ObjectKey{Name: meta.DestroyJobName, Namespace: meta.Namespace}, &destroyJob); err != nil {
  return err
  }
  if destroyJob.Status.Succeeded == int32(1) || deleteConfigurationDirectly {
- // 1. delete Terraform input Configuration ConfigMap
- if err := meta.deleteConfigMap(ctx, k8sClient); err != nil {
- return err
- }
+ return r.cleanUpSubResources(ctx, namespace, configuration, meta)
+ }
 
- // 2. delete connectionSecret
- if configuration.Spec.WriteConnectionSecretToReference != nil {
- secretName := configuration.Spec.WriteConnectionSecretToReference.Name
- secretNameSpace := configuration.Spec.WriteConnectionSecretToReference.Namespace
- if err := deleteConnectionSecret(ctx, k8sClient, secretName, secretNameSpace); err != nil {
- return err
- }
+ return errors.New(types.MessageDestroyJobNotCompleted)
+}
+
+func (r *ConfigurationReconciler) cleanUpSubResources(ctx context.Context, namespace string, configuration v1beta2.Configuration, meta *TFConfigurationMeta) error {
+ var k8sClient = r.Client
+
+ // 1. delete Terraform input Configuration ConfigMap
+ if err := meta.deleteConfigMap(ctx, k8sClient); err != nil {
+ return err
+ }
+
+ // 2. delete connectionSecret
+ if configuration.Spec.WriteConnectionSecretToReference != nil {
+ secretName := configuration.Spec.WriteConnectionSecretToReference.Name
+ secretNameSpace := configuration.Spec.WriteConnectionSecretToReference.Namespace
+ if err := deleteConnectionSecret(ctx, k8sClient, secretName, secretNameSpace); err != nil {
+ return err
  }
+ }
 
- // 3. delete apply job
- var applyJob batchv1.Job
- if err := k8sClient.Get(ctx, client.ObjectKey{Name: meta.ApplyJobName, Namespace: namespace}, &applyJob); err == nil {
- if err := k8sClient.Delete(ctx, &applyJob, client.PropagationPolicy(metav1.DeletePropagationBackground)); err != nil {
- return err
- }
+ // 3. delete apply job
+ var applyJob batchv1.Job
+ if err := k8sClient.Get(ctx, client.ObjectKey{Name: meta.ApplyJobName, Namespace: namespace}, &applyJob); err == nil {
+ if err := k8sClient.Delete(ctx, &applyJob, client.PropagationPolicy(metav1.DeletePropagationBackground)); err != nil {
+ return err
  }
+ }
 
- // 4. delete destroy job
- var j batchv1.Job
- if err := r.Client.Get(ctx, client.ObjectKey{Name: destroyJob.Name, Namespace: destroyJob.Namespace}, &j); err == nil {
- if err := r.Client.Delete(ctx, &j, client.PropagationPolicy(metav1.DeletePropagationBackground)); err != nil {
- return err
- }
+ // 4. delete destroy job
+ var j batchv1.Job
+ if err := r.Client.Get(ctx, client.ObjectKey{Name: meta.DestroyJobName, Namespace: meta.Namespace}, &j); err == nil {
+ if err := r.Client.Delete(ctx, &j, client.PropagationPolicy(metav1.DeletePropagationBackground)); err != nil {
+ return err
  }
+ }
 
- // 5. delete secret which stores variables
- klog.InfoS("Deleting the secret which stores variables", "Name", meta.VariableSecretName)
- var variableSecret v1.Secret
- if err := r.Client.Get(ctx, client.ObjectKey{Name: meta.VariableSecretName, Namespace: meta.Namespace}, &variableSecret); err == nil {
- if err := r.Client.Delete(ctx, &variableSecret); err != nil {
- return err
- }
+ // 5. delete secret which stores variables
+ klog.InfoS("Deleting the secret which stores variables", "Name", meta.VariableSecretName)
+ var variableSecret v1.Secret
+ if err := r.Client.Get(ctx, client.ObjectKey{Name: meta.VariableSecretName, Namespace: meta.Namespace}, &variableSecret); err == nil {
+ if err := r.Client.Delete(ctx, &variableSecret); err != nil {
+ return err
  }
+ }
 
- // 6. delete Kubernetes backend secret
- klog.InfoS("Deleting the secret which stores Kubernetes backend", "Name", meta.BackendSecretName)
- var kubernetesBackendSecret v1.Secret
- if err := r.Client.Get(ctx, client.ObjectKey{Name: meta.BackendSecretName, Namespace: meta.TerraformBackendNamespace}, &kubernetesBackendSecret); err == nil {
- if err := r.Client.Delete(ctx, &kubernetesBackendSecret); err != nil {
- return err
- }
+ // 6. delete Kubernetes backend secret
+ klog.InfoS("Deleting the secret which stores Kubernetes backend", "Name", meta.BackendSecretName)
+ var kubernetesBackendSecret v1.Secret
+ if err := r.Client.Get(ctx, client.ObjectKey{Name: meta.BackendSecretName, Namespace: meta.TerraformBackendNamespace}, &kubernetesBackendSecret); err == nil {
+ if err := r.Client.Delete(ctx, &kubernetesBackendSecret); err != nil {
+ return err
  }
- return nil
  }
- return errors.New(types.MessageDestroyJobNotCompleted)
+ return nil
 }
 
 func (r *ConfigurationReconciler) preCheckResourcesSetting(meta *TFConfigurationMeta) error {

controllers/configuration_controller_test.go

@@ -1125,6 +1125,19 @@ func TestTerraformDestroy(t *testing.T) {
  VariableSecretName: "c",
  }
 
+ r5 := &ConfigurationReconciler{}
+ forceDeleteConfig5 := &v1beta2.Configuration{
+ ObjectMeta: metav1.ObjectMeta{
+ Namespace: "default",
+ Name: "force-delete-config-5",
+ },
+ Spec: v1beta2.ConfigurationSpec{},
+ }
+ var forceDelete = true
+ forceDeleteConfig5.Spec.ForceDelete = &forceDelete
+ k8sClient5 := fake.NewClientBuilder().WithScheme(s).WithObjects(forceDeleteConfig5).Build()
+ r5.Client = k8sClient5
+
  type args struct {
  r *ConfigurationReconciler
  namespace string
@@ -1177,6 +1190,14 @@ func TestTerraformDestroy(t *testing.T) {
  },
  want: want{},
  },
+ {
+ name: "force delete configuration",
+ args: args{
+ r: r5,
+ configuration: forceDeleteConfig5,
+ meta: &TFConfigurationMeta{},
+ },
+ },
  }
  for _, tc := range testcases {
  t.Run(tc.name, func(t *testing.T) {
@@ -1331,8 +1352,6 @@ func TestGetTFOutputs(t *testing.T) {
  k8sClient1 := fake.NewClientBuilder().Build()
  meta1 := &TFConfigurationMeta{}
 
- //scheme := runtime.NewScheme()
- //v1beta2.AddToScheme(scheme)
  secret2 := &corev1.Secret{
  ObjectMeta: metav1.ObjectMeta{
  Name: "a",

e2e/configuration_test.go

@@ -26,6 +26,7 @@ var (
  "examples/alibaba/eip/configuration_eip_remote_subdirectory.yaml",
  "examples/alibaba/oss/configuration_hcl_bucket.yaml",
  }
+ testConfigurationsForceDelete = "examples/random/configuration_force_delete.yaml"
 )
 
 func TestInlineCredentialsConfiguration(t *testing.T) {
@@ -112,7 +113,7 @@ continueCheck:
  }
  if existed {
  if i == 59 {
- t.Error("Configuration is not ready")
+ t.Error("Configuration is not deleted")
  }
 
  time.Sleep(time.Second * 5)
@@ -136,6 +137,52 @@ continueCheck:
  assert.Equal(t, kerrors.IsNotFound(err), true)
 }
 
+func TestForceDeleteConfiguration(t *testing.T) {
+ klog.Info("1. Applying Configuration whose hcl is not valid")
+ pwd, _ := os.Getwd()
+ configuration := filepath.Join(pwd, "..", testConfigurationsForceDelete)
+ cmd := fmt.Sprintf("kubectl apply -f %s", configuration)
+ err := exec.Command("bash", "-c", cmd).Start()
+ assert.NilError(t, err)
+
+ klog.Info("2. Deleting Configuration")
+ cmd = fmt.Sprintf("kubectl delete -f %s", configuration)
+ err = exec.Command("bash", "-c", cmd).Start()
+ assert.NilError(t, err)
+
+ klog.Info("5. Checking Configuration is deleted")
+ for i := 0; i < 60; i++ {
+ var (
+ fields []string
+ existed bool
+ )
+ output, err := exec.Command("bash", "-c", "kubectl get configuration").Output()
+ assert.NilError(t, err)
+
+ lines := strings.Split(string(output), "\n")
+
+ for j, line := range lines {
+ if j == 0 {
+ continue
+ }
+ fields = strings.Fields(line)
+ if len(fields) == 3 && fields[0] == "random-e2e-force-delete" {
+ existed = true
+ }
+ }
+ if existed {
+ if i == 59 {
+ t.Error("Configuration is not deleted")
+ }
+
+ time.Sleep(time.Second * 5)
+ continue
+ } else {
+ break
+ }
+ }
+}
+
 //func TestBasicConfigurationRegression(t *testing.T) {
 // var retryTimes = 120
 //

examples/random/configuration_force_delete.yaml

@@ -0,0 +1,20 @@
+apiVersion: terraform.core.oam.dev/v1beta2
+kind: Configuration
+metadata:
+ name: random-e2e-force-delete
+spec:
+ hcl: |
+ resource "random_id" "server" {
+ byte_length = 8
+ }
+ 
+ output "random_id" {
+ value = random_id.server.hex
+
+ inlineCredentials: true
+
+ forceDelete: true
+
+ writeConnectionSecretToRef:
+ name: random-conn
+ namespace: default