add support for cluster downsize

.github/workflows/helm.yaml

@@ -13,10 +13,10 @@ jobs:
     name: Prepare chart
     steps:
     - name: Checkout Repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - uses: actions/setup-go@v3
       with:
-        go-version: ^1.18.1
+        go-version: ^1.20
     - name: GHCR Login
       if: (github.event_name != 'pull_request')
       uses: docker/login-action@v2
@@ -44,4 +44,4 @@ jobs:
         PKG_RESPONSE=$(helm package ./chart)
         echo "$PKG_RESPONSE"
         CHART_TAR_GZ=$(basename "$PKG_RESPONSE") 
-        helm push "$CHART_TAR_GZ" oci://ghcr.io/flux-framework/flux-operator-helm
+        helm push "$CHART_TAR_GZ" oci://ghcr.io/flux-framework/flux-operator-helm

.github/workflows/main.yaml

@@ -81,7 +81,7 @@ jobs:
 
     steps:
     - name: Clone the code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Setup Go
       uses: actions/setup-go@v3
@@ -92,7 +92,7 @@ jobs:
       run: sudo apt-get update && sudo apt-get install -y libsodium-dev libzmq3-dev libczmq-dev
 
     - name: Start minikube
-      uses: medyagh/setup-minikube@697f2b7aaed5f70bf2a94ee21a4ec3dde7b12f92 # v0.0.9
+      uses: medyagh/setup-minikube@master
 
     - name: Pull Docker Containers to MiniKube
       env:

Makefile

@@ -342,6 +342,7 @@ $(KUSTOMIZE): $(LOCALBIN)
 controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary.
 $(CONTROLLER_GEN): $(LOCALBIN)
 	GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION)
+	go mod tidy
 
 # Build the latest openapi-gen from source
 .PHONY: openapi-gen

api/v1alpha2/minicluster_types.go

@@ -66,6 +66,12 @@ type MiniClusterSpec struct {
 	// +optional
 	ShareProcessNamespace bool `json:"shareProcessNamespace"`
 
+	// Restart failed workers (defaults to true)
+	// This is setting backoffLimitPerIndex to 0 on the backend
+	// This requires an additional feature gate to be enabled.
+	// +optional
+	SuspendWorkers bool `json:"suspendWorkers"`
+
 	// Cleanup the pods and storage when the index broker pod is complete
 	// +kubebuilder:default=false
 	// +default=false

api/v1alpha2/swagger.json

@@ -94,14 +94,12 @@
         "limits": {
           "type": "object",
           "additionalProperties": {
-            "default": {},
             "$ref": "#/definitions/IntOrString"
           }
         },
         "requests": {
           "type": "object",
           "additionalProperties": {
-            "default": {},
             "$ref": "#/definitions/IntOrString"
           }
         }
@@ -622,6 +620,11 @@
           "format": "int32",
           "default": 1
         },
+        "suspendWorkers": {
+          "description": "Restart failed workers (defaults to true) This is setting backoffLimitPerIndex to 0 on the backend This requires an additional feature gate to be enabled.",
+          "type": "boolean",
+          "default": false
+        },
         "tasks": {
           "description": "Total number of CPUs being run across entire cluster",
           "type": "integer",
@@ -735,7 +738,6 @@
           "description": "Resources include limits and requests",
           "type": "object",
           "additionalProperties": {
-            "default": {},
             "$ref": "#/definitions/IntOrString"
           }
         },

chart/templates/minicluster-crd.yaml

@@ -689,6 +689,11 @@ spec:
                   pods) This is also the minimum number required to start Flux
                 format: int32
                 type: integer
+              suspendWorkers:
+                description: Restart failed workers (defaults to true) This is setting
+                  backoffLimitPerIndex to 0 on the backend This requires an additional
+                  feature gate to be enabled.
+                type: boolean
               tasks:
                 default: 1
                 description: Total number of CPUs being run across entire cluster

config/crd/bases/flux-framework.org_miniclusters.yaml

@@ -695,6 +695,11 @@ spec:
                   in pods) This is also the minimum number required to start Flux
                 format: int32
                 type: integer
+              suspendWorkers:
+                description: Restart failed workers (defaults to true) This is setting
+                  backoffLimitPerIndex to 0 on the backend This requires an additional
+                  feature gate to be enabled.
+                type: boolean
               tasks:
                 default: 1
                 description: Total number of CPUs being run across entire cluster

controllers/flux/job.go

@@ -77,6 +77,12 @@ func NewMiniClusterJob(cluster *api.MiniCluster) (*batchv1.Job, error) {
 		},
 	}
 
+	// Does the user want backoff limit per index?
+	if cluster.Spec.SuspendWorkers {
+		var backoffLimitPerIndex int32 = 0
+		job.Spec.BackoffLimitPerIndex = &backoffLimitPerIndex
+	}
+
 	// Add Affinity to map one pod / node only if the user hasn't disbaled it
 	if !cluster.Spec.Network.DisableAffinity {
 		job.Spec.Template.Spec.Affinity = getAffinity(cluster)

docs/getting_started/custom-resource-definition.md

@@ -92,6 +92,19 @@ This would be equivalent to giving a start command of `sleep infinity` however o
 (e.g., if there is a flux shutdown from within the Flux instance) the sleep command would
 not exit with a failed code.
 
+### suspendWorkers
+
+By default, when a pod fails it is attempted to restart. When you use the [JobBackoffPerIndex](https://kubernetes.io/blog/2023/08/21/kubernetes-1-28-jobapi-update/#backoff-limit-per-index) feature gate (Kubernetes 1.28) you can set this to an explicit number of failures allowed. For example, a value of 0 will mean the pod is only allowed to fail once (and not recreated).
+Since we want this primarily to be a case of "restart the workers" or "don't restart them" we expose this as a boolean.
+Setting `suspendWorkers` to true indicates that on a failure, we do not restart.
+
+```yaml
+spec:
+  suspendWorkers: true
+```
+
+This can be useful for cases of autoscaling in the down direction when you need to drain a node, and then delete the pod.
+
 ### launcher
 
 If you are using an executor that launches Flux Jobs (e.g., workflow managers such as Snakemake and Nextflow do!)

examples/dist/flux-operator-arm.yaml

@@ -701,6 +701,11 @@ spec:
                   in pods) This is also the minimum number required to start Flux
                 format: int32
                 type: integer
+              suspendWorkers:
+                description: Restart failed workers (defaults to true) This is setting
+                  backoffLimitPerIndex to 0 on the backend This requires an additional
+                  feature gate to be enabled.
+                type: boolean
               tasks:
                 default: 1
                 description: Total number of CPUs being run across entire cluster

examples/dist/flux-operator.yaml

@@ -701,6 +701,11 @@ spec:
                   in pods) This is also the minimum number required to start Flux
                 format: int32
                 type: integer
+              suspendWorkers:
+                description: Restart failed workers (defaults to true) This is setting
+                  backoffLimitPerIndex to 0 on the backend This requires an additional
+                  feature gate to be enabled.
+                type: boolean
               tasks:
                 default: 1
                 description: Total number of CPUs being run across entire cluster