flux-framework · vsoch · Mar 30, 2023 · Mar 30, 2023 · Mar 30, 2023 · Mar 30, 2023
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -60,20 +60,20 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        test: [["hello-world", "ghcr.io/flux-framework/flux-restful-api:latest", 30],
-               ["post", "ghcr.io/flux-framework/flux-restful-api:latest", 30],
-               ["batch", "ghcr.io/flux-framework/flux-restful-api:latest", 30],
-               ["singularity", "ghcr.io/rse-ops/singularity:tag-mamba", 40],
-               ["nginx-sidecar-service", "ghcr.io/flux-framework/flux-restful-api:latest", 40],
-               ["nginx-service", "ghcr.io/flux-framework/flux-restful-api:latest", 40],
-               ["lammps", "ghcr.io/rse-ops/lammps:flux-sched-focal-v0.24.0", 30],
-               ["pokemon", "ghcr.io/rse-ops/pokemon:app-latest", 30],
-               ["timed", "ghcr.io/flux-framework/flux-restful-api:latest", 30],
-               ["debug", "ghcr.io/flux-framework/flux-restful-api:latest", 30],
-               ["flux-resource-list", "ghcr.io/flux-framework/flux-restful-api:latest", 30],
-               ["existing-volumes", "ghcr.io/flux-framework/flux-restful-api:latest", 30],
-               ["volumes", "ghcr.io/flux-framework/flux-restful-api:latest", 30],
-               ["snakemake", "ghcr.io/rse-ops/atacseq:app-latest", 30]]
+        test: [["hello-world", "ghcr.io/flux-framework/flux-restful-api:latest", 40],
+               ["post", "ghcr.io/flux-framework/flux-restful-api:latest", 40],
+               ["batch", "ghcr.io/flux-framework/flux-restful-api:latest", 40],
+               ["singularity", "ghcr.io/rse-ops/singularity:tag-mamba", 50],
+               ["nginx-sidecar-service", "ghcr.io/flux-framework/flux-restful-api:latest", 50],
+               ["nginx-service", "ghcr.io/flux-framework/flux-restful-api:latest", 50],
+               ["lammps", "ghcr.io/rse-ops/lammps:flux-sched-focal-v0.24.0", 40],
+               ["pokemon", "ghcr.io/rse-ops/pokemon:app-latest", 40],
+               ["timed", "ghcr.io/flux-framework/flux-restful-api:latest", 40],
+               ["debug", "ghcr.io/flux-framework/flux-restful-api:latest", 40],
+               ["flux-resource-list", "ghcr.io/flux-framework/flux-restful-api:latest", 40],
+               ["existing-volumes", "ghcr.io/flux-framework/flux-restful-api:latest", 50],
+               ["volumes", "ghcr.io/flux-framework/flux-restful-api:latest", 50],
+               ["snakemake", "ghcr.io/rse-ops/atacseq:app-latest", 40]]
 
     steps:
     - name: Clone the code
@@ -84,23 +84,28 @@ jobs:
       with:
         go-version: ^1.18
 
-    - name: Start minikube
-      uses: medyagh/setup-minikube@697f2b7aaed5f70bf2a94ee21a4ec3dde7b12f92 # v0.0.9
+    - name: Create k8s Kind Cluster
+      run: |
+        curl -Lo ./kind "https://kind.sigs.k8s.io/dl/v0.17.0/kind-$(uname)-amd64"
+        chmod +x ./kind
+        sudo mv ./kind /usr/local/bin/kind      
+        # prepares host mount at /tmp/data
+        kind create cluster --config ./script/kind-config.yaml
 
     - name: Create the namespace
       run: kubectl create namespace flux-operator
 
-    - name: Pull Docker Containers to MiniKube
+    - name: Pull Docker Containers
       env:
         container: ${{ matrix.test[1] }}
         test: ${{ matrix[0] }}
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: |
-        export SHELL=/bin/bash
-        eval $(minikube -p minikube docker-env)
-        minikube ssh docker pull ${container}
+        docker pull ${container}
+        kind load docker-image ${container}
         if [[ "${test}" == "nginx-service" ]]; then
-            minikube ssh docker pull nginx
+            docker pull nginx
+            kind load docker-image nginx
         fi
         make
         make install
@@ -109,4 +114,4 @@ jobs:
       env:
         name: ${{ matrix.test[0] }}
         jobtime: ${{ matrix.test[2] }}
-      run: /bin/bash ./script/test.sh ${name} ${jobtime}
+      run: /bin/bash ./script/test.sh ${name} ${jobtime}
diff --git a/.github/workflows/test-python.yaml b/.github/workflows/test-python.yaml
@@ -49,25 +49,30 @@ jobs:
       with:
         go-version: ^1.18
 
-    - name: Start minikube
-      uses: medyagh/setup-minikube@697f2b7aaed5f70bf2a94ee21a4ec3dde7b12f92 # v0.0.9
+    - name: Create k8s Kind Cluster
+      run: |
+        curl -Lo ./kind "https://kind.sigs.k8s.io/dl/v0.17.0/kind-$(uname)-amd64"
+        chmod +x ./kind
+        sudo mv ./kind /usr/local/bin/kind      
+        # prepares host mount at /tmp/data
+        kind create cluster --config ./script/kind-config.yaml
 
     - name: Create the namespace
       run: kubectl create namespace flux-operator
 
-    - name: Pull Docker Containers to MiniKube
+    - name: Pull Docker Containers
       env:
         container: ${{ matrix.test[1] }}
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: |
-        export SHELL=/bin/bash
-        eval $(minikube -p minikube docker-env)
-        minikube ssh docker pull ${container}
+        docker pull ${container}
+        kind load docker-image ${container}
         make
         make install
         make docker-build
         printf "Loading Flux Operator latest image...\n"
-        minikube image load ghcr.io/flux-framework/flux-operator:latest
+        docker pull ghcr.io/flux-framework/flux-operator:latest
+        kind load docker-image ghcr.io/flux-framework/flux-operator:latest
         kubectl apply -f ./examples/dist/flux-operator.yaml
 
     - name: Test ${{ matrix.test[0] }}

diff --git a/docs/development/developer-guide.md b/docs/development/developer-guide.md
@@ -174,7 +174,13 @@ examples/tests/
     └── minicluster.yaml
 ```
 
-Thus, to run the full example for hello-world you can do:
+You'll first want to create a kind cluster that mounts `/tmp/data`:
+
+```bash
+$ kind create cluster --config ./script/kind-config.yaml
+```
+
+Then, to run the full example for hello-world you can do:
 
 ```bash
 $ bash script/test.sh hello-world

diff --git a/examples/tests/existing-volumes/post-run.sh b/examples/tests/existing-volumes/post-run.sh
diff --git a/examples/tests/existing-volumes/pre-run.sh b/examples/tests/existing-volumes/pre-run.sh
@@ -3,11 +3,11 @@
 HERE=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 TESTS=$(dirname ${HERE})
 
-echo "Copying local volume to /tmp/data-volumes in minikube"
+echo "Copying local volume to /tmp/data for kind"
+mkdir -p /tmp/data
 
 # The "data" volume will be mounted at /mnt/data
-minikube ssh -- mkdir -p /tmp/data
-minikube cp ${TESTS}/data/pancakes.txt /tmp/data/pancakes.txt
-minikube ssh ls /tmp/data
+cp ${TESTS}/data/pancakes.txt /tmp/data/pancakes.txt
+ls /tmp/data
 kubectl apply -f ${HERE}/pv.yaml 
 kubectl apply -f ${HERE}/pvc.yaml
diff --git a/examples/tests/existing-volumes/test.out.correct b/examples/tests/existing-volumes/test.out.correct
@@ -1 +1 @@
-pancakes.txt
+pancakes.txt
diff --git a/examples/tests/hello-world/test.out.correct b/examples/tests/hello-world/test.out.correct
@@ -1 +1 @@
-hello world
+hello world
diff --git a/examples/tests/hello-world/test.sh b/examples/tests/hello-world/test.sh
diff --git a/examples/tests/nginx-sidecar-service/test.out.correct b/examples/tests/nginx-sidecar-service/test.out.correct
@@ -1,24 +1,24 @@
 Defaulted container "flux-sample" out of: flux-sample, nginx
-<!DOCTYPE html>
-<html>
-<head>
-<title>Welcome to nginx!</title>
-<style>
-html { color-scheme: light dark; }
-body { width: 35em; margin: 0 auto;
-font-family: Tahoma, Verdana, Arial, sans-serif; }
-</style>
-</head>
-<body>
-<h1>Welcome to nginx!</h1>
-<p>If you see this page, the nginx web server is successfully installed and
-working. Further configuration is required.</p>
-
-<p>For online documentation and support please refer to
-<a href="http://nginx.org/">nginx.org</a>.<br/>
-Commercial support is available at
-<a href="http://nginx.com/">nginx.com</a>.</p>
-
-<p><em>Thank you for using nginx.</em></p>
-</body>
-</html>
+<!DOCTYPE html>
+<html>
+<head>
+<title>Welcome to nginx!</title>
+<style>
+html { color-scheme: light dark; }
+body { width: 35em; margin: 0 auto;
+font-family: Tahoma, Verdana, Arial, sans-serif; }
+</style>
+</head>
+<body>
+<h1>Welcome to nginx!</h1>
+<p>If you see this page, the nginx web server is successfully installed and
+working. Further configuration is required.</p>
+
+<p>For online documentation and support please refer to
+<a href="http://nginx.org/">nginx.org</a>.<br/>
+Commercial support is available at
+<a href="http://nginx.com/">nginx.com</a>.</p>
+
+<p><em>Thank you for using nginx.</em></p>
+</body>
+</html>
diff --git a/examples/tests/pokemon/test.out.correct b/examples/tests/pokemon/test.out.correct
@@ -1,32 +1,32 @@
-@@@@@@@@@@,::,@@@@@@@@@@@@@@@@@@@@@@@@,,,,@@@@@@@@@@@@@@@@@@
-@@@@@@@@@,*,,:,@@@@@@@@@@@@@@@@@@@@,,:::*:@@@@@@@@@@@@@@@@@@
-@@@@@@@@,*:,::*@@@@@@@@@@@@@@@@@@,::*:::*,@@@@@@@@@@@@@@@@@@
-@@@@@@@@:*,,:*::@@@@@@@@@@@@@@@,:::::::.:@@@@@@@@@@@@@@@@@@@
-@@@@@@@@::,::.:*@,,::***:::,,@,:::::::::*@@@@@@@@@@@@@@@@@@@
-@@@@@@@@@*,::**.*.......**.*..:,*::::,,*:@@@@@@@@@@@@@@@@@@@
-@@@@@@@@:**,:..*************..,*::::,,*:@@@@@@@@@@@@@@@@@@@@
-@@@@@@@@*::**::*************.*:*,::::*:@@@@@@@@@@@@@@@@@@@@@
-@@@@@@@@,**+:****************.::,,,:*,@@@@@@@@@,,,:::,,@@@@@
-@@@@@@@@@@*..**************.*..*::::,@@@@@@@@,:::,:::::*:@@@
-@@@@@@@@@@:...*********....,*.+++S.:@@@@@@@,*:,,,:***:::*,@@
-@@@@@@@@@@,.:...*****..+%?S@*++++S*.*,@@@@@*:,,::**:::::*:@@
-@@@@@@@@@@@**.%+********S%+,.S++SS.+++.**:*.::::::.*::***,@@
-@@@@@,,:::*.+...*********..+S.SSS+******.......*::*..**.,@@@
-@,::.*******.++++.+.....++SS++.+++.*********.+:*+*:.::::*@@@
-:*.*.*.*****..++*..S.....%.+*::*S+++.....**.:.S...*.****.,@@
-:*SS.*+..++++++:,,,,*S.S+*:,,,,,*+S++++++++S+**:::.*:::::::@
-@@,**.++++++++S,,,,,,,::,,,,,,,,,,.++++.S+++.::::***,:**,,**
-@@@@@@@,,,,,,,*,@,,,,,:*,,,,,,,,,,,::*.,++.+.:::*::****:,,:.
-@@@@@@@@@@@@@@,*,,,,,,,*,,,,,,,,,,:**.+:.S++S::::::::::,:::*
-@@@@@@@@@@@@@@@*:,::::,**,,,,::::****:..*.+S+::::::*:,::::*,
-@@@@@@@@@@@@@@@:.,@,,,,:.:,,,,,,::::*.+S+++..:::::**:::::*,@
-@@@@@@@@@@@@@,*...:,,,,,,*,,,,,::::.+++++++S.:***::::::*:@@@
-@@@@@@@@@@@@,.***..*,,,,,**:::*:::.+++++++++S+*::::::::,@@@@
-@@@@@@@@@@@,.***...+++.**:::*****.+*.***...++S+:::::,@@@@@@@
-@@@@@@@@@@@:S...+++SS..+*:*******.++**:****.++S*,@@@@@@@@@@@
-@@@@@@@@@@@:SSSSSSSSSS:,@@,,,,,,,,,:..**...++++S,@@@@@@@@@@@
-@@@@@@@@@@,.+SSSSSSS+,@@@@@@@@@@@@@@,++++++++++S.@@@@@@@@@@@
-@@@@@@@@@@@@@*+.:*+*@@@@@@@@@@@@@@@@@*S++++++++.+:@@@@@@@@@@
-@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@:*:::*+.:,,:,@@@@@@@@@@
-
-convergedcomputingftw
+@@@@@@@@@@,::,@@@@@@@@@@@@@@@@@@@@@@@@,,,,@@@@@@@@@@@@@@@@@@
+@@@@@@@@@,*,,:,@@@@@@@@@@@@@@@@@@@@,,:::*:@@@@@@@@@@@@@@@@@@
+@@@@@@@@,*:,::*@@@@@@@@@@@@@@@@@@,::*:::*,@@@@@@@@@@@@@@@@@@
+@@@@@@@@:*,,:*::@@@@@@@@@@@@@@@,:::::::.:@@@@@@@@@@@@@@@@@@@
+@@@@@@@@::,::.:*@,,::***:::,,@,:::::::::*@@@@@@@@@@@@@@@@@@@
+@@@@@@@@@*,::**.*.......**.*..:,*::::,,*:@@@@@@@@@@@@@@@@@@@
+@@@@@@@@:**,:..*************..,*::::,,*:@@@@@@@@@@@@@@@@@@@@
+@@@@@@@@*::**::*************.*:*,::::*:@@@@@@@@@@@@@@@@@@@@@
+@@@@@@@@,**+:****************.::,,,:*,@@@@@@@@@,,,:::,,@@@@@
+@@@@@@@@@@*..**************.*..*::::,@@@@@@@@,:::,:::::*:@@@
+@@@@@@@@@@:...*********....,*.+++S.:@@@@@@@,*:,,,:***:::*,@@
+@@@@@@@@@@,.:...*****..+%?S@*++++S*.*,@@@@@*:,,::**:::::*:@@
+@@@@@@@@@@@**.%+********S%+,.S++SS.+++.**:*.::::::.*::***,@@
+@@@@@,,:::*.+...*********..+S.SSS+******.......*::*..**.,@@@
+@,::.*******.++++.+.....++SS++.+++.*********.+:*+*:.::::*@@@
+:*.*.*.*****..++*..S.....%.+*::*S+++.....**.:.S...*.****.,@@
+:*SS.*+..++++++:,,,,*S.S+*:,,,,,*+S++++++++S+**:::.*:::::::@
+@@,**.++++++++S,,,,,,,::,,,,,,,,,,.++++.S+++.::::***,:**,,**
+@@@@@@@,,,,,,,*,@,,,,,:*,,,,,,,,,,,::*.,++.+.:::*::****:,,:.
+@@@@@@@@@@@@@@,*,,,,,,,*,,,,,,,,,,:**.+:.S++S::::::::::,:::*
+@@@@@@@@@@@@@@@*:,::::,**,,,,::::****:..*.+S+::::::*:,::::*,
+@@@@@@@@@@@@@@@:.,@,,,,:.:,,,,,,::::*.+S+++..:::::**:::::*,@
+@@@@@@@@@@@@@,*...:,,,,,,*,,,,,::::.+++++++S.:***::::::*:@@@
+@@@@@@@@@@@@,.***..*,,,,,**:::*:::.+++++++++S+*::::::::,@@@@
+@@@@@@@@@@@,.***...+++.**:::*****.+*.***...++S+:::::,@@@@@@@
+@@@@@@@@@@@:S...+++SS..+*:*******.++**:****.++S*,@@@@@@@@@@@
+@@@@@@@@@@@:SSSSSSSSSS:,@@,,,,,,,,,:..**...++++S,@@@@@@@@@@@
+@@@@@@@@@@,.+SSSSSSS+,@@@@@@@@@@@@@@,++++++++++S.@@@@@@@@@@@
+@@@@@@@@@@@@@*+.:*+*@@@@@@@@@@@@@@@@@*S++++++++.+:@@@@@@@@@@
+@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@:*:::*+.:,,:,@@@@@@@@@@
+
+convergedcomputingftw
diff --git a/examples/tests/post/test.out.correct b/examples/tests/post/test.out.correct
@@ -1,2 +1,2 @@
-hello world
-hello post
+hello world
+hello post
diff --git a/examples/tests/snakemake/minicluster.yaml b/examples/tests/snakemake/minicluster.yaml
@@ -21,8 +21,6 @@ spec:
     data:
       storageClass: hostpath
       path: /tmp/data
-      labels:
-        type: "local"
 
   # This is a list because a pod can support multiple containers
   containers:
@@ -35,11 +33,11 @@ spec:
       # in the container
       volumes:
         data:
-          path: /workflow
+          path: /tmp/data
           # readOnly defaults to false
 
       # This is the directory we will bind the data to
-      workingDir: /workflow
+      workingDir: /tmp/data
       command: snakemake --cores 1 --flux --jobs 1
 
       # Give the command directly to flux start, instead of wrapping in flux submit

diff --git a/examples/tests/snakemake/post-run.sh b/examples/tests/snakemake/post-run.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+echo "Cleaning up /tmp/data bound to kind"
+rm -rf /tmp/data/*
diff --git a/examples/tests/snakemake/pre-run.sh b/examples/tests/snakemake/pre-run.sh
@@ -3,12 +3,9 @@
 HERE=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
 echo "Cloning snakemake run data to /tmp/workflow"
-git clone --depth 1 https://github.com/snakemake/snakemake-tutorial-data /tmp/workflow
 
-wget -O /tmp/workflow/Snakefile https://raw.githubusercontent.com/rse-ops/flux-hpc/main/snakemake/atacseq/Snakefile
-mkdir -p /tmp/workflow/scripts
-wget -O /tmp/workflow/scripts/plot-quals.py https://raw.githubusercontent.com/rse-ops/flux-hpc/main/snakemake/atacseq/scripts/plot-quals.py
+git clone --depth 1 https://github.com/snakemake/snakemake-tutorial-data /tmp/data
 
-echo "Preparing to mount into MiniKube"
-minikube ssh -- mkdir -p /tmp/data
-minikube mount /tmp/workflow:/tmp/data &
+wget -O /tmp/data/Snakefile https://raw.githubusercontent.com/rse-ops/flux-hpc/main/snakemake/atacseq/Snakefile
+mkdir -p /tmp/data/scripts
+wget -O /tmp/data/scripts/plot-quals.py https://raw.githubusercontent.com/rse-ops/flux-hpc/main/snakemake/atacseq/scripts/plot-quals.py
diff --git a/examples/tests/volumes/minicluster.yaml b/examples/tests/volumes/minicluster.yaml
@@ -12,13 +12,11 @@ spec:
   size: 2
 
   # Make this kind of persistent volume and claim available to pods
-  # This is a path in minikube (e.g., minikube ssh)
+  # This is a path in minikube or kind (e.g., minikube ssh)
   volumes:
     data:
       storageClass: hostpath
       path: /tmp/data
-      labels:
-        type: "local"
 
   # This is a list because a pod can support multiple containers
   containers:

diff --git a/examples/tests/volumes/post-run.sh b/examples/tests/volumes/post-run.sh
diff --git a/examples/tests/volumes/pre-run.sh b/examples/tests/volumes/pre-run.sh
@@ -3,9 +3,9 @@
 HERE=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 TESTS=$(dirname ${HERE})
 
-echo "Copying local volume to /tmp/data-volumes in minikube"
+echo "Copying local volume to /tmp/data mounted in kind"
 
 # The "data" volume will be mounted at /mnt/data
-minikube ssh -- mkdir -p /tmp/data
-minikube cp ${TESTS}/data/pancakes.txt /tmp/data/pancakes.txt
-minikube ssh ls /tmp/data
+mkdir -p /tmp/data
+cp ${TESTS}/data/pancakes.txt /tmp/data/pancakes.txt
+ls /tmp/data
diff --git a/examples/tests/volumes/test.out.correct b/examples/tests/volumes/test.out.correct
@@ -1 +1 @@
-pancakes.txt
+pancakes.txt