docker: podman-friendly image locations

Adds fully qualified canonical locations of container images, making the container technology setup podman-friendly. Amends cluster creation scripts to support either Docker or Podman container technologies based on `docker version` information. Closes #729.
reanahub · Jul 13, 2023 · cea49fc · cea49fc
1 parent 6f86e14
commit cea49fc
Show file tree

Hide file tree

Showing 14 changed files with 86 additions and 77 deletions.
diff --git a/helm/configurations/values-dev.yaml b/helm/configurations/values-dev.yaml
@@ -2,25 +2,25 @@
 
 components:
     reana_server:
-      image: reanahub/reana-server
+      image: docker.io/reanahub/reana-server
       environment:
         REANA_SCHEDULER_REQUEUE_SLEEP: 2
         REANA_RATELIMIT_SLOW: "5 per second"
     reana_workflow_controller:
-      image: reanahub/reana-workflow-controller
+      image: docker.io/reanahub/reana-workflow-controller
       environment:
         REANA_RUNTIME_KUBERNETES_KEEP_ALIVE_JOBS_WITH_STATUSES: failed
     reana_workflow_engine_cwl:
-      image: reanahub/reana-workflow-engine-cwl
+      image: docker.io/reanahub/reana-workflow-engine-cwl
     reana_workflow_engine_yadage:
-      image: reanahub/reana-workflow-engine-yadage
+      image: docker.io/reanahub/reana-workflow-engine-yadage
     reana_workflow_engine_serial:
-      image: reanahub/reana-workflow-engine-serial
+      image: docker.io/reanahub/reana-workflow-engine-serial
     reana_workflow_engine_snakemake:
-      image: reanahub/reana-workflow-engine-snakemake
+      image: docker.io/reanahub/reana-workflow-engine-snakemake
     reana_job_controller:
-      image: reanahub/reana-job-controller
+      image: docker.io/reanahub/reana-job-controller
     reana_message_broker:
-      image: reanahub/reana-message-broker
+      image: docker.io/reanahub/reana-message-broker
     reana_ui:
-      image: reanahub/reana-ui
+      image: docker.io/reanahub/reana-ui
diff --git a/helm/reana/README.md b/helm/reana/README.md
@@ -9,8 +9,8 @@ This Helm automatically prefixes all names using the release name to avoid colli
 | Parameter                                                | Description                                                                          | Default value                                   |
 |----------------------------------------------------------|--------------------------------------------------------------------------------------|-------------------------------------------------|
 | `components.reana_db.enabled`                            | Instantiate a PostgreSQL database inside the cluster                                 | true                                            |
-| `components.reana_job_controller.image`                  | [REANA-Job-Controller image](https://hub.docker.com/r/reanahub/reana-job-controller) to use  | `reanahub/reana-job-controller:<chart-release-version>` |
-| `components.reana_message_broker.image`                  | [REANA-Message-Broker image](https://hub.docker.com/r/reanahub/reana-message-broker) to use | `reanahub/reana-message-broker:<chart-release-version>` |
+| `components.reana_job_controller.image`                  | [REANA-Job-Controller image](https://hub.docker.com/r/reanahub/reana-job-controller) to use  | `docker.io/reanahub/reana-job-controller:<chart-release-version>` |
+| `components.reana_message_broker.image`                  | [REANA-Message-Broker image](https://hub.docker.com/r/reanahub/reana-message-broker) to use | `docker.io/reanahub/reana-message-broker:<chart-release-version>` |
 | `components.reana_message_broker.imagePullPolicy`        | REANA-Message-Broker image pull policy                                               | IfNotPresent                                    |
 | `components.reana_server.environment`                    | REANA-Server environment variables                                                   | |
 | `components.reana_server.environment.REANA_MAX_CONCURRENT_BATCH_WORKFLOWS` | Upper limit on concurrent REANA batch workflows running in the cluster. | 30 |
@@ -22,7 +22,7 @@ This Helm automatically prefixes all names using the release name to avoid colli
 | `components.reana_server.environment.REANA_RATELIMIT_GUEST_USER` | Set API limiter config for guest users. Users using reana-client will be treated as guests. | "20 per second" |
 | `components.reana_server.environment.REANA_RATELIMIT_AUTHENTICATED_USER` | Set API limiter config for authenticated web UI users. | "20 per second" |
 | `components.reana_server.environment.REANA_RATELIMIT_SLOW` | Set API limiter config for slow endpoints that need to be protected e.g. launch endpoint. | "1/5 second" |
-| `components.reana_server.image`                          | [REANA-Server image](https://hub.docker.com/r/reanahub/reana-server) to use          | `reanahub/reana-server:<chart-release-version>` |
+| `components.reana_server.image`                          | [REANA-Server image](https://hub.docker.com/r/reanahub/reana-server) to use          | `docker.io/reanahub/reana-server:<chart-release-version>` |
 | `components.reana_server.imagePullPolicy`                | REANA-Server image pull policy                                                       | IfNotPresent                                    |
 | `components.reana_server.uwsgi.processes`                | Number of uWSGI processes                                                            | 6                                               |
 | `components.reana_server.uwsgi.threads`                  | Number of uWSGI threads                                                              | 4                                               |
@@ -31,7 +31,7 @@ This Helm automatically prefixes all names using the release name to avoid colli
 | `components.reana_server.uwsgi.log_5xx`                  | Log only error HTTP requests with status code 5xx. To make this configuration effective `components.reana_server.uwsgi.log_all` must be false. | true |
 | `components.reana_ui.announcement`                       | Announcement message displayed in site top banner                                    | None                                            |
 | `components.reana_ui.enabled`                            | Instantiate the [REANA-UI](https://github.com/reanahub/reana-ui)                     | true                                            |
-| `components.reana_ui.image`                              | [REANA-UI image](https://hub.docker.com/r/reanahub/reana-ui) to use                  | `reanahub/reana-ui:<chart-release-version>`     |
+| `components.reana_ui.image`                              | [REANA-UI image](https://hub.docker.com/r/reanahub/reana-ui) to use                  | `docker.io/reanahub/reana-ui:<chart-release-version>`     |
 | `components.reana_ui.imagePullPolicy`                    | REANA-UI image pull policy                                                           | IfNotPresent                                    |
 | `components.reana_ui.polling_secs`                       | Frequency of workflow list page reload in seconds                                    | 15                                              |
 | `components.reana_ui.client_pyvenv`                      | REANA-Client python environment to source in the welcome example.                    | None                                            |
@@ -43,15 +43,15 @@ This Helm automatically prefixes all names using the release name to avoid colli
 | `components.reana_ui.local_users`                        | Enable local users sign in/up                                                        | true                                            |
 | `components.reana_ui.hide_signup`                        | Hide users sign up form                                                              | false                                           |
 | `components.reana_workflow_controller.environment`       | REANA-Workflow-Controller environment variables                                      | `{SHARED_VOLUME_PATH: /var/reana}`              |
-| `components.reana_workflow_controller.image`             | [REANA-Workflow-Controller image](https://hub.docker.com/r/reanahub/reana-workflow-controller) to use | `reanahub/reana-workflow-controller:<chart-release-version>` |
+| `components.reana_workflow_controller.image`             | [REANA-Workflow-Controller image](https://hub.docker.com/r/reanahub/reana-workflow-controller) to use | `docker.io/reanahub/reana-workflow-controller:<chart-release-version>` |
 | `components.reana_workflow_controller.imagePullPolicy`   | REANA-Workflow-Controller image pull policy                                          | IfNotPresent                                    |
 | `components.reana_workflow_controller.environment.REANA_JOB_HOSTPATH_MOUNTS`   | JSON list of optional hostPath mounts, for all user jobs. Each mount object has a key `name` (name of the mount), `hostPath` (path to the directory to be mounted from the Kubernetes nodes) and `mountPath` (path inside the job containers where the `hostPath` will be mounted)            | None                                            |
 | `components.reana_workflow_controller.environment.REANA_RUNTIME_KUBERNETES_KEEP_ALIVE_JOBS_WITH_STATUSES` | Keep alive Kubernetes user runtime jobs depending on status (`finished` and/or `failed`). | None |
 | `components.reana_workflow_controller.environment.REANA_JOB_STATUS_CONSUMER_PREFETCH_COUNT` | Define max number of unacknowledged deliveries that are permitted on `jobs-status` queue consumer. | 10 |
-| `components.reana_workflow_engine_cwl.image`             | [REANA-Workflow-Engine-CWL image](https://hub.docker.com/r/reanahub/reana-workflow-engine-cwl) to use | `reanahub/reana-workflow-engine-cwl:<chart-release-version>` |
-| `components.reana_workflow_engine_serial.image`          | [REANA-Workflow-Engine-Serial image](https://hub.docker.com/r/reanahub/reana-workflow-engine-serial) to use | `reanahub/reana-workflow-engine-serial:<chart-release-version>` |
-| `components.reana_workflow_engine_yadage.image`          | [REANA-Workflow-Engine-Yadage image](https://hub.docker.com/r/reanahub/reana-workflow-engine-yadage) to use | `reanahub/reana-workflow-engine-yadage:<chart-release-version>` |
-| `components.reana_workflow_engine_snakemake.image`       | [REANA-Workflow-Engine-Snakemake image](https://hub.docker.com/r/reanahub/reana-workflow-engine-snakemake) to use | `reanahub/reana-workflow-engine-snakemake:<chart-release-version>` |
+| `components.reana_workflow_engine_cwl.image`             | [REANA-Workflow-Engine-CWL image](https://hub.docker.com/r/reanahub/reana-workflow-engine-cwl) to use | `docker.io/reanahub/reana-workflow-engine-cwl:<chart-release-version>` |
+| `components.reana_workflow_engine_serial.image`          | [REANA-Workflow-Engine-Serial image](https://hub.docker.com/r/reanahub/reana-workflow-engine-serial) to use | `docker.io/reanahub/reana-workflow-engine-serial:<chart-release-version>` |
+| `components.reana_workflow_engine_yadage.image`          | [REANA-Workflow-Engine-Yadage image](https://hub.docker.com/r/reanahub/reana-workflow-engine-yadage) to use | `docker.io/reanahub/reana-workflow-engine-yadage:<chart-release-version>` |
+| `components.reana_workflow_engine_snakemake.image`       | [REANA-Workflow-Engine-Snakemake image](https://hub.docker.com/r/reanahub/reana-workflow-engine-snakemake) to use | `docker.io/reanahub/reana-workflow-engine-snakemake:<chart-release-version>` |
 | `compute_backends`                                       | List of supported compute backends (kubernetes, htcondorcern, slurmcern)             | "kubernetes"                                    |
 | `db_env_config.REANA_DB_HOST`                            | Environment variable to connect to external databases                                | `<chart-release-name>-db`                       |
 | `db_env_config.REANA_DB_NAME`                            | Environment variable to connect to external databases                                | reana                                           |
@@ -127,4 +127,4 @@ This Helm automatically prefixes all names using the release name to avoid colli
 | `workspaces.retention_rules.cronjob_schedule`            | Cron format string describing how often pending retention rules should be applied.   | "0 2 * * *"                                     |
 | `workspaces.paths`                                       | List of additional workspace paths as strings. Each mount string is composed by a key `hostPath`(path to the directory to be mounted from the Kubernetes nodes) and a cluster_pod_mountpath (path inside the cluster containers where the `mountPath` will be mounted) e.g. `hostPath:mountPath`. The first value listed will be the default workspace root path. Any POSIX filesystem mounted on cluster nodes is supported | None |
 | `interactive_sessions.cronjob_schedule`                  | Cron format string describing how often interactive session cleanup should be performed. | "0 3 * * *"                                 |
-| `interactive_sessions.maximum_inactivity_period`         | Set a limit in days for the maximum inactivity period of interactive sessions. After this period interactive sessions will be automatically closed. To disable autoclosure and allow interactive sessions to run forever, use value "forever". | forever |
+| `interactive_sessions.maximum_inactivity_period`         | Set a limit in days for the maximum inactivity period of interactive sessions. After this period interactive sessions will be automatically closed. To disable autoclosure and allow interactive sessions to run forever, use value "forever". | forever |
diff --git a/helm/reana/templates/reana-cache.yaml b/helm/reana/templates/reana-cache.yaml
@@ -30,7 +30,7 @@ spec:
     spec:
       containers:
       - name: cache
-        image: redis:5.0.5
+        image: docker.io/library/redis:5.0.5
         ports:
         - containerPort: 6379
       {{- if .Values.node_label_infrastructure }}

diff --git a/helm/reana/templates/reana-db.yaml b/helm/reana/templates/reana-db.yaml
@@ -31,7 +31,7 @@ spec:
     spec:
       containers:
       - name: db
-        image: postgres:12.13
+        image: docker.io/library/postgres:12.13
         args:
         - -c
         - max_connections=300

diff --git a/helm/reana/templates/reana-mail.yaml b/helm/reana/templates/reana-mail.yaml
@@ -38,7 +38,7 @@ spec:
     spec:
       containers:
       - name: maildev
-        image: maildev/maildev:1.1.0
+        image: docker.io/maildev/maildev:1.1.0
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 80

diff --git a/helm/reana/templates/reana-wdb.yaml b/helm/reana/templates/reana-wdb.yaml
@@ -37,7 +37,7 @@ spec:
     spec:
       containers:
       - name: wdb
-        image: kozea/wdb:3.2.5
+        image: docker.io/kozea/wdb:3.2.5
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 1984

diff --git a/helm/reana/values.yaml b/helm/reana/values.yaml
@@ -76,12 +76,12 @@ components:
     docs_url: https://docs.reana.io
     forum_url: https://forum.reana.io
     imagePullPolicy: IfNotPresent
-    image: reanahub/reana-ui:0.9.1-alpha.3
+    image: docker.io/reanahub/reana-ui:0.9.1-alpha.3
   reana_db:
     enabled: true
   reana_server:
     imagePullPolicy: IfNotPresent
-    image: reanahub/reana-server:0.9.1-alpha.3
+    image: docker.io/reanahub/reana-server:0.9.1-alpha.3
     environment:
       REANA_MAX_CONCURRENT_BATCH_WORKFLOWS: 30
       REANA_SCHEDULER_REQUEUE_SLEEP: 15
@@ -100,23 +100,23 @@ components:
       log_5xx: true
   reana_workflow_controller:
     imagePullPolicy: IfNotPresent
-    image: reanahub/reana-workflow-controller:0.9.1-alpha.2
+    image: docker.io/reanahub/reana-workflow-controller:0.9.1-alpha.2
     environment:
       SHARED_VOLUME_PATH: /var/reana
       REANA_JOB_STATUS_CONSUMER_PREFETCH_COUNT: 10
   reana_workflow_engine_cwl:
-    image: reanahub/reana-workflow-engine-cwl:0.9.1-alpha.1
+    image: docker.io/reanahub/reana-workflow-engine-cwl:0.9.1-alpha.1
   reana_workflow_engine_yadage:
-    image: reanahub/reana-workflow-engine-yadage:0.9.1-alpha.1
+    image: docker.io/reanahub/reana-workflow-engine-yadage:0.9.1-alpha.1
   reana_workflow_engine_serial:
-    image: reanahub/reana-workflow-engine-serial:0.9.1-alpha.1
+    image: docker.io/reanahub/reana-workflow-engine-serial:0.9.1-alpha.1
   reana_workflow_engine_snakemake:
-    image: reanahub/reana-workflow-engine-snakemake:0.9.1-alpha.1
+    image: docker.io/reanahub/reana-workflow-engine-snakemake:0.9.1-alpha.1
   reana_job_controller:
-    image: reanahub/reana-job-controller:0.9.1-alpha.2
+    image: docker.io/reanahub/reana-job-controller:0.9.1-alpha.2
   reana_message_broker:
     imagePullPolicy: IfNotPresent
-    image: reanahub/reana-message-broker:0.9.1-alpha.1
+    image: docker.io/reanahub/reana-message-broker:0.9.1-alpha.1
 
 notifications:
   enabled: false

diff --git a/reana/config.py b/reana/config.py
@@ -213,39 +213,39 @@
 
 DOCKER_PREFETCH_IMAGES = {
     "reana": [
-        "postgres:12.13",
-        "kozea/wdb:3.2.5",
-        "maildev/maildev:1.1.0",
-        "redis:5.0.5",
+        "docker.io/library/postgres:12.13",
+        "docker.io/kozea/wdb:3.2.5",
+        "docker.io/maildev/maildev:1.1.0",
+        "docker.io/library/redis:5.0.5",
     ],
     "reana-demo-helloworld": [
-        "python:2.7-slim",
+        "docker.io/library/python:2.7-slim",
     ],
     "reana-demo-worldpopulation": [
-        "reanahub/reana-env-jupyter:2.0.0",
+        "docker.io/reanahub/reana-env-jupyter:2.0.0",
     ],
     "reana-demo-root6-roofit": [
-        "reanahub/reana-env-root6:6.18.04",
+        "docker.io/reanahub/reana-env-root6:6.18.04",
     ],
     "reana-demo-atlas-recast": [
-        "reanahub/reana-demo-atlas-recast-eventselection:1.0",
-        "reanahub/reana-demo-atlas-recast-statanalysis:1.0",
+        "docker.io/reanahub/reana-demo-atlas-recast-eventselection:1.0",
+        "docker.io/reanahub/reana-demo-atlas-recast-statanalysis:1.0",
     ],
     "reana-demo-bsm-search": [
-        "reanahub/reana-demo-bsm-search:1.0.0",
-        "reanahub/reana-env-root6:6.18.04",
+        "docker.io/reanahub/reana-demo-bsm-search:1.0.0",
+        "docker.io/reanahub/reana-env-root6:6.18.04",
     ],
     "reana-demo-cms-h4l": [
-        "cmsopendata/cmssw_5_3_32",
+        "docker.io/cmsopendata/cmssw_5_3_32",
     ],
     "reana-demo-cms-dimuon-mass-spectrum": [
-        "cmsopendata/cmssw_5_3_32",
+        "docker.io/cmsopendata/cmssw_5_3_32",
     ],
     "reana-demo-alice-pt-analysis": [
-        "reanahub/reana-env-aliphysics:vAN-20180614-1",
+        "docker.io/reanahub/reana-env-aliphysics:vAN-20180614-1",
     ],
     "reana-demo-alice-lego-train-test-run": [
-        "reanahub/reana-env-aliphysics:vAN-20180614-1",
+        "docker.io/reanahub/reana-env-aliphysics:vAN-20180614-1",
     ],
 }
 """Images to be prefetched depending on the REANA demo to be executed."""

diff --git a/reana/reana_dev/cluster.py b/reana/reana_dev/cluster.py
@@ -157,10 +157,17 @@ def add_volume_mounts(node):
             "podSubnet": "192.168.0.0/16",
         }
 
-    cluster_create = "cat <<EOF | kind create cluster --config=-\n{cluster_config}\nEOF"
-    cluster_create = cluster_create.format(cluster_config=yaml.dump(cluster_config))
+    # detect user container technology (Docker vs Podman)
+    kind_provider = ""
+    docker_version_output = run_command("docker version", return_output=True)
+    if docker_version_output and "Podman Engine" in docker_version_output:
+        kind_provider = "KIND_EXPERIMENTAL_PROVIDER=podman"
 
     # create cluster
+    cluster_create = "cat <<EOF | {kind_provider} kind create cluster --config=-\n{cluster_config}\nEOF"
+    cluster_create = cluster_create.format(
+        kind_provider=kind_provider, cluster_config=yaml.dump(cluster_config)
+    )
     run_command(cluster_create, "reana")
 
     # pull Docker images

diff --git a/reana/reana_dev/docker.py b/reana/reana_dev/docker.py
@@ -158,7 +158,9 @@ def docker_build(
                 cmd += " --no-cache"
             if quiet or parallel > 1:
                 cmd += " --quiet"
-            component_version_tag = "{0}/{1}:{2}".format(user, component, component_tag)
+            component_version_tag = "docker.io/{0}/{1}:{2}".format(
+                user, component, component_tag
+            )
             cmd += " -t {0} .".format(component_version_tag)
             commands.append((_run_command, (cmd, component)))
             built_components_versions_tags.append(component_version_tag)

diff --git a/reana/reana_dev/kind.py b/reana/reana_dev/kind.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of REANA.
-# Copyright (C) 2020, 2021 CERN.
+# Copyright (C) 2020, 2021, 2023 CERN.
 #
 # REANA is free software; you can redistribute it and/or modify it
 # under the terms of the MIT License; see LICENSE file for more details.
@@ -79,7 +79,7 @@ def kind_load_docker_image(user, component, node, exclude_components):  # noqa:
                     cmd = f"{cmd} --nodes {','.join(node)}"
                 run_command(cmd, component)
         elif is_component_dockerised(component):
-            cmd = "kind load docker-image {0}/{1}".format(user, component)
+            cmd = "kind load docker-image docker.io/{0}/{1}".format(user, component)
             if node:
                 cmd = f"{cmd} --nodes {','.join(node)}"
             run_command(cmd, component)