WIP: Allow creation of users through a job

.github/workflows/operatorBuildAndDeploy.yml

@@ -71,7 +71,7 @@ jobs:
  with:
  file: Dockerfile
  context: .
- push: ${{ github.event_name != 'pull_request' }}
+ push: ${{ github.event_name != 'pull_request' && !env.ACT }}
  tags: k8ssandra/cass-operator:${{ steps.vars.outputs.sha_short }}, k8ssandra/cass-operator:latest, k8ssandra/cass-operator:${{ steps.vars.outputs.version }}
  platforms: linux/amd64
  cache-from: type=local,src=/tmp/.buildx-cache
@@ -81,7 +81,7 @@ jobs:
  uses: docker/build-push-action@v3
  with:
  file: logger.Dockerfile
- push: ${{ github.event_name != 'pull_request' }}
+ push: ${{ github.event_name != 'pull_request' && !env.ACT }}
  tags: k8ssandra/system-logger:${{ steps.vars.outputs.sha_short }}, k8ssandra/system-logger:latest
  platforms: linux/amd64
  cache-from: type=local,src=/tmp/.buildx-cache
@@ -106,11 +106,12 @@ jobs:
  build-args: |
  VERSION=${{ steps.vars.outputs.version }}
  context: .
- push: ${{ !env.ACT }}
+ push: ${{ github.event_name != 'pull_request' && !env.ACT }}
  tags: k8ssandra/cass-operator-bundle:v${{ steps.vars.outputs.version }}
  platforms: linux/amd64
  cache-from: type=local,src=/tmp/.buildx-cache
  cache-to: type=local,dest=/tmp/.buildx-cache
  - name: Build and update cass-operator-catalog
  run: |
+ # Load the images here? It wants the bundle..
  make VERSION=${{ steps.vars.outputs.version }} CHANNEL=dev catalog-build catalog-push

apis/cassandra/v1beta1/cassandradatacenter_types.go

@@ -77,6 +77,25 @@ type CassandraUser struct {
  Superuser bool `json:"superuser"`
 }
 
+type UserInfo struct {
+ Annotations map[string]string `json:"annotations,omitempty"`
+ // MountPath tells the script where to read the user information. Required if annotation injection is used, otherwise optional
+ MountPath string `json:"mountPath,omitempty"`
+ CSI *corev1.CSIVolumeSource `json:"csi,omitempty"`
+ SecretName string `json:"secretName,omitempty"`
+
+ // ServiceAccountName override job ServiceAccount, otherwise Spec.ServiceAccount (the one used to create the server pods) is used
+ ServiceAccountName string `json:"serviceAccountName,omitempty"`
+ // TODO Add CSI information here
+ /*
+ TODO Testing:
+ * Add back the Helm utilities
+ * Install Vault + Vault server there
+ * Install CSI driver
+ * Try to create clusters with both methods and see that it succeeds (and uses Vault to fetch the user rights)
+ */
+}
+
 // CassandraDatacenterSpec defines the desired state of a CassandraDatacenter
 // +k8s:openapi-gen=true
 // +kubebuilder:pruning:PreserveUnknownFields
@@ -180,11 +199,13 @@ type CassandraDatacenterSpec struct {
  // podAntiAffinity and requiredDuringSchedulingIgnoredDuringExecution.
  AllowMultipleNodesPerWorker bool `json:"allowMultipleNodesPerWorker,omitempty"`
 
- // This secret defines the username and password for the Cassandra server superuser.
+ // SuperuserSecretName is deprecated. Use UserInfo instead. This secret defines the username and password for the Cassandra server superuser.
  // If it is omitted, we will generate a secret instead.
  SuperuserSecretName string `json:"superuserSecretName,omitempty"`
 
- // The k8s service account to use for the server pods
+ UserInfo *UserInfo `json:"userInfo,omitempty"`
+
+ // ServiceAccount is the k8s service account to use for the server pods
  ServiceAccount string `json:"serviceAccount,omitempty"`
 
  // DEPRECATED. Use CassandraTask for rolling restarts. Whether to do a rolling restart at the next opportunity. The operator will set this back

config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml

@@ -7614,7 +7614,8 @@ spec:
  pattern: (6\.8\.\d+)|(3\.11\.\d+)|(4\.\d+\.\d+)
  type: string
  serviceAccount:
- description: The k8s service account to use for the server pods
+ description: ServiceAccount is the k8s service account to use for
+ the server pods
  type: string
  size:
  description: Desired number of Cassandra server nodes
@@ -7996,9 +7997,9 @@ spec:
  type: object
  type: object
  superuserSecretName:
- description: This secret defines the username and password for the
- Cassandra server superuser. If it is omitted, we will generate a
- secret instead.
+ description: SuperuserSecretName is deprecated. Use UserInfo instead.
+ This secret defines the username and password for the Cassandra
+ server superuser. If it is omitted, we will generate a secret instead.
  type: string
  systemLoggerImage:
  description: Container image for the log tailing sidecar container.
@@ -8072,6 +8073,68 @@ spec:
  type: string
  type: object
  type: array
+ userInfo:
+ properties:
+ annotations:
+ additionalProperties:
+ type: string
+ type: object
+ csi:
+ description: Represents a source location of a volume to mount,
+ managed by an external CSI driver
+ properties:
+ driver:
+ description: driver is the name of the CSI driver that handles
+ this volume. Consult with your admin for the correct name
+ as registered in the cluster.
+ type: string
+ fsType:
+ description: fsType to mount. Ex. "ext4", "xfs", "ntfs". If
+ not provided, the empty value is passed to the associated
+ CSI driver which will determine the default filesystem to
+ apply.
+ type: string
+ nodePublishSecretRef:
+ description: nodePublishSecretRef is a reference to the secret
+ object containing sensitive information to pass to the CSI
+ driver to complete the CSI NodePublishVolume and NodeUnpublishVolume
+ calls. This field is optional, and may be empty if no secret
+ is required. If the secret object contains more than one
+ secret, all secret references are passed.
+ properties:
+ name:
+ description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+ TODO: Add other useful fields. apiVersion, kind, uid?'
+ type: string
+ type: object
+ x-kubernetes-map-type: atomic
+ readOnly:
+ description: readOnly specifies a read-only configuration
+ for the volume. Defaults to false (read/write).
+ type: boolean
+ volumeAttributes:
+ additionalProperties:
+ type: string
+ description: volumeAttributes stores driver-specific properties
+ that are passed to the CSI driver. Consult your driver's
+ documentation for supported values.
+ type: object
+ required:
+ - driver
+ type: object
+ mountPath:
+ description: MountPath tells the script where to read the user
+ information. Required if annotation injection is used, otherwise
+ optional
+ type: string
+ secretName:
+ type: string
+ serviceAccountName:
+ description: ServiceAccountName override job ServiceAccount, otherwise
+ Spec.ServiceAccount (the one used to create the server pods)
+ is used
+ type: string
+ type: object
  users:
  description: Cassandra users to bootstrap
  items:

config/manager/kustomization.yaml

@@ -14,4 +14,4 @@ kind: Kustomization
 images:
 - name: controller
  newName: k8ssandra/cass-operator
- newTag: latest
+ newTag: v1.12.0-dev.28b8ea7-20220629

config/rbac/role.yaml

@@ -5,6 +5,18 @@ metadata:
  creationTimestamp: null
  name: manager-role
 rules:
+- apiGroups:
+ - batch
+ resources:
+ - jobs
+ verbs:
+ - create
+ - delete
+ - get
+ - list
+ - patch
+ - update
+ - watch
 - apiGroups:
  - ""
  resources:

controllers/cassandra/cassandradatacenter_controller.go

@@ -57,6 +57,7 @@ import (
 // +kubebuilder:rbac:groups=core,namespace=cass-operator,resources=pods;endpoints;services;configmaps;secrets;persistentvolumeclaims;events,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,namespace=cass-operator,resources=namespaces,verbs=get
 // +kubebuilder:rbac:groups=core,resources=persistentvolumes;nodes,verbs=get;list;watch
+// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=policy,namespace=cass-operator,resources=poddisruptionbudgets,verbs=get;list;watch;create;update;patch;delete
 
 // CassandraDatacenterReconciler reconciles a cassandraDatacenter object

pkg/reconciliation/reconcile_racks.go

@@ -12,6 +12,7 @@ import (
  "time"
 
  appsv1 "k8s.io/api/apps/v1"
+ batchv1 "k8s.io/api/batch/v1"
  corev1 "k8s.io/api/core/v1"
  policyv1 "k8s.io/api/policy/v1"
  "k8s.io/apimachinery/pkg/api/errors"
@@ -105,6 +106,14 @@ func (rc *ReconciliationContext) CalculateRackInformation() error {
 func (rc *ReconciliationContext) CheckSuperuserSecretCreation() result.ReconcileResult {
  rc.ReqLogger.Info("reconcile_racks::CheckSuperuserSecretCreation")
 
+ if rc.Datacenter.Spec.UserInfo != nil {
+ return result.Continue()
+ }
+
+ if rc.IsInitialized() {
+ return result.Continue()
+ }
+
  _, err := rc.retrieveSuperuserSecretOrCreateDefault()
  if err != nil {
  rc.ReqLogger.Error(err, "error retrieving SuperuserSecret for CassandraDatacenter.")
@@ -876,6 +885,10 @@ func (rc *ReconciliationContext) UpdateSecretWatches() error {
 func (rc *ReconciliationContext) CreateUsers() result.ReconcileResult {
  dc := rc.Datacenter
 
+ if rc.IsInitialized() {
+ return result.Continue()
+ }
+
  if val, found := dc.Annotations[api.SkipUserCreationAnnotation]; found && val == "true" {
  rc.ReqLogger.Info(api.SkipUserCreationAnnotation + " is set, skipping CreateUser")
  return result.Continue()
@@ -888,15 +901,117 @@ func (rc *ReconciliationContext) CreateUsers() result.ReconcileResult {
 
  rc.ReqLogger.Info("reconcile_racks::CreateUsers")
 
+ // TODO We should check if we've already created this ..
+ // TODO This should be cleaned up after a while (TTL)
+ if dc.Spec.UserInfo != nil {
+ // Create the job
+ ttl := int32(86400)
+
+ // TODO Instead of this, I should probably have a "SecretRef" generic structure. That way we could use it for all Secrets (like mgmt-api auth), supporting
+ // both legacy as well as new structure
+
+ // How to get this as input?
+ filePath := dc.Spec.UserInfo.MountPath
+
+ // We want to mount it as a directory and read the files as usernames
+ if dc.Spec.UserInfo.CSI != nil || dc.Spec.UserInfo.SecretName != "" {
+ filePath = "/mnt/secrets/users"
+ }
+
+ // TODO wait for it to complete before we continue..
+
+ job := batchv1.Job{
+ ObjectMeta: metav1.ObjectMeta{
+ Namespace: dc.Namespace,
+ Name: fmt.Sprintf("usercreate-%s", dc.Name),
+ },
+ Spec: batchv1.JobSpec{
+ Template: corev1.PodTemplateSpec{
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "client",
+ Image: "k8ssandra/k8ssandra-client:latest",
+ ImagePullPolicy: corev1.PullIfNotPresent,
+ Args: []string{"users", "add", "--path", filePath, "--dc", dc.Name},
+ },
+ },
+ RestartPolicy: corev1.RestartPolicyNever,
+ },
+ },
+ TTLSecondsAfterFinished: &ttl,
+ },
+ }
+
+ if len(dc.Spec.UserInfo.Annotations) > 0 {
+ job.Spec.Template.ObjectMeta.Annotations = dc.Spec.UserInfo.Annotations
+ }
+
+ // TODO Add verification that we can't have dual injection (CSI + annotations)
+ if dc.Spec.UserInfo.SecretName != "" || dc.Spec.UserInfo.CSI != nil {
+ vol := corev1.Volume{
+ Name: "user-source",
+ }
+ if dc.Spec.UserInfo.SecretName != "" {
+ vol.VolumeSource = corev1.VolumeSource{
+ Secret: &corev1.SecretVolumeSource{
+ SecretName: dc.Spec.UserInfo.SecretName,
+ },
+ }
+ }
+
+ if dc.Spec.UserInfo.CSI != nil {
+ vol.VolumeSource = corev1.VolumeSource{
+ CSI: dc.Spec.UserInfo.CSI,
+ }
+ }
+
+ job.Spec.Template.Spec.Volumes = []corev1.Volume{vol}
+ job.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{
+ {
+ Name: "user-source",
+ ReadOnly: true,
+ MountPath: filePath,
+ },
+ }
+ }
+
+ // labels := dc.GetDatacenterLabels()
+ // oplabels.AddOperatorLabels(labels, dc)
+
+ // Set CassandraDatacenter dc as the owner and controller
+ err := setControllerReference(dc, &job, rc.Scheme)
+ if err != nil {
+ return result.Error(err)
+ }
+
+ if dc.Spec.UserInfo.ServiceAccountName != "" {
+ job.Spec.Template.Spec.ServiceAccountName = dc.Spec.UserInfo.ServiceAccountName
+ } else {
+ job.Spec.Template.Spec.ServiceAccountName = dc.Spec.ServiceAccount
+ }
+
+ if err := rc.Client.Create(rc.Ctx, &job); err != nil {
+ if errors.IsAlreadyExists(err) {
+ return result.Continue()
+ }
+ return result.Error(err)
+ }
+ return result.RequeueSoon(5)
+ }
+
  err := rc.UpdateSecretWatches()
  if err != nil {
  rc.ReqLogger.Error(err, "Failed to update dynamic watches on secrets")
  }
 
  // make sure the default superuser secret exists
- _, err = rc.retrieveSuperuserSecretOrCreateDefault()
- if err != nil {
- rc.ReqLogger.Error(err, "Failed to verify superuser secret status")
+ // TODO Nope..
+ if dc.Spec.UserInfo == nil {
+ _, err = rc.retrieveSuperuserSecretOrCreateDefault()
+ if err != nil {
+ rc.ReqLogger.Error(err, "Failed to verify superuser secret status")
+ }
  }
 
  users := rc.GetUsers()