[main][forwardport] Add validation to Toleration and Affinitys Keys

docs.md

@@ -464,6 +464,21 @@ section. A secondary validator will ensure that the effective data directory for
 from the one chosen during cluster creation. Additionally, the changing of a data directory for the `system-agent`, 
 kubernetes distro (RKE2/K3s), and CAPR components is also prohibited.
 
+#### cluster.spec.clusterAgentDeploymentCustomization and cluster.spec.fleetAgentDeploymentCustomization
+
+The `DeploymentCustomization` fields are of 3 types:
+- `appendTolerations`: adds tolerations to the appropriate deployment (cluster-agent/fleet-agent)
+- `affinity`: adds various affinities to the deployments, which include the following
+ - `nodeAffinity`: where to schedule the workload
+ - `podAffinitity` and `podAntiAffinity`: pods to avoid or prefer when scheduling the workload
+
+A `Toleration` is matched to a regex which is provided by upstream [apimachinery here](https://github.com/kubernetes/apimachinery/blob/02a41040d88da08de6765573ae2b1a51f424e1ca/pkg/apis/meta/v1/validation/validation.go#L96) but it boils down to this regex on the label:
+```regex
+([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]
+```
+
+For the `Affinity` based rules, the `podAffinity`/`podAntiAffinity` are validated via label selectors via [this apimachinery function](https://github.com/kubernetes/apimachinery/blob/02a41040d88da08de6765573ae2b1a51f424e1ca/pkg/apis/meta/v1/validation/validation.go#L56) whereas the `nodeAffinity` `nodeSelectorTerms` are validated via the same `Toleration` function.
+
 ### Mutation Checks
 
 #### On Create

pkg/resources/provisioning.cattle.io/v1/cluster/Cluster.md

@@ -35,6 +35,21 @@ section. A secondary validator will ensure that the effective data directory for
 from the one chosen during cluster creation. Additionally, the changing of a data directory for the `system-agent`, 
 kubernetes distro (RKE2/K3s), and CAPR components is also prohibited.
 
+### cluster.spec.clusterAgentDeploymentCustomization and cluster.spec.fleetAgentDeploymentCustomization
+
+The `DeploymentCustomization` fields are of 3 types:
+- `appendTolerations`: adds tolerations to the appropriate deployment (cluster-agent/fleet-agent)
+- `affinity`: adds various affinities to the deployments, which include the following
+ - `nodeAffinity`: where to schedule the workload
+ - `podAffinitity` and `podAntiAffinity`: pods to avoid or prefer when scheduling the workload
+
+A `Toleration` is matched to a regex which is provided by upstream [apimachinery here](https://github.com/kubernetes/apimachinery/blob/02a41040d88da08de6765573ae2b1a51f424e1ca/pkg/apis/meta/v1/validation/validation.go#L96) but it boils down to this regex on the label:
+```regex
+([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]
+```
+
+For the `Affinity` based rules, the `podAffinity`/`podAntiAffinity` are validated via label selectors via [this apimachinery function](https://github.com/kubernetes/apimachinery/blob/02a41040d88da08de6765573ae2b1a51f424e1ca/pkg/apis/meta/v1/validation/validation.go#L56) whereas the `nodeAffinity` `nodeSelectorTerms` are validated via the same `Toleration` function.
+
 ## Mutation Checks
 
 ### On Create
@@ -51,3 +66,4 @@ Check for the presence of the `provisioning.cattle.io/allow-dynamic-schema-drop`
 perform no mutations. If the value is not present or not `"true"`, compare the value of the `dynamicSchemaSpec` field
 for each `machinePool`, to its' previous value. If the values are not identical, revert the value for the
 `dynamicSchemaSpec` for the specific `machinePool`, but do not reject the request.
+

pkg/resources/provisioning.cattle.io/v1/cluster/validator.go

@@ -23,16 +23,20 @@ import (
  admissionv1 "k8s.io/api/admission/v1"
  admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
  authv1 "k8s.io/api/authorization/v1"
+ k8sv1 "k8s.io/api/core/v1"
  apierrors "k8s.io/apimachinery/pkg/api/errors"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/apis/meta/v1/validation"
  "k8s.io/apimachinery/pkg/runtime/schema"
+ "k8s.io/apimachinery/pkg/util/validation/field"
  authorizationv1 "k8s.io/client-go/kubernetes/typed/authorization/v1"
  "k8s.io/utils/trace"
 )
 
 const (
  globalNamespace = "cattle-global-data"
  systemAgentVarDirEnvVar = "CATTLE_AGENT_VAR_DIR"
+ failureStatus = "Failure"
 )
 
 var (
@@ -111,6 +115,16 @@ func (p *provisioningAdmitter) Admit(request *admission.Request) (*admissionv1.A
  return response, nil
  }
 
+ if response.Result = errorListToStatus(validateAgentDeploymentCustomization(cluster.Spec.ClusterAgentDeploymentCustomization,
+ field.NewPath("spec", "clusterAgentDeploymentCustomization"))); response.Result != nil {
+ return response, nil
+ }
+
+ if response.Result = errorListToStatus(validateAgentDeploymentCustomization(cluster.Spec.FleetAgentDeploymentCustomization,
+ field.NewPath("spec", "fleetAgentDeploymentCustomization"))); response.Result != nil {
+ return response, nil
+ }
+
  if err := p.validateCloudCredentialAccess(request, response, oldCluster, cluster); err != nil || response.Result != nil {
  return response, err
  }
@@ -334,7 +348,7 @@ func (p *provisioningAdmitter) validateCloudCredentialAccess(request *admission.
  }
 
  response.Result = &metav1.Status{
- Status: "Failure",
+ Status: failureStatus,
  Message: resp.Status.Reason,
  Reason: metav1.StatusReasonUnauthorized,
  Code: http.StatusUnauthorized,
@@ -366,7 +380,7 @@ func (p *provisioningAdmitter) validateClusterName(request *admission.Request, r
  }
  if !isValidName(cluster.Name, cluster.Namespace, err == nil) {
  response.Result = &metav1.Status{
- Status: "Failure",
+ Status: failureStatus,
  Message: "cluster name must be 63 characters or fewer, must not begin with a hyphen, cannot be \"local\" nor of the form \"c-xxxxx\", and can only contain lowercase alphanumeric characters or ' - '",
  Reason: metav1.StatusReasonInvalid,
  Code: http.StatusUnprocessableEntity,
@@ -388,7 +402,7 @@ func (p *provisioningAdmitter) validateMachinePoolNames(request *admission.Reque
  for _, pool := range cluster.Spec.RKEConfig.MachinePools {
  if len(pool.Name) > 63 {
  response.Result = &metav1.Status{
- Status: "Failure",
+ Status: failureStatus,
  Message: "pool name must be 63 characters or fewer",
  Reason: metav1.StatusReasonInvalid,
  Code: http.StatusUnprocessableEntity,
@@ -449,7 +463,7 @@ func (p *provisioningAdmitter) validatePSACT(request *admission.Request, respons
  }
  if parsedRangeLessThan123(parsedVersion) {
  response.Result = &metav1.Status{
- Status: "Failure",
+ Status: failureStatus,
  Message: "PodSecurityAdmissionConfigurationTemplate(PSACT) is only supported in k8s version 1.23 and above",
  Reason: metav1.StatusReasonBadRequest,
  Code: http.StatusBadRequest,
@@ -461,7 +475,7 @@ func (p *provisioningAdmitter) validatePSACT(request *admission.Request, respons
  if _, err := p.psactCache.Get(templateName); err != nil {
  if apierrors.IsNotFound(err) {
  response.Result = &metav1.Status{
- Status: "Failure",
+ Status: failureStatus,
  Message: err.Error(),
  Reason: metav1.StatusReasonBadRequest,
  Code: http.StatusBadRequest,
@@ -490,10 +504,155 @@ func (p *provisioningAdmitter) validatePSACT(request *admission.Request, respons
  return nil
 }
 
+func validateAgentDeploymentCustomization(customization *v1.AgentDeploymentCustomization, path *field.Path) field.ErrorList {
+ if customization == nil {
+ return nil
+ }
+ var errList field.ErrorList
+
+ errList = append(errList, validateAppendToleration(customization.AppendTolerations, path.Child("appendTolerations"))...)
+ errList = append(errList, validateAffinity(customization.OverrideAffinity, path.Child("overrideAffinity"))...)
+
+ return errList
+}
+func validateAffinity(overrideAffinity *k8sv1.Affinity, path *field.Path) field.ErrorList {
+ if overrideAffinity == nil {
+ return nil
+ }
+ var errList field.ErrorList
+
+ if affinity := overrideAffinity.NodeAffinity; affinity != nil {
+ errList = append(errList,
+ validatePreferredSchedulingTerms(affinity.PreferredDuringSchedulingIgnoredDuringExecution,
+ path.Child("nodeAffinity").Child("preferredDuringSchedulingIgnoredDuringExecution"))...,
+ )
+ errList = append(errList,
+ validateNodeSelector(affinity.RequiredDuringSchedulingIgnoredDuringExecution,
+ path.Child("nodeAffinity").Child("requiredDuringSchedulingIgnoredDuringExecution"))...,
+ )
+ }
+
+ if podAffinity := overrideAffinity.PodAffinity; podAffinity != nil {
+ errList = append(errList, validatePodAffinityTerms(podAffinity.RequiredDuringSchedulingIgnoredDuringExecution,
+ path.Child("podAffinity").Child("requiredDuringSchedulingIgnoredDuringExecution"))...)
+
+ errList = append(errList, validateWeightedPodAffinityTerms(podAffinity.PreferredDuringSchedulingIgnoredDuringExecution,
+ path.Child("podAffinity").Child("preferredDuringSchedulingIgnoredDuringExecution"))...)
+ }
+
+ if podAntiAffinity := overrideAffinity.PodAntiAffinity; podAntiAffinity != nil {
+ errList = append(errList, validatePodAffinityTerms(podAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution,
+ path.Child("podAntiAffinity").Child("requiredDuringSchedulingIgnoredDuringExecution"))...)
+
+ errList = append(errList, validateWeightedPodAffinityTerms(podAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution,
+ path.Child("podAntiAffinity").Child("preferredDuringSchedulingIgnoredDuringExecution"))...)
+
+ }
+ return errList
+}
+
+func validatePodAffinityTerms(terms []k8sv1.PodAffinityTerm, path *field.Path) field.ErrorList {
+ var errList field.ErrorList
+
+ for k, v := range terms {
+ errList = append(errList, validatePodAffinityTerm(v, path.Index(k))...)
+ }
+ return errList
+}
+
+func validateWeightedPodAffinityTerms(weightedPodAffinityTerm []k8sv1.WeightedPodAffinityTerm, path *field.Path) field.ErrorList {
+ var errList field.ErrorList
+ for k, v := range weightedPodAffinityTerm {
+ errList = append(errList, validatePodAffinityTerm(v.PodAffinityTerm, path.Index(k).Child("podAffinityTerm"))...)
+ }
+ return errList
+}
+
+func validatePodAffinityTerm(podAffinityTerm k8sv1.PodAffinityTerm, path *field.Path) field.ErrorList {
+ var errList field.ErrorList
+ errList = append(errList, validateLabelSelector(podAffinityTerm.LabelSelector, path.Child("labelSelector"))...)
+ errList = append(errList, validateLabelSelector(podAffinityTerm.NamespaceSelector, path.Child("namespaceSelector"))...)
+ return errList
+}
+
+func validateLabelSelector(labelSelector *metav1.LabelSelector, path *field.Path) field.ErrorList {
+ return validation.ValidateLabelSelector(labelSelector, validation.LabelSelectorValidationOptions{}, path)
+
+}
+
+func validatePreferredSchedulingTerms(schedulingTerms []k8sv1.PreferredSchedulingTerm, path *field.Path) field.ErrorList {
+ var errList field.ErrorList
+
+ for k, v := range schedulingTerms {
+ errList = append(errList, validateNodeSelectorTerm(v.Preference, path.Index(k).Child("preferences"))...)
+ }
+ return errList
+}
+
+func validateNodeSelector(nodeSelector *k8sv1.NodeSelector, path *field.Path) field.ErrorList {
+ if nodeSelector == nil {
+ return nil
+ }
+ var errList field.ErrorList
+ nodeSelectorPath := path.Child("nodeSelectorTerms")
+ for k, v := range nodeSelector.NodeSelectorTerms {
+ errList = append(errList, validateNodeSelectorTerm(v, nodeSelectorPath.Index(k))...)
+ }
+ return errList
+}
+
+func validateNodeSelectorTerm(term k8sv1.NodeSelectorTerm, path *field.Path) field.ErrorList {
+ var errList field.ErrorList
+ errList = append(errList, validateNodeSelectorRequirements(term.MatchFields, path.Child("matchFields"))...)
+ errList = append(errList, validateNodeSelectorRequirements(term.MatchExpressions, path.Child("matchExpressions"))...)
+ return errList
+}
+
+// validateNodeSelectorRequirements Validates the NodeSelectors
+// at the moment it only validates the key by calling validation.ValidateLabelName.
+func validateNodeSelectorRequirements(selector []k8sv1.NodeSelectorRequirement, path *field.Path) field.ErrorList {
+ var errList field.ErrorList
+ for k, s := range selector {
+ errList = append(errList, validation.ValidateLabelName(s.Key, path.Index(k).Child("key"))...)
+ }
+ return errList
+}
+
+// validateAppendToleration validate if tolerations follows the k8s standards
+// at the moment it only validates the key by calling validation.ValidateLabelName.
+func validateAppendToleration(toleration []k8sv1.Toleration, path *field.Path) field.ErrorList {
+ var errList field.ErrorList
+ for k, s := range toleration {
+ errList = append(errList, validation.ValidateLabelName(s.Key, path.Index(k))...)
+ }
+ return errList
+}
+
+// errorListToStatus convert an errorList to failure status, it breaks a line for each entry and adds a * in front
+func errorListToStatus(errList field.ErrorList) *metav1.Status {
+ if len(errList) == 0 {
+ return nil
+ }
+ var builder strings.Builder
+ builder.WriteString("* ")
+ for i, fieldErr := range errList {
+ builder.WriteString(fieldErr.Error())
+ if i != len(errList)-1 {
+ builder.WriteString("\n* ")
+ }
+ }
+ return &metav1.Status{
+ Status: failureStatus,
+ Message: builder.String(),
+ Reason: metav1.StatusReasonInvalid,
+ Code: http.StatusUnprocessableEntity,
+ }
+}
+
 func validateACEConfig(cluster *v1.Cluster) *metav1.Status {
  if cluster.Spec.RKEConfig != nil && cluster.Spec.LocalClusterAuthEndpoint.Enabled && cluster.Spec.LocalClusterAuthEndpoint.CACerts != "" && cluster.Spec.LocalClusterAuthEndpoint.FQDN == "" {
  return &metav1.Status{
- Status: "Failure",
+ Status: failureStatus,
  Message: "CACerts defined but FQDN is not defined",
  Reason: metav1.StatusReasonInvalid,
  Code: http.StatusUnprocessableEntity,