From f3b44f5fe0276fb4d11c0530ae67b53e343fd615 Mon Sep 17 00:00:00 2001 From: Daniel Budris Date: Fri, 17 Dec 2021 13:38:39 -0800 Subject: [PATCH 13/34] Support worker node taints seperate taints template into its own template add parse taints method for converting taints config to toml add taints to BottlerocketSettingsInput add template parsing to node userdata generation account for multiple value:effect mappings in each taint key --- .../internal/bottlerocket/bootstrap.go | 11 ++++++ .../internal/bottlerocket/bottlerocket.go | 37 +++++++++++++++++++ .../controllers/kubeadmconfig_controller.go | 3 ++ 3 files changed, 51 insertions(+) diff --git a/bootstrap/kubeadm/internal/bottlerocket/bootstrap.go b/bootstrap/kubeadm/internal/bottlerocket/bootstrap.go index 3ebb9240a..dd769a78d 100644 --- a/bootstrap/kubeadm/internal/bottlerocket/bootstrap.go +++ b/bootstrap/kubeadm/internal/bottlerocket/bootstrap.go @@ -18,6 +18,7 @@ server-tls-bootstrap = false pod-infra-container-image = "{{.PauseContainerSource}}" {{- end -}} ` + bootstrapHostContainerTemplate = `{{define "bootstrapHostContainerSettings" -}} [settings.host-containers.kubeadm-bootstrap] enabled = true @@ -48,6 +49,12 @@ trusted=true {{.NodeLabels}} {{- end -}} ` + taintsTemplate = `{{ define "taintsTemplate" -}} +[settings.kubernetes.node-taints] +{{.Taints}} +{{- end -}} +` + bottlerocketNodeInitSettingsTemplate = `{{template "bootstrapHostContainerSettings" .}} {{template "adminContainerInitSettings" .}} @@ -69,5 +76,9 @@ trusted=true {{- if (ne .NodeLabels "")}} {{template "nodeLabelSettings" .}} {{- end -}} + +{{- if (ne .Taints "")}} +{{template "taintsTemplate" .}} +{{- end -}} ` ) diff --git a/bootstrap/kubeadm/internal/bottlerocket/bottlerocket.go b/bootstrap/kubeadm/internal/bottlerocket/bottlerocket.go index 0ba4318c5..36c3f67a7 100644 --- a/bootstrap/kubeadm/internal/bottlerocket/bottlerocket.go +++ b/bootstrap/kubeadm/internal/bottlerocket/bottlerocket.go @@ -8,6 +8,7 @@ import ( "strings" "text/template" + corev1 "k8s.io/api/core/v1" bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" "github.com/pkg/errors" @@ -27,6 +28,7 @@ type BottlerocketConfig struct { ProxyConfiguration bootstrapv1.ProxyConfiguration RegistryMirrorConfiguration bootstrapv1.RegistryMirrorConfiguration KubeletExtraArgs map[string]string + Taints []corev1.Taint } type BottlerocketSettingsInput struct { @@ -39,6 +41,7 @@ type BottlerocketSettingsInput struct { RegistryMirrorEndpoint string RegistryMirrorCACert string NodeLabels string + Taints string } type HostPath struct { @@ -104,6 +107,9 @@ func generateNodeUserData(kind string, tpl string, data interface{}) ([]byte, er if _, err := tm.Parse(nodeLabelsTemplate); err != nil { return nil, errors.Wrapf(err, "failed to parse node labels %s template", kind) } + if _, err := tm.Parse(taintsTemplate); err != nil { + return nil, errors.Wrapf(err, "failed to parse taints %s template", kind) + } t, err := tm.Parse(tpl) if err != nil { return nil, errors.Wrapf(err, "failed to parse %s template", kind) @@ -139,6 +145,7 @@ func getBottlerocketNodeUserData(bootstrapContainerUserData []byte, users []boot HTTPSProxyEndpoint: config.ProxyConfiguration.HTTPSProxy, RegistryMirrorEndpoint: config.RegistryMirrorConfiguration.Endpoint, NodeLabels: parseNodeLabels(config.KubeletExtraArgs["node-labels"]), // empty string if it does not exist + Taints: parseTaints(config.Taints), //empty string if it does not exist } if len(config.ProxyConfiguration.NoProxy) > 0 { for _, noProxy := range config.ProxyConfiguration.NoProxy { @@ -156,6 +163,36 @@ func getBottlerocketNodeUserData(bootstrapContainerUserData []byte, users []boot return bottlerocketNodeUserData, nil } +// bottlerocket configuration accepts taints in the format +// "key" = ["value:Effect", "value2:Effect2"] +func parseTaints(taints []corev1.Taint) string { + if len(taints) == 0 { + return "" + } + taintValueEffectTemplate := "\"%v:%v\"" + taintsMap := make(map[string][]string) + for _, taint := range taints { + valueEffectString := fmt.Sprintf(taintValueEffectTemplate, taint.Value, taint.Effect) + taintsMap[taint.Key]= append(taintsMap[taint.Key], valueEffectString) + } + + var taintsToml strings.Builder + for k, v := range taintsMap { + // write the taint key and opening bracket: '"key" = [' + taintKey := fmt.Sprintf("\"%v\" = [", k) + taintsToml.WriteString(taintKey) + + // write the value:effect mappings: '"value1:Effect1", "value2:Effect2"' + taintValueEffectMappings := strings.Join(v, ",") + taintsToml.WriteString(taintValueEffectMappings) + + // close the brackets and go to a new line + taintsToml.WriteString("]") + taintsToml.WriteString("\n") + } + return taintsToml.String() +} + func parseNodeLabels(nodeLabels string) string { if nodeLabels == "" { return "" diff --git a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go index 8a8c04c92..4b08e24c8 100644 --- a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go +++ b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go @@ -675,6 +675,9 @@ func (r *KubeadmConfigReconciler) joinWorker(ctx context.Context, scope *Scope) if scope.Config.Spec.JoinConfiguration.NodeRegistration.KubeletExtraArgs != nil { bottlerocketConfig.KubeletExtraArgs = scope.Config.Spec.JoinConfiguration.NodeRegistration.KubeletExtraArgs } + if len(scope.Config.Spec.JoinConfiguration.NodeRegistration.Taints) > 0 { + bottlerocketConfig.Taints = scope.Config.Spec.JoinConfiguration.NodeRegistration.Taints + } bootstrapJoinData, err = bottlerocket.NewNode(nodeInput, bottlerocketConfig) if err != nil { scope.Error(err, "Failed to create a worker bottlerocket join configuration") -- 2.40.0