From b763eaab9c4f4ad2db555315d8a7b3155ec99119 Mon Sep 17 00:00:00 2001 From: Ahree Hong Date: Tue, 7 Mar 2023 14:01:39 -0800 Subject: [PATCH 28/34] add br kernel.sysctl settings Signed-off-by: Ahree Hong --- .../kubeadm/api/v1beta1/kubeadm_types.go | 10 ++++ .../api/v1beta1/zz_generated.deepcopy.go | 27 +++++++++++ ...strap.cluster.x-k8s.io_kubeadmconfigs.yaml | 22 +++++++++ ...uster.x-k8s.io_kubeadmconfigtemplates.yaml | 22 +++++++++ .../internal/bottlerocket/bootstrap.go | 10 ++++ .../internal/bottlerocket/bootstrap_test.go | 48 +++++++++++++++++++ .../internal/bottlerocket/bottlerocket.go | 32 ++++++++++--- .../api/v1alpha3/zz_generated.deepcopy.go | 2 +- .../api/v1alpha4/zz_generated.deepcopy.go | 2 +- ...cluster.x-k8s.io_kubeadmcontrolplanes.yaml | 22 +++++++++ ...x-k8s.io_kubeadmcontrolplanetemplates.yaml | 24 ++++++++++ 11 files changed, 212 insertions(+), 9 deletions(-) diff --git a/bootstrap/kubeadm/api/v1beta1/kubeadm_types.go b/bootstrap/kubeadm/api/v1beta1/kubeadm_types.go index b3814c089..c97ef8604 100644 --- a/bootstrap/kubeadm/api/v1beta1/kubeadm_types.go +++ b/bootstrap/kubeadm/api/v1beta1/kubeadm_types.go @@ -197,6 +197,10 @@ type ClusterConfiguration struct { type BottlerocketSettings struct { // Kubernetes holds the kubernetes settings for bottlerocket nodes. Kubernetes *BottlerocketKubernetesSettings `json:"kubernetes,omitempty"` + + // KernelSettings contains additional kernel settings for Bottlerocket. + // +optional + Kernel *BottlerocketKernelSettings `json:"kernel,omitempty"` } // BottlerocketKubernetesSettings holds the settings for kubernetes on bottlerocket nodes. @@ -212,6 +216,12 @@ type BottlerocketKubernetesSettings struct { ClusterDNSIPs []string `json:"clusterDNSIPs,omitempty"` } +// BottlerocketKernelSettings holds the kernel settings for bottlerocket nodes +type BottlerocketKernelSettings struct { + // SysctlSettings defines the kernel sysctl settings to set for bottlerocket nodes. + SysctlSettings map[string]string `json:"sysctlSettings,omitempty"` +} + // Pause defines the pause image repo and tag that should be run on the bootstrapped nodes. // This setting is ONLY for bottlerocket nodes, as this needs to be set pre-boot time along with user-data type Pause struct { diff --git a/bootstrap/kubeadm/api/v1beta1/zz_generated.deepcopy.go b/bootstrap/kubeadm/api/v1beta1/zz_generated.deepcopy.go index 400e2c343..51c1ed776 100644 --- a/bootstrap/kubeadm/api/v1beta1/zz_generated.deepcopy.go +++ b/bootstrap/kubeadm/api/v1beta1/zz_generated.deepcopy.go @@ -223,6 +223,28 @@ func (in *BottlerocketHostContainer) DeepCopy() *BottlerocketHostContainer { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BottlerocketKernelSettings) DeepCopyInto(out *BottlerocketKernelSettings) { + *out = *in + if in.SysctlSettings != nil { + in, out := &in.SysctlSettings, &out.SysctlSettings + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BottlerocketKernelSettings. +func (in *BottlerocketKernelSettings) DeepCopy() *BottlerocketKernelSettings { + if in == nil { + return nil + } + out := new(BottlerocketKernelSettings) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BottlerocketKubernetesSettings) DeepCopyInto(out *BottlerocketKubernetesSettings) { *out = *in @@ -256,6 +278,11 @@ func (in *BottlerocketSettings) DeepCopyInto(out *BottlerocketSettings) { *out = new(BottlerocketKubernetesSettings) (*in).DeepCopyInto(*out) } + if in.Kernel != nil { + in, out := &in.Kernel, &out.Kernel + *out = new(BottlerocketKernelSettings) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BottlerocketSettings. diff --git a/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml b/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml index c26623a66..a7ae704ec 100644 --- a/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml +++ b/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml @@ -2371,6 +2371,17 @@ spec: description: Bottlerocket holds configuration for certain bottlerocket settings. This is only for bottlerocket. properties: + kernel: + description: KernelSettings contains additional kernel settings + for Bottlerocket. + properties: + sysctlSettings: + additionalProperties: + type: string + description: SysctlSettings defines the kernel sysctl + settings to set for bottlerocket nodes. + type: object + type: object kubernetes: description: Kubernetes holds the kubernetes settings for bottlerocket nodes. @@ -3189,6 +3200,17 @@ spec: description: Bottlerocket holds configuration for certain bottlerocket settings. This is only for bottlerocket. properties: + kernel: + description: KernelSettings contains additional kernel settings + for Bottlerocket. + properties: + sysctlSettings: + additionalProperties: + type: string + description: SysctlSettings defines the kernel sysctl + settings to set for bottlerocket nodes. + type: object + type: object kubernetes: description: Kubernetes holds the kubernetes settings for bottlerocket nodes. diff --git a/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml b/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml index 40dda4464..acf42f3ab 100644 --- a/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml +++ b/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml @@ -2402,6 +2402,17 @@ spec: description: Bottlerocket holds configuration for certain bottlerocket settings. This is only for bottlerocket. properties: + kernel: + description: KernelSettings contains additional kernel + settings for Bottlerocket. + properties: + sysctlSettings: + additionalProperties: + type: string + description: SysctlSettings defines the kernel + sysctl settings to set for bottlerocket nodes. + type: object + type: object kubernetes: description: Kubernetes holds the kubernetes settings for bottlerocket nodes. @@ -3277,6 +3288,17 @@ spec: description: Bottlerocket holds configuration for certain bottlerocket settings. This is only for bottlerocket. properties: + kernel: + description: KernelSettings contains additional kernel + settings for Bottlerocket. + properties: + sysctlSettings: + additionalProperties: + type: string + description: SysctlSettings defines the kernel + sysctl settings to set for bottlerocket nodes. + type: object + type: object kubernetes: description: Kubernetes holds the kubernetes settings for bottlerocket nodes. diff --git a/bootstrap/kubeadm/internal/bottlerocket/bootstrap.go b/bootstrap/kubeadm/internal/bottlerocket/bootstrap.go index e5926de86..a23a43668 100644 --- a/bootstrap/kubeadm/internal/bottlerocket/bootstrap.go +++ b/bootstrap/kubeadm/internal/bottlerocket/bootstrap.go @@ -113,6 +113,12 @@ password = "{{.RegistryMirrorPassword}}" [settings.ntp] time-servers = [{{stringsJoin .NTPServers ", " }}] {{- end -}} +` + + sysctlSettingsTemplate = `{{ define "sysctlSettingsTemplate" -}} +[settings.kernel.sysctl] +{{.SysctlSettings}} +{{- end -}} ` bottlerocketNodeInitSettingsTemplate = `{{template "hostContainerSlice" .}} @@ -149,5 +155,9 @@ time-servers = [{{stringsJoin .NTPServers ", " }}] {{- if .NTPServers}} {{template "ntpSettings" .}} {{- end -}} + +{{- if (ne .SysctlSettings "")}} +{{template "sysctlSettingsTemplate" .}} +{{- end -}} ` ) diff --git a/bootstrap/kubeadm/internal/bottlerocket/bootstrap_test.go b/bootstrap/kubeadm/internal/bottlerocket/bootstrap_test.go index 3463a2769..fdc60204d 100644 --- a/bootstrap/kubeadm/internal/bottlerocket/bootstrap_test.go +++ b/bootstrap/kubeadm/internal/bottlerocket/bootstrap_test.go @@ -253,6 +253,33 @@ essential = false mode = "MODE" source = "BOOTSTRAP_REPO:BOOTSTRAP_TAG" user-data = "BOOTSTRAP_B6_4USERDATA"` + + kernelSettingsUserData = ` +[settings.host-containers.admin] +enabled = true +superpowered = true +source = "ADMIN_REPO:ADMIN_TAG" +user-data = "CnsKCSJzc2giOiB7CgkJImF1dGhvcml6ZWQta2V5cyI6IFsic3NoLXJzYSBBQUEuLi4iXQoJfQp9" +[settings.host-containers.kubeadm-bootstrap] +enabled = true +superpowered = true +source = "BOOTSTRAP_REPO:BOOTSTRAP_TAG" +user-data = "Qk9UVExFUk9DS0VUX0JPT1RTVFJBUF9VU0VSREFUQQ==" + +[settings.kubernetes] +cluster-domain = "cluster.local" +standalone-mode = true +authentication-mode = "tls" +server-tls-bootstrap = false +pod-infra-container-image = "PAUSE_REPO:PAUSE_TAG" +provider-id = "PROVIDERID" + +[settings.network] +hostname = "hostname" +[settings.kernel.sysctl] +"foo" = "bar" +"abc" = "def" +` ) var ( @@ -465,6 +492,27 @@ func TestGetBottlerocketNodeUserData(t *testing.T) { }, output: customBootstrapUserData, }, + { + name: "with kernel settings", + config: &BottlerocketConfig{ + BottlerocketAdmin: brAdmin, + BottlerocketBootstrap: brBootstrap, + Hostname: hostname, + Pause: pause, + KubeletExtraArgs: map[string]string{ + "provider-id": "PROVIDERID", + }, + BottlerocketSettings: &bootstrapv1.BottlerocketSettings{ + Kernel: &bootstrapv1.BottlerocketKernelSettings{ + SysctlSettings: map[string]string{ + "foo": "bar", + "abc": "def", + }, + }, + }, + }, + output: kernelSettingsUserData, + }, } for _, testcase := range testcases { t.Run(testcase.name, func(t *testing.T) { diff --git a/bootstrap/kubeadm/internal/bottlerocket/bottlerocket.go b/bootstrap/kubeadm/internal/bottlerocket/bottlerocket.go index f21247061..fb6a771f6 100644 --- a/bootstrap/kubeadm/internal/bottlerocket/bottlerocket.go +++ b/bootstrap/kubeadm/internal/bottlerocket/bottlerocket.go @@ -54,10 +54,11 @@ type BottlerocketSettingsInput struct { ProviderID string Hostname string AllowedUnsafeSysctls []string - ClusterDNSIPs []string + ClusterDNSIPs []string MaxPods int HostContainers []bootstrapv1.BottlerocketHostContainer BootstrapContainers []bootstrapv1.BottlerocketBootstrapContainer + SysctlSettings string } // HostPath holds the path and type of a host path volume. @@ -155,6 +156,9 @@ func generateNodeUserData(kind string, tpl string, data interface{}) ([]byte, er if _, err := tm.Parse(ntpTemplate); err != nil { return nil, errors.Wrapf(err, "failed to parse NTP %s template", kind) } + if _, err := tm.Parse(sysctlSettingsTemplate); err != nil { + return nil, errors.Wrapf(err, "failed to parse sysctl settings %s template", kind) + } t, err := tm.Parse(tpl) if err != nil { return nil, errors.Wrapf(err, "failed to parse %s template", kind) @@ -238,14 +242,20 @@ func getBottlerocketNodeUserData(bootstrapContainerUserData []byte, users []boot bottlerocketInput.NTPServers = append(bottlerocketInput.NTPServers, strconv.Quote(ntp)) } } - if config.BottlerocketSettings != nil && config.BottlerocketSettings.Kubernetes != nil { - bottlerocketInput.MaxPods = config.BottlerocketSettings.Kubernetes.MaxPods - for _, sysctl := range config.BottlerocketSettings.Kubernetes.AllowedUnsafeSysctls { - bottlerocketInput.AllowedUnsafeSysctls = append(bottlerocketInput.AllowedUnsafeSysctls, strconv.Quote(sysctl)) + if config.BottlerocketSettings != nil { + if config.BottlerocketSettings.Kubernetes != nil { + bottlerocketInput.MaxPods = config.BottlerocketSettings.Kubernetes.MaxPods + for _, sysctl := range config.BottlerocketSettings.Kubernetes.AllowedUnsafeSysctls { + bottlerocketInput.AllowedUnsafeSysctls = append(bottlerocketInput.AllowedUnsafeSysctls, strconv.Quote(sysctl)) + } + for _, ip := range config.BottlerocketSettings.Kubernetes.ClusterDNSIPs { + bottlerocketInput.ClusterDNSIPs = append(bottlerocketInput.ClusterDNSIPs, strconv.Quote(ip)) + } } - for _, ip := range config.BottlerocketSettings.Kubernetes.ClusterDNSIPs { - bottlerocketInput.ClusterDNSIPs = append(bottlerocketInput.ClusterDNSIPs, strconv.Quote(ip)) + if config.BottlerocketSettings.Kernel != nil { + bottlerocketInput.SysctlSettings = parseSysctlSettings(config.BottlerocketSettings.Kernel.SysctlSettings) } + } return generateNodeUserData("InitBottlerocketNode", bottlerocketNodeInitSettingsTemplate, bottlerocketInput) @@ -296,6 +306,14 @@ func parseNodeLabels(nodeLabels string) string { return nodeLabelsToml } +func parseSysctlSettings(sysctlSettings map[string]string) string { + sysctlSettingsToml := "" + for key, value := range sysctlSettings { + sysctlSettingsToml += fmt.Sprintf("\"%v\" = \"%v\"\n", key, value) + } + return sysctlSettingsToml +} + // Parses through all the users and return list of all user's authorized ssh keys func getAllAuthorizedKeys(users []bootstrapv1.User) string { var sshAuthorizedKeys []string diff --git a/controlplane/kubeadm/api/v1alpha3/zz_generated.deepcopy.go b/controlplane/kubeadm/api/v1alpha3/zz_generated.deepcopy.go index 485769c62..b7eb40eef 100644 --- a/controlplane/kubeadm/api/v1alpha3/zz_generated.deepcopy.go +++ b/controlplane/kubeadm/api/v1alpha3/zz_generated.deepcopy.go @@ -23,7 +23,7 @@ package v1alpha3 import ( "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" apiv1alpha3 "sigs.k8s.io/cluster-api/api/v1alpha3" ) diff --git a/controlplane/kubeadm/api/v1alpha4/zz_generated.deepcopy.go b/controlplane/kubeadm/api/v1alpha4/zz_generated.deepcopy.go index bcff12e91..8acd51a25 100644 --- a/controlplane/kubeadm/api/v1alpha4/zz_generated.deepcopy.go +++ b/controlplane/kubeadm/api/v1alpha4/zz_generated.deepcopy.go @@ -23,7 +23,7 @@ package v1alpha4 import ( "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" apiv1alpha4 "sigs.k8s.io/cluster-api/api/v1alpha4" ) diff --git a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml index 077d7493a..2f1f65012 100644 --- a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml +++ b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml @@ -2843,6 +2843,17 @@ spec: description: Bottlerocket holds configuration for certain bottlerocket settings. This is only for bottlerocket. properties: + kernel: + description: KernelSettings contains additional kernel + settings for Bottlerocket. + properties: + sysctlSettings: + additionalProperties: + type: string + description: SysctlSettings defines the kernel sysctl + settings to set for bottlerocket nodes. + type: object + type: object kubernetes: description: Kubernetes holds the kubernetes settings for bottlerocket nodes. @@ -3696,6 +3707,17 @@ spec: description: Bottlerocket holds configuration for certain bottlerocket settings. This is only for bottlerocket. properties: + kernel: + description: KernelSettings contains additional kernel + settings for Bottlerocket. + properties: + sysctlSettings: + additionalProperties: + type: string + description: SysctlSettings defines the kernel sysctl + settings to set for bottlerocket nodes. + type: object + type: object kubernetes: description: Kubernetes holds the kubernetes settings for bottlerocket nodes. diff --git a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml index 1302cc4e1..933f00038 100644 --- a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml +++ b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml @@ -1472,6 +1472,18 @@ spec: certain bottlerocket settings. This is only for bottlerocket. properties: + kernel: + description: KernelSettings contains additional + kernel settings for Bottlerocket. + properties: + sysctlSettings: + additionalProperties: + type: string + description: SysctlSettings defines the kernel + sysctl settings to set for bottlerocket + nodes. + type: object + type: object kubernetes: description: Kubernetes holds the kubernetes settings for bottlerocket nodes. @@ -2384,6 +2396,18 @@ spec: certain bottlerocket settings. This is only for bottlerocket. properties: + kernel: + description: KernelSettings contains additional + kernel settings for Bottlerocket. + properties: + sysctlSettings: + additionalProperties: + type: string + description: SysctlSettings defines the kernel + sysctl settings to set for bottlerocket + nodes. + type: object + type: object kubernetes: description: Kubernetes holds the kubernetes settings for bottlerocket nodes. -- 2.40.0