From 725f03e42dca8dc18787cbb5ac9c9594f1c369c3 Mon Sep 17 00:00:00 2001 From: Ahree Hong Date: Thu, 23 Mar 2023 01:51:16 -0700 Subject: [PATCH 29/34] add boot kernel settings for BR Signed-off-by: Ahree Hong --- .../kubeadm/api/v1beta1/kubeadm_types.go | 8 +++ .../api/v1beta1/zz_generated.deepcopy.go | 35 ++++++++++++ ...strap.cluster.x-k8s.io_kubeadmconfigs.yaml | 22 +++++++ ...uster.x-k8s.io_kubeadmconfigtemplates.yaml | 22 +++++++ .../internal/bottlerocket/bootstrap.go | 13 +++++ .../internal/bottlerocket/bootstrap_test.go | 57 +++++++++++++++++++ .../internal/bottlerocket/bottlerocket.go | 24 ++++++++ ...cluster.x-k8s.io_kubeadmcontrolplanes.yaml | 22 +++++++ ...x-k8s.io_kubeadmcontrolplanetemplates.yaml | 22 +++++++ .../cluster/cluster_controller_phases.go | 3 +- .../machine/machine_controller_phases.go | 5 +- 11 files changed, 230 insertions(+), 3 deletions(-) diff --git a/bootstrap/kubeadm/api/v1beta1/kubeadm_types.go b/bootstrap/kubeadm/api/v1beta1/kubeadm_types.go index c97ef8604..434d2c088 100644 --- a/bootstrap/kubeadm/api/v1beta1/kubeadm_types.go +++ b/bootstrap/kubeadm/api/v1beta1/kubeadm_types.go @@ -201,6 +201,9 @@ type BottlerocketSettings struct { // KernelSettings contains additional kernel settings for Bottlerocket. // +optional Kernel *BottlerocketKernelSettings `json:"kernel,omitempty"` + + // Boot holds the boot-related settings for bottlerocket nodes + Boot *BottlerocketBootSettings `json:"boot,omitempty"` } // BottlerocketKubernetesSettings holds the settings for kubernetes on bottlerocket nodes. @@ -222,6 +225,11 @@ type BottlerocketKernelSettings struct { SysctlSettings map[string]string `json:"sysctlSettings,omitempty"` } +// BottlerocketBootSettings holds the boot-related settings for bottlerocket nodes. +type BottlerocketBootSettings struct { + BootKernelParameters map[string][]string `json:"bootKernelParameters,omitempty"` +} + // Pause defines the pause image repo and tag that should be run on the bootstrapped nodes. // This setting is ONLY for bottlerocket nodes, as this needs to be set pre-boot time along with user-data type Pause struct { diff --git a/bootstrap/kubeadm/api/v1beta1/zz_generated.deepcopy.go b/bootstrap/kubeadm/api/v1beta1/zz_generated.deepcopy.go index 51c1ed776..306ba8931 100644 --- a/bootstrap/kubeadm/api/v1beta1/zz_generated.deepcopy.go +++ b/bootstrap/kubeadm/api/v1beta1/zz_generated.deepcopy.go @@ -159,6 +159,36 @@ func (in *BottlerocketAdmin) DeepCopy() *BottlerocketAdmin { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BottlerocketBootSettings) DeepCopyInto(out *BottlerocketBootSettings) { + *out = *in + if in.BootKernelParameters != nil { + in, out := &in.BootKernelParameters, &out.BootKernelParameters + *out = make(map[string][]string, len(*in)) + for key, val := range *in { + var outVal []string + if val == nil { + (*out)[key] = nil + } else { + in, out := &val, &outVal + *out = make([]string, len(*in)) + copy(*out, *in) + } + (*out)[key] = outVal + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BottlerocketBootSettings. +func (in *BottlerocketBootSettings) DeepCopy() *BottlerocketBootSettings { + if in == nil { + return nil + } + out := new(BottlerocketBootSettings) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BottlerocketBootstrap) DeepCopyInto(out *BottlerocketBootstrap) { *out = *in @@ -283,6 +313,11 @@ func (in *BottlerocketSettings) DeepCopyInto(out *BottlerocketSettings) { *out = new(BottlerocketKernelSettings) (*in).DeepCopyInto(*out) } + if in.Boot != nil { + in, out := &in.Boot, &out.Boot + *out = new(BottlerocketBootSettings) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BottlerocketSettings. diff --git a/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml b/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml index a7ae704ec..df745e7f2 100644 --- a/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml +++ b/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml @@ -2371,6 +2371,17 @@ spec: description: Bottlerocket holds configuration for certain bottlerocket settings. This is only for bottlerocket. properties: + boot: + description: Boot holds the boot-related settings for bottlerocket + nodes + properties: + bootKernelParameters: + additionalProperties: + items: + type: string + type: array + type: object + type: object kernel: description: KernelSettings contains additional kernel settings for Bottlerocket. @@ -3200,6 +3211,17 @@ spec: description: Bottlerocket holds configuration for certain bottlerocket settings. This is only for bottlerocket. properties: + boot: + description: Boot holds the boot-related settings for bottlerocket + nodes + properties: + bootKernelParameters: + additionalProperties: + items: + type: string + type: array + type: object + type: object kernel: description: KernelSettings contains additional kernel settings for Bottlerocket. diff --git a/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml b/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml index acf42f3ab..9c4514353 100644 --- a/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml +++ b/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml @@ -2402,6 +2402,17 @@ spec: description: Bottlerocket holds configuration for certain bottlerocket settings. This is only for bottlerocket. properties: + boot: + description: Boot holds the boot-related settings + for bottlerocket nodes + properties: + bootKernelParameters: + additionalProperties: + items: + type: string + type: array + type: object + type: object kernel: description: KernelSettings contains additional kernel settings for Bottlerocket. @@ -3288,6 +3299,17 @@ spec: description: Bottlerocket holds configuration for certain bottlerocket settings. This is only for bottlerocket. properties: + boot: + description: Boot holds the boot-related settings + for bottlerocket nodes + properties: + bootKernelParameters: + additionalProperties: + items: + type: string + type: array + type: object + type: object kernel: description: KernelSettings contains additional kernel settings for Bottlerocket. diff --git a/bootstrap/kubeadm/internal/bottlerocket/bootstrap.go b/bootstrap/kubeadm/internal/bottlerocket/bootstrap.go index a23a43668..901db1cb7 100644 --- a/bootstrap/kubeadm/internal/bottlerocket/bootstrap.go +++ b/bootstrap/kubeadm/internal/bottlerocket/bootstrap.go @@ -119,6 +119,15 @@ time-servers = [{{stringsJoin .NTPServers ", " }}] [settings.kernel.sysctl] {{.SysctlSettings}} {{- end -}} +` + + bootSettingsTemplate = `{{ define "bootSettings" -}} +[settings.boot] +reboot-to-reconcile = true + +[settings.boot.kernel-parameters] +{{.BootKernel}} +{{- end -}} ` bottlerocketNodeInitSettingsTemplate = `{{template "hostContainerSlice" .}} @@ -159,5 +168,9 @@ time-servers = [{{stringsJoin .NTPServers ", " }}] {{- if (ne .SysctlSettings "")}} {{template "sysctlSettingsTemplate" .}} {{- end -}} + +{{- if .BootKernel}} +{{template "bootSettings" .}} +{{- end -}} ` ) diff --git a/bootstrap/kubeadm/internal/bottlerocket/bootstrap_test.go b/bootstrap/kubeadm/internal/bottlerocket/bootstrap_test.go index fdc60204d..5846524be 100644 --- a/bootstrap/kubeadm/internal/bottlerocket/bootstrap_test.go +++ b/bootstrap/kubeadm/internal/bottlerocket/bootstrap_test.go @@ -279,6 +279,36 @@ hostname = "hostname" [settings.kernel.sysctl] "foo" = "bar" "abc" = "def" +` + + BootSettingsUserData = ` +[settings.host-containers.admin] +enabled = true +superpowered = true +source = "ADMIN_REPO:ADMIN_TAG" +user-data = "CnsKCSJzc2giOiB7CgkJImF1dGhvcml6ZWQta2V5cyI6IFsic3NoLXJzYSBBQUEuLi4iXQoJfQp9" +[settings.host-containers.kubeadm-bootstrap] +enabled = true +superpowered = true +source = "BOOTSTRAP_REPO:BOOTSTRAP_TAG" +user-data = "Qk9UVExFUk9DS0VUX0JPT1RTVFJBUF9VU0VSREFUQQ==" + +[settings.kubernetes] +cluster-domain = "cluster.local" +standalone-mode = true +authentication-mode = "tls" +server-tls-bootstrap = false +pod-infra-container-image = "PAUSE_REPO:PAUSE_TAG" +provider-id = "PROVIDERID" + +[settings.network] +hostname = "hostname" +[settings.boot] +reboot-to-reconcile = true + +[settings.boot.kernel-parameters] +"abc" = ["def","123"] +"foo" = ["bar"] ` ) @@ -513,10 +543,37 @@ func TestGetBottlerocketNodeUserData(t *testing.T) { }, output: kernelSettingsUserData, }, + { + name: "with boot settings", + config: &BottlerocketConfig{ + BottlerocketAdmin: brAdmin, + BottlerocketBootstrap: brBootstrap, + Hostname: hostname, + Pause: pause, + KubeletExtraArgs: map[string]string{ + "provider-id": "PROVIDERID", + }, + BottlerocketSettings: &bootstrapv1.BottlerocketSettings{ + Boot: &bootstrapv1.BottlerocketBootSettings{ + BootKernelParameters: map[string][]string{ + "abc": { + "def", + "123", + }, + "foo": { + "bar", + }, + }, + }, + }, + }, + output: BootSettingsUserData, + }, } for _, testcase := range testcases { t.Run(testcase.name, func(t *testing.T) { b, err := getBottlerocketNodeUserData(brBootstrapUserdata, users, testcase.config) + println(string(b)) g.Expect(err).NotTo(HaveOccurred()) g.Expect(string(b)).To(Equal(testcase.output)) }) diff --git a/bootstrap/kubeadm/internal/bottlerocket/bottlerocket.go b/bootstrap/kubeadm/internal/bottlerocket/bottlerocket.go index fb6a771f6..b588f943e 100644 --- a/bootstrap/kubeadm/internal/bottlerocket/bottlerocket.go +++ b/bootstrap/kubeadm/internal/bottlerocket/bottlerocket.go @@ -56,6 +56,7 @@ type BottlerocketSettingsInput struct { AllowedUnsafeSysctls []string ClusterDNSIPs []string MaxPods int + BootKernel string HostContainers []bootstrapv1.BottlerocketHostContainer BootstrapContainers []bootstrapv1.BottlerocketBootstrapContainer SysctlSettings string @@ -159,6 +160,9 @@ func generateNodeUserData(kind string, tpl string, data interface{}) ([]byte, er if _, err := tm.Parse(sysctlSettingsTemplate); err != nil { return nil, errors.Wrapf(err, "failed to parse sysctl settings %s template", kind) } + if _, err := tm.Parse(bootSettingsTemplate); err != nil { + return nil, errors.Wrapf(err, "failed to parse boot settings %s template", kind) + } t, err := tm.Parse(tpl) if err != nil { return nil, errors.Wrapf(err, "failed to parse %s template", kind) @@ -256,6 +260,10 @@ func getBottlerocketNodeUserData(bootstrapContainerUserData []byte, users []boot bottlerocketInput.SysctlSettings = parseSysctlSettings(config.BottlerocketSettings.Kernel.SysctlSettings) } + if config.BottlerocketSettings.Boot != nil { + bottlerocketInput.BootKernel = parseBootSettings(config.BottlerocketSettings.Boot.BootKernelParameters) + } + } return generateNodeUserData("InitBottlerocketNode", bottlerocketNodeInitSettingsTemplate, bottlerocketInput) @@ -314,6 +322,22 @@ func parseSysctlSettings(sysctlSettings map[string]string) string { return sysctlSettingsToml } +func parseBootSettings(bootSettings map[string][]string) string { + bootSettingsToml := "" + for key, value := range bootSettings { + var values []string + if len(value) != 0 { + for _, val := range value { + quotedVal := "\"" + val + "\"" + values = append(values, quotedVal) + } + } + keyVal := strings.Join(values, ",") + bootSettingsToml += fmt.Sprintf("\"%v\" = [%v]\n", key, keyVal) + } + return bootSettingsToml +} + // Parses through all the users and return list of all user's authorized ssh keys func getAllAuthorizedKeys(users []bootstrapv1.User) string { var sshAuthorizedKeys []string diff --git a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml index 2f1f65012..7b74847bc 100644 --- a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml +++ b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml @@ -2843,6 +2843,17 @@ spec: description: Bottlerocket holds configuration for certain bottlerocket settings. This is only for bottlerocket. properties: + boot: + description: Boot holds the boot-related settings for + bottlerocket nodes + properties: + bootKernelParameters: + additionalProperties: + items: + type: string + type: array + type: object + type: object kernel: description: KernelSettings contains additional kernel settings for Bottlerocket. @@ -3707,6 +3718,17 @@ spec: description: Bottlerocket holds configuration for certain bottlerocket settings. This is only for bottlerocket. properties: + boot: + description: Boot holds the boot-related settings for + bottlerocket nodes + properties: + bootKernelParameters: + additionalProperties: + items: + type: string + type: array + type: object + type: object kernel: description: KernelSettings contains additional kernel settings for Bottlerocket. diff --git a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml index 933f00038..a79e90dea 100644 --- a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml +++ b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml @@ -1472,6 +1472,17 @@ spec: certain bottlerocket settings. This is only for bottlerocket. properties: + boot: + description: Boot holds the boot-related settings + for bottlerocket nodes + properties: + bootKernelParameters: + additionalProperties: + items: + type: string + type: array + type: object + type: object kernel: description: KernelSettings contains additional kernel settings for Bottlerocket. @@ -2396,6 +2407,17 @@ spec: certain bottlerocket settings. This is only for bottlerocket. properties: + boot: + description: Boot holds the boot-related settings + for bottlerocket nodes + properties: + bootKernelParameters: + additionalProperties: + items: + type: string + type: array + type: object + type: object kernel: description: KernelSettings contains additional kernel settings for Bottlerocket. diff --git a/internal/controllers/cluster/cluster_controller_phases.go b/internal/controllers/cluster/cluster_controller_phases.go index 7f2a32379..e85875a83 100644 --- a/internal/controllers/cluster/cluster_controller_phases.go +++ b/internal/controllers/cluster/cluster_controller_phases.go @@ -19,9 +19,10 @@ package cluster import ( "context" "fmt" + "time" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "sigs.k8s.io/controller-runtime/pkg/client" - "time" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" diff --git a/internal/controllers/machine/machine_controller_phases.go b/internal/controllers/machine/machine_controller_phases.go index 5e846594a..9be6ddb34 100644 --- a/internal/controllers/machine/machine_controller_phases.go +++ b/internal/controllers/machine/machine_controller_phases.go @@ -19,9 +19,10 @@ package machine import ( "context" "fmt" - "sigs.k8s.io/controller-runtime/pkg/client" "time" + "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -372,7 +373,7 @@ func (r *Reconciler) reconcileInfrastructure(ctx context.Context, cluster *clust }, }, Data: map[string][]byte{ - "address": []byte(machineIP), + "address": []byte(machineIP), "clientUrls": []byte(fmt.Sprintf("https://%v:2379", machineIP)), }, Type: clusterv1.ClusterSecretType, -- 2.40.0