From 29d3cbd905be4c23924c265dcd1320370d625f85 Mon Sep 17 00:00:00 2001 From: Ahree Hong Date: Wed, 14 Dec 2022 12:47:42 -0800 Subject: [PATCH 24/34] add support for registry credentials Signed-off-by: Ahree Hong --- .../internal/bottlerocket/bootstrap.go | 18 +++++++ .../internal/bottlerocket/bottlerocket.go | 16 ++++++ .../controllers/kubeadmconfig_controller.go | 49 +++++++++++++++++++ .../types/upstreamv1beta1/conversion.go | 2 +- .../types/upstreamv1beta3/conversion.go | 2 +- util/secret/consts.go | 3 ++ 6 files changed, 88 insertions(+), 2 deletions(-) diff --git a/bootstrap/kubeadm/internal/bottlerocket/bootstrap.go b/bootstrap/kubeadm/internal/bottlerocket/bootstrap.go index fe280e091..58f38ca34 100644 --- a/bootstrap/kubeadm/internal/bottlerocket/bootstrap.go +++ b/bootstrap/kubeadm/internal/bottlerocket/bootstrap.go @@ -72,6 +72,20 @@ no-proxy = [{{stringsJoin .NoProxyEndpoints "," }}] data = "{{.RegistryMirrorCACert}}" trusted=true {{- end -}} +` + // We need to assign creds for "public.ecr.aws" because host-ctr expects credentials to be assigned + // to "public.ecr.aws" rather than the mirror's endpoint + // TODO: Once the bottlerocket fixes are in we need to remove the "public.ecr.aws" creds + registryMirrorCredentialsTemplate = `{{define "registryMirrorCredentialsSettings" -}} +[[settings.container-registry.credentials]] +registry = "public.ecr.aws" +username = "{{.RegistryMirrorUsername}}" +password = "{{.RegistryMirrorPassword}}" +[[settings.container-registry.credentials]] +registry = "{{.RegistryMirrorEndpoint}}" +username = "{{.RegistryMirrorUsername}}" +password = "{{.RegistryMirrorPassword}}" +{{- end -}} ` nodeLabelsTemplate = `{{ define "nodeLabelSettings" -}} [settings.kubernetes.node-labels] @@ -104,6 +118,10 @@ trusted=true {{template "registryMirrorCACertSettings" .}} {{- end -}} +{{- if and (ne .RegistryMirrorUsername "") (ne .RegistryMirrorPassword "")}} +{{template "registryMirrorCredentialsSettings" .}} +{{- end -}} + {{- if (ne .NodeLabels "")}} {{template "nodeLabelSettings" .}} {{- end -}} diff --git a/bootstrap/kubeadm/internal/bottlerocket/bottlerocket.go b/bootstrap/kubeadm/internal/bottlerocket/bottlerocket.go index 1859a4816..c37713406 100644 --- a/bootstrap/kubeadm/internal/bottlerocket/bottlerocket.go +++ b/bootstrap/kubeadm/internal/bottlerocket/bottlerocket.go @@ -33,6 +33,7 @@ type BottlerocketConfig struct { Taints []corev1.Taint BottlerocketCustomHostContainers []bootstrapv1.BottlerocketHostContainer BottlerocketCustomBootstrapContainers []bootstrapv1.BottlerocketBootstrapContainer + RegistryMirrorCredentials } type BottlerocketSettingsInput struct { @@ -41,6 +42,8 @@ type BottlerocketSettingsInput struct { NoProxyEndpoints []string RegistryMirrorEndpoint string RegistryMirrorCACert string + RegistryMirrorUsername string + RegistryMirrorPassword string NodeLabels string Taints string ProviderId string @@ -53,6 +56,11 @@ type HostPath struct { Type string } +type RegistryMirrorCredentials struct { + Username string + Password string +} + func generateBootstrapContainerUserData(kind string, tpl string, data interface{}) ([]byte, error) { tm := template.New(kind).Funcs(defaultTemplateFuncMap) if _, err := tm.Parse(filesTemplate); err != nil { @@ -124,6 +132,9 @@ func generateNodeUserData(kind string, tpl string, data interface{}) ([]byte, er if _, err := tm.Parse(registryMirrorCACertTemplate); err != nil { return nil, errors.Wrapf(err, "failed to parse registry mirror ca cert %s template", kind) } + if _, err := tm.Parse(registryMirrorCredentialsTemplate); err != nil { + return nil, errors.Wrapf(err, "failed to parse registry mirror credentials %s template", kind) + } if _, err := tm.Parse(nodeLabelsTemplate); err != nil { return nil, errors.Wrapf(err, "failed to parse node labels %s template", kind) } @@ -204,6 +215,11 @@ func getBottlerocketNodeUserData(bootstrapContainerUserData []byte, users []boot bottlerocketInput.RegistryMirrorCACert = base64.StdEncoding.EncodeToString([]byte(config.RegistryMirrorConfiguration.CACert)) } + if config.RegistryMirrorCredentials.Username != "" && config.RegistryMirrorCredentials.Password != "" { + bottlerocketInput.RegistryMirrorUsername = config.RegistryMirrorCredentials.Username + bottlerocketInput.RegistryMirrorPassword = config.RegistryMirrorCredentials.Password + } + bottlerocketNodeUserData, err := generateNodeUserData("InitBottlerocketNode", bottlerocketNodeInitSettingsTemplate, bottlerocketInput) if err != nil { return nil, err diff --git a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go index 42313cfb9..afaab8a16 100644 --- a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go +++ b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go @@ -70,6 +70,11 @@ const ( DefaultTokenTTL = 15 * time.Minute ) +const ( + RegistryUsername = "username" + RegistryPassword = "password" +) + // InitLocker is a lock that is used around kubeadm init. type InitLocker interface { Lock(ctx context.Context, cluster *clusterv1.Cluster, machine *clusterv1.Machine) bool @@ -477,6 +482,13 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex } if scope.Config.Spec.ClusterConfiguration.RegistryMirror.Endpoint != "" { bottlerocketConfig.RegistryMirrorConfiguration = scope.Config.Spec.ClusterConfiguration.RegistryMirror + registryUsername, registryPassword, err := r.resolveRegistryCredentials(ctx, scope.Config) + if err != nil { + scope.Info("Cannot find secret for registry credentials, proceeding without registry credentials") + } else { + bottlerocketConfig.RegistryMirrorCredentials.Username = string(registryUsername) + bottlerocketConfig.RegistryMirrorCredentials.Password = string(registryPassword) + } } if scope.Config.Spec.InitConfiguration.NodeRegistration.KubeletExtraArgs != nil { bottlerocketConfig.KubeletExtraArgs = scope.Config.Spec.InitConfiguration.NodeRegistration.KubeletExtraArgs @@ -484,6 +496,7 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex if len(scope.Config.Spec.InitConfiguration.NodeRegistration.Taints) > 0 { bottlerocketConfig.Taints = scope.Config.Spec.InitConfiguration.NodeRegistration.Taints } + } clusterdata, err := kubeadmtypes.MarshalClusterConfigurationForVersion(scope.Config.Spec.ClusterConfiguration, parsedVersion) @@ -682,6 +695,13 @@ func (r *KubeadmConfigReconciler) joinWorker(ctx context.Context, scope *Scope) } if scope.Config.Spec.JoinConfiguration.RegistryMirror.Endpoint != "" { bottlerocketConfig.RegistryMirrorConfiguration = scope.Config.Spec.JoinConfiguration.RegistryMirror + registryUsername, registryPassword, err := r.resolveRegistryCredentials(ctx, scope.Config) + if err != nil { + scope.Info("Cannot find secret for registry credentials, proceeding without registry credentials”") + } else { + bottlerocketConfig.RegistryMirrorCredentials.Username = string(registryUsername) + bottlerocketConfig.RegistryMirrorCredentials.Password = string(registryPassword) + } } if scope.Config.Spec.JoinConfiguration.NodeRegistration.KubeletExtraArgs != nil { bottlerocketConfig.KubeletExtraArgs = scope.Config.Spec.JoinConfiguration.NodeRegistration.KubeletExtraArgs @@ -811,6 +831,13 @@ func (r *KubeadmConfigReconciler) joinControlplane(ctx context.Context, scope *S } if scope.Config.Spec.ClusterConfiguration.RegistryMirror.Endpoint != "" { bottlerocketConfig.RegistryMirrorConfiguration = scope.Config.Spec.ClusterConfiguration.RegistryMirror + registryUsername, registryPassword, err := r.resolveRegistryCredentials(ctx, scope.Config) + if err != nil { + scope.Info("Cannot find secret for registry credentials, proceeding without registry credentials”") + } else { + bottlerocketConfig.RegistryMirrorCredentials.Username = string(registryUsername) + bottlerocketConfig.RegistryMirrorCredentials.Password = string(registryPassword) + } } if scope.Config.Spec.JoinConfiguration.NodeRegistration.KubeletExtraArgs != nil { bottlerocketConfig.KubeletExtraArgs = scope.Config.Spec.JoinConfiguration.NodeRegistration.KubeletExtraArgs @@ -917,6 +944,28 @@ func (r *KubeadmConfigReconciler) resolveSecretPasswordContent(ctx context.Conte return data, nil } +// resolveRegistryCredentials returns username and password fetched from a secret object. +func (r *KubeadmConfigReconciler) resolveRegistryCredentials(ctx context.Context, source *bootstrapv1.KubeadmConfig) ([]byte, []byte, error) { + key := types.NamespacedName{Namespace: source.Namespace, Name: secret.RegistryCredentials} + secret := &corev1.Secret{} + if err := r.Client.Get(ctx, key, secret); err != nil { + if apierrors.IsNotFound(err) { + return nil, nil, errors.Wrapf(err, "secret not found: %s", key) + } + return nil, nil, errors.Wrapf(err, "failed to retrieve Secret %q", key) + } + username, ok := secret.Data[RegistryUsername] + if !ok { + return nil, nil, errors.Errorf("secret references non-existent secret key: %q", "username") + } + password, ok := secret.Data[RegistryPassword] + if !ok { + return nil, nil, errors.Errorf("secret references non-existent secret key: %q", "password") + } + return username, password, nil +} + + // ClusterToKubeadmConfigs is a handler.ToRequestsFunc to be used to enqueue // requests for reconciliation of KubeadmConfigs. func (r *KubeadmConfigReconciler) ClusterToKubeadmConfigs(o client.Object) []ctrl.Request { diff --git a/bootstrap/kubeadm/types/upstreamv1beta1/conversion.go b/bootstrap/kubeadm/types/upstreamv1beta1/conversion.go index 6cfa9628b..2be0950c7 100644 --- a/bootstrap/kubeadm/types/upstreamv1beta1/conversion.go +++ b/bootstrap/kubeadm/types/upstreamv1beta1/conversion.go @@ -99,4 +99,4 @@ func Convert_v1beta1_JoinConfiguration_To_upstreamv1beta1_JoinConfiguration(in * func Convert_v1beta1_ClusterConfiguration_To_upstreamv1beta1_ClusterConfiguration(src *bootstrapv1.ClusterConfiguration, dst *ClusterConfiguration, s apimachineryconversion.Scope) error { // ClusterConfiguration.BottlerocketCustomHostContainers exists in bootstrapv1.ClusterConfiguration but not in upstreamv1beta1 return autoConvert_v1beta1_ClusterConfiguration_To_upstreamv1beta1_ClusterConfiguration(src, dst, s) -} +} \ No newline at end of file diff --git a/bootstrap/kubeadm/types/upstreamv1beta3/conversion.go b/bootstrap/kubeadm/types/upstreamv1beta3/conversion.go index 1a96971e5..7c5a40bf9 100644 --- a/bootstrap/kubeadm/types/upstreamv1beta3/conversion.go +++ b/bootstrap/kubeadm/types/upstreamv1beta3/conversion.go @@ -87,4 +87,4 @@ func Convert_v1beta1_ClusterConfiguration_To_upstreamv1beta3_ClusterConfiguratio func Convert_v1beta1_JoinConfiguration_To_upstreamv1beta3_JoinConfiguration(src *bootstrapv1.JoinConfiguration, dst *JoinConfiguration, s apimachineryconversion.Scope) error { // JoinConfiguration.BottlerocketCustomHostContainers exists in bootstrapv1.JoinControlPlane but not in upstreamv1beta3 return autoConvert_v1beta1_JoinConfiguration_To_upstreamv1beta3_JoinConfiguration(src, dst, s) -} +} \ No newline at end of file diff --git a/util/secret/consts.go b/util/secret/consts.go index 043764325..17abd5d70 100644 --- a/util/secret/consts.go +++ b/util/secret/consts.go @@ -25,6 +25,9 @@ const ( // TLSCrtDataName is the key used to store a TLS certificate in the secret's data field. TLSCrtDataName = "tls.crt" + + // RegistryCredentials is the secret name of secret containing registryMirror credentials. + RegistryCredentials = "registry-credentials" ) // Purpose is the name to append to the secret generated for a cluster. -- 2.40.0