package framework import ( "context" "os" "path/filepath" "strconv" "strings" "github.com/aws/eks-anywhere/pkg/api/v1alpha1" ) const ( // NTP configuration environment variables. ntpServersVar = "T_NTP_SERVERS" // Bottlerocket configuration environment variables. maxPodsVar = "T_BR_K8S_SETTINGS_MAX_PODS" clusterDNSIPSVar = "T_BR_K8S_SETTINGS_CLUSTER_DNS_IPS" allowedUnsafeSysctlsVar = "T_BR_K8S_SETTINGS_ALLOWED_UNSAFE_SYSCTLS" // other constants. defaultSSHUsername = "ec2-user" privateKeyFileName = "eks-a-id_rsa" ) var ( ntpServersRequiredVar = []string{ntpServersVar} brKubernetesRequiredVar = []string{maxPodsVar, clusterDNSIPSVar, allowedUnsafeSysctlsVar} ) // RequiredNTPServersEnvVars returns a slice of environment variables required for NTP tests. func RequiredNTPServersEnvVars() []string { return ntpServersRequiredVar } // RequiredBottlerocketKubernetesSettingsEnvVars returns a slice of environment variables required for Bottlerocket Kubernetes tests. func RequiredBottlerocketKubernetesSettingsEnvVars() []string { return brKubernetesRequiredVar } // GetNTPServersFromEnv returns a slice of NTP servers read from the NTP environment veriables. func GetNTPServersFromEnv() []string { serverFromEnv := os.Getenv(ntpServersVar) return strings.Split(serverFromEnv, ",") } // GetBottlerocketKubernetesSettingsFromEnv returns a Bottlerocket Kubernetes settings read from the environment variables. func GetBottlerocketKubernetesSettingsFromEnv() (allowedUnsafeSysclts, clusterDNSIPS []string, maxPods int, err error) { allowedUnsafeSysclts = strings.Split(os.Getenv(allowedUnsafeSysctlsVar), ",") clusterDNSIPS = strings.Split(os.Getenv(clusterDNSIPSVar), ",") maxPods, err = strconv.Atoi(os.Getenv(maxPodsVar)) return allowedUnsafeSysclts, clusterDNSIPS, maxPods, err } // ValidateNTPConfig validates NTP servers are configured properly on all cluster nodes using SSH. func (e *ClusterE2ETest) ValidateNTPConfig(osFamily v1alpha1.OSFamily) { ctx := context.Background() machines, err := e.KubectlClient.GetCAPIMachines(ctx, e.managementCluster(), e.ClusterName) if err != nil { e.T.Fatalf("Error getting machines: %v", err) } for _, machine := range machines { if len(machine.Status.Addresses) > 0 { e.T.Logf("Validating NTP servers for machine %s with IP %s", machine.Name, machine.Status.Addresses[0].Address) e.validateNTP(ctx, osFamily, machine.Status.Addresses[0].Address) } } } func (e *ClusterE2ETest) validateNTP(ctx context.Context, osFamily v1alpha1.OSFamily, IP string) { ssh := buildSSH(e.T) var command []string if osFamily == v1alpha1.Bottlerocket { command = []string{"apiclient", "get", "settings.ntp"} } else { command = []string{"chronyc", "sourcestats"} } out, err := ssh.RunCommand(ctx, filepath.Join(e.ClusterName, privateKeyFileName), defaultSSHUsername, IP, command...) if err != nil { e.T.Fatalf("failed to validate NTP server: %v", err) } for _, server := range GetNTPServersFromEnv() { if !strings.Contains(out, server) { e.T.Fatalf("NTP Server [%s] not configured on machine", server) } e.T.Logf("NTP server [%s] is configured", server) } } // ValidateBottlerocketKubernetesSettings validates Bottlerocket Kubernetes settings are configured properly on all cluster nodes using SSH. func (e *ClusterE2ETest) ValidateBottlerocketKubernetesSettings() { ctx := context.Background() machines, err := e.KubectlClient.GetCAPIMachines(ctx, e.managementCluster(), e.ClusterName) if err != nil { e.T.Fatalf("Error getting machines: %v", err) } for _, machine := range machines { if len(machine.Status.Addresses) > 0 { e.T.Logf("Validating Bottlerocket Kubernetes settings for machine %s with IP %s", machine.Name, machine.Status.Addresses[0].Address) e.validateBottlerocketKubernetesSettings(ctx, machine.Status.Addresses[0].Address) } } } // nolint:gocyclo func (e *ClusterE2ETest) validateBottlerocketKubernetesSettings(ctx context.Context, IP string) { ssh := buildSSH(e.T) command := []string{"apiclient", "get", "settings.network.hostname"} gotHostname, err := ssh.RunCommand(ctx, filepath.Join(e.ClusterName, privateKeyFileName), defaultSSHUsername, IP, command...) if err != nil { e.T.Errorf("failed to validate Bottlerocket Kubernetes settings: %v", err) } if strings.Contains(gotHostname, "etcd") { e.T.Log("Skipping Bottlerocket Kubernetes settings validation for etcd node") return } command = []string{"apiclient", "get", "settings.kubernetes.allowed-unsafe-sysctls"} gotAllowedUnsafeSysctls, err := ssh.RunCommand(ctx, filepath.Join(e.ClusterName, privateKeyFileName), defaultSSHUsername, IP, command...) if err != nil { e.T.Errorf("failed to validate Bottlerocket Kubernetes settings: %v", err) } command = []string{"apiclient", "get", "settings.kubernetes.cluster-dns-ip"} gotClusterDNSIPs, err := ssh.RunCommand(ctx, filepath.Join(e.ClusterName, privateKeyFileName), defaultSSHUsername, IP, command...) if err != nil { e.T.Errorf("failed to validate Bottlerocket Kubernetes settings: %v", err) } command = []string{"apiclient", "get", "settings.kubernetes.max-pods"} gotMaxPods, err := ssh.RunCommand(ctx, filepath.Join(e.ClusterName, privateKeyFileName), defaultSSHUsername, IP, command...) if err != nil { e.T.Errorf("failed to validate Bottlerocket Kubernetes settings: %v", err) } expectedAllowedUnsafeSysctls, expectedClusterDNSIPs, expectedMaxPods, err := GetBottlerocketKubernetesSettingsFromEnv() if err != nil { e.T.Errorf("failed to get Bottlerocket Kubernetes settings from environment variables: %v", err) } for _, sysctl := range expectedAllowedUnsafeSysctls { if !strings.Contains(gotAllowedUnsafeSysctls, sysctl) { e.T.Errorf("Bottlerocket Kubernetes setting [allowed-unsafe-sysctls: %s] not configured on machine", sysctl) } e.T.Logf("Bottlerocket Kubernetes setting [allowed-unsafe-sysctls: %s] is configured", sysctl) } for _, ip := range expectedClusterDNSIPs { if !strings.Contains(gotClusterDNSIPs, ip) { e.T.Errorf("Bottlerocket Kubernetes setting [cluster-dns-ips: %s] not configured on machine", ip) } e.T.Logf("Bottlerocket Kubernetes setting [cluster-dns-ips: %s] is configured", ip) } if !strings.Contains(gotMaxPods, strconv.Itoa(expectedMaxPods)) { e.T.Errorf("Bottlerocket Kubernetes setting [max-pods: %d] not configured on machine", expectedMaxPods) } e.T.Logf("Bottlerocket Kubernetes setting [max-pods: %d] is configured", expectedMaxPods) }