package framework import ( "fmt" "os" "strings" "testing" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" kerrors "k8s.io/apimachinery/pkg/util/errors" "github.com/aws/eks-anywhere/internal/pkg/api" anywherev1 "github.com/aws/eks-anywhere/pkg/api/v1alpha1" releasev1 "github.com/aws/eks-anywhere/release/api/v1alpha1" clusterf "github.com/aws/eks-anywhere/test/framework/cluster" ) const ( snowAMIIDUbuntu123 = "T_SNOW_AMIID_UBUNTU_1_23" snowAMIIDUbuntu124 = "T_SNOW_AMIID_UBUNTU_1_24" snowAMIIDUbuntu125 = "T_SNOW_AMIID_UBUNTU_1_25" snowAMIIDUbuntu126 = "T_SNOW_AMIID_UBUNTU_1_26" snowAMIIDUbuntu127 = "T_SNOW_AMIID_UBUNTU_1_27" snowDevices = "T_SNOW_DEVICES" snowControlPlaneCidr = "T_SNOW_CONTROL_PLANE_CIDR" snowPodCidr = "T_SNOW_POD_CIDR" snowCredentialsFile = "EKSA_AWS_CREDENTIALS_FILE" snowCertificatesFile = "EKSA_AWS_CA_BUNDLES_FILE" snowIPPoolIPStart = "T_SNOW_IPPOOL_IPSTART" snowIPPoolIPEnd = "T_SNOW_IPPOOL_IPEND" snowIPPoolGateway = "T_SNOW_IPPOOL_GATEWAY" snowIPPoolSubnet = "T_SNOW_IPPOOL_SUBNET" snowEc2TagPrefix = "sigs.k8s.io/cluster-api-provider-aws-snow/cluster/" ) var requiredSnowEnvVars = []string{ snowDevices, snowControlPlaneCidr, snowCredentialsFile, snowCertificatesFile, } type Snow struct { t *testing.T fillers []api.SnowFiller clusterFillers []api.ClusterFiller cpCidr string podCidr string } type SnowOpt func(*Snow) func NewSnow(t *testing.T, opts ...SnowOpt) *Snow { checkRequiredEnvVars(t, requiredSnowEnvVars) s := &Snow{ t: t, } s.cpCidr = os.Getenv(snowControlPlaneCidr) s.podCidr = os.Getenv(snowPodCidr) for _, opt := range opts { opt(s) } return s } func (s *Snow) Name() string { return "snow" } func (s *Snow) Setup() {} // UpdateKubeConfig customizes generated kubeconfig for the provider. func (s *Snow) UpdateKubeConfig(content *[]byte, clusterName string) error { return nil } // ClusterConfigUpdates satisfies the test framework Provider. func (s *Snow) ClusterConfigUpdates() []api.ClusterConfigFiller { s.t.Logf("Searching for free IP for Snow Control Plane in CIDR %s", s.cpCidr) ip, err := GenerateUniqueIp(s.cpCidr) if err != nil { s.t.Fatalf("failed to generate control plane ip for snow [cidr=%s]: %v", s.cpCidr, err) } s.t.Logf("Selected IP %s for Snow Control Plane", ip) f := make([]api.ClusterFiller, 0, len(s.clusterFillers)+2) f = append(f, s.clusterFillers...) f = append(f, api.WithControlPlaneEndpointIP(ip)) if s.podCidr != "" { f = append(f, api.WithPodCidr(s.podCidr)) } return []api.ClusterConfigFiller{api.ClusterToConfigFiller(f...), api.SnowToConfigFiller(s.fillers...)} } // CleanupVMs satisfies the test framework Provider. func (s *Snow) CleanupVMs(clusterName string) error { snowDeviceIPs := strings.Split(os.Getenv(snowDevices), ",") s.t.Logf("Cleaning ec2 instances of %s in snow devices: %v", clusterName, snowDeviceIPs) var res []error for _, ip := range snowDeviceIPs { sess, err := newSession(ip) if err != nil { res = append(res, fmt.Errorf("Cannot create session to snow device: %w", err)) continue } ec2Client := ec2.New(sess) // snow device doesn't support filter hitherto out, err := ec2Client.DescribeInstances(&ec2.DescribeInstancesInput{}) if err != nil { res = append(res, fmt.Errorf("Cannot get ec2 instances from snow device: %w", err)) continue } var ownedInstanceIds []*string for _, reservation := range out.Reservations { for _, instance := range reservation.Instances { if isNotTerminatedAndHasTag(instance, snowEc2TagPrefix+clusterName) { ownedInstanceIds = append(ownedInstanceIds, instance.InstanceId) } } } if len(ownedInstanceIds) != 0 { if _, err = ec2Client.TerminateInstances(&ec2.TerminateInstancesInput{ InstanceIds: ownedInstanceIds, }); err != nil { res = append(res, fmt.Errorf("Cannot terminate ec2 instances from snow device: %w", err)) } else { s.t.Logf("Cluster %s EC2 instances have been cleaned from device %s: %+v", clusterName, ip, ownedInstanceIds) } } else { s.t.Logf("No EC2 instances to cleanup for snow device: %s", ip) } cleanedKeys, err := cleanupKeypairs(ec2Client, clusterName) if err != nil { res = append(res, err) } else { s.t.Logf("KeyPairs has been cleaned: %+v", cleanedKeys) } } return kerrors.NewAggregate(res) } func cleanupKeypairs(ec2Client *ec2.EC2, clusterName string) ([]*string, error) { out, err := ec2Client.DescribeKeyPairs(&ec2.DescribeKeyPairsInput{}) if err != nil { return nil, err } var keyPairNames []*string for _, keyPair := range out.KeyPairs { if strings.Contains(*keyPair.KeyName, clusterName) { keyPairNames = append(keyPairNames, keyPair.KeyName) } } var errs []error for _, keyPairName := range keyPairNames { if _, err := ec2Client.DeleteKeyPair(&ec2.DeleteKeyPairInput{ KeyName: keyPairName, }); err != nil { errs = append(errs, err) } } return keyPairNames, kerrors.NewAggregate(errs) } func isNotTerminatedAndHasTag(instance *ec2.Instance, tag string) bool { if *instance.State.Name == "terminated" { return false } for _, t := range instance.Tags { if *t.Key == tag { return true } } return false } func newSession(ip string) (*session.Session, error) { sess, err := session.NewSession(&aws.Config{ Endpoint: aws.String("http://" + ip + ":8008"), Credentials: credentials.NewSharedCredentials(os.Getenv(snowCredentialsFile), ip), Region: aws.String("snow"), }) if err != nil { return nil, fmt.Errorf("Cannot create session to snow device: %v", err) } return sess, nil } func (s *Snow) WithProviderUpgrade(fillers ...api.SnowFiller) ClusterE2ETestOpt { return func(e *ClusterE2ETest) { e.UpdateClusterConfig(api.SnowToConfigFiller(fillers...)) } } // WithBottlerocket123 returns a cluster config filler that sets the kubernetes version of the cluster to 1.23 // as well as the right devices and osFamily for all SnowMachineConfigs. It also sets any // necessary machine config default required for BR, like the container volume size. If the env var is set, this will // also set the AMI ID. Otherwise, it will leave it empty and let CAPAS select one. func (s *Snow) WithBottlerocket123() api.ClusterConfigFiller { s.t.Helper() return s.withBottlerocketForKubeVersion(anywherev1.Kube123) } // WithBottlerocket124 returns a cluster config filler that sets the kubernetes version of the cluster to 1.24 // as well as the right devices and osFamily for all SnowMachineConfigs. It also sets any // necessary machine config default required for BR, like the container volume size. If the env var is set, this will // also set the AMI ID. Otherwise, it will leave it empty and let CAPAS select one. func (s *Snow) WithBottlerocket124() api.ClusterConfigFiller { s.t.Helper() return s.withBottlerocketForKubeVersion(anywherev1.Kube124) } // WithBottlerocket125 returns a cluster config filler that sets the kubernetes version of the cluster to 1.25 // as well as the right devices and osFamily for all SnowMachineConfigs. It also sets any // necessary machine config default required for BR, like the container volume size. If the env var is set, this will // also set the AMI ID. Otherwise, it will leave it empty and let CAPAS select one. func (s *Snow) WithBottlerocket125() api.ClusterConfigFiller { s.t.Helper() return s.withBottlerocketForKubeVersion(anywherev1.Kube125) } // WithBottlerocket126 returns a cluster config filler that sets the kubernetes version of the cluster to 1.26 // as well as the right devices and osFamily for all SnowMachineConfigs. It also sets any // necessary machine config default required for BR, like the container volume size. If the env var is set, this will // also set the AMI ID. Otherwise, it will leave it empty and let CAPAS select one. func (s *Snow) WithBottlerocket126() api.ClusterConfigFiller { s.t.Helper() return s.withBottlerocketForKubeVersion(anywherev1.Kube126) } // WithBottlerocket127 returns a cluster config filler that sets the kubernetes version of the cluster to 1.27 // as well as the right devices and osFamily for all SnowMachineConfigs. It also sets any // necessary machine config default required for BR, like the container volume size. If the env var is set, this will // also set the AMI ID. Otherwise, it will leave it empty and let CAPAS select one. func (s *Snow) WithBottlerocket127() api.ClusterConfigFiller { s.t.Helper() return s.withBottlerocketForKubeVersion(anywherev1.Kube127) } // WithBottlerocketStaticIP123 returns a cluster config filler that sets the kubernetes version of the cluster to 1.23 // as well as the right devices, osFamily and static ip config for all SnowMachineConfigs. Comparing to WithBottlerocket123, // this method also adds a snow ip pool to support static ip configuration. func (s *Snow) WithBottlerocketStaticIP123() api.ClusterConfigFiller { s.t.Helper() return s.withBottlerocketStaticIPForKubeVersion(anywherev1.Kube123) } // WithBottlerocketStaticIP124 returns a cluster config filler that sets the kubernetes version of the cluster to 1.24 // as well as the right devices, osFamily and static ip config for all SnowMachineConfigs. Comparing to WithBottlerocket124, // this method also adds a snow ip pool to support static ip configuration. func (s *Snow) WithBottlerocketStaticIP124() api.ClusterConfigFiller { s.t.Helper() return s.withBottlerocketStaticIPForKubeVersion(anywherev1.Kube124) } // WithBottlerocketStaticIP125 returns a cluster config filler that sets the kubernetes version of the cluster to 1.25 // as well as the right devices, osFamily and static ip config for all SnowMachineConfigs. Comparing to WithBottlerocket125, // this method also adds a snow ip pool to support static ip configuration. func (s *Snow) WithBottlerocketStaticIP125() api.ClusterConfigFiller { s.t.Helper() return s.withBottlerocketStaticIPForKubeVersion(anywherev1.Kube125) } // WithBottlerocketStaticIP126 returns a cluster config filler that sets the kubernetes version of the cluster to 1.26 // as well as the right devices, osFamily and static ip config for all SnowMachineConfigs. Comparing to WithBottlerocket126, // this method also adds a snow ip pool to support static ip configuration. func (s *Snow) WithBottlerocketStaticIP126() api.ClusterConfigFiller { s.t.Helper() return s.withBottlerocketStaticIPForKubeVersion(anywherev1.Kube126) } // WithBottlerocketStaticIP127 returns a cluster config filler that sets the kubernetes version of the cluster to 1.27 // as well as the right devices, osFamily and static ip config for all SnowMachineConfigs. Comparing to WithBottlerocket127, // this method also adds a snow ip pool to support static ip configuration. func (s *Snow) WithBottlerocketStaticIP127() api.ClusterConfigFiller { s.t.Helper() return s.withBottlerocketStaticIPForKubeVersion(anywherev1.Kube127) } // WithUbuntu123 returns a cluster config filler that sets the kubernetes version of the cluster to 1.23 // as well as the right devices and osFamily for all SnowMachineConfigs. If the env var is set, this will // also set the AMI ID. Otherwise, it will leave it empty and let CAPAS select one. func (s *Snow) WithUbuntu123() api.ClusterConfigFiller { s.t.Helper() return s.WithKubeVersionAndOS(anywherev1.Kube123, Ubuntu2004, nil) } // WithUbuntu124 returns a cluster config filler that sets the kubernetes version of the cluster to 1.24 // as well as the right devices and osFamily for all SnowMachineConfigs. If the env var is set, this will // also set the AMI ID. Otherwise, it will leave it empty and let CAPAS select one. func (s *Snow) WithUbuntu124() api.ClusterConfigFiller { s.t.Helper() return s.WithKubeVersionAndOS(anywherev1.Kube124, Ubuntu2004, nil) } // WithUbuntu125 returns a cluster config filler that sets the kubernetes version of the cluster to 1.25 // as well as the right devices and osFamily for all SnowMachineConfigs. If the env var is set, this will // also set the AMI ID. Otherwise, it will leave it empty and let CAPAS select one. func (s *Snow) WithUbuntu125() api.ClusterConfigFiller { s.t.Helper() return s.WithKubeVersionAndOS(anywherev1.Kube125, Ubuntu2004, nil) } // WithUbuntu126 returns a cluster config filler that sets the kubernetes version of the cluster to 1.26 // as well as the right devices and osFamily for all SnowMachineConfigs. If the env var is set, this will // also set the AMI ID. Otherwise, it will leave it empty and let CAPAS select one. func (s *Snow) WithUbuntu126() api.ClusterConfigFiller { s.t.Helper() return s.WithKubeVersionAndOS(anywherev1.Kube126, Ubuntu2004, nil) } // WithUbuntu127 returns a cluster config filler that sets the kubernetes version of the cluster to 1.27 // as well as the right devices and osFamily for all SnowMachineConfigs. If the env var is set, this will // also set the AMI ID. Otherwise, it will leave it empty and let CAPAS select one. func (s *Snow) WithUbuntu127() api.ClusterConfigFiller { s.t.Helper() return s.WithKubeVersionAndOS(anywherev1.Kube127, Ubuntu2004, nil) } func (s *Snow) withBottlerocketForKubeVersion(kubeVersion anywherev1.KubernetesVersion) api.ClusterConfigFiller { return api.JoinClusterConfigFillers( s.WithKubeVersionAndOS(kubeVersion, Bottlerocket1, nil), api.SnowToConfigFiller(api.WithChangeForAllSnowMachines(api.WithSnowContainersVolumeSize(100))), ) } func (s *Snow) withBottlerocketStaticIPForKubeVersion(kubeVersion anywherev1.KubernetesVersion) api.ClusterConfigFiller { poolName := "pool-1" return api.JoinClusterConfigFillers( s.WithKubeVersionAndOS(kubeVersion, Bottlerocket1, nil), api.SnowToConfigFiller(api.WithChangeForAllSnowMachines(api.WithSnowContainersVolumeSize(100))), api.SnowToConfigFiller(api.WithChangeForAllSnowMachines(api.WithStaticIP(poolName))), api.SnowToConfigFiller(s.withIPPoolFromEnvVar(poolName)), ) } // WithKubeVersionAndOS returns a cluster config filler that sets the cluster kube version and the correct AMI ID // and devices for the Snow machine configs. func (s *Snow) WithKubeVersionAndOS(kubeVersion anywherev1.KubernetesVersion, os OS, release *releasev1.EksARelease) api.ClusterConfigFiller { envar := fmt.Sprintf("T_SNOW_AMIID_%s_%s", strings.ToUpper(strings.ReplaceAll(string(os), "-", "_")), strings.ReplaceAll(string(kubeVersion), ".", "_")) return api.JoinClusterConfigFillers( api.ClusterToConfigFiller(api.WithKubernetesVersion(kubeVersion)), api.SnowToConfigFiller( s.withAMIIDFromEnvVar(envar), api.WithSnowStringFromEnvVar(snowDevices, api.WithSnowDevicesForAllMachines), api.WithOsFamilyForAllSnowMachines(osFamiliesForOS[os]), ), ) } func (s *Snow) withAMIIDFromEnvVar(envvar string) api.SnowFiller { val, ok := os.LookupEnv(envvar) if !ok { s.t.Log("% for Snow AMI ID is not set, leaving amiID empty which will let CAPAS select the right AMI from the ones available in the device", envvar) val = "" } return api.WithSnowAMIIDForAllMachines(val) } func (s *Snow) withIPPoolFromEnvVar(name string) api.SnowFiller { envVars := []string{snowIPPoolIPStart, snowIPPoolIPEnd, snowIPPoolGateway, snowIPPoolSubnet} checkRequiredEnvVars(s.t, envVars) return api.WithSnowIPPool(name, os.Getenv(snowIPPoolIPStart), os.Getenv(snowIPPoolIPEnd), os.Getenv(snowIPPoolGateway), os.Getenv(snowIPPoolSubnet)) } func WithSnowUbuntu123() SnowOpt { return func(s *Snow) { s.fillers = append(s.fillers, api.WithSnowStringFromEnvVar(snowAMIIDUbuntu123, api.WithSnowAMIIDForAllMachines), api.WithSnowStringFromEnvVar(snowDevices, api.WithSnowDevicesForAllMachines), api.WithOsFamilyForAllSnowMachines(anywherev1.Ubuntu), ) } } // WithSnowUbuntu124 returns SnowOpt that sets the right devices and osFamily for all SnowMachineConfigs. // If the env var is set, this will also set the AMI ID. // Otherwise, it will leave it empty and let CAPAS select one. func WithSnowUbuntu124() SnowOpt { return func(s *Snow) { s.fillers = append(s.fillers, s.withAMIIDFromEnvVar(snowAMIIDUbuntu124), api.WithSnowStringFromEnvVar(snowDevices, api.WithSnowDevicesForAllMachines), api.WithOsFamilyForAllSnowMachines(anywherev1.Ubuntu), ) } } // WithSnowUbuntu125 returns SnowOpt that sets the right devices and osFamily for all SnowMachineConfigs. // If the env var is set, this will also set the AMI ID. // Otherwise, it will leave it empty and let CAPAS select one. func WithSnowUbuntu125() SnowOpt { return func(s *Snow) { s.fillers = append(s.fillers, api.WithSnowStringFromEnvVar(snowAMIIDUbuntu125, api.WithSnowAMIIDForAllMachines), api.WithSnowStringFromEnvVar(snowDevices, api.WithSnowDevicesForAllMachines), api.WithOsFamilyForAllSnowMachines(anywherev1.Ubuntu), ) } } // WithSnowUbuntu126 returns SnowOpt that sets the right devices and osFamily for all SnowMachineConfigs. // If the env var is set, this will also set the AMI ID. // Otherwise, it will leave it empty and let CAPAS select one. func WithSnowUbuntu126() SnowOpt { return func(s *Snow) { s.fillers = append(s.fillers, api.WithSnowStringFromEnvVar(snowAMIIDUbuntu126, api.WithSnowAMIIDForAllMachines), api.WithSnowStringFromEnvVar(snowDevices, api.WithSnowDevicesForAllMachines), api.WithOsFamilyForAllSnowMachines(anywherev1.Ubuntu), ) } } // WithSnowUbuntu127 returns SnowOpt that sets the right devices and osFamily for all SnowMachineConfigs. // If the env var is set, this will also set the AMI ID. // Otherwise, it will leave it empty and let CAPAS select one. func WithSnowUbuntu127() SnowOpt { return func(s *Snow) { s.fillers = append(s.fillers, api.WithSnowStringFromEnvVar(snowAMIIDUbuntu127, api.WithSnowAMIIDForAllMachines), api.WithSnowStringFromEnvVar(snowDevices, api.WithSnowDevicesForAllMachines), api.WithOsFamilyForAllSnowMachines(anywherev1.Ubuntu), ) } } // WithSnowWorkerNodeGroup stores the necessary fillers to update/create the provided worker node group with its corresponding SnowMachineConfig // and apply the given changes to that machine config. func WithSnowWorkerNodeGroup(name string, workerNodeGroup *WorkerNodeGroup, fillers ...api.SnowMachineConfigFiller) SnowOpt { return func(s *Snow) { s.fillers = append(s.fillers, snowMachineConfig(name, fillers...)) s.clusterFillers = append(s.clusterFillers, buildSnowWorkerNodeGroupClusterFiller(name, workerNodeGroup)) } } // WithWorkerNodeGroup returns a filler that updates/creates the provided worker node group with its corresponding SnowMachineConfig // and applies the given changes to that machine config. func (s *Snow) WithWorkerNodeGroup(name string, workerNodeGroup *WorkerNodeGroup, fillers ...api.SnowMachineConfigFiller) api.ClusterConfigFiller { return api.JoinClusterConfigFillers( api.ClusterToConfigFiller(buildSnowWorkerNodeGroupClusterFiller(name, workerNodeGroup)), api.SnowToConfigFiller(snowMachineConfig(name, fillers...)), ) } // WithNewWorkerNodeGroup returns a filler that updates/creates the provided worker node group with its corresponding SnowMachineConfig. func (s *Snow) WithNewWorkerNodeGroup(name string, workerNodeGroup *WorkerNodeGroup) api.ClusterConfigFiller { return api.JoinClusterConfigFillers( api.ClusterToConfigFiller(buildSnowWorkerNodeGroupClusterFiller(name, workerNodeGroup)), api.SnowToConfigFiller(snowMachineConfig(name)), ) } // WithNewSnowWorkerNodeGroup updates the test cluster Config with the fillers for an specific snow worker node group. // It applies the fillers in WorkerNodeGroup to the named worker node group and the ones for the corresponding SnowMachineConfig. func (s *Snow) WithNewSnowWorkerNodeGroup(name string, workerNodeGroup *WorkerNodeGroup, fillers ...api.SnowMachineConfigFiller) ClusterE2ETestOpt { return func(e *ClusterE2ETest) { e.UpdateClusterConfig( api.SnowToConfigFiller(snowMachineConfig(name, fillers...)), api.ClusterToConfigFiller(buildSnowWorkerNodeGroupClusterFiller(name, workerNodeGroup)), ) } } func snowMachineConfig(name string, fillers ...api.SnowMachineConfigFiller) api.SnowFiller { f := make([]api.SnowMachineConfigFiller, 0, len(fillers)+2) f = append(f, api.WithSnowMachineDefaultValues(), api.WithSnowDevices(os.Getenv(snowDevices)), ) f = append(f, fillers...) return api.WithSnowMachineConfig(name, f...) } func buildSnowWorkerNodeGroupClusterFiller(machineConfigName string, workerNodeGroup *WorkerNodeGroup) api.ClusterFiller { workerNodeGroup.MachineConfigKind = anywherev1.SnowMachineConfigKind workerNodeGroup.MachineConfigName = machineConfigName return workerNodeGroup.ClusterFiller() } func UpdateSnowUbuntuTemplate123Var() api.SnowFiller { return api.WithSnowStringFromEnvVar(snowAMIIDUbuntu123, api.WithSnowAMIIDForAllMachines) } // ClusterStateValidations returns a list of provider specific validations. func (s *Snow) ClusterStateValidations() []clusterf.StateValidation { return []clusterf.StateValidation{} }