package e2e import ( "context" "fmt" "os" "strconv" "time" aws_ssm "github.com/aws/aws-sdk-go/service/ssm" "github.com/go-logr/logr" "gopkg.in/yaml.v2" "github.com/aws/eks-anywhere/internal/pkg/ec2" "github.com/aws/eks-anywhere/internal/pkg/ssm" "github.com/aws/eks-anywhere/internal/pkg/vsphere" "github.com/aws/eks-anywhere/internal/test/cleanup" "github.com/aws/eks-anywhere/pkg/executables" "github.com/aws/eks-anywhere/pkg/retrier" ) const ( testRunnerVCUserEnvVar string = "TEST_RUNNER_GOVC_USERNAME" testRunnerVCPasswordEnvVar string = "TEST_RUNNER_GOVC_PASSWORD" govcUsernameKey string = "GOVC_USERNAME" govcPasswordKey string = "GOVC_PASSWORD" govcURLKey string = "GOVC_URL" govcInsecure string = "GOVC_INSECURE" govcDatacenterKey string = "GOVC_DATACENTER" ssmActivationCodeKey string = "ssm_activation_code" ssmActivationIdKey string = "ssm_activation_id" ssmActivationRegionKey string = "ssm_activation_region" ) type TestRunner interface { createInstance(instanceConf instanceRunConf) (string, error) tagInstance(instanceConf instanceRunConf, key, value string) error decommInstance(instanceRunConf) error } type TestRunnerType string const ( Ec2TestRunnerType TestRunnerType = "ec2" VSphereTestRunnerType TestRunnerType = "vSphere" ) func newTestRunner(runnerType TestRunnerType, config TestInfraConfig) (TestRunner, error) { if runnerType == VSphereTestRunnerType { var err error v := &config.VSphereTestRunner v.envMap, err = v.setEnvironment() if err != nil { return nil, fmt.Errorf("failed to set env for vSphere test runner: %v", err) } return v, nil } else { return &config.Ec2TestRunner, nil } } type TestInfraConfig struct { Ec2TestRunner `yaml:"ec2,omitempty"` VSphereTestRunner `yaml:"vSphere,omitempty"` } func NewTestRunnerConfigFromFile(logger logr.Logger, configFile string) (*TestInfraConfig, error) { file, err := os.ReadFile(configFile) if err != nil { return nil, fmt.Errorf("failed to create test runner config from file: %v", err) } config := TestInfraConfig{} config.VSphereTestRunner.logger = logger config.Ec2TestRunner.logger = logger err = yaml.Unmarshal(file, &config) if err != nil { return nil, fmt.Errorf("failed to create test runner config from file: %v", err) } return &config, nil } type testRunner struct { InstanceID string logger logr.Logger } type Ec2TestRunner struct { testRunner AmiID string `yaml:"amiId"` SubnetID string `yaml:"subnetId"` } type VSphereTestRunner struct { testRunner ActivationId string envMap map[string]string Url string `yaml:"url"` Insecure bool `yaml:"insecure"` Library string `yaml:"library"` Template string `yaml:"template"` Datacenter string `yaml:"datacenter"` Datastore string `yaml:"datastore"` ResourcePool string `yaml:"resourcePool"` Network string `yaml:"network"` Folder string `yaml:"folder"` } func (v *VSphereTestRunner) setEnvironment() (map[string]string, error) { envMap := make(map[string]string) if vSphereUsername, ok := os.LookupEnv(testRunnerVCUserEnvVar); ok && len(vSphereUsername) > 0 { envMap[govcUsernameKey] = vSphereUsername } else { return nil, fmt.Errorf("missing environment variable: %s", testRunnerVCUserEnvVar) } if vSpherePassword, ok := os.LookupEnv(testRunnerVCPasswordEnvVar); ok && len(vSpherePassword) > 0 { envMap[govcPasswordKey] = vSpherePassword } else { return nil, fmt.Errorf("missing environment variable: %s", testRunnerVCPasswordEnvVar) } envMap[govcURLKey] = v.Url envMap[govcInsecure] = strconv.FormatBool(v.Insecure) envMap[govcDatacenterKey] = v.Datacenter v.envMap = envMap return envMap, nil } func (v *VSphereTestRunner) createInstance(c instanceRunConf) (string, error) { name := getTestRunnerName(v.logger, c.jobId) v.logger.V(1).Info("Creating vSphere Test Runner instance", "name", name) ssmActivationInfo, err := ssm.CreateActivation(c.session, name, c.instanceProfileName) if err != nil { return "", fmt.Errorf("unable to create ssm activation: %v", err) } // TODO: import ova template from url if not exist opts := vsphere.OVFDeployOptions{ Name: name, PowerOn: true, DiskProvisioning: "thin", WaitForIP: true, InjectOvfEnv: true, NetworkMappings: []vsphere.NetworkMapping{{Name: v.Network, Network: v.Network}}, PropertyMapping: []vsphere.OVFProperty{ {Key: ssmActivationCodeKey, Value: ssmActivationInfo.ActivationCode}, {Key: ssmActivationIdKey, Value: ssmActivationInfo.ActivationID}, {Key: ssmActivationRegionKey, Value: *c.session.Config.Region}, }, } // deploy template if err := vsphere.DeployTemplate(v.envMap, v.Library, v.Template, name, v.Folder, v.Datacenter, v.Datastore, v.ResourcePool, opts); err != nil { return "", err } var ssmInstance *aws_ssm.InstanceInformation err = retrier.Retry(10, 5*time.Second, func() error { ssmInstance, err = ssm.GetInstanceByActivationId(c.session, ssmActivationInfo.ActivationID) if err != nil { return fmt.Errorf("failed to get ssm instance info post ovf deployment: %v", err) } return nil }) if err != nil { return "", fmt.Errorf("waiting for ssm instance to activate %s : %v", name, err) } v.InstanceID = *ssmInstance.InstanceId v.ActivationId = ssmActivationInfo.ActivationID return *ssmInstance.InstanceId, nil } func (e *Ec2TestRunner) createInstance(c instanceRunConf) (string, error) { name := getTestRunnerName(e.logger, c.jobId) e.logger.V(1).Info("Creating ec2 Test Runner instance", "name", name) instanceId, err := ec2.CreateInstance(c.session, e.AmiID, key, tag, c.instanceProfileName, e.SubnetID, name) if err != nil { return "", fmt.Errorf("creating instance for e2e tests: %v", err) } e.logger.V(1).Info("Instance created", "instance-id", instanceId) e.InstanceID = instanceId return instanceId, nil } func (v *VSphereTestRunner) tagInstance(c instanceRunConf, key, value string) error { vmName := getTestRunnerName(v.logger, c.jobId) vmPath := fmt.Sprintf("/%s/vm/%s/%s", v.Datacenter, v.Folder, vmName) tag := fmt.Sprintf("%s:%s", key, value) if err := vsphere.TagVirtualMachine(v.envMap, vmPath, tag); err != nil { return fmt.Errorf("failed to tag vSphere test runner: %v", err) } return nil } func (e *Ec2TestRunner) tagInstance(c instanceRunConf, key, value string) error { err := ec2.TagInstance(c.session, e.InstanceID, key, value) if err != nil { return fmt.Errorf("failed to tag Ec2 test runner: %v", err) } return nil } func (v *VSphereTestRunner) decommInstance(c instanceRunConf) error { _, deregisterError := ssm.DeregisterInstance(c.session, v.InstanceID) _, deactivateError := ssm.DeleteActivation(c.session, v.ActivationId) deleteError := cleanup.VsphereRmVms(context.Background(), getTestRunnerName(v.logger, c.jobId), executables.WithGovcEnvMap(v.envMap)) if deregisterError != nil { return fmt.Errorf("failed to decommission vsphere test runner ssm instance: %v", deregisterError) } if deactivateError != nil { return fmt.Errorf("failed to decommission vsphere test runner ssm instance: %v", deactivateError) } if deleteError != nil { return fmt.Errorf("failed to decommission vsphere test runner ssm instance: %v", deleteError) } return nil } func (v *Ec2TestRunner) decommInstance(c instanceRunConf) error { return nil } func getTestRunnerName(logger logr.Logger, jobId string) string { name := fmt.Sprintf("eksa-e2e-%s", jobId) if len(name) > 80 { logger.V(1).Info("Truncating test runner name to 80 chars", "original_name", name) name = name[len(name)-80:] logger.V(1).Info("Truncated test runner name", "truncated_name", name) } return name }