--- AWSTemplateFormatVersion: 2010-09-09 Description: Amazon EFS performance tutorial - Creates multiple Auto Scaling groups & mounts an existing EFS file system Metadata: Authors: Description: Darryl Osborne (darrylo@amazon.com) License: Description: 'Copyright 2018 Amazon.com, Inc. and its affiliates. All Rights Reserved. SPDX-License-Identifier: MIT-0' AWS::CloudFormation::Interface: ParameterGroups: - Label: default: File System Parameters Parameters: - FileSystemId - Label: default: Amazon EC2 Instance 1 Parameters: - InstanceType1 - LinuxDistro1 - MountOnBoot1 - MountTls1 - Label: default: Amazon EC2 Instance 2 Parameters: - InstanceType2 - LinuxDistro2 - MountOnBoot2 - MountTls2 - Label: default: Amazon EC2 Instance 3 Parameters: - InstanceType3 - LinuxDistro3 - MountOnBoot3 - MountTls3 - Label: default: Cross instance attributes Parameters: - EfsUtilsSource - KeyName - SecurityGroup - NumberOfSubnets - Subnet ParameterLabels: EfsUtilsSource: default: Source of efs-utils FileSystemId: default: Amazon EFS File System InstanceType1: default: Instance type InstanceType2: default: Instance type InstanceType3: default: Instance type KeyName: default: EC2 key pair LinuxDistro1: default: Linux distro LinuxDistro2: default: Linux distro LinuxDistro3: default: Linux distro MountOnBoot1: default: Auto mount file system MountOnBoot2: default: Auto mount file system MountOnBoot3: default: Auto mount file system MountTls1: default: Mount over TLS MountTls2: default: Mount over TLS MountTls3: default: Mount over TLS NumberOfSubnets: default: Number of subnets SecurityGroup: default: Security Group Subnet: default: Subnets Parameters: EfsUtilsSource: AllowedValues: - GitHub - Repository Default: Repository Description: The source of the EFS mount helper (efs-utils). Type: String FileSystemId: AllowedPattern: ^(fs-)([a-z0-9]{8})$ Description: The Amazon EFS file system id. Type: String InstanceType1: AllowedValues: - t2.nano - t2.micro - t2.small - t2.medium - t2.large - t2.xlarge - t2.2xlarge - t3.2xlarge - m3.medium - m3.large - m3.xlarge - m3.2xlarge - m4.large - m4.xlarge - m4.2xlarge - m4.4xlarge - m4.10xlarge - m4.16xlarge - m5.large - m5.xlarge - m5.2xlarge - m5.4xlarge - m5.12xlarge - m5.24xlarge - c3.large - c3.xlarge - c3.2xlarge - c3.4xlarge - c3.8xlarge - c4.large - c4.xlarge - c4.2xlarge - c4.4xlarge - c4.8xlarge - c5.large - c5.xlarge - c5.2xlarge - c5.4xlarge - c5.9xlarge - c5.18xlarge - r3.large - r3.xlarge - r3.2xlarge - r3.4xlarge - r3.8xlarge - r4.large - r4.xlarge - r4.2xlarge - r4.4xlarge - r4.8xlarge - r4.16xlarge - x1.16xlarge - x1.32xlarge - x1e.xlarge - x1e.2xlarge - x1e.4xlarge - x1e.8xlarge - x1e.16xlarge - x1e.32xlarge - d2.xlarge - d2.2xlarge - d2.4xlarge - d2.8xlarge - h1.2xlarge - h1.4xlarge - h1.8xlarge - h1.16xlarge - i2.xlarge - i2.2xlarge - i2.4xlarge - i2.8xlarge - i3.large - i3.xlarge - i3.2xlarge - i3.4xlarge - i3.8xlarge - i3.16xlarge - f1.2xlarge - f1.16xlarge - g2.2xlarge - g2.8xlarge - g3.4xlarge - g3.8xlarge - g3.16xlarge - p2.xlarge - p2.8xlarge - p2.16xlarge - p3.2xlarge - p3.8xlarge - p3.16xlarge Default: t2.micro Description: The Amazon EC2 instance type for Auto Scaling group 1. Type: String InstanceType2: AllowedValues: - t2.nano - t2.micro - t2.small - t2.medium - t2.large - t2.xlarge - t2.2xlarge - t3.2xlarge - m3.medium - m3.large - m3.xlarge - m3.2xlarge - m4.large - m4.xlarge - m4.2xlarge - m4.4xlarge - m4.10xlarge - m4.16xlarge - m5.large - m5.xlarge - m5.2xlarge - m5.4xlarge - m5.12xlarge - m5.24xlarge - c3.large - c3.xlarge - c3.2xlarge - c3.4xlarge - c3.8xlarge - c4.large - c4.xlarge - c4.2xlarge - c4.4xlarge - c4.8xlarge - c5.large - c5.xlarge - c5.2xlarge - c5.4xlarge - c5.9xlarge - c5.18xlarge - r3.large - r3.xlarge - r3.2xlarge - r3.4xlarge - r3.8xlarge - r4.large - r4.xlarge - r4.2xlarge - r4.4xlarge - r4.8xlarge - r4.16xlarge - x1.16xlarge - x1.32xlarge - x1e.xlarge - x1e.2xlarge - x1e.4xlarge - x1e.8xlarge - x1e.16xlarge - x1e.32xlarge - d2.xlarge - d2.2xlarge - d2.4xlarge - d2.8xlarge - h1.2xlarge - h1.4xlarge - h1.8xlarge - h1.16xlarge - i2.xlarge - i2.2xlarge - i2.4xlarge - i2.8xlarge - i3.large - i3.xlarge - i3.2xlarge - i3.4xlarge - i3.8xlarge - i3.16xlarge - f1.2xlarge - f1.16xlarge - g2.2xlarge - g2.8xlarge - g3.4xlarge - g3.8xlarge - g3.16xlarge - p2.xlarge - p2.8xlarge - p2.16xlarge - p3.2xlarge - p3.8xlarge - p3.16xlarge Default: m4.large Description: The Amazon EC2 instance type for Auto Scaling group 2. Type: String InstanceType3: AllowedValues: - t2.nano - t2.micro - t2.small - t2.medium - t2.large - t2.xlarge - t2.2xlarge - t3.2xlarge - m3.medium - m3.large - m3.xlarge - m3.2xlarge - m4.large - m4.xlarge - m4.2xlarge - m4.4xlarge - m4.10xlarge - m4.16xlarge - m5.large - m5.xlarge - m5.2xlarge - m5.4xlarge - m5.12xlarge - m5.24xlarge - c3.large - c3.xlarge - c3.2xlarge - c3.4xlarge - c3.8xlarge - c4.large - c4.xlarge - c4.2xlarge - c4.4xlarge - c4.8xlarge - c5.large - c5.xlarge - c5.2xlarge - c5.4xlarge - c5.9xlarge - c5.18xlarge - r3.large - r3.xlarge - r3.2xlarge - r3.4xlarge - r3.8xlarge - r4.large - r4.xlarge - r4.2xlarge - r4.4xlarge - r4.8xlarge - r4.16xlarge - x1.16xlarge - x1.32xlarge - x1e.xlarge - x1e.2xlarge - x1e.4xlarge - x1e.8xlarge - x1e.16xlarge - x1e.32xlarge - d2.xlarge - d2.2xlarge - d2.4xlarge - d2.8xlarge - h1.2xlarge - h1.4xlarge - h1.8xlarge - h1.16xlarge - i2.xlarge - i2.2xlarge - i2.4xlarge - i2.8xlarge - i3.large - i3.xlarge - i3.2xlarge - i3.4xlarge - i3.8xlarge - i3.16xlarge - f1.2xlarge - f1.16xlarge - g2.2xlarge - g2.8xlarge - g3.4xlarge - g3.8xlarge - g3.16xlarge - p2.xlarge - p2.8xlarge - p2.16xlarge - p3.2xlarge - p3.8xlarge - p3.16xlarge Default: c5.2xlarge Description: The Amazon EC2 instance type for Auto Scaling group 3. Type: String KeyName: Description: Name of an EC2 KeyPair. Your EC2 instances will launch with this KeyPair. Type: AWS::EC2::KeyPair::KeyName LinuxDistro1: AllowedValues: - "amzn" - "amzn2" Default: "amzn" Description: Linux distribution Type: String LinuxDistro2: AllowedValues: - "amzn" - "amzn2" Default: "amzn" Description: Linux distribution Type: String LinuxDistro3: AllowedValues: - "amzn" - "amzn2" Default: "amzn" Description: Linux distribution Type: String MountOnBoot1: AllowedValues: - true - false Default: true Description: Automount file system on boot Type: String MountOnBoot2: AllowedValues: - true - false Default: true Description: Automount file system on boot Type: String MountOnBoot3: AllowedValues: - true - false Default: true Description: Automount file system on boot Type: String MountTls1: AllowedValues: - true - false Default: false Description: Mount file system over tls (true=encrypted; false=non-encrypted) Type: String MountTls2: AllowedValues: - true - false Default: false Description: Mount file system over tls (true=encrypted; false=non-encrypted) Type: String MountTls3: AllowedValues: - true - false Default: false Description: Mount file system over tls (true=encrypted; false=non-encrypted) Type: String NumberOfSubnets: AllowedValues: - 1 - 2 - 3 - 4 - 5 - 6 Default: 3 Description: Number of subnets. This must match your selections in the list of Subnets below. Type: String SecurityGroup: Description: Select the EC2 security group that allows SSH access to the instance and NFS access to the EFS file system. Type: AWS::EC2::SecurityGroup::Id Subnet: Description: Select existing public subnets. Type: List Conditions: NumberOfSubnets1: !Equals [ 1, !Ref NumberOfSubnets ] NumberOfSubnets2: !Equals [ 2, !Ref NumberOfSubnets ] NumberOfSubnets3: !Equals [ 3, !Ref NumberOfSubnets ] NumberOfSubnets4: !Equals [ 4, !Ref NumberOfSubnets ] NumberOfSubnets5: !Equals [ 5, !Ref NumberOfSubnets ] NumberOfSubnets6: !Equals [ 6, !Ref NumberOfSubnets ] Subnet0: !Or - !Condition NumberOfSubnets1 - !Condition NumberOfSubnets2 - !Condition NumberOfSubnets3 - !Condition NumberOfSubnets4 - !Condition NumberOfSubnets5 - !Condition NumberOfSubnets6 Subnet1: !Or - !Condition NumberOfSubnets2 - !Condition NumberOfSubnets3 - !Condition NumberOfSubnets4 - !Condition NumberOfSubnets5 - !Condition NumberOfSubnets6 Subnet2: !Or - !Condition NumberOfSubnets3 - !Condition NumberOfSubnets4 - !Condition NumberOfSubnets5 - !Condition NumberOfSubnets6 Subnet3: !Or - !Condition NumberOfSubnets4 - !Condition NumberOfSubnets5 - !Condition NumberOfSubnets6 Subnet4: !Or - !Condition NumberOfSubnets5 - !Condition NumberOfSubnets6 Subnet5: !Condition NumberOfSubnets6 MountTls1: !Equals [ true, !Ref MountTls1 ] MountTls2: !Equals [ true, !Ref MountTls2 ] MountTls3: !Equals [ true, !Ref MountTls3 ] MountOnBoot1: !Equals [ true, !Ref MountOnBoot1 ] MountOnBoot2: !Equals [ true, !Ref MountOnBoot2 ] MountOnBoot3: !Equals [ true, !Ref MountOnBoot3 ] AmznBuild1: !And - !Equals [ "amzn", !Ref LinuxDistro1 ] - !Condition BuildInstall AmznBuild2: !And - !Equals [ "amzn", !Ref LinuxDistro2 ] - !Condition BuildInstall AmznBuild3: !And - !Equals [ "amzn", !Ref LinuxDistro3 ] - !Condition BuildInstall Amzn2Build1: !And - !Equals [ "amzn2", !Ref LinuxDistro1 ] - !Condition BuildInstall Amzn2Build2: !And - !Equals [ "amzn2", !Ref LinuxDistro2 ] - !Condition BuildInstall Amzn2Build3: !And - !Equals [ "amzn2", !Ref LinuxDistro3 ] - !Condition BuildInstall RepoInstall: !Equals [ "Repository", !Ref EfsUtilsSource ] BuildInstall: !Equals [ "GitHub", !Ref EfsUtilsSource ] Mappings: RegionMap: us-east-1: amzn: ami-97785bed amzn2: ami-b70554c8 us-east-2: amzn: ami-f63b1193 amzn2: ami-8c122be9 us-west-1: amzn: ami-824c4ee2 amzn2: ami-e0ba5c83 us-west-2: amzn: ami-f2d3638a amzn2: ami-a9d09ed1 eu-west-1: amzn: ami-d834aba1 amzn2: ami-466768ac eu-central-1: amzn: ami-5652ce39 amzn2: ami-7c4f7097 ap-southeast-2: amzn: ami-942dd1f6 amzn2: ami-39f8215b ap-northeast-1: amzn: ami-ceafcba8 amzn2: ami-e99f4896 ap-northeast-2: amzn: ami-863090e8 amzn2: ami-afd86dc1 Resources: InstanceProfile: Type: AWS::IAM::InstanceProfile Properties: Path: / Roles: - !Ref InstanceRole InstanceRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Principal: Service: - ec2.amazonaws.com Action: - sts:AssumeRole Path: / Policies: - PolicyName: efs-performance-tutorial PolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Action: - ec2:CreateTags - ec2:DescribeAvailabilityZones - ec2:DescribeNetworkInterfaceAttribute - ec2:DescribeNetworkInterfaces - ec2:DescribeSecurityGroups - ec2:DescribeSubnets - ec2:DescribeVpcAttribute - ec2:DescribeVpcs - FileSystemId:Describe* - kms:ListAliases Resource: '*' AutoScalingGroup1: Type: AWS::AutoScaling::AutoScalingGroup Properties: Cooldown: 60 HealthCheckGracePeriod: 120 HealthCheckType: EC2 LaunchConfigurationName: !Ref LaunchConfiguration1 MaxSize: 1 MinSize: 1 Tags: - Key: Name Value: 'EFS Performance Tutorial - Launching...' PropagateAtLaunch: true VPCZoneIdentifier: !If [ NumberOfSubnets1, [ !Select [ 0, !Ref Subnet ] ], !If [ NumberOfSubnets2, [ !Select [ 0, !Ref Subnet ], !Select [ 1, !Ref Subnet ] ], !If [ NumberOfSubnets3, [ !Select [ 0, !Ref Subnet ], !Select [ 1, !Ref Subnet ], !Select [ 2, !Ref Subnet ] ], !If [ NumberOfSubnets4, [ !Select [ 0, !Ref Subnet ], !Select [ 1, !Ref Subnet ], !Select [ 2, !Ref Subnet ], !Select [ 3, !Ref Subnet ] ], !If [ NumberOfSubnets5, [ !Select [ 0, !Ref Subnet ], !Select [ 1, !Ref Subnet ], !Select [ 2, !Ref Subnet ], !Select [ 3, !Ref Subnet ], !Select [ 4, !Ref Subnet ] ], [ !Select [ 0, !Ref Subnet ], !Select [ 1, !Ref Subnet ], !Select [ 2, !Ref Subnet ], !Select [ 3, !Ref Subnet ], !Select [ 4, !Ref Subnet ], !Select [ 5, !Ref Subnet ] ] ] ] ] ] ] CreationPolicy: ResourceSignal: Count: 0 Timeout: PT5M LaunchConfiguration1: Type: AWS::AutoScaling::LaunchConfiguration Metadata: AWS::CloudFormation::Init: configSets: install_all_utilities: - tools-install - smallfile-install - fpart-install tools-install: commands: misctools-install: command: yum --enablerepo=epel install nload sysstat ioping -y ignoreErrors: false devtools-install: command: yum groupinstall "Development Tools" -y smallfile-install: files: /tmp/smallfile-install.sh: content: !Sub | #!/bin/bash -xe cd /home/ec2-user/ git clone https://github.com/bengland2/smallfile.git mode: 000777 owner: ec2-user group: ec2-user commands: smallfile-install: command: su -l ec2-user /tmp/smallfile-install.sh ignoreErrors: false fpart-install: files: /tmp/fpart-install.sh: content: !Sub | #!/bin/bash -xe cd /home/ec2-user/ git clone https://github.com/martymac/fpart.git cd /home/ec2-user/fpart autoreconf -i ./configure make sudo make install mode: 000777 owner: ec2-user group: ec2-user commands: fpart-install: command: su -l ec2-user /tmp/fpart-install.sh ignoreErrors: false Properties: BlockDeviceMappings: - DeviceName: /dev/xvda Ebs: DeleteOnTermination: true VolumeSize: 40 VolumeType: gp2 IamInstanceProfile: !Ref InstanceProfile ImageId: !FindInMap [ RegionMap, !Ref 'AWS::Region', !Ref LinuxDistro1 ] InstanceMonitoring: true InstanceType: !Ref InstanceType1 KeyName: !Ref KeyName SecurityGroups: - !Ref SecurityGroup UserData: "Fn::Base64": !Join [ "", [ "#cloud-config\n", "repo_update: true\n", "repo_upgrade: all\n", "\n", "packages:\n", "- parallel\n", !If [ RepoInstall, "- amazon-efs-utils\n", "" ], "- git\n", "- gcc\n", "\n", "runcmd:\n", "- instance_id=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)\n", !If [ BuildInstall, !Join [ "", [ "- git clone https://github.com/aws/efs-utils.git\n", ] ], "\n" ], "- cd /efs-utils\n", "- yum -y install rpm-build \n- make rpm \n- yum -y install build/amazon-efs-utils*rpm\n", "- efs_mount_point=/mnt/efs/01\n", "- mkdir -p ${efs_mount_point}\n", !If [ MountOnBoot3, !Join [ "", [ "- echo \"", !Ref FileSystemId, " ${efs_mount_point} efs _netdev", !If [ MountTls3, ",tls", "" ], " 0 0\" >> /etc/fstab \n- mount -a -t efs\n", "- mkdir -p ${efs_mount_point}/tutorial/touch/${instance_id}\n", "- mkdir -p ${efs_mount_point}/tutorial/dd/${instance_id}\n", "- mkdir -p ${efs_mount_point}/tutorial/cp/${instance_id}\n", "- mkdir -p ${efs_mount_point}/tutorial/rsync/${instance_id}\n", "- mkdir -p ${efs_mount_point}/tutorial/parallelcp/${instance_id}\n", "- mkdir -p ${efs_mount_point}/tutorial/fpsync/${instance_id}\n", "- mkdir -p ${efs_mount_point}/tutorial/parallelcpio/${instance_id}\n", "- chown ec2-user:ec2-user ${efs_mount_point}/tutorial/ -R\n", "\n" ] ], "\n" ], "- ebs_directory=/ebs\n", "- mkdir =p ${ebs_directory}\n", "- mkdir -p ${ebs_directory}/tutorial/dd\n", "- mkdir -p ${ebs_directory}/tutorial/smallfile\n", "- mkdir -p ${ebs_directory}/tutorial/data-1m\n", "- chown ec2-user:ec2-user ${ebs_directory}/tutorial/ -R\n", "\n", "- /opt/aws/bin/cfn-init --configsets install_all_utilities --verbose --stack ", !Ref 'AWS::StackName', " --resource LaunchConfiguration1 --region ", !Ref 'AWS::Region',"\n", "- python /home/ec2-user/smallfile/smallfile_cli.py --operation create --threads 10 --file-size 1024 --file-size-distribution exponential --files 500 --same-dir N --dirs-per-dir 1024 --hash-into-dirs Y --files-per-dir 10240 --top ${ebs_directory}/tutorial/smallfile\n", "- cp -R ${ebs_directory}/tutorial/smallfile/file_srcdir ${ebs_directory}/tutorial/data-1m/\n", "- aws ec2 create-tags --resources ${instance_id} --tags \"Key=Name,Value=EFS Performance Tutorial - Ready\" --region ", !Ref 'AWS::Region',"\n" ] ] AutoScalingGroup2: Type: AWS::AutoScaling::AutoScalingGroup Properties: Cooldown: 60 HealthCheckGracePeriod: 120 HealthCheckType: EC2 LaunchConfigurationName: !Ref LaunchConfiguration2 MaxSize: 1 MinSize: 1 Tags: - Key: Name Value: 'EFS Performance Tutorial - Launching...' PropagateAtLaunch: true VPCZoneIdentifier: !If [ NumberOfSubnets1, [ !Select [ 0, !Ref Subnet ] ], !If [ NumberOfSubnets2, [ !Select [ 0, !Ref Subnet ], !Select [ 1, !Ref Subnet ] ], !If [ NumberOfSubnets3, [ !Select [ 0, !Ref Subnet ], !Select [ 1, !Ref Subnet ], !Select [ 2, !Ref Subnet ] ], !If [ NumberOfSubnets4, [ !Select [ 0, !Ref Subnet ], !Select [ 1, !Ref Subnet ], !Select [ 2, !Ref Subnet ], !Select [ 3, !Ref Subnet ] ], !If [ NumberOfSubnets5, [ !Select [ 0, !Ref Subnet ], !Select [ 1, !Ref Subnet ], !Select [ 2, !Ref Subnet ], !Select [ 3, !Ref Subnet ], !Select [ 4, !Ref Subnet ] ], [ !Select [ 0, !Ref Subnet ], !Select [ 1, !Ref Subnet ], !Select [ 2, !Ref Subnet ], !Select [ 3, !Ref Subnet ], !Select [ 4, !Ref Subnet ], !Select [ 5, !Ref Subnet ] ] ] ] ] ] ] CreationPolicy: ResourceSignal: Count: 0 Timeout: PT5M LaunchConfiguration2: Type: AWS::AutoScaling::LaunchConfiguration Metadata: AWS::CloudFormation::Init: configSets: install_all_utilities: - tools-install - smallfile-install - fpart-install tools-install: commands: misctools-install: command: yum --enablerepo=epel install nload sysstat ioping -y ignoreErrors: false devtools-install: command: yum groupinstall "Development Tools" -y smallfile-install: files: /tmp/smallfile-install.sh: content: !Sub | #!/bin/bash -xe cd /home/ec2-user/ git clone https://github.com/bengland2/smallfile.git mode: 000777 owner: ec2-user group: ec2-user commands: smallfile-install: command: su -l ec2-user /tmp/smallfile-install.sh ignoreErrors: false fpart-install: files: /tmp/fpart-install.sh: content: !Sub | #!/bin/bash -xe cd /home/ec2-user/ git clone https://github.com/martymac/fpart.git cd /home/ec2-user/fpart autoreconf -i ./configure make sudo make install mode: 000777 owner: ec2-user group: ec2-user commands: fpart-install: command: su -l ec2-user /tmp/fpart-install.sh ignoreErrors: false Properties: BlockDeviceMappings: - DeviceName: /dev/xvda Ebs: DeleteOnTermination: true VolumeSize: 40 VolumeType: gp2 IamInstanceProfile: !Ref InstanceProfile ImageId: !FindInMap [ RegionMap, !Ref 'AWS::Region', !Ref LinuxDistro2 ] InstanceMonitoring: true InstanceType: !Ref InstanceType2 KeyName: !Ref KeyName SecurityGroups: - !Ref SecurityGroup UserData: "Fn::Base64": !Join [ "", [ "#cloud-config\n", "repo_update: true\n", "repo_upgrade: all\n", "\n", "packages:\n", "- parallel\n", !If [ RepoInstall, "- amazon-efs-utils\n", "" ], "- git\n", "- gcc\n", "\n", "runcmd:\n", "- instance_id=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)\n", !If [ BuildInstall, !Join [ "", [ "- git clone https://github.com/aws/efs-utils.git\n", ] ], "\n" ], "- cd /efs-utils\n", "- yum -y install rpm-build \n- make rpm \n- yum -y install build/amazon-efs-utils*rpm\n", "- efs_mount_point=/mnt/efs/01\n", "- mkdir -p ${efs_mount_point}\n", !If [ MountOnBoot3, !Join [ "", [ "- echo \"", !Ref FileSystemId, " ${efs_mount_point} efs _netdev", !If [ MountTls3, ",tls", "" ], " 0 0\" >> /etc/fstab \n- mount -a -t efs\n", "- mkdir -p ${efs_mount_point}/tutorial/touch/${instance_id}\n", "- mkdir -p ${efs_mount_point}/tutorial/dd/${instance_id}\n", "- mkdir -p ${efs_mount_point}/tutorial/cp/${instance_id}\n", "- mkdir -p ${efs_mount_point}/tutorial/rsync/${instance_id}\n", "- mkdir -p ${efs_mount_point}/tutorial/parallelcp/${instance_id}\n", "- mkdir -p ${efs_mount_point}/tutorial/fpsync/${instance_id}\n", "- mkdir -p ${efs_mount_point}/tutorial/parallelcpio/${instance_id}\n", "- chown ec2-user:ec2-user ${efs_mount_point}/tutorial/ -R\n", "\n" ] ], "\n" ], "- ebs_directory=/ebs\n", "- mkdir =p ${ebs_directory}\n", "- mkdir -p ${ebs_directory}/tutorial/dd\n", "- mkdir -p ${ebs_directory}/tutorial/smallfile\n", "- mkdir -p ${ebs_directory}/tutorial/data-1m\n", "- chown ec2-user:ec2-user ${ebs_directory}/tutorial/ -R\n", "\n", "- /opt/aws/bin/cfn-init --configsets install_all_utilities --verbose --stack ", !Ref 'AWS::StackName', " --resource LaunchConfiguration2 --region ", !Ref 'AWS::Region',"\n", "- python /home/ec2-user/smallfile/smallfile_cli.py --operation create --threads 10 --file-size 1024 --file-size-distribution exponential --files 500 --same-dir N --dirs-per-dir 1024 --hash-into-dirs Y --files-per-dir 10240 --top ${ebs_directory}/tutorial/smallfile\n", "- cp -R ${ebs_directory}/tutorial/smallfile/file_srcdir ${ebs_directory}/tutorial/data-1m/\n", "- aws ec2 create-tags --resources ${instance_id} --tags \"Key=Name,Value=EFS Performance Tutorial - Ready\" --region ", !Ref 'AWS::Region',"\n" ] ] AutoScalingGroup3: Type: AWS::AutoScaling::AutoScalingGroup Properties: Cooldown: 60 HealthCheckGracePeriod: 120 HealthCheckType: EC2 LaunchConfigurationName: !Ref LaunchConfiguration3 MaxSize: 1 MinSize: 1 Tags: - Key: Name Value: 'EFS Performance Tutorial - Launching...' PropagateAtLaunch: true VPCZoneIdentifier: !If [ NumberOfSubnets1, [ !Select [ 0, !Ref Subnet ] ], !If [ NumberOfSubnets2, [ !Select [ 0, !Ref Subnet ], !Select [ 1, !Ref Subnet ] ], !If [ NumberOfSubnets3, [ !Select [ 0, !Ref Subnet ], !Select [ 1, !Ref Subnet ], !Select [ 2, !Ref Subnet ] ], !If [ NumberOfSubnets4, [ !Select [ 0, !Ref Subnet ], !Select [ 1, !Ref Subnet ], !Select [ 2, !Ref Subnet ], !Select [ 3, !Ref Subnet ] ], !If [ NumberOfSubnets5, [ !Select [ 0, !Ref Subnet ], !Select [ 1, !Ref Subnet ], !Select [ 2, !Ref Subnet ], !Select [ 3, !Ref Subnet ], !Select [ 4, !Ref Subnet ] ], [ !Select [ 0, !Ref Subnet ], !Select [ 1, !Ref Subnet ], !Select [ 2, !Ref Subnet ], !Select [ 3, !Ref Subnet ], !Select [ 4, !Ref Subnet ], !Select [ 5, !Ref Subnet ] ] ] ] ] ] ] CreationPolicy: ResourceSignal: Count: 0 Timeout: PT5M LaunchConfiguration3: Type: AWS::AutoScaling::LaunchConfiguration Metadata: AWS::CloudFormation::Init: configSets: install_all_utilities: - tools-install - smallfile-install - fpart-install tools-install: commands: misctools-install: command: yum --enablerepo=epel install nload sysstat ioping -y ignoreErrors: false devtools-install: command: yum groupinstall "Development Tools" -y smallfile-install: files: /tmp/smallfile-install.sh: content: !Sub | #!/bin/bash -xe cd /home/ec2-user/ git clone https://github.com/bengland2/smallfile.git mode: 000777 owner: ec2-user group: ec2-user commands: smallfile-install: command: su -l ec2-user /tmp/smallfile-install.sh ignoreErrors: false fpart-install: files: /tmp/fpart-install.sh: content: !Sub | #!/bin/bash -xe cd /home/ec2-user/ git clone https://github.com/martymac/fpart.git cd /home/ec2-user/fpart autoreconf -i ./configure make sudo make install mode: 000777 owner: ec2-user group: ec2-user commands: fpart-install: command: su -l ec2-user /tmp/fpart-install.sh ignoreErrors: false Properties: BlockDeviceMappings: - DeviceName: /dev/xvda Ebs: DeleteOnTermination: true VolumeSize: 40 VolumeType: gp2 IamInstanceProfile: !Ref InstanceProfile ImageId: !FindInMap [ RegionMap, !Ref 'AWS::Region', !Ref LinuxDistro3 ] InstanceMonitoring: true InstanceType: !Ref InstanceType3 KeyName: !Ref KeyName SecurityGroups: - !Ref SecurityGroup UserData: "Fn::Base64": !Join [ "", [ "#cloud-config\n", "repo_update: true\n", "repo_upgrade: all\n", "\n", "packages:\n", "- parallel\n", !If [ RepoInstall, "- amazon-efs-utils\n", "" ], "- git\n", "- gcc\n", "\n", "runcmd:\n", "- instance_id=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)\n", !If [ BuildInstall, !Join [ "", [ "- git clone https://github.com/aws/efs-utils.git\n", ] ], "\n" ], "- cd /efs-utils\n", "- yum -y install rpm-build \n- make rpm \n- yum -y install build/amazon-efs-utils*rpm\n", "- efs_mount_point=/mnt/efs/01\n", "- mkdir -p ${efs_mount_point}\n", !If [ MountOnBoot3, !Join [ "", [ "- echo \"", !Ref FileSystemId, " ${efs_mount_point} efs _netdev", !If [ MountTls3, ",tls", "" ], " 0 0\" >> /etc/fstab \n- mount -a -t efs\n", "- mkdir -p ${efs_mount_point}/tutorial/touch/${instance_id}\n", "- mkdir -p ${efs_mount_point}/tutorial/dd/${instance_id}\n", "- mkdir -p ${efs_mount_point}/tutorial/cp/${instance_id}\n", "- mkdir -p ${efs_mount_point}/tutorial/rsync/${instance_id}\n", "- mkdir -p ${efs_mount_point}/tutorial/parallelcp/${instance_id}\n", "- mkdir -p ${efs_mount_point}/tutorial/fpsync/${instance_id}\n", "- mkdir -p ${efs_mount_point}/tutorial/parallelcpio/${instance_id}\n", "- chown ec2-user:ec2-user ${efs_mount_point}/tutorial/ -R\n", "\n" ] ], "\n" ], "- ebs_directory=/ebs\n", "- mkdir =p ${ebs_directory}\n", "- mkdir -p ${ebs_directory}/tutorial/dd\n", "- mkdir -p ${ebs_directory}/tutorial/smallfile\n", "- mkdir -p ${ebs_directory}/tutorial/data-1m\n", "- chown ec2-user:ec2-user ${ebs_directory}/tutorial/ -R\n", "\n", "- /opt/aws/bin/cfn-init --configsets install_all_utilities --verbose --stack ", !Ref 'AWS::StackName', " --resource LaunchConfiguration3 --region ", !Ref 'AWS::Region',"\n", "- python /home/ec2-user/smallfile/smallfile_cli.py --operation create --threads 10 --file-size 1024 --file-size-distribution exponential --files 500 --same-dir N --dirs-per-dir 1024 --hash-into-dirs Y --files-per-dir 10240 --top ${ebs_directory}/tutorial/smallfile\n", "- cp -R ${ebs_directory}/tutorial/smallfile/file_srcdir ${ebs_directory}/tutorial/data-1m/\n", "- aws ec2 create-tags --resources ${instance_id} --tags \"Key=Name,Value=EFS Performance Tutorial - Ready\" --region ", !Ref 'AWS::Region',"\n" ] ]