#!/bin/bash # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 OR ISC set -exuo pipefail # -e: Exit on any failure # -x: Print the command before running # -u: Any variable that is not set will cause an error if used # -o pipefail: Makes sure to exit a pipeline with a non-zero error code if any command in the pipeline exists with a # non-zero error code. function delete_s3_buckets() { aws s3api list-buckets --query "Buckets[].Name" | jq '.[]' | while read -r i; do bucket_name=$(echo "${i}" | tr -d '"') # Delete the bucket if its name uses AWS_LC_S3_BUCKET_PREFIX. if [[ "${bucket_name}" == *"${AWS_LC_S3_BUCKET_PREFIX}"* ]]; then aws s3 rm "s3://${bucket_name}" --recursive aws s3api delete-bucket --bucket "${bucket_name}" # Delete bm-framework buckets if we're not on the team account elif [[ "${CDK_DEPLOY_ACCOUNT}" != "620771051181" ]] && [[ "${bucket_name}" == *"${aws-lc-ci-bm-framework}"* ]]; then aws s3 rm "s3://${bucket_name}" --recursive aws s3api delete-bucket --bucket "${bucket_name}" fi done } function delete_container_repositories() { ecr_repos=$(aws ecr describe-repositories) if [[ "${ecr_repos}" == *"${ECR_LINUX_AARCH_REPO_NAME}"* ]]; then aws ecr delete-repository --repository-name "${ECR_LINUX_AARCH_REPO_NAME}" --force fi if [[ "${ecr_repos}" == *"${ECR_LINUX_X86_REPO_NAME}"* ]]; then aws ecr delete-repository --repository-name "${ECR_LINUX_X86_REPO_NAME}" --force fi if [[ "${ecr_repos}" == *"${ECR_WINDOWS_X86_REPO_NAME}"* ]]; then aws ecr delete-repository --repository-name "${ECR_WINDOWS_X86_REPO_NAME}" --force fi } function destroy_ci() { if [[ "${CDK_DEPLOY_ACCOUNT}" == "620771051181" ]]; then echo "destroy_ci should not be executed on team account." exit 1 fi cdk destroy aws-lc-* --force # CDK stack destroy does not delete s3 bucket automatically. delete_s3_buckets # CDK stack destroy does not delete ecr automatically. delete_container_repositories } function destroy_docker_img_build_stack() { if [[ "${IMG_BUILD_STATUS}" == "Failed" ]]; then echo "Docker images build failed. AWS resources of building Docker images is kept for debug." exit 1 fi # Destroy all temporary resources created for all docker image build. cdk destroy aws-lc-docker-image-build-* --force # CDK stack destroy does not delete s3 bucket automatically. delete_s3_buckets } function create_linux_docker_img_build_stack() { # Clean up build stacks if exists. destroy_docker_img_build_stack # Deploy aws-lc ci stacks. # When repeatedly deploy, error 'EIP failed Reason: Maximum number of addresses has been reached' can happen. # https://forums.aws.amazon.com/thread.jspa?messageID=952368 # Workaround: go to AWS EIP console, release unused IP. cdk deploy aws-lc-docker-image-build-linux --require-approval never } function create_win_docker_img_build_stack() { # Clean up build stacks if exists. destroy_docker_img_build_stack # Deploy aws-lc ci stacks. # When repeatedly deploy, error 'EIP failed Reason: Maximum number of addresses has been reached' can happen. # https://forums.aws.amazon.com/thread.jspa?messageID=952368 # Workaround: go to AWS EIP console, release unused IP. cdk deploy aws-lc-docker-image-build-windows --require-approval never } function create_github_ci_stack() { cdk deploy aws-lc-ci-* --require-approval never } function run_linux_img_build() { # https://awscli.amazonaws.com/v2/documentation/api/latest/reference/codebuild/start-build-batch.html build_id=$(aws codebuild start-build-batch --project-name aws-lc-docker-image-build-linux | jq -r '.buildBatch.id') export AWS_LC_LINUX_BUILD_BATCH_ID="${build_id}" } function run_windows_img_build() { # EC2 takes several minutes to be ready for running command. echo "Wait 3 min for EC2 ready for SSM command execution." sleep 180 # Run commands on windows EC2 instance to build windows docker images. for i in {1..60}; do instance_id=$(aws ec2 describe-instances \ --filters "Name=tag:${WIN_EC2_TAG_KEY},Values=${WIN_EC2_TAG_VALUE}" | jq -r '.Reservations[0].Instances[0].InstanceId') if [[ "${instance_id}" == "null" ]]; then sleep 60 continue fi instance_ping_status=$(aws ssm describe-instance-information \ --filters "Key=InstanceIds,Values=${instance_id}" | jq -r '.InstanceInformationList[0].PingStatus') if [[ "${instance_ping_status}" == "Online" ]]; then # https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ssm/send-command.html command_id=$(aws ssm send-command \ --instance-ids "${instance_id}" \ --document-name "${WIN_DOCKER_BUILD_SSM_DOCUMENT}" \ --output-s3-bucket-name "${S3_FOR_WIN_DOCKER_IMG_BUILD}" \ --output-s3-key-prefix 'runcommand' | jq -r '.Command.CommandId') # Export for checking command run status. export WINDOWS_DOCKER_IMG_BUILD_COMMAND_ID="${command_id}" echo "Windows ec2 is executing SSM command." return else echo "${i}: Current instance ping status: ${instance_ping_status}. Wait 1 minute to retry SSM command execution." sleep 60 fi done echo "After 30 minutes, Windows ec2 is still not ready for SSM commands execution. Exit." exit 1 } function linux_docker_img_build_status_check() { export IMG_BUILD_STATUS='Failed' # Every 5 min, this function checks if the linux docker image batch code build finished successfully. # Normally, docker img build can take up to 1 hour. Here, we wait up to 30 * 5 min. for i in {1..30}; do # https://docs.aws.amazon.com/cli/latest/reference/codebuild/batch-get-build-batches.html build_batch_status=$(aws codebuild batch-get-build-batches --ids "${AWS_LC_LINUX_BUILD_BATCH_ID}" | jq -r '.buildBatches[0].buildBatchStatus') if [[ ${build_batch_status} == 'SUCCEEDED' ]]; then export IMG_BUILD_STATUS='Success' echo "Code build ${AWS_LC_LINUX_BUILD_BATCH_ID} finished successfully." return elif [[ ${build_batch_status} == 'FAILED' ]]; then echo "Code build ${AWS_LC_LINUX_BUILD_BATCH_ID} failed." exit 1 else echo "${i}: Wait 5 min for docker image build job finish." sleep 300 fi done echo "Code build ${AWS_LC_LINUX_BUILD_BATCH_ID} takes more time than expected." exit 1 } function win_docker_img_build_status_check() { export IMG_BUILD_STATUS='Failed' # Every 5 min, this function checks if the windows docker image build is finished successfully. # Normally, docker img build can take up to 1 hour. Here, we wait up to 30 * 5 min. for i in {1..30}; do # https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ssm/list-commands.html command_run_status=$(aws ssm list-commands --command-id "${WINDOWS_DOCKER_IMG_BUILD_COMMAND_ID}" | jq -r '.Commands[0].Status') if [[ ${command_run_status} == 'Success' ]]; then export IMG_BUILD_STATUS='Success' echo "SSM command ${WINDOWS_DOCKER_IMG_BUILD_COMMAND_ID} finished successfully." return elif [[ ${command_run_status} == 'Failed' ]]; then echo "SSM command ${WINDOWS_DOCKER_IMG_BUILD_COMMAND_ID} failed." exit 1 else echo "${i}: Wait 5 min for windows docker image build job finish." sleep 300 fi done echo "SSM command ${WINDOWS_DOCKER_IMG_BUILD_COMMAND_ID} takes more time than expected." exit 1 } function build_linux_docker_images() { # Always destroy docker build stacks (which include EC2 instance) on EXIT. trap destroy_docker_img_build_stack EXIT # Create/update aws-ecr repo. cdk deploy aws-lc-ecr-linux-* --require-approval never # Create docker image build stack. create_linux_docker_img_build_stack echo "Activating AWS CodeBuild to build Linux aarch & x86 docker images." run_linux_img_build echo "Waiting for docker images creation. Building the docker images need to take 1 hour." # TODO(CryptoAlg-624): These image build may fail due to the Docker Hub pull limits made on 2020-11-01. linux_docker_img_build_status_check } function build_win_docker_images() { # Always destroy docker build stacks (which include EC2 instance) on EXIT. trap destroy_docker_img_build_stack EXIT # Create/update aws-ecr repo. cdk deploy aws-lc-ecr-windows-* --require-approval never # Create aws windows build stack create_win_docker_img_build_stack echo "Executing AWS SSM commands to build Windows docker images." run_windows_img_build echo "Waiting for docker images creation. Building the docker images need to take 1 hour." # TODO(CryptoAlg-624): These image build may fail due to the Docker Hub pull limits made on 2020-11-01. win_docker_img_build_status_check } function setup_ci() { build_linux_docker_images build_win_docker_images create_github_ci_stack create_android_resources } function create_android_resources() { # Use aws cli to create Device Farm project and get project arn to create device pools. # TODO: Move resource creation to aws cdk when cdk has support for device form resource constructs. # Issue: https://github.com/aws/aws-cdk/issues/17893 DEVICEFARM_PROJECT=`aws devicefarm create-project --name aws-lc-android-ci | \ python -c 'import json,sys;obj=json.load(sys.stdin);print(obj["project"]["arn"])'` DEVICEFARM_DEVICE_POOL=`aws devicefarm create-device-pool --project-arn ${DEVICEFARM_PROJECT} \ --name "aws-lc-device-pool" \ --description "AWS-LC Device Pool" \ --rules file://../android/devicepool_rules.json --max-devices 2 | \ python -c 'import json,sys;obj=json.load(sys.stdin);print(obj["devicePool"]["arn"])'` DEVICEFARM_DEVICE_POOL_FIPS=`aws devicefarm create-device-pool --project-arn ${DEVICEFARM_PROJECT} \ --name "aws-lc-device-pool-fips" \ --description "AWS-LC FIPS Device Pool" \ --rules file://../android/devicepool_rules_fips.json --max-devices 2 | \ python -c 'import json,sys;obj=json.load(sys.stdin);print(obj["devicePool"]["arn"])'` cat <