#!/bin/bash # /* # __author__ = "Srikanth Kodali" # edited = "Jack Tanny and Joyson Neville Lewis " # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: MIT-0 # */ # MUST BE RUN AS ROOT # MUST FIRST PASS DEV_IOT_THING / DEV_IOT_THING_GROUP AND AWS_ACCOUNT_NUMBER TO THE SCRIPT WHEN RUNNING IT # MUST INSTALL SIGIL # Purpose: This bash script deploys an AWS IoT Greengrass component to AWS IoT Thing(s) which are a part of an AWS IoT Thing Group. The script takes inputs such as the component artifacts, recipe template, and configuration details, and then creates the component and deploys it on the Thing(s). # Pre-Requisites: Greengrass must be installed on the edge device, the component artifact files must be in the SRC_FOLDER on the Linux machine running this script, and a recipe-file-template.yaml file must exist on the Linux machine for the component you are creating and deploying. One of the best ways to accomplish this is to clone the repository where these items are kept, or pull recent changes/updates if the repository is already cloned on the Linux machine you are using. # This version has been adapted to run in code deploy on a linux 2 ubuntu instance. # Here you can set environment variables _setEnv() { AWS="aws" # Set either the AWS Greengrass Thing or Thing Group in which you are deploying a component #### WHEN SWITCHING BETWEEN USING THIS SCRIPT FOR THINGS, AND THING GROUPS, CHECK THAT EACH FUNCTION CALLS THE CORRECT DEV_IOT_THING* VARIABLE. I suggest using ctrl+f for this check. echo "DEV_IOT_THING_GROUP: ${DEV_IOT_THING_GROUP}" # the Greengrass Thing group which you want to deploy a component to. All Things in Thing Group will get component. # echo "DEV_IOT_THING: ${DEV_IOT_THING}" # the Greengrass Thing you want to deploy a component on echo "AWS_ACCOUNT_NUMBER: ${AWS_ACCOUNT_NUMBER}" # account number of the AWS account that holds the AWS IoT Greengrass resources # ROLE_ARN="arn:aws:iam::593512547852:role/admin" # Set the role ARN for use by STS AWS_REGION="us-west-2" # AWS region where the Greengrass Resources are deployed # CURRENT_VERSION_NUMBER="1.0.0" # Set the current version of the component here # START_VERSION_NUMBER="1.0.0" # Set the first version of the component here # COMPONENT_NAME="influxdb-subscriber" # Name of the component that is being deployed. Also serves as a Prefix for S3_BUCKET and is the prefix for S3_PATH. # export LC_CTYPE=C #If you run on mac < bigsur # export LC_ALL=C # If you run on mac bigsur # RANDOM_SUFFIX=`cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 8 | head -n 1` # create a random suffix for the new S3 bucket. This makes sure each version of the component lives in its own S3 bucket # echo ${RANDOM_SUFFIX} # S3_BUCKET="ggv2-mqtt-to-sitewise-component-${RANDOM_SUFFIX}" # S3 bucket where deployed component will live # S3_KEY="artifacts" # Prefix in S3_BUCKET denoting the component artifacts and other associated items # S3_PATH="s3://${S3_BUCKET}/${S3_KEY}/${COMPONENT_NAME}" # SRC_FOLDER="src" # folder name of the source directory. All items in the source directory will be zipped to create the component. # ARTIFACTS_ARCHIVE_FILE_NAME="influxdb-subscriber" # the file name of the artifacts that will be zipped up. MAIN_ARTIFACT_FILE="main.py" # the artifact file on the Thing DEPLOYMENT_CONFIG_FILE="deployment_configuration.json" # the configuration file that will be created using Sigil in the _prepare_deployment_config_file_based_on_deployment_name ## If using a model, specify these variables, edit the Sigil command in MAIN, and use a recipe template similar to https://github.com/aws-samples/use-influxdb-and-grafana-to-visualize-ml-output-with-awsiot-greengrass-v2/blob/main/modules/edge-inference/aws-gg-deploy/recipe-file-template.yaml #COMPILATION_NAME="" #MODEL_NAME='' #NEO_COMPILED_MODEL_PATH="s3://$S3_BUCKET/$COMPILATION_NAME/output/model-LINUX_X86_64.zip" } # Sigil is a command line tool that helps us create configuration templates. Read more about it on GitHub _installSigil() { if ! [ -x "$(command -v sigil)" ]; then echo 'Error: sigil is not installed.' >&2 echo "https://github.com/gliderlabs/sigil/releases/download/v0.6.0/sigil_0.6.0_$(uname -sm|tr \ _).tgz" curl -kL "https://github.com/gliderlabs/sigil/releases/download/v0.6.0/sigil_0.6.0_$(uname -sm|tr \ _).tgz" | tar -zxC /usr/local/bin which sigil else echo "Sigil is already installed." fi } # jq is a Linux command line utility that is easily used to extract data from JSON documents. _check_if_jq_exists() { JQ=`which jq` if [ $? -eq 0 ]; then # checking if JQ (represented by $?) exists echo "JQ exists." else echo "jq does not exists, please install it." echo "EXITING the program." exit 1; fi } # This function checks if a bucket with the prefix already exists. If it doesn't exist, it creates the bucket using the AWS CLI. Then, the S3_PATH variable is created as a concatenation of S3_BUCKET + S3_KEY + COMPONENT_NAME _checkIfABucketWithPrefixExists() { bucket_list="" bucket_list=`aws s3 ls | awk '{print $3}'` # lists S3 buckets if [[ ${bucket_list[@]} ]]; then # if aws returns bucket list echo ${bucket_list[@]} if printf '%s\n' "${bucket_list[@]}" | grep 'ggv2-mqtt-to-sitewise-component-'; then # if a bucket exists with 'ggv2-mqtt-to-sitewise-component-' in the name echo "Bucket with prefix exists." S3_BUCKET=`printf '%s\n' "${bucket_list[@]}" | grep 'ggv2-mqtt-to-sitewise-component-'` # set S3_BUCKET as the bucket name of the bucket with prefix 'ggv2-mqtt-to-sitewise-component-' echo $S3_BUCKET S3_KEY="artifacts" # Prefix in S3_BUCKET denoting the component artifacts and other associated items S3_PATH="s3://${S3_BUCKET}/${S3_KEY}/${COMPONENT_NAME}" # creating the path, using the bucket name that already exists echo "S3_PATH IS: ${S3_PATH}" else echo "Bucket is being created." aws s3api create-bucket --bucket ${S3_BUCKET} --region ${AWS_REGION} --create-bucket-configuration LocationConstraint=us-west-2 # creates S3 bucket if it does exist already, using the RANDOM_SUFFIX so the bucket name is unique fi else echo "Error accessing S3 buckets, please check the error. Exiting." exit 1; fi } # This function will set the NEXT_VERSION variable _getNextVersion() { DELIMETER="." target_index=${2} # 2nd argument of the function. A number to represent the index that is being iterated. (i.e. If you want 1.0.1 to be 1.1.1, set this target_index=1) full_version="" IFS='.' read -r -a full_version <<< "$1" # reads the 1st argument of the function, which should be the CURRENT_VERSION_NUMBER, and separates it on the '.' character # The number of values in $1 tha are delimited by "." For example, CURRENT_VERSION_NUMBER = 1.0.1. # the _getNextVersion() CURRENT_VERSION_NUMBER 2 is called. the for loop will loop 3 times, with index values of 0, 1, and 2. # when index = 2 (the second argument, $2), the value of the index, 1, is increased by 1. then the NEXT_VERSION is set as 1.0.2 for index in ${!full_version[@]}; do if [ $index -eq $2 ]; then # if the index equals the iteration number local value=full_version[$index] value=$((value+1)) # increase the value by one of the iteration number full_version[$index]=$value break fi done NEXT_VERSION=`echo $(IFS=${DELIMETER} ; echo "${full_version[*]}")` echo "Next Version: " ${NEXT_VERSION} } # this function will zip up all the items in the directory passed to it by the first argument, and copy it to S3 in the ${S3_PATH}/${NEXT_VERSION}/ location. The 2nd argument serves as the file name for the zipped file. _compressArtifactsAndPushToCloud() { SRC_FOLDER_ARG=$1 PRJ_ARG=$2 PWDIR=`pwd` echo $PWDIR cd ../${SRC_FOLDER_ARG} echo $PRJ_ARG zip -r ${PRJ_ARG}.zip . ls -ltra ${AWS} s3 cp ${PRJ_ARG}.zip ${S3_PATH}/${NEXT_VERSION}/ # copy to S3 cd ${PWDIR} ls -ltra } # This function creates a Greengrass component, following the recipe passed to it. The recipe is created in MAIN. # See more about the recipe here: https://docs.aws.amazon.com/greengrass/v2/developerguide/component-recipe-reference.html _create_gg_component_in_cloud() { REC_FILE_ARG=$1 #1st argument passed to it. The recipe file contains the components details, dependencies, artifacts, and lifecycles. It is a .yaml file. RECIPE_URI="fileb://${REC_FILE_ARG}" echo "Creating components in the cloud." ARN=$(${AWS} greengrassv2 create-component-version --inline-recipe ${RECIPE_URI} --region ${AWS_REGION} | jq -r ".arn") echo "ARN is : " echo ${ARN} AWS_ACCOUNT_NUM=`echo ${ARN} | cut -d ":" -f 5` echo "AWS Account Number" echo ${AWS_ACCOUNT_NUM} #${AWS} greengrassv2 describe-component --arn "" --region ${AWS_REGION} } # This function creates the configuration file. It is passed the DEPLOY_FILE_ARG argument, which serves as the name of the configuration file returned # by the function. The COMP_NAME is the second argument, which serves as the componentName. For more information about this configuration file, see # https://docs.aws.amazon.com/greengrass/v2/developerguide/gdk-cli-configuration-file.html # If the deployment exists for the DEV_IOT_THING or DEV_IOT_THING_GROUP, the CURRENT_VERSION_NUMBER is incremented to the new version. Otherwise, the CURRENT_VERSION_NUMBER set in the _setEnv() is used as the current version. # This function also returns the NEXT_VERSION variable to be used when creating the new recipe file _prepare_deployment_config_file_based_on_deployment_name() { DEPLOY_FILE_ARG=$1 COMP_NAME=$2 START_VERSION=$3 COMP_VERSION="" # If using a THING_GROUP, uncomment the below line, and comment out the line after it. THING_GROUP_ARN="arn:aws:iot:${AWS_REGION}:${AWS_ACCOUNT_NUMBER}:thinggroup/${DEV_IOT_THING_GROUP}" deployment_id=`aws greengrassv2 list-deployments --target-arn ${THING_GROUP_ARN} | jq -r '.deployments[]' | jq -r .deploymentId` #THING_ARN="arn:aws:iot:${AWS_REGION}:${AWS_ACCOUNT_NUMBER}:thing/${DEV_IOT_THING}" #deployment_id=`aws greengrassv2 list-deployments --target-arn ${THING_ARN} | jq -r '.deployments[]' | jq -r .deploymentId` echo "Deployment Id is : ${deployment_id}" ### Check if the component was created previously and used in other thing groups. EXITING_COMP_VERSION=`aws greengrassv2 list-components | jq -r '.components[] | select(.componentName == "'"${COMP_NAME}"'").latestVersion' | jq -r '.componentVersion'` # any component with COMP_NAME echo "Existing comp version is : ${EXITING_COMP_VERSION}" if [[ ! ${EXITING_COMP_VERSION} = "null" ]]; then # if the component already exists, incredment the CURRENT_VERSION_NUMBER using _getNextVersion method, as this is a new version of that component _getNextVersion ${EXITING_COMP_VERSION} 2 CURRENT_VERSION_NUMBER=${NEXT_VERSION} echo "When existing component found in this account, then CURRENT_VERSION_NUMBER is : ${CURRENT_VERSION_NUMBER}" else # if the component does not already exist, set the CURRENT_VERSION_NUMBER = START_VERSION_NUMBER CURRENT_VERSION_NUMBER=${START_VERSION} fi if [[ -z "${deployment_id}" ]]; then # checks if the deployment ID exists for the DEV_IOT_THING_GROUP by checking if deployment_id is an empty string echo "There is no deployment for this thinggroup : ${DEV_IOT_THING_GROUP} yet." #echo "There is no deployment for this thing : ${DEV_IOT_THING} yet." STR1='{"' STR2=${COMP_NAME} STR3='": {"componentVersion": ' STR4=${CURRENT_VERSION_NUMBER} STR5=',"configurationUpdate":{"reset":[""]}}}' # resets the configuration in the deployment so it will not re-use the old configuration and instead use the default configuration we set NEW_CONFIG_JSON=$STR1$STR2$STR3\"$STR4\"$STR5 # composes a NEW_CONFIG_JSON, following the GDK CLI file format https://docs.aws.amazon.com/greengrass/v2/developerguide/gdk-cli-configuration-file.html#:~:text=configuration%20file%20examples-,GDK%20CLI%20configuration%20file%20format,-When%20you%20define echo ${NEW_CONFIG_JSON} NEXT_VERSION=${CURRENT_VERSION_NUMBER} else # if deployemnt ID exists already _getNextVersion ${CURRENT_VERSION_NUMBER} 2 NEW_CONFIG_JSON=`aws greengrassv2 get-deployment --deployment-id ${deployment_id} | jq .'components' | jq 'del(."$COMP_NAME")' | jq '. += {"'"$COMP_NAME"'": {"componentVersion": "'"$NEXT_VERSION"'","configurationUpdate":{"reset":[""]}}}'` # compose a NEW_CONFIG_JSON, following the GDK CLI file format, using the NEXT_VERSION returned from _getNextVersion 2 lines above fi FINAL_CONFIG_JSON='{"components":'$NEW_CONFIG_JSON'}' echo $(echo "$FINAL_CONFIG_JSON" | jq '.') > ${DEPLOY_FILE_ARG} # creates DEPLOY_FILE_ARG file, named as the first argument passed to the functon, comprised of the contents of the FINAL_CONFIG_JSON variable cat ${DEPLOY_FILE_ARG} | jq # displays contents of the configuration file } # this function simply deploys the configuration to all things in the DEV_IOT_THING_GROUP _deploy_configuration_on_devices() { echo "ARN is in deployment : ${ARN}" CONFIG_FILE_ARG=$1 CONFIG_URI="fileb://${CONFIG_FILE_ARG}" # If using a THING_GROUP, uncomment the below line, and comment out the line after it. THING_GROUP_ARN="arn:aws:iot:${AWS_REGION}:${AWS_ACCOUNT_NUM}:thinggroup/${DEV_IOT_THING_GROUP}" #THING_ARN="arn:aws:iot:${AWS_REGION}:${AWS_ACCOUNT_NUM}:thing/${DEV_IOT_THING}" RES=`${AWS} greengrassv2 create-deployment --target-arn ${THING_GROUP_ARN} --cli-input-json ${CONFIG_URI} --region ${AWS_REGION} --deployment-policies failureHandlingPolicy=DO_NOTHING` #RES=`${AWS} greengrassv2 create-deployment --target-arn ${THING_ARN} --cli-input-json ${CONFIG_URI} --region ${AWS_REGION} --deployment-policies failureHandlingPolicy=DO_NOTHING` echo ${RES} } ########################## MAIN ############################### # ############################################################### # checks the number of arguments passed to the script equals 2. Must pass DEV_IOT_THING and AWS_ACCOUNT_NUMBER echo $# if [ "$#" -ne 2 ]; then echo "usage: ddeploy-mqtt-connector.sh " echo $# exit 1 fi # If using a THING_GROUP, uncomment the below line, and comment out the line after it. DEV_IOT_THING_GROUP=${1} #DEV_IOT_THING=${1} AWS_ACCOUNT_NUMBER=${2} include_public_comps=true _setEnv #_exportAWSCreds _installSigil _check_if_jq_exists #_checkIfABucketWithPrefixExists ## This function checks if a bucket with the prefix 'ggv2-mqtt-to-sitewise-component-' already exists. If it doesn't exist, it creates the bucket using the AWS CLI. Then, the S3_PATH variable is created as a concatenation of S3_BUCKET + S3_KEY + COMPONENT_NAME _prepare_deployment_config_file_based_on_deployment_name ${DEPLOYMENT_CONFIG_FILE} ${COMPONENT_NAME} ${START_VERSION_NUMBER} # creates the configuration file, named DEPLOYMENT_CONFIG_FILE, with the ComponentName as COMPONENT_NAME # this block sets the PREVIOUS_RECIPE_FILE_NAME so it may be removed. It does so based on whether or not the COMP_VERSION exists # RECIPE_FILE_NAME="${COMPONENT_NAME}-${NEXT_VERSION}.yaml" # sets the new RECIPE_FILE_NAME to be written to using sigil below # if [[ ! ${COMP_VERSION} = "" ]]; then # if the component already exists, then COMP_VERSION holds the old version number # PREVIOUS_RECIPE_FILE_NAME="${COMPONENT_NAME}-${COMP_VERSION}.yaml" # fi # if [[ ! ${EXITING_COMP_VERSION} = "" ]]; then # if there is an existing component used by other things/thing groups already # PREVIOUS_RECIPE_FILE_NAME="${COMPONENT_NAME}-${EXITING_COMP_VERSION}.yaml" # fi # echo ${COMPONENT_NAME} # echo ${NEXT_VERSION} # echo ${SRC_FOLDER} # echo ${RECIPE_FILE_NAME} ## Removing old recipe file from the local disk if test -f "${PREVIOUS_RECIPE_FILE_NAME}"; then # if the PREVIOUS_FILE_RECIPE_NAME exists on the disk echo "Previous version recipe file : ${PREVIOUS_RECIPE_FILE_NAME} exists. Removing it from local disk." rm ${PREVIOUS_RECIPE_FILE_NAME} # remove file fi #_compressArtifactsAndPushToCloud ${SRC_FOLDER} ${ARTIFACTS_ARCHIVE_FILE_NAME} # compress the items found in the SRC_FOLDER directory, name the zip file ARCHIVE_ARTIFACTS_FILE_NAME and put the ziped file in the s3 location ${S3_PATH}/${NEXT_VERSION}/ # generating the recipe file using the recipe-file-template.yaml template stored on the Linux machine and Sigil library. #sigil -p -f recipe-file-template.yaml s3_path=${S3_PATH} next_version=${NEXT_VERSION} component_name=${COMPONENT_NAME} component_version_number=${NEXT_VERSION} artifacts_zip_file_name=${ARTIFACTS_ARCHIVE_FILE_NAME} artifacts_entry_file=${MAIN_ARTIFACT_FILE} | tee ${RECIPE_FILE_NAME} > /dev/null echo "======== Generated recipe file is : ========" cat ${RECIPE_FILE_NAME} echo "======== End of recipe file : ==============" _create_gg_component_in_cloud ${RECIPE_FILE_NAME} # creating the AWS IoT Greengrass component based on the RECIPE_FILE_NAME _deploy_configuration_on_devices ${DEPLOYMENT_CONFIG_FILE} # deploys component using configuration file on all things in the DEV_IOT_THING_GROUP