# ================================================================================== # Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. # Permission is hereby granted, free of charge, to any person obtaining a copy of this # software and associated documentation files (the "Software"), to deal in the Software # without restriction, including without limitation the rights to use, copy, modify, # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # ================================================================================== # # stvblogPrepareProcess # by: Rob Dachowski # For questions or feedback, please contact robdac@amazon.com # # Purpose: The labmda does the initial setup for running the process # # Change Log: # 3/1/21: Initial version # # ================================================================================== import json import uuid import boto3 from botocore.exceptions import ClientError import ssmparms as sp import stmparms as stm import stverrors print( "BOTO3 VERSION:", boto3.__version__) # ================================================================================== # Function: labmda_handler # Purpose: This is the "main" code for this lambda function # Parameters: # event - the JSON input structure containing the parameters from the step function process # ================================================================================== def lambda_handler(event, context): #debugging message print("===> stvblogPrepareProcess: " + "\nEvent: " + str(event) ) # set up S3 s = boto3.client( 's3' ) s3 = boto3.resource('s3') # set up Dynamo DB dd = boto3.resource('dynamodb') table = dd.Table("stvblog") # create the unique identifier uid = uuid.uuid4().hex[:6] print( '\t---> UUID: ' + uid ) # Get the values from the parameter store to put in the dynamo table print( '\t---> Reading parameters from Systems Manager Parameter Store') baseBucketName = sp.get_parameter( "/stvblog/baseBucketName")['Parameter']['Value'] mediaInput = sp.get_parameter( "/stvblog/mediaInput")['Parameter']['Value'] transcriptionInput = sp.get_parameter( "/stvblog/transcriptionInput")['Parameter']['Value'] transcriptionOutput = sp.get_parameter( "/stvblog/transcriptionOutput")['Parameter']['Value'] translationInput = sp.get_parameter( "/stvblog/translationInput")['Parameter']['Value'] translationOutput = sp.get_parameter( "/stvblog/translationOutput")['Parameter']['Value'] subtitleInput = sp.get_parameter( "/stvblog/subtitleInput")['Parameter']['Value'] subtitleOutput = sp.get_parameter( "/stvblog/subtitleOutput")['Parameter']['Value'] pollyInput = sp.get_parameter( "/stvblog/pollyInput")['Parameter']['Value'] pollyOutput = sp.get_parameter( "/stvblog/pollyOutput")['Parameter']['Value'] finalInput = sp.get_parameter( "/stvblog/finalInput")['Parameter']['Value'] finalOutput = sp.get_parameter( "/stvblog/finalOutput")['Parameter']['Value'] region = sp.get_parameter( "/stvblog/region")['Parameter']['Value'] dataAccessRoleARN = sp.get_parameter( "/stvblog/translateDataAccessRoleARN")['Parameter']['Value'] # Check that we got everything we expected from the Parameter Store.... if not (baseBucketName and mediaInput and transcriptionInput and transcriptionOutput and translationInput and translationOutput and subtitleInput and subtitleOutput and pollyInput and pollyOutput and finalInput and finalOutput): # if we got here, then something went wrong reading the parameters from Systems Manager raise stvParameterStoreError( "*** Parameters not read from SSM correctly. Aborting! ***\n" ) else: print( '\t---> Parameters read successfully') # Create the folder structure by appending the uuid to the base bucket and include other processing folders # The Structure is: # baseBucketName + uuid # |_ mediaInput # |_ transcriptionInput # |_ transcriptionOutput # |_ translationInput # |_ translationOutput # |_ subtitleInput # |_ subtitleOutput # |_ pollyInput # |_ pollyOutput # |_ finalInput # |_ finalOutput print( '\t---> Creating folder structure in ', baseBucketName ) result = s.put_object(Bucket=baseBucketName, Key=uid + '/' + mediaInput + '/') result = s.put_object(Bucket=baseBucketName, Key=uid + '/' + transcriptionInput+ '/') result = s.put_object(Bucket=baseBucketName, Key=uid + '/' + transcriptionOutput + '/') result = s.put_object(Bucket=baseBucketName, Key=uid + '/' + translationInput + '/') result = s.put_object(Bucket=baseBucketName, Key=uid + '/' + translationOutput + '/') result = s.put_object(Bucket=baseBucketName, Key=uid + '/' + subtitleInput + '/') result = s.put_object(Bucket=baseBucketName, Key=uid + '/' + subtitleOutput + '/') result = s.put_object(Bucket=baseBucketName, Key=uid + '/' + pollyInput+ '/') result = s.put_object(Bucket=baseBucketName, Key=uid + '/' + pollyOutput + '/') result = s.put_object(Bucket=baseBucketName, Key=uid + '/' + finalInput + '/') result = s.put_object(Bucket=baseBucketName, Key=uid + '/' + finalOutput + '/') print( '\t---> Folder structure created in ', baseBucketName ) # Create the sub folders needed for the target language processing for lang in event['Targets']: result = s.put_object(Bucket=baseBucketName, Key=uid + '/' + translationOutput + '/' + lang['translate']['targetLanguageShort'] + '/') print( '\t---> Added language specific folder structure in ', uid + '/' + translationOutput + '/' + lang['translate']['targetLanguageShort'] + '/' ) # Copy the input file to the mediaInput "folder" source = {"Bucket": event['Inputs']['inputMediaBucket'], "Key": event['Inputs']['inputMediaKey'] } destination = s3.Bucket( baseBucketName ) destinationKey = uid + '/' + mediaInput + '/' + event['Inputs']['mediaFile'] print( "\t---> Copying ", str(source), " to ", str(destination) + "/" + destinationKey) destination.copy( source, destinationKey) # copy the input file to the finalInput as well... destinationKey = uid + '/' + finalInput + '/' + event['Inputs']['mediaFile'] print( "\t---> Copying ", str(source), " to ", str(destination) + "/" + destinationKey) destination.copy( source, destinationKey) # set up the mediaUri mediaUri = "https://" + "s3-" + region + ".amazonaws.com/" + baseBucketName + '/' + uid + '/' + mediaInput + '/' + event['Inputs']['mediaFile'] print( "\t---> mediaURI: " + mediaUri ) # set up parms for the overall Process. The path is defined in the Step Function response = {} response['Config'] = {} response['Config']['baseBucketName'] = baseBucketName response['Config']['mediaInput'] = mediaInput response['Config']['transcriptionInput'] = transcriptionInput response['Config']['transcriptionOutput'] = transcriptionOutput response['Config']['translationInput'] = translationInput response['Config']['translationOutput'] = translationOutput response['Config']['subtitleInput'] = subtitleInput response['Config']['subtitleOutput'] = subtitleOutput response['Config']['pollyInput'] = pollyInput response['Config']['pollyOutput'] = pollyOutput response['Config']['finalInput'] = finalInput response['Config']['finalOutput'] = finalOutput response['Config']['region'] = region response['Config']['translateDataAccessRoleARN'] = dataAccessRoleARN response['Inputs'] = event['Inputs'] response['Inputs']['mediaURI'] = mediaUri response['Outputs'] = {} response['Outputs']['process'] = {} response['Outputs']['process']['uuid'] = uid response['Outputs']['process']['ProcessName'] = "process-" + uid response['Outputs']['process']['status'] = "IN_PROGRESS" response['Targets'] = event['Targets'] # Finally, put the parameters in DynamoDB. We will update things there as we progress if stm.put_new_stm_parms( response['Outputs']['process']['ProcessName'], response['Config'], response['Inputs'],response['Outputs'], event['Targets'] ): print('===> stvblogPrepareProcessing Complete') return response else: raise stvError( "*** Error writing to the stvblog table ***" )