using System;
using System.Collections.Generic;
using System.Data;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Amazon.Common.DotNetCli.Tools;
using Amazon.Common.DotNetCli.Tools.Commands;
using Amazon.Runtime;
using Amazon.S3;
using Amazon.S3.Transfer;
using Newtonsoft.Json.Schema;
using Amazon.Lambda.Tools.Commands;
using Amazon.Common.DotNetCli.Tools.Options;
namespace Amazon.Lambda.Tools.TemplateProcessor
{
///
/// This class is the entry point to traversing a CloudFormation template and looking for any resources that are pointing to local
/// paths. Those local paths are uploaded to S3 and the template is updated with the location in S3. Once this is complete this
/// template can be deployed to CloudFormation to create or update a stack.
///
public class TemplateProcessorManager
{
IToolLogger Logger { get; }
IAmazonS3 S3Client { get; }
///
/// The S3 bucket used to store all local paths in S3.
///
string S3Bucket { get; }
///
/// Prefix for any S3 objects uploaded to S3.
///
string S3Prefix { get; }
///
/// The command that initiated the template processor
///
public LambdaBaseCommand OriginatingCommand { get; }
///
/// Options to use when a local path is pointing to the current directory. This is needed to maintain backwards compatibility
/// with the original version of the deploy-serverless and package-ci commands.
///
DefaultLocationOption DefaultOptions { get; }
public TemplateProcessorManager(LambdaBaseCommand originatingCommand, string s3Bucket, string s3Prefix, DefaultLocationOption defaultOptions)
{
this.OriginatingCommand = originatingCommand;
this.Logger = originatingCommand.Logger;
this.S3Client = originatingCommand.S3Client;
this.S3Bucket = s3Bucket;
this.S3Prefix = s3Prefix;
this.DefaultOptions = defaultOptions;
}
///
/// Transforms the provided template by uploading to S3 any local resources the template is pointing to,
/// like .NET projects for a Lambda project, and then updating the CloudFormation resources to point to the
/// S3 locations.
///
/// The directory where the template was found.
/// The template to search for updatable resources. The file isn't just read from
/// templateDirectory because template substitutions might have occurred before this was executed.
///
public async Task TransformTemplateAsync(string templateDirectory, string templateBody, string[] args)
{
// Remove Project Location switch from arguments list since this should not be used for code base.
string[] modifiedArguments = RemoveProjectLocationArgument(args);
// If templateDirectory is actually pointing the CloudFormation template then grab its root.
if (File.Exists(templateDirectory))
templateDirectory = Path.GetDirectoryName(templateDirectory);
// Maintain a cache of local paths to S3 Keys so if the same local path is referred to for
// multiple Lambda functions it is only built and uploaded once.
var cacheOfLocalPathsToS3Keys = new Dictionary();
var parser = CreateTemplateParser(templateBody);
foreach(var updatableResource in parser.UpdatableResources())
{
this.Logger?.WriteLine($"Processing CloudFormation resource {updatableResource.Name}");
foreach (var field in updatableResource.Fields)
{
var localPath = field.GetLocalPath();
if (localPath == null)
continue;
UpdateResourceResults updateResults;
if (!cacheOfLocalPathsToS3Keys.TryGetValue(localPath, out updateResults))
{
this.Logger?.WriteLine(
$"Initiate packaging of {field.GetLocalPath()} for resource {updatableResource.Name}");
updateResults = await ProcessUpdatableResourceAsync(templateDirectory, field, modifiedArguments);
cacheOfLocalPathsToS3Keys[localPath] = updateResults;
}
else
{
this.Logger?.WriteLine(
$"Using previous upload artifact s3://{this.S3Bucket}/{updateResults.S3Key} for resource {updatableResource.Name}");
}
if(updatableResource.UploadType == CodeUploadType.Zip)
{
field.SetS3Location(this.S3Bucket, updateResults.S3Key);
}
else if(updatableResource.UploadType == CodeUploadType.Image)
{
field.SetImageUri(updateResults.ImageUri);
}
else
{
throw new LambdaToolsException($"Unknown upload type for setting resource: {updatableResource.UploadType}", LambdaToolsException.LambdaErrorCode.ServerlessTemplateParseError);
}
if (!string.IsNullOrEmpty(updateResults.DotnetShareStoreEnv))
{
field.Resource.SetEnvironmentVariable(LambdaConstants.ENV_DOTNET_SHARED_STORE, updateResults.DotnetShareStoreEnv);
}
}
}
var newTemplate = parser.GetUpdatedTemplate();
return newTemplate;
}
///
/// Determine the action to be done for the local path, like building a .NET Core package, then uploading the
/// package to S3. The S3 key is returned to be updated in the template.
///
///
///
///
///
private async Task ProcessUpdatableResourceAsync(string templateDirectory, IUpdateResourceField field, string[] args)
{
UpdateResourceResults results;
var localPath = field.GetLocalPath();
if (!field.IsImagePushed && !Path.IsPathRooted(localPath))
localPath = Path.Combine(templateDirectory, localPath);
bool deleteArchiveAfterUploaded = false;
// If ImageUri needs to be processed.
if (field.IsImagePushed)
{
results = new UpdateResourceResults { ImageUri = localPath };
}
// Uploading a single file as the code for the resource. If the single file is not a zip file then zip the file first.
else if (File.Exists(localPath))
{
if(field.IsCode && !string.Equals(Path.GetExtension(localPath), ".zip", StringComparison.OrdinalIgnoreCase))
{
this.Logger.WriteLine($"Creating zip archive for {localPath} file");
results = new UpdateResourceResults { ZipArchivePath = GenerateOutputZipFilename(field) };
LambdaPackager.BundleFiles(results.ZipArchivePath, Path.GetDirectoryName(localPath), new string[] { localPath }, this.Logger);
}
else
{
results = new UpdateResourceResults { ZipArchivePath = localPath };
}
}
// If IsCode is false then the local path needs to point to a file and not a directory. When IsCode is true
// it can point either to a file or a directory.
else if (!field.IsCode && !File.Exists(localPath))
{
throw new LambdaToolsException($"File that the field {field.Resource.Name}/{field.Name} is pointing to doesn't exist", LambdaToolsException.LambdaErrorCode.ServerlessTemplateMissingLocalPath);
}
else if (!Directory.Exists(localPath))
{
throw new LambdaToolsException($"Directory that the field {field.Resource.Name}/{field.Name} is pointing doesn't exist", LambdaToolsException.LambdaErrorCode.ServerlessTemplateMissingLocalPath);
}
// To maintain compatibility if the field is point to current directory or not set at all but a prepackaged zip archive is given
// then use it as the package source.
else if (IsCurrentDirectory(field.GetLocalPath()) && !string.IsNullOrEmpty(this.DefaultOptions.Package))
{
results = new UpdateResourceResults { ZipArchivePath = this.DefaultOptions.Package };
}
else if(field.IsCode)
{
// If the function is image upload then run the .NET tools to handle running
// docker build even if the current folder is not a .NET project. The .NET
// could be in a sub folder or be a self contained Docker build.
if (IsDotnetProjectDirectory(localPath) || field.Resource.UploadType == CodeUploadType.Image)
{
results = await PackageDotnetProjectAsync(field, localPath, args);
}
else
{
results = new UpdateResourceResults { ZipArchivePath = GenerateOutputZipFilename(field) };
LambdaPackager.BundleDirectory(results.ZipArchivePath, localPath, false, this.Logger);
}
deleteArchiveAfterUploaded = true;
}
else
{
throw new LambdaToolsException($"Unable to determine package action for the field {field.Resource.Name}/{field.Name}", LambdaToolsException.LambdaErrorCode.ServerlessTemplateUnknownActionForLocalPath);
}
if(!string.IsNullOrEmpty(results.ZipArchivePath))
{
string s3Key;
using (var stream = File.OpenRead(results.ZipArchivePath))
{
s3Key = await Utilities.UploadToS3Async(this.Logger, this.S3Client, this.S3Bucket, this.S3Prefix, Path.GetFileName(results.ZipArchivePath), stream);
results.S3Key = s3Key;
}
// Now that the temp zip file is uploaded to S3 clean up by deleting the temp file.
if (deleteArchiveAfterUploaded)
{
try
{
File.Delete(results.ZipArchivePath);
}
catch (Exception e)
{
this.Logger?.WriteLine($"Warning: Unable to delete temporary archive, {results.ZipArchivePath}, after uploading to S3: {e.Message}");
}
}
}
return results;
}
///
/// Executes the package command to create the deployment bundle for the .NET project and returns the path.
///
///
///
///
///
private async Task PackageDotnetProjectAsync(IUpdateResourceField field, string location, string[] args)
{
if (field.Resource.UploadType == CodeUploadType.Zip)
{
var command = new Commands.PackageCommand(this.Logger, location, args);
command.LambdaClient = this.OriginatingCommand?.LambdaClient;
command.S3Client = this.OriginatingCommand?.S3Client;
command.IAMClient = this.OriginatingCommand?.IAMClient;
command.CloudFormationClient = this.OriginatingCommand?.CloudFormationClient;
command.DisableRegionAndCredentialsCheck = true;
var outputPackage = GenerateOutputZipFilename(field);
command.OutputPackageFileName = outputPackage;
command.TargetFramework =
LambdaUtilities.DetermineTargetFrameworkFromLambdaRuntime(field.Resource.LambdaRuntime, location);
command.Architecture = field.Resource.LambdaArchitecture;
command.LayerVersionArns = field.Resource.LambdaLayers;
// If the project is in the same directory as the CloudFormation template then use any parameters
// that were specified on the command to build the project.
if (IsCurrentDirectory(field.GetLocalPath()))
{
if (!string.IsNullOrEmpty(this.DefaultOptions.TargetFramework))
command.TargetFramework = this.DefaultOptions.TargetFramework;
command.Configuration = this.DefaultOptions.Configuration;
command.DisableVersionCheck = this.DefaultOptions.DisableVersionCheck;
command.MSBuildParameters = this.DefaultOptions.MSBuildParameters;
}
if (!await command.ExecuteAsync())
{
var message = $"Error packaging up project in {location} for CloudFormation resource {field.Resource.Name}";
if (command.LastToolsException != null)
message += $": {command.LastToolsException.Message}";
throw new LambdaToolsException(message, ToolsException.CommonErrorCode.DotnetPublishFailed, command.LastToolsException);
}
var results = new UpdateResourceResults() { ZipArchivePath = outputPackage };
if (!string.IsNullOrEmpty(command.NewDotnetSharedStoreValue))
{
results.DotnetShareStoreEnv = command.NewDotnetSharedStoreValue;
}
return results;
}
else if (field.Resource.UploadType == CodeUploadType.Image)
{
this.Logger.WriteLine($"Building Docker image for {location}");
var pushCommand = new PushDockerImageCommand(Logger, location, args);
pushCommand.ECRClient = OriginatingCommand.ECRClient;
pushCommand.IAMClient = OriginatingCommand.IAMClient;
pushCommand.DisableInteractive = true;
pushCommand.PushDockerImageProperties.DockerFile = field.GetMetadataDockerfile();
pushCommand.PushDockerImageProperties.DockerImageTag = field.GetMetadataDockerTag();
pushCommand.ImageTagUniqueSeed = field.Resource.Name;
// Refer https://docs.docker.com/engine/reference/commandline/build/#set-build-time-variables---build-arg
Dictionary dockerBuildArgs = field.GetMetadataDockerBuildArgs();
if (dockerBuildArgs != null && dockerBuildArgs.Count > 0)
{
StringBuilder buildArgs = new StringBuilder();
foreach (var keyValuePair in dockerBuildArgs)
{
if (keyValuePair.Value != null)
{
buildArgs.Append($"--build-arg {keyValuePair.Key}={keyValuePair.Value} ");
}
else
{
// --build-arg flag could be used without a value, in which case the value from the local environment will be propagated into the Docker container.
buildArgs.Append($"--build-arg {keyValuePair.Key} ");
}
}
pushCommand.PushDockerImageProperties.DockerBuildOptions = buildArgs.ToString().TrimEnd();
}
await pushCommand.PushImageAsync();
if (pushCommand.LastToolsException != null)
throw pushCommand.LastToolsException;
return new UpdateResourceResults { ImageUri = pushCommand.PushedImageUri };
}
else
{
throw new LambdaToolsException($"Unknown upload type for packaging: {field.Resource.UploadType}", LambdaToolsException.LambdaErrorCode.ServerlessTemplateParseError);
}
}
private string[] RemoveProjectLocationArgument(string[] args)
{
List argumentList;
if (args == null || args.Length == 0) return args;
argumentList = new List();
for (int counter = 0; counter < args.Length; counter++)
{
// Skip project location switch and it's value.
if (string.Equals(args[counter], CommonDefinedCommandOptions.ARGUMENT_PROJECT_LOCATION.ShortSwitch, StringComparison.OrdinalIgnoreCase) ||
string.Equals(args[counter], CommonDefinedCommandOptions.ARGUMENT_PROJECT_LOCATION.Switch, StringComparison.OrdinalIgnoreCase))
{
counter += 1;
}
else
{
argumentList.Add(args[counter]);
}
}
return argumentList.ToArray();
}
private static string GenerateOutputZipFilename(IUpdateResourceField field)
{
var outputPackage = Path.Combine(Path.GetTempPath(), $"{field.Resource.Name}-{field.Name}-{DateTime.Now.Ticks}.zip");
return outputPackage;
}
///
/// Check to see if the directory contains a .NET project file.
///
///
///
private bool IsDotnetProjectDirectory(string localPath)
{
if (!Directory.Exists(localPath))
return false;
var projectFiles = Directory.GetFiles(localPath, "*.??proj", SearchOption.TopDirectoryOnly)
.Where(x => LambdaUtilities.ValidProjectExtensions.Contains(Path.GetExtension(x))).ToArray();
return projectFiles.Length == 1;
}
private bool IsCurrentDirectory(string localPath)
{
if (string.IsNullOrEmpty(localPath))
return true;
if (string.Equals(".", localPath))
return true;
if (string.Equals("./", localPath))
return true;
if (string.Equals(@".\", localPath))
return true;
return false;
}
///
/// Create the appropriate parser depending whether the template is JSON or YAML.
///
///
///
///
public static ITemplateParser CreateTemplateParser(string templateBody)
{
switch (LambdaUtilities.DetermineTemplateFormat(templateBody))
{
case TemplateFormat.Json:
return new JsonTemplateParser(templateBody);
case TemplateFormat.Yaml:
return new YamlTemplateParser(templateBody);
default:
throw new LambdaToolsException("Unable to determine template file format", LambdaToolsException.LambdaErrorCode.ServerlessTemplateParseError);
}
}
class UpdateResourceResults
{
public string ZipArchivePath { get; set; }
public string S3Key { get; set; }
public string DotnetShareStoreEnv { get; set; }
public string ImageUri { get; set; }
}
}
}