using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Json.LitJson;
using ServiceClientGenerator.DefaultConfiguration;
using ServiceClientGenerator.Endpoints;
using ServiceClientGenerator.Endpoints.Tests;
namespace ServiceClientGenerator
{
///
/// Loads the generator control manifest document. This document
/// yields the Visual Studio project metadata (for new project files
/// that we create) and the set of service configurations available
/// to process.
///
public class GenerationManifest
{
abstract class ModelsSectionKeys
{
public const string ActiveKey = "active";
public const string NamespaceKey = "namespace";
public const string BaseNameKey = "base-name";
public const string NugetPackageTitleSuffix = "nuget-package-title-suffix";
public const string DefaultRegionKey = "default-region";
public const string GenerateClientConstructorsKey = "generate-client-constructors";
public const string CustomizationFileKey = "customization-file";
public const string MaxRetriesKey = "max-retries";
public const string SynopsisKey = "synopsis";
public const string NetStandardSupportKey = "netstandard-support";
public const string DependenciesKey = "dependencies";
public const string ReferenceDependenciesKey = "reference-dependencies";
public const string NugetDependenciesKey = "nuget-dependencies";
public const string DependencyNameKey = "name";
public const string DependencyVersionKey = "version";
public const string DependencyHintPathKey = "hint-path";
public const string ParentBaseNameKey = "parent-base-name";
public const string TagsKey = "tags";
public const string LicenseUrlKey = "license-url";
public const string TestServiceKey = "test-service";
public const string LegacyServiceIdKey = "legacy-service-id";
}
abstract class ProjectsSectionKeys
{
public const string ProjectsKey = "projects";
public const string NameKey = "name";
public const string ConfigurationsKey = "configurations";
public const string TargetFrameworksKey = "targetFrameworks";
public const string DefineConstantsKey = "defineConstants";
public const string BinSubFolderKey = "binSubFolder";
public const string TemplateKey = "template";
public const string PlatformCodeFoldersKey = "platformCodeFolders";
public const string ExtraTestProjects = "extraTestProjects";
public const string NuGetTargetFrameworkKey = "nugetTargetPlatform";
public const string PlatformExcludeFoldersKey = "excludeFolders";
public const string FrameworkPathOverrideKey = "frameworkPathOverride";
public const string FrameworkRefernecesKey = "frameworkReferences";
public const string EmbeddedResourcesKey = "embeddedResources";
public const string UnitTestProjectsKey = "unittestprojects";
public const string NoWarn = "noWarn";
public const string PackageReferences = "packageReferences";
public const string OutputPathOverrideKey = "outputPathOverride";
}
///
/// URL for Apache License 2.0
///
public const string ApacheLicenseURL = @"http://aws.amazon.com/apache2.0/";
///
/// The set of services declared in the manifest as supporting generation.
///
public IEnumerable ServiceConfigurations { get; private set; }
//public IDictionary ServiceVersions { get; private set; }
///
/// The set of per-platform project metadata needed to generate a platform
/// specific project file for a service.
///
public IEnumerable ProjectFileConfigurations { get; private set; }
public IEnumerable UnitTestProjectFileConfigurations { get; private set; }
public string CoreFileVersion { get; private set; }
public string CoreVersion { get; private set; }
public bool DefaultToPreview
{
get;
private set;
}
public string PreviewLabel
{
get;
private set;
}
///
/// Model representing the default configuration modes as built
/// from the sdk-default-configurations.json file.
///
public DefaultConfiguration.DefaultConfigurationModel DefaultConfiguration { get; set; }
//This should be the same version number as SdkVersioning.DefaultAssemblyVersion in BuildTasks
private const string DefaultAssemblyVersion = "3.3";
///
/// Processes the control manifest to yield the set of services available to
/// generate and the Visual Studio project file information used to create
/// new projects for services.
///
/// GeneratorOptions containing information requred to load GenerationManifest
public static GenerationManifest Load(GeneratorOptions options)
{
var generationManifest =
new GenerationManifest(
new DefaultConfigurationController(
new FileReader(),
new DefaultConfigurationParser()));
var manifest = LoadJsonFromFile(options.Manifest);
var versionsManifest = LoadJsonFromFile(options.Versions);
generationManifest.CoreFileVersion = versionsManifest["CoreVersion"].ToString();
generationManifest.CoreVersion = Utils.GetVersion(versionsManifest["OverrideCoreVersion"]?.ToString() ?? generationManifest.CoreFileVersion);
generationManifest.DefaultToPreview = (bool)versionsManifest["DefaultToPreview"];
if (generationManifest.DefaultToPreview)
{
generationManifest.PreviewLabel = (string)versionsManifest["PreviewLabel"];
}
if (!string.IsNullOrEmpty(generationManifest.PreviewLabel))
generationManifest.PreviewLabel = "-" + generationManifest.PreviewLabel;
generationManifest.LoadDefaultConfiguration(options.ModelsFolder);
generationManifest.LoadServiceConfigurations(manifest, versionsManifest["ServiceVersions"], options);
generationManifest.LoadProjectConfigurations(manifest);
generationManifest.LoadUnitTestProjectConfigurations(manifest);
return generationManifest;
}
///
/// Loads the sdk-default-configurations.json file.
///
private void LoadDefaultConfiguration(string manifestPath)
{
var repositoryRootDirectoryPath = Utils.PathCombineAlt(manifestPath, "..","..");
DefaultConfiguration = _defaultConfigurationController.LoadDefaultConfiguration(repositoryRootDirectoryPath);
}
///
/// Recursively walk through the ServiceModels folder and load/parse the
/// model files to generate ServiceConfiguration objects.
///
/// loaded _manifest.json file
/// loaded _sdk-versions.json file
/// generator options
void LoadServiceConfigurations(JsonData manifest, JsonData serviceVersions, GeneratorOptions options)
{
List> modelConfigList = new List>();
var serviceConfigurations = new List();
var serviceDirectories = Utils.GetServiceDirectories(options);
foreach (string serviceDirectory in serviceDirectories)
{
string metadataJsonFile = Utils.PathCombineAlt(serviceDirectory, "metadata.json");
if (File.Exists(metadataJsonFile))
{
JsonData metadataNode = LoadJsonFromFile(metadataJsonFile);
var activeNode = metadataNode[ModelsSectionKeys.ActiveKey];
if (activeNode != null
&& activeNode.IsBoolean
&& !(bool)activeNode )
{
continue;
}
var serviceModelFileName = GetLatestModel(serviceDirectory);
string paginatorsFileName = GetLatestPaginators(serviceDirectory);
var config = CreateServiceConfiguration(metadataNode, serviceVersions, serviceDirectory, serviceModelFileName, paginatorsFileName);
serviceConfigurations.Add(config);
modelConfigList.Add(new Tuple(metadataNode, config));
}
}
if (string.IsNullOrEmpty(options.ServiceModels))
{
// We skip this check if ServiceModels provided, as we would never satisfy condition below.
//
// We need to make sure that we have configuration files for all expected services and
// that there aren't mismatches in the service names.
foreach (string serviceVersionEntry in serviceVersions.GetMap().Keys)
{
if (!serviceConfigurations.Any(config => config.ServiceNameRoot == serviceVersionEntry))
{
throw new Exception($"Service entry {serviceVersionEntry} doesn't match any of the available service configurations.");
}
}
}
// The parent model for current model, if set, the client will be generated
// in the same namespace and share common types.
foreach (var modelConfig in modelConfigList)
{
var modelNode = modelConfig.Item1;
var config = modelConfig.Item2;
var parentClassName = modelNode[ModelsSectionKeys.ParentBaseNameKey] != null ? modelNode[ModelsSectionKeys.ParentBaseNameKey].ToString() : null;
if (parentClassName != null)
{
try
{
config.ParentConfig = serviceConfigurations.Single(c => c.ClassName.Equals(parentClassName));
}
catch (KeyNotFoundException exception)
{
// Note : the parent model should be defined in the manifest before being referred by a child model
throw new KeyNotFoundException(
string.Format("A parent model with name {0} is not defined in the manifest", parentClassName),
exception); ;
}
}
}
ServiceConfigurations = serviceConfigurations
.OrderBy(sc => sc.ServiceDependencies.Count)
.ToList();
}
///
/// Use the date order of the models combined with default string sort
/// to find the latest models file
///
///
///
private static string GetLatestModel(string serviceDirectory)
{
string latestModelName = string.Empty;
foreach (string modelName in Directory.GetFiles(serviceDirectory, "*.normal.json", SearchOption.TopDirectoryOnly))
{
if (string.Compare(latestModelName, modelName) < 0)
{
latestModelName = modelName;
}
}
if (string.IsNullOrEmpty(latestModelName))
{
throw new FileNotFoundException("Failed to find a model file in " + serviceDirectory);
}
return Path.GetFileName(latestModelName);
}
///
/// Use the date order of the paginators combined with default string sort
/// to find the latest paginators file
///
///
///
private static string GetLatestPaginators(string serviceDirectory)
{
var latestPaginatorsName = Directory.GetFiles(serviceDirectory, "*.paginators.json", SearchOption.TopDirectoryOnly)
.OrderBy(x => x).FirstOrDefault() ?? "";
return Path.GetFileName(latestPaginatorsName);
}
private const string EndpointRuleSetFile = "endpoint-rule-set.json";
private const string EndpointRuleSetTestsFile = "endpoint-tests.json";
private ServiceConfiguration CreateServiceConfiguration(JsonData modelNode, JsonData serviceVersions, string serviceDirectoryPath, string serviceModelFileName, string servicePaginatorsFileName)
{
var modelFullPath = Utils.PathCombineAlt(serviceDirectoryPath, serviceModelFileName);
var paginatorsFullPath = Utils.PathCombineAlt(serviceDirectoryPath, servicePaginatorsFileName);
JsonData metadata = JsonMapper.ToObject(File.ReadAllText(modelFullPath))[ServiceModel.MetadataKey];
// A new config that the api generates from
var modelName = Path.GetFileName(serviceDirectoryPath);
var config = new ServiceConfiguration
{
ModelName = modelName,
ModelPath = modelFullPath,
PaginatorsPath = paginatorsFullPath,
Namespace = Utils.JsonDataToString(modelNode[ModelsSectionKeys.NamespaceKey]), // Namespace of the service if it's different from basename
ClassNameOverride = Utils.JsonDataToString(modelNode[ModelsSectionKeys.BaseNameKey]),
DefaultRegion = Utils.JsonDataToString(modelNode[ModelsSectionKeys.DefaultRegionKey]),
GenerateConstructors = modelNode[ModelsSectionKeys.GenerateClientConstructorsKey] == null || (bool)modelNode[ModelsSectionKeys.GenerateClientConstructorsKey], // A way to prevent generating basic constructors
IsTestService = modelNode[ModelsSectionKeys.TestServiceKey] != null && (bool)modelNode[ModelsSectionKeys.TestServiceKey]
};
// Load endpoints ruleset and tests if present
var rulesetFileName = Directory.GetFiles(serviceDirectoryPath, "*." + EndpointRuleSetFile, SearchOption.TopDirectoryOnly).FirstOrDefault();
var testsFileName = Directory.GetFiles(serviceDirectoryPath, "*." + EndpointRuleSetTestsFile, SearchOption.TopDirectoryOnly).FirstOrDefault();
// We have found tests but not rules, something is wrong!
if (rulesetFileName == null && testsFileName != null)
{
throw new FileNotFoundException($"We have found endpoints tests but endpoints rules are missing. Expected file suffix is .{EndpointRuleSetFile}");
}
if (rulesetFileName != null)
{
var json = File.ReadAllText(rulesetFileName);
config.EndpointsRuleSet = JsonMapper.ToObject(json);
if (testsFileName == null)
{
throw new FileNotFoundException($"Endpoints tests are missing. Expected file suffix is .{EndpointRuleSetTestsFile}");
}
json = File.ReadAllText(testsFileName);
config.EndpointTests = JsonMapper.ToObject(json);
}
if (modelNode[ModelsSectionKeys.NugetPackageTitleSuffix] != null)
config.NugetPackageTitleSuffix = modelNode[ModelsSectionKeys.NugetPackageTitleSuffix].ToString();
if (modelNode[ModelsSectionKeys.ReferenceDependenciesKey] != null)
{
config.ReferenceDependencies = new Dictionary>();
foreach (KeyValuePair kvp in modelNode[ModelsSectionKeys.ReferenceDependenciesKey])
{
var platformDependencies = new List();
foreach (JsonData item in kvp.Value)
{
var platformDependency = new Dependency
{
Name = item[ModelsSectionKeys.DependencyNameKey].ToString(),
Version = item.PropertyNames.Contains(ModelsSectionKeys.DependencyVersionKey) ? item[ModelsSectionKeys.DependencyVersionKey].ToString() : "0.0.0.0",
HintPath = item[ModelsSectionKeys.DependencyHintPathKey].ToString(),
};
platformDependencies.Add(platformDependency);
}
config.ReferenceDependencies.Add(kvp.Key, platformDependencies);
}
}
if (modelNode[ModelsSectionKeys.NugetDependenciesKey] != null)
{
config.NugetDependencies = new Dictionary>();
foreach (KeyValuePair kvp in modelNode[ModelsSectionKeys.NugetDependenciesKey])
{
var nugetDependencies = new List();
foreach (JsonData item in kvp.Value)
{
var nugetDependency = new Dependency
{
Name = item[ModelsSectionKeys.DependencyNameKey].ToString(),
Version = item[ModelsSectionKeys.DependencyVersionKey].ToString(),
};
nugetDependencies.Add(nugetDependency);
}
config.NugetDependencies.Add(kvp.Key, nugetDependencies);
}
}
config.Tags = new List();
if (modelNode[ModelsSectionKeys.TagsKey] != null)
{
foreach (JsonData tag in modelNode[ModelsSectionKeys.TagsKey])
{
config.Tags.Add(tag.ToString());
}
}
// Provides a way to specify a customizations file rather than using a generated one
config.CustomizationsPath = modelNode[ModelsSectionKeys.CustomizationFileKey] == null
? DetermineCustomizationsPath(config.ServiceDirectoryName)
: Utils.PathCombineAlt(serviceDirectoryPath, modelNode[ModelsSectionKeys.CustomizationFileKey].ToString());
if (modelNode[ModelsSectionKeys.MaxRetriesKey] != null && modelNode[ModelsSectionKeys.MaxRetriesKey].IsInt)
config.OverrideMaxRetries = Convert.ToInt32(modelNode[ModelsSectionKeys.MaxRetriesKey].ToString());
if (modelNode[ModelsSectionKeys.SynopsisKey] != null)
config.Synopsis = (string)modelNode[ModelsSectionKeys.SynopsisKey];
if (modelNode[ModelsSectionKeys.LegacyServiceIdKey] != null)
config.LegacyServiceId = (string)modelNode[ModelsSectionKeys.LegacyServiceIdKey];
if (modelNode[ModelsSectionKeys.NetStandardSupportKey] != null)
config.NetStandardSupport = (bool)modelNode[ModelsSectionKeys.NetStandardSupportKey];
else
config.NetStandardSupport = true;
config.ServiceDependencies = new Dictionary(StringComparer.Ordinal);
if (modelNode[ModelsSectionKeys.DependenciesKey] != null && modelNode[ModelsSectionKeys.DependenciesKey].IsArray)
{
foreach (var d in modelNode[ModelsSectionKeys.DependenciesKey])
{
config.ServiceDependencies.Add(d.ToString(), null);
}
}
if (modelNode[ModelsSectionKeys.LicenseUrlKey] != null && modelNode[ModelsSectionKeys.LicenseUrlKey].IsString)
{
config.LicenseUrl = modelNode[ModelsSectionKeys.LicenseUrlKey].ToString();
config.RequireLicenseAcceptance = true;
}
else
config.LicenseUrl = ApacheLicenseURL;
var serviceName = config.ServiceNameRoot;
var versionInfoJson = serviceVersions[serviceName];
if (versionInfoJson != null)
{
var dependencies = versionInfoJson["Dependencies"];
foreach (var name in dependencies.PropertyNames)
{
var version = dependencies[name].ToString();
config.ServiceDependencies[name] = version;
}
var versionText = versionInfoJson["Version"].ToString();
config.ServiceFileVersion = versionText;
var assemblyVersionOverride = versionInfoJson["AssemblyVersionOverride"];
if (assemblyVersionOverride != null)
{
config.ServiceAssemblyVersionOverride = assemblyVersionOverride.ToString();
}
if(versionInfoJson["InPreview"] != null && (bool)versionInfoJson["InPreview"])
config.InPreview = true;
else
config.InPreview = this.DefaultToPreview;
}
else
{
config.ServiceDependencies["Core"] = CoreFileVersion;
config.InPreview = this.DefaultToPreview;
config.ServiceFileVersion = DefaultAssemblyVersion;
var versionTokens = CoreVersion.Split('.');
if (!DefaultAssemblyVersion.StartsWith($"{versionTokens[0]}.{versionTokens[1]}"))
{
throw new NotImplementedException($"{nameof(DefaultAssemblyVersion)} should be updated to match the AWSSDK.Core minor version number.");
}
}
return config;
}
private static List SafeGetStringList(JsonData data, string key)
{
var t = data.SafeGet(key);
if (t != null)
{
return (from object obj in t select obj.ToString()).ToList();
}
else
{
return new List();
}
}
private static List SafeGetObjectList(JsonData data, string key, Func converter)
{
var t = data.SafeGet(key);
if (t != null)
{
return (from JsonData obj in t select converter(obj)).ToList();
}
else
{
return new List();
}
}
private static ProjectFileConfiguration LoadProjectFileConfiguration(JsonData node)
{
return new ProjectFileConfiguration{
Name = node.SafeGetString(ProjectsSectionKeys.NameKey),
TargetFrameworkVersions = SafeGetStringList(node, ProjectsSectionKeys.TargetFrameworksKey),
CompilationConstants = SafeGetStringList(node, ProjectsSectionKeys.DefineConstantsKey),
BinSubFolder = node.SafeGetString(ProjectsSectionKeys.BinSubFolderKey),
Template = node.SafeGetString(ProjectsSectionKeys.TemplateKey),
NuGetTargetPlatform = node.SafeGetString(ProjectsSectionKeys.NuGetTargetFrameworkKey),
FrameworkPathOverride = node.SafeGetString(ProjectsSectionKeys.FrameworkPathOverrideKey),
NoWarn = node.SafeGetString(ProjectsSectionKeys.NoWarn),
OutputPathOverride = node.SafeGetString(ProjectsSectionKeys.OutputPathOverrideKey),
Configurations = SafeGetStringList(node, ProjectsSectionKeys.ConfigurationsKey),
EmbeddedResources = SafeGetStringList(node, ProjectsSectionKeys.EmbeddedResourcesKey),
PlatformCodeFolders = SafeGetStringList(node, ProjectsSectionKeys.PlatformCodeFoldersKey),
PlatformExcludeFolders = SafeGetStringList(node, ProjectsSectionKeys.PlatformExcludeFoldersKey),
DllReferences = SafeGetObjectList(node, ProjectsSectionKeys.FrameworkRefernecesKey, Dependency.ParseJson),
PackageReferences = SafeGetObjectList(node, ProjectsSectionKeys.PackageReferences, ProjectFileCreator.PackageReference.ParseJson),
};
}
///
/// Parses the Visual Studio project metadata entries from the manifest. These
/// are used when generating project files for a service.
/// Sets the ProjectFileConfigurations member on exit with the collection of loaded
/// configurations.
///
///
void LoadProjectConfigurations(JsonData document)
{
var projectConfigurations = new List();
var projectsNode = document[ProjectsSectionKeys.ProjectsKey];
foreach (JsonData projectNode in projectsNode)
{
var config = LoadProjectFileConfiguration(projectNode);
var extraTestProjects = projectNode.SafeGet(ProjectsSectionKeys.ExtraTestProjects);
if (extraTestProjects == null)
{
config.ExtraTestProjects = new List();
}
else
{
config.ExtraTestProjects = (from object etp in extraTestProjects
select etp.ToString()).ToList();
}
projectConfigurations.Add(config);
}
ProjectFileConfigurations = projectConfigurations;
}
void LoadUnitTestProjectConfigurations(JsonData document)
{
IList configuraitons = new List();
var projectsNode = document[ProjectsSectionKeys.UnitTestProjectsKey];
foreach (JsonData projectNode in projectsNode)
{
var configuration = LoadProjectFileConfiguration(projectNode);
configuraitons.Add(configuration);
}
UnitTestProjectFileConfigurations = configuraitons;
}
///
/// Finds the customizations file in \customizations as model.customizations.json if it's there
///
/// The name of the model as defined in the _manifest
/// Full path to the customization if it exists, null if it wasn't found
private static string DetermineCustomizationsPath(string serviceKey)
{
if (!Directory.Exists("customizations"))
{
return null;
}
var files = Directory.GetFiles("customizations", serviceKey + ".customizations.json").OrderByDescending(x => x);
return !files.Any() ? null : files.Single();
}
///
/// Loads a JsonData object for data in a given file.
///
/// Path to the JSON file.
/// JsonData corresponding to JSON in the file.
private static JsonData LoadJsonFromFile(string path)
{
JsonData data;
using (var reader = new StreamReader(path))
data = JsonMapper.ToObject(reader);
return data;
}
private readonly IDefaultConfigurationController _defaultConfigurationController;
private GenerationManifest(IDefaultConfigurationController defaultConfigurationController)
{
_defaultConfigurationController = defaultConfigurationController;
}
public GenerationManifest()
{
}
}
}