using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Management.Automation; using System.Text; using System.Xml; using AWSPowerShellGenerator.Utils; using AWSPowerShellGenerator.Writers.Help; using System.Reflection; using System.Net; using System.Text.RegularExpressions; using System.Threading.Tasks; namespace AWSPowerShellGenerator.Generators { public class WebHelpGenerator : HelpGeneratorBase { const int MAX_FILE_SIZE = 160; // matching sdk private static readonly Regex TypeNameRegex = new Regex("Amazon(\\.[A-Za-z0-9]+)+", RegexOptions.Compiled); #region Public properties // the root path of the AWS .Net sdk docs; eg http://docs.aws.amazon.com/sdkfornet/v3/apidocs public string SDKHelpRoot { get; set; } // the root path of the docs domain for cn-north-1 region public string CNNorth1RegionDocsDomain { get; set; } public static string CNNorth1RegionDisclaimerTemplate = "AWS services or capabilities described in AWS Documentation may vary by region/location. " + "Click Getting Started with Amazon AWS to see specific differences applicable to the China (Beijing) Region."; public static HashSet NonModularizedServices = new HashSet { "CloudHSM", "ElasticLoadBalancing", "CloudWatchEvents", "KinesisAnalytics" }; public string BJSRegionDisclaimer { get { return string.Format(CNNorth1RegionDisclaimerTemplate, CNNorth1RegionDocsDomain); } } #endregion protected static Dictionary _msdnDocLinks = new Dictionary(StringComparer.OrdinalIgnoreCase) { { "System.String", "https://docs.microsoft.com/en-us/dotnet/api/system.string" }, { "System.Byte", "https://docs.microsoft.com/en-us/dotnet/api/system.byte" }, { "System.Int32", "https://docs.microsoft.com/en-us/dotnet/api/system.int32" }, { "System.Single", "https://docs.microsoft.com/en-us/dotnet/api/system.single" }, { "System.Double", "https://docs.microsoft.com/en-us/dotnet/api/system.double" }, { "System.Boolean", "https://docs.microsoft.com/en-us/dotnet/api/system.boolean" }, { "System.Collections.Hashtable", "https://docs.microsoft.com/en-us/dotnet/api/system.collections.hashtable" }, { "System.Management.Automation.SwitchParameter", "https://docs.microsoft.com/en-us/dotnet/api/system.management.automation.switchparameter" } }; private static readonly HashSet CommonParameters = new HashSet() { "AccessKey", "Credential", "ProfileLocation", "ProfileName", "NetworkCredential", "SecretKey", "SessionToken", "Region", "EndpointUrl" }; /// /// As we process each cmdlet we build a collection organized by service detailing any backwards /// compatibility aliases that we'll output into a set of tables into the pstoolsref_legacyaliases.html /// page. In the tuples, T1 is the alias name, T2 is the actual cmdlet name. /// private Dictionary>> _legacyAliasesByService = new Dictionary>>(); protected override void GenerateHelper() { base.GenerateHelper(); if (string.IsNullOrEmpty(SDKHelpRoot)) SDKHelpRoot = "http://docs.aws.amazon.com/sdkfornet/v3/apidocs/"; else if (!SDKHelpRoot.EndsWith("/")) SDKHelpRoot = SDKHelpRoot + "/"; Console.WriteLine("Generating web help documentation:"); Console.WriteLine(".... SDK help base URI set to {0}", SDKHelpRoot); Console.WriteLine(".... writing doc output to {0}", OutputFolder); var buildLogsPath = Path.Combine(this.Options.RootPath, "logs"); if (!Directory.Exists(buildLogsPath)) Directory.CreateDirectory(buildLogsPath); var logFile = Path.Combine(buildLogsPath, Name + ".dll-WebHelp.log"); var oldWriter = Console.Out; try { using (var consoleWriter = new StreamWriter(File.OpenWrite(logFile))) { Console.SetOut(consoleWriter); CleanWebHelpOutputFolder(OutputFolder); CopyWebHelpStaticFiles(OutputFolder); CreateVersionInfoFile(Path.Combine(OutputFolder, "items")); var tocWriter = new TOCWriter(Options, OutputFolder); tocWriter.AddFixedSection(); Parallel.ForEach(CmdletTypes, (cmdletType) => { var (moduleName, serviceName) = DetermineCmdletServiceOwner(cmdletType); var cmdletInfo = InspectCmdletAttributes(cmdletType); string synopsis = null; if (cmdletInfo.AWSCmdletAttribute == null) { Console.WriteLine("Unable to find AWSCmdletAttribute for type " + cmdletType.FullName); } else { var cmdletReturnAttributeType = cmdletInfo.AWSCmdletAttribute.GetType(); synopsis = cmdletReturnAttributeType.GetProperty("Synopsis").GetValue(cmdletInfo.AWSCmdletAttribute, null) as string; } foreach (var cmdletAttribute in cmdletInfo.CmdletAttributes) { var typeDocumentation = DocumentationUtils.GetTypeDocumentation(cmdletType, AssemblyDocumentation); typeDocumentation = DocumentationUtils.FormatXMLForPowershell(typeDocumentation, true); Console.WriteLine($"Cmdlet = {cmdletType.FullName}"); Console.WriteLine($"Documentation = {typeDocumentation}"); var cmdletName = cmdletAttribute.VerbName + "-" + cmdletAttribute.NounName; var allProperties = GetRootSimpleProperties(cmdletType); var parameterPartitioning = new CmdletParameterSetPartitions(allProperties, cmdletAttribute.DefaultParameterSetName); var serviceAbbreviation = GetServiceAbbreviation(cmdletType); var cmdletPageWriter = new CmdletPageWriter(Options, OutputFolder, serviceName, moduleName, cmdletName); WriteDetails(cmdletPageWriter, typeDocumentation, synopsis, cmdletInfo.AWSCmdletAttribute); WriteSyntax(cmdletPageWriter, cmdletName, parameterPartitioning); WriteParameters(cmdletPageWriter, cmdletName, allProperties, false); WriteParameters(cmdletPageWriter, cmdletName, allProperties, true); WriteOutputs(cmdletPageWriter, cmdletInfo.AWSCmdletOutputAttributes); WriteNotes(cmdletPageWriter); WriteExamples(cmdletPageWriter, cmdletName); WriteRelatedLinks(cmdletPageWriter, serviceAbbreviation, cmdletName); cmdletPageWriter.Write(); lock (tocWriter) { var legacyAlias = InspectForLegacyAliasAttribution(moduleName, cmdletName, cmdletInfo.AWSCmdletAttribute); tocWriter.AddServiceCmdlet(moduleName, serviceName, cmdletName, cmdletPageWriter.GetTOCID(), synopsis, legacyAlias); } } }); tocWriter.Write(); WriteLegacyAliasesPage(); } } finally { Console.SetOut(oldWriter); } } private void WriteDetails(CmdletPageWriter writer, string typeDocumentation, string synopsis, Attribute awsCmdletAttribute) { if (!string.IsNullOrEmpty(synopsis)) writer.AddPageElement(CmdletPageWriter.SynopsisElementKey, synopsis); var doc = new StringBuilder(); if (!string.IsNullOrEmpty(typeDocumentation)) doc.Append(typeDocumentation); var legacyAlias = ExtractLegacyAlias(awsCmdletAttribute); if (!string.IsNullOrEmpty(legacyAlias)) { doc.AppendLine("
"); doc.AppendLine("
"); doc.AppendFormat("Note: For scripts written against earlier versions of this module this cmdlet can also be invoked with the alias, {0}.", legacyAlias); } if (doc.Length > 0) writer.AddPageElement(CmdletPageWriter.DescriptionElementKey, doc.ToString()); } private static void WriteSyntax(CmdletPageWriter writer, string cmdletName, CmdletParameterSetPartitions parameterSetPartitioning) { var sb = new StringBuilder(); if (parameterSetPartitioning.HasNamedParameterSets) { var sets = parameterSetPartitioning.NamedParameterSets; foreach (var set in sets) { AppendSyntaxChart(cmdletName, set, parameterSetPartitioning, sb); } } else { AppendSyntaxChart(cmdletName, CmdletParameterSetPartitions.AllSetsKey, parameterSetPartitioning, sb); } writer.AddPageElement(CmdletPageWriter.SyntaxElementKey, sb.ToString()); } private static void AppendSyntaxChart(string cmdletName, string setName, CmdletParameterSetPartitions parameterSetPartitioning, StringBuilder sb) { var isCustomNamedSet = !setName.Equals(CmdletParameterSetPartitions.AllSetsKey); var isDefaultSet = isCustomNamedSet && setName.Equals(parameterSetPartitioning.DefaultParameterSetName, StringComparison.Ordinal); var setParameterNames = parameterSetPartitioning.ParameterNamesForSet(setName, parameterSetPartitioning.HasNamedParameterSets); if (isCustomNamedSet) { sb.Append( isDefaultSet ? $"

{setName} (Default)

" : $"

{setName}

"); } sb.Append("
"); // Microsoft cmdlets show params in syntax in defined but non-alpha order. Use the ordering we found // during reflection here in the hope the sdk has them in 'most important' order. Also, if there is // four or less params, keep the syntax on one line. sb.Append($"
{cmdletName}
"); var allParameters = parameterSetPartitioning.Parameters; var paramCount = setParameterNames.Count; if (paramCount > 0) { sb.Append("
"); foreach (var p in allParameters .Where(param => !CommonParameters.Contains(param.CmdletParameterName) && setParameterNames.Contains(param.CmdletParameterName))) { sb.Append($"
-{p.CmdletParameterName} <{GetTypeDisplayName(p.PropertyType, false)}>
"); } sb.Append("
"); } sb.Append("
"); if (isCustomNamedSet) { sb.Append("
"); } } private void WriteParameters(CmdletPageWriter writer, string cmdletName, IEnumerable allProperties, bool commonParameters) { var pageElementKey = commonParameters ? CmdletPageWriter.CommonParametersElementKey : CmdletPageWriter.ParametersElementKey; var sb = new StringBuilder(); // Microsoft cmdlets list the parameters in alpha order here, so do the same (leaving metadata/paging // params in the correct order) foreach (var property in allProperties.Where(p => CommonParameters.Contains(p.Name) ^ !commonParameters).OrderBy(p => p.Name)) { sb.Append("
"); var typeName = GetTypeDisplayName(property.PropertyType, true); var inputTypeDocLink = PredictHtmlDocsLink(typeName); if (!string.IsNullOrEmpty(inputTypeDocLink)) { if (_msdnDocLinks.ContainsKey(typeName)) { // Open the MSDN links with _blank to open in a new tab sb.Append($""); } else { // For links to elsewhere in PowerShell or the SDK, open in the same tab via _parent to preserve the referrer for analytics sb.Append($""); } } else { sb.Append($"
-{property.Name} <{GetTypeDisplayName(property.PropertyType, false)}>
"); } sb.Append($"
{property.PowershellWebDocumentation}
"); InspectParameter(property, out var isRequiredForParameterSets, out var pipelineInput, out var position, out var aliases); sb.Append("
"); sb.Append($""); sb.Append($""); //sb.AppendFormat("", ); sb.Append($""); //sb.AppendFormat("", ); if (aliases.Length > 0) { sb.AppendFormat($""); } sb.Append("
Required?{FormatIsRequired(isRequiredForParameterSets)}
Position?{position}
Default value{0}
Accept pipeline input?{pipelineInput}
Accept wildcard characters?{0}
Aliases{string.Join(", ", aliases)}
"); sb.Append("
"); } if (sb.Length != 0) { writer.AddPageElement(pageElementKey, sb.ToString()); } } private static string FormatIsRequired(HashSet parameterSets) { if (parameterSets == null) { return "False"; } if (parameterSets.Count == 0) { return "True"; } return $"True ({string.Join(", ", parameterSets)})"; } private void WriteOutputs(CmdletPageWriter writer, IEnumerable outputAttributes) { var sb = new StringBuilder(); // attributing describing outputs means we don't need to worry about detecting sb.Length > 0 on exit foreach (var outputAttribute in outputAttributes) { var attributeType = outputAttribute.GetType(); var returnType = attributeType.GetProperty("ReturnType").GetValue(outputAttribute, null) as string; var description = attributeType.GetProperty("Description").GetValue(outputAttribute, null) as string; sb.Append("
"); if (!string.IsNullOrEmpty(returnType)) { returnType = TypeNameRegex.Replace(returnType, (match) => ChangeTypeNameStringIntoLink(match.Value)); sb.Append($"
{returnType}
"); } sb.Append($"
{WebUtility.HtmlEncode(description)}
"); sb.Append(""); } writer.AddPageElement(CmdletPageWriter.OutputsElementKey, sb.ToString()); } private string ChangeTypeNameStringIntoLink(string text) { var uri = PredictHtmlDocsLink(text); if (string.IsNullOrEmpty(uri)) { return WebUtility.HtmlEncode(text); } return $"{WebUtility.HtmlEncode(text)}"; } private static void WriteNotes(CmdletPageWriter writer) { } private void WriteExamples(CmdletPageWriter writer, string cmdletName) { XmlDocument document; // lack of examples will be reported in the logs by the native help generator if (!ExamplesCache.TryGetValue(cmdletName, out document)) return; var set = document.SelectSingleNode("examples"); if (set == null) return; var sb = new StringBuilder(); int exampleIndex = 1; var examples = set.SelectNodes("example"); foreach (XmlNode example in examples) { sb.AppendFormat("

Example {0}

", exampleIndex); sb.Append("
");

                var code = example.SelectSingleNode("code");
                if (code == null)
                    Logger.LogError("Unable to find examples  tag for cmdlet " + cmdletName);

                var codeSample = code.InnerText.Trim('\r', '\n');
                
                codeSample = codeSample.Replace("\r\n", "
").Replace("\n", "
"); sb.AppendFormat("
{0}
", codeSample); var description = example.SelectSingleNode("description"); if (description == null) Logger.LogError("Unable to find examples tag for cmdlet " + cmdletName); // use InnerXml here to allow for
and other layout format tags, these get stripped // if we use InnerText var innerXml = description.InnerXml; // convert link elements to anchors (pshelp strips the tags to leave the link) /*if (innerXml.Contains("")) { }*/ sb.AppendFormat("
{0}
", innerXml); sb.Append("
"); exampleIndex++; } writer.AddPageElement(CmdletPageWriter.ExamplesElementKey, sb.ToString()); } private void WriteRelatedLinks(CmdletPageWriter writer, string serviceAbbreviation, string cmdletName) { var sb = new StringBuilder(); // putting common credential and region parameters into a related link is the simplest // approach, but only do it for service cmdlets if (!serviceAbbreviation.Equals("Common", StringComparison.Ordinal)) { XmlDocument document; if (LinksCache.TryGetValue(serviceAbbreviation, out document)) { ConstructLinks(sb, document, "*"); ConstructLinks(sb, document, cmdletName); } } // Add link for User Guide to all cmdlets AppendLink(sb, "AWS Tools for PowerShell User Guide", "http://docs.aws.amazon.com/powershell/latest/userguide/"); writer.AddPageElement(CmdletPageWriter.RelatedLinksElementKey, sb.ToString()); } public void ConstructLinks(StringBuilder sb, XmlDocument document, string target) { var links = GetRelatedLinks(document, target); if (links != null) { foreach (XmlNode link in links) { string displayname = null; try { displayname = link.Attributes["name"].InnerText; } catch { } if (string.IsNullOrEmpty(displayname) || string.IsNullOrEmpty(link.InnerText)) { Logger.LogError("Malformed link {0}, skipping" + link.OuterXml.ToString()); } AppendLink(sb, displayname, link.InnerText); } } } private void AppendLink(StringBuilder sb, string linkText, string linkAddress) { sb.Append("
"); sb.AppendFormat("{0}", linkText, linkAddress); sb.Append("
"); } /// /// Inspects supplied type name to see if it might have a predictable aws/msdn /// doc url. /// /// private string PredictHtmlDocsLink(string typeName) { if (!typeName.Equals("None", StringComparison.Ordinal)) { var t = typeName.TrimEnd(new char[] { '[', ']' }); if (t.StartsWith("Amazon.PowerShell", StringComparison.Ordinal)) { switch (t) { case "Amazon.PowerShell.Cmdlets.DDB.Model.TableSchema": return "Amazon_DynamoDB_TableSchema.html"; default: return null; } } if (t.StartsWith("Amazon.", StringComparison.Ordinal)) { // generate the old 'full' namepath and then shrink it down in the same way // as the sdk to avoid filepath limitations. Example SDK path: // http://docs.aws.amazon.com/sdkfornet/v3/apidocs/index.html?page=EC2/TEC2ScheduledInstance.html&tocid=Amazon_EC2_Model_ScheduledInstance // Note how the pages are arranged beneath an extra folder var nameComponents = t.Split(new char[] { '.' }, StringSplitOptions.RemoveEmptyEntries); var serviceName = nameComponents[1]; if (ServiceNamespaceContractions.ContainsKey(serviceName)) serviceName = ServiceNamespaceContractions[serviceName]; var tName = nameComponents[nameComponents.Length - 1]; var sdkTypePagePath = ShrinkSdkLongFilepath($"T{serviceName}{tName}"); return $"{SDKHelpRoot}index.html?page={serviceName}/{sdkTypePagePath}.html&tocid={t.Replace('.', '_')}"; } // 'spot' some common types we see in external apis if (_msdnDocLinks.ContainsKey(t)) return _msdnDocLinks[t]; } return null; } /// /// Inspects the cmdlet to see if it has a backwards compatibility 'legacy' alias /// and if so, adds it to the collection to be emitted into the pstoolsref-legacyaliases.html /// page. /// /// /// /// The legacy alias, if one exists private string InspectForLegacyAliasAttribution(string serviceName, string cmdletName, Attribute awsCmdletAttribute) { var legacyAlias = ExtractLegacyAlias(awsCmdletAttribute); if (!string.IsNullOrEmpty(legacyAlias)) { List> aliases; if (_legacyAliasesByService.ContainsKey(serviceName)) aliases = _legacyAliasesByService[serviceName]; else { aliases = new List>(); _legacyAliasesByService.Add(serviceName, aliases); } aliases.Add(new Tuple(legacyAlias, cmdletName)); } return legacyAlias; } // copied from the sdk help generator - we should share this one day, or generate both // docs from one tool private static string ShrinkSdkLongFilepath(string name) { var fixedUpName = name.Replace('.', '_'); // don't use encoded <> in filename, as browsers re-encode it in links to %3C // and the link fails fixedUpName = fixedUpName.Replace("<", "!").Replace(">", "!"); fixedUpName = fixedUpName.Replace("Amazon", ""); fixedUpName = fixedUpName.Replace("_Model_", ""); fixedUpName = fixedUpName.Replace("_Begin", ""); fixedUpName = fixedUpName.Replace("_End", ""); fixedUpName = fixedUpName.Replace("Client_", ""); fixedUpName = fixedUpName.Replace("+", ""); fixedUpName = fixedUpName.Replace("_", ""); foreach (var k in ServiceNamespaceContractions.Keys) { fixedUpName = fixedUpName.Replace(k, ServiceNamespaceContractions[k]); } if (fixedUpName.Length > MAX_FILE_SIZE) { throw new ApplicationException(string.Format("Filename: {0} is too long", fixedUpName)); } return fixedUpName; } private static bool RequiresLinkTarget(string typeName) { // bit of a hack to make sure links to ps model types we might have // stay in the frame. sdk and msdn links go to a new tab. return !(typeName.StartsWith("Amazon.PowerShell.", StringComparison.Ordinal)); } private static void CopyWebHelpStaticFiles(string webFilesRoot) { Console.WriteLine("Copying Web Help DocSet Static Resources"); var sourceLocation = Directory.GetParent(typeof(PsHelpGenerator).Assembly.Location).FullName; IOUtils.DirectoryCopy(Path.Combine(sourceLocation, "..", "..", "..", "..", "AWSPSGeneratorLib", "HelpMaterials", "WebHelp", "StaticContent"), webFilesRoot, true); } private void CreateVersionInfoFile(string path) { if (!Directory.Exists(path)) Directory.CreateDirectory(path); // write the json file containing the assembly version; this can then // be injected onto each page using load-time script var v = CmdletAssembly.GetName().Version.ToString(); var assemblyVersionsContent = string.Format("{{ \"awspowershell.dll\" : \"{0}\" }}", v); File.WriteAllText(Path.Combine(path, "assemblyversions.json"), assemblyVersionsContent); } private static void CleanWebHelpOutputFolder(string outputFolder) { if (!Directory.Exists(outputFolder)) return; try { Console.WriteLine("Cleaning out web help output folder"); Directory.Delete(outputFolder, true); } catch (IOException) { Console.WriteLine("WARNING: Failed to clean web help output folder '{0}'", outputFolder); } } private static string ExtractLegacyAlias(object awsCmdletAttribute) { if (awsCmdletAttribute == null) return null; var awsCmdletAttributeType = awsCmdletAttribute.GetType(); return awsCmdletAttributeType.GetProperty("LegacyAlias").GetValue(awsCmdletAttribute, null) as string; } /// /// Injects the detected legacy aliases into the pstoolsref-legacyaliases.html template /// page and copies it to the output folder. /// private void WriteLegacyAliasesPage() { var templateFilename = "pstoolsref-legacyaliases.html"; using (var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream("AWSPowerShellGenerator.HelpMaterials.WebHelp.Templates." + templateFilename)) using (var reader = new StreamReader(stream)) { var finalBody = reader.ReadToEnd(); var aliasTables = new StringBuilder(); if (_legacyAliasesByService.ContainsKey(TOCWriter.CommonTOCName)) { var aliases = _legacyAliasesByService[TOCWriter.CommonTOCName]; WriteLegacyAliasesForService(aliasTables, TOCWriter.CommonTOCName, aliases); } var services = _legacyAliasesByService.Keys.ToList(); services.Sort(); foreach (var service in services) { if (service.Equals(TOCWriter.CommonTOCName, StringComparison.Ordinal)) continue; var aliases = _legacyAliasesByService[service]; WriteLegacyAliasesForService(aliasTables, service, aliases); } finalBody = finalBody.Replace("{LEGACY_ALIASES_SNIPPET}", aliasTables.ToString()); var filename = Path.Combine(OutputFolder, "items", templateFilename); using (var writer = new StreamWriter(filename)) { writer.Write(finalBody); } } } private void WriteLegacyAliasesForService(StringBuilder aliasTables, string serviceName, IEnumerable> aliases) { aliasTables.AppendFormat("

{0}

", serviceName); aliasTables.AppendLine(""); aliasTables.AppendLine(""); aliasTables.AppendLine(""); foreach (var a in aliases.OrderBy(alias => alias.Item1)) { var alias = a.Item1; var cmdlet = a.Item2; var cmdletLink = string.Format("{0}", cmdlet); aliasTables.AppendFormat("", alias, cmdletLink); aliasTables.AppendLine(); } aliasTables.AppendLine(""); aliasTables.AppendLine("
AliasCmdlet
{0}{1}
"); } } }