using System; using System.CodeDom; using System.Collections.Generic; using System.IO; using System.Linq; using System.Reflection; using System.Text; using System.Threading.Tasks; using System.Xml.XPath; using System.Xml.Linq; using SDKDocGenerator.Writers; using System.Xml; using System.Diagnostics; using System.Text.RegularExpressions; using System.Web; namespace SDKDocGenerator { public static class NDocUtilities { public const string MSDN_TYPE_URL_PATTERN = "https://msdn.microsoft.com/en-us/library/{0}.aspx"; public const string DOC_SAMPLES_SUBFOLDER = "AWSSDKDocSamples"; public const string crossReferenceOpeningTagText = " and tags public const string crossReferenceClosingTagText = "/>"; public const string crefAttributeName = "cref"; public const string hrefAttributeName = "href"; public const string nameAttributeName = "name"; public const string targetAttributeName = "target"; // inner attribute of a cross reference tag we're interested in public static readonly string innerCrefAttributeText = crefAttributeName + "=\""; public static readonly string innerHrefAttributeText = hrefAttributeName + "=\""; private static readonly Dictionary NdocToHtmlElementMapping = new Dictionary(StringComparer.Ordinal) { { "summary", "p" }, { "para", "p" }, { "see", "a" }, { "paramref", "code" } }; #region manage ndoc instances // The reason we cache the doc data on the side instead of directly referencing doc instances from // the type information is becasue we are loading the assemblies for reflection in a separate app domain. private static IDictionary> _ndocCache = new Dictionary>(); public static string GenerateDocId(string serviceName, string platform) { // platform can be null; in which case we just use an empty string to construct the id. return string.Format("{0}:{1}", serviceName, platform == null ? "" : platform); } public static void LoadDocumentation(string assemblyName, string serviceName, string platform, GeneratorOptions options) { var ndocFilename = assemblyName + ".xml"; var platformSpecificNdocFile = Path.Combine(options.SDKAssembliesRoot, platform, ndocFilename); if (File.Exists(platformSpecificNdocFile)) { var docId = GenerateDocId(serviceName, platform); _ndocCache.Add(docId, CreateNDocTable(platformSpecificNdocFile, serviceName, options)); } } public static void UnloadDocumentation(string serviceName, string platform) { var docId = GenerateDocId(serviceName, platform); _ndocCache.Remove(docId); } public static IDictionary GetDocumentationInstance(string serviceName, string platform) { return GetDocumentationInstance(GenerateDocId(serviceName, platform)); } public static IDictionary GetDocumentationInstance(string docId) { IDictionary doc = null; if (_ndocCache.TryGetValue(docId, out doc)) { return doc; } return null; } private static IDictionary CreateNDocTable(string filePath, string serviceName, GeneratorOptions options) { var dict = new Dictionary(); var document = LoadAssemblyDocumentationWithSamples(filePath, options.CodeSamplesRootFolder, serviceName); PreprocessCodeBlocksToPreTags(options, document); foreach (var element in document.XPathSelectElements("//members/member")) { var xattribute = element.Attributes().FirstOrDefault(x => x.Name.LocalName == "name"); if (xattribute == null) continue; dict[xattribute.Value] = element; } return dict; } #endregion public static XElement FindDocumentation(AbstractWrapper wrapper) { var ndoc = GetDocumentationInstance(wrapper.DocId); return FindDocumentation(ndoc, wrapper); } public static XElement FindDocumentation(IDictionary ndoc, AbstractWrapper wrapper) { if (ndoc == null) return null; if (wrapper is TypeWrapper) return FindDocumentation(ndoc, (TypeWrapper)wrapper); if (wrapper is PropertyInfoWrapper) return FindDocumentation(ndoc, (PropertyInfoWrapper)wrapper); if (wrapper is MethodInfoWrapper) return FindDocumentation(ndoc, (MethodInfoWrapper)wrapper); if (wrapper is ConstructorInfoWrapper) return FindDocumentation(ndoc, (ConstructorInfoWrapper)wrapper); if (wrapper is FieldInfoWrapper) return FindDocumentation(ndoc, (FieldInfoWrapper)wrapper); return null; } public static XElement FindDocumentation(IDictionary ndoc, FieldInfoWrapper info) { var signature = string.Format("F:{0}.{1}", info.DeclaringType.FullName, info.Name); XElement element; if (!ndoc.TryGetValue(signature, out element)) return null; return element; } public static XElement FindFieldDocumentation(TypeWrapper type, string fieldName) { var ndoc = GetDocumentationInstance(type.DocId); return FindFieldDocumentation(ndoc, type, fieldName); } public static XElement FindFieldDocumentation(IDictionary ndoc, TypeWrapper type, string fieldName) { var signature = string.Format("F:{0}.{1}", type.FullName, fieldName); XElement element; if (!ndoc.TryGetValue(signature, out element)) return null; return element; } public static XElement FindDocumentation(IDictionary ndoc, TypeWrapper type) { var signature = "T:" + type.FullName; XElement element; if (!ndoc.TryGetValue(signature, out element)) return null; return element; } public static XElement FindDocumentation(IDictionary ndoc, PropertyInfoWrapper info) { var type = info.DeclaringType; var signature = string.Format("P:{0}.{1}", type.FullName, info.Name); XElement element; if (!ndoc.TryGetValue(signature, out element)) return null; return element; } public static XElement FindDocumentation(IDictionary ndoc, EventInfoWrapper info) { var type = info.DeclaringType; var signature = string.Format("E:{0}.{1}", type.FullName, info.Name); XElement element; if (!ndoc.TryGetValue(signature, out element)) return null; return element; } public static string DetermineNDocNameLookupSignature(MethodInfo info, string docId) { return DetermineNDocNameLookupSignature(new MethodInfoWrapper(info, docId)); } public static string DetermineNDocNameLookupSignature(MethodInfoWrapper info) { var type = info.DeclaringType; var fullName = type.FullName ?? type.Namespace + "." + type.Name; var typeGenericParameters = type.GetGenericArguments(); var parameters = new StringBuilder(); foreach (var param in info.GetParameters()) { if (parameters.Length > 0) parameters.Append(","); DetermineParameterName(param.ParameterType, parameters, typeGenericParameters); if (param.IsOut) { parameters.Append("@"); } } var genericTag = ""; if (info.IsGenericMethod) { genericTag = "``" + info.GetGenericArguments().Length; } var signature = parameters.Length > 0 ? string.Format("M:{0}.{1}{2}({3})", fullName, info.Name, genericTag, parameters) : string.Format("M:{0}.{1}{2}", fullName, info.Name, genericTag); return signature; } private static void DetermineParameterName(TypeWrapper parameterTypeInfo, StringBuilder parameters, IList typeGenericParameters) { if (parameterTypeInfo.IsGenericParameter) { var typeGenericParameterIndex = typeGenericParameters.IndexOf(parameterTypeInfo); var isClassGenericParameter = typeGenericParameterIndex >= 0; if (isClassGenericParameter) parameters.AppendFormat("`{0}", typeGenericParameterIndex); else parameters.AppendFormat("``{0}", 0); } else if (parameterTypeInfo.IsGenericType) { parameters .Append(parameterTypeInfo.GenericTypeName) .Append("{"); IList args = parameterTypeInfo.GenericTypeArguments(); for (var i = 0; i < args.Count; i++) { if (i != 0) { parameters.Append(","); } DetermineParameterName(args[i], parameters, typeGenericParameters); } parameters.Append("}"); } else { parameters.Append(parameterTypeInfo.FullName); } } public static XElement FindDocumentation(IDictionary ndoc, MethodInfoWrapper info) { var signature = DetermineNDocNameLookupSignature(info); XElement element; if (!ndoc.TryGetValue(signature, out element)) return null; return element; } public static XElement FindDocumentation(IDictionary ndoc, ConstructorInfoWrapper info) { var type = info.DeclaringType; var parameters = new StringBuilder(); var typeGenericParameters = type.GetGenericArguments(); foreach (var param in info.GetParameters()) { if (parameters.Length > 0) parameters.Append(","); DetermineParameterName(param.ParameterType, parameters, typeGenericParameters); if (param.IsOut) { parameters.Append("@"); } } var formattedParmaters = parameters.Length > 0 ? string.Format("({0})", parameters) : parameters.ToString(); var signature = string.Format("M:{0}.#ctor{1}", type.FullName, formattedParmaters); XElement element; if (!ndoc.TryGetValue(signature, out element)) return null; return element; } public static string FindParameterDocumentation(XElement ndoc, string name) { if (ndoc == null) return string.Empty; var node = ndoc.XPathSelectElement(string.Format("./param[@name = '{0}']", name)); if (node == null) return string.Empty; return node.Value; } public static string FindReturnDocumentation(XElement ndoc) { if (ndoc == null) return string.Empty; var node = ndoc.XPathSelectElement("returns"); if (node == null) return string.Empty; return node.Value; } /// /// Finds the Async version of the specified non async method info. /// /// /// /// public static XElement FindDocumentationAsync(IDictionary ndoc, MethodInfoWrapper info) { if (ndoc == null) return null; var type = info.DeclaringType; if (type.FullName == null) return null; var parameters = new StringBuilder(); foreach (var param in info.GetParameters()) { if (parameters.Length > 0) parameters.Append(","); if (param.ParameterType.IsGenericType) { parameters .Append(param.ParameterType.GenericTypeName) .Append("{") .Append(string.Join(",", param.ParameterType.GenericTypeArguments().Select(a => a.FullName))) .Append("}"); } else { parameters.Append(param.ParameterType.FullName); if (param.IsOut) parameters.Append("@"); } } if (parameters.Length > 0) parameters.Append(","); // Async methods have this additional parameter parameters.Append("System.Threading.CancellationToken"); var signature = parameters.Length > 0 ? string.Format("M:{0}.{1}({2})", type.FullName, info.Name + "Async", parameters) : string.Format("M:{0}.{1}", type.FullName, info.Name + "Async"); XElement element; if (!ndoc.TryGetValue(signature, out element)) return null; return element; } public static string TransformDocumentationToHTML(XElement element, string rootNodeName, AbstractTypeProvider typeProvider, FrameworkVersion version) { if (element == null) return string.Empty; var rootNode = element.XPathSelectElement(rootNodeName); if (rootNode == null) return string.Empty; if (rootNodeName.Equals("seealso", StringComparison.OrdinalIgnoreCase)) return SeeAlsoElementToHTML(rootNode, typeProvider, version); else return DocBlobToHTML(rootNode, typeProvider, version); } private static string SeeAlsoElementToHTML(XElement rootNode, AbstractTypeProvider typeProvider, FrameworkVersion version) { var reader = rootNode.CreateReader(); reader.MoveToContent(); var innerXml = reader.ReadInnerXml(); string content = ""; var href = rootNode.Attribute("href"); if (href != null) { content += string.Format(@"", href.Value, innerXml); } var cref = rootNode.Attribute("cref"); if (cref != null) { content += BaseWriter.CreateCrossReferenceTagReplacement(typeProvider, cref.Value, version); } return content; } private static string DocBlobToHTML(XElement rootNode, AbstractTypeProvider typeProvider, FrameworkVersion version) { using (var textWriter = new StringWriter()) { var writerSettings = new XmlWriterSettings { OmitXmlDeclaration = true }; using (var writer = XmlWriter.Create(textWriter, writerSettings)) { var reader = rootNode.CreateReader(); while (reader.Read()) { switch (reader.NodeType) { case XmlNodeType.Element: // handle self-closing element, like // this must be read before any other reading is done var selfClosingElement = reader.IsEmptyElement; // element name substitution, if necessary string elementName; if (!NdocToHtmlElementMapping.TryGetValue(reader.LocalName, out elementName)) elementName = reader.LocalName; // some elements can't be empty, use this variable for that string emptyElementContents = null; // start element writer.WriteStartElement(elementName); // copy over attributes if (reader.HasAttributes) { var isAbsoluteLink = false; var hasTarget = false; for (int i = 0; i < reader.AttributeCount; i++) { reader.MoveToAttribute(i); var attributeName = reader.Name; var attributeValue = reader.Value; var isCref = string.Equals(attributeName, crefAttributeName, StringComparison.Ordinal); var isHref = string.Equals(attributeName, hrefAttributeName, StringComparison.Ordinal); var isName = string.Equals(attributeName, nameAttributeName, StringComparison.Ordinal); var isTarget = string.Equals(attributeName, targetAttributeName, StringComparison.Ordinal); var writeAttribute = true; if (isCref) { // replace cref with href attributeName = hrefAttributeName; // extract type name from cref value for emptyElementContents var crefParts = attributeValue.Split(':'); if (crefParts.Length != 2) throw new InvalidOperationException(); var typeName = crefParts[1]; var targetType = typeProvider.GetType(typeName); if (targetType == null) { emptyElementContents = typeName; //If the type cannot be found do not render out the href attribute. //This will make it so things such as properties which we do not have //specific doc pages for do not render as a broken link but we can still //use the crefs in the code correctly. writeAttribute = false; } else emptyElementContents = targetType.CreateReferenceHtml(fullTypeName: true); } else if (isHref) { // extract href value for emptyElementContents emptyElementContents = attributeValue; if(attributeValue.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || attributeValue.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) { isAbsoluteLink = true; } } else if (isName) { emptyElementContents = attributeValue; } else if (isTarget) { hasTarget = true; } if(writeAttribute) { writer.WriteAttributeString(attributeName, attributeValue); } } if(elementName == "a" && isAbsoluteLink && !hasTarget) { //Add a target=\"_blank\" to allow the absolute link to break out //of the frame. writer.WriteAttributeString(targetAttributeName, "_blank"); } } // if this is a self-closing element, close it if (selfClosingElement) { // write empty element contents, if any if (!string.IsNullOrEmpty(emptyElementContents)) { writer.WriteRaw(emptyElementContents); } // close element now writer.WriteEndElement(); } break; case XmlNodeType.EndElement: writer.WriteEndElement(); break; case XmlNodeType.Text: writer.WriteRaw(reader.Value); break; default: throw new InvalidOperationException(); } } } return textWriter.ToString(); } } public static void PreprocessCodeBlocksToPreTags(GeneratorOptions options, XDocument doc) { var nodesToRemove = new List(); var codeNodes = doc.XPathSelectElements("//code"); foreach (var codeNode in codeNodes) { string processedCodeSample = null; var xattribute = codeNode.Attributes().FirstOrDefault(x => x.Name.LocalName == "source"); if (xattribute != null) { var sourceRelativePath = xattribute.Value; xattribute = codeNode.Attributes().FirstOrDefault(x => x.Name.LocalName == "region"); if (xattribute == null) continue; var regionName = xattribute.Value; var samplePath = FindSampleCodePath(options.CodeSamplesRootFolder, sourceRelativePath); if (samplePath == null) { Console.Error.WriteLine("Error finding sample path for {0}", sourceRelativePath); continue; } var content = File.ReadAllText(samplePath); var startPos = content.IndexOf("#region " + regionName); if (startPos == -1) { Console.Error.WriteLine("Error finding region for {0}", regionName); continue; } startPos = content.IndexOf('\n', startPos); var endPos = content.IndexOf("#endregion", startPos); var sampleCode = content.Substring(startPos, endPos - startPos); processedCodeSample = HttpUtility.HtmlEncode(sampleCode); } else { processedCodeSample = HttpUtility.HtmlEncode(codeNode.Value); } if (processedCodeSample != null && processedCodeSample.IndexOf('\n') > -1) { processedCodeSample = LeftJustifyCodeBlocks(processedCodeSample); var preElement = new XElement("pre", processedCodeSample); preElement.SetAttributeValue("class", "brush: csharp"); codeNode.AddAfterSelf(preElement); nodesToRemove.Add(codeNode); string title = null; xattribute = codeNode.Attributes().FirstOrDefault(x => x.Name.LocalName == "title"); if (xattribute != null) title = xattribute.Value; if (title != null) { var titleElement = new XElement("h4", title); titleElement.SetAttributeValue("class", "csharp-code-sample-title"); preElement.AddBeforeSelf(titleElement); } } } nodesToRemove.ForEach(x => x.Remove()); } private static string FindSampleCodePath(string codeSampleRootDirectory, string relativePath) { if (string.IsNullOrEmpty(codeSampleRootDirectory)) return null; var fullPath = Path.Combine(codeSampleRootDirectory, relativePath); return !File.Exists(fullPath) ? null : fullPath; } private static string LeftJustifyCodeBlocks(string codeBlock) { // Switch tabs to 4 spaces var block = new StringBuilder(codeBlock).Replace("\t", new string(' ', 4)).ToString(); // Find the nearest indent location var nearestIndent = int.MaxValue; using (var reader = new StringReader(block)) { string line; while ((line = reader.ReadLine()) != null) { int indent = FindFirstNoWhitePosition(line); if (indent != -1 && indent < nearestIndent) nearestIndent = indent; } } // Substring all lines with content to the indent location; var reformattedBuilder = new StringBuilder(); using (var reader = new StringReader(block)) { string line; while ((line = reader.ReadLine()) != null) { if (string.IsNullOrWhiteSpace(line)) reformattedBuilder.AppendLine(line); else { var trimedLine = line.Substring(nearestIndent); reformattedBuilder.AppendLine(trimedLine); } } } return reformattedBuilder.ToString(); } private static int FindFirstNoWhitePosition(string line) { if (string.IsNullOrWhiteSpace(line)) return -1; for (int space = 0; space < line.Length; space++) { if (!Char.IsWhiteSpace(line[space])) return space; } return -1; } public static XDocument LoadAssemblyDocumentationWithSamples(string filePath, string samplesDir, string serviceName) { if (!string.IsNullOrEmpty(samplesDir)) { var extraDocNodes = new List(); foreach (var pattern in new[] { ".extra.xml", ".GeneratedSamples.extra.xml" }) { var extraFile = Path.Combine(samplesDir, DOC_SAMPLES_SUBFOLDER, serviceName + pattern); if (File.Exists(extraFile)) { var extraDoc = new XmlDocument(); extraDoc.Load(extraFile); foreach (XmlNode node in extraDoc.SelectNodes("docs/doc")) { extraDocNodes.Add(node); } } } if (extraDocNodes.Any()) { Trace.WriteLine(String.Format("Merging {0} code samples into {1}", serviceName, filePath)); var sdkDoc = new XmlDocument(); sdkDoc.Load(filePath); var examplesMap = BuildExamplesMap(extraDocNodes); ProcessExtraDoc(sdkDoc, examplesMap); return XDocument.Load(new XmlNodeReader(sdkDoc), LoadOptions.PreserveWhitespace); } } return XDocument.Load(filePath, LoadOptions.PreserveWhitespace); } private static IDictionary BuildExamplesMap(List docNodes) { Trace.WriteLine(String.Format("Found {0} extra doc nodes", docNodes.Count), "verbose"); var map = new Dictionary(StringComparer.Ordinal); foreach (var docNode in docNodes) { var members = docNode.SelectNodes("members/member"); foreach (XmlNode memberNode in members) { var nameAttribute = memberNode.Attributes["name"]; if (null == nameAttribute) throw new InvalidDataException("unable to retrieve 'name' attribute for member node."); var memberSpec = nameAttribute.Value; var exampleNode = docNode.SelectSingleNode("value/example"); var content = exampleNode.InnerXml; if (map.ContainsKey(memberSpec)) map[memberSpec] += content; else map[memberSpec] = content; } } return map; } private static void ProcessExtraDoc(XmlDocument sdkDocument, IDictionary examplesMap) { foreach (var memberSpec in examplesMap.Keys) { var docNode = sdkDocument.SelectSingleNode(string.Format("doc/members/member[@name='{0}']", memberSpec)); if (null == docNode) { Trace.WriteLine(String.Format("** member name not found, skipping: {0}", memberSpec), "verbose"); continue; } XmlNode sdkExampleNode = docNode.SelectSingleNode("example"); if (null != sdkExampleNode) { sdkExampleNode.InnerXml = examplesMap[memberSpec]; } else { string sdkXml = docNode.InnerXml; sdkXml += String.Format("{0}", examplesMap[memberSpec]); docNode.InnerXml = sdkXml; } Trace.WriteLine(string.Format("Successfully updated SDK XML for member {0}", memberSpec), "verbose"); } } } }