using System; using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using System.IO; using System.IO.Compression; using PortingAssistant.Client.Model; using Newtonsoft.Json.Linq; using Amazon.S3; using PortingAssistant.Client.NuGet.Interfaces; namespace PortingAssistant.Client.NuGet { /// /// Compatibility checker for Portability Analyzer results /// public class PortabilityAnalyzerCompatibilityChecker : ICompatibilityChecker { private const string NamespaceLookupFile = "microsoftlibs.namespace.lookup.json"; private readonly ILogger _logger; private readonly IHttpService _httpService; private Dictionary _manifest; private static readonly int _maxProcessConcurrency = 3; private static readonly SemaphoreSlim _semaphore = new SemaphoreSlim(_maxProcessConcurrency); public PackageSourceType CompatibilityCheckerType => PackageSourceType.PORTABILITY_ANALYZER; /// /// Creates a new instance of Portability Analyzer compatibility checker /// /// The transferUtility object to read data from S3 /// Logger object public PortabilityAnalyzerCompatibilityChecker( IHttpService httpService, ILogger logger ) { _logger = logger; _httpService = httpService; _manifest = null; } /// /// Checks the packages in Portability Analyzer /// /// The package versions to check /// Path to the solution to check /// The results of the compatibility check public Dictionary> Check( IEnumerable packageVersions, string pathToSolution, bool isIncremental, bool incrementalRefresh) { var compatibilityTaskCompletionSources = new Dictionary>(); try { if (_manifest == null) { var manifestTask = GetManifestAsync(); manifestTask.Wait(); _manifest = manifestTask.Result; } var foundPackages = new Dictionary>(); packageVersions.ToList().ForEach(p => { if (p.PackageSourceType != PackageSourceType.SDK) { return; } var value = _manifest.GetValueOrDefault(p.PackageId, null); if (value != null) { compatibilityTaskCompletionSources.Add(p, new TaskCompletionSource()); if (!foundPackages.ContainsKey(value)) { foundPackages.Add(value, new List()); } foundPackages[value].Add(p); } }); _logger.LogInformation("Checking Portability Analyzer source for compatibility of {0} package(s)", foundPackages.Count); if (foundPackages.Any()) { Task.Run(() => { _semaphore.Wait(); try { ProcessCompatibility(packageVersions, foundPackages, compatibilityTaskCompletionSources); } finally { _semaphore.Release(); } }); } return compatibilityTaskCompletionSources.ToDictionary(t => t.Key, t => t.Value.Task); } catch (Exception ex) { foreach (var packageVersion in packageVersions) { if (compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var taskCompletionSource)) { taskCompletionSource.TrySetException( new PortingAssistantClientException(ExceptionMessage.PackageNotFound(packageVersion), ex)); } } return compatibilityTaskCompletionSources.ToDictionary(t => t.Key, t => t.Value.Task); } } /// /// Processes the package compatibility /// /// Collection of package versions to check /// Collection of packages found /// The results of the compatibility check to process private async void ProcessCompatibility(IEnumerable packageVersions, Dictionary> foundPackages, Dictionary> compatibilityTaskCompletionSources) { var packageVersionsFound = new HashSet(); var packageVersionsWithErrors = new HashSet(); foreach (var url in foundPackages) { try { _logger.LogInformation("Downloading {0} from {1}", url.Key, CompatibilityCheckerType); using var stream = await _httpService.DownloadS3FileAsync(url.Key); using var gzipStream = new GZipStream(stream, CompressionMode.Decompress); using var streamReader = new StreamReader(gzipStream); var packageFromS3 = JsonConvert.DeserializeObject(streamReader.ReadToEnd()); packageFromS3.Package.Name = url.Value.First().PackageId; foreach (var packageVersion in url.Value) { if (compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var taskCompletionSource)) { taskCompletionSource.SetResult(packageFromS3.Package); packageVersionsFound.Add(packageVersion); } } } catch (Exception ex) { if (ex.Message.Contains("404")) { _logger.LogInformation($"Encountered {ex.GetType()} while downloading and parsing {url.Key} " + $"from {CompatibilityCheckerType}, but it was ignored. " + $"ErrorMessage: {ex.Message}."); // filter all 404 errors ex = null; } else { _logger.LogError("Failed when downloading and parsing {0} from {1}, {2}", url.Key, CompatibilityCheckerType, ex); } foreach (var packageVersion in url.Value) { if (compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var taskCompletionSource)) { taskCompletionSource.SetException(new PortingAssistantClientException(ExceptionMessage.PackageNotFound(packageVersion), ex)); packageVersionsWithErrors.Add(packageVersion); } } } } foreach (var packageVersion in packageVersions) { if (packageVersionsFound.Contains(packageVersion) || packageVersionsWithErrors.Contains(packageVersion)) { continue; } if (compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var taskCompletionSource)) { var errorMessage = $"Could not find package {packageVersion} in external source; try checking an internal source."; _logger.LogInformation(errorMessage); var innerException = new PackageNotFoundException(errorMessage); taskCompletionSource.TrySetException(new PortingAssistantClientException(ExceptionMessage.PackageNotFound(packageVersion), innerException)); } } } private async Task> GetManifestAsync() { using var stream = await _httpService.DownloadS3FileAsync(NamespaceLookupFile); using var streamReader = new StreamReader(stream); var result = streamReader.ReadToEnd(); return JsonConvert.DeserializeObject(result).ToObject>(); } private class PackageFromS3 { public PackageDetails Package { get; set; } } } }