using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using System.IO;
using System.IO.Compression;
using PortingAssistant.Client.Model;
using Newtonsoft.Json.Linq;
using Amazon.S3;
using PortingAssistant.Client.NuGet.Interfaces;
namespace PortingAssistant.Client.NuGet
{
///
/// Compatibility checker for Portability Analyzer results
///
public class PortabilityAnalyzerCompatibilityChecker : ICompatibilityChecker
{
private const string NamespaceLookupFile = "microsoftlibs.namespace.lookup.json";
private readonly ILogger _logger;
private readonly IHttpService _httpService;
private Dictionary _manifest;
private static readonly int _maxProcessConcurrency = 3;
private static readonly SemaphoreSlim _semaphore = new SemaphoreSlim(_maxProcessConcurrency);
public PackageSourceType CompatibilityCheckerType => PackageSourceType.PORTABILITY_ANALYZER;
///
/// Creates a new instance of Portability Analyzer compatibility checker
///
/// The transferUtility object to read data from S3
/// Logger object
public PortabilityAnalyzerCompatibilityChecker(
IHttpService httpService,
ILogger logger
)
{
_logger = logger;
_httpService = httpService;
_manifest = null;
}
///
/// Checks the packages in Portability Analyzer
///
/// The package versions to check
/// Path to the solution to check
/// The results of the compatibility check
public Dictionary> Check(
IEnumerable packageVersions,
string pathToSolution, bool isIncremental, bool incrementalRefresh)
{
var compatibilityTaskCompletionSources = new Dictionary>();
try
{
if (_manifest == null)
{
var manifestTask = GetManifestAsync();
manifestTask.Wait();
_manifest = manifestTask.Result;
}
var foundPackages = new Dictionary>();
packageVersions.ToList().ForEach(p =>
{
if (p.PackageSourceType != PackageSourceType.SDK)
{
return;
}
var value = _manifest.GetValueOrDefault(p.PackageId, null);
if (value != null)
{
compatibilityTaskCompletionSources.Add(p, new TaskCompletionSource());
if (!foundPackages.ContainsKey(value))
{
foundPackages.Add(value, new List());
}
foundPackages[value].Add(p);
}
});
_logger.LogInformation("Checking Portability Analyzer source for compatibility of {0} package(s)", foundPackages.Count);
if (foundPackages.Any())
{
Task.Run(() =>
{
_semaphore.Wait();
try
{
ProcessCompatibility(packageVersions, foundPackages, compatibilityTaskCompletionSources);
}
finally
{
_semaphore.Release();
}
});
}
return compatibilityTaskCompletionSources.ToDictionary(t => t.Key, t => t.Value.Task);
}
catch (Exception ex)
{
foreach (var packageVersion in packageVersions)
{
if (compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var taskCompletionSource))
{
taskCompletionSource.TrySetException(
new PortingAssistantClientException(ExceptionMessage.PackageNotFound(packageVersion), ex));
}
}
return compatibilityTaskCompletionSources.ToDictionary(t => t.Key, t => t.Value.Task);
}
}
///
/// Processes the package compatibility
///
/// Collection of package versions to check
/// Collection of packages found
/// The results of the compatibility check to process
private async void ProcessCompatibility(IEnumerable packageVersions,
Dictionary> foundPackages,
Dictionary> compatibilityTaskCompletionSources)
{
var packageVersionsFound = new HashSet();
var packageVersionsWithErrors = new HashSet();
foreach (var url in foundPackages)
{
try
{
_logger.LogInformation("Downloading {0} from {1}", url.Key, CompatibilityCheckerType);
using var stream = await _httpService.DownloadS3FileAsync(url.Key);
using var gzipStream = new GZipStream(stream, CompressionMode.Decompress);
using var streamReader = new StreamReader(gzipStream);
var packageFromS3 = JsonConvert.DeserializeObject(streamReader.ReadToEnd());
packageFromS3.Package.Name = url.Value.First().PackageId;
foreach (var packageVersion in url.Value)
{
if (compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var taskCompletionSource))
{
taskCompletionSource.SetResult(packageFromS3.Package);
packageVersionsFound.Add(packageVersion);
}
}
}
catch (Exception ex)
{
if (ex.Message.Contains("404"))
{
_logger.LogInformation($"Encountered {ex.GetType()} while downloading and parsing {url.Key} " +
$"from {CompatibilityCheckerType}, but it was ignored. " +
$"ErrorMessage: {ex.Message}.");
// filter all 404 errors
ex = null;
}
else
{
_logger.LogError("Failed when downloading and parsing {0} from {1}, {2}", url.Key, CompatibilityCheckerType, ex);
}
foreach (var packageVersion in url.Value)
{
if (compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var taskCompletionSource))
{
taskCompletionSource.SetException(new PortingAssistantClientException(ExceptionMessage.PackageNotFound(packageVersion), ex));
packageVersionsWithErrors.Add(packageVersion);
}
}
}
}
foreach (var packageVersion in packageVersions)
{
if (packageVersionsFound.Contains(packageVersion) || packageVersionsWithErrors.Contains(packageVersion))
{
continue;
}
if (compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var taskCompletionSource))
{
var errorMessage = $"Could not find package {packageVersion} in external source; try checking an internal source.";
_logger.LogInformation(errorMessage);
var innerException = new PackageNotFoundException(errorMessage);
taskCompletionSource.TrySetException(new PortingAssistantClientException(ExceptionMessage.PackageNotFound(packageVersion), innerException));
}
}
}
private async Task> GetManifestAsync()
{
using var stream = await _httpService.DownloadS3FileAsync(NamespaceLookupFile);
using var streamReader = new StreamReader(stream);
var result = streamReader.ReadToEnd();
return JsonConvert.DeserializeObject(result).ToObject>();
}
private class PackageFromS3
{
public PackageDetails Package { get; set; }
}
}
}