using Amazon.S3; using Amazon.S3.Util; using Amazon.S3.Model; using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; using System.Threading; using System.Net; using Amazon.Runtime.Internal; using Amazon.Runtime; using System.IO; using System.Reflection; using Amazon.SimpleNotificationService; using Amazon.SimpleNotificationService.Model; using System.Security.Cryptography; using Xunit; namespace Amazon.DNXCore.IntegrationTests { internal class UtilityMethods { public const string SDK_TEST_PREFIX = "aws-net-sdk"; static string _accountId; /// /// There is not a good way to get account id. This hack creates /// a topic gets the account id out of the ARN and then deletes the topic. /// public static string AccountId { get { if (_accountId == null) { var createRequest = new CreateTopicRequest { Name = "sdk-accountid-lookup" + DateTime.Now.Ticks }; using (var snsClient = CreateClient()) { var response = snsClient.CreateTopicAsync(createRequest).Result; var tokens = response.TopicArn.Split(':'); _accountId = tokens[4]; snsClient.DeleteTopicAsync(new DeleteTopicRequest { TopicArn = response.TopicArn }).Wait(); } } return _accountId; } } public async static Task CreateBucketAsync(IAmazonS3 s3Client, string testName, bool setPublicACLs = false) { string bucketName = string.Format("{0}-{1}-{2}", UtilityMethods.SDK_TEST_PREFIX, testName, DateTime.Now.Ticks).ToLower().Replace('_','-'); await s3Client.PutBucketAsync(new PutBucketRequest { BucketName = bucketName }).ConfigureAwait(false); if (setPublicACLs) { await SetPublicBucketACLs(s3Client, bucketName); } return bucketName; } private static async Task SetPublicBucketACLs(IAmazonS3 client, string bucketName) { await client.PutBucketOwnershipControlsAsync(new PutBucketOwnershipControlsRequest { BucketName = bucketName, OwnershipControls = new OwnershipControls { Rules = new List { new OwnershipControlsRule{ObjectOwnership = ObjectOwnership.BucketOwnerPreferred} } } }); await client.PutPublicAccessBlockAsync(new PutPublicAccessBlockRequest { BucketName = bucketName, PublicAccessBlockConfiguration = new PublicAccessBlockConfiguration { BlockPublicAcls = false } }); } public static string GenerateName() { return GenerateName(SDK_TEST_PREFIX + "-"); } public static string GenerateName(string name) { return (SDK_TEST_PREFIX + "-" + name + "-" + new Random().Next()).ToLower().Replace('_', '-'); } public static void WriteFile(string path, string contents) { string fullPath = Path.GetFullPath(path); new DirectoryInfo(Path.GetDirectoryName(fullPath)).Create(); File.WriteAllText(fullPath, contents); } public static void GenerateFile(string path, long size) { string contents = GenerateTestContents(size); WriteFile(path, contents); } public static string GenerateTestContents(long size) { StringBuilder sb = new StringBuilder(); for (long i = 0; i < size; i++) { char c = (char)('a' + (i % 26)); sb.Append(c); } string contents = sb.ToString(); return contents; } public static Task DeleteBucketWithObjectsAsync(IAmazonS3 s3Client, string bucketName) { return DeleteS3BucketWithObjectsAsync(s3Client, bucketName); } private async static Task DeleteS3BucketWithObjectsAsync(IAmazonS3 s3Client, string bucketName, CancellationToken cancellationToken = new CancellationToken()) { // Validations. if (s3Client == null) { throw new ArgumentNullException("s3Client", "The s3Client cannot be null!"); } if (string.IsNullOrEmpty(bucketName)) { throw new ArgumentNullException("bucketName", "The bucketName cannot be null or empty string!"); } var listVersionsRequest = new ListVersionsRequest { BucketName = bucketName }; ListVersionsResponse listVersionsResponse; string lastRequestId = null; // Iterate through the objects in the bucket and delete them. do { // Check if the operation has been canceled. cancellationToken.ThrowIfCancellationRequested(); // List all the versions of all the objects in the bucket. listVersionsResponse = await s3Client.ListVersionsAsync(listVersionsRequest).ConfigureAwait(false); // Silverlight uses HTTP caching, so avoid an infinite loop by throwing an exception if (string.Equals(lastRequestId, listVersionsResponse.ResponseMetadata.RequestId, StringComparison.OrdinalIgnoreCase)) { throw new InvalidOperationException(); } lastRequestId = listVersionsResponse.ResponseMetadata.RequestId; if (listVersionsResponse.Versions.Count == 0) { // If the bucket has no objects break the loop. break; } var keyVersionList = new List(listVersionsResponse.Versions.Count); for (int index = 0; index < listVersionsResponse.Versions.Count; index++) { keyVersionList.Add(new KeyVersion { Key = listVersionsResponse.Versions[index].Key, VersionId = listVersionsResponse.Versions[index].VersionId }); } try { // Delete the current set of objects. var deleteObjectsResponse = await s3Client.DeleteObjectsAsync(new DeleteObjectsRequest { BucketName = bucketName, Objects = keyVersionList, Quiet = true }).ConfigureAwait(false); //if (!deleteOptions.QuietMode) //{ // // If quiet mode is not set, update the client with list of deleted objects. // InvokeS3DeleteBucketWithObjectsUpdateCallback( // updateCallback, // new S3DeleteBucketWithObjectsUpdate // { // DeletedObjects = deleteObjectsResponse.DeletedObjects // } // ); //} } catch //(DeleteObjectsException deleteObjectsException) { //if (deleteOptions.ContinueOnError) //{ // // Continue the delete operation if an error was encountered. // // Update the client with the list of objects that were deleted and the // // list of objects on which the delete failed. // InvokeS3DeleteBucketWithObjectsUpdateCallback( // updateCallback, // new S3DeleteBucketWithObjectsUpdate // { // DeletedObjects = deleteObjectsException.Response.DeletedObjects, // DeleteErrors = deleteObjectsException.Response.DeleteErrors // } // ); //} //else //{ // // Re-throw the exception if an error was encountered. // throw; //} throw; } // Set the markers to get next set of objects from the bucket. listVersionsRequest.KeyMarker = listVersionsResponse.NextKeyMarker; listVersionsRequest.VersionIdMarker = listVersionsResponse.NextVersionIdMarker; } // Continue listing objects and deleting them until the bucket is empty. while (listVersionsResponse.IsTruncated); const int maxRetries = 10; for (int retries = 1; retries <= maxRetries; retries++) { try { // Bucket is empty, delete the bucket. await s3Client.DeleteBucketAsync(new DeleteBucketRequest { BucketName = bucketName }).ConfigureAwait(false); break; } catch (AmazonS3Exception e) { if (e.StatusCode != HttpStatusCode.Conflict || retries == maxRetries) throw; else DefaultRetryPolicy.WaitBeforeRetry(retries, 5000); } } //// Signal that the operation is completed. //asyncCancelableResult.SignalWaitHandleOnCompleted(); } public static AWSCredentials CreateTemporaryCredentials() { using (var sts = TestBase.CreateClient()) { var creds = sts.GetSessionTokenAsync().Result.Credentials; return creds; } } public static T WaitUntilSuccess(Func loadFunction, int sleepSeconds = 5, int maxWaitSeconds = 300) { T result = default(T); WaitUntil(() => { try { result = loadFunction(); return true; } catch { return false; } }, sleepSeconds, maxWaitSeconds); return result; } public static void WaitUntilSuccess(Action action, int sleepSeconds = 5, int maxWaitSeconds = 300) { WaitUntil(() => { try { action(); return true; } catch { return false; } }, sleepSeconds, maxWaitSeconds); } public static void WaitUntil(Func matchFunction, int sleepSeconds = 5, int maxWaitSeconds = 300, bool failIfNotCompleted = true) { if (sleepSeconds < 0) throw new ArgumentOutOfRangeException("sleepSeconds"); if (maxWaitSeconds < 0) throw new ArgumentOutOfRangeException("maxWaitSeconds"); var sleepTime = TimeSpan.FromSeconds(sleepSeconds); var maxTime = TimeSpan.FromSeconds(maxWaitSeconds); var endTime = DateTime.Now + maxTime; while (DateTime.Now < endTime) { if (matchFunction()) return; Sleep(sleepTime); } if(failIfNotCompleted) throw new TimeoutException(string.Format("Wait condition was not satisfied for {0} seconds", maxWaitSeconds)); } public static void Sleep(TimeSpan ts) { Task.Delay(ts).Wait(); } public static async Task SleepAsync(TimeSpan ts) { await Task.Delay(ts).ConfigureAwait(false); } public static void RunAsSync(Func asyncFunc) { Task.Run(asyncFunc).Wait(); } public static string GetResourceText(string resourceName) { using (StreamReader reader = new StreamReader(GetResourceStream(resourceName))) { return reader.ReadToEnd(); } } public static string GetFileText(string resourceName) { return File.ReadAllText(resourceName); } public static Stream GetResourceStream(string resourceName) { Assembly assembly = typeof(UtilityMethods).GetTypeInfo().Assembly; var resource = FindResourceName(resourceName); Stream stream = assembly.GetManifestResourceStream(resource); return stream; } public static string FindResourceName(string partialName) { return FindResourceName(s => s.IndexOf(partialName, StringComparison.OrdinalIgnoreCase) >= 0).Single(); } public static IEnumerable FindResourceName(Predicate match) { Assembly assembly = typeof(UtilityMethods).GetTypeInfo().Assembly; var allResources = assembly.GetManifestResourceNames(); foreach (var resource in allResources) { if (match(resource)) yield return resource; } } public static T CreateClient() where T : AmazonServiceClient { return TestBase.CreateClient(); } private static byte[] computeHash(string file) { byte[] fileMD5; using (Stream fileStream = File.OpenRead(file)) { fileMD5 = MD5.Create().ComputeHash(fileStream); } return fileMD5; } public static void CompareFiles(string file1, string file2) { byte[] file1MD5 = computeHash(file1); byte[] file2MD5 = computeHash(file2); Assert.Equal(file1MD5.Length, file2MD5.Length); for (int i = 0; i < file1MD5.Length; i++) { Assert.Equal(file1MD5[i], file2MD5[i]); } } } }