/*******************************************************************************
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use
* this file except in compliance with the License. A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file.
* This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
* *****************************************************************************
*
* AWS Tools for Windows (TM) PowerShell (TM)
*
*/
using System;
using System.Linq;
using System.Management.Automation;
using Amazon.PowerShell.Common;
using Amazon.S3;
using Amazon.S3.Model;
using Amazon.S3.Transfer;
using System.IO;
using Amazon.PowerShell.Utils;
namespace Amazon.PowerShell.Cmdlets.S3
{
///
///
/// Downloads an S3 object, optionally including sub-objects, to a local file or folder location. Returns a
/// FileInfo or DirectoryInfo instance to the downloaded file or the containing folder.
///
///
/// Note that you can pipe an Amazon.S3.Model.S3Object instance to this cmdlet and its members will be used to
/// satisfy the BucketName, Key and optionally VersionId (if an S3ObjectVersion instance is supplied), parameters.
///
///
[Cmdlet("Read", "S3Object", DefaultParameterSetName = ParamSet_ToLocalFile)]
[OutputType(new Type[] { typeof(System.IO.FileInfo), typeof(System.IO.DirectoryInfo) })]
[AWSCmdlet("Downloads one or more objects from an S3 bucket to the local file system.", Operation = new[] { "GetObject" })]
[AWSCmdletOutput("System.IO.FileInfo instance if reading a single object or System.IO.DirectoryInfo instance for multi-object read.",
"Returns a System.IO.FileInfo instance representing the local file if reading a single object or a System.IO.DirectoryInfo instance to the root parent folder if reading multiple objects."
)]
public class ReadS3ObjectCmdlet : AmazonS3ClientCmdlet, IExecutor
{
const string ParamSet_ToLocalFile = "DownloadFile";
const string ParamSet_ToLocalFolder = "DownloadFolder";
const string ParamSet_FromS3Object = "FromS3ObjectToFileOrFolder";
// try and anticipate all the ways a user might mean 'get everything from root'
internal static readonly string[] rootIndicators = new string[] { "/", @"\", "*", "/*", @"\*" };
protected override bool IsSensitiveRequest { get; set; } = true;
#region Bucket Params
#region Parameter BucketName
///
///
/// Name of the bucket that holds the content to be downloaded.
///
///
///
/// When using this action with an access point, you must direct requests to the access
/// point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com.
/// When using this action with an access point through the Amazon Web Services SDKs,
/// you provide the access point ARN in place of the bucket name. For more information
/// about access point ARNs, see Using
/// access points in the Amazon S3 User Guide.
///
///
///
/// When you use this action with Amazon S3 on Outposts, you must direct requests to the
/// S3 on Outposts hostname. The S3 on Outposts hostname takes the form AccessPointName-AccountId.outpostID.s3-outposts.Region.amazonaws.com
.
/// When you use this action with S3 on Outposts through the Amazon Web Services SDKs,
/// you provide the Outposts access point ARN in place of the bucket name. For more information
/// about S3 on Outposts ARNs, see What
/// is S3 on Outposts in the Amazon S3 User Guide.
///
///
[Parameter(Position = 0, Mandatory = true, ValueFromPipelineByPropertyName = true, ValueFromPipeline = true)]
[Amazon.PowerShell.Common.AWSRequiredParameter]
public System.String BucketName { get; set; }
#endregion
#endregion
#region File Download Parameters
#region Parameter Key
///
/// The key that identifies the single object in S3.
///
[Parameter(Position = 1, ParameterSetName = ParamSet_ToLocalFile, Mandatory = true, ValueFromPipelineByPropertyName = true)]
[Amazon.PowerShell.Common.AWSRequiredParameter]
public System.String Key { get; set; }
#endregion
#region Parameter File
///
/// The full path to the local file that will be created.
///
[Parameter(Position = 2, ParameterSetName = ParamSet_ToLocalFile, Mandatory = true, ValueFromPipelineByPropertyName = true)]
[Parameter(ParameterSetName = ParamSet_FromS3Object, ValueFromPipelineByPropertyName = true)]
[Amazon.PowerShell.Common.AWSRequiredParameter(ParameterSets = new[] { ParamSet_ToLocalFile })]
public System.String File { get; set; }
#endregion
#region Parameter Version
///
/// If specified, the specific version of the S3 object is returned.
///
[Parameter(Position = 3, ParameterSetName = ParamSet_ToLocalFile, ValueFromPipelineByPropertyName = true)]
public System.String Version { get; set; }
#endregion
#endregion
#region Folder Download Parameters
#region Parameter KeyPrefix
///
///
/// The key prefix that identifies the set of S3 objects to be downloaded. The
/// key structure is preserved as the folder hierarchy under the destination folder.
///
///
/// To indicate that all content in the bucket is to be downloaded, values of
/// '/', '\', '*', '/*' or '\*' may be used for this parameter.
///
[Alias("Prefix")]
[Parameter(Position = 1, ParameterSetName = ParamSet_ToLocalFolder, ValueFromPipelineByPropertyName = true)]
public System.String KeyPrefix { get; set; }
#endregion
#region Parameter Folder
///
/// The full path to a local folder; all downloaded content will be placed under this folder,
/// with subfolders maintaining the S3 object key hierarchies.
///
[Alias("Directory")]
[Parameter(Position = 2, ParameterSetName = ParamSet_ToLocalFolder, Mandatory = true, ValueFromPipelineByPropertyName = true)]
[Parameter(ParameterSetName = ParamSet_FromS3Object, ValueFromPipelineByPropertyName = true)]
[Amazon.PowerShell.Common.AWSRequiredParameter(ParameterSets = new[] { ParamSet_ToLocalFolder })]
public System.String Folder { get; set; }
#endregion
#region Parameter DisableSlashCorrection
///
/// By default if KeyPrefix doesn't have a trailing '/' then a '/' is appended to mimic a virtual S3 directory. If the
/// KeyPrefix is not meant to be S3 virtual directory set DisableSlashCorrection to true to disable the behavior
/// for adding a trailing '/' to the KeyPrefix value.
///
[Parameter(ParameterSetName = ParamSet_ToLocalFolder, ValueFromPipelineByPropertyName = true)]
public bool DisableSlashCorrection { get; set; }
#endregion
#endregion
#region S3Object Download Parameters
#region Parameter S3Object
///
///
/// Amazon.S3.Model.S3Object instance containing the bucketname and key of the object to download.
/// If the supplied object is an Amazon.S3.Model.S3ObjectVersion instance (derived from S3Object),
/// the version of the object to download will be inferred automatically.
///
///
/// The object identified by the supplied S3Object can be downloaded to a specific file (by supplying
/// a value for the -File parameter) or to a folder (specified using the -Folder parameter). When
/// downloading to a folder, the object key is used as the filename. Note that object keys that are not
/// valid filenames for the host system could cause an exception to be thrown.
///
///
[Parameter(ValueFromPipeline=true, ParameterSetName=ParamSet_FromS3Object, Mandatory=true, ValueFromPipelineByPropertyName = true)]
[Amazon.PowerShell.Common.AWSRequiredParameter]
public S3Object S3Object { get; set; }
#endregion
#endregion
#region Common Optional Parameters
#region Parameter ModifiedSinceDate
///
/// If specified, only objects that have been modified since this date will be downloaded.
/// This parameter is deprecated.
///
[Parameter(ValueFromPipelineByPropertyName = true)]
[System.ObsoleteAttribute("This parameter is deprecated because it doesn't honor DateTimeKind. Use UtcModifiedSinceDate instead")]
public System.DateTime ModifiedSinceDate { get; set; }
#endregion
#region Parameter UnmodifiedSinceDate
///
/// If specified, only objects that have not been modified since this date will be downloaded.
/// This parameter is deprecated.
///
[Parameter(ValueFromPipelineByPropertyName = true)]
[System.ObsoleteAttribute("This parameter is deprecated because it doesn't honor DateTimeKind. Use UtcUnmodifiedSinceDate instead")]
public System.DateTime UnmodifiedSinceDate { get; set; }
#endregion
#region Parameter UtcModifiedSinceDate
///
/// If specified, only objects that have been modified since this date will be downloaded.
///
[Parameter(ValueFromPipelineByPropertyName = true)]
public System.DateTime UtcModifiedSinceDate { get; set; }
#endregion
#region Parameter UtcUnmodifiedSinceDate
///
/// If specified, only objects that have not been modified since this date will be downloaded.
///
[Parameter(ValueFromPipelineByPropertyName = true)]
public System.DateTime UtcUnmodifiedSinceDate { get; set; }
#endregion
#region Parameter ServerSideEncryptionCustomerMethod
///
/// Specifies the server-side encryption algorithm to be used with the customer provided key.
/// Allowable values: None or AES256.
///
[Parameter(ParameterSetName = ParamSet_ToLocalFile, ValueFromPipelineByPropertyName = true)]
[Parameter(ParameterSetName = ParamSet_FromS3Object, ValueFromPipelineByPropertyName = true)]
[AWSConstantClassSource("Amazon.S3.ServerSideEncryptionCustomerMethod")]
public Amazon.S3.ServerSideEncryptionCustomerMethod ServerSideEncryptionCustomerMethod { get; set; }
#endregion
#region Parameter ServerSideEncryptionCustomerProvidedKey
///
/// Specifies base64-encoded encryption key for Amazon S3 to use to decrypt the object.
///
[Parameter(ParameterSetName = ParamSet_ToLocalFile, ValueFromPipelineByPropertyName = true)]
[Parameter(ParameterSetName = ParamSet_FromS3Object, ValueFromPipelineByPropertyName = true)]
public System.String ServerSideEncryptionCustomerProvidedKey { get; set; }
#endregion
#region Parameter ServerSideEncryptionCustomerProvidedKeyMD5
///
/// Specifies base64-encoded MD5 of the encryption key for Amazon S3 to use to decrypt the object. This field is optional, the SDK will calculate the MD5 if this is not set.
///
[Parameter(ParameterSetName = ParamSet_ToLocalFile, ValueFromPipelineByPropertyName = true)]
[Parameter(ParameterSetName = ParamSet_FromS3Object, ValueFromPipelineByPropertyName = true)]
public System.String ServerSideEncryptionCustomerProvidedKeyMD5 { get; set; }
#endregion
#region Parameter ChecksumMode
///
/// This must be enabled to retrieve the checksum. In addition, if you enable ChecksumMode
/// and the object is KMS encrypted, you must have permission to the kms:Decrypt
action
/// for the request to succeed.
///
[Parameter(ParameterSetName = ParamSet_ToLocalFile, ValueFromPipelineByPropertyName = true)]
[Parameter(ParameterSetName = ParamSet_FromS3Object, ValueFromPipelineByPropertyName = true)]
[AWSConstantClassSource("Amazon.S3.ChecksumMode")]
public ChecksumMode ChecksumMode { get; set; }
#endregion
#endregion
protected override void ProcessRecord()
{
base.ProcessRecord();
var context = new CmdletContext
{
BucketName = this.BucketName
};
switch (this.ParameterSetName)
{
case ParamSet_ToLocalFile:
{
context.Key = AmazonS3Helper.CleanKey(this.Key);
context.File = PSHelpers.PSPathToAbsolute(this.SessionState.Path, this.File);
context.Version = this.Version;
}
break;
case ParamSet_ToLocalFolder:
{
context.OriginalKeyPrefix = this.KeyPrefix;
context.KeyPrefix = rootIndicators.Contains(this.KeyPrefix, StringComparer.OrdinalIgnoreCase)
? "/" : AmazonS3Helper.CleanKey(this.KeyPrefix);
context.Folder = PSHelpers.PSPathToAbsolute(this.SessionState.Path, this.Folder);
context.DisableSlashCorrection = this.DisableSlashCorrection;
}
break;
case ParamSet_FromS3Object:
{
context.BucketName = this.S3Object.BucketName;
context.Key = this.S3Object.Key;
var s3ObjectVersion = this.S3Object as S3ObjectVersion;
context.Version = s3ObjectVersion == null ? null : s3ObjectVersion.VersionId;
if (this.ParameterWasBound("File"))
{
context.File = PSHelpers.PSPathToAbsolute(this.SessionState.Path, this.File);
}
else
{
var path = PSHelpers.PSPathToAbsolute(this.SessionState.Path, this.Folder);
context.File = Path.Combine(path, S3Object.Key);
}
}
break;
}
if (ParameterWasBound("UtcModifiedSinceDate"))
context.UtcModifiedSinceDate = this.UtcModifiedSinceDate;
if (ParameterWasBound("UtcUnmodifiedSinceDate"))
context.UtcUnmodifiedSinceDate = this.UtcUnmodifiedSinceDate;
#pragma warning disable CS0618, CS0612 //A class member was marked with the Obsolete attribute
if (ParameterWasBound("ModifiedSinceDate"))
context.ModifiedSinceDate = this.ModifiedSinceDate;
if (ParameterWasBound("UnmodifiedSinceDate"))
context.UnmodifiedSinceDate = this.UnmodifiedSinceDate;
#pragma warning restore CS0618, CS0612 //A class member was marked with the Obsolete attribute
if (ParameterWasBound("ServerSideEncryptionCustomerMethod"))
context.ServerSideEncryptionCustomerMethod = this.ServerSideEncryptionCustomerMethod;
context.ServerSideEncryptionCustomerProvidedKey = this.ServerSideEncryptionCustomerProvidedKey;
context.ServerSideEncryptionCustomerProvidedKeyMD5 = this.ServerSideEncryptionCustomerProvidedKeyMD5;
if (ParameterWasBound("ChecksumMode"))
context.ChecksumMode = this.ChecksumMode;
var output = Execute(context) as CmdletOutput;
ProcessOutput(output);
}
#region IExecutor Members
public object Execute(ExecutorContext context)
{
var cmdletContext = context as CmdletContext;
if (!string.IsNullOrEmpty(cmdletContext.File))
return DownloadFileFromS3(cmdletContext);
else
return DownloadFolderFromS3(cmdletContext);
}
CmdletOutput DownloadFileFromS3(ExecutorContext context)
{
var cmdletContext = context as CmdletContext;
var request = new TransferUtilityDownloadRequest
{
BucketName = cmdletContext.BucketName,
FilePath = cmdletContext.File,
Key = cmdletContext.Key
};
if (!string.IsNullOrEmpty(cmdletContext.Version))
request.VersionId = cmdletContext.Version;
if (cmdletContext.UtcModifiedSinceDate.HasValue)
request.ModifiedSinceDateUtc = cmdletContext.UtcModifiedSinceDate.Value;
if (cmdletContext.UtcUnmodifiedSinceDate.HasValue)
request.UnmodifiedSinceDateUtc = cmdletContext.UtcUnmodifiedSinceDate.Value;
#pragma warning disable CS0618, CS0612 //A class member was marked with the Obsolete attribute
if (cmdletContext.ModifiedSinceDate.HasValue)
{
if (cmdletContext.UtcModifiedSinceDate != null)
{
throw new ArgumentException("Parameters ModifiedSinceDate and UtcModifiedSinceDate are mutually exclusive.");
}
request.ModifiedSinceDate = cmdletContext.ModifiedSinceDate.Value;
}
if (cmdletContext.UnmodifiedSinceDate.HasValue)
{
if (cmdletContext.UtcUnmodifiedSinceDate != null)
{
throw new ArgumentException("Parameters UnmodifiedSinceDate and UtcUnmodifiedSinceDate are mutually exclusive.");
}
request.UnmodifiedSinceDate = cmdletContext.UnmodifiedSinceDate.Value;
}
#pragma warning restore CS0618, CS0612 //A class member was marked with the Obsolete attribute
request.ServerSideEncryptionCustomerMethod = cmdletContext.ServerSideEncryptionCustomerMethod;
request.ServerSideEncryptionCustomerProvidedKey = cmdletContext.ServerSideEncryptionCustomerProvidedKey;
request.ServerSideEncryptionCustomerProvidedKeyMD5 = cmdletContext.ServerSideEncryptionCustomerProvidedKeyMD5;
request.ChecksumMode = cmdletContext.ChecksumMode;
CmdletOutput output;
using (var tu = new TransferUtility(Client ?? CreateClient(_CurrentCredentials, _RegionEndpoint)))
{
Utils.Common.WriteVerboseEndpointMessage(this, Client.Config, "Amazon S3 object download APIs");
var runner = new ProgressRunner(this);
var tracker = new DownloadFileProgressTracker(runner, handler => request.WriteObjectProgressEvent += handler, cmdletContext.Key);
output = runner.SafeRun(() => tu.Download(request), tracker);
if (output.ErrorResponse == null)
output.PipelineOutput = new FileInfo(cmdletContext.File);
}
return output;
}
CmdletOutput DownloadFolderFromS3(ExecutorContext context)
{
var cmdletContext = context as CmdletContext;
var request = new TransferUtilityDownloadDirectoryRequest
{
BucketName = cmdletContext.BucketName,
LocalDirectory = cmdletContext.Folder,
S3Directory = cmdletContext.KeyPrefix,
DisableSlashCorrection = cmdletContext.DisableSlashCorrection
};
if (cmdletContext.UtcModifiedSinceDate.HasValue)
request.ModifiedSinceDateUtc = cmdletContext.UtcModifiedSinceDate.Value;
if (cmdletContext.UtcUnmodifiedSinceDate.HasValue)
request.UnmodifiedSinceDateUtc = cmdletContext.UtcUnmodifiedSinceDate.Value;
#pragma warning disable CS0618, CS0612 //A class member was marked with the Obsolete attribute
if (cmdletContext.ModifiedSinceDate.HasValue)
{
if (cmdletContext.UtcModifiedSinceDate != null)
{
throw new ArgumentException("Parameters ModifiedSinceDate and UtcModifiedSinceDate are mutually exclusive.");
}
request.ModifiedSinceDate = cmdletContext.ModifiedSinceDate.Value;
}
if (cmdletContext.UnmodifiedSinceDate.HasValue)
{
if (cmdletContext.UtcUnmodifiedSinceDate != null)
{
throw new ArgumentException("Parameters UnmodifiedSinceDate and UtcUnmodifiedSinceDate are mutually exclusive.");
}
request.UnmodifiedSinceDate = cmdletContext.UnmodifiedSinceDate.Value;
}
#pragma warning restore CS0618, CS0612 //A class member was marked with the Obsolete attribute
CmdletOutput output;
using (var tu = new TransferUtility(Client ?? CreateClient(_CurrentCredentials, _RegionEndpoint)))
{
Utils.Common.WriteVerboseEndpointMessage(this, Client.Config, "Amazon S3 object download APIs");
var runner = new ProgressRunner(this);
var tracker = new DownloadFolderProgressTracker(runner, handler => request.DownloadedDirectoryProgressEvent += handler);
output = runner.SafeRun(() => tu.DownloadDirectory(request), tracker);
if (output.ErrorResponse == null)
output.PipelineOutput = new DirectoryInfo(cmdletContext.Folder);
WriteVerbose(string.Format("Downloaded {0} object(s) from bucket '{1}' with keyprefix '{2}' to '{3}'",
tracker.DownloadedCount,
cmdletContext.BucketName,
cmdletContext.OriginalKeyPrefix,
cmdletContext.Folder));
}
return output;
}
public ExecutorContext CreateContext()
{
return new CmdletContext();
}
#endregion
internal class CmdletContext : ExecutorContext
{
public String BucketName { get; set; }
public String Key { get; set; }
public String File { get; set; }
public string Version { get; set; }
public String OriginalKeyPrefix { get; set; }
public String KeyPrefix { get; set; }
public String Folder { get; set; }
[System.ObsoleteAttribute]
public DateTime? ModifiedSinceDate { get; set; }
[System.ObsoleteAttribute]
public DateTime? UnmodifiedSinceDate { get; set; }
public DateTime? UtcModifiedSinceDate { get; set; }
public DateTime? UtcUnmodifiedSinceDate { get; set; }
public ServerSideEncryptionCustomerMethod ServerSideEncryptionCustomerMethod { get; set; }
public string ServerSideEncryptionCustomerProvidedKey { get; set; }
public string ServerSideEncryptionCustomerProvidedKeyMD5 { get; set; }
public ChecksumMode ChecksumMode { get; set; }
public bool DisableSlashCorrection { get; set; }
}
internal class DownloadFileProgressTracker : ProgressTracker
{
int _currentPercent = 0;
readonly string _key;
const string DownloadingFileActivity = "Downloading";
const string ProgressMsgFormat = "File {0}...{1}%";
public override string Activity
{
get { return DownloadingFileActivity; }
}
public DownloadFileProgressTracker(ProgressRunner runner, Action> subscribe, string key)
: base(runner, subscribe)
{
this._key = key;
ReportProgress(0, ProgressMsgFormat, _key, 0);
}
public override void ReportProgress(WriteObjectProgressArgs args)
{
if (args.PercentDone != _currentPercent)
{
_currentPercent = args.PercentDone;
ReportProgress(args.PercentDone, ProgressMsgFormat, _key, args.PercentDone);
}
}
}
internal class DownloadFolderProgressTracker : ProgressTracker
{
int _fileDownloadCount = 0;
string _currentFile = string.Empty;
const string DownloadingFolderActivity = "Downloading";
public override string Activity
{
get { return DownloadingFolderActivity; }
}
public DownloadFolderProgressTracker(ProgressRunner runner, Action> subscribe)
: base(runner, subscribe)
{
ReportProgress(0, "Downloading files...");
}
public int DownloadedCount { get { return _fileDownloadCount; } }
public override void ReportProgress(DownloadDirectoryProgressArgs args)
{
if (string.Compare(_currentFile, args.CurrentFile, StringComparison.CurrentCultureIgnoreCase) != 0)
{
_currentFile = args.CurrentFile;
_fileDownloadCount = args.NumberOfFilesDownloaded;
ReportProgress(args.NumberOfFilesDownloaded, args.TotalNumberOfFiles, "Downloaded {0} files, current file {1}", args.NumberOfFilesDownloaded, args.CurrentFile);
}
}
}
}
}