/*******************************************************************************
* Copyright 2012-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use
* this file except in compliance with the License. A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file.
* This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
* *****************************************************************************
*
* AWS Tools for Windows (TM) PowerShell (TM)
*
*/
using System;
using System.Collections.Generic;
using System.Linq;
using System.Management.Automation;
using System.Text;
using Amazon.PowerShell.Common;
using Amazon.Runtime;
using Amazon.S3;
using Amazon.S3.Model;
namespace Amazon.PowerShell.Cmdlets.S3
{
///
/// This action filters the contents of an Amazon S3 object based on a simple structured
/// query language (SQL) statement. In the request, along with the SQL expression, you
/// must also specify a data serialization format (JSON, CSV, or Apache Parquet) of the
/// object. Amazon S3 uses this format to parse object data into records, and returns
/// only records that match the specified SQL expression. You must also specify the data
/// serialization format for the response.
///
///
///
/// This action is not supported by Amazon S3 on Outposts.
///
/// For more information about Amazon S3 Select, see Selecting
/// Content from Objects and SELECT
/// Command in the Amazon S3 User Guide.
/// - Permissions
-
/// You must have
s3:GetObject
permission for this operation. Amazon S3 Select
/// does not support anonymous access. For more information about permissions, see Specifying
/// Permissions in a Policy in the Amazon S3 User Guide.
/// - Object Data Formats
-
/// You can use Amazon S3 Select to query objects that have the following format properties:
///
- CSV, JSON, and Parquet - Objects must be in CSV, JSON, or Parquet format.
///
- UTF-8 - UTF-8 is the only encoding type Amazon S3 Select supports.
///
- GZIP or BZIP2 - CSV and JSON files can be compressed using GZIP or BZIP2.
/// GZIP and BZIP2 are the only compression formats that Amazon S3 Select supports for
/// CSV and JSON files. Amazon S3 Select supports columnar compression for Parquet using
/// GZIP or Snappy. Amazon S3 Select does not support whole-object compression for Parquet
/// objects.
///
- Server-side encryption - Amazon S3 Select supports querying objects that are
/// protected with server-side encryption.
///
/// For objects that are encrypted with customer-provided encryption keys (SSE-C), you
/// must use HTTPS, and you must use the headers that are documented in the GetObject.
/// For more information about SSE-C, see Server-Side
/// Encryption (Using Customer-Provided Encryption Keys) in the Amazon S3 User
/// Guide.
///
/// For objects that are encrypted with Amazon S3 managed keys (SSE-S3) and Amazon Web
/// Services KMS keys (SSE-KMS), server-side encryption is handled transparently, so you
/// don't need to specify anything. For more information about server-side encryption,
/// including SSE-S3 and SSE-KMS, see Protecting
/// Data Using Server-Side Encryption in the Amazon S3 User Guide.
///
- Working with the Response Body
-
/// Given the response size is unknown, Amazon S3 Select streams the response as a series
/// of messages and includes a
Transfer-Encoding
header with chunked
/// as its value in the response. For more information, see Appendix:
/// SelectObjectContent Response.
/// - GetObject Support
-
/// The
SelectObjectContent
action does not support the following GetObject
/// functionality. For more information, see GetObject.
/// Range
: Although you can specify a scan range for an Amazon S3 Select
/// request (see SelectObjectContentRequest
/// - ScanRange in the request parameters), you cannot specify the range of bytes
/// of an object to return.
/// -
/// The
GLACIER
, DEEP_ARCHIVE
, and REDUCED_REDUNDANCY
/// storage classes, or the ARCHIVE_ACCESS
and DEEP_ARCHIVE_ACCESS
/// access tiers of the INTELLIGENT_TIERING
storage class: You cannot query
/// objects in the GLACIER
, DEEP_ARCHIVE
, or REDUCED_REDUNDANCY
/// storage classes, nor objects in the ARCHIVE_ACCESS
or DEEP_ARCHIVE_ACCESS
/// access tiers of the INTELLIGENT_TIERING
storage class. For more information
/// about storage classes, see Using
/// Amazon S3 storage classes in the Amazon S3 User Guide.
///
- Special Errors
-
/// For a list of special errors for this operation, see List
/// of SELECT Object Content Error Codes
/// The following operations are related to SelectObjectContent
:
///
///
[Cmdlet("Select", "S3ObjectContent")]
[OutputType("Amazon.S3.Model.ISelectObjectContentEventStream")]
[AWSCmdlet("Calls the Amazon Simple Storage Service (S3) SelectObjectContent API operation.", Operation = new[] {"SelectObjectContent"}, SelectReturnType = typeof(Amazon.S3.Model.SelectObjectContentResponse))]
[AWSCmdletOutput("Amazon.S3.Model.ISelectObjectContentEventStream or Amazon.S3.Model.SelectObjectContentResponse",
"This cmdlet returns an Amazon.S3.Model.ISelectObjectContentEventStream object.",
"The service call response (type Amazon.S3.Model.SelectObjectContentResponse) can also be referenced from properties attached to the cmdlet entry in the $AWSHistory stack."
)]
public partial class SelectS3ObjectContentCmdlet : AmazonS3ClientCmdlet, IExecutor
{
protected override bool IsSensitiveRequest { get; set; } = true;
#region Parameter BucketName
///
///
/// The S3 Bucket name.
///
///
[System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)]
public System.String BucketName { get; set; }
#endregion
#region Parameter ScanRange_End
///
///
/// Specifies the end of the byte range. This parameter is optional. Valid values: non-negative integers. The default value is one less than the size of the object being queried.
///
///
[System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)]
public System.Int64? ScanRange_End { get; set; }
#endregion
#region Parameter ExpectedBucketOwner
///
///
/// The account ID of the expected bucket owner.
/// If the bucket is owned by a different account, the request will fail with an HTTP 403 (Access Denied) error.
///
///
[System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)]
public System.String ExpectedBucketOwner { get; set; }
#endregion
#region Parameter Expression
///
///
/// The expression that is used to query the object.
///
///
[System.Management.Automation.Parameter(Position = 0, ValueFromPipelineByPropertyName = true, ValueFromPipeline = true)]
public System.String Expression { get; set; }
#endregion
#region Parameter ExpressionType
///
///
/// The type of the provided expression (e.g., SQL).
///
///
[System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)]
[AWSConstantClassSource("Amazon.S3.ExpressionType")]
public Amazon.S3.ExpressionType ExpressionType { get; set; }
#endregion
#region Parameter InputSerialization
///
///
/// Describes the format of the data in the object that is being queried.
///
///
[System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)]
public Amazon.S3.Model.InputSerialization InputSerialization { get; set; }
#endregion
#region Parameter Key
///
///
/// The Object Key.
///
///
[System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)]
public System.String Key { get; set; }
#endregion
#region Parameter OutputSerialization
///
///
/// Describes the format of the data that you want Amazon S3 to return in response.
///
///
[System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)]
public Amazon.S3.Model.OutputSerialization OutputSerialization { get; set; }
#endregion
#region Parameter RequestProgress
///
///
/// Specifies if periodic request progress information should be enabled.
///
///
[System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)]
public System.Boolean? RequestProgress { get; set; }
#endregion
#region Parameter ServerSideCustomerEncryptionMethod
///
///
/// The SSE Algorithm used to encrypt the object.
///
///
[System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)]
[AWSConstantClassSource("Amazon.S3.ServerSideEncryptionCustomerMethod")]
public Amazon.S3.ServerSideEncryptionCustomerMethod ServerSideCustomerEncryptionMethod { get; set; }
#endregion
#region Parameter ServerSideEncryptionCustomerProvidedKey
///
///
/// The SSE Customer Key.
///
///
[System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)]
public System.String ServerSideEncryptionCustomerProvidedKey { get; set; }
#endregion
#region Parameter ServerSideEncryptionCustomerProvidedKeyMD5
///
///
/// The SSE Customer Key MD5.
///
///
[System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)]
public System.String ServerSideEncryptionCustomerProvidedKeyMD5 { get; set; }
#endregion
#region Parameter ScanRange_Start
///
///
/// Specifies the start of the byte range. This parameter is optional. Valid values: non-negative integers. The default value is 0.
///
///
[System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)]
public System.Int64? ScanRange_Start { get; set; }
#endregion
#region Parameter Bucket
///
///
/// The S3 Bucket.
///
/// This parameter is deprecated.
///
[System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)]
[System.ObsoleteAttribute("Use BucketName instead")]
public System.String Bucket { get; set; }
#endregion
#region Parameter Select
///
/// Use the -Select parameter to control the cmdlet output. The default value is 'Payload'.
/// Specifying -Select '*' will result in the cmdlet returning the whole service response (Amazon.S3.Model.SelectObjectContentResponse).
/// Specifying the name of a property of type Amazon.S3.Model.SelectObjectContentResponse will result in that property being returned.
/// Specifying -Select '^ParameterName' will result in the cmdlet returning the selected cmdlet parameter value.
///
[System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)]
public string Select { get; set; } = "Payload";
#endregion
#region Parameter PassThru
///
/// Changes the cmdlet behavior to return the value passed to the Expression parameter.
/// The -PassThru parameter is deprecated, use -Select '^Expression' instead. This parameter will be removed in a future version.
///
[System.Obsolete("The -PassThru parameter is deprecated, use -Select '^Expression' instead. This parameter will be removed in a future version.")]
[System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)]
public SwitchParameter PassThru { get; set; }
#endregion
protected override void ProcessRecord()
{
this._AWSSignerType = "s3";
base.ProcessRecord();
var context = new CmdletContext();
// allow for manipulation of parameters prior to loading into context
PreExecutionContextLoad(context);
#pragma warning disable CS0618, CS0612 //A class member was marked with the Obsolete attribute
if (ParameterWasBound(nameof(this.Select)))
{
context.Select = CreateSelectDelegate(Select) ??
throw new System.ArgumentException("Invalid value for -Select parameter.", nameof(this.Select));
if (this.PassThru.IsPresent)
{
throw new System.ArgumentException("-PassThru cannot be used when -Select is specified.", nameof(this.Select));
}
}
else if (this.PassThru.IsPresent)
{
context.Select = (response, cmdlet) => this.Expression;
}
#pragma warning restore CS0618, CS0612 //A class member was marked with the Obsolete attribute
#pragma warning disable CS0618, CS0612 //A class member was marked with the Obsolete attribute
context.Bucket = this.Bucket;
#pragma warning restore CS0618, CS0612 //A class member was marked with the Obsolete attribute
context.BucketName = this.BucketName;
context.Key = this.Key;
context.ServerSideCustomerEncryptionMethod = this.ServerSideCustomerEncryptionMethod;
context.ServerSideEncryptionCustomerProvidedKey = this.ServerSideEncryptionCustomerProvidedKey;
context.ServerSideEncryptionCustomerProvidedKeyMD5 = this.ServerSideEncryptionCustomerProvidedKeyMD5;
context.Expression = this.Expression;
context.ExpressionType = this.ExpressionType;
context.RequestProgress = this.RequestProgress;
context.InputSerialization = this.InputSerialization;
context.OutputSerialization = this.OutputSerialization;
context.ScanRange_Start = this.ScanRange_Start;
context.ScanRange_End = this.ScanRange_End;
context.ExpectedBucketOwner = this.ExpectedBucketOwner;
// allow further manipulation of loaded context prior to processing
PostExecutionContextLoad(context);
var output = Execute(context) as CmdletOutput;
ProcessOutput(output);
}
#region IExecutor Members
public object Execute(ExecutorContext context)
{
var cmdletContext = context as CmdletContext;
// create request
var request = new Amazon.S3.Model.SelectObjectContentRequest();
#pragma warning disable CS0618, CS0612 //A class member was marked with the Obsolete attribute
if (cmdletContext.Bucket != null)
{
request.Bucket = cmdletContext.Bucket;
}
#pragma warning restore CS0618, CS0612 //A class member was marked with the Obsolete attribute
if (cmdletContext.BucketName != null)
{
request.BucketName = cmdletContext.BucketName;
}
if (cmdletContext.Key != null)
{
request.Key = cmdletContext.Key;
}
if (cmdletContext.ServerSideCustomerEncryptionMethod != null)
{
request.ServerSideCustomerEncryptionMethod = cmdletContext.ServerSideCustomerEncryptionMethod;
}
if (cmdletContext.ServerSideEncryptionCustomerProvidedKey != null)
{
request.ServerSideEncryptionCustomerProvidedKey = cmdletContext.ServerSideEncryptionCustomerProvidedKey;
}
if (cmdletContext.ServerSideEncryptionCustomerProvidedKeyMD5 != null)
{
request.ServerSideEncryptionCustomerProvidedKeyMD5 = cmdletContext.ServerSideEncryptionCustomerProvidedKeyMD5;
}
if (cmdletContext.Expression != null)
{
request.Expression = cmdletContext.Expression;
}
if (cmdletContext.ExpressionType != null)
{
request.ExpressionType = cmdletContext.ExpressionType;
}
if (cmdletContext.RequestProgress != null)
{
request.RequestProgress = cmdletContext.RequestProgress.Value;
}
if (cmdletContext.InputSerialization != null)
{
request.InputSerialization = cmdletContext.InputSerialization;
}
if (cmdletContext.OutputSerialization != null)
{
request.OutputSerialization = cmdletContext.OutputSerialization;
}
// populate ScanRange
var requestScanRangeIsNull = true;
request.ScanRange = new Amazon.S3.Model.ScanRange();
System.Int64? requestScanRange_scanRange_Start = null;
if (cmdletContext.ScanRange_Start != null)
{
requestScanRange_scanRange_Start = cmdletContext.ScanRange_Start.Value;
}
if (requestScanRange_scanRange_Start != null)
{
request.ScanRange.Start = requestScanRange_scanRange_Start.Value;
requestScanRangeIsNull = false;
}
System.Int64? requestScanRange_scanRange_End = null;
if (cmdletContext.ScanRange_End != null)
{
requestScanRange_scanRange_End = cmdletContext.ScanRange_End.Value;
}
if (requestScanRange_scanRange_End != null)
{
request.ScanRange.End = requestScanRange_scanRange_End.Value;
requestScanRangeIsNull = false;
}
// determine if request.ScanRange should be set to null
if (requestScanRangeIsNull)
{
request.ScanRange = null;
}
if (cmdletContext.ExpectedBucketOwner != null)
{
request.ExpectedBucketOwner = cmdletContext.ExpectedBucketOwner;
}
CmdletOutput output;
// issue call
var client = Client ?? CreateClient(_CurrentCredentials, _RegionEndpoint);
try
{
var response = CallAWSServiceOperation(client, request);
object pipelineOutput = null;
pipelineOutput = cmdletContext.Select(response, this);
output = new CmdletOutput
{
PipelineOutput = pipelineOutput,
ServiceResponse = response
};
}
catch (Exception e)
{
output = new CmdletOutput { ErrorResponse = e };
}
return output;
}
public ExecutorContext CreateContext()
{
return new CmdletContext();
}
#endregion
#region AWS Service Operation Call
private Amazon.S3.Model.SelectObjectContentResponse CallAWSServiceOperation(IAmazonS3 client, Amazon.S3.Model.SelectObjectContentRequest request)
{
Utils.Common.WriteVerboseEndpointMessage(this, client.Config, "Amazon Simple Storage Service (S3)", "SelectObjectContent");
try
{
#if DESKTOP
return client.SelectObjectContent(request);
#elif CORECLR
return client.SelectObjectContentAsync(request).GetAwaiter().GetResult();
#else
#error "Unknown build edition"
#endif
}
catch (AmazonServiceException exc)
{
var webException = exc.InnerException as System.Net.WebException;
if (webException != null)
{
throw new Exception(Utils.Common.FormatNameResolutionFailureMessage(client.Config, webException.Message), webException);
}
throw;
}
}
#endregion
internal partial class CmdletContext : ExecutorContext
{
[System.ObsoleteAttribute]
public System.String Bucket { get; set; }
public System.String BucketName { get; set; }
public System.String Key { get; set; }
public Amazon.S3.ServerSideEncryptionCustomerMethod ServerSideCustomerEncryptionMethod { get; set; }
public System.String ServerSideEncryptionCustomerProvidedKey { get; set; }
public System.String ServerSideEncryptionCustomerProvidedKeyMD5 { get; set; }
public System.String Expression { get; set; }
public Amazon.S3.ExpressionType ExpressionType { get; set; }
public System.Boolean? RequestProgress { get; set; }
public Amazon.S3.Model.InputSerialization InputSerialization { get; set; }
public Amazon.S3.Model.OutputSerialization OutputSerialization { get; set; }
public System.Int64? ScanRange_Start { get; set; }
public System.Int64? ScanRange_End { get; set; }
public System.String ExpectedBucketOwner { get; set; }
public System.Func Select { get; set; } =
(response, cmdlet) => response.Payload;
}
}
}