/******************************************************************************* * Copyright 2012-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * * http://aws.amazon.com/apache2.0 * * or in the "license" file accompanying this file. * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR * CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. * ***************************************************************************** * * AWS Tools for Windows (TM) PowerShell (TM) * */ using System; using System.Collections.Generic; using System.Linq; using System.Management.Automation; using System.Text; using Amazon.PowerShell.Common; using Amazon.Runtime; using Amazon.S3; using Amazon.S3.Model; namespace Amazon.PowerShell.Cmdlets.S3 { /// /// This action filters the contents of an Amazon S3 object based on a simple structured /// query language (SQL) statement. In the request, along with the SQL expression, you /// must also specify a data serialization format (JSON, CSV, or Apache Parquet) of the /// object. Amazon S3 uses this format to parse object data into records, and returns /// only records that match the specified SQL expression. You must also specify the data /// serialization format for the response. /// /// /// /// This action is not supported by Amazon S3 on Outposts. /// /// For more information about Amazon S3 Select, see Selecting /// Content from Objects and SELECT /// Command in the Amazon S3 User Guide. ///
Permissions
/// You must have s3:GetObject permission for this operation. Amazon S3 Select /// does not support anonymous access. For more information about permissions, see Specifying /// Permissions in a Policy in the Amazon S3 User Guide. ///
Object Data Formats
/// You can use Amazon S3 Select to query objects that have the following format properties: ///
  • CSV, JSON, and Parquet - Objects must be in CSV, JSON, or Parquet format. ///
  • UTF-8 - UTF-8 is the only encoding type Amazon S3 Select supports. ///
  • GZIP or BZIP2 - CSV and JSON files can be compressed using GZIP or BZIP2. /// GZIP and BZIP2 are the only compression formats that Amazon S3 Select supports for /// CSV and JSON files. Amazon S3 Select supports columnar compression for Parquet using /// GZIP or Snappy. Amazon S3 Select does not support whole-object compression for Parquet /// objects. ///
  • Server-side encryption - Amazon S3 Select supports querying objects that are /// protected with server-side encryption. /// /// For objects that are encrypted with customer-provided encryption keys (SSE-C), you /// must use HTTPS, and you must use the headers that are documented in the GetObject. /// For more information about SSE-C, see Server-Side /// Encryption (Using Customer-Provided Encryption Keys) in the Amazon S3 User /// Guide. /// /// For objects that are encrypted with Amazon S3 managed keys (SSE-S3) and Amazon Web /// Services KMS keys (SSE-KMS), server-side encryption is handled transparently, so you /// don't need to specify anything. For more information about server-side encryption, /// including SSE-S3 and SSE-KMS, see Protecting /// Data Using Server-Side Encryption in the Amazon S3 User Guide. ///
Working with the Response Body
/// Given the response size is unknown, Amazon S3 Select streams the response as a series /// of messages and includes a Transfer-Encoding header with chunked /// as its value in the response. For more information, see Appendix: /// SelectObjectContent Response. ///
GetObject Support
/// The SelectObjectContent action does not support the following GetObject /// functionality. For more information, see GetObject. ///
  • Range: Although you can specify a scan range for an Amazon S3 Select /// request (see SelectObjectContentRequest /// - ScanRange in the request parameters), you cannot specify the range of bytes /// of an object to return. ///
  • /// The GLACIER, DEEP_ARCHIVE, and REDUCED_REDUNDANCY /// storage classes, or the ARCHIVE_ACCESS and DEEP_ARCHIVE_ACCESS /// access tiers of the INTELLIGENT_TIERING storage class: You cannot query /// objects in the GLACIER, DEEP_ARCHIVE, or REDUCED_REDUNDANCY /// storage classes, nor objects in the ARCHIVE_ACCESS or DEEP_ARCHIVE_ACCESS /// access tiers of the INTELLIGENT_TIERING storage class. For more information /// about storage classes, see Using /// Amazon S3 storage classes in the Amazon S3 User Guide. ///
Special Errors
/// For a list of special errors for this operation, see List /// of SELECT Object Content Error Codes
/// The following operations are related to SelectObjectContent: /// ///
[Cmdlet("Select", "S3ObjectContent")] [OutputType("Amazon.S3.Model.ISelectObjectContentEventStream")] [AWSCmdlet("Calls the Amazon Simple Storage Service (S3) SelectObjectContent API operation.", Operation = new[] {"SelectObjectContent"}, SelectReturnType = typeof(Amazon.S3.Model.SelectObjectContentResponse))] [AWSCmdletOutput("Amazon.S3.Model.ISelectObjectContentEventStream or Amazon.S3.Model.SelectObjectContentResponse", "This cmdlet returns an Amazon.S3.Model.ISelectObjectContentEventStream object.", "The service call response (type Amazon.S3.Model.SelectObjectContentResponse) can also be referenced from properties attached to the cmdlet entry in the $AWSHistory stack." )] public partial class SelectS3ObjectContentCmdlet : AmazonS3ClientCmdlet, IExecutor { protected override bool IsSensitiveRequest { get; set; } = true; #region Parameter BucketName /// /// /// The S3 Bucket name. /// /// [System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)] public System.String BucketName { get; set; } #endregion #region Parameter ScanRange_End /// /// /// Specifies the end of the byte range. This parameter is optional. Valid values: non-negative integers. The default value is one less than the size of the object being queried. /// /// [System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)] public System.Int64? ScanRange_End { get; set; } #endregion #region Parameter ExpectedBucketOwner /// /// /// The account ID of the expected bucket owner. /// If the bucket is owned by a different account, the request will fail with an HTTP 403 (Access Denied) error. /// /// [System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)] public System.String ExpectedBucketOwner { get; set; } #endregion #region Parameter Expression /// /// /// The expression that is used to query the object. /// /// [System.Management.Automation.Parameter(Position = 0, ValueFromPipelineByPropertyName = true, ValueFromPipeline = true)] public System.String Expression { get; set; } #endregion #region Parameter ExpressionType /// /// /// The type of the provided expression (e.g., SQL). /// /// [System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)] [AWSConstantClassSource("Amazon.S3.ExpressionType")] public Amazon.S3.ExpressionType ExpressionType { get; set; } #endregion #region Parameter InputSerialization /// /// /// Describes the format of the data in the object that is being queried. /// /// [System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)] public Amazon.S3.Model.InputSerialization InputSerialization { get; set; } #endregion #region Parameter Key /// /// /// The Object Key. /// /// [System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)] public System.String Key { get; set; } #endregion #region Parameter OutputSerialization /// /// /// Describes the format of the data that you want Amazon S3 to return in response. /// /// [System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)] public Amazon.S3.Model.OutputSerialization OutputSerialization { get; set; } #endregion #region Parameter RequestProgress /// /// /// Specifies if periodic request progress information should be enabled. /// /// [System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)] public System.Boolean? RequestProgress { get; set; } #endregion #region Parameter ServerSideCustomerEncryptionMethod /// /// /// The SSE Algorithm used to encrypt the object. /// /// [System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)] [AWSConstantClassSource("Amazon.S3.ServerSideEncryptionCustomerMethod")] public Amazon.S3.ServerSideEncryptionCustomerMethod ServerSideCustomerEncryptionMethod { get; set; } #endregion #region Parameter ServerSideEncryptionCustomerProvidedKey /// /// /// The SSE Customer Key. /// /// [System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)] public System.String ServerSideEncryptionCustomerProvidedKey { get; set; } #endregion #region Parameter ServerSideEncryptionCustomerProvidedKeyMD5 /// /// /// The SSE Customer Key MD5. /// /// [System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)] public System.String ServerSideEncryptionCustomerProvidedKeyMD5 { get; set; } #endregion #region Parameter ScanRange_Start /// /// /// Specifies the start of the byte range. This parameter is optional. Valid values: non-negative integers. The default value is 0. /// /// [System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)] public System.Int64? ScanRange_Start { get; set; } #endregion #region Parameter Bucket /// /// /// The S3 Bucket. /// /// This parameter is deprecated. /// [System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)] [System.ObsoleteAttribute("Use BucketName instead")] public System.String Bucket { get; set; } #endregion #region Parameter Select /// /// Use the -Select parameter to control the cmdlet output. The default value is 'Payload'. /// Specifying -Select '*' will result in the cmdlet returning the whole service response (Amazon.S3.Model.SelectObjectContentResponse). /// Specifying the name of a property of type Amazon.S3.Model.SelectObjectContentResponse will result in that property being returned. /// Specifying -Select '^ParameterName' will result in the cmdlet returning the selected cmdlet parameter value. /// [System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)] public string Select { get; set; } = "Payload"; #endregion #region Parameter PassThru /// /// Changes the cmdlet behavior to return the value passed to the Expression parameter. /// The -PassThru parameter is deprecated, use -Select '^Expression' instead. This parameter will be removed in a future version. /// [System.Obsolete("The -PassThru parameter is deprecated, use -Select '^Expression' instead. This parameter will be removed in a future version.")] [System.Management.Automation.Parameter(ValueFromPipelineByPropertyName = true)] public SwitchParameter PassThru { get; set; } #endregion protected override void ProcessRecord() { this._AWSSignerType = "s3"; base.ProcessRecord(); var context = new CmdletContext(); // allow for manipulation of parameters prior to loading into context PreExecutionContextLoad(context); #pragma warning disable CS0618, CS0612 //A class member was marked with the Obsolete attribute if (ParameterWasBound(nameof(this.Select))) { context.Select = CreateSelectDelegate(Select) ?? throw new System.ArgumentException("Invalid value for -Select parameter.", nameof(this.Select)); if (this.PassThru.IsPresent) { throw new System.ArgumentException("-PassThru cannot be used when -Select is specified.", nameof(this.Select)); } } else if (this.PassThru.IsPresent) { context.Select = (response, cmdlet) => this.Expression; } #pragma warning restore CS0618, CS0612 //A class member was marked with the Obsolete attribute #pragma warning disable CS0618, CS0612 //A class member was marked with the Obsolete attribute context.Bucket = this.Bucket; #pragma warning restore CS0618, CS0612 //A class member was marked with the Obsolete attribute context.BucketName = this.BucketName; context.Key = this.Key; context.ServerSideCustomerEncryptionMethod = this.ServerSideCustomerEncryptionMethod; context.ServerSideEncryptionCustomerProvidedKey = this.ServerSideEncryptionCustomerProvidedKey; context.ServerSideEncryptionCustomerProvidedKeyMD5 = this.ServerSideEncryptionCustomerProvidedKeyMD5; context.Expression = this.Expression; context.ExpressionType = this.ExpressionType; context.RequestProgress = this.RequestProgress; context.InputSerialization = this.InputSerialization; context.OutputSerialization = this.OutputSerialization; context.ScanRange_Start = this.ScanRange_Start; context.ScanRange_End = this.ScanRange_End; context.ExpectedBucketOwner = this.ExpectedBucketOwner; // allow further manipulation of loaded context prior to processing PostExecutionContextLoad(context); var output = Execute(context) as CmdletOutput; ProcessOutput(output); } #region IExecutor Members public object Execute(ExecutorContext context) { var cmdletContext = context as CmdletContext; // create request var request = new Amazon.S3.Model.SelectObjectContentRequest(); #pragma warning disable CS0618, CS0612 //A class member was marked with the Obsolete attribute if (cmdletContext.Bucket != null) { request.Bucket = cmdletContext.Bucket; } #pragma warning restore CS0618, CS0612 //A class member was marked with the Obsolete attribute if (cmdletContext.BucketName != null) { request.BucketName = cmdletContext.BucketName; } if (cmdletContext.Key != null) { request.Key = cmdletContext.Key; } if (cmdletContext.ServerSideCustomerEncryptionMethod != null) { request.ServerSideCustomerEncryptionMethod = cmdletContext.ServerSideCustomerEncryptionMethod; } if (cmdletContext.ServerSideEncryptionCustomerProvidedKey != null) { request.ServerSideEncryptionCustomerProvidedKey = cmdletContext.ServerSideEncryptionCustomerProvidedKey; } if (cmdletContext.ServerSideEncryptionCustomerProvidedKeyMD5 != null) { request.ServerSideEncryptionCustomerProvidedKeyMD5 = cmdletContext.ServerSideEncryptionCustomerProvidedKeyMD5; } if (cmdletContext.Expression != null) { request.Expression = cmdletContext.Expression; } if (cmdletContext.ExpressionType != null) { request.ExpressionType = cmdletContext.ExpressionType; } if (cmdletContext.RequestProgress != null) { request.RequestProgress = cmdletContext.RequestProgress.Value; } if (cmdletContext.InputSerialization != null) { request.InputSerialization = cmdletContext.InputSerialization; } if (cmdletContext.OutputSerialization != null) { request.OutputSerialization = cmdletContext.OutputSerialization; } // populate ScanRange var requestScanRangeIsNull = true; request.ScanRange = new Amazon.S3.Model.ScanRange(); System.Int64? requestScanRange_scanRange_Start = null; if (cmdletContext.ScanRange_Start != null) { requestScanRange_scanRange_Start = cmdletContext.ScanRange_Start.Value; } if (requestScanRange_scanRange_Start != null) { request.ScanRange.Start = requestScanRange_scanRange_Start.Value; requestScanRangeIsNull = false; } System.Int64? requestScanRange_scanRange_End = null; if (cmdletContext.ScanRange_End != null) { requestScanRange_scanRange_End = cmdletContext.ScanRange_End.Value; } if (requestScanRange_scanRange_End != null) { request.ScanRange.End = requestScanRange_scanRange_End.Value; requestScanRangeIsNull = false; } // determine if request.ScanRange should be set to null if (requestScanRangeIsNull) { request.ScanRange = null; } if (cmdletContext.ExpectedBucketOwner != null) { request.ExpectedBucketOwner = cmdletContext.ExpectedBucketOwner; } CmdletOutput output; // issue call var client = Client ?? CreateClient(_CurrentCredentials, _RegionEndpoint); try { var response = CallAWSServiceOperation(client, request); object pipelineOutput = null; pipelineOutput = cmdletContext.Select(response, this); output = new CmdletOutput { PipelineOutput = pipelineOutput, ServiceResponse = response }; } catch (Exception e) { output = new CmdletOutput { ErrorResponse = e }; } return output; } public ExecutorContext CreateContext() { return new CmdletContext(); } #endregion #region AWS Service Operation Call private Amazon.S3.Model.SelectObjectContentResponse CallAWSServiceOperation(IAmazonS3 client, Amazon.S3.Model.SelectObjectContentRequest request) { Utils.Common.WriteVerboseEndpointMessage(this, client.Config, "Amazon Simple Storage Service (S3)", "SelectObjectContent"); try { #if DESKTOP return client.SelectObjectContent(request); #elif CORECLR return client.SelectObjectContentAsync(request).GetAwaiter().GetResult(); #else #error "Unknown build edition" #endif } catch (AmazonServiceException exc) { var webException = exc.InnerException as System.Net.WebException; if (webException != null) { throw new Exception(Utils.Common.FormatNameResolutionFailureMessage(client.Config, webException.Message), webException); } throw; } } #endregion internal partial class CmdletContext : ExecutorContext { [System.ObsoleteAttribute] public System.String Bucket { get; set; } public System.String BucketName { get; set; } public System.String Key { get; set; } public Amazon.S3.ServerSideEncryptionCustomerMethod ServerSideCustomerEncryptionMethod { get; set; } public System.String ServerSideEncryptionCustomerProvidedKey { get; set; } public System.String ServerSideEncryptionCustomerProvidedKeyMD5 { get; set; } public System.String Expression { get; set; } public Amazon.S3.ExpressionType ExpressionType { get; set; } public System.Boolean? RequestProgress { get; set; } public Amazon.S3.Model.InputSerialization InputSerialization { get; set; } public Amazon.S3.Model.OutputSerialization OutputSerialization { get; set; } public System.Int64? ScanRange_Start { get; set; } public System.Int64? ScanRange_End { get; set; } public System.String ExpectedBucketOwner { get; set; } public System.Func Select { get; set; } = (response, cmdlet) => response.Payload; } } }