/**
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: Apache-2.0.
*/
#pragma once
#include Specifies a command-separated value (CSV) data store stored in Amazon
* S3.See Also:
AWS
* API Reference
The name of the data store.
*/ inline const Aws::String& GetName() const{ return m_name; } /** *The name of the data store.
*/ inline bool NameHasBeenSet() const { return m_nameHasBeenSet; } /** *The name of the data store.
*/ inline void SetName(const Aws::String& value) { m_nameHasBeenSet = true; m_name = value; } /** *The name of the data store.
*/ inline void SetName(Aws::String&& value) { m_nameHasBeenSet = true; m_name = std::move(value); } /** *The name of the data store.
*/ inline void SetName(const char* value) { m_nameHasBeenSet = true; m_name.assign(value); } /** *The name of the data store.
*/ inline S3CsvSource& WithName(const Aws::String& value) { SetName(value); return *this;} /** *The name of the data store.
*/ inline S3CsvSource& WithName(Aws::String&& value) { SetName(std::move(value)); return *this;} /** *The name of the data store.
*/ inline S3CsvSource& WithName(const char* value) { SetName(value); return *this;} /** *A list of the Amazon S3 paths to read from.
*/ inline const Aws::VectorA list of the Amazon S3 paths to read from.
*/ inline bool PathsHasBeenSet() const { return m_pathsHasBeenSet; } /** *A list of the Amazon S3 paths to read from.
*/ inline void SetPaths(const Aws::VectorA list of the Amazon S3 paths to read from.
*/ inline void SetPaths(Aws::VectorA list of the Amazon S3 paths to read from.
*/ inline S3CsvSource& WithPaths(const Aws::VectorA list of the Amazon S3 paths to read from.
*/ inline S3CsvSource& WithPaths(Aws::VectorA list of the Amazon S3 paths to read from.
*/ inline S3CsvSource& AddPaths(const Aws::String& value) { m_pathsHasBeenSet = true; m_paths.push_back(value); return *this; } /** *A list of the Amazon S3 paths to read from.
*/ inline S3CsvSource& AddPaths(Aws::String&& value) { m_pathsHasBeenSet = true; m_paths.push_back(std::move(value)); return *this; } /** *A list of the Amazon S3 paths to read from.
*/ inline S3CsvSource& AddPaths(const char* value) { m_pathsHasBeenSet = true; m_paths.push_back(value); return *this; } /** *Specifies how the data is compressed. This is generally not necessary if the
* data has a standard file extension. Possible values are "gzip"
and
* "bzip"
).
Specifies how the data is compressed. This is generally not necessary if the
* data has a standard file extension. Possible values are "gzip"
and
* "bzip"
).
Specifies how the data is compressed. This is generally not necessary if the
* data has a standard file extension. Possible values are "gzip"
and
* "bzip"
).
Specifies how the data is compressed. This is generally not necessary if the
* data has a standard file extension. Possible values are "gzip"
and
* "bzip"
).
Specifies how the data is compressed. This is generally not necessary if the
* data has a standard file extension. Possible values are "gzip"
and
* "bzip"
).
Specifies how the data is compressed. This is generally not necessary if the
* data has a standard file extension. Possible values are "gzip"
and
* "bzip"
).
A string containing a JSON list of Unix-style glob patterns to exclude. For * example, "[\"**.pdf\"]" excludes all PDF files.
*/ inline const Aws::VectorA string containing a JSON list of Unix-style glob patterns to exclude. For * example, "[\"**.pdf\"]" excludes all PDF files.
*/ inline bool ExclusionsHasBeenSet() const { return m_exclusionsHasBeenSet; } /** *A string containing a JSON list of Unix-style glob patterns to exclude. For * example, "[\"**.pdf\"]" excludes all PDF files.
*/ inline void SetExclusions(const Aws::VectorA string containing a JSON list of Unix-style glob patterns to exclude. For * example, "[\"**.pdf\"]" excludes all PDF files.
*/ inline void SetExclusions(Aws::VectorA string containing a JSON list of Unix-style glob patterns to exclude. For * example, "[\"**.pdf\"]" excludes all PDF files.
*/ inline S3CsvSource& WithExclusions(const Aws::VectorA string containing a JSON list of Unix-style glob patterns to exclude. For * example, "[\"**.pdf\"]" excludes all PDF files.
*/ inline S3CsvSource& WithExclusions(Aws::VectorA string containing a JSON list of Unix-style glob patterns to exclude. For * example, "[\"**.pdf\"]" excludes all PDF files.
*/ inline S3CsvSource& AddExclusions(const Aws::String& value) { m_exclusionsHasBeenSet = true; m_exclusions.push_back(value); return *this; } /** *A string containing a JSON list of Unix-style glob patterns to exclude. For * example, "[\"**.pdf\"]" excludes all PDF files.
*/ inline S3CsvSource& AddExclusions(Aws::String&& value) { m_exclusionsHasBeenSet = true; m_exclusions.push_back(std::move(value)); return *this; } /** *A string containing a JSON list of Unix-style glob patterns to exclude. For * example, "[\"**.pdf\"]" excludes all PDF files.
*/ inline S3CsvSource& AddExclusions(const char* value) { m_exclusionsHasBeenSet = true; m_exclusions.push_back(value); return *this; } /** *The target group size in bytes. The default is computed based on the input
* data size and the size of your cluster. When there are fewer than 50,000 input
* files, "groupFiles"
must be set to "inPartition"
for
* this to take effect.
The target group size in bytes. The default is computed based on the input
* data size and the size of your cluster. When there are fewer than 50,000 input
* files, "groupFiles"
must be set to "inPartition"
for
* this to take effect.
The target group size in bytes. The default is computed based on the input
* data size and the size of your cluster. When there are fewer than 50,000 input
* files, "groupFiles"
must be set to "inPartition"
for
* this to take effect.
The target group size in bytes. The default is computed based on the input
* data size and the size of your cluster. When there are fewer than 50,000 input
* files, "groupFiles"
must be set to "inPartition"
for
* this to take effect.
The target group size in bytes. The default is computed based on the input
* data size and the size of your cluster. When there are fewer than 50,000 input
* files, "groupFiles"
must be set to "inPartition"
for
* this to take effect.
The target group size in bytes. The default is computed based on the input
* data size and the size of your cluster. When there are fewer than 50,000 input
* files, "groupFiles"
must be set to "inPartition"
for
* this to take effect.
The target group size in bytes. The default is computed based on the input
* data size and the size of your cluster. When there are fewer than 50,000 input
* files, "groupFiles"
must be set to "inPartition"
for
* this to take effect.
The target group size in bytes. The default is computed based on the input
* data size and the size of your cluster. When there are fewer than 50,000 input
* files, "groupFiles"
must be set to "inPartition"
for
* this to take effect.
Grouping files is turned on by default when the input contains more than
* 50,000 files. To turn on grouping with fewer than 50,000 files, set this
* parameter to "inPartition". To disable grouping when there are more than 50,000
* files, set this parameter to "none"
.
Grouping files is turned on by default when the input contains more than
* 50,000 files. To turn on grouping with fewer than 50,000 files, set this
* parameter to "inPartition". To disable grouping when there are more than 50,000
* files, set this parameter to "none"
.
Grouping files is turned on by default when the input contains more than
* 50,000 files. To turn on grouping with fewer than 50,000 files, set this
* parameter to "inPartition". To disable grouping when there are more than 50,000
* files, set this parameter to "none"
.
Grouping files is turned on by default when the input contains more than
* 50,000 files. To turn on grouping with fewer than 50,000 files, set this
* parameter to "inPartition". To disable grouping when there are more than 50,000
* files, set this parameter to "none"
.
Grouping files is turned on by default when the input contains more than
* 50,000 files. To turn on grouping with fewer than 50,000 files, set this
* parameter to "inPartition". To disable grouping when there are more than 50,000
* files, set this parameter to "none"
.
Grouping files is turned on by default when the input contains more than
* 50,000 files. To turn on grouping with fewer than 50,000 files, set this
* parameter to "inPartition". To disable grouping when there are more than 50,000
* files, set this parameter to "none"
.
Grouping files is turned on by default when the input contains more than
* 50,000 files. To turn on grouping with fewer than 50,000 files, set this
* parameter to "inPartition". To disable grouping when there are more than 50,000
* files, set this parameter to "none"
.
Grouping files is turned on by default when the input contains more than
* 50,000 files. To turn on grouping with fewer than 50,000 files, set this
* parameter to "inPartition". To disable grouping when there are more than 50,000
* files, set this parameter to "none"
.
If set to true, recursively reads files in all subdirectories under the * specified paths.
*/ inline bool GetRecurse() const{ return m_recurse; } /** *If set to true, recursively reads files in all subdirectories under the * specified paths.
*/ inline bool RecurseHasBeenSet() const { return m_recurseHasBeenSet; } /** *If set to true, recursively reads files in all subdirectories under the * specified paths.
*/ inline void SetRecurse(bool value) { m_recurseHasBeenSet = true; m_recurse = value; } /** *If set to true, recursively reads files in all subdirectories under the * specified paths.
*/ inline S3CsvSource& WithRecurse(bool value) { SetRecurse(value); return *this;} /** *This option controls the duration in milliseconds after which the s3 listing * is likely to be consistent. Files with modification timestamps falling within * the last maxBand milliseconds are tracked specially when using JobBookmarks to * account for Amazon S3 eventual consistency. Most users don't need to set this * option. The default is 900000 milliseconds, or 15 minutes.
*/ inline int GetMaxBand() const{ return m_maxBand; } /** *This option controls the duration in milliseconds after which the s3 listing * is likely to be consistent. Files with modification timestamps falling within * the last maxBand milliseconds are tracked specially when using JobBookmarks to * account for Amazon S3 eventual consistency. Most users don't need to set this * option. The default is 900000 milliseconds, or 15 minutes.
*/ inline bool MaxBandHasBeenSet() const { return m_maxBandHasBeenSet; } /** *This option controls the duration in milliseconds after which the s3 listing * is likely to be consistent. Files with modification timestamps falling within * the last maxBand milliseconds are tracked specially when using JobBookmarks to * account for Amazon S3 eventual consistency. Most users don't need to set this * option. The default is 900000 milliseconds, or 15 minutes.
*/ inline void SetMaxBand(int value) { m_maxBandHasBeenSet = true; m_maxBand = value; } /** *This option controls the duration in milliseconds after which the s3 listing * is likely to be consistent. Files with modification timestamps falling within * the last maxBand milliseconds are tracked specially when using JobBookmarks to * account for Amazon S3 eventual consistency. Most users don't need to set this * option. The default is 900000 milliseconds, or 15 minutes.
*/ inline S3CsvSource& WithMaxBand(int value) { SetMaxBand(value); return *this;} /** *This option specifies the maximum number of files to save from the last * maxBand seconds. If this number is exceeded, extra files are skipped and only * processed in the next job run.
*/ inline int GetMaxFilesInBand() const{ return m_maxFilesInBand; } /** *This option specifies the maximum number of files to save from the last * maxBand seconds. If this number is exceeded, extra files are skipped and only * processed in the next job run.
*/ inline bool MaxFilesInBandHasBeenSet() const { return m_maxFilesInBandHasBeenSet; } /** *This option specifies the maximum number of files to save from the last * maxBand seconds. If this number is exceeded, extra files are skipped and only * processed in the next job run.
*/ inline void SetMaxFilesInBand(int value) { m_maxFilesInBandHasBeenSet = true; m_maxFilesInBand = value; } /** *This option specifies the maximum number of files to save from the last * maxBand seconds. If this number is exceeded, extra files are skipped and only * processed in the next job run.
*/ inline S3CsvSource& WithMaxFilesInBand(int value) { SetMaxFilesInBand(value); return *this;} /** *Specifies additional connection options.
*/ inline const S3DirectSourceAdditionalOptions& GetAdditionalOptions() const{ return m_additionalOptions; } /** *Specifies additional connection options.
*/ inline bool AdditionalOptionsHasBeenSet() const { return m_additionalOptionsHasBeenSet; } /** *Specifies additional connection options.
*/ inline void SetAdditionalOptions(const S3DirectSourceAdditionalOptions& value) { m_additionalOptionsHasBeenSet = true; m_additionalOptions = value; } /** *Specifies additional connection options.
*/ inline void SetAdditionalOptions(S3DirectSourceAdditionalOptions&& value) { m_additionalOptionsHasBeenSet = true; m_additionalOptions = std::move(value); } /** *Specifies additional connection options.
*/ inline S3CsvSource& WithAdditionalOptions(const S3DirectSourceAdditionalOptions& value) { SetAdditionalOptions(value); return *this;} /** *Specifies additional connection options.
*/ inline S3CsvSource& WithAdditionalOptions(S3DirectSourceAdditionalOptions&& value) { SetAdditionalOptions(std::move(value)); return *this;} /** *Specifies the delimiter character. The default is a comma: ",", but any other * character can be specified.
*/ inline const Separator& GetSeparator() const{ return m_separator; } /** *Specifies the delimiter character. The default is a comma: ",", but any other * character can be specified.
*/ inline bool SeparatorHasBeenSet() const { return m_separatorHasBeenSet; } /** *Specifies the delimiter character. The default is a comma: ",", but any other * character can be specified.
*/ inline void SetSeparator(const Separator& value) { m_separatorHasBeenSet = true; m_separator = value; } /** *Specifies the delimiter character. The default is a comma: ",", but any other * character can be specified.
*/ inline void SetSeparator(Separator&& value) { m_separatorHasBeenSet = true; m_separator = std::move(value); } /** *Specifies the delimiter character. The default is a comma: ",", but any other * character can be specified.
*/ inline S3CsvSource& WithSeparator(const Separator& value) { SetSeparator(value); return *this;} /** *Specifies the delimiter character. The default is a comma: ",", but any other * character can be specified.
*/ inline S3CsvSource& WithSeparator(Separator&& value) { SetSeparator(std::move(value)); return *this;} /** *Specifies a character to use for escaping. This option is used only when
* reading CSV files. The default value is none
. If enabled, the
* character which immediately follows is used as-is, except for a small set of
* well-known escapes (\n
, \r
, \t
, and
* \0
).
Specifies a character to use for escaping. This option is used only when
* reading CSV files. The default value is none
. If enabled, the
* character which immediately follows is used as-is, except for a small set of
* well-known escapes (\n
, \r
, \t
, and
* \0
).
Specifies a character to use for escaping. This option is used only when
* reading CSV files. The default value is none
. If enabled, the
* character which immediately follows is used as-is, except for a small set of
* well-known escapes (\n
, \r
, \t
, and
* \0
).
Specifies a character to use for escaping. This option is used only when
* reading CSV files. The default value is none
. If enabled, the
* character which immediately follows is used as-is, except for a small set of
* well-known escapes (\n
, \r
, \t
, and
* \0
).
Specifies a character to use for escaping. This option is used only when
* reading CSV files. The default value is none
. If enabled, the
* character which immediately follows is used as-is, except for a small set of
* well-known escapes (\n
, \r
, \t
, and
* \0
).
Specifies a character to use for escaping. This option is used only when
* reading CSV files. The default value is none
. If enabled, the
* character which immediately follows is used as-is, except for a small set of
* well-known escapes (\n
, \r
, \t
, and
* \0
).
Specifies a character to use for escaping. This option is used only when
* reading CSV files. The default value is none
. If enabled, the
* character which immediately follows is used as-is, except for a small set of
* well-known escapes (\n
, \r
, \t
, and
* \0
).
Specifies a character to use for escaping. This option is used only when
* reading CSV files. The default value is none
. If enabled, the
* character which immediately follows is used as-is, except for a small set of
* well-known escapes (\n
, \r
, \t
, and
* \0
).
Specifies the character to use for quoting. The default is a double quote:
* '"'
. Set this to -1
to turn off quoting entirely.
Specifies the character to use for quoting. The default is a double quote:
* '"'
. Set this to -1
to turn off quoting entirely.
Specifies the character to use for quoting. The default is a double quote:
* '"'
. Set this to -1
to turn off quoting entirely.
Specifies the character to use for quoting. The default is a double quote:
* '"'
. Set this to -1
to turn off quoting entirely.
Specifies the character to use for quoting. The default is a double quote:
* '"'
. Set this to -1
to turn off quoting entirely.
Specifies the character to use for quoting. The default is a double quote:
* '"'
. Set this to -1
to turn off quoting entirely.
A Boolean value that specifies whether a single record can span multiple
* lines. This can occur when a field contains a quoted new-line character. You
* must set this option to True if any record spans multiple lines. The default
* value is False
, which allows for more aggressive file-splitting
* during parsing.
A Boolean value that specifies whether a single record can span multiple
* lines. This can occur when a field contains a quoted new-line character. You
* must set this option to True if any record spans multiple lines. The default
* value is False
, which allows for more aggressive file-splitting
* during parsing.
A Boolean value that specifies whether a single record can span multiple
* lines. This can occur when a field contains a quoted new-line character. You
* must set this option to True if any record spans multiple lines. The default
* value is False
, which allows for more aggressive file-splitting
* during parsing.
A Boolean value that specifies whether a single record can span multiple
* lines. This can occur when a field contains a quoted new-line character. You
* must set this option to True if any record spans multiple lines. The default
* value is False
, which allows for more aggressive file-splitting
* during parsing.
A Boolean value that specifies whether to treat the first line as a header.
* The default value is False
.
A Boolean value that specifies whether to treat the first line as a header.
* The default value is False
.
A Boolean value that specifies whether to treat the first line as a header.
* The default value is False
.
A Boolean value that specifies whether to treat the first line as a header.
* The default value is False
.
A Boolean value that specifies whether to write the header to output. The
* default value is True
.
A Boolean value that specifies whether to write the header to output. The
* default value is True
.
A Boolean value that specifies whether to write the header to output. The
* default value is True
.
A Boolean value that specifies whether to write the header to output. The
* default value is True
.
A Boolean value that specifies whether to skip the first data line. The
* default value is False
.
A Boolean value that specifies whether to skip the first data line. The
* default value is False
.
A Boolean value that specifies whether to skip the first data line. The
* default value is False
.
A Boolean value that specifies whether to skip the first data line. The
* default value is False
.
A Boolean value that specifies whether to use the advanced SIMD CSV reader * along with Apache Arrow based columnar memory formats. Only available in Glue * version 3.0.
*/ inline bool GetOptimizePerformance() const{ return m_optimizePerformance; } /** *A Boolean value that specifies whether to use the advanced SIMD CSV reader * along with Apache Arrow based columnar memory formats. Only available in Glue * version 3.0.
*/ inline bool OptimizePerformanceHasBeenSet() const { return m_optimizePerformanceHasBeenSet; } /** *A Boolean value that specifies whether to use the advanced SIMD CSV reader * along with Apache Arrow based columnar memory formats. Only available in Glue * version 3.0.
*/ inline void SetOptimizePerformance(bool value) { m_optimizePerformanceHasBeenSet = true; m_optimizePerformance = value; } /** *A Boolean value that specifies whether to use the advanced SIMD CSV reader * along with Apache Arrow based columnar memory formats. Only available in Glue * version 3.0.
*/ inline S3CsvSource& WithOptimizePerformance(bool value) { SetOptimizePerformance(value); return *this;} /** *Specifies the data schema for the S3 CSV source.
*/ inline const Aws::VectorSpecifies the data schema for the S3 CSV source.
*/ inline bool OutputSchemasHasBeenSet() const { return m_outputSchemasHasBeenSet; } /** *Specifies the data schema for the S3 CSV source.
*/ inline void SetOutputSchemas(const Aws::VectorSpecifies the data schema for the S3 CSV source.
*/ inline void SetOutputSchemas(Aws::VectorSpecifies the data schema for the S3 CSV source.
*/ inline S3CsvSource& WithOutputSchemas(const Aws::VectorSpecifies the data schema for the S3 CSV source.
*/ inline S3CsvSource& WithOutputSchemas(Aws::VectorSpecifies the data schema for the S3 CSV source.
*/ inline S3CsvSource& AddOutputSchemas(const GlueSchema& value) { m_outputSchemasHasBeenSet = true; m_outputSchemas.push_back(value); return *this; } /** *Specifies the data schema for the S3 CSV source.
*/ inline S3CsvSource& AddOutputSchemas(GlueSchema&& value) { m_outputSchemasHasBeenSet = true; m_outputSchemas.push_back(std::move(value)); return *this; } private: Aws::String m_name; bool m_nameHasBeenSet = false; Aws::Vector