/**
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: Apache-2.0.
*/
#pragma once
#include Specifies a JSON data store stored in Amazon S3.See Also:
* AWS
* API Reference
The name of the data store.
*/ inline const Aws::String& GetName() const{ return m_name; } /** *The name of the data store.
*/ inline bool NameHasBeenSet() const { return m_nameHasBeenSet; } /** *The name of the data store.
*/ inline void SetName(const Aws::String& value) { m_nameHasBeenSet = true; m_name = value; } /** *The name of the data store.
*/ inline void SetName(Aws::String&& value) { m_nameHasBeenSet = true; m_name = std::move(value); } /** *The name of the data store.
*/ inline void SetName(const char* value) { m_nameHasBeenSet = true; m_name.assign(value); } /** *The name of the data store.
*/ inline S3JsonSource& WithName(const Aws::String& value) { SetName(value); return *this;} /** *The name of the data store.
*/ inline S3JsonSource& WithName(Aws::String&& value) { SetName(std::move(value)); return *this;} /** *The name of the data store.
*/ inline S3JsonSource& WithName(const char* value) { SetName(value); return *this;} /** *A list of the Amazon S3 paths to read from.
*/ inline const Aws::VectorA list of the Amazon S3 paths to read from.
*/ inline bool PathsHasBeenSet() const { return m_pathsHasBeenSet; } /** *A list of the Amazon S3 paths to read from.
*/ inline void SetPaths(const Aws::VectorA list of the Amazon S3 paths to read from.
*/ inline void SetPaths(Aws::VectorA list of the Amazon S3 paths to read from.
*/ inline S3JsonSource& WithPaths(const Aws::VectorA list of the Amazon S3 paths to read from.
*/ inline S3JsonSource& WithPaths(Aws::VectorA list of the Amazon S3 paths to read from.
*/ inline S3JsonSource& AddPaths(const Aws::String& value) { m_pathsHasBeenSet = true; m_paths.push_back(value); return *this; } /** *A list of the Amazon S3 paths to read from.
*/ inline S3JsonSource& AddPaths(Aws::String&& value) { m_pathsHasBeenSet = true; m_paths.push_back(std::move(value)); return *this; } /** *A list of the Amazon S3 paths to read from.
*/ inline S3JsonSource& AddPaths(const char* value) { m_pathsHasBeenSet = true; m_paths.push_back(value); return *this; } /** *Specifies how the data is compressed. This is generally not necessary if the
* data has a standard file extension. Possible values are "gzip"
and
* "bzip"
).
Specifies how the data is compressed. This is generally not necessary if the
* data has a standard file extension. Possible values are "gzip"
and
* "bzip"
).
Specifies how the data is compressed. This is generally not necessary if the
* data has a standard file extension. Possible values are "gzip"
and
* "bzip"
).
Specifies how the data is compressed. This is generally not necessary if the
* data has a standard file extension. Possible values are "gzip"
and
* "bzip"
).
Specifies how the data is compressed. This is generally not necessary if the
* data has a standard file extension. Possible values are "gzip"
and
* "bzip"
).
Specifies how the data is compressed. This is generally not necessary if the
* data has a standard file extension. Possible values are "gzip"
and
* "bzip"
).
A string containing a JSON list of Unix-style glob patterns to exclude. For * example, "[\"**.pdf\"]" excludes all PDF files.
*/ inline const Aws::VectorA string containing a JSON list of Unix-style glob patterns to exclude. For * example, "[\"**.pdf\"]" excludes all PDF files.
*/ inline bool ExclusionsHasBeenSet() const { return m_exclusionsHasBeenSet; } /** *A string containing a JSON list of Unix-style glob patterns to exclude. For * example, "[\"**.pdf\"]" excludes all PDF files.
*/ inline void SetExclusions(const Aws::VectorA string containing a JSON list of Unix-style glob patterns to exclude. For * example, "[\"**.pdf\"]" excludes all PDF files.
*/ inline void SetExclusions(Aws::VectorA string containing a JSON list of Unix-style glob patterns to exclude. For * example, "[\"**.pdf\"]" excludes all PDF files.
*/ inline S3JsonSource& WithExclusions(const Aws::VectorA string containing a JSON list of Unix-style glob patterns to exclude. For * example, "[\"**.pdf\"]" excludes all PDF files.
*/ inline S3JsonSource& WithExclusions(Aws::VectorA string containing a JSON list of Unix-style glob patterns to exclude. For * example, "[\"**.pdf\"]" excludes all PDF files.
*/ inline S3JsonSource& AddExclusions(const Aws::String& value) { m_exclusionsHasBeenSet = true; m_exclusions.push_back(value); return *this; } /** *A string containing a JSON list of Unix-style glob patterns to exclude. For * example, "[\"**.pdf\"]" excludes all PDF files.
*/ inline S3JsonSource& AddExclusions(Aws::String&& value) { m_exclusionsHasBeenSet = true; m_exclusions.push_back(std::move(value)); return *this; } /** *A string containing a JSON list of Unix-style glob patterns to exclude. For * example, "[\"**.pdf\"]" excludes all PDF files.
*/ inline S3JsonSource& AddExclusions(const char* value) { m_exclusionsHasBeenSet = true; m_exclusions.push_back(value); return *this; } /** *The target group size in bytes. The default is computed based on the input
* data size and the size of your cluster. When there are fewer than 50,000 input
* files, "groupFiles"
must be set to "inPartition"
for
* this to take effect.
The target group size in bytes. The default is computed based on the input
* data size and the size of your cluster. When there are fewer than 50,000 input
* files, "groupFiles"
must be set to "inPartition"
for
* this to take effect.
The target group size in bytes. The default is computed based on the input
* data size and the size of your cluster. When there are fewer than 50,000 input
* files, "groupFiles"
must be set to "inPartition"
for
* this to take effect.
The target group size in bytes. The default is computed based on the input
* data size and the size of your cluster. When there are fewer than 50,000 input
* files, "groupFiles"
must be set to "inPartition"
for
* this to take effect.
The target group size in bytes. The default is computed based on the input
* data size and the size of your cluster. When there are fewer than 50,000 input
* files, "groupFiles"
must be set to "inPartition"
for
* this to take effect.
The target group size in bytes. The default is computed based on the input
* data size and the size of your cluster. When there are fewer than 50,000 input
* files, "groupFiles"
must be set to "inPartition"
for
* this to take effect.
The target group size in bytes. The default is computed based on the input
* data size and the size of your cluster. When there are fewer than 50,000 input
* files, "groupFiles"
must be set to "inPartition"
for
* this to take effect.
The target group size in bytes. The default is computed based on the input
* data size and the size of your cluster. When there are fewer than 50,000 input
* files, "groupFiles"
must be set to "inPartition"
for
* this to take effect.
Grouping files is turned on by default when the input contains more than
* 50,000 files. To turn on grouping with fewer than 50,000 files, set this
* parameter to "inPartition". To disable grouping when there are more than 50,000
* files, set this parameter to "none"
.
Grouping files is turned on by default when the input contains more than
* 50,000 files. To turn on grouping with fewer than 50,000 files, set this
* parameter to "inPartition". To disable grouping when there are more than 50,000
* files, set this parameter to "none"
.
Grouping files is turned on by default when the input contains more than
* 50,000 files. To turn on grouping with fewer than 50,000 files, set this
* parameter to "inPartition". To disable grouping when there are more than 50,000
* files, set this parameter to "none"
.
Grouping files is turned on by default when the input contains more than
* 50,000 files. To turn on grouping with fewer than 50,000 files, set this
* parameter to "inPartition". To disable grouping when there are more than 50,000
* files, set this parameter to "none"
.
Grouping files is turned on by default when the input contains more than
* 50,000 files. To turn on grouping with fewer than 50,000 files, set this
* parameter to "inPartition". To disable grouping when there are more than 50,000
* files, set this parameter to "none"
.
Grouping files is turned on by default when the input contains more than
* 50,000 files. To turn on grouping with fewer than 50,000 files, set this
* parameter to "inPartition". To disable grouping when there are more than 50,000
* files, set this parameter to "none"
.
Grouping files is turned on by default when the input contains more than
* 50,000 files. To turn on grouping with fewer than 50,000 files, set this
* parameter to "inPartition". To disable grouping when there are more than 50,000
* files, set this parameter to "none"
.
Grouping files is turned on by default when the input contains more than
* 50,000 files. To turn on grouping with fewer than 50,000 files, set this
* parameter to "inPartition". To disable grouping when there are more than 50,000
* files, set this parameter to "none"
.
If set to true, recursively reads files in all subdirectories under the * specified paths.
*/ inline bool GetRecurse() const{ return m_recurse; } /** *If set to true, recursively reads files in all subdirectories under the * specified paths.
*/ inline bool RecurseHasBeenSet() const { return m_recurseHasBeenSet; } /** *If set to true, recursively reads files in all subdirectories under the * specified paths.
*/ inline void SetRecurse(bool value) { m_recurseHasBeenSet = true; m_recurse = value; } /** *If set to true, recursively reads files in all subdirectories under the * specified paths.
*/ inline S3JsonSource& WithRecurse(bool value) { SetRecurse(value); return *this;} /** *This option controls the duration in milliseconds after which the s3 listing * is likely to be consistent. Files with modification timestamps falling within * the last maxBand milliseconds are tracked specially when using JobBookmarks to * account for Amazon S3 eventual consistency. Most users don't need to set this * option. The default is 900000 milliseconds, or 15 minutes.
*/ inline int GetMaxBand() const{ return m_maxBand; } /** *This option controls the duration in milliseconds after which the s3 listing * is likely to be consistent. Files with modification timestamps falling within * the last maxBand milliseconds are tracked specially when using JobBookmarks to * account for Amazon S3 eventual consistency. Most users don't need to set this * option. The default is 900000 milliseconds, or 15 minutes.
*/ inline bool MaxBandHasBeenSet() const { return m_maxBandHasBeenSet; } /** *This option controls the duration in milliseconds after which the s3 listing * is likely to be consistent. Files with modification timestamps falling within * the last maxBand milliseconds are tracked specially when using JobBookmarks to * account for Amazon S3 eventual consistency. Most users don't need to set this * option. The default is 900000 milliseconds, or 15 minutes.
*/ inline void SetMaxBand(int value) { m_maxBandHasBeenSet = true; m_maxBand = value; } /** *This option controls the duration in milliseconds after which the s3 listing * is likely to be consistent. Files with modification timestamps falling within * the last maxBand milliseconds are tracked specially when using JobBookmarks to * account for Amazon S3 eventual consistency. Most users don't need to set this * option. The default is 900000 milliseconds, or 15 minutes.
*/ inline S3JsonSource& WithMaxBand(int value) { SetMaxBand(value); return *this;} /** *This option specifies the maximum number of files to save from the last * maxBand seconds. If this number is exceeded, extra files are skipped and only * processed in the next job run.
*/ inline int GetMaxFilesInBand() const{ return m_maxFilesInBand; } /** *This option specifies the maximum number of files to save from the last * maxBand seconds. If this number is exceeded, extra files are skipped and only * processed in the next job run.
*/ inline bool MaxFilesInBandHasBeenSet() const { return m_maxFilesInBandHasBeenSet; } /** *This option specifies the maximum number of files to save from the last * maxBand seconds. If this number is exceeded, extra files are skipped and only * processed in the next job run.
*/ inline void SetMaxFilesInBand(int value) { m_maxFilesInBandHasBeenSet = true; m_maxFilesInBand = value; } /** *This option specifies the maximum number of files to save from the last * maxBand seconds. If this number is exceeded, extra files are skipped and only * processed in the next job run.
*/ inline S3JsonSource& WithMaxFilesInBand(int value) { SetMaxFilesInBand(value); return *this;} /** *Specifies additional connection options.
*/ inline const S3DirectSourceAdditionalOptions& GetAdditionalOptions() const{ return m_additionalOptions; } /** *Specifies additional connection options.
*/ inline bool AdditionalOptionsHasBeenSet() const { return m_additionalOptionsHasBeenSet; } /** *Specifies additional connection options.
*/ inline void SetAdditionalOptions(const S3DirectSourceAdditionalOptions& value) { m_additionalOptionsHasBeenSet = true; m_additionalOptions = value; } /** *Specifies additional connection options.
*/ inline void SetAdditionalOptions(S3DirectSourceAdditionalOptions&& value) { m_additionalOptionsHasBeenSet = true; m_additionalOptions = std::move(value); } /** *Specifies additional connection options.
*/ inline S3JsonSource& WithAdditionalOptions(const S3DirectSourceAdditionalOptions& value) { SetAdditionalOptions(value); return *this;} /** *Specifies additional connection options.
*/ inline S3JsonSource& WithAdditionalOptions(S3DirectSourceAdditionalOptions&& value) { SetAdditionalOptions(std::move(value)); return *this;} /** *A JsonPath string defining the JSON data.
*/ inline const Aws::String& GetJsonPath() const{ return m_jsonPath; } /** *A JsonPath string defining the JSON data.
*/ inline bool JsonPathHasBeenSet() const { return m_jsonPathHasBeenSet; } /** *A JsonPath string defining the JSON data.
*/ inline void SetJsonPath(const Aws::String& value) { m_jsonPathHasBeenSet = true; m_jsonPath = value; } /** *A JsonPath string defining the JSON data.
*/ inline void SetJsonPath(Aws::String&& value) { m_jsonPathHasBeenSet = true; m_jsonPath = std::move(value); } /** *A JsonPath string defining the JSON data.
*/ inline void SetJsonPath(const char* value) { m_jsonPathHasBeenSet = true; m_jsonPath.assign(value); } /** *A JsonPath string defining the JSON data.
*/ inline S3JsonSource& WithJsonPath(const Aws::String& value) { SetJsonPath(value); return *this;} /** *A JsonPath string defining the JSON data.
*/ inline S3JsonSource& WithJsonPath(Aws::String&& value) { SetJsonPath(std::move(value)); return *this;} /** *A JsonPath string defining the JSON data.
*/ inline S3JsonSource& WithJsonPath(const char* value) { SetJsonPath(value); return *this;} /** *A Boolean value that specifies whether a single record can span multiple
* lines. This can occur when a field contains a quoted new-line character. You
* must set this option to True if any record spans multiple lines. The default
* value is False
, which allows for more aggressive file-splitting
* during parsing.
A Boolean value that specifies whether a single record can span multiple
* lines. This can occur when a field contains a quoted new-line character. You
* must set this option to True if any record spans multiple lines. The default
* value is False
, which allows for more aggressive file-splitting
* during parsing.
A Boolean value that specifies whether a single record can span multiple
* lines. This can occur when a field contains a quoted new-line character. You
* must set this option to True if any record spans multiple lines. The default
* value is False
, which allows for more aggressive file-splitting
* during parsing.
A Boolean value that specifies whether a single record can span multiple
* lines. This can occur when a field contains a quoted new-line character. You
* must set this option to True if any record spans multiple lines. The default
* value is False
, which allows for more aggressive file-splitting
* during parsing.
Specifies the data schema for the S3 JSON source.
*/ inline const Aws::VectorSpecifies the data schema for the S3 JSON source.
*/ inline bool OutputSchemasHasBeenSet() const { return m_outputSchemasHasBeenSet; } /** *Specifies the data schema for the S3 JSON source.
*/ inline void SetOutputSchemas(const Aws::VectorSpecifies the data schema for the S3 JSON source.
*/ inline void SetOutputSchemas(Aws::VectorSpecifies the data schema for the S3 JSON source.
*/ inline S3JsonSource& WithOutputSchemas(const Aws::VectorSpecifies the data schema for the S3 JSON source.
*/ inline S3JsonSource& WithOutputSchemas(Aws::VectorSpecifies the data schema for the S3 JSON source.
*/ inline S3JsonSource& AddOutputSchemas(const GlueSchema& value) { m_outputSchemasHasBeenSet = true; m_outputSchemas.push_back(value); return *this; } /** *Specifies the data schema for the S3 JSON source.
*/ inline S3JsonSource& AddOutputSchemas(GlueSchema&& value) { m_outputSchemasHasBeenSet = true; m_outputSchemas.push_back(std::move(value)); return *this; } private: Aws::String m_name; bool m_nameHasBeenSet = false; Aws::Vector