/** * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0. */ #pragma once #include #include #include #include #include #include #include namespace Aws { namespace Utils { namespace Json { class JsonValue; class JsonView; } // namespace Json } // namespace Utils namespace Glue { namespace Model { /** *

Specifies a JSON data store stored in Amazon S3.

See Also:

* AWS * API Reference

*/ class S3JsonSource { public: AWS_GLUE_API S3JsonSource(); AWS_GLUE_API S3JsonSource(Aws::Utils::Json::JsonView jsonValue); AWS_GLUE_API S3JsonSource& operator=(Aws::Utils::Json::JsonView jsonValue); AWS_GLUE_API Aws::Utils::Json::JsonValue Jsonize() const; /** *

The name of the data store.

*/ inline const Aws::String& GetName() const{ return m_name; } /** *

The name of the data store.

*/ inline bool NameHasBeenSet() const { return m_nameHasBeenSet; } /** *

The name of the data store.

*/ inline void SetName(const Aws::String& value) { m_nameHasBeenSet = true; m_name = value; } /** *

The name of the data store.

*/ inline void SetName(Aws::String&& value) { m_nameHasBeenSet = true; m_name = std::move(value); } /** *

The name of the data store.

*/ inline void SetName(const char* value) { m_nameHasBeenSet = true; m_name.assign(value); } /** *

The name of the data store.

*/ inline S3JsonSource& WithName(const Aws::String& value) { SetName(value); return *this;} /** *

The name of the data store.

*/ inline S3JsonSource& WithName(Aws::String&& value) { SetName(std::move(value)); return *this;} /** *

The name of the data store.

*/ inline S3JsonSource& WithName(const char* value) { SetName(value); return *this;} /** *

A list of the Amazon S3 paths to read from.

*/ inline const Aws::Vector& GetPaths() const{ return m_paths; } /** *

A list of the Amazon S3 paths to read from.

*/ inline bool PathsHasBeenSet() const { return m_pathsHasBeenSet; } /** *

A list of the Amazon S3 paths to read from.

*/ inline void SetPaths(const Aws::Vector& value) { m_pathsHasBeenSet = true; m_paths = value; } /** *

A list of the Amazon S3 paths to read from.

*/ inline void SetPaths(Aws::Vector&& value) { m_pathsHasBeenSet = true; m_paths = std::move(value); } /** *

A list of the Amazon S3 paths to read from.

*/ inline S3JsonSource& WithPaths(const Aws::Vector& value) { SetPaths(value); return *this;} /** *

A list of the Amazon S3 paths to read from.

*/ inline S3JsonSource& WithPaths(Aws::Vector&& value) { SetPaths(std::move(value)); return *this;} /** *

A list of the Amazon S3 paths to read from.

*/ inline S3JsonSource& AddPaths(const Aws::String& value) { m_pathsHasBeenSet = true; m_paths.push_back(value); return *this; } /** *

A list of the Amazon S3 paths to read from.

*/ inline S3JsonSource& AddPaths(Aws::String&& value) { m_pathsHasBeenSet = true; m_paths.push_back(std::move(value)); return *this; } /** *

A list of the Amazon S3 paths to read from.

*/ inline S3JsonSource& AddPaths(const char* value) { m_pathsHasBeenSet = true; m_paths.push_back(value); return *this; } /** *

Specifies how the data is compressed. This is generally not necessary if the * data has a standard file extension. Possible values are "gzip" and * "bzip").

*/ inline const CompressionType& GetCompressionType() const{ return m_compressionType; } /** *

Specifies how the data is compressed. This is generally not necessary if the * data has a standard file extension. Possible values are "gzip" and * "bzip").

*/ inline bool CompressionTypeHasBeenSet() const { return m_compressionTypeHasBeenSet; } /** *

Specifies how the data is compressed. This is generally not necessary if the * data has a standard file extension. Possible values are "gzip" and * "bzip").

*/ inline void SetCompressionType(const CompressionType& value) { m_compressionTypeHasBeenSet = true; m_compressionType = value; } /** *

Specifies how the data is compressed. This is generally not necessary if the * data has a standard file extension. Possible values are "gzip" and * "bzip").

*/ inline void SetCompressionType(CompressionType&& value) { m_compressionTypeHasBeenSet = true; m_compressionType = std::move(value); } /** *

Specifies how the data is compressed. This is generally not necessary if the * data has a standard file extension. Possible values are "gzip" and * "bzip").

*/ inline S3JsonSource& WithCompressionType(const CompressionType& value) { SetCompressionType(value); return *this;} /** *

Specifies how the data is compressed. This is generally not necessary if the * data has a standard file extension. Possible values are "gzip" and * "bzip").

*/ inline S3JsonSource& WithCompressionType(CompressionType&& value) { SetCompressionType(std::move(value)); return *this;} /** *

A string containing a JSON list of Unix-style glob patterns to exclude. For * example, "[\"**.pdf\"]" excludes all PDF files.

*/ inline const Aws::Vector& GetExclusions() const{ return m_exclusions; } /** *

A string containing a JSON list of Unix-style glob patterns to exclude. For * example, "[\"**.pdf\"]" excludes all PDF files.

*/ inline bool ExclusionsHasBeenSet() const { return m_exclusionsHasBeenSet; } /** *

A string containing a JSON list of Unix-style glob patterns to exclude. For * example, "[\"**.pdf\"]" excludes all PDF files.

*/ inline void SetExclusions(const Aws::Vector& value) { m_exclusionsHasBeenSet = true; m_exclusions = value; } /** *

A string containing a JSON list of Unix-style glob patterns to exclude. For * example, "[\"**.pdf\"]" excludes all PDF files.

*/ inline void SetExclusions(Aws::Vector&& value) { m_exclusionsHasBeenSet = true; m_exclusions = std::move(value); } /** *

A string containing a JSON list of Unix-style glob patterns to exclude. For * example, "[\"**.pdf\"]" excludes all PDF files.

*/ inline S3JsonSource& WithExclusions(const Aws::Vector& value) { SetExclusions(value); return *this;} /** *

A string containing a JSON list of Unix-style glob patterns to exclude. For * example, "[\"**.pdf\"]" excludes all PDF files.

*/ inline S3JsonSource& WithExclusions(Aws::Vector&& value) { SetExclusions(std::move(value)); return *this;} /** *

A string containing a JSON list of Unix-style glob patterns to exclude. For * example, "[\"**.pdf\"]" excludes all PDF files.

*/ inline S3JsonSource& AddExclusions(const Aws::String& value) { m_exclusionsHasBeenSet = true; m_exclusions.push_back(value); return *this; } /** *

A string containing a JSON list of Unix-style glob patterns to exclude. For * example, "[\"**.pdf\"]" excludes all PDF files.

*/ inline S3JsonSource& AddExclusions(Aws::String&& value) { m_exclusionsHasBeenSet = true; m_exclusions.push_back(std::move(value)); return *this; } /** *

A string containing a JSON list of Unix-style glob patterns to exclude. For * example, "[\"**.pdf\"]" excludes all PDF files.

*/ inline S3JsonSource& AddExclusions(const char* value) { m_exclusionsHasBeenSet = true; m_exclusions.push_back(value); return *this; } /** *

The target group size in bytes. The default is computed based on the input * data size and the size of your cluster. When there are fewer than 50,000 input * files, "groupFiles" must be set to "inPartition" for * this to take effect.

*/ inline const Aws::String& GetGroupSize() const{ return m_groupSize; } /** *

The target group size in bytes. The default is computed based on the input * data size and the size of your cluster. When there are fewer than 50,000 input * files, "groupFiles" must be set to "inPartition" for * this to take effect.

*/ inline bool GroupSizeHasBeenSet() const { return m_groupSizeHasBeenSet; } /** *

The target group size in bytes. The default is computed based on the input * data size and the size of your cluster. When there are fewer than 50,000 input * files, "groupFiles" must be set to "inPartition" for * this to take effect.

*/ inline void SetGroupSize(const Aws::String& value) { m_groupSizeHasBeenSet = true; m_groupSize = value; } /** *

The target group size in bytes. The default is computed based on the input * data size and the size of your cluster. When there are fewer than 50,000 input * files, "groupFiles" must be set to "inPartition" for * this to take effect.

*/ inline void SetGroupSize(Aws::String&& value) { m_groupSizeHasBeenSet = true; m_groupSize = std::move(value); } /** *

The target group size in bytes. The default is computed based on the input * data size and the size of your cluster. When there are fewer than 50,000 input * files, "groupFiles" must be set to "inPartition" for * this to take effect.

*/ inline void SetGroupSize(const char* value) { m_groupSizeHasBeenSet = true; m_groupSize.assign(value); } /** *

The target group size in bytes. The default is computed based on the input * data size and the size of your cluster. When there are fewer than 50,000 input * files, "groupFiles" must be set to "inPartition" for * this to take effect.

*/ inline S3JsonSource& WithGroupSize(const Aws::String& value) { SetGroupSize(value); return *this;} /** *

The target group size in bytes. The default is computed based on the input * data size and the size of your cluster. When there are fewer than 50,000 input * files, "groupFiles" must be set to "inPartition" for * this to take effect.

*/ inline S3JsonSource& WithGroupSize(Aws::String&& value) { SetGroupSize(std::move(value)); return *this;} /** *

The target group size in bytes. The default is computed based on the input * data size and the size of your cluster. When there are fewer than 50,000 input * files, "groupFiles" must be set to "inPartition" for * this to take effect.

*/ inline S3JsonSource& WithGroupSize(const char* value) { SetGroupSize(value); return *this;} /** *

Grouping files is turned on by default when the input contains more than * 50,000 files. To turn on grouping with fewer than 50,000 files, set this * parameter to "inPartition". To disable grouping when there are more than 50,000 * files, set this parameter to "none".

*/ inline const Aws::String& GetGroupFiles() const{ return m_groupFiles; } /** *

Grouping files is turned on by default when the input contains more than * 50,000 files. To turn on grouping with fewer than 50,000 files, set this * parameter to "inPartition". To disable grouping when there are more than 50,000 * files, set this parameter to "none".

*/ inline bool GroupFilesHasBeenSet() const { return m_groupFilesHasBeenSet; } /** *

Grouping files is turned on by default when the input contains more than * 50,000 files. To turn on grouping with fewer than 50,000 files, set this * parameter to "inPartition". To disable grouping when there are more than 50,000 * files, set this parameter to "none".

*/ inline void SetGroupFiles(const Aws::String& value) { m_groupFilesHasBeenSet = true; m_groupFiles = value; } /** *

Grouping files is turned on by default when the input contains more than * 50,000 files. To turn on grouping with fewer than 50,000 files, set this * parameter to "inPartition". To disable grouping when there are more than 50,000 * files, set this parameter to "none".

*/ inline void SetGroupFiles(Aws::String&& value) { m_groupFilesHasBeenSet = true; m_groupFiles = std::move(value); } /** *

Grouping files is turned on by default when the input contains more than * 50,000 files. To turn on grouping with fewer than 50,000 files, set this * parameter to "inPartition". To disable grouping when there are more than 50,000 * files, set this parameter to "none".

*/ inline void SetGroupFiles(const char* value) { m_groupFilesHasBeenSet = true; m_groupFiles.assign(value); } /** *

Grouping files is turned on by default when the input contains more than * 50,000 files. To turn on grouping with fewer than 50,000 files, set this * parameter to "inPartition". To disable grouping when there are more than 50,000 * files, set this parameter to "none".

*/ inline S3JsonSource& WithGroupFiles(const Aws::String& value) { SetGroupFiles(value); return *this;} /** *

Grouping files is turned on by default when the input contains more than * 50,000 files. To turn on grouping with fewer than 50,000 files, set this * parameter to "inPartition". To disable grouping when there are more than 50,000 * files, set this parameter to "none".

*/ inline S3JsonSource& WithGroupFiles(Aws::String&& value) { SetGroupFiles(std::move(value)); return *this;} /** *

Grouping files is turned on by default when the input contains more than * 50,000 files. To turn on grouping with fewer than 50,000 files, set this * parameter to "inPartition". To disable grouping when there are more than 50,000 * files, set this parameter to "none".

*/ inline S3JsonSource& WithGroupFiles(const char* value) { SetGroupFiles(value); return *this;} /** *

If set to true, recursively reads files in all subdirectories under the * specified paths.

*/ inline bool GetRecurse() const{ return m_recurse; } /** *

If set to true, recursively reads files in all subdirectories under the * specified paths.

*/ inline bool RecurseHasBeenSet() const { return m_recurseHasBeenSet; } /** *

If set to true, recursively reads files in all subdirectories under the * specified paths.

*/ inline void SetRecurse(bool value) { m_recurseHasBeenSet = true; m_recurse = value; } /** *

If set to true, recursively reads files in all subdirectories under the * specified paths.

*/ inline S3JsonSource& WithRecurse(bool value) { SetRecurse(value); return *this;} /** *

This option controls the duration in milliseconds after which the s3 listing * is likely to be consistent. Files with modification timestamps falling within * the last maxBand milliseconds are tracked specially when using JobBookmarks to * account for Amazon S3 eventual consistency. Most users don't need to set this * option. The default is 900000 milliseconds, or 15 minutes.

*/ inline int GetMaxBand() const{ return m_maxBand; } /** *

This option controls the duration in milliseconds after which the s3 listing * is likely to be consistent. Files with modification timestamps falling within * the last maxBand milliseconds are tracked specially when using JobBookmarks to * account for Amazon S3 eventual consistency. Most users don't need to set this * option. The default is 900000 milliseconds, or 15 minutes.

*/ inline bool MaxBandHasBeenSet() const { return m_maxBandHasBeenSet; } /** *

This option controls the duration in milliseconds after which the s3 listing * is likely to be consistent. Files with modification timestamps falling within * the last maxBand milliseconds are tracked specially when using JobBookmarks to * account for Amazon S3 eventual consistency. Most users don't need to set this * option. The default is 900000 milliseconds, or 15 minutes.

*/ inline void SetMaxBand(int value) { m_maxBandHasBeenSet = true; m_maxBand = value; } /** *

This option controls the duration in milliseconds after which the s3 listing * is likely to be consistent. Files with modification timestamps falling within * the last maxBand milliseconds are tracked specially when using JobBookmarks to * account for Amazon S3 eventual consistency. Most users don't need to set this * option. The default is 900000 milliseconds, or 15 minutes.

*/ inline S3JsonSource& WithMaxBand(int value) { SetMaxBand(value); return *this;} /** *

This option specifies the maximum number of files to save from the last * maxBand seconds. If this number is exceeded, extra files are skipped and only * processed in the next job run.

*/ inline int GetMaxFilesInBand() const{ return m_maxFilesInBand; } /** *

This option specifies the maximum number of files to save from the last * maxBand seconds. If this number is exceeded, extra files are skipped and only * processed in the next job run.

*/ inline bool MaxFilesInBandHasBeenSet() const { return m_maxFilesInBandHasBeenSet; } /** *

This option specifies the maximum number of files to save from the last * maxBand seconds. If this number is exceeded, extra files are skipped and only * processed in the next job run.

*/ inline void SetMaxFilesInBand(int value) { m_maxFilesInBandHasBeenSet = true; m_maxFilesInBand = value; } /** *

This option specifies the maximum number of files to save from the last * maxBand seconds. If this number is exceeded, extra files are skipped and only * processed in the next job run.

*/ inline S3JsonSource& WithMaxFilesInBand(int value) { SetMaxFilesInBand(value); return *this;} /** *

Specifies additional connection options.

*/ inline const S3DirectSourceAdditionalOptions& GetAdditionalOptions() const{ return m_additionalOptions; } /** *

Specifies additional connection options.

*/ inline bool AdditionalOptionsHasBeenSet() const { return m_additionalOptionsHasBeenSet; } /** *

Specifies additional connection options.

*/ inline void SetAdditionalOptions(const S3DirectSourceAdditionalOptions& value) { m_additionalOptionsHasBeenSet = true; m_additionalOptions = value; } /** *

Specifies additional connection options.

*/ inline void SetAdditionalOptions(S3DirectSourceAdditionalOptions&& value) { m_additionalOptionsHasBeenSet = true; m_additionalOptions = std::move(value); } /** *

Specifies additional connection options.

*/ inline S3JsonSource& WithAdditionalOptions(const S3DirectSourceAdditionalOptions& value) { SetAdditionalOptions(value); return *this;} /** *

Specifies additional connection options.

*/ inline S3JsonSource& WithAdditionalOptions(S3DirectSourceAdditionalOptions&& value) { SetAdditionalOptions(std::move(value)); return *this;} /** *

A JsonPath string defining the JSON data.

*/ inline const Aws::String& GetJsonPath() const{ return m_jsonPath; } /** *

A JsonPath string defining the JSON data.

*/ inline bool JsonPathHasBeenSet() const { return m_jsonPathHasBeenSet; } /** *

A JsonPath string defining the JSON data.

*/ inline void SetJsonPath(const Aws::String& value) { m_jsonPathHasBeenSet = true; m_jsonPath = value; } /** *

A JsonPath string defining the JSON data.

*/ inline void SetJsonPath(Aws::String&& value) { m_jsonPathHasBeenSet = true; m_jsonPath = std::move(value); } /** *

A JsonPath string defining the JSON data.

*/ inline void SetJsonPath(const char* value) { m_jsonPathHasBeenSet = true; m_jsonPath.assign(value); } /** *

A JsonPath string defining the JSON data.

*/ inline S3JsonSource& WithJsonPath(const Aws::String& value) { SetJsonPath(value); return *this;} /** *

A JsonPath string defining the JSON data.

*/ inline S3JsonSource& WithJsonPath(Aws::String&& value) { SetJsonPath(std::move(value)); return *this;} /** *

A JsonPath string defining the JSON data.

*/ inline S3JsonSource& WithJsonPath(const char* value) { SetJsonPath(value); return *this;} /** *

A Boolean value that specifies whether a single record can span multiple * lines. This can occur when a field contains a quoted new-line character. You * must set this option to True if any record spans multiple lines. The default * value is False, which allows for more aggressive file-splitting * during parsing.

*/ inline bool GetMultiline() const{ return m_multiline; } /** *

A Boolean value that specifies whether a single record can span multiple * lines. This can occur when a field contains a quoted new-line character. You * must set this option to True if any record spans multiple lines. The default * value is False, which allows for more aggressive file-splitting * during parsing.

*/ inline bool MultilineHasBeenSet() const { return m_multilineHasBeenSet; } /** *

A Boolean value that specifies whether a single record can span multiple * lines. This can occur when a field contains a quoted new-line character. You * must set this option to True if any record spans multiple lines. The default * value is False, which allows for more aggressive file-splitting * during parsing.

*/ inline void SetMultiline(bool value) { m_multilineHasBeenSet = true; m_multiline = value; } /** *

A Boolean value that specifies whether a single record can span multiple * lines. This can occur when a field contains a quoted new-line character. You * must set this option to True if any record spans multiple lines. The default * value is False, which allows for more aggressive file-splitting * during parsing.

*/ inline S3JsonSource& WithMultiline(bool value) { SetMultiline(value); return *this;} /** *

Specifies the data schema for the S3 JSON source.

*/ inline const Aws::Vector& GetOutputSchemas() const{ return m_outputSchemas; } /** *

Specifies the data schema for the S3 JSON source.

*/ inline bool OutputSchemasHasBeenSet() const { return m_outputSchemasHasBeenSet; } /** *

Specifies the data schema for the S3 JSON source.

*/ inline void SetOutputSchemas(const Aws::Vector& value) { m_outputSchemasHasBeenSet = true; m_outputSchemas = value; } /** *

Specifies the data schema for the S3 JSON source.

*/ inline void SetOutputSchemas(Aws::Vector&& value) { m_outputSchemasHasBeenSet = true; m_outputSchemas = std::move(value); } /** *

Specifies the data schema for the S3 JSON source.

*/ inline S3JsonSource& WithOutputSchemas(const Aws::Vector& value) { SetOutputSchemas(value); return *this;} /** *

Specifies the data schema for the S3 JSON source.

*/ inline S3JsonSource& WithOutputSchemas(Aws::Vector&& value) { SetOutputSchemas(std::move(value)); return *this;} /** *

Specifies the data schema for the S3 JSON source.

*/ inline S3JsonSource& AddOutputSchemas(const GlueSchema& value) { m_outputSchemasHasBeenSet = true; m_outputSchemas.push_back(value); return *this; } /** *

Specifies the data schema for the S3 JSON source.

*/ inline S3JsonSource& AddOutputSchemas(GlueSchema&& value) { m_outputSchemasHasBeenSet = true; m_outputSchemas.push_back(std::move(value)); return *this; } private: Aws::String m_name; bool m_nameHasBeenSet = false; Aws::Vector m_paths; bool m_pathsHasBeenSet = false; CompressionType m_compressionType; bool m_compressionTypeHasBeenSet = false; Aws::Vector m_exclusions; bool m_exclusionsHasBeenSet = false; Aws::String m_groupSize; bool m_groupSizeHasBeenSet = false; Aws::String m_groupFiles; bool m_groupFilesHasBeenSet = false; bool m_recurse; bool m_recurseHasBeenSet = false; int m_maxBand; bool m_maxBandHasBeenSet = false; int m_maxFilesInBand; bool m_maxFilesInBandHasBeenSet = false; S3DirectSourceAdditionalOptions m_additionalOptions; bool m_additionalOptionsHasBeenSet = false; Aws::String m_jsonPath; bool m_jsonPathHasBeenSet = false; bool m_multiline; bool m_multilineHasBeenSet = false; Aws::Vector m_outputSchemas; bool m_outputSchemasHasBeenSet = false; }; } // namespace Model } // namespace Glue } // namespace Aws