/*
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Text;
#pragma warning disable 1591
namespace Amazon.ElasticMapReduce.Model
{
///
/// The action to take if your step is waiting for the instance group to start
/// and it enters the Arrested state.
///
/// Fail - Fail the step.
/// Wait - Continue waiting until the instance group is no longer arrested (requires
/// manual intervention).
/// Continue - Proceed onto the next step.
///
///
public enum OnArrested { Fail, Wait, Continue }
///
/// Action to take if there is a failure modifying your cluster composition.
/// Fail - Fail the step.
/// Continue - Proceed on to the next step.
///
public enum OnFailure { Fail, Continue }
///
/// This class provides some helper methods for creating a Resize Job Flow step
/// as part of your job flow. The resize step can be used to automatically
/// adjust the composition of your cluster while it is running. For example, if
/// you have a large workflow with different compute requirements, you can use
/// this step to automatically add a task instance group before your most compute
/// intensive step.
///
///
/// AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
/// IAmazonElasticMapReduce emr = new AmazonElasticMapReduceClient(credentials);
///
/// var resize = new ResizeJobFlowStep
/// {
/// OnArrested = OnArrested.Continue,
/// OnFailure = OnFailure.Continue
/// };
///
/// resize.AddResizeAction(new AddInstanceGroup
/// {
/// InstanceGroup = "core",
/// InstanceCount = 10
/// });
///
/// resize.AddResizeAction(new AddInstanceGroup
/// {
/// InstanceGroup = "task",
/// InstanceCount = 10,
/// WithInstanceType = "m1.small"
/// });
///
/// HadoopJarStepConfig config = resize.ToHadoopJarStepConfig();
///
/// StepConfig resizeJobFlow = new StepConfig {
/// Name = "Resize job flow",
/// ActionOnFailure = "TERMINATE_JOB_FLOW",
/// HadoopJarStep = config,
/// };
///
/// RunJobFlowRequest request = new RunJobFlowRequest {
/// Name = "Resize job flow",
/// Steps = new List<StepConfig> { resizeJobFlow },
/// LogUri = "s3://log-bucket/",
/// Instances = new JobFlowInstancesConfig {
/// Ec2KeyName = "keypair",
/// HadoopVersion = "0.20",
/// InstanceCount = 5,
/// KeepJobFlowAliveWhenNoSteps = true,
/// MasterInstanceType = "m1.small",
/// SlaveInstanceType = "m1.small"
/// }
/// };
///
/// RunJobFlowResponse response = emr.RunJobFlow(request);
///
///
public class ResizeJobFlowStep
{
string bucket;
List args = new List();
bool wait = true;
OnArrested? onArrested = null;
OnFailure? onFailure = null;
///
/// Creates a new ResizeJobFlowStep using the default Elastic Map Reduce
/// bucket (us-east-1.elasticmapreduce) for the default (us-east-1) region.
///
public ResizeJobFlowStep()
: this("us-east-1.elasticmapreduce")
{
}
///
/// Creates a new ResizeJobFlowStep using the specified Amazon S3 bucket to
/// load resources.
///
/// The official bucket format is "<region>.elasticmapreduce", so if
/// you're using the us-east-1 region, you should use the bucket
/// "us-east-1.elasticmapreduce".
///
///
/// The Amazon S3 bucket from which to load resources.
public ResizeJobFlowStep(string bucket)
{
this.bucket = bucket;
}
///
/// Add a new action for this step to perform. These actions can be to modify
/// or add instance groups. This step supports multiple actions, but requires
/// at least one be specified.
///
public void AddResizeAction(ResizeAction resizeAction)
{
this.args.AddRange(resizeAction.Args);
}
///
/// Whether the step should wait for the modification to complete
/// or if it should just continue onto the next step once the modification
/// request is received. Defaults to true.
///
public bool Wait
{
get { return this.wait; }
set { this.wait = value; }
}
///
/// What action this step should take if any of the instance group modifications result
/// in the instance group entering Arrested state. This can happen when the bootstrap
/// actions on the newly launched instances are continuously failing.
///
public OnArrested? OnArrested
{
get { return this.onArrested; }
set { this.onArrested = value; }
}
///
/// What action this step should take if the modification fails. This can happen when
/// you request to perform an invalid action, such as shrink a core instance group.
///
public OnFailure? OnFailure
{
get { return this.OnFailure; }
set { this.onFailure = value; }
}
///
/// Creates the final HadoopJarStepConfig once you are done configuring the step. You can use
/// this as you would any other HadoopJarStepConfig.
///
/// HadoopJarStepConfig configured to perform the specified actions.
public HadoopJarStepConfig ToHadoopJarStepConfig()
{
if (args.Count == 0)
{
throw new AmazonElasticMapReduceException("Cannot create a ResizeJobFlowStep with no resize actions.");
}
if (wait == false)
{
args.Add("--no-wait");
}
if (onArrested != null)
{
args.Add("--on-arrested");
args.Add(onArrested.ToString());
}
if (onFailure != null)
{
args.Add("--on-failure");
args.Add(onFailure.ToString());
}
return new HadoopJarStepConfig
{
Jar = string.Format(CultureInfo.InvariantCulture, "s3://{0}/libs/resize-job-flow/0.1/resize-job-flow.jar", bucket),
Args = args
};
}
}
public interface ResizeAction
{
List Args { get; }
}
/**
* Class representing a change to an existing instance group.
*/
public class ModifyInstanceGroup : ResizeAction
{
string instanceGroup;
int? instanceCount;
///
/// The identification for the instance group to modify. You can specify either
/// core or task if there is only one instance group of that role. Otherwise you can
/// specify the instance group ID (such as ig-ABABABABABAB).
///
public string InstanceGroup
{
get { return this.instanceGroup; }
set { this.instanceGroup = value; }
}
///
/// Tthe new size of the instance group.
///
public int? InstanceCount
{
get { return this.instanceCount; }
set { this.instanceCount = value; }
}
public List Args
{
get
{
if (instanceGroup == null)
{
throw new AmazonElasticMapReduceException("InstanceGroup must not be null.");
}
if (instanceCount == null)
{
throw new AmazonElasticMapReduceException("InstanceCount must not be null.");
}
List args = new List();
args.Add("--modify-instance-group");
args.Add(instanceGroup);
args.Add("--instance-count");
args.Add(instanceCount.ToString());
return args;
}
}
}
///
/// Class representing creating a new instance group.
///
public class AddInstanceGroup : ResizeAction
{
string instanceGroup;
int? instanceCount;
string instanceType;
///
/// The role of the new instance group (core, task).
///
public string InstanceGroup
{
get { return this.instanceGroup; }
set { this.instanceGroup = value; }
}
///
/// The count of the new instance group.
///
public int? InstanceCount
{
get { return this.instanceCount; }
set { this.instanceCount = value; }
}
///
/// The instance type to use for this instance group.
///
public string WithInstanceType
{
get { return this.instanceType; }
set { this.instanceType = value; }
}
public List Args
{
get
{
if (instanceGroup == null)
{
throw new AmazonElasticMapReduceException("InstanceGroup must not be null.");
}
if (instanceCount == null)
{
throw new AmazonElasticMapReduceException("InstanceCount must not be null.");
}
if (instanceType == null)
{
throw new AmazonElasticMapReduceException("InstanceType must not be null.");
}
List args = new List();
args.Add("--add-instance-group");
args.Add(instanceGroup);
args.Add("--instance-count");
args.Add(instanceCount.ToString());
args.Add("--instance-type");
args.Add(instanceType);
return args;
}
}
}
}