Skip to content

Commit fb84aaf

Browse files
sdiaosdiao
committed
EMR: Added default configuration for create-cluster. Fixed create-cluster.rst format. Added shortcut options --instance-type and --instance-count to --instance-groups.
1 parent a4a5985 commit fb84aaf

8 files changed

Lines changed: 310 additions & 107 deletions

File tree

‎awscli/customizations/emr/createcluster.py‎

Lines changed: 23 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
from awscli.customizations.commands import BasicCommand
1616
from awscli.customizations.emr import constants
17+
from awscli.customizations.emr import defaultconfig
1718
from awscli.customizations.emr import emrutils
1819
from awscli.customizations.emr import steputils
1920
from awscli.customizations.emr import hbaseutils
@@ -32,18 +33,19 @@ class CreateCluster(BasicCommand):
3233
DESCRIPTION = (
3334
'Creates and starts running an EMR cluster.\n'
3435
'\nQuick start:\n'
35-
'\naws emr create-cluster --ami-version 3.1.0 '
36-
' --instance-groups InstanceGroupType=MASTER,InstanceCount=1,'
37-
'InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,'
38-
'InstanceType=m3.xlarge --auto-terminate\n')
36+
'\naws emr create-cluster --ami-version <ami-version> '
37+
'--instance-type <instance-type> --instance-count <instance-count>\n')
3938
ARG_TABLE = [
4039
{'name': 'ami-version',
41-
'required': True,
42-
'help_text': helptext.AMI_VERSION},
40+
'help_text': helptext.AMI_VERSION,
41+
'required': True},
4342
{'name': 'instance-groups',
44-
'required': True,
4543
'schema': argumentschema.INSTANCE_GROUPS_SCHEMA,
4644
'help_text': helptext.INSTANCE_GROUPS},
45+
{'name': 'instance-type',
46+
'help_text': helptext.INSTANCE_TYPE},
47+
{'name': 'instance-count',
48+
'help_text': helptext.INSTANCE_COUNT},
4749
{'name': 'auto-terminate', 'action': 'store_true',
4850
'group_name': 'auto_terminate',
4951
'help_text': helptext.AUTO_TERMINATE},
@@ -83,7 +85,8 @@ class CreateCluster(BasicCommand):
8385
'schema': argumentschema.BOOTSTRAP_ACTIONS_SCHEMA},
8486
{'name': 'applications',
8587
'help_text': helptext.APPLICATIONS,
86-
'schema': argumentschema.APPLICATIONS_SCHEMA},
88+
'schema': argumentschema.APPLICATIONS_SCHEMA,
89+
'default': defaultconfig.APPLICATIONS},
8790
{'name': 'steps',
8891
'schema': argumentschema.STEPS_SCHEMA,
8992
'help_text': helptext.STEPS},
@@ -102,10 +105,17 @@ def _run_main(self, parsed_args, parsed_globals):
102105
bootstrap_actions = []
103106
params['Name'] = parsed_args.name
104107

105-
is_valid_ami = re.match('\d?\..*', parsed_args.ami_version)
106-
if is_valid_ami is None:
107-
raise exceptions.\
108-
InvalidAmiVersionError(ami_version=parsed_args.ami_version)
108+
instances_config = {}
109+
instances_config['InstanceGroups'] = \
110+
instancegroupsutils.validate_and_build_instance_groups(
111+
instance_groups=parsed_args.instance_groups,
112+
instance_type=parsed_args.instance_type,
113+
instance_count=parsed_args.instance_count)
114+
115+
is_valid_ami_version = re.match('\d?\..*', parsed_args.ami_version)
116+
if is_valid_ami_version is None:
117+
raise exceptions.InvalidAmiVersionError(
118+
ami_version=parsed_args.ami_version)
109119
params['AmiVersion'] = parsed_args.ami_version
110120
emrutils.apply_dict(
111121
params, 'AdditionalInfo', parsed_args.additional_info)
@@ -117,18 +127,11 @@ def _run_main(self, parsed_args, parsed_globals):
117127
parsed_args.ec2_attributes['InstanceProfile'] = EC2_ROLE_NAME
118128

119129
emrutils.apply_dict(params, 'ServiceRole', parsed_args.service_role)
120-
instances_config = {}
121-
instances_config['InstanceGroups'] = \
122-
instancegroupsutils.build_instance_groups(
123-
parsed_args.instance_groups)
124130

125131
if (
126132
parsed_args.no_auto_terminate is False and
127133
parsed_args.auto_terminate is False):
128-
raise exceptions.\
129-
MissingBooleanOptionsError(
130-
true_option='--auto-terminate',
131-
false_option='--no-auto-terminate')
134+
parsed_args.no_auto_terminate = True
132135

133136
instances_config['KeepJobFlowAliveWhenNoSteps'] = \
134137
emrutils.apply_boolean_options(

‎awscli/customizations/emr/defaultconfig.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License"). You
4+
# may not use this file except in compliance with the License. A copy of
5+
# the License is located at
6+
#
7+
# http://aws.amazon.com/apache2.0/
8+
#
9+
# or in the "license" file accompanying this file. This file is
10+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11+
# ANY KIND, either express or implied. See the License for the specific
12+
# language governing permissions and limitations under the License.
13+
14+
# Declare all the constants used by EMR in this file.
15+
16+
17+
# create-cluster default config
18+
19+
INSTANCE_GROUPS = \
20+
('[{"InstanceGroupType": "MASTER","InstanceCount": 1, '
21+
'"Name": "Master Instance Group","InstanceType": "m3.xlarge"},'
22+
'{"InstanceGroupType": "CORE", "InstanceCount": 2,'
23+
'"Name": "Core Instance Group", "InstanceType": "m3.xlarge"}]')
24+
25+
APPLICATIONS = '[{"Name": "Hive"}, {"Name": "Pig"}]'
26+
27+
RELEASE_LABEL = '3.1.0'

‎awscli/customizations/emr/exceptions.py‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,12 +43,34 @@ class MissingParametersError(EmrError):
4343
'{object_name}: {missing}.')
4444

4545

46+
class MissingRequiredInstanceGroupsError(EmrError):
47+
"""
48+
In create-cluster command, none of --instance-group,
49+
--instance-count nor --instance-type were not supplied.
50+
"""
51+
fmt = ('aws: error: Must specify either --instance-groups or '
52+
'--instance-type with --instance-count(optional) to '
53+
'configure instance groups.')
54+
55+
56+
class InstanceGroupsValidationError(EmrError):
57+
"""
58+
--instance-type and --instance-count are shortcut option
59+
for --instance-groups and they cannot be specified
60+
together with --instance-groups
61+
"""
62+
fmt = ('aws: error: You may not specify --instance-type '
63+
'or --instance-count with --instance-groups, '
64+
'because --instance-type and --instance-count are '
65+
'shortcut options for --instance-groups.')
66+
67+
4668
class InvalidAmiVersionError(EmrError):
4769
"""
4870
The supplied ami-version is invalid.
49-
:ivar ami_version: The provided ami-version.
71+
:ivar ami_version: The provided ami_version.
5072
"""
51-
fmt = ('aws: error: The supplied AMI version {ami_version} is invalid.'
73+
fmt = ('aws: error: The supplied AMI version "{ami_version}" is invalid.'
5274
' Please see AMI Versions Supported in Amazon EMR in '
5375
'Amazon Elastic MapReduce Developer Guide: '
5476
'http://docs.aws.amazon.com/ElasticMapReduce/'

‎awscli/customizations/emr/helptext.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,19 @@
8080
'<code>[Name], InstanceGroupType, InstanceType, InstanceCount,'
8181
' [BidPrice]</code></p>')
8282

83+
INSTANCE_TYPE = (
84+
'<p>Shortcut option for --instance-groups. A specification of the '
85+
'type of Amazon EC2 instances used together with --instance-count '
86+
'(optional) to create instance groups in a cluster. '
87+
'If --instance-count is not provided, a single node cluster '
88+
'will be created by default. </p>')
89+
90+
INSTANCE_COUNT = (
91+
'<p>Shortcut option for --instance-groups. '
92+
'A specification of the number of Amazon EC2 instances used '
93+
'together with --instance-type to create instance groups in '
94+
'a cluster. </p>')
95+
8396
ADDITIONAL_INFO = (
8497
'<p>Specifies additional information during cluster creation</p>')
8598

‎awscli/customizations/emr/instancegroupsutils.py‎

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
# language governing permissions and limitations under the License.
1313

1414
from awscli.customizations.emr import constants
15+
from awscli.customizations.emr import exceptions
1516

1617

1718
def build_instance_groups(parsed_instance_groups):
@@ -39,4 +40,44 @@ def build_instance_groups(parsed_instance_groups):
3940
ig_config['Market'] = constants.ON_DEMAND
4041
instance_groups.append(ig_config)
4142

42-
return instance_groups
43+
return instance_groups
44+
45+
46+
def _build_instance_group(
47+
instance_type, instance_count, instance_group_type):
48+
ig_config = {}
49+
ig_config['InstanceType'] = instance_type
50+
ig_config['InstanceCount'] = instance_count
51+
ig_config['InstanceRole'] = instance_group_type.upper()
52+
ig_config['Name'] = ig_config['InstanceRole']
53+
ig_config['Market'] = constants.ON_DEMAND
54+
return ig_config
55+
56+
57+
def validate_and_build_instance_groups(
58+
instance_groups, instance_type, instance_count):
59+
if (instance_groups is None and instance_type is None):
60+
raise exceptions.MissingRequiredInstanceGroupsError
61+
62+
if (instance_groups is not None and
63+
(instance_type is not None or
64+
instance_count is not None)):
65+
raise exceptions.InstanceGroupsValidationError
66+
67+
if instance_groups is not None:
68+
return build_instance_groups(instance_groups)
69+
else:
70+
instance_groups = []
71+
master_ig = _build_instance_group(
72+
instance_type=instance_type,
73+
instance_count=1,
74+
instance_group_type="MASTER")
75+
instance_groups.append(master_ig)
76+
if instance_count is not None and int(instance_count) > 1:
77+
core_ig = _build_instance_group(
78+
instance_type=instance_type,
79+
instance_count=int(instance_count)-1,
80+
instance_group_type="CORE")
81+
instance_groups.append(core_ig)
82+
83+
return instance_groups

0 commit comments

Comments
 (0)