Skip to content

Commit d64bced

Browse files
kyleknapkyleknap
committed
Merge branch 'emr' into release-1.4.0
2 parents 499d3af + 780deb1 commit d64bced

26 files changed

Lines changed: 777 additions & 385 deletions

‎awscli/customizations/emr/applicationutils.py‎

Lines changed: 37 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,9 @@ def build_applications(parsed_applications, parsed_globals, ami_version=None):
2929
build_supported_product(
3030
app_config['Name'], app_config.get('Args')))
3131
elif app_name == constants.HIVE:
32-
hive_version = app_config.get('Version')
33-
if hive_version is None:
34-
hive_version = constants.LATEST
32+
hive_version = constants.LATEST
3533
step_list.append(
36-
_build_install_hive_step(
37-
region=parsed_globals.region,
38-
version=hive_version))
34+
_build_install_hive_step(region=parsed_globals.region))
3935
args = app_config.get('Args')
4036
if args is not None:
4137
hive_site_path = _find_matching_arg(
@@ -44,41 +40,36 @@ def build_applications(parsed_applications, parsed_globals, ami_version=None):
4440
step_list.append(
4541
_build_install_hive_site_step(
4642
region=parsed_globals.region,
47-
version=hive_version,
4843
hive_site_path=hive_site_path))
4944
elif app_name == constants.PIG:
50-
pig_version = app_config.get('Version')
51-
if pig_version is None:
52-
pig_version = constants.LATEST
45+
pig_version = constants.LATEST
5346
step_list.append(
54-
emrutils.build_pig_install_step(
55-
region=parsed_globals.region,
56-
version=pig_version))
47+
_build_pig_install_step(
48+
region=parsed_globals.region))
5749
elif app_name == constants.GANGLIA:
5850
ba_list.append(
59-
build_ganglia_install_bootstrap_action(
51+
_build_ganglia_install_bootstrap_action(
6052
region=parsed_globals.region))
6153
elif app_name == constants.HBASE:
6254
ba_list.append(
63-
build_hbase_install_bootstrap_action(
55+
_build_hbase_install_bootstrap_action(
6456
region=parsed_globals.region))
6557
if ami_version >= '3.0':
6658
step_list.append(
67-
build_hbase_install_step(
59+
_build_hbase_install_step(
6860
constants.HBASE_PATH_HADOOP2_INSTALL_JAR))
6961
elif ami_version >= '2.1':
7062
step_list.append(
71-
build_hbase_install_step(
63+
_build_hbase_install_step(
7264
constants.HBASE_PATH_HADOOP1_INSTALL_JAR))
7365
else:
7466
raise ValueError('aws: error: AMI version ' + ami_version +
7567
'is not compatible with HBase.')
7668
elif app_name == constants.IMPALA:
7769
ba_list.append(
78-
build_impala_install_bootstrap_action(
70+
_build_impala_install_bootstrap_action(
7971
region=parsed_globals.region,
80-
args=app_config.get('Args'),
81-
version=app_config.get('Version')))
72+
args=app_config.get('Args')))
8273
else:
8374
raise exceptions.UnknownApplicationError(app_name=app_name)
8475

@@ -92,38 +83,36 @@ def build_supported_product(name, args):
9283
return config
9384

9485

95-
def build_ganglia_install_bootstrap_action(region):
86+
def _build_ganglia_install_bootstrap_action(region):
9687
return emrutils.build_bootstrap_action(
9788
name=constants.INSTALL_GANGLIA_NAME,
9889
path=emrutils.build_s3_link(
9990
relative_path=constants.GANGLIA_INSTALL_BA_PATH,
10091
region=region))
10192

10293

103-
def build_hbase_install_bootstrap_action(region):
94+
def _build_hbase_install_bootstrap_action(region):
10495
return emrutils.build_bootstrap_action(
10596
name=constants.INSTALL_HBASE_NAME,
10697
path=emrutils.build_s3_link(
10798
relative_path=constants.HBASE_INSTALL_BA_PATH,
10899
region=region))
109100

110101

111-
def build_hbase_install_step(jar):
102+
def _build_hbase_install_step(jar):
112103
return emrutils.build_step(
113104
jar=jar,
114105
name=constants.START_HBASE_NAME,
115106
action_on_failure=constants.TERMINATE_CLUSTER,
116107
args=constants.HBASE_INSTALL_ARG)
117108

118109

119-
def build_impala_install_bootstrap_action(region, version, args=None):
120-
if version is None:
121-
version = 'latest'
110+
def _build_impala_install_bootstrap_action(region, args=None):
122111
args_list = [
123112
constants.BASE_PATH_ARG,
124113
emrutils.build_s3_link(region=region),
125114
constants.IMPALA_VERSION,
126-
version]
115+
constants.LATEST]
127116
if args is not None:
128117
args_list.append(constants.IMPALA_CONF)
129118
args_list += args
@@ -135,15 +124,15 @@ def build_impala_install_bootstrap_action(region, version, args=None):
135124
args=args_list)
136125

137126

138-
def _build_install_hive_step(region, version,
127+
def _build_install_hive_step(region,
139128
action_on_failure=constants.TERMINATE_CLUSTER):
140129
step_args = [
141130
emrutils.build_s3_link(constants.HIVE_SCRIPT_PATH, region),
142131
constants.INSTALL_HIVE_ARG,
143132
constants.BASE_PATH_ARG,
144133
emrutils.build_s3_link(constants.HIVE_BASE_PATH),
145134
constants.HIVE_VERSIONS,
146-
version]
135+
constants.LATEST]
147136
step = emrutils.build_step(
148137
name=constants.INSTALL_HIVE_NAME,
149138
action_on_failure=action_on_failure,
@@ -152,7 +141,7 @@ def _build_install_hive_step(region, version,
152141
return step
153142

154143

155-
def _build_install_hive_site_step(region, version, hive_site_path,
144+
def _build_install_hive_site_step(region, hive_site_path,
156145
action_on_failure=constants.CANCEL_AND_WAIT):
157146
step_args = [
158147
emrutils.build_s3_link(constants.HIVE_SCRIPT_PATH, region),
@@ -161,7 +150,7 @@ def _build_install_hive_site_step(region, version, hive_site_path,
161150
constants.INSTALL_HIVE_SITE_ARG,
162151
hive_site_path,
163152
constants.HIVE_VERSIONS,
164-
version]
153+
constants.LATEST]
165154
step = emrutils.build_step(
166155
name=constants.INSTALL_HIVE_SITE_NAME,
167156
action_on_failure=action_on_failure,
@@ -170,6 +159,23 @@ def _build_install_hive_site_step(region, version, hive_site_path,
170159
return step
171160

172161

162+
def _build_pig_install_step(region,
163+
action_on_failure=constants.TERMINATE_CLUSTER):
164+
step_args = [
165+
emrutils.build_s3_link(constants.PIG_SCRIPT_PATH, region),
166+
constants.INSTALL_PIG_ARG,
167+
constants.BASE_PATH_ARG,
168+
emrutils.build_s3_link(constants.PIG_BASE_PATH, region),
169+
constants.PIG_VERSIONS,
170+
constants.LATEST]
171+
step = emrutils.build_step(
172+
name=constants.INSTALL_PIG_NAME,
173+
action_on_failure=action_on_failure,
174+
jar=emrutils.build_s3_link(constants.SCRIPT_RUNNER_PATH, region),
175+
args=step_args)
176+
return step
177+
178+
173179
def _find_matching_arg(key, args_list):
174180
for arg in args_list:
175181
if key in arg:

‎awscli/customizations/emr/argumentschema.py‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -104,11 +104,6 @@
104104
"enum": ["MapR", "HIVE", "PIG", "HBASE", "IMPALA", "GANGLIA"],
105105
"required": True
106106
},
107-
"Version": {
108-
"type": "string",
109-
"description":
110-
"Version number of the application to be installed."
111-
},
112107
"Args": {
113108
"type": "array",
114109
"description":
@@ -168,11 +163,6 @@
168163
"type": "string",
169164
"description": "The name of the step. ",
170165
},
171-
"Version": {
172-
"type": "string",
173-
"default": "latest",
174-
"description": "The application version. "
175-
},
176166
"ActionOnFailure": {
177167
"type": "string",
178168
"description": "The action to take if the cluster step fails.",

‎awscli/customizations/emr/constants.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,3 +135,9 @@
135135
SSH_USER = 'hadoop'
136136
STARTING_STATES = ['STARTING', 'BOOTSTRAPPING']
137137
TERMINATED_STATES = ['TERMINATED', 'TERMINATING', 'TERMINATED_WITH_ERRORS']
138+
139+
# list-clusters
140+
LIST_CLUSTERS_ACTIVE_STATES = ['STARTING', 'BOOTSTRAPPING', 'RUNNING',
141+
'WAITING', 'TERMINATING']
142+
LIST_CLUSTERS_TERMINATED_STATES = ['TERMINATED']
143+
LIST_CLUSTERS_FAILED_STATES = ['TERMINATED_WITH_ERRORS']

‎awscli/customizations/emr/createcluster.py‎

Lines changed: 28 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
from awscli.customizations.commands import BasicCommand
1616
from awscli.customizations.emr import constants
17+
from awscli.customizations.emr import defaultconfig
1718
from awscli.customizations.emr import emrutils
1819
from awscli.customizations.emr import steputils
1920
from awscli.customizations.emr import hbaseutils
@@ -32,18 +33,19 @@ class CreateCluster(BasicCommand):
3233
DESCRIPTION = (
3334
'Creates and starts running an EMR cluster.\n'
3435
'\nQuick start:\n'
35-
'\naws emr create-cluster --ami-version 3.1.0 '
36-
' --instance-groups InstanceGroupType=MASTER,InstanceCount=1,'
37-
'InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,'
38-
'InstanceType=m3.xlarge --auto-terminate\n')
36+
'\naws emr create-cluster --ami-version <ami-version> '
37+
'--instance-type <instance-type> [--instance-count <instance-count>]\n')
3938
ARG_TABLE = [
4039
{'name': 'ami-version',
41-
'required': True,
42-
'help_text': helptext.AMI_VERSION},
40+
'help_text': helptext.AMI_VERSION,
41+
'required': True},
4342
{'name': 'instance-groups',
44-
'required': True,
4543
'schema': argumentschema.INSTANCE_GROUPS_SCHEMA,
4644
'help_text': helptext.INSTANCE_GROUPS},
45+
{'name': 'instance-type',
46+
'help_text': helptext.INSTANCE_TYPE},
47+
{'name': 'instance-count',
48+
'help_text': helptext.INSTANCE_COUNT},
4749
{'name': 'auto-terminate', 'action': 'store_true',
4850
'group_name': 'auto_terminate',
4951
'help_text': helptext.AUTO_TERMINATE},
@@ -83,7 +85,8 @@ class CreateCluster(BasicCommand):
8385
'schema': argumentschema.BOOTSTRAP_ACTIONS_SCHEMA},
8486
{'name': 'applications',
8587
'help_text': helptext.APPLICATIONS,
86-
'schema': argumentschema.APPLICATIONS_SCHEMA},
88+
'schema': argumentschema.APPLICATIONS_SCHEMA,
89+
'default': defaultconfig.APPLICATIONS},
8790
{'name': 'steps',
8891
'schema': argumentschema.STEPS_SCHEMA,
8992
'help_text': helptext.STEPS},
@@ -102,10 +105,17 @@ def _run_main(self, parsed_args, parsed_globals):
102105
bootstrap_actions = []
103106
params['Name'] = parsed_args.name
104107

105-
is_valid_ami = re.match('\d?\..*', parsed_args.ami_version)
106-
if is_valid_ami is None:
107-
raise exceptions.\
108-
InvalidAmiVersionError(ami_version=parsed_args.ami_version)
108+
instances_config = {}
109+
instances_config['InstanceGroups'] = \
110+
instancegroupsutils.validate_and_build_instance_groups(
111+
instance_groups=parsed_args.instance_groups,
112+
instance_type=parsed_args.instance_type,
113+
instance_count=parsed_args.instance_count)
114+
115+
is_valid_ami_version = re.match('\d?\..*', parsed_args.ami_version)
116+
if is_valid_ami_version is None:
117+
raise exceptions.InvalidAmiVersionError(
118+
ami_version=parsed_args.ami_version)
109119
params['AmiVersion'] = parsed_args.ami_version
110120
emrutils.apply_dict(
111121
params, 'AdditionalInfo', parsed_args.additional_info)
@@ -117,18 +127,11 @@ def _run_main(self, parsed_args, parsed_globals):
117127
parsed_args.ec2_attributes['InstanceProfile'] = EC2_ROLE_NAME
118128

119129
emrutils.apply_dict(params, 'ServiceRole', parsed_args.service_role)
120-
instances_config = {}
121-
instances_config['InstanceGroups'] = \
122-
instancegroupsutils.build_instance_groups(
123-
parsed_args.instance_groups)
124130

125131
if (
126132
parsed_args.no_auto_terminate is False and
127133
parsed_args.auto_terminate is False):
128-
raise exceptions.\
129-
MissingBooleanOptionsError(
130-
true_option='--auto-terminate',
131-
false_option='--no-auto-terminate')
134+
parsed_args.no_auto_terminate = True
132135

133136
instances_config['KeepJobFlowAliveWhenNoSteps'] = \
134137
emrutils.apply_boolean_options(
@@ -144,6 +147,11 @@ def _run_main(self, parsed_args, parsed_globals):
144147
parsed_args.no_termination_protected,
145148
'--no-termination-protected')
146149

150+
if (
151+
parsed_args.visible_to_all_users is False and
152+
parsed_args.no_visible_to_all_users is False):
153+
parsed_args.visible_to_all_users = True
154+
147155
params['VisibleToAllUsers'] = \
148156
emrutils.apply_boolean_options(
149157
parsed_args.visible_to_all_users,

‎awscli/customizations/emr/defaultconfig.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License"). You
4+
# may not use this file except in compliance with the License. A copy of
5+
# the License is located at
6+
#
7+
# http://aws.amazon.com/apache2.0/
8+
#
9+
# or in the "license" file accompanying this file. This file is
10+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11+
# ANY KIND, either express or implied. See the License for the specific
12+
# language governing permissions and limitations under the License.
13+
14+
# Declare all the constants used by EMR in this file.
15+
16+
17+
# create-cluster default config
18+
19+
INSTANCE_GROUPS = \
20+
('[{"InstanceGroupType": "MASTER","InstanceCount": 1, '
21+
'"Name": "Master Instance Group","InstanceType": "m3.xlarge"},'
22+
'{"InstanceGroupType": "CORE", "InstanceCount": 2,'
23+
'"Name": "Core Instance Group", "InstanceType": "m3.xlarge"}]')
24+
25+
APPLICATIONS = '[{"Name": "Hive"}, {"Name": "Pig"}]'
26+
27+
RELEASE_LABEL = '3.1.0'

0 commit comments

Comments
 (0)