Skip to content

Commit ea421f0

Browse files
sdiaosdiao
committed
EMR: Implemented create-cluster, add-steps, add-tags, create-default-roles, create-hbase-backup, disable-hbase-backup, schedule-hbase-backup, add-instacne-groups, ssh, socks, put, get, install-applications commands. Refactored describe-cluster. Added unit tests and examples.
EMR: Implemented add-steps. EMR: Added create-hbase-backup and schedule-hbase-backup EMR: Add --restore-from-hbase-backup option in create-cluster command. EMR: Implemented disable-hbase-backup command. EMR: Implemented create-default-roles. EMR: Added unit tests for add-steps; Fixed bugs in create-cluster; Code cleaned up. EMR: Refactored describe-cluster command to include bootstrap actions and instance-groups in the output. EMR: Added create-cluster unit tests; Minor bug fix. EMR: Add argument schema EMR: Modified help texts and examples for: add-steps, create-default-roles, describe-cluster and add-tags. Removed set-visible-to-all-users and set-termination-protection commands. EMR: Minor bug fix in adding-steps; Integrated with CLI parser. EMR create-cluster: Integrates --instance-groups, --ec2-attributes,--applications,--steps and --bootstrap-actions options. EMR: Adds help texts and examples for create-cluster command. EMR: Updated add-steps unit tests EMR: Modified install-applications and output of create-default-roles and added unit tests. EMR: Added AddInstanceGroups command. EMR: Added ssh, socks, put and get commands. EMR: Add unit tests for create-cluster
1 parent 91cdd64 commit ea421f0

42 files changed

Lines changed: 5273 additions & 206 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎awscli/customizations/emr/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
1+
# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
22
#
33
# Licensed under the Apache License, Version 2.0 (the "License"). You
44
# may not use this file except in compliance with the License. A copy of

‎awscli/customizations/emr/addinstancegroups.py‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License"). You
4+
# may not use this file except in compliance with the License. A copy of
5+
# the License is located at
6+
#
7+
# http://aws.amazon.com/apache2.0/
8+
#
9+
# or in the "license" file accompanying this file. This file is
10+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11+
# ANY KIND, either express or implied. See the License for the specific
12+
# language governing permissions and limitations under the License.
13+
14+
from awscli.customizations.commands import BasicCommand
15+
from awscli.clidriver import CLIOperationCaller
16+
17+
import argumentschema
18+
import helptext
19+
import instancegroupsutils
20+
21+
22+
class AddInstanceGroups(BasicCommand):
23+
NAME = 'add-instance-groups'
24+
DESCRIPTION = 'Adds an instance group to a running cluster.'
25+
ARG_TABLE = [
26+
{'name': 'cluster-id', 'required': True,
27+
'help_text': helptext.CLUSTER_ID},
28+
{'name': 'instance-groups', 'required': True,
29+
'help_text': helptext.INSTANCE_GROUPS,
30+
'schema': argumentschema.INSTANCE_GROUPS_SCHEMA}
31+
]
32+
33+
def _run_main(self, parsed_args, parsed_globals):
34+
emr = self._session.get_service('emr')
35+
parameters = {'JobFlowId': parsed_args.cluster_id}
36+
parameters['InstanceGroups'] = \
37+
instancegroupsutils.build_instance_groups(
38+
parsed_args.instance_groups)
39+
cli_operation_caller = CLIOperationCaller(self._session)
40+
cli_operation_caller.invoke(
41+
emr.get_operation('AddInstanceGroups'),
42+
parameters, parsed_globals)
43+
return 0

‎awscli/customizations/emr/addsteps.py‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License"). You
4+
# may not use this file except in compliance with the License. A copy of
5+
# the License is located at
6+
#
7+
# http://aws.amazon.com/apache2.0/
8+
#
9+
# or in the "license" file accompanying this file. This file is
10+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11+
# ANY KIND, either express or implied. See the License for the specific
12+
# language governing permissions and limitations under the License.
13+
14+
from awscli.customizations.commands import BasicCommand
15+
from awscli.clidriver import CLIOperationCaller
16+
17+
import steputils
18+
import argumentschema
19+
import helptext
20+
import emrutils
21+
22+
23+
class AddSteps(BasicCommand):
24+
NAME = 'add-steps'
25+
DESCRIPTION = ('Add a list of steps to a cluster. ')
26+
ARG_TABLE = [
27+
{'name': 'cluster-id', 'required': True,
28+
'help_text': helptext.CLUSTER_ID
29+
},
30+
{'name': 'steps',
31+
'required': True,
32+
'nargs': '+',
33+
'schema': argumentschema.STEPS_SCHEMA,
34+
'help_text': helptext.STEPS
35+
}
36+
]
37+
EXAMPLES = emrutils.get_example_file(NAME).read()
38+
39+
def _run_main(self, parsed_args, parsed_globals):
40+
emr = self._session.get_service('emr')
41+
parsed_steps = parsed_args.steps
42+
step_list = steputils.build_step_config_list(
43+
parsed_step_list=parsed_steps, region=parsed_globals.region)
44+
parameters = {
45+
'JobFlowId': parsed_args.cluster_id,
46+
'Steps': step_list
47+
}
48+
49+
cli_operation_caller = CLIOperationCaller(self._session)
50+
cli_operation_caller.invoke(
51+
emr.get_operation('AddJobFlowSteps'), parameters, parsed_globals)
52+
53+
return 0

‎awscli/customizations/emr/applicationutils.py‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License"). You
4+
# may not use this file except in compliance with the License. A copy of
5+
# the License is located at
6+
#
7+
# http://aws.amazon.com/apache2.0/
8+
#
9+
# or in the "license" file accompanying this file. This file is
10+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11+
# ANY KIND, either express or implied. See the License for the specific
12+
# language governing permissions and limitations under the License.
13+
14+
import constants
15+
import emrutils
16+
import exceptions
17+
18+
19+
def build_applications(parsed_applications, parsed_globals, ami_version=None):
20+
app_list = []
21+
step_list = []
22+
ba_list = []
23+
24+
for app_config in parsed_applications:
25+
app_name = app_config['Name'].lower()
26+
27+
if app_name in constants.MAPR_NAMES:
28+
app_list.append(
29+
build_supported_product(
30+
app_config['Name'], app_config['Args']))
31+
elif app_name == constants.HIVE:
32+
hive_version = app_config.get('Version')
33+
if hive_version is None:
34+
hive_version = constants.LATEST
35+
step_list.append(
36+
emrutils.build_hive_install_step(
37+
region=parsed_globals.region,
38+
version=hive_version))
39+
elif app_name == constants.PIG:
40+
pig_version = app_config.get('Version')
41+
if pig_version is None:
42+
pig_version = constants.LATEST
43+
step_list.append(
44+
emrutils.build_pig_install_step(
45+
region=parsed_globals.region,
46+
version=pig_version))
47+
elif app_name == constants.GANGLIA:
48+
ba_list.append(
49+
build_ganglia_install_bootstrap_action(
50+
region=parsed_globals.region))
51+
elif app_name == constants.HBASE:
52+
ba_list.append(
53+
build_hbase_install_bootstrap_action(
54+
region=parsed_globals.region))
55+
if ami_version >= '3.0':
56+
step_list.append(
57+
build_hbase_install_step(
58+
constants.HBASE_PATH_HADOOP2_INSTALL_JAR))
59+
elif ami_version >= '2.1':
60+
step_list.append(
61+
build_hbase_install_step(
62+
constants.HBASE_PATH_HADOOP1_INSTALL_JAR))
63+
else:
64+
raise ValueError('aws: error: AMI version ' + ami_version +
65+
'is not compatible with HBase.')
66+
elif app_name == constants.IMPALA:
67+
ba_list.append(
68+
build_impala_install_bootstrap_action(
69+
region=parsed_globals.region,
70+
args=app_config.get('Args'),
71+
version=app_config.get('Version')))
72+
else:
73+
raise exceptions.UnknownApplicationError(app_name=app_name)
74+
75+
return app_list, ba_list, step_list
76+
77+
78+
def build_supported_product(name, args):
79+
if args is None:
80+
args = []
81+
config = {'Name': name, 'Args': args}
82+
return config
83+
84+
85+
def build_ganglia_install_bootstrap_action(region):
86+
return emrutils.build_bootstrap_action(
87+
name=constants.INSTALL_GANGLIA_NAME,
88+
path=emrutils.build_s3_link(
89+
relative_path=constants.GANGLIA_INSTALL_BA_PATH,
90+
region=region))
91+
92+
93+
def build_hbase_install_bootstrap_action(region):
94+
return emrutils.build_bootstrap_action(
95+
name=constants.INSTALL_HBASE_NAME,
96+
path=emrutils.build_s3_link(
97+
relative_path=constants.HBASE_INSTALL_BA_PATH,
98+
region=region))
99+
100+
101+
def build_hbase_install_step(jar):
102+
return emrutils.build_step(
103+
jar=jar,
104+
name=constants.START_HBASE_NAME,
105+
action_on_failure=constants.TERMINATE_CLUSTER,
106+
args=constants.HBASE_INSTALL_ARG)
107+
108+
109+
def build_impala_install_bootstrap_action(region, version, args=None):
110+
if version is None:
111+
version = 'latest'
112+
args_list = [
113+
constants.BASE_PATH_ARG,
114+
emrutils.build_s3_link(region=region),
115+
constants.IMPALA_VERSION,
116+
version]
117+
if args is not None:
118+
args_list.append(constants.IMPALA_CONF)
119+
args_list += args
120+
return emrutils.build_bootstrap_action(
121+
name=constants.INSTALL_IMPALA_NAME,
122+
path=emrutils.build_s3_link(
123+
relative_path=constants.IMPALA_INSTALL_PATH,
124+
region=region),
125+
args=args_list)

0 commit comments

Comments
 (0)