Skip to content

Commit bd4bbbe

Browse files
sdiaojamesls
sdiao
authored and
jamesls
committed
EMR: changed to use regionalized buckets.
1 parent 4cf42e9 commit bd4bbbe

4 files changed

Lines changed: 77 additions & 44 deletions

File tree

‎awscli/customizations/emr/emrutils.py‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -129,14 +129,15 @@ def build_bootstrap_action(
129129
return ba_config
130130

131131

132-
def build_s3_link(relative_path='', region=None):
133-
if region and region != 'us-east-1':
134-
return 's3://{0}.elasticmapreduce{1}'.format(region, relative_path)
135-
else:
136-
return 's3://elasticmapreduce{0}'.format(relative_path)
132+
def build_s3_link(relative_path='', region='us-east-1'):
133+
if region is None:
134+
region = 'us-east-1'
135+
return 's3://{0}.elasticmapreduce{1}'.format(region, relative_path)
137136

138137

139-
def get_script_runner(region=None):
138+
def get_script_runner(region='us-east-1'):
139+
if region is None:
140+
region = 'us-east-1'
140141
return build_s3_link(
141142
relative_path=constants.SCRIPT_RUNNER_PATH, region=region)
142143

‎tests/unit/customizations/emr/test_add_steps.py‎

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,11 @@ class TestAddSteps(BaseAWSCommandParamsTest):
4545
's3://elasticmapreduce/samples/hive-ads/libs/model-build.q'
4646

4747
HIVE_DEFAULT_HADOOP_JAR_STEP = \
48-
{'Jar': 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
48+
{'Jar':
49+
('s3://us-east-1.elasticmapreduce/'
50+
'libs/script-runner/script-runner.jar'),
4951
'Args':
50-
['s3://elasticmapreduce/libs/hive/hive-script',
52+
['s3://us-east-1.elasticmapreduce/libs/hive/hive-script',
5153
'--run-hive-script',
5254
'--hive-versions',
5355
'latest',
@@ -58,9 +60,11 @@ class TestAddSteps(BaseAWSCommandParamsTest):
5860
}
5961

6062
HIVE_BASIC_HADOOP_JAR_STEP = \
61-
{'Jar': 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
63+
{'Jar':
64+
('s3://us-east-1.elasticmapreduce/libs/'
65+
'script-runner/script-runner.jar'),
6266
'Args':
63-
['s3://elasticmapreduce/libs/hive/hive-script',
67+
['s3://us-east-1.elasticmapreduce/libs/hive/hive-script',
6468
'--run-hive-script',
6569
'--hive-versions',
6670
'0.11.0.1',
@@ -74,9 +78,11 @@ class TestAddSteps(BaseAWSCommandParamsTest):
7478
's3://elasticmapreduce/samples/pig-apache/do-reports2.pig'
7579

7680
PIG_DEFAULT_HADOOP_JAR_STEP = \
77-
{'Jar': 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
81+
{'Jar':
82+
('s3://us-east-1.elasticmapreduce/libs/'
83+
'script-runner/script-runner.jar'),
7884
'Args':
79-
['s3://elasticmapreduce/libs/pig/pig-script',
85+
['s3://us-east-1.elasticmapreduce/libs/pig/pig-script',
8086
'--run-pig-script',
8187
'--pig-versions',
8288
'latest',
@@ -87,9 +93,11 @@ class TestAddSteps(BaseAWSCommandParamsTest):
8793
]}
8894

8995
PIG_BASIC_HADOOP_JAR_STEP = \
90-
{'Jar': 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
96+
{'Jar':
97+
('s3://us-east-1.elasticmapreduce/libs/'
98+
'script-runner/script-runner.jar'),
9199
'Args':
92-
['s3://elasticmapreduce/libs/pig/pig-script',
100+
['s3://us-east-1.elasticmapreduce/libs/pig/pig-script',
93101
'--run-pig-script',
94102
'--pig-versions',
95103
'0.11.1.0',
@@ -104,9 +112,11 @@ class TestAddSteps(BaseAWSCommandParamsTest):
104112
'--console-output-path,s3://myimpala/output'
105113

106114
IMPALA_BASIC_HADOOP_JAR_STEP = \
107-
{'Jar': 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
115+
{'Jar':
116+
('s3://us-east-1.elasticmapreduce/libs/'
117+
'script-runner/script-runner.jar'),
108118
'Args':
109-
['s3://elasticmapreduce/libs/impala/setup-impala',
119+
['s3://us-east-1.elasticmapreduce/libs/impala/setup-impala',
110120
'--run-impala-script',
111121
'--impala-script',
112122
's3://myimpala/input',

‎tests/unit/customizations/emr/test_create_cluster.py‎

Lines changed: 40 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -91,11 +91,13 @@
9191

9292
INSTALL_HIVE_STEP = {
9393
'HadoopJarStep': {
94-
'Args': ['s3://elasticmapreduce/libs/hive/hive-script',
94+
'Args': ['s3://us-east-1.elasticmapreduce/libs/hive/hive-script',
9595
'--install-hive', '--base-path',
96-
's3://elasticmapreduce/libs/hive',
96+
's3://us-east-1.elasticmapreduce/libs/hive',
9797
'--hive-versions', 'latest'],
98-
'Jar': 's3://elasticmapreduce/libs/script-runner/script-runner.jar'
98+
'Jar':
99+
('s3://us-east-1.elasticmapreduce/libs/'
100+
'script-runner/script-runner.jar')
99101
},
100102
'Name': 'Install Hive',
101103
'ActionOnFailure': 'TERMINATE_CLUSTER'
@@ -119,11 +121,13 @@
119121

120122
INSTALL_PIG_STEP = {
121123
'HadoopJarStep': {
122-
'Args': ['s3://elasticmapreduce/libs/pig/pig-script',
124+
'Args': ['s3://us-east-1.elasticmapreduce/libs/pig/pig-script',
123125
'--install-pig', '--base-path',
124-
's3://elasticmapreduce/libs/pig',
126+
's3://us-east-1.elasticmapreduce/libs/pig',
125127
'--pig-versions', 'latest'],
126-
'Jar': 's3://elasticmapreduce/libs/script-runner/script-runner.jar'
128+
'Jar':
129+
('s3://us-east-1.elasticmapreduce/libs/'
130+
'script-runner/script-runner.jar')
127131
},
128132
'Name': 'Install Pig',
129133
'ActionOnFailure': 'TERMINATE_CLUSTER'
@@ -141,22 +145,24 @@
141145

142146
INSTALL_GANGLIA_BA = {
143147
'ScriptBootstrapAction': {
144-
'Path': 's3://elasticmapreduce/bootstrap-actions/install-ganglia'
148+
'Path':
149+
('s3://us-east-1.elasticmapreduce/'
150+
'bootstrap-actions/install-ganglia')
145151
},
146152
'Name': 'Install Ganglia'
147153
}
148154

149155
INSTALL_HBASE_BA = {
150156
'ScriptBootstrapAction': {
151-
'Path': 's3://elasticmapreduce/bootstrap-actions/setup-hbase'
157+
'Path': 's3://us-east-1.elasticmapreduce/bootstrap-actions/setup-hbase'
152158
},
153159
'Name': 'Install HBase'
154160
}
155161

156162
INSTALL_IMPALA_BA = {
157163
'ScriptBootstrapAction': {
158-
'Path': 's3://elasticmapreduce/libs/impala/setup-impala',
159-
'Args': ['--base-path', 's3://elasticmapreduce',
164+
'Path': 's3://us-east-1.elasticmapreduce/libs/impala/setup-impala',
165+
'Args': ['--base-path', 's3://us-east-1.elasticmapreduce',
160166
'--impala-version', 'latest']
161167
},
162168
'Name': 'Install Impala'
@@ -214,9 +220,11 @@
214220
'Name': 'Hive program',
215221
'ActionOnFailure': 'CONTINUE',
216222
'HadoopJarStep': {
217-
'Jar': 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
223+
'Jar':
224+
('s3://us-east-1.elasticmapreduce/libs/'
225+
'script-runner/script-runner.jar'),
218226
'Args': [
219-
's3://elasticmapreduce/libs/hive/hive-script',
227+
's3://us-east-1.elasticmapreduce/libs/hive/hive-script',
220228
'--run-hive-script',
221229
'--hive-versions',
222230
'latest',
@@ -229,9 +237,11 @@
229237
'Name': 'HiveBasicStep',
230238
'ActionOnFailure': 'CANCEL_AND_WAIT',
231239
'HadoopJarStep': {
232-
'Jar': 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
240+
'Jar':
241+
('s3://us-east-1.elasticmapreduce/libs/'
242+
'script-runner/script-runner.jar'),
233243
'Args': [
234-
's3://elasticmapreduce/libs/hive/hive-script',
244+
's3://us-east-1.elasticmapreduce/libs/hive/hive-script',
235245
'--run-hive-script',
236246
'--hive-versions',
237247
'0.11.0.1',
@@ -247,9 +257,11 @@
247257
'Name': 'Pig program',
248258
'ActionOnFailure': 'CONTINUE',
249259
'HadoopJarStep': {
250-
'Jar': 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
260+
'Jar':
261+
('s3://us-east-1.elasticmapreduce/libs/'
262+
'script-runner/script-runner.jar'),
251263
'Args': [
252-
's3://elasticmapreduce/libs/pig/pig-script',
264+
's3://us-east-1.elasticmapreduce/libs/pig/pig-script',
253265
'--run-pig-script',
254266
'--pig-versions',
255267
'latest',
@@ -263,9 +275,11 @@
263275
'Name': 'PigBasicStep',
264276
'ActionOnFailure': 'CANCEL_AND_WAIT',
265277
'HadoopJarStep': {
266-
'Jar': 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
278+
'Jar':
279+
('s3://us-east-1.elasticmapreduce/libs/'
280+
'script-runner/script-runner.jar'),
267281
'Args': [
268-
's3://elasticmapreduce/libs/pig/pig-script',
282+
's3://us-east-1.elasticmapreduce/libs/pig/pig-script',
269283
'--run-pig-script',
270284
'--pig-versions',
271285
'0.11.1.0',
@@ -282,9 +296,11 @@
282296
'Name': 'Impala program',
283297
'ActionOnFailure': 'CONTINUE',
284298
'HadoopJarStep': {
285-
'Jar': 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
299+
'Jar':
300+
('s3://us-east-1.elasticmapreduce/libs/'
301+
'script-runner/script-runner.jar'),
286302
'Args': [
287-
's3://elasticmapreduce/libs/impala/setup-impala',
303+
's3://us-east-1.elasticmapreduce/libs/impala/setup-impala',
288304
'--run-impala-script',
289305
'--impala-script',
290306
's3://myimpala/input',
@@ -448,9 +464,11 @@ def test_enable_debugging(self):
448464
[{'Name': 'Setup Hadoop Debugging',
449465
'ActionOnFailure': 'TERMINATE_CLUSTER',
450466
'HadoopJarStep':
451-
{'Args': ['s3://elasticmapreduce/libs/state-pusher/0.1/fetch'],
467+
{'Args':
468+
[('s3://us-east-1.elasticmapreduce/libs/'
469+
'state-pusher/0.1/fetch')],
452470
'Jar':
453-
's3://elasticmapreduce/libs/' +
471+
's3://us-east-1.elasticmapreduce/libs/' +
454472
'script-runner/script-runner.jar'
455473
}
456474
}]

‎tests/unit/customizations/emr/test_install_applications.py‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,13 @@
1818

1919
INSTALL_HIVE_STEP = {
2020
'HadoopJarStep': {
21-
'Args': ['s3://elasticmapreduce/libs/hive/hive-script',
21+
'Args': ['s3://us-east-1.elasticmapreduce/libs/hive/hive-script',
2222
'--install-hive', '--base-path',
23-
's3://elasticmapreduce/libs/hive',
23+
's3://us-east-1.elasticmapreduce/libs/hive',
2424
'--hive-versions', 'latest'],
25-
'Jar': 's3://elasticmapreduce/libs/script-runner/script-runner.jar'
25+
'Jar':
26+
('s3://us-east-1.elasticmapreduce/libs/'
27+
'script-runner/script-runner.jar')
2628
},
2729
'Name': 'Install Hive',
2830
'ActionOnFailure': 'TERMINATE_CLUSTER'
@@ -46,11 +48,13 @@
4648

4749
INSTALL_PIG_STEP = {
4850
'HadoopJarStep': {
49-
'Args': ['s3://elasticmapreduce/libs/pig/pig-script',
51+
'Args': ['s3://us-east-1.elasticmapreduce/libs/pig/pig-script',
5052
'--install-pig', '--base-path',
51-
's3://elasticmapreduce/libs/pig',
53+
's3://us-east-1.elasticmapreduce/libs/pig',
5254
'--pig-versions', 'latest'],
53-
'Jar': 's3://elasticmapreduce/libs/script-runner/script-runner.jar'
55+
'Jar':
56+
('s3://us-east-1.elasticmapreduce/libs/'
57+
'script-runner/script-runner.jar')
5458
},
5559
'Name': 'Install Pig',
5660
'ActionOnFailure': 'TERMINATE_CLUSTER'

0 commit comments

Comments
 (0)