CoCalc -- test_add

GitHub Repository: aws/aws-cli
Path: blob/develop/tests/unit/customizations/emr/test_add_steps.py
¹⁵⁶⁹ views
1
# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2
#
3
# Licensed under the Apache License, Version 2.0 (the "License"). You
4
# may not use this file except in compliance with the License. A copy of
5
# the License is located at
6
#
7
#     http://aws.amazon.com/apache2.0/
8
#
9
# or in the "license" file accompanying this file. This file is
10
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11
# ANY KIND, either express or implied. See the License for the specific
12
# language governing permissions and limitations under the License.
13
import os
14
import copy
15

16

17
from awscli.testutils import mock
18
from tests.unit.customizations.emr import EMRBaseAWSCommandParamsTest as \
19
    BaseAWSCommandParamsTest
20

21

22
class TestAddSteps(BaseAWSCommandParamsTest):
23
    prefix = 'emr add-steps --cluster-id j-ABC --steps '
24

25
    STREAMING_ARGS = 'Args=-files,' + \
26
        's3://elasticmapreduce/samples/wordcount/wordSplitter.py,' + \
27
        '-mapper,wordSplitter.py,' + \
28
        '-reducer,aggregate,' + \
29
        '-input,s3://elasticmapreduce/samples/wordcount/input,' + \
30
        '-output,s3://mybucket/wordcount/output/2014-04-18/12-15-24'
31
    STREAMING_HADOOP_SCRIPT_RUNNER_STEP = \
32
        {'Jar': '/home/hadoop/contrib/streaming/hadoop-streaming.jar',
33
         'Args':
34
            ['-files',
35
             's3://elasticmapreduce/samples/wordcount/wordSplitter.py',
36
             '-mapper',
37
             'wordSplitter.py',
38
             '-reducer',
39
             'aggregate',
40
             '-input',
41
             's3://elasticmapreduce/samples/wordcount/input',
42
             '-output',
43
             's3://mybucket/wordcount/output/2014-04-18/12-15-24']
44
         }
45
    STREAMING_HADOOP_COMMAND_RUNNER_STEP = \
46
        {'Jar': 'command-runner.jar',
47
         'Args':
48
            ['hadoop-streaming',
49
             '-files',
50
             's3://elasticmapreduce/samples/wordcount/wordSplitter.py',
51
             '-mapper',
52
             'wordSplitter.py',
53
             '-reducer',
54
             'aggregate',
55
             '-input',
56
             's3://elasticmapreduce/samples/wordcount/input',
57
             '-output',
58
             's3://mybucket/wordcount/output/2014-04-18/12-15-24']
59
         }
60

61
    HIVE_BASIC_ARGS = 'Args=-f,' + \
62
        's3://elasticmapreduce/samples/hive-ads/libs/model-build.q'
63

64
    HIVE_DEFAULT_SCRIPT_RUNNER_STEP = \
65
        {'Jar':
66
            ('s3://us-east-1.elasticmapreduce/'
67
             'libs/script-runner/script-runner.jar'),
68
         'Args':
69
            ['s3://us-east-1.elasticmapreduce/libs/hive/hive-script',
70
             '--run-hive-script',
71
             '--hive-versions',
72
             'latest',
73
             '--args',
74
             '-f',
75
             's3://elasticmapreduce/samples/hive-ads/libs/model-build.q'
76
             ]
77
         }
78

79
    HIVE_DEFAULT_COMMAND_RUNNER_STEP = \
80
        {'Jar':
81
            ('command-runner.jar'),
82
         'Args':
83
            ['hive-script',
84
             '--run-hive-script',
85
             '--args',
86
             '-f',
87
             's3://elasticmapreduce/samples/hive-ads/libs/model-build.q'
88
             ]
89
         }
90

91
    PIG_BASIC_ARGS = 'Args=-f,' + \
92
        's3://elasticmapreduce/samples/pig-apache/do-reports2.pig'
93

94
    PIG_DEFAULT_SCRIPT_RUNNER_STEP = \
95
        {'Jar':
96
            ('s3://us-east-1.elasticmapreduce/libs/'
97
             'script-runner/script-runner.jar'),
98
         'Args':
99
            ['s3://us-east-1.elasticmapreduce/libs/pig/pig-script',
100
             '--run-pig-script',
101
             '--pig-versions',
102
             'latest',
103
             '--args',
104
             '-f',
105
             's3://elasticmapreduce/samples/'
106
             'pig-apache/do-reports2.pig',
107
             ]}
108

109
    PIG_DEFAULT_COMMAND_RUNNER_STEP = \
110
        {'Jar':
111
            ('command-runner.jar'),
112
         'Args':
113
            ['pig-script',
114
             '--run-pig-script',
115
             '--args',
116
             '-f',
117
             's3://elasticmapreduce/samples/'
118
             'pig-apache/do-reports2.pig',
119
             ]}
120

121
    IMPALA_BASIC_ARGS = 'Args=' + \
122
        '--impala-script,s3://myimpala/input,' + \
123
        '--console-output-path,s3://myimpala/output'
124

125
    IMPALA_BASIC_SCRIPT_RUNNER_STEP = \
126
        {'Jar':
127
            ('s3://us-east-1.elasticmapreduce/libs/'
128
             'script-runner/script-runner.jar'),
129
         'Args':
130
            ['s3://us-east-1.elasticmapreduce/libs/impala/setup-impala',
131
             '--run-impala-script',
132
             '--impala-script',
133
             's3://myimpala/input',
134
             '--console-output-path',
135
             's3://myimpala/output'
136
             ]
137
         }
138

139
    SPARK_SUBMIT_BASIC_ARGS = 'Args=' + \
140
        '[--deploy-mode,' + \
141
        'cluster,' + \
142
        '--conf,' + \
143
        'k1=v1,' + \
144
        's3://mybucket/myfolder/app.jar,' + \
145
        'k2=v2]'
146

147
    SPARK_SUBMIT_SCRIPT_RUNNER_STEP = \
148
        {
149
            'Jar':
150
            ('s3://us-east-1.elasticmapreduce/libs/'
151
             'script-runner/script-runner.jar'),
152
            'Args':
153
            ['/home/hadoop/spark/bin/spark-submit',
154
             '--deploy-mode',
155
             'cluster',
156
             '--conf',
157
             'k1=v1',
158
             's3://mybucket/myfolder/app.jar',
159
             'k2=v2'
160
             ]
161
        }
162

163
    SPARK_SUBMIT_COMMAND_RUNNER_STEP = \
164
        {
165
            'Jar': 'command-runner.jar',
166
            'Args':
167
            ['spark-submit',
168
             '--deploy-mode',
169
             'cluster',
170
             '--conf',
171
             'k1=v1',
172
             's3://mybucket/myfolder/app.jar',
173
             'k2=v2'
174
             ]
175
        }
176

177
    def test_unknown_step_type(self):
178
        cmd = self.prefix + 'Type=unknown'
179
        expected_error_msg = '\naws: error: ' + \
180
            'The step type unknown is not supported.\n'
181
        self.assert_error_for_ami_and_release_based_clusters(
182
            cmd=cmd, expected_error_msg=expected_error_msg,
183
            expected_result_release=expected_error_msg)
184

185
    def test_default_step_type_name_action_on_failure(self):
186
        cmd = self.prefix + 'Jar=s3://mybucket/mytest.jar'
187
        expected_result = {
188
            'JobFlowId': 'j-ABC',
189
            'Steps': [
190
                {'Name': 'Custom JAR',
191
                 'ActionOnFailure': 'CONTINUE',
192
                 'HadoopJarStep': {'Jar': 's3://mybucket/mytest.jar'}
193
                 }
194
            ]
195
        }
196

197
        self.assert_params_for_ami_and_release_based_clusters(
198
            cmd=cmd, expected_result=expected_result,
199
            expected_result_release=expected_result)
200

201
    def test_custom_jar_step_missing_jar(self):
202
        cmd = self.prefix + 'Name=CustomJarMissingJar'
203
        expected_error_msg = '\naws: error: The following ' + \
204
            'required parameters are missing for CustomJARStepConfig: Jar.\n'
205
        self.assert_error_for_ami_and_release_based_clusters(
206
            cmd=cmd, expected_error_msg=expected_error_msg,
207
            expected_result_release=expected_error_msg)
208

209
    def test_custom_jar_step_with_all_fields(self):
210
        cmd = self.prefix + (
211
            'Name=Custom,Type=Custom_jar,'
212
            'Jar=s3://mybucket/mytest.jar,'
213
            'Args=arg1,arg2,MainClass=mymainclass,'
214
            'ActionOnFailure=TERMINATE_CLUSTER,'
215
            'Properties=k1=v1\,k2=v2\,k3')
216
        expected_result = {
217
            'JobFlowId': 'j-ABC',
218
            'Steps': [
219
                {'Name': 'Custom',
220
                 'ActionOnFailure': 'TERMINATE_CLUSTER',
221
                 'HadoopJarStep':
222
                    {'Jar': 's3://mybucket/mytest.jar',
223
                     'Args': ['arg1', 'arg2'],
224
                     'MainClass': 'mymainclass',
225
                     'Properties':
226
                        [{'Key': 'k1', 'Value': 'v1'},
227
                         {'Key': 'k2', 'Value': 'v2'},
228
                         {'Key': 'k3', 'Value': ''}
229
                         ]
230
                     }
231
                 }
232
            ]
233
        }
234

235
        self.assert_params_for_ami_and_release_based_clusters(
236
            cmd=cmd, expected_result=expected_result,
237
            expected_result_release=expected_result)
238

239
    def test_streaming_step_with_default_fields(self):
240
        cmd = self.prefix + 'Type=Streaming,' + self.STREAMING_ARGS
241
        expected_result = {
242
            'JobFlowId': 'j-ABC',
243
            'Steps': [
244
                {'Name': 'Streaming program',
245
                 'ActionOnFailure': 'CONTINUE',
246
                 'HadoopJarStep': self.STREAMING_HADOOP_SCRIPT_RUNNER_STEP
247
                 }
248
            ]
249
        }
250
        expected_result_release = copy.deepcopy(expected_result)
251
        expected_result_release['Steps'][0]['HadoopJarStep'] = \
252
            self.STREAMING_HADOOP_COMMAND_RUNNER_STEP
253

254
        self.assert_params_for_ami_and_release_based_clusters(
255
            cmd=cmd, expected_result=expected_result,
256
            expected_result_release=expected_result_release)
257

258
    def test_step_with_execution_role_arn(self):
259
        cmd = self.prefix + 'Type=Streaming,' + self.STREAMING_ARGS
260
        cmd += ' --execution-role-arn arn:aws:iam::123456789010:role/sample '
261
        expected_result = {
262
            'ExecutionRoleArn': 'arn:aws:iam::123456789010:role/sample',
263
            'JobFlowId': 'j-ABC',
264
            'Steps': [
265
                {'Name': 'Streaming program',
266
                 'ActionOnFailure': 'CONTINUE',
267
                 'HadoopJarStep': self.STREAMING_HADOOP_SCRIPT_RUNNER_STEP
268
                 }
269
            ]
270
        }
271
        expected_result_release = copy.deepcopy(expected_result)
272
        expected_result_release['Steps'][0]['HadoopJarStep'] = \
273
            self.STREAMING_HADOOP_COMMAND_RUNNER_STEP
274

275
        self.assert_params_for_ami_and_release_based_clusters(
276
            cmd=cmd, expected_result=expected_result,
277
            expected_result_release=expected_result_release)
278

279
    def test_streaming_step_missing_args(self):
280
        cmd = self.prefix + 'Type=Streaming'
281
        expected_error_msg = '\naws: error: The following ' + \
282
            'required parameters are missing for StreamingStepConfig: Args.\n'
283
        self.assert_error_for_ami_and_release_based_clusters(
284
            cmd=cmd, expected_error_msg=expected_error_msg,
285
            expected_result_release=expected_error_msg)
286

287
    def test_streaming_jar_with_all_fields(self):
288
        test_step_config = 'Type=Streaming,' + \
289
            'Name=StreamingStepAllFields,' + \
290
            'ActionOnFailure=CANCEL_AND_WAIT,' + \
291
            self.STREAMING_ARGS
292
        cmd = self.prefix + test_step_config
293
        expected_result = {
294
            'JobFlowId': 'j-ABC',
295
            'Steps': [
296
                {'Name': 'StreamingStepAllFields',
297
                 'ActionOnFailure': 'CANCEL_AND_WAIT',
298
                 'HadoopJarStep': self.STREAMING_HADOOP_SCRIPT_RUNNER_STEP
299
                 }
300
            ]
301
        }
302

303
        expected_result_release = copy.deepcopy(expected_result)
304
        expected_result_release['Steps'][0]['HadoopJarStep'] = \
305
            self.STREAMING_HADOOP_COMMAND_RUNNER_STEP
306

307
        self.assert_params_for_ami_and_release_based_clusters(
308
            cmd=cmd, expected_result=expected_result,
309
            expected_result_release=expected_result_release)
310

311
    def test_hive_step_with_default_fields(self):
312
        cmd = self.prefix + 'Type=Hive,' + self.HIVE_BASIC_ARGS
313
        expected_result = {
314
            'JobFlowId': 'j-ABC',
315
            'Steps':    [
316
                {'Name': 'Hive program',
317
                 'ActionOnFailure': 'CONTINUE',
318
                 'HadoopJarStep': self.HIVE_DEFAULT_SCRIPT_RUNNER_STEP
319
                 }]
320
            }
321
        expected_result_release = copy.deepcopy(expected_result)
322
        expected_result_release['Steps'][0]['HadoopJarStep'] = \
323
            self.HIVE_DEFAULT_COMMAND_RUNNER_STEP
324

325
        self.assert_params_for_ami_and_release_based_clusters(
326
            cmd=cmd, expected_result=expected_result,
327
            expected_result_release=expected_result_release)
328

329
    def test_hive_step_missing_args(self):
330
        cmd = self.prefix + 'Type=Hive'
331
        expected_error_msg = '\naws: error: The following ' + \
332
            'required parameters are missing for HiveStepConfig: Args.\n'
333

334
        self.assert_error_for_ami_and_release_based_clusters(
335
            cmd=cmd, expected_error_msg=expected_error_msg,
336
            expected_result_release=expected_error_msg)
337

338
    def test_hive_step_with_all_fields(self):
339
        test_step_config = \
340
            'Type=Hive,' + \
341
            'ActionOnFailure=CANCEL_AND_WAIT,' + \
342
            'Name=HiveWithAllFields,' + \
343
            self.HIVE_BASIC_ARGS
344
        cmd = self.prefix + test_step_config
345
        expected_result = {
346
            'JobFlowId': 'j-ABC',
347
            'Steps': [
348
                {'Name': 'HiveWithAllFields',
349
                 'ActionOnFailure': 'CANCEL_AND_WAIT',
350
                 'HadoopJarStep': self.HIVE_DEFAULT_SCRIPT_RUNNER_STEP
351
                 }]
352
        }
353
        expected_result_release = copy.deepcopy(expected_result)
354
        expected_result_release['Steps'][0]['HadoopJarStep'] = \
355
            self.HIVE_DEFAULT_COMMAND_RUNNER_STEP
356

357
        self.assert_params_for_ami_and_release_based_clusters(
358
            cmd=cmd, expected_result=expected_result,
359
            expected_result_release=expected_result_release)
360

361
    def test_pig_step_with_default_fields(self):
362
        cmd = self.prefix + 'Type=Pig,' + self.PIG_BASIC_ARGS
363
        expected_result = {
364
            'JobFlowId': 'j-ABC',
365
            'Steps':    [
366
                {'Name': 'Pig program',
367
                 'ActionOnFailure': 'CONTINUE',
368
                 'HadoopJarStep': self.PIG_DEFAULT_SCRIPT_RUNNER_STEP
369
                 }]
370
        }
371
        expected_result_release = copy.deepcopy(expected_result)
372
        expected_result_release['Steps'][0]['HadoopJarStep'] = \
373
            self.PIG_DEFAULT_COMMAND_RUNNER_STEP
374

375
        self.assert_params_for_ami_and_release_based_clusters(
376
            cmd=cmd, expected_result=expected_result,
377
            expected_result_release=expected_result_release)
378

379
    def test_pig_missing_args(self):
380
        cmd = self.prefix + 'Type=Pig'
381
        expected_error_msg = '\naws: error: The following ' + \
382
            'required parameters are missing for PigStepConfig: Args.\n'
383
        self.assert_error_for_ami_and_release_based_clusters(
384
            cmd=cmd, expected_error_msg=expected_error_msg,
385
            expected_result_release=expected_error_msg)
386

387
    def test_pig_step_with_all_fields(self):
388
        test_step_config = \
389
            'Name=PigWithAllFields,' + \
390
            'Type=Pig,' + \
391
            self.PIG_BASIC_ARGS + ',' + \
392
            'ActionOnFailure=CANCEL_AND_WAIT'
393
        cmd = self.prefix + test_step_config
394
        expected_result = {
395
            'JobFlowId': 'j-ABC',
396
            'Steps': [
397
                {'Name': 'PigWithAllFields',
398
                 'ActionOnFailure': 'CANCEL_AND_WAIT',
399
                 'HadoopJarStep': self.PIG_DEFAULT_SCRIPT_RUNNER_STEP
400
                 }
401
            ]
402
        }
403
        expected_result_release = copy.deepcopy(expected_result)
404
        expected_result_release['Steps'][0]['HadoopJarStep'] = \
405
            self.PIG_DEFAULT_COMMAND_RUNNER_STEP
406

407
        self.assert_params_for_ami_and_release_based_clusters(
408
            cmd=cmd, expected_result=expected_result,
409
            expected_result_release=expected_result_release)
410

411
    def test_impala_step_with_default_fields(self):
412
        test_step_config = 'Type=Impala,' + \
413
            self.IMPALA_BASIC_ARGS
414
        cmd = self.prefix + test_step_config
415
        expected_result = {
416
            'JobFlowId': 'j-ABC',
417
            'Steps':    [
418
                {'Name': 'Impala program',
419
                 'ActionOnFailure': 'CONTINUE',
420
                 'HadoopJarStep': self.IMPALA_BASIC_SCRIPT_RUNNER_STEP
421
                 }]
422
        }
423
        self.assert_params_for_cmd(cmd, expected_result)
424

425
    def test_SPARK_SUBMIT_SCRIPT_RUNNER_STEP(self):
426
        cmd = self.prefix + 'Type=SPARK,' + \
427
            self.SPARK_SUBMIT_BASIC_ARGS
428
        expected_result = {
429
            'JobFlowId': 'j-ABC',
430
            'Steps':    [
431
                {'Name': 'Spark application',
432
                 'ActionOnFailure': 'CONTINUE',
433
                 'HadoopJarStep': self.SPARK_SUBMIT_SCRIPT_RUNNER_STEP
434
                 }]
435
        }
436
        expected_result_release = copy.deepcopy(expected_result)
437
        expected_result_release['Steps'][0]['HadoopJarStep'] = \
438
            self.SPARK_SUBMIT_COMMAND_RUNNER_STEP
439

440
        self.assert_params_for_ami_and_release_based_clusters(
441
            cmd=cmd, expected_result=expected_result,
442
            expected_result_release=expected_result_release)
443

444
    def test_spark_missing_arg(self):
445
        cmd = self.prefix + 'Type=SPARK'
446
        expected_error_msg = '\naws: error: The following ' + \
447
            'required parameters are missing for SparkStepConfig: Args.\n'
448
        self.assert_error_for_ami_and_release_based_clusters(
449
            cmd=cmd, expected_error_msg=expected_error_msg,
450
            expected_result_release=expected_error_msg)
451

452
    def test_impala_missing_args(self):
453
        cmd = self.prefix + 'Type=Impala'
454
        expected_error_msg = '\naws: error: The following ' + \
455
            'required parameters are missing for ImpalaStepConfig: Args.\n'
456
        self.assert_error_for_ami_and_release_based_clusters(
457
            cmd=cmd, expected_error_msg=expected_error_msg,
458
            expected_result_release=None)
459

460
    def test_impala_step_with_all_fields(self):
461
        test_step_config = \
462
            'Name=ImpalaWithAllFields,' + \
463
            'Type=Impala,' + \
464
            self.IMPALA_BASIC_ARGS + ',' + \
465
            'ActionOnFailure=CANCEL_AND_WAIT'
466
        cmd = self.prefix + test_step_config
467
        expected_result = {
468
            'JobFlowId': 'j-ABC',
469
            'Steps': [
470
                {'Name': 'ImpalaWithAllFields',
471
                 'ActionOnFailure': 'CANCEL_AND_WAIT',
472
                 'HadoopJarStep': self.IMPALA_BASIC_SCRIPT_RUNNER_STEP
473
                 }
474
            ]
475
        }
476
        self.assert_params_for_ami_and_release_based_clusters(
477
            cmd=cmd, expected_result=expected_result,
478
            expected_result_release=None)
479

480
    def test_impala_step_with_release(self):
481
        test_step_config = 'Type=Impala,' + self.IMPALA_BASIC_ARGS
482
        cmd = self.prefix + test_step_config
483
        expected_result_release = '\naws: error: The step type impala ' + \
484
            'is not supported.\n'
485

486
        self.assert_error_for_ami_and_release_based_clusters(
487
            cmd=cmd, expected_error_msg=None,
488
            expected_result_release=expected_result_release)
489

490
    def test_empty_step_args(self):
491
        cmd = self.prefix + 'Type=Streaming,Args='
492
        expected_error_msg = ('\naws: error: The prameter Args cannot '
493
                              'be an empty list.\n')
494
        self.assert_error_for_ami_and_release_based_clusters(
495
            cmd=cmd, expected_error_msg=expected_error_msg,
496
            expected_result_release=expected_error_msg)
497

498
        cmd = self.prefix + 'Type=Pig,Args='
499
        self.assert_error_for_ami_and_release_based_clusters(
500
            cmd=cmd, expected_error_msg=expected_error_msg,
501
            expected_result_release=expected_error_msg)
502

503
        cmd = self.prefix + 'Type=Hive,Args='
504
        self.assert_error_for_ami_and_release_based_clusters(
505
            cmd=cmd, expected_error_msg=expected_error_msg,
506
            expected_result_release=expected_error_msg)
507

508
        cmd = self.prefix + 'Args='
509
        expected_error_msg = ('\naws: error: The following required parameters'
510
                              ' are missing for CustomJARStepConfig: Jar.\n')
511
        self.assert_error_for_ami_and_release_based_clusters(
512
            cmd=cmd, expected_error_msg=expected_error_msg,
513
            expected_result_release=expected_error_msg)
514

515
    def test_all_step_types(self):
516
        test_step_config = 'Jar=s3://mybucket/mytest.jar ' + \
517
            ' Type=Streaming,' + self.STREAMING_ARGS + \
518
            ' Type=Hive,' + self.HIVE_BASIC_ARGS + \
519
            ' Type=Pig,' + self.PIG_BASIC_ARGS + \
520
            ' Type=Impala,' + self.IMPALA_BASIC_ARGS
521
        cmd = self.prefix + test_step_config
522
        expected_result = {
523
            'JobFlowId': 'j-ABC',
524
            'Steps': [
525
                {'Name': 'Custom JAR',
526
                 'ActionOnFailure': 'CONTINUE',
527
                 'HadoopJarStep': {'Jar': 's3://mybucket/mytest.jar'}
528
                 },
529
                {'Name': 'Streaming program',
530
                 'ActionOnFailure': 'CONTINUE',
531
                 'HadoopJarStep': self.STREAMING_HADOOP_SCRIPT_RUNNER_STEP
532
                 },
533
                {'Name': 'Hive program',
534
                 'ActionOnFailure': 'CONTINUE',
535
                 'HadoopJarStep': self.HIVE_DEFAULT_SCRIPT_RUNNER_STEP
536
                 },
537
                {'Name': 'Pig program',
538
                 'ActionOnFailure': 'CONTINUE',
539
                 'HadoopJarStep': self.PIG_DEFAULT_SCRIPT_RUNNER_STEP
540
                 },
541
                {'Name': 'Impala program',
542
                 'ActionOnFailure': 'CONTINUE',
543
                 'HadoopJarStep': self.IMPALA_BASIC_SCRIPT_RUNNER_STEP
544
                 }
545
            ]
546
        }
547

548
        self.assert_params_for_ami_and_release_based_clusters(
549
            cmd=cmd, expected_result=expected_result,
550
            expected_result_release=None)
551

552
    def test_all_step_types_release(self):
553
        test_step_config = 'Jar=s3://mybucket/mytest.jar ' + \
554
            ' Type=Streaming,' + self.STREAMING_ARGS + \
555
            ' Type=Hive,' + self.HIVE_BASIC_ARGS + \
556
            ' Type=Pig,' + self.PIG_BASIC_ARGS
557

558
        cmd = self.prefix + test_step_config
559
        expected_result_release = {
560
            'JobFlowId': 'j-ABC',
561
            'Steps': [
562
                {'Name': 'Custom JAR',
563
                 'ActionOnFailure': 'CONTINUE',
564
                 'HadoopJarStep': {'Jar': 's3://mybucket/mytest.jar'}
565
                 },
566
                {'Name': 'Streaming program',
567
                 'ActionOnFailure': 'CONTINUE',
568
                 'HadoopJarStep': self.STREAMING_HADOOP_COMMAND_RUNNER_STEP
569
                 },
570
                {'Name': 'Hive program',
571
                 'ActionOnFailure': 'CONTINUE',
572
                 'HadoopJarStep': self.HIVE_DEFAULT_COMMAND_RUNNER_STEP
573
                 },
574
                {'Name': 'Pig program',
575
                 'ActionOnFailure': 'CONTINUE',
576
                 'HadoopJarStep': self.PIG_DEFAULT_COMMAND_RUNNER_STEP
577
                 }
578
            ]
579
        }
580

581
        self.assert_params_for_ami_and_release_based_clusters(
582
            cmd=cmd, expected_result=None,
583
            expected_result_release=expected_result_release)
584

585
    def test_all_step_types_from_json(self):
586
        data_path = os.path.join(
587
            os.path.dirname(__file__), 'input_steps.json')
588
        cmd = self.prefix + 'file://' + data_path
589
        hive_script_runner_step = copy.deepcopy(
590
            self.HIVE_DEFAULT_SCRIPT_RUNNER_STEP)
591
        hive_script_runner_step['Args'] += \
592
            ['-d',
593
             'INPUT=s3://elasticmapreduce/samples/hive-ads/tables',
594
             '-d',
595
             'OUTPUT=s3://mybucket/hive-ads/output/2014-04-18/11-07-32',
596
             '-d',
597
             'LIBS=s3://elasticmapreduce/samples/hive-ads/libs'
598
             ]
599
        pig_script_runner_step = copy.deepcopy(
600
            self.PIG_DEFAULT_SCRIPT_RUNNER_STEP)
601
        pig_script_runner_step['Args'] += \
602
            ['-p',
603
             'INPUT=s3://elasticmapreduce/samples/pig-apache/input',
604
             '-p',
605
             'OUTPUT=s3://mybucket/pig-apache/output/2014-04-21/20-09-28'
606
             ]
607

608
        expected_result = {
609
            'JobFlowId': 'j-ABC',
610
            'Steps': [
611
                {'Name': 'Custom JAR step',
612
                 'ActionOnFailure': 'CANCEL_AND_WAIT',
613
                 'HadoopJarStep': {'Jar': 's3://mybucket/mytest.jar'}
614
                 },
615
                {'Name': 'Streaming step',
616
                 'ActionOnFailure': 'CANCEL_AND_WAIT',
617
                 'HadoopJarStep': self.STREAMING_HADOOP_SCRIPT_RUNNER_STEP
618
                 },
619
                {'Name': 'Hive step',
620
                 'ActionOnFailure': 'TERMINATE_CLUSTER',
621
                 'HadoopJarStep': hive_script_runner_step
622
                 },
623
                {'Name': 'Pig step',
624
                 'ActionOnFailure': 'TERMINATE_CLUSTER',
625
                 'HadoopJarStep': pig_script_runner_step
626
                 },
627
                {'Name': 'Impala step',
628
                 'ActionOnFailure': 'CANCEL_AND_WAIT',
629
                 'HadoopJarStep': self.IMPALA_BASIC_SCRIPT_RUNNER_STEP
630
                 }
631
            ]
632
        }
633
        self.assert_params_for_cmd(cmd, expected_result)
634

635
    @mock.patch('awscli.customizations.emr.'
636
                'emrutils.get_release_label')
637
    def assert_params_for_ami_and_release_based_clusters(
638
            self, grl_patch, cmd, expected_result, expected_result_release):
639
        if expected_result:
640
            grl_patch.return_value = None
641
            self.assert_params_for_cmd(cmd, expected_result)
642
        if expected_result_release:
643
            grl_patch.return_value = 'emr-4.0'
644
            self.assert_params_for_cmd(cmd, expected_result_release)
645

646
    @mock.patch('awscli.customizations.emr.'
647
                'emrutils.get_release_label')
648
    def assert_error_for_ami_and_release_based_clusters(
649
            self, grl_patch, cmd, expected_error_msg,
650
            expected_result_release):
651
        if expected_error_msg:
652
            grl_patch.return_value = None
653
            result = self.run_cmd(cmd, 255)
654
            self.assertEqual(expected_error_msg, result[1])
655
        if expected_result_release:
656
            grl_patch.return_value = 'emr-4.0'
657
            result = self.run_cmd(cmd, 255)
658
            self.assertEqual(expected_result_release, result[1])
659

660
if __name__ == "__main__":
661
    unittest.main()
662

663
Product

Resources

Company
