CoCalc -- test_create_cluster_ami

GitHub Repository: aws/aws-cli
Path: blob/develop/tests/unit/customizations/emr/test_create_cluster_ami_version.py
¹⁵⁶⁹ views
1
# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2
#
3
# Licensed under the Apache License, Version 2.0 (the "License"). You
4
# may not use this file except in compliance with the License. A copy of
5
# the License is located at
6
#
7
#     http://aws.amazon.com/apache2.0/
8
#
9
# or in the "license" file accompanying this file. This file is
10
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11
# ANY KIND, either express or implied. See the License for the specific
12
# language governing permissions and limitations under the License.
13

14
import copy
15
import json
16
import os
17

18
from awscli.testutils import mock, unittest
19
from tests.unit.customizations.emr import (
20
    EMRBaseAWSCommandParamsTest as BaseAWSCommandParamsTest,
21
)
22
from tests.unit.customizations.emr import test_constants as CONSTANTS
23
from tests.unit.customizations.emr import (
24
    test_constants_instance_fleets as CONSTANTS_FLEET,
25
)
26

27
DEFAULT_CLUSTER_NAME = "Development Cluster"
28

29
DEFAULT_INSTANCE_GROUPS_ARG = (
30
    'InstanceGroupType=MASTER,Name=MASTER,'
31
    'InstanceCount=1,InstanceType=m1.large,CustomAmiId=ami-deadbeef '
32
    'InstanceGroupType=CORE,Name=CORE,'
33
    'InstanceCount=1,InstanceType=m1.large,CustomAmiId=ami-deadbeef '
34
    'InstanceGroupType=TASK,Name=TASK,'
35
    'InstanceCount=1,InstanceType=m1.large,CustomAmiId=ami-deadpork '
36
)
37

38
DEFAULT_INSTANCE_GROUPS = [
39
    {
40
        'InstanceRole': 'MASTER',
41
        'InstanceCount': 1,
42
        'Name': 'MASTER',
43
        'CustomAmiId': 'ami-deadbeef',
44
        'Market': 'ON_DEMAND',
45
        'InstanceType': 'm1.large',
46
    },
47
    {
48
        'InstanceRole': 'CORE',
49
        'InstanceCount': 1,
50
        'Name': 'CORE',
51
        'CustomAmiId': 'ami-deadbeef',
52
        'Market': 'ON_DEMAND',
53
        'InstanceType': 'm1.large',
54
    },
55
    {
56
        'InstanceRole': 'TASK',
57
        'InstanceCount': 1,
58
        'Name': 'TASK',
59
        'CustomAmiId': 'ami-deadpork',
60
        'Market': 'ON_DEMAND',
61
        'InstanceType': 'm1.large',
62
    },
63
]
64

65
DEFAULT_CMD = (
66
    'emr create-cluster --ami-version 3.0.4 --use-default-roles'
67
    ' --instance-groups ' + DEFAULT_INSTANCE_GROUPS_ARG + ' '
68
)
69

70
DEFAULT_INSTANCES = {
71
    'KeepJobFlowAliveWhenNoSteps': True,
72
    'TerminationProtected': False,
73
    'InstanceGroups': DEFAULT_INSTANCE_GROUPS,
74
}
75

76
EC2_ROLE_NAME = "EMR_EC2_DefaultRole"
77
EMR_ROLE_NAME = "EMR_DefaultRole"
78

79
TEST_BA = [
80
    {
81
        'ScriptBootstrapAction': {
82
            'Path': 's3://test/ba1',
83
            'Args': ['arg1', 'arg2', 'arg3'],
84
        },
85
        'Name': 'ba1',
86
    },
87
    {
88
        'ScriptBootstrapAction': {
89
            'Path': 's3://test/ba2',
90
            'Args': ['arg1', 'arg2', 'arg3'],
91
        },
92
        'Name': 'ba2',
93
    },
94
]
95

96
INSTALL_HIVE_STEP = {
97
    'HadoopJarStep': {
98
        'Args': [
99
            's3://us-east-1.elasticmapreduce/libs/hive/hive-script',
100
            '--install-hive',
101
            '--base-path',
102
            's3://us-east-1.elasticmapreduce/libs/hive',
103
            '--hive-versions',
104
            'latest',
105
        ],
106
        'Jar': (
107
            's3://us-east-1.elasticmapreduce/libs/'
108
            'script-runner/script-runner.jar'
109
        ),
110
    },
111
    'Name': 'Install Hive',
112
    'ActionOnFailure': 'TERMINATE_CLUSTER',
113
}
114

115
INSTALL_HIVE_SITE_STEP = {
116
    'HadoopJarStep': {
117
        'Args': [
118
            's3://us-east-1.elasticmapreduce/libs/hive/hive-script',
119
            '--base-path',
120
            's3://us-east-1.elasticmapreduce/libs/hive',
121
            '--install-hive-site',
122
            '--hive-site=s3://test/hive-conf/hive-site.xml',
123
            '--hive-versions',
124
            'latest',
125
        ],
126
        'Jar': (
127
            's3://us-east-1.elasticmapreduce/libs/'
128
            'script-runner/script-runner.jar'
129
        ),
130
    },
131
    'Name': 'Install Hive Site Configuration',
132
    'ActionOnFailure': 'CANCEL_AND_WAIT',
133
}
134

135
INSTALL_PIG_STEP = {
136
    'HadoopJarStep': {
137
        'Args': [
138
            's3://us-east-1.elasticmapreduce/libs/pig/pig-script',
139
            '--install-pig',
140
            '--base-path',
141
            's3://us-east-1.elasticmapreduce/libs/pig',
142
            '--pig-versions',
143
            'latest',
144
        ],
145
        'Jar': (
146
            's3://us-east-1.elasticmapreduce/libs/'
147
            'script-runner/script-runner.jar'
148
        ),
149
    },
150
    'Name': 'Install Pig',
151
    'ActionOnFailure': 'TERMINATE_CLUSTER',
152
}
153

154
INSTALL_HBASE_STEP = {
155
    'HadoopJarStep': {
156
        'Args': ['emr.hbase.backup.Main', '--start-master'],
157
        'Jar': '/home/hadoop/lib/hbase.jar',
158
    },
159
    'Name': 'Start HBase',
160
    'ActionOnFailure': 'TERMINATE_CLUSTER',
161
}
162

163
INSTALL_GANGLIA_BA = {
164
    'ScriptBootstrapAction': {
165
        'Path': (
166
            's3://us-east-1.elasticmapreduce/'
167
            'bootstrap-actions/install-ganglia'
168
        )
169
    },
170
    'Name': 'Install Ganglia',
171
}
172

173
INSTALL_HBASE_BA = {
174
    'ScriptBootstrapAction': {
175
        'Path': 's3://us-east-1.elasticmapreduce/bootstrap-actions/setup-hbase'
176
    },
177
    'Name': 'Install HBase',
178
}
179

180
INSTALL_IMPALA_BA = {
181
    'ScriptBootstrapAction': {
182
        'Path': 's3://us-east-1.elasticmapreduce/libs/impala/setup-impala',
183
        'Args': [
184
            '--base-path',
185
            's3://us-east-1.elasticmapreduce',
186
            '--impala-version',
187
            'latest',
188
        ],
189
    },
190
    'Name': 'Install Impala',
191
}
192

193
INSTALL_MAPR_PRODUCT = {
194
    'Name': 'mapr',
195
    'Args': ['--edition', 'm5', '--version', '3.0.2'],
196
}
197

198
INSTALL_SUPPORTED_PRODUCTS = [
199
    {
200
        'Name': 'hue',
201
        'Args': ['--hue-config', 's3://elasticmapreduce/hue-config'],
202
    },
203
    {'Name': 'mapr', 'Args': ['--edition', 'm7']},
204
    {'Name': 'unknown', 'Args': ['arg1', 'k1=v1']},
205
]
206

207
CUSTOM_JAR_STEP = {
208
    'Name': 'Custom JAR',
209
    'ActionOnFailure': 'CONTINUE',
210
    'HadoopJarStep': {'Jar': 's3://mybucket/mytest.jar'},
211
}
212

213
STREAMING_ARGS = (
214
    'Args=-files,'
215
    's3://elasticmapreduce/samples/wordcount/wordSplitter.py,'
216
    '-mapper,wordSplitter.py,'
217
    '-reducer,aggregate,'
218
    '-input,s3://elasticmapreduce/samples/wordcount/input,'
219
    '-output,s3://mybucket/wordcount/output/2014-04-18/12-15-24'
220
)
221

222
STREAMING_HADOOP_JAR_STEP = {
223
    'Jar': '/home/hadoop/contrib/streaming/hadoop-streaming.jar',
224
    'Args': [
225
        '-files',
226
        's3://elasticmapreduce/samples/wordcount/wordSplitter.py',
227
        '-mapper',
228
        'wordSplitter.py',
229
        '-reducer',
230
        'aggregate',
231
        '-input',
232
        's3://elasticmapreduce/samples/wordcount/input',
233
        '-output',
234
        's3://mybucket/wordcount/output/2014-04-18/12-15-24',
235
    ],
236
}
237

238
HIVE_BASIC_ARGS = (
239
    'Args=-f,s3://elasticmapreduce/samples/hive-ads/libs/model-build.q'
240
)
241

242
HIVE_DEFAULT_STEP = {
243
    'Name': 'Hive program',
244
    'ActionOnFailure': 'CONTINUE',
245
    'HadoopJarStep': {
246
        'Jar': (
247
            's3://us-east-1.elasticmapreduce/libs/'
248
            'script-runner/script-runner.jar'
249
        ),
250
        'Args': [
251
            's3://us-east-1.elasticmapreduce/libs/hive/hive-script',
252
            '--run-hive-script',
253
            '--hive-versions',
254
            'latest',
255
            '--args',
256
            '-f',
257
            's3://elasticmapreduce/samples/hive-ads/libs/model-build.q',
258
        ],
259
    },
260
}
261

262
HIVE_BASIC_STEP = {
263
    'Name': 'HiveBasicStep',
264
    'ActionOnFailure': 'CANCEL_AND_WAIT',
265
    'HadoopJarStep': {
266
        'Jar': (
267
            's3://us-east-1.elasticmapreduce/libs/'
268
            'script-runner/script-runner.jar'
269
        ),
270
        'Args': [
271
            's3://us-east-1.elasticmapreduce/libs/hive/hive-script',
272
            '--run-hive-script',
273
            '--hive-versions',
274
            'latest',
275
            '--args',
276
            '-f',
277
            's3://elasticmapreduce/samples/hive-ads/libs/model-build.q',
278
        ],
279
    },
280
}
281

282
PIG_BASIC_ARGS = (
283
    'Args=-f,' + 's3://elasticmapreduce/samples/pig-apache/do-reports2.pig'
284
)
285

286
PIG_DEFAULT_STEP = {
287
    'Name': 'Pig program',
288
    'ActionOnFailure': 'CONTINUE',
289
    'HadoopJarStep': {
290
        'Jar': (
291
            's3://us-east-1.elasticmapreduce/libs/'
292
            'script-runner/script-runner.jar'
293
        ),
294
        'Args': [
295
            's3://us-east-1.elasticmapreduce/libs/pig/pig-script',
296
            '--run-pig-script',
297
            '--pig-versions',
298
            'latest',
299
            '--args',
300
            '-f',
301
            's3://elasticmapreduce/samples/' 'pig-apache/do-reports2.pig',
302
        ],
303
    },
304
}
305

306
PIG_BASIC_STEP = {
307
    'Name': 'PigBasicStep',
308
    'ActionOnFailure': 'CANCEL_AND_WAIT',
309
    'HadoopJarStep': {
310
        'Jar': (
311
            's3://us-east-1.elasticmapreduce/libs/'
312
            'script-runner/script-runner.jar'
313
        ),
314
        'Args': [
315
            's3://us-east-1.elasticmapreduce/libs/pig/pig-script',
316
            '--run-pig-script',
317
            '--pig-versions',
318
            'latest',
319
            '--args',
320
            '-f',
321
            's3://elasticmapreduce/samples/' 'pig-apache/do-reports2.pig',
322
        ],
323
    },
324
}
325
IMPALA_BASIC_ARGS = (
326
    'Args=--impala-script,s3://myimpala/input,'
327
    '--console-output-path,s3://myimpala/output'
328
)
329

330
IMPALA_DEFAULT_STEP = {
331
    'Name': 'Impala program',
332
    'ActionOnFailure': 'CONTINUE',
333
    'HadoopJarStep': {
334
        'Jar': (
335
            's3://us-east-1.elasticmapreduce/libs/'
336
            'script-runner/script-runner.jar'
337
        ),
338
        'Args': [
339
            's3://us-east-1.elasticmapreduce/libs/impala/setup-impala',
340
            '--run-impala-script',
341
            '--impala-script',
342
            's3://myimpala/input',
343
            '--console-output-path',
344
            's3://myimpala/output',
345
        ],
346
    },
347
}
348

349
CREATE_CLUSTER_RESULT = {
350
    "JobFlowId": "j-XXXX",
351
    "ClusterArn": "arn:aws:elasticmapreduce:region:012345678910:cluster/j-XXXX",
352
}
353

354
CONSTRUCTED_RESULT = {
355
    "ClusterId": "j-XXXX",
356
    "ClusterArn": "arn:aws:elasticmapreduce:region:012345678910:cluster/j-XXXX",
357
}
358

359
DEFAULT_RESULT = {
360
    'Name': DEFAULT_CLUSTER_NAME,
361
    'Instances': DEFAULT_INSTANCES,
362
    'AmiVersion': '3.0.4',
363
    'VisibleToAllUsers': True,
364
    'JobFlowRole': EC2_ROLE_NAME,
365
    'ServiceRole': EMR_ROLE_NAME,
366
    'Tags': [],
367
}
368

369
EMR_MANAGED_MASTER_SECURITY_GROUP = 'sg-master1'
370

371
EMR_MANAGED_SLAVE_SECURITY_GROUP = 'sg-slave1'
372

373
SERVICE_ACCESS_SECURITY_GROUP = "sg-service-access"
374

375
ADDITIONAL_MASTER_SECURITY_GROUPS = [
376
    'sg-addMaster1',
377
    'sg-addMaster2',
378
    'sg-addMaster3',
379
    'sg-addMaster4',
380
]
381

382
ADDITIONAL_SLAVE_SECURITY_GROUPS = [
383
    'sg-addSlave1',
384
    'sg-addSlave2',
385
    'sg-addSlave3',
386
    'sg-addSlave4',
387
]
388

389

390
class TestCreateCluster(BaseAWSCommandParamsTest):
391
    prefix = 'emr create-cluster '
392

393
    def test_quick_start(self):
394
        cmd = (
395
            self.prefix
396
            + '--ami-version 3.1.0 --instance-groups '
397
            + DEFAULT_INSTANCE_GROUPS_ARG
398
        )
399
        result = {
400
            'Name': DEFAULT_CLUSTER_NAME,
401
            'Instances': DEFAULT_INSTANCES,
402
            'AmiVersion': '3.1.0',
403
            'VisibleToAllUsers': True,
404
            'Tags': [],
405
        }
406
        self.assert_params_for_cmd(cmd, result)
407

408
    def assert_error_message_has_field_name(self, error_msg, field_name):
409
        self.assertIn('Missing required parameter', error_msg)
410
        self.assertIn(field_name, error_msg)
411

412
    def test_default_cmd(self):
413
        self.assert_params_for_cmd(DEFAULT_CMD, DEFAULT_RESULT)
414

415
    def test_cluster_without_service_role_and_instance_profile(self):
416
        cmd = (
417
            'emr create-cluster --ami-version 3.0.4 '
418
            '--instance-groups ' + DEFAULT_INSTANCE_GROUPS_ARG
419
        )
420
        result = copy.deepcopy(DEFAULT_RESULT)
421
        del result['JobFlowRole']
422
        del result['ServiceRole']
423
        self.assert_params_for_cmd(cmd, result)
424

425
    def test_cluster_with_service_role_and_instance_profile(self):
426
        cmd = (
427
            'emr create-cluster --ami-version 3.0.4'
428
            ' --service-role ServiceRole --ec2-attributes '
429
            'InstanceProfile=Ec2_InstanceProfile '
430
            '--instance-groups ' + DEFAULT_INSTANCE_GROUPS_ARG
431
        )
432
        result = copy.deepcopy(DEFAULT_RESULT)
433
        result['JobFlowRole'] = 'Ec2_InstanceProfile'
434
        result['ServiceRole'] = 'ServiceRole'
435
        self.assert_params_for_cmd(cmd, result)
436

437
    def test_mutual_exclusive_use_default_roles_and_service_role(self):
438
        cmd = (
439
            DEFAULT_CMD
440
            + '--ec2-attributes InstanceProfile=Ec2_InstanceProfile'
441
        )
442
        expected_error_msg = (
443
            '\naws: error: You cannot specify both --use-default-roles '
444
            'and --ec2-attributes InstanceProfile options together. Either '
445
            'choose --use-default-roles or use both --service-role <roleName>'
446
            ' and --ec2-attributes InstanceProfile=<profileName>.\n'
447
        )
448
        result = self.run_cmd(cmd, 255)
449
        self.assertEqual(expected_error_msg, result[1])
450

451
    def test_mutual_exclusive_use_default_roles_and_instance_profile(self):
452
        cmd = (
453
            DEFAULT_CMD + '--service-role ServiceRole '
454
            '--ec2-attributes InstanceProfile=Ec2_InstanceProfile'
455
        )
456
        expected_error_msg = (
457
            '\naws: error: You cannot specify both --use-default-roles '
458
            'and --service-role options together. Either choose '
459
            '--use-default-roles or use both --service-role <roleName> '
460
            'and --ec2-attributes InstanceProfile=<profileName>.\n'
461
        )
462
        result = self.run_cmd(cmd, 255)
463
        self.assertEqual(expected_error_msg, result[1])
464

465
    def test_cluster_name_no_space(self):
466
        cmd = DEFAULT_CMD + '--name MyCluster'
467
        result = copy.deepcopy(DEFAULT_RESULT)
468
        result['Name'] = 'MyCluster'
469
        self.assert_params_for_cmd(cmd, result)
470

471
    def test_cluster_name_with_space(self):
472
        cmd = DEFAULT_CMD.split() + ['--name', 'My Cluster']
473
        result = copy.deepcopy(DEFAULT_RESULT)
474
        result['Name'] = 'My Cluster'
475
        self.assert_params_for_cmd(cmd, result)
476

477
    def test_ami_version(self):
478
        cmd = DEFAULT_CMD + '--ami-version 3.0.4'
479
        result = copy.deepcopy(DEFAULT_RESULT)
480
        result['AmiVersion'] = '3.0.4'
481
        self.assert_params_for_cmd(cmd, result)
482

483
    def test_log_uri(self):
484
        test_log_uri = 's3://test/logs'
485
        cmd = DEFAULT_CMD + '--log-uri ' + test_log_uri
486
        result = copy.deepcopy(DEFAULT_RESULT)
487
        result['LogUri'] = test_log_uri
488
        self.assert_params_for_cmd(cmd, result)
489

490
    def test_additional_info(self):
491
        test_info = '{ami32: "ami-82e305f5"}'
492
        cmd = DEFAULT_CMD.split() + ['--additional-info', test_info]
493
        result = copy.deepcopy(DEFAULT_RESULT)
494
        result['AdditionalInfo'] = test_info
495
        self.assert_params_for_cmd(cmd, result)
496

497
    def test_auto_terminte(self):
498
        cmd = (
499
            'emr create-cluster --use-default-roles --ami-version 3.0.4 '
500
            '--auto-terminate --instance-groups ' + DEFAULT_INSTANCE_GROUPS_ARG
501
        )
502
        result = copy.deepcopy(DEFAULT_RESULT)
503
        instances = copy.deepcopy(DEFAULT_INSTANCES)
504
        instances['KeepJobFlowAliveWhenNoSteps'] = False
505
        result['Instances'] = instances
506
        self.assert_params_for_cmd(cmd, result)
507

508
    def test_auto_terminate_and_no_auto_terminate(self):
509
        cmd = (
510
            DEFAULT_CMD
511
            + '--ami-version 3.0.4 '
512
            + '--auto-terminate --no-auto-terminate'
513
        )
514
        expected_error_msg = (
515
            '\naws: error: cannot use both --no-auto-terminate and'
516
            ' --auto-terminate options together.\n'
517
        )
518
        result = self.run_cmd(cmd, 255)
519
        self.assertEqual(expected_error_msg, result[1])
520

521
    def test_termination_protected(self):
522
        cmd = DEFAULT_CMD + '--termination-protected'
523
        result = copy.deepcopy(DEFAULT_RESULT)
524
        instances = copy.deepcopy(DEFAULT_INSTANCES)
525
        instances['TerminationProtected'] = True
526
        result['Instances'] = instances
527
        self.assert_params_for_cmd(cmd, result)
528

529
    def test_no_termination_protected(self):
530
        cmd = DEFAULT_CMD + '--no-termination-protected'
531
        self.assert_params_for_cmd(cmd, DEFAULT_RESULT)
532

533
    def test_termination_protected_and_no_termination_protected(self):
534
        cmd = (
535
            DEFAULT_CMD + '--termination-protected --no-termination-protected'
536
        )
537
        expected_error_msg = (
538
            '\naws: error: cannot use both --termination-protected'
539
            ' and --no-termination-protected options together.\n'
540
        )
541
        result = self.run_cmd(cmd, 255)
542
        self.assertEqual(expected_error_msg, result[1])
543

544
    def test_unhealthy_node_replacement(self):
545
        cmd = DEFAULT_CMD + '--unhealthy-node-replacement'
546
        result = copy.deepcopy(DEFAULT_RESULT)
547
        instances = copy.deepcopy(DEFAULT_INSTANCES)
548
        instances['UnhealthyNodeReplacement'] = True
549
        result['Instances'] = instances
550
        self.assert_params_for_cmd(cmd, result)
551

552
    def test_no_unhealthy_node_replacement(self):
553
        cmd = DEFAULT_CMD + '--no-unhealthy-node-replacement'
554
        result = copy.deepcopy(DEFAULT_RESULT)
555
        instances = copy.deepcopy(DEFAULT_INSTANCES)
556
        instances['UnhealthyNodeReplacement'] = False
557
        result['Instances'] = instances
558
        self.assert_params_for_cmd(cmd, result)
559

560
    def test_unhealthy_node_replacement_and_no_unhealthy_node_replacement(
561
        self,
562
    ):
563
        cmd = (
564
            DEFAULT_CMD
565
            + '--unhealthy-node-replacement --no-unhealthy-node-replacement'
566
        )
567
        expected_error_msg = (
568
            '\naws: error: cannot use both --unhealthy-node-replacement'
569
            ' and --no-unhealthy-node-replacement options together.\n'
570
        )
571
        result = self.run_cmd(cmd, 255)
572
        self.assertEqual(expected_error_msg, result[1])
573

574
    def test_visible_to_all_users(self):
575
        cmd = DEFAULT_CMD + '--visible-to-all-users'
576
        self.assert_params_for_cmd(cmd, DEFAULT_RESULT)
577

578
    def test_no_visible_to_all_users(self):
579
        cmd = DEFAULT_CMD + '--no-visible-to-all-users'
580
        result = copy.deepcopy(DEFAULT_RESULT)
581
        result['VisibleToAllUsers'] = False
582
        self.assert_params_for_cmd(cmd, result)
583

584
    def test_visible_to_all_users_and_no_visible_to_all_users(self):
585
        cmd = DEFAULT_CMD + '--visible-to-all-users --no-visible-to-all-users'
586
        expected_error_msg = (
587
            '\naws: error: cannot use both --visible-to-all-users and '
588
            '--no-visible-to-all-users options together.\n'
589
        )
590
        result = self.run_cmd(cmd, 255)
591
        self.assertEqual(expected_error_msg, result[1])
592

593
    def test_extended_support(self):
594
        cmd = DEFAULT_CMD + '--extended-support'
595
        result = copy.deepcopy(DEFAULT_RESULT)
596
        result['ExtendedSupport'] = True
597
        self.assert_params_for_cmd(cmd, result)
598

599
    def test_no_extended_support(self):
600
        cmd = DEFAULT_CMD + '--no-extended-support'
601
        result = copy.deepcopy(DEFAULT_RESULT)
602
        result['ExtendedSupport'] = False
603
        self.assert_params_for_cmd(cmd, result)
604

605
    def test_extended_support_and_no_extended_support(self):
606
        cmd = DEFAULT_CMD + '--extended-support --no-extended-support'
607
        expected_error_msg = (
608
            '\naws: error: cannot use both --extended-support'
609
            ' and --no-extended-support options together.\n'
610
        )
611
        result = self.run_cmd(cmd, 255)
612
        self.assertEqual(expected_error_msg, result[1])
613

614
    def test_tags(self):
615
        cmd = DEFAULT_CMD.split() + ['--tags', 'k1=v1', 'k2', 'k3=spaces  v3']
616
        result = copy.deepcopy(DEFAULT_RESULT)
617
        tags = [
618
            {'Key': 'k1', 'Value': 'v1'},
619
            {'Key': 'k2', 'Value': ''},
620
            {'Key': 'k3', 'Value': 'spaces  v3'},
621
        ]
622
        result['Tags'] = tags
623
        self.assert_params_for_cmd(cmd, result)
624

625
    def test_enable_debugging(self):
626
        cmd = DEFAULT_CMD + '--log-uri s3://test/logs --enable-debugging'
627
        result = copy.deepcopy(DEFAULT_RESULT)
628
        result['LogUri'] = 's3://test/logs'
629
        debugging_config = [
630
            {
631
                'Name': 'Setup Hadoop Debugging',
632
                'ActionOnFailure': 'TERMINATE_CLUSTER',
633
                'HadoopJarStep': {
634
                    'Args': [
635
                        (
636
                            's3://us-east-1.elasticmapreduce/libs/'
637
                            'state-pusher/0.1/fetch'
638
                        )
639
                    ],
640
                    'Jar': 's3://us-east-1.elasticmapreduce/libs/'
641
                    + 'script-runner/script-runner.jar',
642
                },
643
            }
644
        ]
645
        result['Steps'] = debugging_config
646
        self.assert_params_for_cmd(cmd, result)
647

648
        cmd = DEFAULT_CMD + (
649
            '--log-uri s3://test/logs --enable-debugging ' '--region us-west-2'
650
        )
651
        debugging_config = [
652
            {
653
                'Name': 'Setup Hadoop Debugging',
654
                'ActionOnFailure': 'TERMINATE_CLUSTER',
655
                'HadoopJarStep': {
656
                    'Args': [
657
                        (
658
                            's3://us-west-2.elasticmapreduce/libs/'
659
                            'state-pusher/0.1/fetch'
660
                        )
661
                    ],
662
                    'Jar': 's3://us-west-2.elasticmapreduce/libs/'
663
                    + 'script-runner/script-runner.jar',
664
                },
665
            }
666
        ]
667
        result['Steps'] = debugging_config
668
        self.assert_params_for_cmd(cmd, result)
669

670
    def test_enable_debugging_no_log_uri(self):
671
        cmd = DEFAULT_CMD + '--enable-debugging'
672
        expected_error_msg = (
673
            '\naws: error: LogUri not specified. You must specify a logUri'
674
            ' if you enable debugging when creating a cluster.\n'
675
        )
676
        result = self.run_cmd(cmd, 255)
677
        self.assertEqual(expected_error_msg, result[1])
678

679
    def test_enable_debugging_and_no_enable_debugging(self):
680
        cmd = (
681
            DEFAULT_CMD
682
            + '--enable-debugging --no-enable-debugging'
683
            + ' --log-uri s3://test/logs'
684
        )
685
        expected_error_msg = (
686
            '\naws: error: cannot use both --enable-debugging and '
687
            '--no-enable-debugging options together.\n'
688
        )
689
        result = self.run_cmd(cmd, 255)
690
        self.assertEqual(expected_error_msg, result[1])
691

692
    def test_instance_groups_default_name_market(self):
693
        cmd = (
694
            'emr create-cluster --use-default-roles --ami-version 3.0.4 '
695
            '--instance-groups '
696
            'InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m1.large,CustomAmiId=ami-deadbeef '
697
            'InstanceGroupType=CORE,InstanceCount=1,InstanceType=m1.large,CustomAmiId=ami-deadbeef '
698
            'InstanceGroupType=TASK,InstanceCount=1,InstanceType=m1.large,CustomAmiId=ami-deadpork '
699
        )
700
        self.assert_params_for_cmd(cmd, DEFAULT_RESULT)
701

702
    def test_instance_groups_instance_group_type_mismatch_cases(self):
703
        cmd = (
704
            'emr create-cluster --use-default-roles --ami-version 3.0.4 '
705
            '--instance-groups '
706
            'Name=MASTER,InstanceGroupType=MaSter,InstanceCount=1,'
707
            'InstanceType=m1.large,CustomAmiId=ami-deadbeef Name=CORE,InstanceGroupType=cORE,'
708
            'InstanceCount=1,InstanceType=m1.large,CustomAmiId=ami-deadbeef Name=TASK,'
709
            'InstanceGroupType=tAsK,InstanceCount=1,InstanceType=m1.large,CustomAmiId=ami-deadpork'
710
        )
711
        self.assert_params_for_cmd(cmd, DEFAULT_RESULT)
712

713
    def test_instance_groups_instance_type_and_count(self):
714
        cmd = (
715
            'emr create-cluster --use-default-roles --ami-version 3.0.4 '
716
            '--instance-type m1.large'
717
        )
718
        expected_result = copy.deepcopy(DEFAULT_RESULT)
719
        expected_result['Instances'] = {
720
            'KeepJobFlowAliveWhenNoSteps': True,
721
            'TerminationProtected': False,
722
            'InstanceGroups': [
723
                {
724
                    'InstanceRole': 'MASTER',
725
                    'InstanceCount': 1,
726
                    'Name': 'MASTER',
727
                    'Market': 'ON_DEMAND',
728
                    'InstanceType': 'm1.large',
729
                }
730
            ],
731
        }
732
        self.assert_params_for_cmd(cmd, expected_result)
733
        cmd = (
734
            'emr create-cluster --use-default-roles --ami-version 3.0.4 '
735
            '--instance-type m1.large --instance-count 3'
736
        )
737
        expected_result = copy.deepcopy(DEFAULT_RESULT)
738
        expected_result['Instances'] = {
739
            'KeepJobFlowAliveWhenNoSteps': True,
740
            'TerminationProtected': False,
741
            'InstanceGroups': [
742
                {
743
                    'InstanceRole': 'MASTER',
744
                    'InstanceCount': 1,
745
                    'Name': 'MASTER',
746
                    'Market': 'ON_DEMAND',
747
                    'InstanceType': 'm1.large',
748
                },
749
                {
750
                    'InstanceRole': 'CORE',
751
                    'InstanceCount': 2,
752
                    'Name': 'CORE',
753
                    'Market': 'ON_DEMAND',
754
                    'InstanceType': 'm1.large',
755
                },
756
            ],
757
        }
758
        self.assert_params_for_cmd(cmd, expected_result)
759

760
    def test_instance_groups_missing_required_parameter_error(self):
761
        cmd = 'emr create-cluster --use-default-roles --ami-version 3.0.4 '
762
        expect_error_msg = (
763
            '\naws: error: Must specify either --instance-groups or '
764
            '--instance-type with --instance-count(optional) to '
765
            'configure instance groups.\n'
766
        )
767
        result = self.run_cmd(cmd, 255)
768
        self.assertEqual(expect_error_msg, result[1])
769

770
        cmd = (
771
            'emr create-cluster --use-default-roles --ami-version 3.0.4 '
772
            '--instance-count 2'
773
        )
774
        expect_error_msg = (
775
            '\naws: error: Must specify either --instance-groups or '
776
            '--instance-type with --instance-count(optional) to '
777
            'configure instance groups.\n'
778
        )
779
        result = self.run_cmd(cmd, 255)
780
        self.assertEqual(expect_error_msg, result[1])
781

782
    def test_instance_groups_exclusive_parameter_validation_error(self):
783
        cmd = (
784
            'emr create-cluster --use-default-roles --ami-version 3.0.4 '
785
            '--instance-type m1.large --instance-groups '
786
            + DEFAULT_INSTANCE_GROUPS_ARG
787
        )
788
        expect_error_msg = (
789
            '\naws: error: You may not specify --instance-type '
790
            'or --instance-count with --instance-groups, '
791
            'because --instance-type and --instance-count are '
792
            'shortcut options for --instance-groups.\n'
793
        )
794
        result = self.run_cmd(cmd, 255)
795
        self.assertEqual(expect_error_msg, result[1])
796

797
        cmd = (
798
            'emr create-cluster --use-default-roles --ami-version 3.0.4 '
799
            '--instance-type m1.large --instance-count 2 '
800
            '--instance-groups ' + DEFAULT_INSTANCE_GROUPS_ARG
801
        )
802
        expect_error_msg = (
803
            '\naws: error: You may not specify --instance-type '
804
            'or --instance-count with --instance-groups, '
805
            'because --instance-type and --instance-count are '
806
            'shortcut options for --instance-groups.\n'
807
        )
808
        result = self.run_cmd(cmd, 255)
809
        self.assertEqual(expect_error_msg, result[1])
810

811
    def test_instance_groups_missing_instance_group_type_error(self):
812
        cmd = (
813
            'emr create-cluster --use-default-roles --ami-version 3.0.4 '
814
            '--auto-terminate '
815
            '--instance-groups '
816
            'Name=Master,InstanceCount=1,InstanceType=m1.small'
817
        )
818
        stderr = self.run_cmd(cmd, 255)[1]
819
        self.assert_error_message_has_field_name(stderr, 'InstanceGroupType')
820

821
    def test_instance_groups_missing_instance_type_error(self):
822
        cmd = (
823
            'emr create-cluster --use-default-roles --ami-version 3.0.4 '
824
            '--auto-terminate '
825
            '--instance-groups '
826
            'Name=Master,InstanceGroupType=MASTER,InstanceCount=1'
827
        )
828
        stderr = self.run_cmd(cmd, 255)[1]
829
        self.assert_error_message_has_field_name(stderr, 'InstanceType')
830

831
    def test_instance_groups_missing_instance_count_error(self):
832
        cmd = (
833
            'emr create-cluster --use-default-roles --ami-version 3.0.4 '
834
            '--auto-terminate '
835
            '--instance-groups '
836
            'Name=Master,InstanceGroupType=MASTER,InstanceType=m1.xlarge'
837
        )
838
        stderr = self.run_cmd(cmd, 255)[1]
839
        self.assert_error_message_has_field_name(stderr, 'InstanceCount')
840

841
    def test_instance_groups_from_json_file(self):
842
        data_path = os.path.join(
843
            os.path.dirname(__file__), 'input_instance_groups.json'
844
        )
845
        cmd = (
846
            'emr create-cluster --use-default-roles --ami-version 3.0.4  '
847
            '--instance-groups file://' + data_path
848
        )
849
        result = copy.deepcopy(DEFAULT_RESULT)
850
        result['Instances']['InstanceGroups'] = [
851
            {
852
                'InstanceRole': 'MASTER',
853
                'InstanceCount': 1,
854
                'Name': 'Master Instance Group',
855
                'Market': 'ON_DEMAND',
856
                'InstanceType': 'm1.large',
857
            },
858
            {
859
                'InstanceRole': 'CORE',
860
                'InstanceCount': 2,
861
                'Name': 'Core Instance Group',
862
                'Market': 'ON_DEMAND',
863
                'InstanceType': 'm1.xlarge',
864
            },
865
            {
866
                'InstanceRole': 'TASK',
867
                'InstanceCount': 3,
868
                'Name': 'Task Instance Group',
869
                'Market': 'SPOT',
870
                'BidPrice': '3.45',
871
                'InstanceType': 'm1.xlarge',
872
            },
873
        ]
874
        self.assert_params_for_cmd(cmd, result)
875

876
    def test_instance_groups_from_json_file_spot_bidprice_equals_ondemandprice(
877
        self,
878
    ):
879
        data_path = os.path.join(
880
            os.path.dirname(__file__),
881
            'input_instance_groups_spot_bidprice_equals_ondemandprice.json',
882
        )
883
        cmd = (
884
            'emr create-cluster --use-default-roles --ami-version 3.0.4  '
885
            '--instance-groups file://' + data_path
886
        )
887
        result = copy.deepcopy(DEFAULT_RESULT)
888
        result['Instances']['InstanceGroups'] = [
889
            {
890
                'InstanceRole': 'MASTER',
891
                'InstanceCount': 1,
892
                'Name': 'Master Instance Group',
893
                'Market': 'SPOT',
894
                'InstanceType': 'm1.large',
895
            },
896
            {
897
                'InstanceRole': 'CORE',
898
                'InstanceCount': 2,
899
                'Name': 'Core Instance Group',
900
                'Market': 'SPOT',
901
                'InstanceType': 'm1.xlarge',
902
            },
903
            {
904
                'InstanceRole': 'TASK',
905
                'InstanceCount': 3,
906
                'Name': 'Task Instance Group',
907
                'Market': 'SPOT',
908
                'InstanceType': 'm1.xlarge',
909
            },
910
        ]
911
        self.assert_params_for_cmd(cmd, result)
912

913
    def test_ec2_attributes_no_az(self):
914
        cmd = (
915
            'emr create-cluster --ami-version 3.0.4 '
916
            '--instance-groups '
917
            + DEFAULT_INSTANCE_GROUPS_ARG
918
            + ' --ec2-attributes KeyName=testkey,SubnetId=subnet-123456,'
919
            'InstanceProfile=EMR_EC2_DefaultRole '
920
            '--service-role EMR_DefaultRole'
921
        )
922
        result = copy.deepcopy(DEFAULT_RESULT)
923
        result['Instances']['Ec2KeyName'] = 'testkey'
924
        result['Instances']['Ec2SubnetId'] = 'subnet-123456'
925
        result['JobFlowRole'] = 'EMR_EC2_DefaultRole'
926
        self.assert_params_for_cmd(cmd, result)
927

928
    def test_ec2_attributes_az(self):
929
        cmd = DEFAULT_CMD + '--ec2-attributes AvailabilityZone=us-east-1a'
930
        result = copy.deepcopy(DEFAULT_RESULT)
931
        result['Instances']['Placement'] = {'AvailabilityZone': 'us-east-1a'}
932
        self.assert_params_for_cmd(cmd, result)
933

934
    def test_ec2_attributes_subnet_az_error(self):
935
        cmd = (
936
            DEFAULT_CMD
937
            + '--ec2-attributes '
938
            + 'SubnetId=subnet-123456,AvailabilityZone=us-east-1a'
939
        )
940
        expect_error_msg = (
941
            '\naws: error: You may not specify both a SubnetId and an Availab'
942
            'ilityZone (placement) because ec2SubnetId implies a placement.\n'
943
        )
944
        result = self.run_cmd(cmd, 255)
945
        self.assertEqual(expect_error_msg, result[1])
946

947
    def test_ec2_attributes_with_subnet_from_json_file(self):
948
        data_path = os.path.join(
949
            os.path.dirname(__file__), 'input_ec2_attributes_with_subnet.json'
950
        )
951
        cmd = (
952
            'emr create-cluster --ami-version 3.0.4 '
953
            '--instance-groups '
954
            + DEFAULT_INSTANCE_GROUPS_ARG
955
            + ' --ec2-attributes file://'
956
            + data_path
957
            + ' --service-role EMR_DefaultRole'
958
        )
959
        result = copy.deepcopy(DEFAULT_RESULT)
960
        result['Instances']['Ec2KeyName'] = 'testkey'
961
        result['Instances']['Ec2SubnetId'] = 'subnet-123456'
962
        result['JobFlowRole'] = 'EMR_EC2_DefaultRole'
963
        self.assert_params_for_cmd(cmd, result)
964

965
    def test_ec2_attributes_with_az_from_json_file(self):
966
        data_path = os.path.join(
967
            os.path.dirname(__file__), 'input_ec2_attributes_with_az.json'
968
        )
969
        cmd = (
970
            'emr create-cluster --ami-version 3.0.4 '
971
            '--instance-groups '
972
            + DEFAULT_INSTANCE_GROUPS_ARG
973
            + ' --ec2-attributes file://'
974
            + data_path
975
            + ' --service-role EMR_DefaultRole'
976
        )
977
        result = copy.deepcopy(DEFAULT_RESULT)
978
        result['Instances']['Ec2KeyName'] = 'testkey'
979
        result['Instances']['Placement'] = {'AvailabilityZone': 'us-east-1a'}
980
        result['JobFlowRole'] = 'EMR_EC2_DefaultRole'
981
        self.assert_params_for_cmd(cmd, result)
982

983
    # Bootstrap Actions test cases
984
    def test_bootstrap_actions_missing_path_error(self):
985
        cmd = DEFAULT_CMD + '--bootstrap-actions Name=ba1,Args=arg1,arg2'
986
        stderr = self.run_cmd(cmd, 255)[1]
987
        self.assert_error_message_has_field_name(stderr, 'Path')
988

989
    def test_bootstrap_actions_with_all_fields(self):
990
        cmd = DEFAULT_CMD + (
991
            ' --bootstrap-actions '
992
            'Path=s3://test/ba1,Name=ba1,Args=arg1,arg2,arg3 '
993
            'Path=s3://test/ba2,Name=ba2,Args=arg1,arg2,arg3'
994
        )
995
        result = copy.deepcopy(DEFAULT_RESULT)
996
        result['BootstrapActions'] = TEST_BA
997

998
        self.assert_params_for_cmd(cmd, result)
999

1000
    def test_bootstrap_actions_exceed_maximum_error(self):
1001
        cmd = DEFAULT_CMD + ' --bootstrap-actions'
1002
        ba_cmd = ' Path=s3://test/ba1,Name=ba1,Args=arg1,arg2,arg3'
1003
        for i in range(1, 18):
1004
            cmd += ba_cmd
1005

1006
        expected_error_msg = (
1007
            '\naws: error: maximum number of '
1008
            + 'bootstrap actions for a cluster exceeded.\n'
1009
        )
1010
        result = self.run_cmd(cmd, 255)
1011

1012
        self.assertEqual(expected_error_msg, result[1])
1013

1014
    def test_bootstrap_actions_exceed_maximum_with_applications_error(self):
1015
        cmd = (
1016
            DEFAULT_CMD
1017
            + ' --applications Name=GANGLIA Name=HBASE'
1018
            + ' Name=IMPALA,Args=arg1,arg2 --bootstrap-actions'
1019
        )
1020
        ba_cmd = ' Path=s3://test/ba1,Name=ba1,Args=arg1,arg2,arg3'
1021
        for i in range(1, 15):
1022
            cmd += ba_cmd
1023
        expected_error_msg = (
1024
            '\naws: error: maximum number of '
1025
            + 'bootstrap actions for a cluster exceeded.\n'
1026
        )
1027
        result = self.run_cmd(cmd, 255)
1028
        self.assertEqual(expected_error_msg, result[1])
1029

1030
    def test_boostrap_actions_with_default_fields(self):
1031
        cmd = DEFAULT_CMD + (
1032
            ' --bootstrap-actions Path=s3://test/ba1 Path=s3://test/ba2'
1033
        )
1034
        result = copy.deepcopy(DEFAULT_RESULT)
1035
        result['BootstrapActions'] = [
1036
            {
1037
                'Name': 'Bootstrap action',
1038
                'ScriptBootstrapAction': {'Path': 's3://test/ba1'},
1039
            },
1040
            {
1041
                'Name': 'Bootstrap action',
1042
                'ScriptBootstrapAction': {'Path': 's3://test/ba2'},
1043
            },
1044
        ]
1045
        self.assert_params_for_cmd(cmd, result)
1046

1047
    def test_bootstrap_actions_from_json_file(self):
1048
        data_path = os.path.join(
1049
            os.path.dirname(__file__), 'input_bootstrap_actions.json'
1050
        )
1051
        cmd = DEFAULT_CMD + ' --bootstrap-actions file://' + data_path
1052
        result = copy.deepcopy(DEFAULT_RESULT)
1053
        result['BootstrapActions'] = [
1054
            {
1055
                "Name": "Bootstrap Action 1",
1056
                "ScriptBootstrapAction": {
1057
                    "Path": "s3://mybucket/test1",
1058
                    "Args": ["arg1", "arg2"],
1059
                },
1060
            },
1061
            {
1062
                "Name": "Bootstrap Action 2",
1063
                "ScriptBootstrapAction": {
1064
                    "Path": "s3://mybucket/test2",
1065
                    "Args": ["arg1", "arg2"],
1066
                },
1067
            },
1068
        ]
1069
        self.assert_params_for_cmd(cmd, result)
1070

1071
    # Applications test cases
1072
    def test_install_hive_with_defaults(self):
1073
        cmd = DEFAULT_CMD + '--applications Name=Hive'
1074
        result = copy.deepcopy(DEFAULT_RESULT)
1075
        result['Steps'] = [INSTALL_HIVE_STEP]
1076
        self.assert_params_for_cmd(cmd, result)
1077

1078
    def test_install_hive_with_profile_region(self):
1079
        self.driver.session.set_config_variable('region', 'cn-north-1')
1080
        cmd = DEFAULT_CMD + '--applications Name=Hive'
1081
        HIVE_STEP = json.dumps(INSTALL_HIVE_STEP).replace(
1082
            'us-east-1', 'cn-north-1'
1083
        )
1084
        result = copy.deepcopy(DEFAULT_RESULT)
1085
        result['Steps'] = [json.loads(HIVE_STEP)]
1086
        self.assert_params_for_cmd(cmd, result)
1087

1088
    def test_install_hive_site(self):
1089
        cmdline = (
1090
            DEFAULT_CMD + '--applications Name=Hive,'
1091
            'Args=[--hive-site=s3://test/hive-conf/hive-site.xml]'
1092
        )
1093
        result = copy.deepcopy(DEFAULT_RESULT)
1094
        result['Steps'] = [INSTALL_HIVE_STEP, INSTALL_HIVE_SITE_STEP]
1095
        self.assert_params_for_cmd(cmdline, result)
1096
        cmdline = (
1097
            DEFAULT_CMD + '--applications Name=Hive,'
1098
            'Args=[--hive-site=s3://test/hive-conf/hive-site.xml,k1]'
1099
        )
1100
        self.assert_params_for_cmd(cmdline, result)
1101

1102
    def test_install_pig_with_defaults(self):
1103
        cmd = DEFAULT_CMD + '--applications Name=Pig'
1104
        result = copy.deepcopy(DEFAULT_RESULT)
1105
        result['Steps'] = [INSTALL_PIG_STEP]
1106
        self.assert_params_for_cmd(cmd, result)
1107

1108
    def test_install_ganglia(self):
1109
        cmd = DEFAULT_CMD + '--applications Name=Ganglia'
1110
        result = copy.deepcopy(DEFAULT_RESULT)
1111
        result['BootstrapActions'] = [INSTALL_GANGLIA_BA]
1112
        self.assert_params_for_cmd(cmd, result)
1113

1114
    def test_install_impala_with_defaults(self):
1115
        cmd = DEFAULT_CMD + '--applications Name=Impala'
1116
        result = copy.deepcopy(DEFAULT_RESULT)
1117
        result['BootstrapActions'] = [INSTALL_IMPALA_BA]
1118
        self.assert_params_for_cmd(cmd, result)
1119

1120
    def test_install_impala_with_all_fields(self):
1121
        cmd = DEFAULT_CMD + '--applications Name=Impala,Args=arg1,arg2'
1122
        result = copy.deepcopy(DEFAULT_RESULT)
1123
        ba = copy.deepcopy(INSTALL_IMPALA_BA)
1124
        ba['ScriptBootstrapAction']['Args'] += ['--impala-conf', 'arg1,arg2']
1125
        result['BootstrapActions'] = [ba]
1126
        self.assert_params_for_cmd(cmd, result)
1127

1128
    def test_install_hbase(self):
1129
        cmd = DEFAULT_CMD + '--applications Name=hbase'
1130
        result = copy.deepcopy(DEFAULT_RESULT)
1131
        result['BootstrapActions'] = [INSTALL_HBASE_BA]
1132
        result['Steps'] = [INSTALL_HBASE_STEP]
1133
        self.assert_params_for_cmd(cmd, result)
1134

1135
    def test_install_mapr_with_args(self):
1136
        cmd = (
1137
            DEFAULT_CMD
1138
            + '--applications Name=mapr,Args=--edition,m5,--version,3.0.2'
1139
        )
1140
        result = copy.deepcopy(DEFAULT_RESULT)
1141
        result['NewSupportedProducts'] = [INSTALL_MAPR_PRODUCT]
1142
        self.assert_params_for_cmd(cmd, result)
1143

1144
    def test_install_mapr_without_args(self):
1145
        cmd = DEFAULT_CMD + '--applications Name=mapr'
1146
        result = copy.deepcopy(DEFAULT_RESULT)
1147
        result['NewSupportedProducts'] = [{'Name': 'mapr', 'Args': []}]
1148
        self.assert_params_for_cmd(cmd, result)
1149

1150
    def test_supported_products(self):
1151
        cmd = DEFAULT_CMD + (
1152
            '--applications '
1153
            'Name=hue,Args=--hue-config,s3://elasticmapreduce/hue-config '
1154
            'Name=mapr,Args=--edition,m7 '
1155
            'Name=unknown,Args=[arg1,k1=v1]'
1156
        )
1157
        result = copy.deepcopy(DEFAULT_RESULT)
1158
        result['NewSupportedProducts'] = INSTALL_SUPPORTED_PRODUCTS
1159
        self.assert_params_for_cmd(cmd, result)
1160

1161
    def test_applications_all_types(self):
1162
        cmd = DEFAULT_CMD + (
1163
            '--applications '
1164
            'Name=hive Name=pig Name=ganglia Name=hbase Name=impala '
1165
            'Name=mapr,Args=--edition,m5,--version,3.0.2'
1166
        )
1167
        ba_list = [INSTALL_GANGLIA_BA, INSTALL_HBASE_BA, INSTALL_IMPALA_BA]
1168
        step_list = [INSTALL_HIVE_STEP, INSTALL_PIG_STEP, INSTALL_HBASE_STEP]
1169
        result = copy.deepcopy(DEFAULT_RESULT)
1170
        result['Steps'] = step_list
1171
        result['BootstrapActions'] = ba_list
1172
        result['NewSupportedProducts'] = [INSTALL_MAPR_PRODUCT]
1173
        self.assert_params_for_cmd(cmd, result)
1174

1175
    def test_applications_all_types_from_json_file(self):
1176
        data_path = os.path.join(
1177
            os.path.dirname(__file__), 'input_applications.json'
1178
        )
1179
        cmd = DEFAULT_CMD + '--applications file://' + data_path
1180
        impala_ba = copy.deepcopy(INSTALL_IMPALA_BA)
1181
        impala_ba['ScriptBootstrapAction']['Args'] += [
1182
            '--impala-conf',
1183
            'IMPALA_BACKEND_PORT=22001,IMPALA_MEM_LIMIT=70%',
1184
        ]
1185
        ba_list = [INSTALL_GANGLIA_BA, INSTALL_HBASE_BA, impala_ba]
1186
        step_list = [INSTALL_HIVE_STEP, INSTALL_PIG_STEP, INSTALL_HBASE_STEP]
1187
        result = copy.deepcopy(DEFAULT_RESULT)
1188
        result['Steps'] = step_list
1189
        result['BootstrapActions'] = ba_list
1190
        result['NewSupportedProducts'] = [INSTALL_MAPR_PRODUCT]
1191
        self.assert_params_for_cmd(cmd, result)
1192

1193
    # Steps test cases
1194
    def test_wrong_step_type_error(self):
1195
        cmd = DEFAULT_CMD + '--steps Type=unknown'
1196
        expected_error_msg = (
1197
            '\naws: error: The step type unknown is not supported.\n'
1198
        )
1199
        result = self.run_cmd(cmd, 255)
1200
        self.assertEqual(expected_error_msg, result[1])
1201

1202
    def test_default_step_type_name_action_on_failure(self):
1203
        cmd = DEFAULT_CMD + '--steps Jar=s3://mybucket/mytest.jar'
1204
        result = copy.deepcopy(DEFAULT_RESULT)
1205
        result['Steps'] = [CUSTOM_JAR_STEP]
1206
        self.assert_params_for_cmd(cmd, result)
1207

1208
    def test_custom_jar_step_missing_jar(self):
1209
        cmd = DEFAULT_CMD + '--steps Name=CustomJarMissingJar'
1210
        expect_error_msg = (
1211
            '\naws: error: The following '
1212
            + 'required parameters are missing for CustomJARStepConfig: Jar.\n'
1213
        )
1214
        result = self.run_cmd(cmd, 255)
1215
        self.assertEqual(expect_error_msg, result[1])
1216

1217
    def test_custom_jar_step_with_all_fields(self):
1218
        cmd = (
1219
            DEFAULT_CMD
1220
            + '--steps '
1221
            + (
1222
                'Name=Custom,Type=Custom_JAR,'
1223
                'Jar=s3://mybucket/mytest.jar,'
1224
                'Args=arg1,arg2,MainClass=mymainclass,'
1225
                'ActionOnFailure=TERMINATE_CLUSTER'
1226
            )
1227
        )
1228
        expected_steps = [
1229
            {
1230
                'Name': 'Custom',
1231
                'ActionOnFailure': 'TERMINATE_CLUSTER',
1232
                'HadoopJarStep': {
1233
                    'Jar': 's3://mybucket/mytest.jar',
1234
                    'Args': ['arg1', 'arg2'],
1235
                    'MainClass': 'mymainclass',
1236
                },
1237
            }
1238
        ]
1239
        result = copy.deepcopy(DEFAULT_RESULT)
1240
        result['Steps'] = expected_steps
1241
        self.assert_params_for_cmd(cmd, result)
1242

1243
    def test_streaming_step_with_default_fields(self):
1244
        cmd = DEFAULT_CMD + '--steps Type=Streaming,' + STREAMING_ARGS
1245
        result = copy.deepcopy(DEFAULT_RESULT)
1246
        result['Steps'] = [
1247
            {
1248
                'Name': 'Streaming program',
1249
                'ActionOnFailure': 'CONTINUE',
1250
                'HadoopJarStep': STREAMING_HADOOP_JAR_STEP,
1251
            }
1252
        ]
1253
        self.assert_params_for_cmd(cmd, result)
1254

1255
    def test_streaming_step_missing_args(self):
1256
        cmd = DEFAULT_CMD + '--steps Type=Streaming'
1257
        expect_error_msg = (
1258
            '\naws: error: The following '
1259
            + 'required parameters are missing for StreamingStepConfig: Args.\n'
1260
        )
1261
        result = self.run_cmd(cmd, 255)
1262
        self.assertEqual(expect_error_msg, result[1])
1263

1264
    def test_streaming_jar_with_all_fields(self):
1265
        test_step_config = (
1266
            '--steps Type=Streaming,Name=StreamingStepAllFields,'
1267
            'ActionOnFailure=CANCEL_AND_WAIT,' + STREAMING_ARGS
1268
        )
1269
        cmd = DEFAULT_CMD + test_step_config
1270
        result = copy.deepcopy(DEFAULT_RESULT)
1271
        result['Steps'] = [
1272
            {
1273
                'Name': 'StreamingStepAllFields',
1274
                'ActionOnFailure': 'CANCEL_AND_WAIT',
1275
                'HadoopJarStep': STREAMING_HADOOP_JAR_STEP,
1276
            }
1277
        ]
1278
        self.assert_params_for_cmd(cmd, result)
1279

1280
    def test_hive_step_with_default_fields(self):
1281
        cmd = DEFAULT_CMD + (
1282
            '--applications Name=Hive --steps Type=Hive,' + HIVE_BASIC_ARGS
1283
        )
1284
        result = copy.deepcopy(DEFAULT_RESULT)
1285
        result['Steps'] = [INSTALL_HIVE_STEP, HIVE_DEFAULT_STEP]
1286
        self.assert_params_for_cmd(cmd, result)
1287

1288
    def test_hive_step_missing_args(self):
1289
        cmd = DEFAULT_CMD + '--applications Name=Hive --steps Type=Hive'
1290
        expect_error_msg = (
1291
            '\naws: error: The following '
1292
            + 'required parameters are missing for HiveStepConfig: Args.\n'
1293
        )
1294
        result = self.run_cmd(cmd, 255)
1295
        self.assertEqual(expect_error_msg, result[1])
1296

1297
    def test_hive_step_with_all_fields(self):
1298
        test_step_config = (
1299
            'Type=Hive,ActionOnFailure=CANCEL_AND_WAIT,'
1300
            'Name=HiveBasicStep,' + HIVE_BASIC_ARGS
1301
        )
1302
        cmd = DEFAULT_CMD + (
1303
            '--applications Name=Hive --steps ' + test_step_config
1304
        )
1305
        result = copy.deepcopy(DEFAULT_RESULT)
1306
        install_step = copy.deepcopy(INSTALL_HIVE_STEP)
1307
        result['Steps'] = [install_step, HIVE_BASIC_STEP]
1308
        self.assert_params_for_cmd(cmd, result)
1309

1310
    def test_pig_step_with_default_fields(self):
1311
        cmd = DEFAULT_CMD + (
1312
            '--applications Name=Pig --steps Type=Pig,' + PIG_BASIC_ARGS
1313
        )
1314
        result = copy.deepcopy(DEFAULT_RESULT)
1315
        result['Steps'] = [INSTALL_PIG_STEP, PIG_DEFAULT_STEP]
1316
        self.assert_params_for_cmd(cmd, result)
1317

1318
    def test_pig_missing_args(self):
1319
        cmd = DEFAULT_CMD + '--applications Name=Pig --steps Type=Pig'
1320
        expect_error_msg = (
1321
            '\naws: error: The following '
1322
            + 'required parameters are missing for PigStepConfig: Args.\n'
1323
        )
1324
        result = self.run_cmd(cmd, 255)
1325
        self.assertEqual(expect_error_msg, result[1])
1326

1327
    def test_pig_step_with_all_fields(self):
1328
        test_step_config = (
1329
            'Name=PigBasicStep,Type=Pig,'
1330
            + PIG_BASIC_ARGS
1331
            + ',ActionOnFailure=CANCEL_AND_WAIT'
1332
        )
1333
        cmd = DEFAULT_CMD + (
1334
            '--applications Name=Pig --steps ' + test_step_config
1335
        )
1336
        result = copy.deepcopy(DEFAULT_RESULT)
1337
        install_step = copy.deepcopy(INSTALL_PIG_STEP)
1338
        result['Steps'] = [install_step, PIG_BASIC_STEP]
1339
        self.assert_params_for_cmd(cmd, result)
1340

1341
    def test_impala_step_with_default_fields(self):
1342
        cmd = DEFAULT_CMD + (
1343
            '--applications Name=Impala --steps Type=Impala,'
1344
            + IMPALA_BASIC_ARGS
1345
        )
1346
        result = copy.deepcopy(DEFAULT_RESULT)
1347
        result['BootstrapActions'] = [INSTALL_IMPALA_BA]
1348
        result['Steps'] = [IMPALA_DEFAULT_STEP]
1349
        self.assert_params_for_cmd(cmd, result)
1350

1351
    def test_impala_missing_args(self):
1352
        cmd = DEFAULT_CMD + '--applications Name=Impala --steps Type=Impala'
1353
        expect_error_msg = (
1354
            '\naws: error: The following '
1355
            + 'required parameters are missing for ImpalaStepConfig: Args.\n'
1356
        )
1357
        result = self.run_cmd(cmd, 255)
1358
        self.assertEqual(expect_error_msg, result[1])
1359

1360
    def test_impala_step_with_all_fields(self):
1361
        test_step_config = (
1362
            'Name=ImpalaBasicStep,Type=Impala,'
1363
            + IMPALA_BASIC_ARGS
1364
            + ',ActionOnFailure=CANCEL_AND_WAIT'
1365
        )
1366
        cmd = DEFAULT_CMD + (
1367
            '--applications Name=Impala --steps ' + test_step_config
1368
        )
1369
        result = copy.deepcopy(DEFAULT_RESULT)
1370
        result['BootstrapActions'] = [INSTALL_IMPALA_BA]
1371
        step = copy.deepcopy(IMPALA_DEFAULT_STEP)
1372
        step['Name'] = 'ImpalaBasicStep'
1373
        step['ActionOnFailure'] = 'CANCEL_AND_WAIT'
1374
        result['Steps'] = [step]
1375
        self.assert_params_for_cmd(cmd, result)
1376

1377
    def test_restore_from_hbase(self):
1378
        cmd = DEFAULT_CMD + (
1379
            '--applications Name=hbase --restore-from-hbase-backup '
1380
            'Dir=s3://mybucket/test,BackupVersion=test_version'
1381
        )
1382
        result = copy.deepcopy(DEFAULT_RESULT)
1383
        result['BootstrapActions'] = [INSTALL_HBASE_BA]
1384
        result['Steps'] = [
1385
            INSTALL_HBASE_STEP,
1386
            {
1387
                'Name': 'Restore HBase',
1388
                'ActionOnFailure': 'CANCEL_AND_WAIT',
1389
                'HadoopJarStep': {
1390
                    'Args': [
1391
                        'emr.hbase.backup.Main',
1392
                        '--restore',
1393
                        '--backup-dir',
1394
                        's3://mybucket/test',
1395
                        '--backup-version',
1396
                        'test_version',
1397
                    ],
1398
                    'Jar': '/home/hadoop/lib/hbase.jar',
1399
                },
1400
            },
1401
        ]
1402
        self.assert_params_for_cmd(cmd, result)
1403
        data_path = os.path.join(
1404
            os.path.dirname(__file__), 'input_hbase_restore_from_backup.json'
1405
        )
1406
        cmd = DEFAULT_CMD + (
1407
            '--applications Name=hbase --restore-from-hbase-backup '
1408
            'file://' + data_path
1409
        )
1410
        self.assert_params_for_cmd(cmd, result)
1411

1412
    def test_empty_step_args(self):
1413
        cmd = DEFAULT_CMD + '--steps Type=Streaming,Args= '
1414
        expect_error_msg = (
1415
            '\naws: error: The prameter Args cannot ' 'be an empty list.\n'
1416
        )
1417
        result = self.run_cmd(cmd, 255)
1418
        self.assertEqual(expect_error_msg, result[1])
1419

1420
        cmd = DEFAULT_CMD + '--steps Type=Pig,Args= '
1421
        result = self.run_cmd(cmd, 255)
1422
        self.assertEqual(expect_error_msg, result[1])
1423

1424
        cmd = DEFAULT_CMD + '--steps Type=Hive,Args= '
1425
        result = self.run_cmd(cmd, 255)
1426
        self.assertEqual(expect_error_msg, result[1])
1427

1428
        cmd = DEFAULT_CMD + '--steps Args= '
1429
        expect_error_msg = (
1430
            '\naws: error: The following required parameters '
1431
            'are missing for CustomJARStepConfig: Jar.\n'
1432
        )
1433
        result = self.run_cmd(cmd, 255)
1434
        self.assertEqual(expect_error_msg, result[1])
1435

1436
    def test_missing_applications_for_steps(self):
1437
        cmd = (
1438
            DEFAULT_CMD
1439
            + '--steps Jar=s3://test/customJar.jar '
1440
            + 'Type=HIVE,Args=-f,s3://test/hive '
1441
            + 'Type=PIG,Args=-f,s3://test/pig '
1442
            + 'Type=IMPALA,Args=--impala-script,s3://test/impala '
1443
            + 'Type=Streaming,Args=-files,s3://test/mapper.py,-mapper,'
1444
            + 'mapper.py,-reducer,aggregator,-input,s3://test/input,-output,'
1445
            + 's3://test/output '
1446
            + 'Type=PIG,Args=-f,s3://test/pig2 '
1447
            + 'Type=IMPALA,Args=--impala-script,s3://test/impala2 '
1448
            + 'Type=PIG,Args=-f,s3://test/pig3 '
1449
            + ' Jar=s3://test/customJar2.jar '
1450
            + ' --applications Name=Hive'
1451
        )
1452

1453
        expected_error_msg1 = (
1454
            '\naws: error: Some of the steps require the following'
1455
            ' applications to be installed: Impala, Pig. '
1456
            'Please install the applications using --applications.\n'
1457
        )
1458
        expected_error_msg2 = (
1459
            '\naws: error: Some of the steps require the following'
1460
            ' applications to be installed: Pig, Impala. '
1461
            'Please install the applications using --applications.\n'
1462
        )
1463
        result = self.run_cmd(cmd, 255)
1464

1465
        if (
1466
            result[1] == expected_error_msg1
1467
            or result[1] == expected_error_msg2
1468
        ):
1469
            self.assertTrue(True)
1470
        else:
1471
            self.assertTrue(False)
1472

1473
    def test_missing_applications_with_hbase(self):
1474
        cmd = (
1475
            DEFAULT_CMD
1476
            + '--steps Jar=s3://test/customJar.jar '
1477
            + 'Type=HIVE,Args=-f,s3://test/hive '
1478
            + 'Type=PIG,Args=-f,s3://test/pig '
1479
            + 'Type=IMPALA,Args=--impala-script,s3://test/impala '
1480
            + 'Type=Streaming,Args=-files,s3://test/mapper.py,-mapper,'
1481
            + 'mapper.py,-reducer,aggregator,-input,s3://test/input,-output,'
1482
            + 's3://test/output'
1483
            + ' --applications Name=Hive Name=Pig'
1484
            + ' --restore-from-hbase-backup Dir=s3://myBucket/myDir'
1485
        )
1486

1487
        expected_error_msg1 = (
1488
            '\naws: error: Some of the steps require the following'
1489
            ' applications to be installed: Hbase, Impala. '
1490
            'Please install the applications using --applications.\n'
1491
        )
1492
        expected_error_msg2 = (
1493
            '\naws: error: Some of the steps require the following'
1494
            ' applications to be installed: Impala, Hbase. '
1495
            'Please install the applications using --applications.\n'
1496
        )
1497
        result = self.run_cmd(cmd, 255)
1498

1499
        if (
1500
            result[1] == expected_error_msg1
1501
            or result[1] == expected_error_msg2
1502
        ):
1503
            self.assertTrue(True)
1504
        else:
1505
            self.assertTrue(False)
1506

1507
    @mock.patch('awscli.customizations.emr.emrutils.call')
1508
    def test_constructed_result(self, call_patch):
1509
        call_patch.return_value = CREATE_CLUSTER_RESULT
1510
        cmd = DEFAULT_CMD
1511
        result = self.run_cmd(cmd, expected_rc=0)
1512
        result_json = json.loads(result[0])
1513
        self.assertEqual(result_json, CONSTRUCTED_RESULT)
1514

1515
    def test_all_security_groups(self):
1516
        cmd = DEFAULT_CMD + (
1517
            '--ec2-attributes EmrManagedMasterSecurityGroup=sg-master1,'
1518
            'EmrManagedSlaveSecurityGroup=sg-slave1,'
1519
            'ServiceAccessSecurityGroup=sg-service-access,'
1520
            'AdditionalMasterSecurityGroups='
1521
            '[sg-addMaster1,sg-addMaster2,sg-addMaster3,'
1522
            'sg-addMaster4],AdditionalSlaveSecurityGroups=[sg-addSlave1,'
1523
            'sg-addSlave2,sg-addSlave3,sg-addSlave4]'
1524
        )
1525

1526
        result = copy.deepcopy(DEFAULT_RESULT)
1527
        instances = result['Instances']
1528
        instances['EmrManagedMasterSecurityGroup'] = (
1529
            EMR_MANAGED_MASTER_SECURITY_GROUP
1530
        )
1531
        instances['EmrManagedSlaveSecurityGroup'] = (
1532
            EMR_MANAGED_SLAVE_SECURITY_GROUP
1533
        )
1534
        instances['AdditionalMasterSecurityGroups'] = (
1535
            ADDITIONAL_MASTER_SECURITY_GROUPS
1536
        )
1537
        instances['ServiceAccessSecurityGroup'] = SERVICE_ACCESS_SECURITY_GROUP
1538
        instances['AdditionalSlaveSecurityGroups'] = (
1539
            ADDITIONAL_SLAVE_SECURITY_GROUPS
1540
        )
1541

1542
        self.assert_params_for_cmd(cmd, result)
1543

1544
    def test_emr_managed_security_groups(self):
1545
        cmd = DEFAULT_CMD + (
1546
            '--ec2-attributes EmrManagedMasterSecurityGroup=sg-master1,'
1547
            'EmrManagedSlaveSecurityGroup=sg-slave1,'
1548
            'ServiceAccessSecurityGroup=sg-service-access'
1549
        )
1550

1551
        result = copy.deepcopy(DEFAULT_RESULT)
1552
        instances = result['Instances']
1553
        instances['EmrManagedMasterSecurityGroup'] = (
1554
            EMR_MANAGED_MASTER_SECURITY_GROUP
1555
        )
1556
        instances['EmrManagedSlaveSecurityGroup'] = (
1557
            EMR_MANAGED_SLAVE_SECURITY_GROUP
1558
        )
1559
        instances['ServiceAccessSecurityGroup'] = SERVICE_ACCESS_SECURITY_GROUP
1560

1561
        self.assert_params_for_cmd(cmd, result)
1562

1563
    def test_additional_security_groups(self):
1564
        cmd = DEFAULT_CMD + (
1565
            '--ec2-attributes AdditionalMasterSecurityGroups=[sg-addMaster1'
1566
            ',sg-addMaster2,sg-addMaster3,sg-addMaster4],AdditionalSlaveSecu'
1567
            'rityGroups=[sg-addSlave1,sg-addSlave2,sg-addSlave3,sg-addSlave4]'
1568
        )
1569

1570
        result = copy.deepcopy(DEFAULT_RESULT)
1571
        instances = result['Instances']
1572
        instances['AdditionalMasterSecurityGroups'] = (
1573
            ADDITIONAL_MASTER_SECURITY_GROUPS
1574
        )
1575
        instances['AdditionalSlaveSecurityGroups'] = (
1576
            ADDITIONAL_SLAVE_SECURITY_GROUPS
1577
        )
1578

1579
        self.assert_params_for_cmd(cmd, result)
1580

1581
    def test_security_groups_from_json_file(self):
1582
        data_path = os.path.join(
1583
            os.path.dirname(__file__),
1584
            'input_ec2_attributes_with_security_groups.json',
1585
        )
1586
        cmd = DEFAULT_CMD + '--ec2-attributes file://' + data_path
1587

1588
        result = copy.deepcopy(DEFAULT_RESULT)
1589
        instances = result['Instances']
1590
        instances['EmrManagedMasterSecurityGroup'] = (
1591
            EMR_MANAGED_MASTER_SECURITY_GROUP
1592
        )
1593
        instances['EmrManagedSlaveSecurityGroup'] = (
1594
            EMR_MANAGED_SLAVE_SECURITY_GROUP
1595
        )
1596
        instances['ServiceAccessSecurityGroup'] = SERVICE_ACCESS_SECURITY_GROUP
1597
        instances['AdditionalMasterSecurityGroups'] = (
1598
            ADDITIONAL_MASTER_SECURITY_GROUPS
1599
        )
1600
        instances['AdditionalSlaveSecurityGroups'] = (
1601
            ADDITIONAL_SLAVE_SECURITY_GROUPS
1602
        )
1603

1604
        self.assert_params_for_cmd(cmd, result)
1605

1606
    def test_instance_group_with_autoscaling_policy(self):
1607
        cmd = (
1608
            self.prefix
1609
            + '--ami-version 3.1.0 --auto-scaling-role EMR_AUTOSCALING_ROLE --instance-groups '
1610
            + CONSTANTS.INSTANCE_GROUPS_WITH_AUTOSCALING_POLICY_ARG
1611
        )
1612
        result = {
1613
            'Name': DEFAULT_CLUSTER_NAME,
1614
            'Instances': {
1615
                'KeepJobFlowAliveWhenNoSteps': True,
1616
                'TerminationProtected': False,
1617
                'InstanceGroups': CONSTANTS.INSTANCE_GROUPS_WITH_AUTOSCALING_POLICY,
1618
            },
1619
            'AmiVersion': '3.1.0',
1620
            'AutoScalingRole': 'EMR_AUTOSCALING_ROLE',
1621
            'VisibleToAllUsers': True,
1622
            'Tags': [],
1623
        }
1624
        self.assert_params_for_cmd(cmd, result)
1625

1626
    def test_instance_group_with_autoscaling_policy_missing_autoscaling_role(
1627
        self,
1628
    ):
1629
        cmd = (
1630
            self.prefix
1631
            + '--ami-version 3.1.0 --instance-groups '
1632
            + CONSTANTS.INSTANCE_GROUPS_WITH_AUTOSCALING_POLICY_ARG
1633
        )
1634
        expected_error_msg = (
1635
            '\naws: error: Must specify --auto-scaling-role when'
1636
            ' configuring an AutoScaling policy for an instance group.\n'
1637
        )
1638
        result = self.run_cmd(cmd, 255)
1639
        self.assertEqual(expected_error_msg, result[1])
1640

1641
    def test_scale_down_behavior(self):
1642
        cmd = (
1643
            self.prefix
1644
            + '--ami-version 3.1.0 --scale-down-behavior TERMINATE_AT_TASK_COMPLETION '
1645
            '--instance-groups ' + DEFAULT_INSTANCE_GROUPS_ARG
1646
        )
1647
        result = {
1648
            'Name': DEFAULT_CLUSTER_NAME,
1649
            'Instances': DEFAULT_INSTANCES,
1650
            'AmiVersion': '3.1.0',
1651
            'VisibleToAllUsers': True,
1652
            'ScaleDownBehavior': 'TERMINATE_AT_TASK_COMPLETION',
1653
            'Tags': [],
1654
        }
1655
        self.assert_params_for_cmd(cmd, result)
1656

1657
    def test_instance_group_with_ebs_config(self):
1658
        cmd = (
1659
            self.prefix
1660
            + '--ami-version 3.1.0 --instance-groups '
1661
            + CONSTANTS.INSTANCE_GROUPS_WITH_EBS_VOLUME_ARG
1662
        )
1663
        result = {
1664
            'Name': DEFAULT_CLUSTER_NAME,
1665
            'Instances': {
1666
                'KeepJobFlowAliveWhenNoSteps': True,
1667
                'TerminationProtected': False,
1668
                'InstanceGroups': CONSTANTS.INSTANCE_GROUPS_WITH_EBS,
1669
            },
1670
            'AmiVersion': '3.1.0',
1671
            'VisibleToAllUsers': True,
1672
            'Tags': [],
1673
        }
1674
        self.assert_params_for_cmd(cmd, result)
1675

1676
    def test_instance_groups_with_ebs_config_missing_volume_type(self):
1677
        cmd = (
1678
            self.prefix
1679
            + '--ami-version 3.1.0 --instance-groups '
1680
            + CONSTANTS.INSTANCE_GROUPS_WITH_EBS_VOLUME_MISSING_VOLTYPE_ARG
1681
        )
1682
        stderr = self.run_cmd(cmd, 255)[1]
1683
        self.assert_error_message_has_field_name(stderr, 'VolumeType')
1684

1685
    def test_instance_groups_with_ebs_config_missing_size(self):
1686
        cmd = (
1687
            self.prefix
1688
            + '--ami-version 3.1.0 --instance-groups '
1689
            + CONSTANTS.INSTANCE_GROUPS_WITH_EBS_VOLUME_MISSING_SIZE_ARG
1690
        )
1691
        stderr = self.run_cmd(cmd, 255)[1]
1692
        self.assert_error_message_has_field_name(stderr, 'SizeInGB')
1693

1694
    def test_instance_groups_with_ebs_config_missing_volume_spec(self):
1695
        cmd = (
1696
            self.prefix
1697
            + '--ami-version 3.1.0 --instance-groups '
1698
            + CONSTANTS.INSTANCE_GROUPS_WITH_EBS_VOLUME_MISSING_VOLSPEC_ARG
1699
        )
1700
        result = {
1701
            'Name': DEFAULT_CLUSTER_NAME,
1702
            'Instances': {
1703
                'KeepJobFlowAliveWhenNoSteps': True,
1704
                'TerminationProtected': False,
1705
                'InstanceGroups': CONSTANTS.INSTANCE_GROUPS_WITH_EBS_VOLUME_MISSING_VOLSPEC,
1706
            },
1707
            'AmiVersion': '3.1.0',
1708
            'VisibleToAllUsers': True,
1709
            'Tags': [],
1710
        }
1711
        self.assert_params_for_cmd(cmd, result)
1712

1713
    def test_instance_groups_with_ebs_config_missing_iops(self):
1714
        cmd = (
1715
            self.prefix
1716
            + '--ami-version 3.1.0 --instance-groups '
1717
            + CONSTANTS.INSTANCE_GROUPS_WITH_EBS_VOLUME_MISSING_IOPS_ARG
1718
        )
1719
        result = {
1720
            'Name': DEFAULT_CLUSTER_NAME,
1721
            'Instances': {
1722
                'KeepJobFlowAliveWhenNoSteps': True,
1723
                'TerminationProtected': False,
1724
                'InstanceGroups': CONSTANTS.INSTANCE_GROUPS_WITH_EBS_VOLUME_MISSING_IOPS,
1725
            },
1726
            'AmiVersion': '3.1.0',
1727
            'VisibleToAllUsers': True,
1728
            'Tags': [],
1729
        }
1730
        self.assert_params_for_cmd(cmd, result)
1731

1732
    def test_instance_groups_with_ebs_config_multiple_instance_groups(self):
1733
        cmd = (
1734
            self.prefix
1735
            + '--ami-version 3.1.0 --instance-groups '
1736
            + CONSTANTS.MULTIPLE_INSTANCE_GROUPS_WITH_EBS_VOLUMES_VOLUME_ARG
1737
        )
1738
        result = {
1739
            'Name': DEFAULT_CLUSTER_NAME,
1740
            'Instances': {
1741
                'KeepJobFlowAliveWhenNoSteps': True,
1742
                'TerminationProtected': False,
1743
                'InstanceGroups': CONSTANTS.MULTIPLE_INSTANCE_GROUPS_WITH_EBS_VOLUMES,
1744
            },
1745
            'AmiVersion': '3.1.0',
1746
            'VisibleToAllUsers': True,
1747
            'Tags': [],
1748
        }
1749
        self.assert_params_for_cmd(cmd, result)
1750

1751
    def test_instance_group_with_ebs_config_from_json(self):
1752
        data_path = os.path.join(
1753
            os.path.dirname(__file__), 'input_instance_groups_ebs_config.json'
1754
        )
1755
        cmd = (
1756
            'emr create-cluster --use-default-roles --ami-version 3.0.4  '
1757
            '--instance-groups file://' + data_path
1758
        )
1759
        result = copy.deepcopy(DEFAULT_RESULT)
1760
        result['Instances']['InstanceGroups'] = [
1761
            {
1762
                'InstanceRole': 'MASTER',
1763
                'InstanceCount': 1,
1764
                'Name': 'Master Instance Group',
1765
                'Market': 'ON_DEMAND',
1766
                'InstanceType': 'd2.xlarge',
1767
                'EbsConfiguration': {
1768
                    'EbsBlockDeviceConfigs': [
1769
                        {
1770
                            'VolumeSpecification': {
1771
                                'VolumeType': 'standard',
1772
                                'SizeInGB': 10,
1773
                            },
1774
                            'VolumesPerInstance': 4,
1775
                        }
1776
                    ],
1777
                    'EbsOptimized': True,
1778
                },
1779
            },
1780
            {
1781
                'InstanceRole': 'CORE',
1782
                'InstanceCount': 2,
1783
                'Name': 'Core Instance Group',
1784
                'Market': 'ON_DEMAND',
1785
                'InstanceType': 'd2.xlarge',
1786
            },
1787
            {
1788
                'InstanceRole': 'TASK',
1789
                'InstanceCount': 3,
1790
                'Name': 'Task Instance Group',
1791
                'Market': 'SPOT',
1792
                'BidPrice': '3.45',
1793
                'InstanceType': 'd2.xlarge',
1794
            },
1795
        ]
1796
        self.assert_params_for_cmd(cmd, result)
1797

1798
    def test_instance_fleets_with_on_demand_master_only(self):
1799
        cmd = (
1800
            self.prefix
1801
            + '--ami-version 3.1.0 --instance-fleets '
1802
            + CONSTANTS_FLEET.INSTANCE_FLEETS_WITH_ON_DEMAND_MASTER_ONLY
1803
        )
1804
        result = {
1805
            'Name': DEFAULT_CLUSTER_NAME,
1806
            'Instances': {
1807
                'KeepJobFlowAliveWhenNoSteps': True,
1808
                'TerminationProtected': False,
1809
                'InstanceFleets': CONSTANTS_FLEET.RES_INSTANCE_FLEETS_WITH_ON_DEMAND_MASTER_ONLY,
1810
            },
1811
            'AmiVersion': '3.1.0',
1812
            'VisibleToAllUsers': True,
1813
            'Tags': [],
1814
        }
1815
        self.assert_params_for_cmd(cmd, result)
1816

1817
    def test_instance_fleets_with_on_demand_master_only_with_targeted_odcr(
1818
        self,
1819
    ):
1820
        cmd = (
1821
            self.prefix
1822
            + '--ami-version 3.1.0 --instance-fleets '
1823
            + CONSTANTS_FLEET.INSTANCE_FLEETS_WITH_ON_DEMAND_MASTER_ONLY_WITH_TARGETED_ODCR
1824
        )
1825
        result = {
1826
            'Name': DEFAULT_CLUSTER_NAME,
1827
            'Instances': {
1828
                'KeepJobFlowAliveWhenNoSteps': True,
1829
                'TerminationProtected': False,
1830
                'InstanceFleets': CONSTANTS_FLEET.RES_INSTANCE_FLEETS_WITH_ON_DEMAND_MASTER_ONLY_WITH_TARGETED_ODCR,
1831
            },
1832
            'AmiVersion': '3.1.0',
1833
            'VisibleToAllUsers': True,
1834
            'Tags': [],
1835
        }
1836
        self.assert_params_for_cmd(cmd, result)
1837

1838
    def test_instance_fleets_with_spot_master_only(self):
1839
        cmd = (
1840
            self.prefix
1841
            + '--ami-version 3.1.0 --instance-fleets '
1842
            + CONSTANTS_FLEET.INSTANCE_FLEETS_WITH_SPOT_MASTER_ONLY
1843
        )
1844
        result = {
1845
            'Name': DEFAULT_CLUSTER_NAME,
1846
            'Instances': {
1847
                'KeepJobFlowAliveWhenNoSteps': True,
1848
                'TerminationProtected': False,
1849
                'InstanceFleets': CONSTANTS_FLEET.RES_INSTANCE_FLEETS_WITH_SPOT_MASTER_ONLY,
1850
            },
1851
            'AmiVersion': '3.1.0',
1852
            'VisibleToAllUsers': True,
1853
            'Tags': [],
1854
        }
1855
        self.assert_params_for_cmd(cmd, result)
1856

1857
    def test_instance_fleets_with_spot_master_only_with_ebs_conf(self):
1858
        cmd = (
1859
            self.prefix
1860
            + '--ami-version 3.1.0 --instance-fleets '
1861
            + CONSTANTS_FLEET.INSTANCE_FLEETS_WITH_SPOT_MASTER_ONLY_WITH_EBS_CONF
1862
        )
1863
        result = {
1864
            'Name': DEFAULT_CLUSTER_NAME,
1865
            'Instances': {
1866
                'KeepJobFlowAliveWhenNoSteps': True,
1867
                'TerminationProtected': False,
1868
                'InstanceFleets': CONSTANTS_FLEET.RES_INSTANCE_FLEETS_WITH_SPOT_MASTER_ONLY_WITH_EBS_CONF,
1869
            },
1870
            'AmiVersion': '3.1.0',
1871
            'VisibleToAllUsers': True,
1872
            'Tags': [],
1873
        }
1874
        self.assert_params_for_cmd(cmd, result)
1875

1876
    def test_instance_fleets_with_spot_master_specific_azs(self):
1877
        cmd = (
1878
            self.prefix
1879
            + '--ami-version 3.1.0 --instance-fleets '
1880
            + CONSTANTS_FLEET.INSTANCE_FLEETS_WITH_SPOT_MASTER_ONLY
1881
            + ' --ec2-attributes AvailabilityZones=[us-east-1a,us-east-1b]'
1882
        )
1883
        result = {
1884
            'Name': DEFAULT_CLUSTER_NAME,
1885
            'Instances': {
1886
                'KeepJobFlowAliveWhenNoSteps': True,
1887
                'TerminationProtected': False,
1888
                'InstanceFleets': CONSTANTS_FLEET.RES_INSTANCE_FLEETS_WITH_SPOT_MASTER_ONLY,
1889
                'Placement': {
1890
                    'AvailabilityZones': ['us-east-1a', 'us-east-1b']
1891
                },
1892
            },
1893
            'AmiVersion': '3.1.0',
1894
            'VisibleToAllUsers': True,
1895
            'Tags': [],
1896
        }
1897
        self.assert_params_for_cmd(cmd, result)
1898

1899
    def test_instance_fleets_with_spot_master_subnet_ids(self):
1900
        cmd = (
1901
            self.prefix
1902
            + '--ami-version 3.1.0 --instance-fleets '
1903
            + CONSTANTS_FLEET.INSTANCE_FLEETS_WITH_SPOT_MASTER_ONLY
1904
            + ' --ec2-attributes SubnetIds=[subnetid-1,subnetid-2]'
1905
        )
1906
        result = {
1907
            'Name': DEFAULT_CLUSTER_NAME,
1908
            'Instances': {
1909
                'KeepJobFlowAliveWhenNoSteps': True,
1910
                'TerminationProtected': False,
1911
                'InstanceFleets': CONSTANTS_FLEET.RES_INSTANCE_FLEETS_WITH_SPOT_MASTER_ONLY,
1912
                'Ec2SubnetIds': ['subnetid-1', 'subnetid-2'],
1913
            },
1914
            'AmiVersion': '3.1.0',
1915
            'VisibleToAllUsers': True,
1916
            'Tags': [],
1917
        }
1918
        self.assert_params_for_cmd(cmd, result)
1919

1920
    def test_instance_fleets_with_spot_master_core_cluster_multiple_instance_types(
1921
        self,
1922
    ):
1923
        cmd = (
1924
            self.prefix
1925
            + '--ami-version 3.1.0 --instance-fleets '
1926
            + CONSTANTS_FLEET.INSTANCE_FLEETS_WITH_SPOT_MASTER_CORE_CLUSTER
1927
        )
1928
        result = {
1929
            'Name': DEFAULT_CLUSTER_NAME,
1930
            'Instances': {
1931
                'KeepJobFlowAliveWhenNoSteps': True,
1932
                'TerminationProtected': False,
1933
                'InstanceFleets': CONSTANTS_FLEET.RES_INSTANCE_FLEETS_WITH_SPOT_MASTER_CORE_CLUSTER,
1934
            },
1935
            'AmiVersion': '3.1.0',
1936
            'VisibleToAllUsers': True,
1937
            'Tags': [],
1938
        }
1939
        self.assert_params_for_cmd(cmd, result)
1940

1941
    def test_instance_fleets_with_spot_master_core_cluster_multiple_custom_amis(
1942
        self,
1943
    ):
1944
        cmd = (
1945
            self.prefix
1946
            + '--ami-version 3.1.0 --instance-fleets '
1947
            + CONSTANTS_FLEET.INSTANCE_FLEETS_WITH_SPOT_MASTER_CORE_CLUSTER_WITH_CUSTOM_AMI
1948
        )
1949
        result = {
1950
            'Name': DEFAULT_CLUSTER_NAME,
1951
            'Instances': {
1952
                'KeepJobFlowAliveWhenNoSteps': True,
1953
                'TerminationProtected': False,
1954
                'InstanceFleets': CONSTANTS_FLEET.RES_INSTANCE_FLEETS_WITH_SPOT_MASTER_CORE_CLUSTER_WITH_CUSTOM_AMI,
1955
            },
1956
            'AmiVersion': '3.1.0',
1957
            'VisibleToAllUsers': True,
1958
            'Tags': [],
1959
        }
1960
        self.assert_params_for_cmd(cmd, result)
1961

1962
    def test_instance_fleets_with_complex_config_from_json(self):
1963
        data_path = os.path.join(
1964
            os.path.dirname(__file__), 'input_instance_fleets.json'
1965
        )
1966
        cmd = (
1967
            'emr create-cluster --use-default-roles --ami-version 3.1.0  '
1968
            '--instance-fleets file://' + data_path
1969
        )
1970
        result = {
1971
            'Name': DEFAULT_CLUSTER_NAME,
1972
            'Instances': {
1973
                'KeepJobFlowAliveWhenNoSteps': True,
1974
                'TerminationProtected': False,
1975
                'InstanceFleets': CONSTANTS_FLEET.RES_INSTANCE_FLEETS_WITH_COMPLEX_CONFIG_FROM_JSON,
1976
            },
1977
            'AmiVersion': '3.1.0',
1978
            'VisibleToAllUsers': True,
1979
            'Tags': [],
1980
            'JobFlowRole': 'EMR_EC2_DefaultRole',
1981
            'ServiceRole': 'EMR_DefaultRole',
1982
        }
1983
        self.assert_params_for_cmd(cmd, result)
1984

1985
    def test_instance_fleets_with_both_az_azs_specified(self):
1986
        cmd = (
1987
            self.prefix
1988
            + '--ami-version 3.1.0 --instance-fleets '
1989
            + CONSTANTS_FLEET.INSTANCE_FLEETS_WITH_SPOT_MASTER_ONLY
1990
            + ' --ec2-attributes AvailabilityZone=us-east-1a,AvailabilityZones=[us-east-1a,us-east-1b]'
1991
        )
1992
        expected_error_msg = (
1993
            '\naws: error: You cannot specify both AvailabilityZone'
1994
            ' and AvailabilityZones options together.\n'
1995
        )
1996
        result = self.run_cmd(cmd, 255)
1997
        self.assertEqual(expected_error_msg, result[1])
1998

1999

2000
if __name__ == "__main__":
2001
    unittest.main()
2002

2003
Product

Resources

Company
