CoCalc -- test_create_cluster_release

GitHub Repository: aws/aws-cli
Path: blob/develop/tests/unit/customizations/emr/test_create_cluster_release_label.py
¹⁵⁶⁹ views
1
# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2
#
3
# Licensed under the Apache License, Version 2.0 (the "License"). You
4
# may not use this file except in compliance with the License. A copy of
5
# the License is located at
6
#
7
#     http://aws.amazon.com/apache2.0/
8
#
9
# or in the "license" file accompanying this file. This file is
10
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11
# ANY KIND, either express or implied. See the License for the specific
12
# language governing permissions and limitations under the License.
13

14
import copy
15
import os
16

17
from botocore.compat import OrderedDict, json
18

19
from awscli.testutils import mock, unittest
20
from tests.unit.customizations.emr import (
21
    EMRBaseAWSCommandParamsTest as BaseAWSCommandParamsTest,
22
)
23
from tests.unit.customizations.emr import test_constants as CONSTANTS
24
from tests.unit.customizations.emr import (
25
    test_constants_instance_fleets as CONSTANTS_FLEET,
26
)
27

28
DEFAULT_CLUSTER_NAME = "Development Cluster"
29

30
DEFAULT_INSTANCE_GROUPS_ARG = (
31
    'InstanceGroupType=MASTER,Name=MASTER,'
32
    'InstanceCount=1,InstanceType=m1.large '
33
    'InstanceGroupType=CORE,Name=CORE,'
34
    'InstanceCount=1,InstanceType=m1.large '
35
    'InstanceGroupType=TASK,Name=TASK,'
36
    'InstanceCount=1,InstanceType=m1.large '
37
)
38

39
HA_INSTANCE_GROUPS_ARG = (
40
    'InstanceGroupType=MASTER,Name=MASTER,'
41
    'InstanceCount=3,InstanceType=m1.large '
42
    'InstanceGroupType=CORE,Name=CORE,'
43
    'InstanceCount=1,InstanceType=m1.large '
44
)
45

46
DEFAULT_INSTANCE_GROUPS = [
47
    {
48
        'InstanceRole': 'MASTER',
49
        'InstanceCount': 1,
50
        'Name': 'MASTER',
51
        'Market': 'ON_DEMAND',
52
        'InstanceType': 'm1.large',
53
    },
54
    {
55
        'InstanceRole': 'CORE',
56
        'InstanceCount': 1,
57
        'Name': 'CORE',
58
        'Market': 'ON_DEMAND',
59
        'InstanceType': 'm1.large',
60
    },
61
    {
62
        'InstanceRole': 'TASK',
63
        'InstanceCount': 1,
64
        'Name': 'TASK',
65
        'Market': 'ON_DEMAND',
66
        'InstanceType': 'm1.large',
67
    },
68
]
69

70
HA_INSTANCE_GROUPS = [
71
    {
72
        'InstanceRole': 'MASTER',
73
        'InstanceCount': 3,
74
        'Name': 'MASTER',
75
        'Market': 'ON_DEMAND',
76
        'InstanceType': 'm1.large',
77
    },
78
    {
79
        'InstanceRole': 'CORE',
80
        'InstanceCount': 1,
81
        'Name': 'CORE',
82
        'Market': 'ON_DEMAND',
83
        'InstanceType': 'm1.large',
84
    },
85
]
86

87

88
DEFAULT_CMD = (
89
    'emr create-cluster --release-label emr-4.0.0'
90
    ' --use-default-roles'
91
    ' --instance-groups ' + DEFAULT_INSTANCE_GROUPS_ARG + ' '
92
)
93

94
DEFAULT_INSTANCES = {
95
    'KeepJobFlowAliveWhenNoSteps': True,
96
    'TerminationProtected': False,
97
    'InstanceGroups': DEFAULT_INSTANCE_GROUPS,
98
}
99

100
HA_INSTANCES = {
101
    'KeepJobFlowAliveWhenNoSteps': True,
102
    'TerminationProtected': False,
103
    'InstanceGroups': HA_INSTANCE_GROUPS,
104
}
105

106
EC2_ROLE_NAME = "EMR_EC2_DefaultRole"
107
EMR_ROLE_NAME = "EMR_DefaultRole"
108

109
DEFAULT_KERBEROS_ATTRIBUTES_ARGS = (
110
    'Realm=EC2.INTERNAL,'
111
    'KdcAdminPassword=123,'
112
    'CrossRealmTrustPrincipalPassword=123,'
113
    'ADDomainJoinUser=aws,'
114
    'ADDomainJoinPassword=123'
115
)
116

117
KERBEROS_ATTRIBUTES = {
118
    'Realm': 'EC2.INTERNAL',
119
    'KdcAdminPassword': '123',
120
    'CrossRealmTrustPrincipalPassword': '123',
121
    'ADDomainJoinUser': 'aws',
122
    'ADDomainJoinPassword': '123',
123
}
124

125
TEST_BA = [
126
    {
127
        'ScriptBootstrapAction': {
128
            'Path': 's3://test/ba1',
129
            'Args': ['arg1', 'arg2', 'arg3'],
130
        },
131
        'Name': 'ba1',
132
    },
133
    {
134
        'ScriptBootstrapAction': {
135
            'Path': 's3://test/ba2',
136
            'Args': ['arg1', 'arg2', 'arg3'],
137
        },
138
        'Name': 'ba2',
139
    },
140
]
141

142
CUSTOM_JAR_STEP = {
143
    'Name': 'Custom JAR',
144
    'ActionOnFailure': 'CONTINUE',
145
    'HadoopJarStep': {'Jar': 's3://mybucket/mytest.jar'},
146
}
147

148
STREAMING_ARGS = (
149
    'Args=-files,'
150
    's3://elasticmapreduce/samples/wordcount/wordSplitter.py,'
151
    '-mapper,wordSplitter.py,'
152
    '-reducer,aggregate,'
153
    '-input,s3://elasticmapreduce/samples/wordcount/input,'
154
    '-output,s3://mybucket/wordcount/output/2014-04-18/12-15-24'
155
)
156

157
STREAMING_HADOOP_JAR_STEP = {
158
    'Jar': 'command-runner.jar',
159
    'Args': [
160
        'hadoop-streaming',
161
        '-files',
162
        's3://elasticmapreduce/samples/wordcount/wordSplitter.py',
163
        '-mapper',
164
        'wordSplitter.py',
165
        '-reducer',
166
        'aggregate',
167
        '-input',
168
        's3://elasticmapreduce/samples/wordcount/input',
169
        '-output',
170
        's3://mybucket/wordcount/output/2014-04-18/12-15-24',
171
    ],
172
}
173

174
CREATE_CLUSTER_RESULT = {
175
    "JobFlowId": "j-XXXX",
176
    "ClusterArn": "arn:aws:elasticmapreduce:region:012345678910:cluster/j-XXXX",
177
}
178

179
CONSTRUCTED_RESULT = {
180
    "ClusterId": "j-XXXX",
181
    "ClusterArn": "arn:aws:elasticmapreduce:region:012345678910:cluster/j-XXXX",
182
}
183

184
DEFAULT_RESULT = {
185
    'Name': DEFAULT_CLUSTER_NAME,
186
    'Instances': DEFAULT_INSTANCES,
187
    'ReleaseLabel': 'emr-4.0.0',
188
    'VisibleToAllUsers': True,
189
    'JobFlowRole': EC2_ROLE_NAME,
190
    'ServiceRole': EMR_ROLE_NAME,
191
    'Tags': [],
192
}
193

194
EMR_MANAGED_MASTER_SECURITY_GROUP = 'sg-master1'
195

196
EMR_MANAGED_SLAVE_SECURITY_GROUP = 'sg-slave1'
197

198
SERVICE_ACCESS_SECURITY_GROUP = "sg-service-access"
199

200
ADDITIONAL_MASTER_SECURITY_GROUPS = [
201
    'sg-addMaster1',
202
    'sg-addMaster2',
203
    'sg-addMaster3',
204
    'sg-addMaster4',
205
]
206

207
ADDITIONAL_SLAVE_SECURITY_GROUPS = [
208
    'sg-addSlave1',
209
    'sg-addSlave2',
210
    'sg-addSlave3',
211
    'sg-addSlave4',
212
]
213

214
HIVE_BASIC_ARGS = (
215
    'Args=-f,s3://elasticmapreduce/samples/hive-ads/libs/model-build.q'
216
)
217

218
HIVE_DEFAULT_STEP = {
219
    'Name': 'Hive program',
220
    'ActionOnFailure': 'CONTINUE',
221
    'HadoopJarStep': {
222
        'Jar': ('command-runner.jar'),
223
        'Args': [
224
            'hive-script',
225
            '--run-hive-script',
226
            '--args',
227
            '-f',
228
            's3://elasticmapreduce/samples/hive-ads/libs/model-build.q',
229
        ],
230
    },
231
}
232

233
HIVE_STEP = {
234
    'Name': 'HiveBasicStep',
235
    'ActionOnFailure': 'CANCEL_AND_WAIT',
236
    'HadoopJarStep': {
237
        'Jar': ('command-runner.jar'),
238
        'Args': [
239
            'hive-script',
240
            '--run-hive-script',
241
            '--args',
242
            '-f',
243
            's3://elasticmapreduce/samples/hive-ads/libs/model-build.q',
244
        ],
245
    },
246
}
247

248
PIG_BASIC_ARGS = (
249
    'Args=-f,' + 's3://elasticmapreduce/samples/pig-apache/do-reports2.pig'
250
)
251

252
PIG_DEFAULT_STEP = {
253
    'Name': 'Pig program',
254
    'ActionOnFailure': 'CONTINUE',
255
    'HadoopJarStep': {
256
        'Jar': ('command-runner.jar'),
257
        'Args': [
258
            'pig-script',
259
            '--run-pig-script',
260
            '--args',
261
            '-f',
262
            's3://elasticmapreduce/samples/' 'pig-apache/do-reports2.pig',
263
        ],
264
    },
265
}
266

267
PIG_STEP = {
268
    'Name': 'PigBasicStep',
269
    'ActionOnFailure': 'CANCEL_AND_WAIT',
270
    'HadoopJarStep': {
271
        'Jar': ('command-runner.jar'),
272
        'Args': [
273
            'pig-script',
274
            '--run-pig-script',
275
            '--args',
276
            '-f',
277
            's3://elasticmapreduce/samples/' 'pig-apache/do-reports2.pig',
278
        ],
279
    },
280
}
281

282

283
class TestCreateCluster(BaseAWSCommandParamsTest):
284
    prefix = 'emr create-cluster '
285

286
    def test_ami_version_release_label_exclusive_validation(self):
287
        cmd = (
288
            self.prefix + '--ami-version 3.8.0 --release-label '
289
            'emr-4.0.0 --instance-groups ' + DEFAULT_INSTANCE_GROUPS_ARG
290
        )
291
        expected_error_msg = (
292
            '\naws: error: You cannot specify both --ami-version'
293
            ' and --release-label options together.\n'
294
        )
295
        result = self.run_cmd(cmd, 255)
296
        self.assertEqual(expected_error_msg, result[1])
297

298
    def test_if_ami_version_or_release_label_is_provided(self):
299
        cmd = self.prefix + ' --instance-groups ' + DEFAULT_INSTANCE_GROUPS_ARG
300
        expected_error_msg = (
301
            '\naws: error: Either --ami-version or'
302
            ' --release-label is required.\n'
303
        )
304
        result = self.run_cmd(cmd, 255)
305
        self.assertEqual(expected_error_msg, result[1])
306

307
    def test_no_special_steps_added_for_applications(self):
308
        cmd = DEFAULT_CMD + '--applications Name=MapR'
309
        result = copy.deepcopy(DEFAULT_RESULT)
310
        result['Applications'] = [{'Name': 'MapR'}]
311
        self.assert_params_for_cmd(cmd, result)
312

313
        cmd = DEFAULT_CMD + '--applications Name=HUE'
314
        result = copy.deepcopy(DEFAULT_RESULT)
315
        result['Applications'] = [{'Name': 'HUE'}]
316
        self.assert_params_for_cmd(cmd, result)
317

318
        cmd = DEFAULT_CMD + '--applications Name=HIVE'
319
        result = copy.deepcopy(DEFAULT_RESULT)
320
        result['Applications'] = [{'Name': 'HIVE'}]
321
        self.assert_params_for_cmd(cmd, result)
322

323
        cmd = DEFAULT_CMD + '--applications Name=PIG'
324
        result = copy.deepcopy(DEFAULT_RESULT)
325
        result['Applications'] = [{'Name': 'PIG'}]
326
        self.assert_params_for_cmd(cmd, result)
327

328
        cmd = DEFAULT_CMD + '--applications Name=HBASE'
329
        result = copy.deepcopy(DEFAULT_RESULT)
330
        result['Applications'] = [{'Name': 'HBASE'}]
331
        self.assert_params_for_cmd(cmd, result)
332

333
        cmd = DEFAULT_CMD + '--applications Name=IMPALA'
334
        result = copy.deepcopy(DEFAULT_RESULT)
335
        result['Applications'] = [{'Name': 'IMPALA'}]
336
        self.assert_params_for_cmd(cmd, result)
337

338
        cmd = DEFAULT_CMD + '--applications Name=GANGLIA'
339
        result = copy.deepcopy(DEFAULT_RESULT)
340
        result['Applications'] = [{'Name': 'GANGLIA'}]
341
        self.assert_params_for_cmd(cmd, result)
342

343
    def test_configurations(self):
344
        cmd = (
345
            DEFAULT_CMD + ' --configurations '
346
            '[{"Classification":"hadoop-env","Properties":{'
347
            '"someProperty":"someValue"},"Configurations":[]}]'
348
        )
349
        result = copy.deepcopy(DEFAULT_RESULT)
350
        result['Configurations'] = [
351
            {
352
                'Classification': 'hadoop-env',
353
                'Configurations': [],
354
                'Properties': {'someProperty': 'someValue'},
355
            }
356
        ]
357
        self.assert_params_for_cmd(cmd, result)
358

359
    def test_quick_start(self):
360
        cmd = (
361
            self.prefix
362
            + '--release-label emr-4.0.0 --instance-groups '
363
            + DEFAULT_INSTANCE_GROUPS_ARG
364
        )
365
        result = {
366
            'Name': DEFAULT_CLUSTER_NAME,
367
            'Instances': DEFAULT_INSTANCES,
368
            'ReleaseLabel': 'emr-4.0.0',
369
            'VisibleToAllUsers': True,
370
            'Tags': [],
371
        }
372
        self.assert_params_for_cmd(cmd, result)
373

374
    def assert_error_message_has_field_name(self, error_msg, field_name):
375
        self.assertIn('Missing required parameter', error_msg)
376
        self.assertIn(field_name, error_msg)
377

378
    def test_default_cmd(self):
379
        self.assert_params_for_cmd(DEFAULT_CMD, DEFAULT_RESULT)
380

381
    def test_cluster_without_service_role_and_instance_profile(self):
382
        cmd = (
383
            'emr create-cluster --release-label emr-4.0.0 '
384
            '--instance-groups ' + DEFAULT_INSTANCE_GROUPS_ARG
385
        )
386
        result = copy.deepcopy(DEFAULT_RESULT)
387
        del result['JobFlowRole']
388
        del result['ServiceRole']
389
        self.assert_params_for_cmd(cmd, result)
390

391
    def test_cluster_with_service_role_and_instance_profile(self):
392
        cmd = (
393
            'emr create-cluster --release-label emr-4.0.0'
394
            ' --service-role ServiceRole --ec2-attributes '
395
            'InstanceProfile=Ec2_InstanceProfile '
396
            '--instance-groups ' + DEFAULT_INSTANCE_GROUPS_ARG
397
        )
398
        result = copy.deepcopy(DEFAULT_RESULT)
399
        result['JobFlowRole'] = 'Ec2_InstanceProfile'
400
        result['ServiceRole'] = 'ServiceRole'
401
        self.assert_params_for_cmd(cmd, result)
402

403
    def test_mutual_exclusive_use_default_roles_and_service_role(self):
404
        cmd = (
405
            DEFAULT_CMD
406
            + '--ec2-attributes InstanceProfile=Ec2_InstanceProfile'
407
        )
408
        expected_error_msg = (
409
            '\naws: error: You cannot specify both --use-default-roles '
410
            'and --ec2-attributes InstanceProfile options together. Either '
411
            'choose --use-default-roles or use both --service-role <roleName>'
412
            ' and --ec2-attributes InstanceProfile=<profileName>.\n'
413
        )
414
        result = self.run_cmd(cmd, 255)
415
        self.assertEqual(expected_error_msg, result[1])
416

417
    def test_mutual_exclusive_use_default_roles_and_instance_profile(self):
418
        cmd = (
419
            DEFAULT_CMD + '--service-role ServiceRole '
420
            '--ec2-attributes InstanceProfile=Ec2_InstanceProfile'
421
        )
422
        expected_error_msg = (
423
            '\naws: error: You cannot specify both --use-default-roles '
424
            'and --service-role options together. Either choose '
425
            '--use-default-roles or use both --service-role <roleName> '
426
            'and --ec2-attributes InstanceProfile=<profileName>.\n'
427
        )
428
        result = self.run_cmd(cmd, 255)
429
        self.assertEqual(expected_error_msg, result[1])
430

431
    def test_cluster_name_no_space(self):
432
        cmd = DEFAULT_CMD + '--name MyCluster'
433
        result = copy.deepcopy(DEFAULT_RESULT)
434
        result['Name'] = 'MyCluster'
435
        self.assert_params_for_cmd(cmd, result)
436

437
    def test_cluster_name_with_space(self):
438
        cmd = DEFAULT_CMD.split() + ['--name', 'My Cluster']
439
        result = copy.deepcopy(DEFAULT_RESULT)
440
        result['Name'] = 'My Cluster'
441
        self.assert_params_for_cmd(cmd, result)
442

443
    def test_ami_version(self):
444
        cmd = DEFAULT_CMD + '--release-label emr-4.0.0'
445
        result = copy.deepcopy(DEFAULT_RESULT)
446
        result['ReleaseLabel'] = 'emr-4.0.0'
447
        self.assert_params_for_cmd(cmd, result)
448

449
    def test_log_uri(self):
450
        test_log_uri = 's3://test/logs'
451
        cmd = DEFAULT_CMD + '--log-uri ' + test_log_uri
452
        result = copy.deepcopy(DEFAULT_RESULT)
453
        result['LogUri'] = test_log_uri
454
        self.assert_params_for_cmd(cmd, result)
455

456
    def test_additional_info(self):
457
        test_info = '{ami32: "ami-82e305f5"}'
458
        cmd = DEFAULT_CMD.split() + ['--additional-info', test_info]
459
        result = copy.deepcopy(DEFAULT_RESULT)
460
        result['AdditionalInfo'] = test_info
461
        self.assert_params_for_cmd(cmd, result)
462

463
    def test_auto_terminte(self):
464
        cmd = (
465
            'emr create-cluster --use-default-roles'
466
            ' --release-label emr-4.0.0 '
467
            '--auto-terminate --instance-groups ' + DEFAULT_INSTANCE_GROUPS_ARG
468
        )
469
        result = copy.deepcopy(DEFAULT_RESULT)
470
        instances = copy.deepcopy(DEFAULT_INSTANCES)
471
        instances['KeepJobFlowAliveWhenNoSteps'] = False
472
        result['Instances'] = instances
473
        self.assert_params_for_cmd(cmd, result)
474

475
    def test_auto_terminate_and_no_auto_terminate(self):
476
        cmd = (
477
            DEFAULT_CMD
478
            + '--release-label emr-4.0.0 '
479
            + '--auto-terminate --no-auto-terminate'
480
        )
481
        expected_error_msg = (
482
            '\naws: error: cannot use both --no-auto-terminate and'
483
            ' --auto-terminate options together.\n'
484
        )
485
        result = self.run_cmd(cmd, 255)
486
        self.assertEqual(expected_error_msg, result[1])
487

488
    def test_termination_protected(self):
489
        cmd = DEFAULT_CMD + '--termination-protected'
490
        result = copy.deepcopy(DEFAULT_RESULT)
491
        instances = copy.deepcopy(DEFAULT_INSTANCES)
492
        instances['TerminationProtected'] = True
493
        result['Instances'] = instances
494
        self.assert_params_for_cmd(cmd, result)
495

496
    def test_no_termination_protected(self):
497
        cmd = DEFAULT_CMD + '--no-termination-protected'
498
        self.assert_params_for_cmd(cmd, DEFAULT_RESULT)
499

500
    def test_termination_protected_and_no_termination_protected(self):
501
        cmd = (
502
            DEFAULT_CMD + '--termination-protected --no-termination-protected'
503
        )
504
        expected_error_msg = (
505
            '\naws: error: cannot use both --termination-protected'
506
            ' and --no-termination-protected options together.\n'
507
        )
508
        result = self.run_cmd(cmd, 255)
509
        self.assertEqual(expected_error_msg, result[1])
510

511
    def test_unhealthy_node_replacement(self):
512
        cmd = DEFAULT_CMD + '--unhealthy-node-replacement'
513
        result = copy.deepcopy(DEFAULT_RESULT)
514
        instances = copy.deepcopy(DEFAULT_INSTANCES)
515
        instances['UnhealthyNodeReplacement'] = True
516
        result['Instances'] = instances
517
        self.assert_params_for_cmd(cmd, result)
518

519
    def test_no_unhealthy_node_replacement(self):
520
        cmd = DEFAULT_CMD + '--no-unhealthy-node-replacement'
521
        result = copy.deepcopy(DEFAULT_RESULT)
522
        instances = copy.deepcopy(DEFAULT_INSTANCES)
523
        instances['UnhealthyNodeReplacement'] = False
524
        result['Instances'] = instances
525
        self.assert_params_for_cmd(cmd, result)
526

527
    def test_unhealthy_node_replacement_and_no_unhealthy_node_replacement(
528
        self,
529
    ):
530
        cmd = (
531
            DEFAULT_CMD
532
            + '--unhealthy-node-replacement --no-unhealthy-node-replacement'
533
        )
534
        expected_error_msg = (
535
            '\naws: error: cannot use both --unhealthy-node-replacement'
536
            ' and --no-unhealthy-node-replacement options together.\n'
537
        )
538
        result = self.run_cmd(cmd, 255)
539
        self.assertEqual(expected_error_msg, result[1])
540

541
    def test_visible_to_all_users(self):
542
        cmd = DEFAULT_CMD + '--visible-to-all-users'
543
        self.assert_params_for_cmd(cmd, DEFAULT_RESULT)
544

545
    def test_no_visible_to_all_users(self):
546
        cmd = DEFAULT_CMD + '--no-visible-to-all-users'
547
        result = copy.deepcopy(DEFAULT_RESULT)
548
        result['VisibleToAllUsers'] = False
549
        self.assert_params_for_cmd(cmd, result)
550

551
    def test_visible_to_all_users_and_no_visible_to_all_users(self):
552
        cmd = DEFAULT_CMD + '--visible-to-all-users --no-visible-to-all-users'
553
        expected_error_msg = (
554
            '\naws: error: cannot use both --visible-to-all-users and '
555
            '--no-visible-to-all-users options together.\n'
556
        )
557
        result = self.run_cmd(cmd, 255)
558
        self.assertEqual(expected_error_msg, result[1])
559

560
    def test_extended_support(self):
561
        cmd = DEFAULT_CMD + '--extended-support'
562
        result = copy.deepcopy(DEFAULT_RESULT)
563
        result['ExtendedSupport'] = True
564
        self.assert_params_for_cmd(cmd, result)
565

566
    def test_no_extended_support(self):
567
        cmd = DEFAULT_CMD + '--no-extended-support'
568
        result = copy.deepcopy(DEFAULT_RESULT)
569
        result['ExtendedSupport'] = False
570
        self.assert_params_for_cmd(cmd, result)
571

572
    def test_extended_support_and_no_extended_support(self):
573
        cmd = DEFAULT_CMD + '--extended-support --no-extended-support'
574
        expected_error_msg = (
575
            '\naws: error: cannot use both --extended-support'
576
            ' and --no-extended-support options together.\n'
577
        )
578
        result = self.run_cmd(cmd, 255)
579
        self.assertEqual(expected_error_msg, result[1])
580

581
    def test_tags(self):
582
        cmd = DEFAULT_CMD.split() + ['--tags', 'k1=v1', 'k2', 'k3=spaces  v3']
583
        result = copy.deepcopy(DEFAULT_RESULT)
584
        tags = [
585
            {'Key': 'k1', 'Value': 'v1'},
586
            {'Key': 'k2', 'Value': ''},
587
            {'Key': 'k3', 'Value': 'spaces  v3'},
588
        ]
589
        result['Tags'] = tags
590
        self.assert_params_for_cmd(cmd, result)
591

592
    def test_enable_debugging(self):
593
        cmd = DEFAULT_CMD + '--log-uri s3://test/logs --enable-debugging'
594
        result = copy.deepcopy(DEFAULT_RESULT)
595
        result['LogUri'] = 's3://test/logs'
596
        debugging_config = [
597
            {
598
                'Name': 'Setup Hadoop Debugging',
599
                'ActionOnFailure': 'TERMINATE_CLUSTER',
600
                'HadoopJarStep': {
601
                    'Args': ['state-pusher-script'],
602
                    'Jar': 'command-runner.jar',
603
                },
604
            }
605
        ]
606
        result['Steps'] = debugging_config
607
        self.assert_params_for_cmd(cmd, result)
608

609
        cmd = DEFAULT_CMD + (
610
            '--log-uri s3://test/logs --enable-debugging ' '--region us-west-2'
611
        )
612
        debugging_config = [
613
            {
614
                'Name': 'Setup Hadoop Debugging',
615
                'ActionOnFailure': 'TERMINATE_CLUSTER',
616
                'HadoopJarStep': {
617
                    'Args': ['state-pusher-script'],
618
                    'Jar': 'command-runner.jar',
619
                },
620
            }
621
        ]
622
        result['Steps'] = debugging_config
623
        self.assert_params_for_cmd(cmd, result)
624

625
    def test_enable_debugging_no_log_uri(self):
626
        cmd = DEFAULT_CMD + '--enable-debugging'
627
        expected_error_msg = (
628
            '\naws: error: LogUri not specified. You must specify a logUri'
629
            ' if you enable debugging when creating a cluster.\n'
630
        )
631
        result = self.run_cmd(cmd, 255)
632
        self.assertEqual(expected_error_msg, result[1])
633

634
    def test_enable_debugging_and_no_enable_debugging(self):
635
        cmd = (
636
            DEFAULT_CMD
637
            + '--enable-debugging --no-enable-debugging'
638
            + ' --log-uri s3://test/logs'
639
        )
640
        expected_error_msg = (
641
            '\naws: error: cannot use both --enable-debugging and '
642
            '--no-enable-debugging options together.\n'
643
        )
644
        result = self.run_cmd(cmd, 255)
645
        self.assertEqual(expected_error_msg, result[1])
646

647
    def test_instance_groups_default_name_market(self):
648
        cmd = (
649
            'emr create-cluster --use-default-roles --release-label emr-4.0.0 '
650
            '--instance-groups '
651
            'InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m1.large '
652
            'InstanceGroupType=CORE,InstanceCount=1,InstanceType=m1.large '
653
            'InstanceGroupType=TASK,InstanceCount=1,InstanceType=m1.large '
654
        )
655
        self.assert_params_for_cmd(cmd, DEFAULT_RESULT)
656

657
    def test_instance_groups_instance_group_type_mismatch_cases(self):
658
        cmd = (
659
            'emr create-cluster --use-default-roles --release-label emr-4.0.0 '
660
            '--instance-groups '
661
            'Name=MASTER,InstanceGroupType=MaSter,InstanceCount=1,'
662
            'InstanceType=m1.large Name=CORE,InstanceGroupType=cORE,'
663
            'InstanceCount=1,InstanceType=m1.large Name=TASK,'
664
            'InstanceGroupType=tAsK,InstanceCount=1,InstanceType=m1.large'
665
        )
666
        self.assert_params_for_cmd(cmd, DEFAULT_RESULT)
667

668
    def test_instance_groups_instance_type_and_count(self):
669
        cmd = (
670
            'emr create-cluster --use-default-roles --release-label emr-4.0.0 '
671
            '--instance-type m1.large'
672
        )
673
        expected_result = copy.deepcopy(DEFAULT_RESULT)
674
        expected_result['Instances'] = {
675
            'KeepJobFlowAliveWhenNoSteps': True,
676
            'TerminationProtected': False,
677
            'InstanceGroups': [
678
                {
679
                    'InstanceRole': 'MASTER',
680
                    'InstanceCount': 1,
681
                    'Name': 'MASTER',
682
                    'Market': 'ON_DEMAND',
683
                    'InstanceType': 'm1.large',
684
                }
685
            ],
686
        }
687
        self.assert_params_for_cmd(cmd, expected_result)
688
        cmd = (
689
            'emr create-cluster --use-default-roles --release-label emr-4.0.0 '
690
            '--instance-type m1.large --instance-count 3'
691
        )
692
        expected_result = copy.deepcopy(DEFAULT_RESULT)
693
        expected_result['Instances'] = {
694
            'KeepJobFlowAliveWhenNoSteps': True,
695
            'TerminationProtected': False,
696
            'InstanceGroups': [
697
                {
698
                    'InstanceRole': 'MASTER',
699
                    'InstanceCount': 1,
700
                    'Name': 'MASTER',
701
                    'Market': 'ON_DEMAND',
702
                    'InstanceType': 'm1.large',
703
                },
704
                {
705
                    'InstanceRole': 'CORE',
706
                    'InstanceCount': 2,
707
                    'Name': 'CORE',
708
                    'Market': 'ON_DEMAND',
709
                    'InstanceType': 'm1.large',
710
                },
711
            ],
712
        }
713
        self.assert_params_for_cmd(cmd, expected_result)
714

715
    def test_instance_groups_missing_required_parameter_error(self):
716
        cmd = (
717
            'emr create-cluster --use-default-roles --release-label'
718
            ' emr-4.0.0 '
719
        )
720
        expect_error_msg = (
721
            '\naws: error: Must specify either --instance-groups or '
722
            '--instance-type with --instance-count(optional) to '
723
            'configure instance groups.\n'
724
        )
725
        result = self.run_cmd(cmd, 255)
726
        self.assertEqual(expect_error_msg, result[1])
727

728
        cmd = (
729
            'emr create-cluster --use-default-roles --release-label emr-4.0.0 '
730
            '--instance-count 2'
731
        )
732
        expect_error_msg = (
733
            '\naws: error: Must specify either --instance-groups or '
734
            '--instance-type with --instance-count(optional) to '
735
            'configure instance groups.\n'
736
        )
737
        result = self.run_cmd(cmd, 255)
738
        self.assertEqual(expect_error_msg, result[1])
739

740
    def test_instance_groups_exclusive_parameter_validation_error(self):
741
        cmd = (
742
            'emr create-cluster --use-default-roles --release-label 4.0.0 '
743
            '--instance-type m1.large --instance-groups '
744
            + DEFAULT_INSTANCE_GROUPS_ARG
745
        )
746
        expect_error_msg = (
747
            '\naws: error: You may not specify --instance-type '
748
            'or --instance-count with --instance-groups, '
749
            'because --instance-type and --instance-count are '
750
            'shortcut options for --instance-groups.\n'
751
        )
752
        result = self.run_cmd(cmd, 255)
753
        self.assertEqual(expect_error_msg, result[1])
754

755
        cmd = (
756
            'emr create-cluster --use-default-roles --release-label emr-4.0.0 '
757
            '--instance-type m1.large --instance-count 2 '
758
            '--instance-groups ' + DEFAULT_INSTANCE_GROUPS_ARG
759
        )
760
        expect_error_msg = (
761
            '\naws: error: You may not specify --instance-type '
762
            'or --instance-count with --instance-groups, '
763
            'because --instance-type and --instance-count are '
764
            'shortcut options for --instance-groups.\n'
765
        )
766
        result = self.run_cmd(cmd, 255)
767
        self.assertEqual(expect_error_msg, result[1])
768

769
    def test_instance_groups_missing_instance_group_type_error(self):
770
        cmd = (
771
            'emr create-cluster --use-default-roles --release-label emr-4.0.0 '
772
            '--auto-terminate '
773
            '--instance-groups '
774
            'Name=Master,InstanceCount=1,InstanceType=m1.small'
775
        )
776
        stderr = self.run_cmd(cmd, 255)[1]
777
        self.assert_error_message_has_field_name(stderr, 'InstanceGroupType')
778

779
    def test_instance_groups_missing_instance_type_error(self):
780
        cmd = (
781
            'emr create-cluster --use-default-roles --release-label emr-4.0.0 '
782
            '--auto-terminate '
783
            '--instance-groups '
784
            'Name=Master,InstanceGroupType=MASTER,InstanceCount=1'
785
        )
786
        stderr = self.run_cmd(cmd, 255)[1]
787
        self.assert_error_message_has_field_name(stderr, 'InstanceType')
788

789
    def test_instance_groups_missing_instance_count_error(self):
790
        cmd = (
791
            'emr create-cluster --use-default-roles --release-label emr-4.0.0 '
792
            '--auto-terminate '
793
            '--instance-groups '
794
            'Name=Master,InstanceGroupType=MASTER,InstanceType=m1.xlarge'
795
        )
796
        stderr = self.run_cmd(cmd, 255)[1]
797
        self.assert_error_message_has_field_name(stderr, 'InstanceCount')
798

799
    def test_instance_groups_from_json_file(self):
800
        data_path = os.path.join(
801
            os.path.dirname(__file__), 'input_instance_groups.json'
802
        )
803
        cmd = (
804
            'emr create-cluster --use-default-roles'
805
            ' --release-label emr-4.0.0  '
806
            '--instance-groups file://' + data_path
807
        )
808
        result = copy.deepcopy(DEFAULT_RESULT)
809
        result['Instances']['InstanceGroups'] = [
810
            {
811
                'InstanceRole': 'MASTER',
812
                'InstanceCount': 1,
813
                'Name': 'Master Instance Group',
814
                'Market': 'ON_DEMAND',
815
                'InstanceType': 'm1.large',
816
            },
817
            {
818
                'InstanceRole': 'CORE',
819
                'InstanceCount': 2,
820
                'Name': 'Core Instance Group',
821
                'Market': 'ON_DEMAND',
822
                'InstanceType': 'm1.xlarge',
823
            },
824
            {
825
                'InstanceRole': 'TASK',
826
                'InstanceCount': 3,
827
                'Name': 'Task Instance Group',
828
                'Market': 'SPOT',
829
                'BidPrice': '3.45',
830
                'InstanceType': 'm1.xlarge',
831
            },
832
        ]
833
        self.assert_params_for_cmd(cmd, result)
834

835
    def test_instance_groups_from_json_file_spot_bidprice_equals_ondemandprice(
836
        self,
837
    ):
838
        data_path = os.path.join(
839
            os.path.dirname(__file__),
840
            'input_instance_groups_spot_bidprice_equals_ondemandprice.json',
841
        )
842
        cmd = (
843
            'emr create-cluster --use-default-roles'
844
            ' --release-label emr-4.0.0  '
845
            '--instance-groups file://' + data_path
846
        )
847
        result = copy.deepcopy(DEFAULT_RESULT)
848
        result['Instances']['InstanceGroups'] = [
849
            {
850
                'InstanceRole': 'MASTER',
851
                'InstanceCount': 1,
852
                'Name': 'Master Instance Group',
853
                'Market': 'SPOT',
854
                'InstanceType': 'm1.large',
855
            },
856
            {
857
                'InstanceRole': 'CORE',
858
                'InstanceCount': 2,
859
                'Name': 'Core Instance Group',
860
                'Market': 'SPOT',
861
                'InstanceType': 'm1.xlarge',
862
            },
863
            {
864
                'InstanceRole': 'TASK',
865
                'InstanceCount': 3,
866
                'Name': 'Task Instance Group',
867
                'Market': 'SPOT',
868
                'InstanceType': 'm1.xlarge',
869
            },
870
        ]
871
        self.assert_params_for_cmd(cmd, result)
872

873
    def test_instance_groups_adds_configurations(self):
874
        data_path = os.path.join(
875
            os.path.dirname(__file__),
876
            'input_instance_groups_with_configurations.json',
877
        )
878
        cmd = (
879
            'emr create-cluster --use-default-roles'
880
            ' --release-label emr-4.0.0 '
881
            '--instance-groups file://' + data_path
882
        )
883
        result = copy.deepcopy(DEFAULT_RESULT)
884
        result['Instances']['InstanceGroups'][1]['Configurations'] = [
885
            OrderedDict(
886
                [
887
                    ("Classification", "hdfs-site"),
888
                    (
889
                        "Properties",
890
                        OrderedDict(
891
                            [
892
                                ("test-key1", "test-value1"),
893
                                ("test-key2", "test-value2"),
894
                            ]
895
                        ),
896
                    ),
897
                ]
898
            )
899
        ]
900
        self.assert_params_for_cmd(cmd, result)
901

902
    def test_ec2_attributes_no_az(self):
903
        cmd = (
904
            'emr create-cluster --release-label emr-4.0.0 '
905
            '--instance-groups '
906
            + DEFAULT_INSTANCE_GROUPS_ARG
907
            + ' --ec2-attributes KeyName=testkey,SubnetId=subnet-123456,'
908
            'InstanceProfile=EMR_EC2_DefaultRole '
909
            '--service-role EMR_DefaultRole'
910
        )
911
        result = copy.deepcopy(DEFAULT_RESULT)
912
        result['Instances']['Ec2KeyName'] = 'testkey'
913
        result['Instances']['Ec2SubnetId'] = 'subnet-123456'
914
        result['JobFlowRole'] = 'EMR_EC2_DefaultRole'
915
        self.assert_params_for_cmd(cmd, result)
916

917
    def test_ec2_attributes_az(self):
918
        cmd = DEFAULT_CMD + '--ec2-attributes AvailabilityZone=us-east-1a'
919
        result = copy.deepcopy(DEFAULT_RESULT)
920
        result['Instances']['Placement'] = {'AvailabilityZone': 'us-east-1a'}
921
        self.assert_params_for_cmd(cmd, result)
922

923
    def test_ec2_attributes_subnet_az_error(self):
924
        cmd = (
925
            DEFAULT_CMD
926
            + '--ec2-attributes '
927
            + 'SubnetId=subnet-123456,AvailabilityZone=us-east-1a'
928
        )
929
        expect_error_msg = (
930
            '\naws: error: You may not specify both a SubnetId and an Availab'
931
            'ilityZone (placement) because ec2SubnetId implies a placement.\n'
932
        )
933
        result = self.run_cmd(cmd, 255)
934
        self.assertEqual(expect_error_msg, result[1])
935

936
    def test_ec2_attributes_with_subnet_from_json_file(self):
937
        data_path = os.path.join(
938
            os.path.dirname(__file__), 'input_ec2_attributes_with_subnet.json'
939
        )
940
        cmd = (
941
            'emr create-cluster --release-label emr-4.0.0 '
942
            '--instance-groups '
943
            + DEFAULT_INSTANCE_GROUPS_ARG
944
            + ' --ec2-attributes file://'
945
            + data_path
946
            + ' --service-role EMR_DefaultRole'
947
        )
948
        result = copy.deepcopy(DEFAULT_RESULT)
949
        result['Instances']['Ec2KeyName'] = 'testkey'
950
        result['Instances']['Ec2SubnetId'] = 'subnet-123456'
951
        result['JobFlowRole'] = 'EMR_EC2_DefaultRole'
952
        self.assert_params_for_cmd(cmd, result)
953

954
    def test_ec2_attributes_with_az_from_json_file(self):
955
        data_path = os.path.join(
956
            os.path.dirname(__file__), 'input_ec2_attributes_with_az.json'
957
        )
958
        cmd = (
959
            'emr create-cluster --release-label emr-4.0.0 '
960
            '--instance-groups '
961
            + DEFAULT_INSTANCE_GROUPS_ARG
962
            + ' --ec2-attributes file://'
963
            + data_path
964
            + ' --service-role EMR_DefaultRole'
965
        )
966
        result = copy.deepcopy(DEFAULT_RESULT)
967
        result['Instances']['Ec2KeyName'] = 'testkey'
968
        result['Instances']['Placement'] = {'AvailabilityZone': 'us-east-1a'}
969
        result['JobFlowRole'] = 'EMR_EC2_DefaultRole'
970
        self.assert_params_for_cmd(cmd, result)
971

972
    # Bootstrap Actions test cases
973
    def test_bootstrap_actions_missing_path_error(self):
974
        cmd = DEFAULT_CMD + '--bootstrap-actions Name=ba1,Args=arg1,arg2'
975
        stderr = self.run_cmd(cmd, 255)[1]
976
        self.assert_error_message_has_field_name(stderr, 'Path')
977

978
    def test_bootstrap_actions_with_all_fields(self):
979
        cmd = DEFAULT_CMD + (
980
            ' --bootstrap-actions '
981
            'Path=s3://test/ba1,Name=ba1,Args=arg1,arg2,arg3 '
982
            'Path=s3://test/ba2,Name=ba2,Args=arg1,arg2,arg3'
983
        )
984
        result = copy.deepcopy(DEFAULT_RESULT)
985
        result['BootstrapActions'] = TEST_BA
986

987
        self.assert_params_for_cmd(cmd, result)
988

989
    def test_bootstrap_actions_exceed_maximum_error(self):
990
        cmd = DEFAULT_CMD + ' --bootstrap-actions'
991
        ba_cmd = ' Path=s3://test/ba1,Name=ba1,Args=arg1,arg2,arg3'
992
        for i in range(1, 18):
993
            cmd += ba_cmd
994

995
        expected_error_msg = (
996
            '\naws: error: maximum number of '
997
            + 'bootstrap actions for a cluster exceeded.\n'
998
        )
999
        result = self.run_cmd(cmd, 255)
1000

1001
        self.assertEqual(expected_error_msg, result[1])
1002

1003
    def test_bootstrap_actions_exceed_maximum_with_applications_error(self):
1004
        cmd = DEFAULT_CMD + ' --bootstrap-actions'
1005
        ba_cmd = ' Path=s3://test/ba1,Name=ba1,Args=arg1,arg2,arg3'
1006
        for i in range(1, 20):
1007
            cmd += ba_cmd
1008
        expected_error_msg = (
1009
            '\naws: error: maximum number of '
1010
            + 'bootstrap actions for a cluster exceeded.\n'
1011
        )
1012
        result = self.run_cmd(cmd, 255)
1013
        self.assertEqual(expected_error_msg, result[1])
1014

1015
    def test_boostrap_actions_with_default_fields(self):
1016
        cmd = DEFAULT_CMD + (
1017
            ' --bootstrap-actions Path=s3://test/ba1 Path=s3://test/ba2'
1018
        )
1019
        result = copy.deepcopy(DEFAULT_RESULT)
1020
        result['BootstrapActions'] = [
1021
            {
1022
                'Name': 'Bootstrap action',
1023
                'ScriptBootstrapAction': {'Path': 's3://test/ba1'},
1024
            },
1025
            {
1026
                'Name': 'Bootstrap action',
1027
                'ScriptBootstrapAction': {'Path': 's3://test/ba2'},
1028
            },
1029
        ]
1030
        self.assert_params_for_cmd(cmd, result)
1031

1032
    def test_bootstrap_actions_from_json_file(self):
1033
        data_path = os.path.join(
1034
            os.path.dirname(__file__), 'input_bootstrap_actions.json'
1035
        )
1036
        cmd = DEFAULT_CMD + ' --bootstrap-actions file://' + data_path
1037
        result = copy.deepcopy(DEFAULT_RESULT)
1038
        result['BootstrapActions'] = [
1039
            {
1040
                "Name": "Bootstrap Action 1",
1041
                "ScriptBootstrapAction": {
1042
                    "Path": "s3://mybucket/test1",
1043
                    "Args": ["arg1", "arg2"],
1044
                },
1045
            },
1046
            {
1047
                "Name": "Bootstrap Action 2",
1048
                "ScriptBootstrapAction": {
1049
                    "Path": "s3://mybucket/test2",
1050
                    "Args": ["arg1", "arg2"],
1051
                },
1052
            },
1053
        ]
1054
        self.assert_params_for_cmd(cmd, result)
1055

1056
    # Steps test cases
1057
    def test_wrong_step_type_error(self):
1058
        cmd = DEFAULT_CMD + '--steps Type=unknown'
1059
        expected_error_msg = (
1060
            '\naws: error: The step type unknown is not supported.\n'
1061
        )
1062
        result = self.run_cmd(cmd, 255)
1063
        self.assertEqual(expected_error_msg, result[1])
1064

1065
    def test_default_step_type_name_action_on_failure(self):
1066
        cmd = DEFAULT_CMD + '--steps Jar=s3://mybucket/mytest.jar'
1067
        result = copy.deepcopy(DEFAULT_RESULT)
1068
        result['Steps'] = [CUSTOM_JAR_STEP]
1069
        self.assert_params_for_cmd(cmd, result)
1070

1071
    def test_custom_jar_step_missing_jar(self):
1072
        cmd = DEFAULT_CMD + '--steps Name=CustomJarMissingJar'
1073
        expect_error_msg = (
1074
            '\naws: error: The following '
1075
            + 'required parameters are missing for CustomJARStepConfig: Jar.\n'
1076
        )
1077
        result = self.run_cmd(cmd, 255)
1078
        self.assertEqual(expect_error_msg, result[1])
1079

1080
    def test_custom_jar_step_with_all_fields(self):
1081
        cmd = (
1082
            DEFAULT_CMD
1083
            + '--steps '
1084
            + (
1085
                'Name=Custom,Type=Custom_JAR,'
1086
                'Jar=s3://mybucket/mytest.jar,'
1087
                'Args=arg1,arg2,MainClass=mymainclass,'
1088
                'ActionOnFailure=TERMINATE_CLUSTER'
1089
            )
1090
        )
1091
        expected_steps = [
1092
            {
1093
                'Name': 'Custom',
1094
                'ActionOnFailure': 'TERMINATE_CLUSTER',
1095
                'HadoopJarStep': {
1096
                    'Jar': 's3://mybucket/mytest.jar',
1097
                    'Args': ['arg1', 'arg2'],
1098
                    'MainClass': 'mymainclass',
1099
                },
1100
            }
1101
        ]
1102
        result = copy.deepcopy(DEFAULT_RESULT)
1103
        result['Steps'] = expected_steps
1104
        self.assert_params_for_cmd(cmd, result)
1105

1106
    def test_streaming_step_with_default_fields(self):
1107
        cmd = DEFAULT_CMD + '--steps Type=Streaming,' + STREAMING_ARGS
1108
        result = copy.deepcopy(DEFAULT_RESULT)
1109
        result['Steps'] = [
1110
            {
1111
                'Name': 'Streaming program',
1112
                'ActionOnFailure': 'CONTINUE',
1113
                'HadoopJarStep': STREAMING_HADOOP_JAR_STEP,
1114
            }
1115
        ]
1116
        self.assert_params_for_cmd(cmd, result)
1117

1118
    def test_streaming_step_missing_args(self):
1119
        cmd = DEFAULT_CMD + '--steps Type=Streaming'
1120
        expect_error_msg = (
1121
            '\naws: error: The following '
1122
            + 'required parameters are missing for StreamingStepConfig: Args.\n'
1123
        )
1124
        result = self.run_cmd(cmd, 255)
1125
        self.assertEqual(expect_error_msg, result[1])
1126

1127
    def test_streaming_jar_with_all_fields(self):
1128
        test_step_config = (
1129
            '--steps Type=Streaming,Name=StreamingStepAllFields,'
1130
            'ActionOnFailure=CANCEL_AND_WAIT,' + STREAMING_ARGS
1131
        )
1132
        cmd = DEFAULT_CMD + test_step_config
1133
        result = copy.deepcopy(DEFAULT_RESULT)
1134
        result['Steps'] = [
1135
            {
1136
                'Name': 'StreamingStepAllFields',
1137
                'ActionOnFailure': 'CANCEL_AND_WAIT',
1138
                'HadoopJarStep': STREAMING_HADOOP_JAR_STEP,
1139
            }
1140
        ]
1141
        self.assert_params_for_cmd(cmd, result)
1142

1143
    def test_hive_step_with_default_fields(self):
1144
        cmd = DEFAULT_CMD + (
1145
            '--applications Name=Hive --steps Type=Hive,' + HIVE_BASIC_ARGS
1146
        )
1147
        result = copy.deepcopy(DEFAULT_RESULT)
1148
        result['Applications'] = [{'Name': 'Hive'}]
1149
        result['Steps'] = [HIVE_DEFAULT_STEP]
1150
        self.assert_params_for_cmd(cmd, result)
1151

1152
    def test_hive_step_missing_args(self):
1153
        cmd = DEFAULT_CMD + '--applications Name=Hive --steps Type=Hive'
1154
        expect_error_msg = (
1155
            '\naws: error: The following '
1156
            + 'required parameters are missing for HiveStepConfig: Args.\n'
1157
        )
1158
        result = self.run_cmd(cmd, 255)
1159
        self.assertEqual(expect_error_msg, result[1])
1160

1161
    def test_hive_step_with_all_fields(self):
1162
        test_step_config = (
1163
            'Type=Hive,ActionOnFailure=CANCEL_AND_WAIT,'
1164
            'Name=HiveBasicStep,' + HIVE_BASIC_ARGS
1165
        )
1166
        cmd = DEFAULT_CMD + (
1167
            '--applications Name=Hive --steps ' + test_step_config
1168
        )
1169
        result = copy.deepcopy(DEFAULT_RESULT)
1170
        result['Steps'] = [HIVE_STEP]
1171
        result['Applications'] = [{'Name': 'Hive'}]
1172
        self.assert_params_for_cmd(cmd, result)
1173

1174
    def test_pig_step_with_default_fields(self):
1175
        cmd = DEFAULT_CMD + (
1176
            '--applications Name=Pig --steps Type=Pig,' + PIG_BASIC_ARGS
1177
        )
1178
        result = copy.deepcopy(DEFAULT_RESULT)
1179
        result['Steps'] = [PIG_DEFAULT_STEP]
1180
        result['Applications'] = [{'Name': 'Pig'}]
1181
        self.assert_params_for_cmd(cmd, result)
1182

1183
    def test_pig_missing_args(self):
1184
        cmd = DEFAULT_CMD + '--applications Name=Pig --steps Type=Pig'
1185
        expect_error_msg = (
1186
            '\naws: error: The following '
1187
            + 'required parameters are missing for PigStepConfig: Args.\n'
1188
        )
1189
        result = self.run_cmd(cmd, 255)
1190
        self.assertEqual(expect_error_msg, result[1])
1191

1192
    def test_pig_step_with_all_fields(self):
1193
        test_step_config = (
1194
            'Name=PigBasicStep,Type=Pig,'
1195
            + PIG_BASIC_ARGS
1196
            + ',ActionOnFailure=CANCEL_AND_WAIT'
1197
        )
1198
        cmd = DEFAULT_CMD + (
1199
            '--applications Name=Pig --steps ' + test_step_config
1200
        )
1201
        result = copy.deepcopy(DEFAULT_RESULT)
1202
        result['Applications'] = [{'Name': 'Pig'}]
1203
        result['Steps'] = [PIG_STEP]
1204
        self.assert_params_for_cmd(cmd, result)
1205

1206
    @mock.patch('awscli.customizations.emr.emrutils.call')
1207
    def test_constructed_result(self, call_patch):
1208
        call_patch.return_value = CREATE_CLUSTER_RESULT
1209
        cmd = DEFAULT_CMD
1210
        result = self.run_cmd(cmd, expected_rc=0)
1211
        result_json = json.loads(result[0])
1212
        self.assertEqual(result_json, CONSTRUCTED_RESULT)
1213

1214
    def test_all_security_groups(self):
1215
        cmd = DEFAULT_CMD + (
1216
            '--ec2-attributes EmrManagedMasterSecurityGroup=sg-master1,'
1217
            'EmrManagedSlaveSecurityGroup=sg-slave1,'
1218
            'ServiceAccessSecurityGroup=sg-service-access,'
1219
            'AdditionalMasterSecurityGroups='
1220
            '[sg-addMaster1,sg-addMaster2,sg-addMaster3,'
1221
            'sg-addMaster4],AdditionalSlaveSecurityGroups=[sg-addSlave1,'
1222
            'sg-addSlave2,sg-addSlave3,sg-addSlave4]'
1223
        )
1224

1225
        result = copy.deepcopy(DEFAULT_RESULT)
1226
        instances = result['Instances']
1227
        instances['EmrManagedMasterSecurityGroup'] = (
1228
            EMR_MANAGED_MASTER_SECURITY_GROUP
1229
        )
1230
        instances['EmrManagedSlaveSecurityGroup'] = (
1231
            EMR_MANAGED_SLAVE_SECURITY_GROUP
1232
        )
1233
        instances['ServiceAccessSecurityGroup'] = SERVICE_ACCESS_SECURITY_GROUP
1234
        instances['AdditionalMasterSecurityGroups'] = (
1235
            ADDITIONAL_MASTER_SECURITY_GROUPS
1236
        )
1237
        instances['AdditionalSlaveSecurityGroups'] = (
1238
            ADDITIONAL_SLAVE_SECURITY_GROUPS
1239
        )
1240

1241
        self.assert_params_for_cmd(cmd, result)
1242

1243
    def test_emr_managed_security_groups(self):
1244
        cmd = DEFAULT_CMD + (
1245
            '--ec2-attributes EmrManagedMasterSecurityGroup=sg-master1,'
1246
            'EmrManagedSlaveSecurityGroup=sg-slave1,'
1247
            'ServiceAccessSecurityGroup=sg-service-access'
1248
        )
1249

1250
        result = copy.deepcopy(DEFAULT_RESULT)
1251
        instances = result['Instances']
1252
        instances['EmrManagedMasterSecurityGroup'] = (
1253
            EMR_MANAGED_MASTER_SECURITY_GROUP
1254
        )
1255
        instances['EmrManagedSlaveSecurityGroup'] = (
1256
            EMR_MANAGED_SLAVE_SECURITY_GROUP
1257
        )
1258
        instances['ServiceAccessSecurityGroup'] = SERVICE_ACCESS_SECURITY_GROUP
1259

1260
        self.assert_params_for_cmd(cmd, result)
1261

1262
    def test_additional_security_groups(self):
1263
        cmd = DEFAULT_CMD + (
1264
            '--ec2-attributes AdditionalMasterSecurityGroups=[sg-addMaster1'
1265
            ',sg-addMaster2,sg-addMaster3,sg-addMaster4],AdditionalSlaveSecu'
1266
            'rityGroups=[sg-addSlave1,sg-addSlave2,sg-addSlave3,sg-addSlave4]'
1267
        )
1268

1269
        result = copy.deepcopy(DEFAULT_RESULT)
1270
        instances = result['Instances']
1271
        instances['AdditionalMasterSecurityGroups'] = (
1272
            ADDITIONAL_MASTER_SECURITY_GROUPS
1273
        )
1274
        instances['AdditionalSlaveSecurityGroups'] = (
1275
            ADDITIONAL_SLAVE_SECURITY_GROUPS
1276
        )
1277

1278
        self.assert_params_for_cmd(cmd, result)
1279

1280
    def test_security_groups_from_json_file(self):
1281
        data_path = os.path.join(
1282
            os.path.dirname(__file__),
1283
            'input_ec2_attributes_with_security_groups.json',
1284
        )
1285
        cmd = DEFAULT_CMD + '--ec2-attributes file://' + data_path
1286

1287
        result = copy.deepcopy(DEFAULT_RESULT)
1288
        instances = result['Instances']
1289
        instances['EmrManagedMasterSecurityGroup'] = (
1290
            EMR_MANAGED_MASTER_SECURITY_GROUP
1291
        )
1292
        instances['EmrManagedSlaveSecurityGroup'] = (
1293
            EMR_MANAGED_SLAVE_SECURITY_GROUP
1294
        )
1295
        instances['ServiceAccessSecurityGroup'] = SERVICE_ACCESS_SECURITY_GROUP
1296
        instances['AdditionalMasterSecurityGroups'] = (
1297
            ADDITIONAL_MASTER_SECURITY_GROUPS
1298
        )
1299
        instances['AdditionalSlaveSecurityGroups'] = (
1300
            ADDITIONAL_SLAVE_SECURITY_GROUPS
1301
        )
1302

1303
        self.assert_params_for_cmd(cmd, result)
1304

1305
    def test_instance_group_with_autoscaling_policy(self):
1306
        cmd = (
1307
            self.prefix
1308
            + '--release-label emr-4.2.0 --auto-scaling-role EMR_AUTOSCALING_ROLE --instance-groups '
1309
            + CONSTANTS.INSTANCE_GROUPS_WITH_AUTOSCALING_POLICY_ARG
1310
        )
1311
        result = {
1312
            'Name': DEFAULT_CLUSTER_NAME,
1313
            'Instances': {
1314
                'KeepJobFlowAliveWhenNoSteps': True,
1315
                'TerminationProtected': False,
1316
                'InstanceGroups': CONSTANTS.INSTANCE_GROUPS_WITH_AUTOSCALING_POLICY,
1317
            },
1318
            'ReleaseLabel': 'emr-4.2.0',
1319
            'AutoScalingRole': 'EMR_AUTOSCALING_ROLE',
1320
            'VisibleToAllUsers': True,
1321
            'Tags': [],
1322
        }
1323
        self.assert_params_for_cmd(cmd, result)
1324

1325
    def test_instance_group_with_autoscaling_policy_missing_autoscaling_role(
1326
        self,
1327
    ):
1328
        cmd = (
1329
            self.prefix
1330
            + '--release-label emr-4.2.0 --instance-groups '
1331
            + CONSTANTS.INSTANCE_GROUPS_WITH_AUTOSCALING_POLICY_ARG
1332
        )
1333
        expected_error_msg = (
1334
            '\naws: error: Must specify --auto-scaling-role when'
1335
            ' configuring an AutoScaling policy for an instance group.\n'
1336
        )
1337
        result = self.run_cmd(cmd, 255)
1338
        self.assertEqual(expected_error_msg, result[1])
1339

1340
    def test_scale_down_behavior(self):
1341
        cmd = (
1342
            self.prefix
1343
            + '--release-label emr-4.0.0 --scale-down-behavior TERMINATE_AT_INSTANCE_HOUR '
1344
            '--instance-groups ' + DEFAULT_INSTANCE_GROUPS_ARG
1345
        )
1346
        result = {
1347
            'Name': DEFAULT_CLUSTER_NAME,
1348
            'Instances': DEFAULT_INSTANCES,
1349
            'ReleaseLabel': 'emr-4.0.0',
1350
            'VisibleToAllUsers': True,
1351
            'ScaleDownBehavior': 'TERMINATE_AT_INSTANCE_HOUR',
1352
            'Tags': [],
1353
        }
1354
        self.assert_params_for_cmd(cmd, result)
1355

1356
    def test_instance_group_with_ebs_config(self):
1357
        cmd = (
1358
            self.prefix
1359
            + '--release-label emr-4.2.0 --instance-groups '
1360
            + CONSTANTS.INSTANCE_GROUPS_WITH_EBS_VOLUME_ARG
1361
        )
1362
        result = {
1363
            'Name': DEFAULT_CLUSTER_NAME,
1364
            'Instances': {
1365
                'KeepJobFlowAliveWhenNoSteps': True,
1366
                'TerminationProtected': False,
1367
                'InstanceGroups': CONSTANTS.INSTANCE_GROUPS_WITH_EBS,
1368
            },
1369
            'ReleaseLabel': 'emr-4.2.0',
1370
            'VisibleToAllUsers': True,
1371
            'Tags': [],
1372
        }
1373
        self.assert_params_for_cmd(cmd, result)
1374

1375
    def test_instance_groups_with_ebs_config_missing_volume_type(self):
1376
        cmd = (
1377
            self.prefix
1378
            + '--release-label emr-4.2.0 --instance-groups '
1379
            + CONSTANTS.INSTANCE_GROUPS_WITH_EBS_VOLUME_MISSING_VOLTYPE_ARG
1380
        )
1381
        stderr = self.run_cmd(cmd, 255)[1]
1382
        self.assert_error_message_has_field_name(stderr, 'VolumeType')
1383

1384
    def test_instance_groups_with_ebs_config_missing_size(self):
1385
        cmd = (
1386
            self.prefix
1387
            + '--release-label emr-4.2.0 --instance-groups '
1388
            + CONSTANTS.INSTANCE_GROUPS_WITH_EBS_VOLUME_MISSING_SIZE_ARG
1389
        )
1390
        stderr = self.run_cmd(cmd, 255)[1]
1391
        self.assert_error_message_has_field_name(stderr, 'SizeInGB')
1392

1393
    def test_instance_groups_with_ebs_config_missing_volume_spec(self):
1394
        cmd = (
1395
            self.prefix
1396
            + '--release-label emr-4.2.0 --instance-groups '
1397
            + CONSTANTS.INSTANCE_GROUPS_WITH_EBS_VOLUME_MISSING_VOLSPEC_ARG
1398
        )
1399
        result = {
1400
            'Name': DEFAULT_CLUSTER_NAME,
1401
            'Instances': {
1402
                'KeepJobFlowAliveWhenNoSteps': True,
1403
                'TerminationProtected': False,
1404
                'InstanceGroups': CONSTANTS.INSTANCE_GROUPS_WITH_EBS_VOLUME_MISSING_VOLSPEC,
1405
            },
1406
            'ReleaseLabel': 'emr-4.2.0',
1407
            'VisibleToAllUsers': True,
1408
            'Tags': [],
1409
        }
1410
        self.assert_params_for_cmd(cmd, result)
1411

1412
    def test_instance_groups_with_ebs_config_missing_iops(self):
1413
        cmd = (
1414
            self.prefix
1415
            + '--release-label emr-4.2.0 --instance-groups '
1416
            + CONSTANTS.INSTANCE_GROUPS_WITH_EBS_VOLUME_MISSING_IOPS_ARG
1417
        )
1418
        result = {
1419
            'Name': DEFAULT_CLUSTER_NAME,
1420
            'Instances': {
1421
                'KeepJobFlowAliveWhenNoSteps': True,
1422
                'TerminationProtected': False,
1423
                'InstanceGroups': CONSTANTS.INSTANCE_GROUPS_WITH_EBS_VOLUME_MISSING_IOPS,
1424
            },
1425
            'ReleaseLabel': 'emr-4.2.0',
1426
            'VisibleToAllUsers': True,
1427
            'Tags': [],
1428
        }
1429
        self.assert_params_for_cmd(cmd, result)
1430

1431
    def test_instance_groups_with_ebs_config_multiple_instance_groups(self):
1432
        cmd = (
1433
            self.prefix
1434
            + '--release-label emr-4.2.0 --instance-groups '
1435
            + CONSTANTS.MULTIPLE_INSTANCE_GROUPS_WITH_EBS_VOLUMES_VOLUME_ARG
1436
        )
1437
        result = {
1438
            'Name': DEFAULT_CLUSTER_NAME,
1439
            'Instances': {
1440
                'KeepJobFlowAliveWhenNoSteps': True,
1441
                'TerminationProtected': False,
1442
                'InstanceGroups': CONSTANTS.MULTIPLE_INSTANCE_GROUPS_WITH_EBS_VOLUMES,
1443
            },
1444
            'ReleaseLabel': 'emr-4.2.0',
1445
            'VisibleToAllUsers': True,
1446
            'Tags': [],
1447
        }
1448
        self.assert_params_for_cmd(cmd, result)
1449

1450
    def test_instance_fleets_with_on_demand_master_only(self):
1451
        cmd = (
1452
            self.prefix
1453
            + '--release-label emr-4.2.0 --instance-fleets '
1454
            + CONSTANTS_FLEET.INSTANCE_FLEETS_WITH_ON_DEMAND_MASTER_ONLY
1455
        )
1456
        result = {
1457
            'Name': DEFAULT_CLUSTER_NAME,
1458
            'Instances': {
1459
                'KeepJobFlowAliveWhenNoSteps': True,
1460
                'TerminationProtected': False,
1461
                'InstanceFleets': CONSTANTS_FLEET.RES_INSTANCE_FLEETS_WITH_ON_DEMAND_MASTER_ONLY,
1462
            },
1463
            'ReleaseLabel': 'emr-4.2.0',
1464
            'VisibleToAllUsers': True,
1465
            'Tags': [],
1466
        }
1467
        self.assert_params_for_cmd(cmd, result)
1468

1469
    def test_instance_fleets_with_spot_master_only_with_ebs_conf(self):
1470
        cmd = (
1471
            self.prefix
1472
            + '--release-label emr-4.2.0 --instance-fleets '
1473
            + CONSTANTS_FLEET.INSTANCE_FLEETS_WITH_SPOT_MASTER_ONLY_WITH_EBS_CONF
1474
        )
1475
        result = {
1476
            'Name': DEFAULT_CLUSTER_NAME,
1477
            'Instances': {
1478
                'KeepJobFlowAliveWhenNoSteps': True,
1479
                'TerminationProtected': False,
1480
                'InstanceFleets': CONSTANTS_FLEET.RES_INSTANCE_FLEETS_WITH_SPOT_MASTER_ONLY_WITH_EBS_CONF,
1481
            },
1482
            'ReleaseLabel': 'emr-4.2.0',
1483
            'VisibleToAllUsers': True,
1484
            'Tags': [],
1485
        }
1486
        self.assert_params_for_cmd(cmd, result)
1487

1488
    def test_instance_fleets_with_spot_master_specific_azs(self):
1489
        cmd = (
1490
            self.prefix
1491
            + '--release-label emr-4.2.0 --instance-fleets '
1492
            + CONSTANTS_FLEET.INSTANCE_FLEETS_WITH_SPOT_MASTER_ONLY
1493
            + ' --ec2-attributes AvailabilityZones=[us-east-1a,us-east-1b]'
1494
        )
1495
        result = {
1496
            'Name': DEFAULT_CLUSTER_NAME,
1497
            'Instances': {
1498
                'KeepJobFlowAliveWhenNoSteps': True,
1499
                'TerminationProtected': False,
1500
                'InstanceFleets': CONSTANTS_FLEET.RES_INSTANCE_FLEETS_WITH_SPOT_MASTER_ONLY,
1501
                'Placement': {
1502
                    'AvailabilityZones': ['us-east-1a', 'us-east-1b']
1503
                },
1504
            },
1505
            'ReleaseLabel': 'emr-4.2.0',
1506
            'VisibleToAllUsers': True,
1507
            'Tags': [],
1508
        }
1509
        self.assert_params_for_cmd(cmd, result)
1510

1511
    def test_instance_fleets_with_spot_master_subnet_ids(self):
1512
        cmd = (
1513
            self.prefix
1514
            + '--release-label emr-4.2.0 --instance-fleets '
1515
            + CONSTANTS_FLEET.INSTANCE_FLEETS_WITH_SPOT_MASTER_ONLY
1516
            + ' --ec2-attributes SubnetIds=[subnetid-1,subnetid-2]'
1517
        )
1518
        result = {
1519
            'Name': DEFAULT_CLUSTER_NAME,
1520
            'Instances': {
1521
                'KeepJobFlowAliveWhenNoSteps': True,
1522
                'TerminationProtected': False,
1523
                'InstanceFleets': CONSTANTS_FLEET.RES_INSTANCE_FLEETS_WITH_SPOT_MASTER_ONLY,
1524
                'Ec2SubnetIds': ['subnetid-1', 'subnetid-2'],
1525
            },
1526
            'ReleaseLabel': 'emr-4.2.0',
1527
            'VisibleToAllUsers': True,
1528
            'Tags': [],
1529
        }
1530
        self.assert_params_for_cmd(cmd, result)
1531

1532
    def test_instance_fleets_with_spot_master_core_cluster_multiple_instance_types(
1533
        self,
1534
    ):
1535
        cmd = (
1536
            self.prefix
1537
            + '--release-label emr-4.2.0 --instance-fleets '
1538
            + CONSTANTS_FLEET.INSTANCE_FLEETS_WITH_SPOT_MASTER_CORE_CLUSTER
1539
        )
1540
        result = {
1541
            'Name': DEFAULT_CLUSTER_NAME,
1542
            'Instances': {
1543
                'KeepJobFlowAliveWhenNoSteps': True,
1544
                'TerminationProtected': False,
1545
                'InstanceFleets': CONSTANTS_FLEET.RES_INSTANCE_FLEETS_WITH_SPOT_MASTER_CORE_CLUSTER,
1546
            },
1547
            'ReleaseLabel': 'emr-4.2.0',
1548
            'VisibleToAllUsers': True,
1549
            'Tags': [],
1550
        }
1551
        self.assert_params_for_cmd(cmd, result)
1552

1553
    def test_instance_fleets_with_complex_config_from_json(self):
1554
        data_path = os.path.join(
1555
            os.path.dirname(__file__), 'input_instance_fleets.json'
1556
        )
1557
        cmd = (
1558
            'emr create-cluster --use-default-roles --release-label emr-4.2.0  '
1559
            '--instance-fleets file://' + data_path
1560
        )
1561
        result = {
1562
            'Name': DEFAULT_CLUSTER_NAME,
1563
            'Instances': {
1564
                'KeepJobFlowAliveWhenNoSteps': True,
1565
                'TerminationProtected': False,
1566
                'InstanceFleets': CONSTANTS_FLEET.RES_INSTANCE_FLEETS_WITH_COMPLEX_CONFIG_FROM_JSON,
1567
            },
1568
            'ReleaseLabel': 'emr-4.2.0',
1569
            'VisibleToAllUsers': True,
1570
            'Tags': [],
1571
            'JobFlowRole': 'EMR_EC2_DefaultRole',
1572
            'ServiceRole': 'EMR_DefaultRole',
1573
        }
1574
        self.assert_params_for_cmd(cmd, result)
1575

1576
    def test_instance_fleets_with_both_fleet_group_specified(self):
1577
        cmd = (
1578
            self.prefix
1579
            + '--release-label emr-4.2.0 --instance-fleets '
1580
            + CONSTANTS_FLEET.INSTANCE_FLEETS_WITH_SPOT_MASTER_ONLY
1581
            + ' --instance-groups '
1582
            + CONSTANTS.INSTANCE_GROUPS_WITH_EBS_VOLUME_ARG
1583
        )
1584
        expected_error_msg = (
1585
            '\naws: error: You cannot specify both --instance-groups'
1586
            ' and --instance-fleets options together.\n'
1587
        )
1588
        result = self.run_cmd(cmd, 255)
1589
        self.assertEqual(expected_error_msg, result[1])
1590

1591
    def test_instance_fleets_with_both_subnetid_subnetids_specified(self):
1592
        cmd = (
1593
            self.prefix
1594
            + '--release-label emr-4.2.0 --instance-fleets '
1595
            + CONSTANTS_FLEET.INSTANCE_FLEETS_WITH_SPOT_MASTER_ONLY
1596
            + ' --ec2-attributes SubnetId=subnetid-1,SubnetIds=[subnetid-1,subnetid-2]'
1597
        )
1598
        expected_error_msg = (
1599
            '\naws: error: You cannot specify both SubnetId'
1600
            ' and SubnetIds options together.\n'
1601
        )
1602
        result = self.run_cmd(cmd, 255)
1603
        self.assertEqual(expected_error_msg, result[1])
1604

1605
    def test_create_cluster_with_security_config(self):
1606
        cmd = (
1607
            self.prefix
1608
            + '--release-label emr-4.7.2 --security-configuration MySecurityConfig '
1609
            + '--instance-groups '
1610
            + DEFAULT_INSTANCE_GROUPS_ARG
1611
        )
1612
        result = {
1613
            'Name': DEFAULT_CLUSTER_NAME,
1614
            'Instances': DEFAULT_INSTANCES,
1615
            'ReleaseLabel': 'emr-4.7.2',
1616
            'VisibleToAllUsers': True,
1617
            'Tags': [],
1618
            'SecurityConfiguration': 'MySecurityConfig',
1619
        }
1620
        self.assert_params_for_cmd(cmd, result)
1621

1622
    def test_create_cluster_with_security_config_and_kerberos_attributes(self):
1623
        cmd = (
1624
            self.prefix
1625
            + '--release-label emr-4.7.2 --security-configuration MySecurityConfig'
1626
            + ' --kerberos-attributes '
1627
            + DEFAULT_KERBEROS_ATTRIBUTES_ARGS
1628
            + ' --instance-groups '
1629
            + DEFAULT_INSTANCE_GROUPS_ARG
1630
        )
1631
        result = {
1632
            'Name': DEFAULT_CLUSTER_NAME,
1633
            'Instances': DEFAULT_INSTANCES,
1634
            'KerberosAttributes': KERBEROS_ATTRIBUTES,
1635
            'ReleaseLabel': 'emr-4.7.2',
1636
            'VisibleToAllUsers': True,
1637
            'Tags': [],
1638
            'SecurityConfiguration': 'MySecurityConfig',
1639
        }
1640
        self.assert_params_for_cmd(cmd, result)
1641

1642
    def test_create_cluster_with_custom_ami_id(self):
1643
        cmd = (
1644
            self.prefix
1645
            + '--release-label emr-4.7.2 --security-configuration MySecurityConfig '
1646
            + ' --custom-ami-id ami-9be6f38c'
1647
            + ' --instance-groups '
1648
            + DEFAULT_INSTANCE_GROUPS_ARG
1649
        )
1650
        result = {
1651
            'Name': DEFAULT_CLUSTER_NAME,
1652
            'Instances': DEFAULT_INSTANCES,
1653
            'ReleaseLabel': 'emr-4.7.2',
1654
            'VisibleToAllUsers': True,
1655
            'Tags': [],
1656
            'CustomAmiId': 'ami-9be6f38c',
1657
            'SecurityConfiguration': 'MySecurityConfig',
1658
        }
1659
        self.assert_params_for_cmd(cmd, result)
1660

1661
    def test_create_cluster_with_ebs_root_volume_size(self):
1662
        cmd = (
1663
            self.prefix
1664
            + '--release-label emr-4.7.2 --security-configuration MySecurityConfig '
1665
            + ' --ebs-root-volume-size 50'
1666
            + ' --instance-groups '
1667
            + DEFAULT_INSTANCE_GROUPS_ARG
1668
        )
1669
        result = {
1670
            'Name': DEFAULT_CLUSTER_NAME,
1671
            'Instances': DEFAULT_INSTANCES,
1672
            'ReleaseLabel': 'emr-4.7.2',
1673
            'VisibleToAllUsers': True,
1674
            'Tags': [],
1675
            'EbsRootVolumeSize': 50,
1676
            'SecurityConfiguration': 'MySecurityConfig',
1677
        }
1678
        self.assert_params_for_cmd(cmd, result)
1679

1680
    def test_create_cluster_with_ebs_root_volume_iops(self):
1681
        cmd = (
1682
            self.prefix
1683
            + '--release-label emr-6.15.0 --security-configuration MySecurityConfig '
1684
            + ' --ebs-root-volume-iops 3000'
1685
            + ' --instance-groups '
1686
            + DEFAULT_INSTANCE_GROUPS_ARG
1687
        )
1688
        result = {
1689
            'Name': DEFAULT_CLUSTER_NAME,
1690
            'Instances': DEFAULT_INSTANCES,
1691
            'ReleaseLabel': 'emr-6.15.0',
1692
            'VisibleToAllUsers': True,
1693
            'Tags': [],
1694
            'EbsRootVolumeIops': 3000,
1695
            'SecurityConfiguration': 'MySecurityConfig',
1696
        }
1697
        self.assert_params_for_cmd(cmd, result)
1698

1699
    def test_create_cluster_with_ebs_root_volume_throughput(self):
1700
        cmd = (
1701
            self.prefix
1702
            + '--release-label emr-6.15.0 --security-configuration MySecurityConfig '
1703
            + ' --ebs-root-volume-throughput 125'
1704
            + ' --instance-groups '
1705
            + DEFAULT_INSTANCE_GROUPS_ARG
1706
        )
1707
        result = {
1708
            'Name': DEFAULT_CLUSTER_NAME,
1709
            'Instances': DEFAULT_INSTANCES,
1710
            'ReleaseLabel': 'emr-6.15.0',
1711
            'VisibleToAllUsers': True,
1712
            'Tags': [],
1713
            'EbsRootVolumeThroughput': 125,
1714
            'SecurityConfiguration': 'MySecurityConfig',
1715
        }
1716
        self.assert_params_for_cmd(cmd, result)
1717

1718
    def test_create_cluster_with_repo_upgrade_on_boot(self):
1719
        cmd = (
1720
            self.prefix
1721
            + '--release-label emr-4.7.2 --security-configuration MySecurityConfig '
1722
            + ' --repo-upgrade-on-boot NONE'
1723
            + ' --instance-groups '
1724
            + DEFAULT_INSTANCE_GROUPS_ARG
1725
        )
1726
        result = {
1727
            'Name': DEFAULT_CLUSTER_NAME,
1728
            'Instances': DEFAULT_INSTANCES,
1729
            'ReleaseLabel': 'emr-4.7.2',
1730
            'VisibleToAllUsers': True,
1731
            'Tags': [],
1732
            'RepoUpgradeOnBoot': 'NONE',
1733
            'SecurityConfiguration': 'MySecurityConfig',
1734
        }
1735
        self.assert_params_for_cmd(cmd, result)
1736

1737
    def test_create_cluster_with_step_concurrency_level(self):
1738
        cmd = (
1739
            self.prefix
1740
            + '--release-label emr-5.28.0 --security-configuration MySecurityConfig '
1741
            + '--step-concurrency-level 30 '
1742
            + '--instance-groups '
1743
            + DEFAULT_INSTANCE_GROUPS_ARG
1744
        )
1745
        result = {
1746
            'Name': DEFAULT_CLUSTER_NAME,
1747
            'Instances': DEFAULT_INSTANCES,
1748
            'ReleaseLabel': 'emr-5.28.0',
1749
            'VisibleToAllUsers': True,
1750
            'Tags': [],
1751
            'StepConcurrencyLevel': 30,
1752
            'SecurityConfiguration': 'MySecurityConfig',
1753
        }
1754
        self.assert_params_for_cmd(cmd, result)
1755

1756
    def test_create_cluster_with_managed_scaling_policy(self):
1757
        cmd = (
1758
            self.prefix
1759
            + '--release-label emr-5.28.0 --security-configuration MySecurityConfig '
1760
            + '--managed-scaling-policy ComputeLimits={MinimumCapacityUnits=2,MaximumCapacityUnits=4,UnitType=Instances,MaximumCoreCapacityUnits=1} '
1761
            + '--instance-groups '
1762
            + DEFAULT_INSTANCE_GROUPS_ARG
1763
        )
1764
        result = {
1765
            'Name': DEFAULT_CLUSTER_NAME,
1766
            'Instances': DEFAULT_INSTANCES,
1767
            'ReleaseLabel': 'emr-5.28.0',
1768
            'VisibleToAllUsers': True,
1769
            'Tags': [],
1770
            'ManagedScalingPolicy': {
1771
                'ComputeLimits': {
1772
                    'MinimumCapacityUnits': 2,
1773
                    'MaximumCapacityUnits': 4,
1774
                    'UnitType': 'Instances',
1775
                    'MaximumCoreCapacityUnits': 1,
1776
                }
1777
            },
1778
            'SecurityConfiguration': 'MySecurityConfig',
1779
        }
1780
        self.assert_params_for_cmd(cmd, result)
1781

1782
    def test_create_cluster_with_managed_scaling_policy_customer_knobs(self):
1783
        cmd = (
1784
            self.prefix
1785
            + '--release-label emr-5.28.0 --security-configuration MySecurityConfig '
1786
            + '--managed-scaling-policy ComputeLimits={MinimumCapacityUnits=2,MaximumCapacityUnits=4,'
1787
            + 'UnitType=Instances,MaximumCoreCapacityUnits=1},ScalingStrategy=ADVANCED,'
1788
            + 'UtilizationPerformanceIndex=1 --instance-groups '
1789
            + DEFAULT_INSTANCE_GROUPS_ARG
1790
        )
1791
        result = {
1792
            'Name': DEFAULT_CLUSTER_NAME,
1793
            'Instances': DEFAULT_INSTANCES,
1794
            'ReleaseLabel': 'emr-5.28.0',
1795
            'VisibleToAllUsers': True,
1796
            'Tags': [],
1797
            'ManagedScalingPolicy': {
1798
                'ComputeLimits': {
1799
                    'MinimumCapacityUnits': 2,
1800
                    'MaximumCapacityUnits': 4,
1801
                    'UnitType': 'Instances',
1802
                    'MaximumCoreCapacityUnits': 1,
1803
                },
1804
                'ScalingStrategy': 'ADVANCED',
1805
                'UtilizationPerformanceIndex': 1,
1806
            },
1807
            'SecurityConfiguration': 'MySecurityConfig',
1808
        }
1809
        self.assert_params_for_cmd(cmd, result)
1810

1811
    def test_create_cluster_with_auto_termination_policy(self):
1812
        cmd = (
1813
            self.prefix
1814
            + '--release-label emr-5.34.0 '
1815
            + '--auto-termination-policy IdleTimeout=100 '
1816
            + '--instance-groups '
1817
            + DEFAULT_INSTANCE_GROUPS_ARG
1818
        )
1819
        result = {
1820
            'Name': DEFAULT_CLUSTER_NAME,
1821
            'Instances': DEFAULT_INSTANCES,
1822
            'ReleaseLabel': 'emr-5.34.0',
1823
            'VisibleToAllUsers': True,
1824
            'Tags': [],
1825
            'AutoTerminationPolicy': {
1826
                'IdleTimeout': 100,
1827
            },
1828
        }
1829
        self.assert_params_for_cmd(cmd, result)
1830

1831
    def test_create_cluster_with_log_encryption_kms_key_id(self):
1832
        test_log_uri = 's3://test/logs'
1833
        test_log_encryption_kms_key_id = 'valid_kms_key'
1834
        cmd = (
1835
            self.prefix
1836
            + '--release-label emr-5.30.0 '
1837
            + '--log-uri '
1838
            + test_log_uri
1839
            + ' --log-encryption-kms-key-id '
1840
            + test_log_encryption_kms_key_id
1841
            + ' --instance-groups '
1842
            + DEFAULT_INSTANCE_GROUPS_ARG
1843
        )
1844
        result = {
1845
            'Name': DEFAULT_CLUSTER_NAME,
1846
            'Instances': DEFAULT_INSTANCES,
1847
            'ReleaseLabel': 'emr-5.30.0',
1848
            'VisibleToAllUsers': True,
1849
            'Tags': [],
1850
            'LogUri': test_log_uri,
1851
            'LogEncryptionKmsKeyId': test_log_encryption_kms_key_id,
1852
        }
1853
        self.assert_params_for_cmd(cmd, result)
1854

1855
    def test_create_cluster_with_placement_groups(self):
1856
        cmd = (
1857
            self.prefix
1858
            + '--release-label emr-5.28.0 --security-configuration '
1859
            + 'MySecurityConfig --placement-group-configs '
1860
            + 'InstanceRole=MASTER,PlacementStrategy=SPREAD '
1861
            + '--instance-groups '
1862
            + HA_INSTANCE_GROUPS_ARG
1863
        )
1864
        result = {
1865
            'Name': DEFAULT_CLUSTER_NAME,
1866
            'Instances': HA_INSTANCES,
1867
            'ReleaseLabel': 'emr-5.28.0',
1868
            'VisibleToAllUsers': True,
1869
            'Tags': [],
1870
            'PlacementGroupConfigs': [
1871
                {'InstanceRole': 'MASTER', 'PlacementStrategy': 'SPREAD'}
1872
            ],
1873
            'SecurityConfiguration': 'MySecurityConfig',
1874
        }
1875
        self.assert_params_for_cmd(cmd, result)
1876

1877
    def test_instance_fleets_with_resize_allocation_strategy_spot_od(self):
1878
        cmd = (
1879
            self.prefix
1880
            + '--release-label emr-4.2.0 --instance-fleets '
1881
            + CONSTANTS_FLEET.INSTANCE_FLEETS_WITH_RESIZE_ALLOCATION_STRATEGY_SPOT_AND_OD
1882
            + ' --ec2-attributes AvailabilityZones=[us-east-1a,us-east-1b]'
1883
        )
1884
        instance_fleets = CONSTANTS_FLEET.RES_INSTANCE_FLEETS_WITH_RESIZE_ALLOCATION_STRATEGY_SPOT_AND_OD
1885
        result = {
1886
            'Name': DEFAULT_CLUSTER_NAME,
1887
            'Instances': {
1888
                'KeepJobFlowAliveWhenNoSteps': True,
1889
                'TerminationProtected': False,
1890
                'InstanceFleets': instance_fleets,
1891
                'Placement': {
1892
                    'AvailabilityZones': ['us-east-1a', 'us-east-1b']
1893
                },
1894
            },
1895
            'ReleaseLabel': 'emr-4.2.0',
1896
            'VisibleToAllUsers': True,
1897
            'Tags': [],
1898
        }
1899
        self.assert_params_for_cmd(cmd, result)
1900

1901
    def test_create_cluster_with_os_release_label(self):
1902
        test_os_release_label = '2.0.20220406.1'
1903
        cmd = (
1904
            self.prefix
1905
            + '--release-label emr-6.6.0'
1906
            + ' --os-release-label '
1907
            + test_os_release_label
1908
            + ' --instance-groups '
1909
            + DEFAULT_INSTANCE_GROUPS_ARG
1910
        )
1911
        result = {
1912
            'Name': DEFAULT_CLUSTER_NAME,
1913
            'Instances': DEFAULT_INSTANCES,
1914
            'ReleaseLabel': 'emr-6.6.0',
1915
            'VisibleToAllUsers': True,
1916
            'Tags': [],
1917
            'OSReleaseLabel': test_os_release_label,
1918
        }
1919
        self.assert_params_for_cmd(cmd, result)
1920

1921
    def test_instance_fleets_with_spot_allocation_strategy(self):
1922
        cmd = (
1923
            self.prefix
1924
            + '--release-label emr-4.2.0 --instance-fleets '
1925
            + CONSTANTS_FLEET.INSTANCE_FLEETS_WITH_SPOT_ALLOCATION_STRATEGY
1926
            + ' --ec2-attributes AvailabilityZones=[us-east-1a,us-east-1b]'
1927
        )
1928
        result = {
1929
            'Name': DEFAULT_CLUSTER_NAME,
1930
            'Instances': {
1931
                'KeepJobFlowAliveWhenNoSteps': True,
1932
                'TerminationProtected': False,
1933
                'InstanceFleets': CONSTANTS_FLEET.RES_INSTANCE_FLEETS_WITH_SPOT_ALLOCATION_STRATEGY,
1934
                'Placement': {
1935
                    'AvailabilityZones': ['us-east-1a', 'us-east-1b']
1936
                },
1937
            },
1938
            'ReleaseLabel': 'emr-4.2.0',
1939
            'VisibleToAllUsers': True,
1940
            'Tags': [],
1941
        }
1942
        self.assert_params_for_cmd(cmd, result)
1943

1944
    def test_instance_fleets_with_prioritized_allocation_strategy_spot_ondemand(
1945
        self,
1946
    ):
1947
        cmd = (
1948
            self.prefix
1949
            + '--release-label emr-4.2.0 --instance-fleets '
1950
            + CONSTANTS_FLEET.INSTANCE_FLEETS_WITH_PRIORITIZED_ALLOCATION_STRATEGY_SPOT_AND_OD
1951
            + ' --ec2-attributes AvailabilityZones=[us-east-1a,us-east-1b]'
1952
        )
1953
        result = {
1954
            'Name': DEFAULT_CLUSTER_NAME,
1955
            'Instances': {
1956
                'KeepJobFlowAliveWhenNoSteps': True,
1957
                'TerminationProtected': False,
1958
                'InstanceFleets': CONSTANTS_FLEET.RES_INSTANCE_FLEETS_WITH_PRIORITIZED_ALLOCATION_STRATEGY_SPOT_AND_OD,
1959
                'Placement': {
1960
                    'AvailabilityZones': ['us-east-1a', 'us-east-1b']
1961
                },
1962
            },
1963
            'ReleaseLabel': 'emr-4.2.0',
1964
            'VisibleToAllUsers': True,
1965
            'Tags': [],
1966
        }
1967
        self.assert_params_for_cmd(cmd, result)
1968

1969

1970
if __name__ == "__main__":
1971
    unittest.main()
1972

1973
Product

Resources

Company
