CoCalc -- test_add_instance

GitHub Repository: aws/aws-cli
Path: blob/develop/tests/unit/customizations/emr/test_add_instance_groups.py
¹⁵⁶⁹ views
1
# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2
#
3
# Licensed under the Apache License, Version 2.0 (the "License"). You
4
# may not use this file except in compliance with the License. A copy of
5
# the License is located at
6
#
7
#     http://aws.amazon.com/apache2.0/
8
#
9
# or in the "license" file accompanying this file. This file is
10
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11
# ANY KIND, either express or implied. See the License for the specific
12
# language governing permissions and limitations under the License.
13

14
from tests.unit.customizations.emr import EMRBaseAWSCommandParamsTest as \
15
    BaseAWSCommandParamsTest
16
from tests.unit.customizations.emr import test_constants as \
17
    CONSTANTS
18
import json
19
from awscli.testutils import mock
20

21

22
INSTANCE_GROUPS_WITH_AUTOSCALING_POLICY = (
23
    ' InstanceGroupType=TASK,InstanceType=d2.xlarge,InstanceCount=2,'
24
    'AutoScalingPolicy={Constraints={MinCapacity=1,MaxCapacity=2},'
25
    'Rules=[{Name=TestRule,Description=TestDescription,Action={Market=ON_DEMAND,'
26
    'SimpleScalingPolicyConfiguration={AdjustmentType=EXACT_CAPACITY,ScalingAdjustment=2,CoolDown=5}},'
27
    'Trigger={CloudWatchAlarmDefinition={ComparisonOperator=GREATER_THAN,'
28
    'EvaluationPeriods=5,MetricName=TestMetric,Namespace=EMR,Period=3,Statistic=MAXIMUM,'
29
    'Threshold=4.565,Unit=NONE,Dimensions=[{Key=TestKey,Value=TestValue}]}}}]}'
30
)
31

32
INSTANCE_GROUPS_WITH_EBS_VOLUME_ARG = (
33
    ' InstanceGroupType=TASK,InstanceType=d2.xlarge,InstanceCount=2,EbsConfiguration={EbsOptimized=true,EbsBlockDeviceConfigs=[{VolumeSpecification={VolumeType=gp2,SizeInGB=100,Iops=100},VolumesPerInstance=4},{VolumeSpecification={VolumeType=gp2,SizeInGB=100,Iops=100}}]}')
34

35
INSTANCE_GROUPS_WITH_EBS_VOLUME_MISSING_VOLTYPE_ARG = (
36
    ' InstanceGroupType=TASK,InstanceType=d2.xlarge,InstanceCount=2,EbsConfiguration={EbsOptimized=true,EbsBlockDeviceConfigs=[{VolumeSpecification={SizeInGB=100,Iops=100},VolumesPerInstance=4},{VolumeSpecification={VolumeType=gp2,SizeInGB=100,Iops=100}}]}')
37

38
INSTANCE_GROUPS_WITH_EBS_VOLUME_MISSING_SIZE_ARG = (
39
    ' InstanceGroupType=TASK,InstanceType=d2.xlarge,InstanceCount=2,EbsConfiguration={EbsOptimized=true,EbsBlockDeviceConfigs=[{VolumeSpecification={VolumeType=gp2,Iops=100},VolumesPerInstance=4},{VolumeSpecification={VolumeType=gp2,SizeInGB=100,Iops=100}}]}')
40

41
INSTANCE_GROUPS_WITH_EBS_VOLUME_MISSING_VOLSPEC_ARG = (
42
    ' InstanceGroupType=TASK,InstanceType=d2.xlarge,InstanceCount=2,EbsConfiguration={EbsOptimized=true}')
43

44
INSTANCE_GROUPS_WITH_EBS_VOLUME_MISSING_IOPS_ARG = (
45
    ' InstanceGroupType=TASK,InstanceType=d2.xlarge,InstanceCount=2,EbsConfiguration={EbsOptimized=true,EbsBlockDeviceConfigs=[{VolumeSpecification={VolumeType=gp2,SizeInGB=100},VolumesPerInstance=4}]}')
46

47
MULTIPLE_INSTANCE_GROUPS_WITH_EBS_VOLUMES_VOLUME_ARG = (
48
    ' InstanceGroupType=TASK,InstanceType=d2.xlarge,InstanceCount=2,EbsConfiguration={EbsOptimized=true,EbsBlockDeviceConfigs=[{VolumeSpecification={VolumeType=gp2,SizeInGB=100},VolumesPerInstance=4}]} InstanceGroupType=CORE,InstanceType=d2.xlarge,InstanceCount=2,EbsConfiguration={EbsOptimized=true,EbsBlockDeviceConfigs=[{VolumeSpecification={VolumeType=gp2,SizeInGB=100,Iops=20}},{VolumeSpecification={VolumeType=gp2,SizeInGB=100,Iops=40}}]}')
49

50
INSTANCE_GROUPS_WITH_CUSTOM_AMI_ARG = (
51
    ' InstanceGroupType=TASK,InstanceType=d2.xlarge,InstanceCount=2,CustomAmiId=ami-deadbeef'
52
)
53

54
DEFAULT_INSTANCE_GROUPS = [{'InstanceRole': 'TASK',
55
                            'InstanceCount': 10,
56
                            'Name': 'TASK',
57
                            'Market': 'ON_DEMAND',
58
                            'InstanceType': 'm2.large'
59
                            }]
60

61

62
DEFAULT_INSTANCE_GROUPS_WITH_CUSTOM_AMI = \
63
    [{'CustomAmiId':'ami-deadbeef',
64
    'InstanceRole': 'TASK',
65
    'InstanceCount': 2,
66
    'Name': 'TASK',
67
    'Market': 'ON_DEMAND',
68
    'InstanceType': 'd2.xlarge'}]
69

70
DEFAULT_INSTANCE_GROUPS_WITH_EBS_CONFIG = \
71
    [{'EbsConfiguration': 
72
        {'EbsOptimized': True,
73
            'EbsBlockDeviceConfigs': 
74
                [
75
                 {'VolumeSpecification':
76
                      {'Iops': 100,
77
                      'SizeInGB': 100,
78
                      'VolumeType': 'gp2'},
79
                      'VolumesPerInstance': 4},
80
                  {'VolumeSpecification':
81
                     {'Iops': 100,
82
                      'SizeInGB': 100,
83
                      'VolumeType': 'gp2'}}]},
84
    'InstanceCount': 2,
85
    'InstanceRole': 'TASK',
86
    'InstanceType': 'd2.xlarge',
87
    'Market': 'ON_DEMAND',
88
    'Name': 'TASK'}]
89

90
DEFAULT_INSTANCE_GROUPS_WITH_EBS_CONFIG_MISSING_IOPS = \
91
    [{'EbsConfiguration': 
92
        {'EbsOptimized': True,
93
            'EbsBlockDeviceConfigs': 
94
                [{'VolumeSpecification':
95
                  {'SizeInGB': 100,
96
                  'VolumeType': 'gp2'},
97
                  'VolumesPerInstance': 4}]},
98

99
     'InstanceCount': 2,
100
     'InstanceRole': 'TASK',
101
     'InstanceType': 'd2.xlarge',
102
     'Market': 'ON_DEMAND',
103
     'Name': 'TASK'}]
104

105
DEFAULT_INSTANCE_GROUPS_WITH_EBS_CONFIG_MISSING_VOLSPEC = \
106
    [{'EbsConfiguration': {'EbsOptimized': True},
107
    'InstanceCount': 2,
108
    'InstanceRole': 'TASK',
109
    'InstanceType': 'd2.xlarge',
110
    'Market': 'ON_DEMAND',
111
    'Name': 'TASK'}]
112

113
DEFAULT_INSTANCE_GROUPS_WITH_AUTOSCALING_POLICY = \
114
[{'InstanceCount': 2,
115
  'InstanceRole': 'TASK',
116
  'InstanceType': 'd2.xlarge',
117
  'Market': 'ON_DEMAND',
118
  'Name': 'TASK',
119
  'AutoScalingPolicy': {
120
      'Constraints': {
121
          'MinCapacity': 1,
122
          'MaxCapacity': 2
123
      },
124
      'Rules': [
125
          {
126
             'Name': 'TestRule',
127
             'Description': 'TestDescription',
128
             'Action': {
129
                 'Market': 'ON_DEMAND',
130
                 'SimpleScalingPolicyConfiguration': {
131
                     'AdjustmentType': 'EXACT_CAPACITY',
132
                     'ScalingAdjustment': 2,
133
                     'CoolDown': 5
134
                 }
135
             },
136
             'Trigger': {
137
                 'CloudWatchAlarmDefinition': {
138
                     'ComparisonOperator': 'GREATER_THAN',
139
                     'Dimensions': [
140
                         {
141
                             'Key': 'TestKey',
142
                             'Value': 'TestValue'
143
                         }],
144
                     'EvaluationPeriods': 5,
145
                     'MetricName': 'TestMetric',
146
                     'Namespace': 'EMR',
147
                     'Period': 3,
148
                     'Statistic': 'MAXIMUM',
149
                     'Threshold': 4.565,
150
                     'Unit': 'NONE'
151
                 }
152
             }
153
          }
154
      ]
155
  }
156
  }]
157

158
DEFAULT_MULTIPLE_INSTANCE_GROUPS_WITH_EBS_CONFIG = \
159
    [{'EbsConfiguration': 
160
        {'EbsOptimized': True,
161
            'EbsBlockDeviceConfigs': 
162
                [{'VolumeSpecification':
163
                  {'SizeInGB': 100,
164
                  'VolumeType': 'gp2'},
165
                  'VolumesPerInstance': 4}]},
166
    'InstanceCount': 2,
167
    'InstanceRole': 'TASK',
168
    'InstanceType': 'd2.xlarge',
169
    'Market': 'ON_DEMAND',
170
    'Name': 'TASK'},
171
   {'EbsConfiguration': 
172
        {'EbsOptimized': True,
173
            'EbsBlockDeviceConfigs': 
174
                [{'VolumeSpecification':
175
                   {'Iops': 20,
176
                    'SizeInGB': 100,
177
                    'VolumeType': 'gp2'
178
                    }},
179
                   {'VolumeSpecification':
180
                    {'Iops': 40,
181
                    'SizeInGB': 100,
182
                    'VolumeType': 'gp2'}}]},
183
    'InstanceCount': 2,
184
    'InstanceRole': 'CORE',
185
    'InstanceType': 'd2.xlarge',
186
    'Market': 'ON_DEMAND',
187
    'Name': 'CORE'}]
188

189

190
ADD_INSTANCE_GROUPS_RESULT = {
191
    "InstanceGroupIds": [
192
        "ig-XXXX"
193
    ],
194
    "ClusterArn": "arn:aws:elasticmapreduce:region:012345678910:cluster/j-XXXX",
195
    "JobFlowId": "j-YYYY"
196
}
197

198
CONSTRUCTED_RESULT = {
199
    "InstanceGroupIds": [
200
        "ig-XXXX"
201
    ],
202
    "ClusterArn": "arn:aws:elasticmapreduce:region:012345678910:cluster/j-XXXX",
203
    "ClusterId": "j-YYYY"
204
}
205

206

207
class TestAddInstanceGroups(BaseAWSCommandParamsTest):
208
    prefix = 'emr add-instance-groups --cluster-id J-ABCD --instance-groups'
209

210
    def test_instance_groups_with_autoscaling_policy(self):
211
        cmd = self.prefix
212
        cmd += INSTANCE_GROUPS_WITH_AUTOSCALING_POLICY
213
        result = {'JobFlowId': 'J-ABCD',
214
                  'InstanceGroups': DEFAULT_INSTANCE_GROUPS_WITH_AUTOSCALING_POLICY}
215
        self.assert_params_for_cmd(cmd, result)
216

217
    def assert_error_message_has_field_name(self, error_msg, field_name):
218
        self.assertIn('Missing required parameter', error_msg)
219
        self.assertIn(field_name, error_msg)
220

221
    def test_instance_groups_default_name_market(self):
222
        cmd = self.prefix
223
        cmd += ' InstanceGroupType=TASK,InstanceCount=10,InstanceType=m2.large'
224
        result = {'JobFlowId': 'J-ABCD',
225
                  'InstanceGroups': DEFAULT_INSTANCE_GROUPS}
226

227
        self.assert_params_for_cmd(cmd, result)
228

229
    def test_instance_groups_missing_instance_group_type_error(self):
230
        cmd = self.prefix + ' Name=Task,InstanceType=m1.small,' +\
231
            'InstanceCount=5'
232
        result = self.run_cmd(cmd, 255)
233
        self.assert_error_message_has_field_name(result[1],
234
                                                 'InstanceGroupType')
235

236
    def test_instance_groups_missing_instance_type_error(self):
237
        cmd = self.prefix + ' Name=Task,InstanceGroupType=Task,' +\
238
            'InstanceCount=5'
239
        stderr = self.run_cmd(cmd, 255)[1]
240
        self.assert_error_message_has_field_name(stderr, 'InstanceType')
241

242
    def test_instance_groups_missing_instance_count_error(self):
243
        cmd = self.prefix + ' Name=Task,InstanceGroupType=Task,' +\
244
            'InstanceType=m1.xlarge'
245
        stderr = self.run_cmd(cmd, 255)[1]
246
        self.assert_error_message_has_field_name(stderr, 'InstanceCount')
247

248
    def test_instance_groups_all_fields(self):
249
        cmd = self.prefix + ' InstanceGroupType=MASTER,Name="MasterGroup",' +\
250
            'InstanceCount=1,InstanceType=m1.large'
251
        cmd += ' InstanceGroupType=CORE,Name="CoreGroup",InstanceCount=1,' +\
252
            'InstanceType=m1.xlarge,BidPrice=1.234'
253
        cmd += ' InstanceGroupType=TASK,Name="TaskGroup",InstanceCount=2,' +\
254
            'InstanceType=m1.large'
255

256
        expected_instance_groups = [
257
            {'InstanceRole': 'MASTER',
258
             'InstanceCount': 1,
259
             'Name': 'MasterGroup',
260
             'Market': 'ON_DEMAND',
261
             'InstanceType': 'm1.large'
262
             },
263
            {'InstanceRole': 'CORE',
264
             'InstanceCount': 1,
265
             'Name': 'CoreGroup',
266
             'Market': 'SPOT',
267
             'BidPrice': '1.234',
268
             'InstanceType': 'm1.xlarge'
269
             },
270
            {'InstanceRole': 'TASK',
271
             'InstanceCount': 2,
272
             'Name': 'TaskGroup',
273
             'Market': 'ON_DEMAND',
274
             'InstanceType': 'm1.large'
275
             }
276
        ]
277
        result = {'JobFlowId': 'J-ABCD',
278
                  'InstanceGroups': expected_instance_groups}
279

280
        self.assert_params_for_cmd(cmd, result)
281

282
    def test_instance_groups_spot_bidprice_equals_ondemandprice(self):
283
        cmd = self.prefix + ' InstanceGroupType=MASTER,Name="MasterGroup",' +\
284
            'InstanceCount=1,InstanceType=m1.large,BidPrice=OnDemandPrice'
285
        cmd += ' InstanceGroupType=CORE,Name="CoreGroup",InstanceCount=1,' +\
286
            'InstanceType=m1.xlarge,BidPrice=OnDemandPrice'
287
        cmd += ' InstanceGroupType=TASK,Name="TaskGroup",InstanceCount=2,' +\
288
            'InstanceType=m1.large,BidPrice=OnDemandPrice'
289

290
        expected_instance_groups = [
291
            {'InstanceRole': 'MASTER',
292
             'InstanceCount': 1,
293
             'Name': 'MasterGroup',
294
             'Market': 'SPOT',
295
             'InstanceType': 'm1.large'
296
             },
297
            {'InstanceRole': 'CORE',
298
             'InstanceCount': 1,
299
             'Name': 'CoreGroup',
300
             'Market': 'SPOT',
301
             'InstanceType': 'm1.xlarge'
302
             },
303
            {'InstanceRole': 'TASK',
304
             'InstanceCount': 2,
305
             'Name': 'TaskGroup',
306
             'Market': 'SPOT',
307
             'InstanceType': 'm1.large'
308
             }
309
        ]
310
        result = {'JobFlowId': 'J-ABCD',
311
                  'InstanceGroups': expected_instance_groups}
312

313
        self.assert_params_for_cmd(cmd, result)
314

315
    def test_instance_groups_with_ebs_config(self):
316
        cmd = self.prefix
317
        cmd += INSTANCE_GROUPS_WITH_EBS_VOLUME_ARG
318
        result = {'JobFlowId': 'J-ABCD',
319
                  'InstanceGroups': DEFAULT_INSTANCE_GROUPS_WITH_EBS_CONFIG}
320
        self.assert_params_for_cmd(cmd, result)
321

322
    def test_instance_groups_with_ebs_config_missing_volume_type(self):
323
        cmd = self.prefix
324
        cmd += INSTANCE_GROUPS_WITH_EBS_VOLUME_MISSING_VOLTYPE_ARG
325
        stderr = self.run_cmd(cmd, 255)[1]
326
        self.assert_error_message_has_field_name(stderr, 'VolumeType')
327

328
    def test_instance_groups_with_ebs_config_missing_size(self):
329
        cmd = self.prefix
330
        cmd += INSTANCE_GROUPS_WITH_EBS_VOLUME_MISSING_SIZE_ARG
331
        stderr = self.run_cmd(cmd, 255)[1]
332
        self.assert_error_message_has_field_name(stderr, 'SizeInGB')
333

334
    def test_instance_groups_with_ebs_config_missing_volume_spec(self):
335
        cmd = self.prefix
336
        cmd += INSTANCE_GROUPS_WITH_EBS_VOLUME_MISSING_VOLSPEC_ARG
337
        result = {'JobFlowId': 'J-ABCD',
338
                  'InstanceGroups': DEFAULT_INSTANCE_GROUPS_WITH_EBS_CONFIG_MISSING_VOLSPEC}
339
        self.assert_params_for_cmd(cmd, result)
340

341
    def test_instance_groups_with_ebs_config_missing_iops(self):
342
        cmd = self.prefix
343
        cmd += INSTANCE_GROUPS_WITH_EBS_VOLUME_MISSING_IOPS_ARG
344
        result = {'JobFlowId': 'J-ABCD',
345
                  'InstanceGroups': DEFAULT_INSTANCE_GROUPS_WITH_EBS_CONFIG_MISSING_IOPS}
346
        self.assert_params_for_cmd(cmd, result)
347

348
    def test_instance_groups_with_ebs_config_multiple_instance_groups(self):
349
        cmd = self.prefix
350
        cmd += MULTIPLE_INSTANCE_GROUPS_WITH_EBS_VOLUMES_VOLUME_ARG
351
        result = {'JobFlowId': 'J-ABCD',
352
                  'InstanceGroups': DEFAULT_MULTIPLE_INSTANCE_GROUPS_WITH_EBS_CONFIG}
353
        self.assert_params_for_cmd(cmd, result)
354

355
    def test_instance_groups_with_custom_ami_instance_groups(self):
356
        cmd = self.prefix
357
        cmd += INSTANCE_GROUPS_WITH_CUSTOM_AMI_ARG
358
        result = {'JobFlowId': 'J-ABCD',
359
                  'InstanceGroups': DEFAULT_INSTANCE_GROUPS_WITH_CUSTOM_AMI}
360
        self.assert_params_for_cmd(cmd, result)
361

362
    @mock.patch('awscli.customizations.emr.emrutils.call')
363
    def test_constructed_result(self, call_patch):
364
        call_patch.return_value = ADD_INSTANCE_GROUPS_RESULT
365
        cmd = self.prefix
366
        cmd += ' InstanceGroupType=TASK,InstanceCount=10,InstanceType=m2.large'
367

368
        result = self.run_cmd(cmd, expected_rc=0)
369
        result_json = json.loads(result[0])
370

371
        self.assertEqual(result_json, CONSTRUCTED_RESULT)
372

373
if __name__ == "__main__":
374
    unittest.main()
375

376
Product

Resources

Company
