Path: blob/develop/awscli/customizations/emr/argumentschema.py
1567 views
# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.1#2# Licensed under the Apache License, Version 2.0 (the "License"). You3# may not use this file except in compliance with the License. A copy of4# the License is located at5#6# http://aws.amazon.com/apache2.0/7#8# or in the "license" file accompanying this file. This file is9# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF10# ANY KIND, either express or implied. See the License for the specific11# language governing permissions and limitations under the License.1213from awscli.customizations.emr import helptext14from awscli.customizations.emr.createdefaultroles import EC2_ROLE_NAME1516CONFIGURATIONS_PROPERTIES_SCHEMA = {17"type": "map",18"key": {19"type": "string",20"description": "Configuration key"21},22"value": {23"type": "string",24"description": "Configuration value"25},26"description": "Application configuration properties"27}2829CONFIGURATIONS_CLASSIFICATION_SCHEMA = {30"type": "string",31"description": "Application configuration classification name",32}3334INNER_CONFIGURATIONS_SCHEMA = {35"type": "array",36"items": {37"type": "object",38"properties": {39"Classification": CONFIGURATIONS_CLASSIFICATION_SCHEMA,40"Properties": CONFIGURATIONS_PROPERTIES_SCHEMA41}42},43"description": "Instance group application configurations."44}4546OUTER_CONFIGURATIONS_SCHEMA = {47"type": "array",48"items": {49"type": "object",50"properties": {51"Classification": CONFIGURATIONS_CLASSIFICATION_SCHEMA,52"Properties": CONFIGURATIONS_PROPERTIES_SCHEMA,53"Configurations": INNER_CONFIGURATIONS_SCHEMA54}55},56"description": "Instance group application configurations."57}5859ONDEMAND_CAPACITY_RESERVATION_OPTIONS_SCHEMA = {60"type": "object",61"properties" : {62"UsageStrategy": {63"type": "string",64"description": "The strategy of whether to use available capacity reservations to fulfill On-Demand capacity.",65"enum": ["use-capacity-reservations-first"]66},67"CapacityReservationPreference": {68"type": "string",69"description": "The preference of the capacity reservation of the instance.",70"enum": [71"open",72"none"73]74},75"CapacityReservationResourceGroupArn": {76"type": "string",77"description": "The ARN of the capacity reservation resource group in which to run the instance."78}79}80}8182SPOT_ALLOCATION_STRATEGY_SCHEMA = {83"type": "string",84"description": "The strategy to use to launch Spot instance fleets.",85"enum": ["capacity-optimized", "price-capacity-optimized", "lowest-price", "diversified", "capacity-optimized-prioritized"]86}8788ONDEMAND_ALLOCATION_STRATEGY_SCHEMA = {89"type": "string",90"description": "The strategy to use to launch On-Demand instance fleets.",91"enum": ["lowest-price", "prioritized"]92}9394INSTANCE_GROUPS_SCHEMA = {95"type": "array",96"items": {97"type": "object",98"properties": {99"Name": {100"type": "string",101"description":102"Friendly name given to the instance group."103},104"InstanceGroupType": {105"type": "string",106"description":107"The type of the instance group in the cluster.",108"enum": ["MASTER", "CORE", "TASK"],109"required": True110},111"BidPrice": {112"type": "string",113"description":114"Bid price for each Amazon EC2 instance in the "115"instance group when launching nodes as Spot Instances, "116"expressed in USD."117},118"InstanceType": {119"type": "string",120"description":121"The Amazon EC2 instance type for all instances "122"in the instance group.",123"required": True124},125"InstanceCount": {126"type": "integer",127"description": "Target number of Amazon EC2 instances "128"for the instance group",129"required": True130},131"CustomAmiId": {132"type": "string",133"description": "The AMI ID of a custom AMI to use when Amazon EMR provisions EC2 instances."134},135"EbsConfiguration": {136"type": "object",137"description": "EBS configuration that will be associated with the instance group.",138"properties": {139"EbsOptimized": {140"type": "boolean",141"description": "Boolean flag used to tag EBS-optimized instances.",142},143"EbsBlockDeviceConfigs": {144"type": "array",145"items": {146"type": "object",147"properties": {148"VolumeSpecification" : {149"type": "object",150"description": "The EBS volume specification that will be created and attached to every instance in this instance group.",151"properties": {152"VolumeType": {153"type": "string",154"description": "The EBS volume type that is attached to all the instances in the instance group. Valid types are: gp2, io1, and standard.",155"required": True156},157"SizeInGB": {158"type": "integer",159"description": "The EBS volume size, in GB, that is attached to all the instances in the instance group.",160"required": True161},162"Iops": {163"type": "integer",164"description": "The IOPS of the EBS volume that is attached to all the instances in the instance group.",165},166"Throughput": {167"type": "integer",168"description": "The throughput of the EBS volume that is attached to all the instances in the instance group.",169}170}171},172"VolumesPerInstance": {173"type": "integer",174"description": "The number of EBS volumes that will be created and attached to each instance in the instance group.",175}176}177}178}179}180},181"AutoScalingPolicy": {182"type": "object",183"description": "Auto Scaling policy that will be associated with the instance group.",184"properties": {185"Constraints": {186"type": "object",187"description": "The Constraints that will be associated to an Auto Scaling policy.",188"properties": {189"MinCapacity": {190"type": "integer",191"description": "The minimum value for the instances to scale in"192" to in response to scaling activities."193},194"MaxCapacity": {195"type": "integer",196"description": "The maximum value for the instances to scale out to in response"197" to scaling activities"198}199}200},201"Rules": {202"type": "array",203"description": "The Rules associated to an Auto Scaling policy.",204"items": {205"type": "object",206"properties": {207"Name": {208"type": "string",209"description": "Name of the Auto Scaling rule."210},211"Description": {212"type": "string",213"description": "Description of the Auto Scaling rule."214},215"Action": {216"type": "object",217"description": "The Action associated to an Auto Scaling rule.",218"properties": {219"Market": { # Required for Instance Fleets220"type": "string",221"description": "Market type of the Amazon EC2 instances used to create a "222"cluster node by Auto Scaling action.",223"enum": ["ON_DEMAND", "SPOT"]224},225"SimpleScalingPolicyConfiguration": {226"type": "object",227"description": "The Simple scaling configuration that will be associated"228"to Auto Scaling action.",229"properties": {230"AdjustmentType": {231"type": "string",232"description": "Specifies how the ScalingAdjustment parameter is "233"interpreted.",234"enum": ["CHANGE_IN_CAPACITY", "PERCENT_CHANGE_IN_CAPACITY",235"EXACT_CAPACITY"]236},237"ScalingAdjustment": {238"type": "integer",239"description": "The amount by which to scale, based on the "240"specified adjustment type."241},242"CoolDown": {243"type": "integer",244"description": "The amount of time, in seconds, after a scaling "245"activity completes and before the next scaling "246"activity can start."247}248}249}250}251},252"Trigger": {253"type": "object",254"description": "The Trigger associated to an Auto Scaling rule.",255"properties": {256"CloudWatchAlarmDefinition": {257"type": "object",258"description": "The Alarm to be registered with CloudWatch, to trigger"259" scaling activities.",260"properties": {261"ComparisonOperator": {262"type": "string",263"description": "The arithmetic operation to use when comparing the"264" specified Statistic and Threshold."265},266"EvaluationPeriods": {267"type": "integer",268"description": "The number of periods over which data is compared"269" to the specified threshold."270},271"MetricName": {272"type": "string",273"description": "The name for the alarm's associated metric."274},275"Namespace": {276"type": "string",277"description": "The namespace for the alarm's associated metric."278},279"Period": {280"type": "integer",281"description": "The period in seconds over which the specified "282"statistic is applied."283},284"Statistic": {285"type": "string",286"description": "The statistic to apply to the alarm's associated "287"metric."288},289"Threshold": {290"type": "double",291"description": "The value against which the specified statistic is "292"compared."293},294"Unit": {295"type": "string",296"description": "The statistic's unit of measure."297},298"Dimensions": {299"type": "array",300"description": "The dimensions for the alarm's associated metric.",301"items": {302"type": "object",303"properties": {304"Key": {305"type": "string",306"description": "Dimension Key."307},308"Value": {309"type": "string",310"description": "Dimension Value."311}312}313}314}315}316}317}318}319}320}321}322}323},324"Configurations": OUTER_CONFIGURATIONS_SCHEMA325}326}327}328329INSTANCE_FLEETS_SCHEMA = {330"type": "array",331"items": {332"type": "object",333"properties": {334"Name": {335"type": "string",336"description": "Friendly name given to the instance fleet."337},338"InstanceFleetType": {339"type": "string",340"description": "The type of the instance fleet in the cluster.",341"enum": ["MASTER", "CORE", "TASK"],342"required": True343},344"TargetOnDemandCapacity": {345"type": "integer",346"description": "Target on-demand capacity for the instance fleet."347},348"TargetSpotCapacity": {349"type": "integer",350"description": "Target spot capacity for the instance fleet."351},352"InstanceTypeConfigs": {353"type": "array",354"items": {355"type": "object",356"properties": {357"InstanceType": {358"type": "string",359"description": "The Amazon EC2 instance type for the instance fleet.",360"required": True361},362"WeightedCapacity": {363"type": "integer",364"description": "The weight assigned to an instance type, which will impact the overall fulfillment of the capacity."365},366"BidPrice": {367"type": "string",368"description": "Bid price for each Amazon EC2 instance in the "369"instance fleet when launching nodes as Spot Instances, "370"expressed in USD."371},372"BidPriceAsPercentageOfOnDemandPrice": {373"type": "double",374"description": "Bid price as percentage of on-demand price."375},376"CustomAmiId": {377"type": "string",378"description": "The AMI ID of a custom AMI to use when Amazon EMR provisions EC2 instances."379},380"Priority": {381"type": "double",382"description": "The priority at which Amazon EMR launches the EC2 instances with this instance type. "383"Priority starts at 0, which is the highest priority. Amazon EMR considers the highest priority first."384},385"EbsConfiguration": {386"type": "object",387"description": "EBS configuration that is associated with the instance group.",388"properties": {389"EbsOptimized": {390"type": "boolean",391"description": "Boolean flag used to tag EBS-optimized instances.",392},393"EbsBlockDeviceConfigs": {394"type": "array",395"items": {396"type": "object",397"properties": {398"VolumeSpecification" : {399"type": "object",400"description": "The EBS volume specification that is created "401"and attached to each instance in the instance group.",402"properties": {403"VolumeType": {404"type": "string",405"description": "The EBS volume type that is attached to all "406"the instances in the instance group. Valid types are: "407"gp2, io1, and standard.",408"required": True409},410"SizeInGB": {411"type": "integer",412"description": "The EBS volume size, in GB, that is attached "413"to all the instances in the instance group.",414"required": True415},416"Iops": {417"type": "integer",418"description": "The IOPS of the EBS volume that is attached to "419"all the instances in the instance group.",420},421"Throughput": {422"type": "integer",423"description": "The throughput of the EBS volume that is attached to "424"all the instances in the instance group.",425}426}427},428"VolumesPerInstance": {429"type": "integer",430"description": "The number of EBS volumes that will be created and "431"attached to each instance in the instance group.",432}433}434}435}436}437},438"Configurations": OUTER_CONFIGURATIONS_SCHEMA439}440}441},442"LaunchSpecifications": {443"type": "object",444"properties" : {445"OnDemandSpecification": {446"type": "object",447"properties": {448"AllocationStrategy": ONDEMAND_ALLOCATION_STRATEGY_SCHEMA,449"CapacityReservationOptions": ONDEMAND_CAPACITY_RESERVATION_OPTIONS_SCHEMA450}451},452"SpotSpecification": {453"type": "object",454"properties": {455"TimeoutDurationMinutes": {456"type": "integer",457"description": "The time, in minutes, after which the action specified in TimeoutAction field will be performed if requested resources are unavailable."458},459"TimeoutAction": {460"type": "string",461"description": "The action that is performed after TimeoutDurationMinutes.",462"enum": [463"TERMINATE_CLUSTER",464"SWITCH_TO_ONDEMAND"465]466},467"BlockDurationMinutes": {468"type": "integer",469"description": "Block duration in minutes."470},471"AllocationStrategy": SPOT_ALLOCATION_STRATEGY_SCHEMA472}473}474}475},476"ResizeSpecifications": {477"type": "object",478"properties": {479"SpotResizeSpecification": {480"type": "object",481"properties": {482"TimeoutDurationMinutes": {483"type" : "integer",484"description": "The time, in minutes, after which the resize will be stopped if requested resources are unavailable."485},486"AllocationStrategy": SPOT_ALLOCATION_STRATEGY_SCHEMA487}488},489"OnDemandResizeSpecification": {490"type": "object",491"properties": {492"TimeoutDurationMinutes": {493"type" : "integer",494"description": "The time, in minutes, after which the resize will be stopped if requested resources are unavailable."495},496"AllocationStrategy": ONDEMAND_ALLOCATION_STRATEGY_SCHEMA,497"CapacityReservationOptions": ONDEMAND_CAPACITY_RESERVATION_OPTIONS_SCHEMA498}499}500}501},502"Context": {503"type": "string",504"description": "Reserved."505}506}507}508}509510EC2_ATTRIBUTES_SCHEMA = {511"type": "object",512"properties": {513"KeyName": {514"type": "string",515"description":516"The name of the Amazon EC2 key pair that can "517"be used to ssh to the master node as the user 'hadoop'."518},519"SubnetId": {520"type": "string",521"description":522"To launch the cluster in Amazon "523"Virtual Private Cloud (Amazon VPC), set this parameter to "524"the identifier of the Amazon VPC subnet where you want "525"the cluster to launch. If you do not specify this value, "526"the cluster is launched in the normal Amazon Web Services "527"cloud, outside of an Amazon VPC. "528},529"SubnetIds": {530"type": "array",531"description":532"List of SubnetIds.",533"items": {534"type": "string"535}536},537"AvailabilityZone": {538"type": "string",539"description": "The Availability Zone the cluster will run in."540},541"AvailabilityZones": {542"type": "array",543"description": "List of AvailabilityZones.",544"items": {545"type": "string"546}547},548"InstanceProfile": {549"type": "string",550"description":551"An IAM role for the cluster. The EC2 instances of the cluster"552" assume this role. The default role is " +553EC2_ROLE_NAME + ". In order to use the default"554" role, you must have already created it using the "555"<code>create-default-roles</code> command. "556},557"EmrManagedMasterSecurityGroup": {558"type": "string",559"description": helptext.EMR_MANAGED_MASTER_SECURITY_GROUP560},561"EmrManagedSlaveSecurityGroup": {562"type": "string",563"description": helptext.EMR_MANAGED_SLAVE_SECURITY_GROUP564},565"ServiceAccessSecurityGroup": {566"type": "string",567"description": helptext.SERVICE_ACCESS_SECURITY_GROUP568},569"AdditionalMasterSecurityGroups": {570"type": "array",571"description": helptext.ADDITIONAL_MASTER_SECURITY_GROUPS,572"items": {573"type": "string"574}575},576"AdditionalSlaveSecurityGroups": {577"type": "array",578"description": helptext.ADDITIONAL_SLAVE_SECURITY_GROUPS,579"items": {580"type": "string"581}582}583}584}585586587APPLICATIONS_SCHEMA = {588"type": "array",589"items": {590"type": "object",591"properties": {592"Name": {593"type": "string",594"description": "Application name.",595"enum": ["MapR", "HUE", "HIVE", "PIG", "HBASE",596"IMPALA", "GANGLIA", "HADOOP", "SPARK"],597"required": True598},599"Args": {600"type": "array",601"description":602"A list of arguments to pass to the application.",603"items": {604"type": "string"605}606}607}608}609}610611BOOTSTRAP_ACTIONS_SCHEMA = {612"type": "array",613"items": {614"type": "object",615"properties": {616"Name": {617"type": "string",618"default": "Bootstrap Action"619},620"Path": {621"type": "string",622"description":623"Location of the script to run during a bootstrap action. "624"Can be either a location in Amazon S3 or "625"on a local file system.",626"required": True627},628"Args": {629"type": "array",630"description":631"A list of command line arguments to pass to "632"the bootstrap action script",633"items": {634"type": "string"635}636}637}638}639}640641642STEPS_SCHEMA = {643"type": "array",644"items": {645"type": "object",646"properties": {647"Type": {648"type": "string",649"description":650"The type of a step to be added to the cluster.",651"default": "custom_jar",652"enum": ["CUSTOM_JAR", "STREAMING", "HIVE", "PIG", "IMPALA"],653},654"Name": {655"type": "string",656"description": "The name of the step. ",657},658"ActionOnFailure": {659"type": "string",660"description": "The action to take if the cluster step fails.",661"enum": ["TERMINATE_CLUSTER", "CANCEL_AND_WAIT", "CONTINUE"],662"default": "CONTINUE"663},664"Jar": {665"type": "string",666"description": "A path to a JAR file run during the step.",667},668"Args": {669"type": "array",670"description":671"A list of command line arguments to pass to the step.",672"items": {673"type": "string"674}675},676"MainClass": {677"type": "string",678"description":679"The name of the main class in the specified "680"Java file. If not specified, the JAR file should "681"specify a Main-Class in its manifest file."682},683"Properties": {684"type": "string",685"description":686"A list of Java properties that are set when the step "687"runs. You can use these properties to pass key value "688"pairs to your main function."689}690}691}692}693694HBASE_RESTORE_FROM_BACKUP_SCHEMA = {695"type": "object",696"properties": {697"Dir": {698"type": "string",699"description": helptext.HBASE_BACKUP_DIR700},701"BackupVersion": {702"type": "string",703"description": helptext.HBASE_BACKUP_VERSION704}705}706}707708EMR_FS_SCHEMA = {709"type": "object",710"properties": {711"Consistent": {712"type": "boolean",713"description": "Enable EMRFS consistent view."714},715"SSE": {716"type": "boolean",717"description": "Enable Amazon S3 server-side encryption on files "718"written to S3 by EMRFS."719},720"RetryCount": {721"type": "integer",722"description":723"The maximum number of times to retry upon S3 inconsistency."724},725"RetryPeriod": {726"type": "integer",727"description": "The amount of time (in seconds) until the first "728"retry. Subsequent retries use an exponential "729"back-off."730},731"Args": {732"type": "array",733"description": "A list of arguments to pass for additional "734"EMRFS configuration.",735"items": {736"type": "string"737}738},739"Encryption": {740"type": "string",741"description": "EMRFS encryption type.",742"enum": ["SERVERSIDE", "CLIENTSIDE"]743},744"ProviderType": {745"type": "string",746"description": "EMRFS client-side encryption provider type.",747"enum": ["KMS", "CUSTOM"]748},749"KMSKeyId": {750"type": "string",751"description": "AWS KMS's customer master key identifier",752},753"CustomProviderLocation": {754"type": "string",755"description": "Custom encryption provider JAR location."756},757"CustomProviderClass": {758"type": "string",759"description": "Custom encryption provider full class name."760}761}762}763764TAGS_SCHEMA = {765"type": "array",766"items": {767"type": "string"768}769}770771KERBEROS_ATTRIBUTES_SCHEMA = {772"type": "object",773"properties": {774"Realm": {775"type": "string",776"description": "The name of Kerberos realm."777},778"KdcAdminPassword": {779"type": "string",780"description": "The password of Kerberos administrator."781},782"CrossRealmTrustPrincipalPassword": {783"type": "string",784"description": "The password to establish cross-realm trusts."785},786"ADDomainJoinUser": {787"type": "string",788"description": "The name of the user with privileges to join instances to Active Directory."789},790"ADDomainJoinPassword": {791"type": "string",792"description": "The password of the user with privileges to join instances to Active Directory."793}794}795}796797MANAGED_SCALING_POLICY_SCHEMA = {798"type": "object",799"properties": {800"ComputeLimits": {801"type": "object",802"description":803"The EC2 unit limits for a managed scaling policy. "804"The managed scaling activity of a cluster is not allowed to go above "805"or below these limits. The limits apply to CORE and TASK groups "806"and exclude the capacity of the MASTER group.",807"properties": {808"MinimumCapacityUnits": {809"type": "integer",810"description":811"The lower boundary of EC2 units. It is measured through "812"VCPU cores or instances for instance groups and measured "813"through units for instance fleets. Managed scaling "814"activities are not allowed beyond this boundary.",815"required": True816},817"MaximumCapacityUnits": {818"type": "integer",819"description":820"The upper boundary of EC2 units. It is measured through "821"VCPU cores or instances for instance groups and measured "822"through units for instance fleets. Managed scaling "823"activities are not allowed beyond this boundary.",824"required": True825},826"MaximumOnDemandCapacityUnits": {827"type": "integer",828"description":829"The upper boundary of on-demand EC2 units. It is measured through "830"VCPU cores or instances for instance groups and measured "831"through units for instance fleets. The on-demand units are not "832"allowed to scale beyond this boundary. "833"This value must be lower than MaximumCapacityUnits."834},835"UnitType": {836"type": "string",837"description": "The unit type used for specifying a managed scaling policy.",838"enum": ["VCPU", "Instances", "InstanceFleetUnits"],839"required": True840},841"MaximumCoreCapacityUnits": {842"type": "integer",843"description":844"The upper boundary of EC2 units for core node type in a cluster. "845"It is measured through VCPU cores or instances for instance groups "846"and measured through units for instance fleets. "847"The core units are not allowed to scale beyond this boundary. "848"The parameter is used to split capacity allocation between core and task nodes."849}850}851},852"ScalingStrategy": {853"type": "string",854"enum": ["DEFAULT", "ADVANCED"],855"description":856"Determines whether a custom scaling utilization performance index can be set. "857"Possible values include ADVANCED or DEFAULT."858},859"UtilizationPerformanceIndex": {860"type": "integer",861"description":862"An integer value that represents an advanced scaling strategy. "863"Setting a higher value optimizes for performance. "864"Setting a lower value optimizes for resource conservation. "865"Setting the value to 50 balances performance and resource conservation. "866"Possible values are 1, 25, 50, 75, and 100."867}868}869}870871PLACEMENT_GROUP_CONFIGS_SCHEMA = {872"type": "array",873"items": {874"type": "object",875"properties": {876"InstanceRole": {877"type": "string",878"description": "Role of the instance in the cluster.",879"enum": ["MASTER", "CORE", "TASK"],880"required": True881},882"PlacementStrategy": {883"type": "string",884"description": "EC2 Placement Group strategy associated "885"with instance role.",886"enum": ["SPREAD", "PARTITION", "CLUSTER", "NONE"]887}888}889}890}891892AUTO_TERMINATION_POLICY_SCHEMA = {893"type": "object",894"properties": {895"IdleTimeout": {896"type": "long",897"description":898"Specifies the amount of idle time in seconds after which the cluster automatically terminates. "899"You can specify a minimum of 60 seconds and a maximum of 604800 seconds (seven days).",900}901}902}903904905