constants.py

# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

# Declare all the constants used by EMR in this file.

EC2_ROLE_NAME = "EMR_EC2_DefaultRole"
EMR_ROLE_NAME = "EMR_DefaultRole"
EMR_AUTOSCALING_ROLE_NAME = "EMR_AutoScaling_DefaultRole"
ROLE_ARN_PATTERN = "arn:{{region_suffix}}:iam::aws:policy/service-role/{{policy_name}}"
EC2_ROLE_POLICY_NAME = "AmazonElasticMapReduceforEC2Role"
EMR_ROLE_POLICY_NAME = "AmazonElasticMapReduceRole"
EMR_AUTOSCALING_ROLE_POLICY_NAME = "AmazonElasticMapReduceforAutoScalingRole"
EMR_AUTOSCALING_SERVICE_NAME = "application-autoscaling"
EMR_AUTOSCALING_SERVICE_PRINCIPAL = "application-autoscaling.amazonaws.com"
EC2_SERVICE_PRINCIPAL = "ec2.amazonaws.com"

# Action on failure
CONTINUE = 'CONTINUE'
CANCEL_AND_WAIT = 'CANCEL_AND_WAIT'
TERMINATE_CLUSTER = 'TERMINATE_CLUSTER'
DEFAULT_FAILURE_ACTION = CONTINUE

# Market type
SPOT = 'SPOT'
ON_DEMAND = 'ON_DEMAND'

SCRIPT_RUNNER_PATH = '/libs/script-runner/script-runner.jar'
COMMAND_RUNNER = 'command-runner.jar'
DEBUGGING_PATH = '/libs/state-pusher/0.1/fetch'
DEBUGGING_COMMAND = 'state-pusher-script'
DEBUGGING_NAME = 'Setup Hadoop Debugging'

CONFIG_HADOOP_PATH = '/bootstrap-actions/configure-hadoop'

# S3 copy bootstrap action
S3_GET_BA_NAME = 'S3 get'
S3_GET_BA_SRC = '-s'
S3_GET_BA_DEST = '-d'
S3_GET_BA_FORCE = '-f'

# EMRFS
EMRFS_BA_NAME = 'Setup EMRFS'
EMRFS_BA_ARG_KEY = '-e'
EMRFS_CONSISTENT_KEY = 'fs.s3.consistent'
EMRFS_SSE_KEY = 'fs.s3.enableServerSideEncryption'
EMRFS_RETRY_COUNT_KEY = 'fs.s3.consistent.retryCount'
EMRFS_RETRY_PERIOD_KEY = 'fs.s3.consistent.retryPeriodSeconds'
EMRFS_CSE_KEY = 'fs.s3.cse.enabled'
EMRFS_CSE_KMS_KEY_ID_KEY = 'fs.s3.cse.kms.keyId'
EMRFS_CSE_ENCRYPTION_MATERIALS_PROVIDER_KEY = \
    'fs.s3.cse.encryptionMaterialsProvider'
EMRFS_CSE_CUSTOM_PROVIDER_URI_KEY = 'fs.s3.cse.encryptionMaterialsProvider.uri'

EMRFS_CSE_KMS_PROVIDER_FULL_CLASS_NAME = ('com.amazon.ws.emr.hadoop.fs.cse.'
                                          'KMSEncryptionMaterialsProvider')
EMRFS_CSE_CUSTOM_S3_GET_BA_PATH = 'file:/usr/share/aws/emr/scripts/s3get'
EMRFS_CUSTOM_DEST_PATH = '/usr/share/aws/emr/auxlib'

EMRFS_SERVER_SIDE = 'SERVERSIDE'
EMRFS_CLIENT_SIDE = 'CLIENTSIDE'
EMRFS_KMS = 'KMS'
EMRFS_CUSTOM = 'CUSTOM'

EMRFS_SITE = 'emrfs-site'

MAX_BOOTSTRAP_ACTION_NUMBER = 16
BOOTSTRAP_ACTION_NAME = 'Bootstrap action'

HIVE_BASE_PATH = '/libs/hive'
HIVE_SCRIPT_PATH = '/libs/hive/hive-script'
HIVE_SCRIPT_COMMAND = 'hive-script'

PIG_BASE_PATH = '/libs/pig'
PIG_SCRIPT_PATH = '/libs/pig/pig-script'
PIG_SCRIPT_COMMAND = 'pig-script'

GANGLIA_INSTALL_BA_PATH = '/bootstrap-actions/install-ganglia'

# HBase
HBASE_INSTALL_BA_PATH = '/bootstrap-actions/setup-hbase'
HBASE_PATH_HADOOP1_INSTALL_JAR = '/home/hadoop/lib/hbase-0.92.0.jar'
HBASE_PATH_HADOOP2_INSTALL_JAR = '/home/hadoop/lib/hbase.jar'
HBASE_INSTALL_ARG = ['emr.hbase.backup.Main', '--start-master']
HBASE_JAR_PATH = '/home/hadoop/lib/hbase.jar'
HBASE_MAIN = 'emr.hbase.backup.Main'

# HBase commands
HBASE_RESTORE = '--restore'
HBASE_BACKUP_DIR_FOR_RESTORE = '--backup-dir-to-restore'
HBASE_BACKUP_VERSION_FOR_RESTORE = '--backup-version'
HBASE_BACKUP = '--backup'
HBASE_SCHEDULED_BACKUP = '--set-scheduled-backup'
HBASE_BACKUP_DIR = '--backup-dir'
HBASE_INCREMENTAL_BACKUP_INTERVAL = '--incremental-backup-time-interval'
HBASE_INCREMENTAL_BACKUP_INTERVAL_UNIT = '--incremental-backup-time-unit'
HBASE_FULL_BACKUP_INTERVAL = '--full-backup-time-interval'
HBASE_FULL_BACKUP_INTERVAL_UNIT = '--full-backup-time-unit'
HBASE_DISABLE_FULL_BACKUP = '--disable-full-backups'
HBASE_DISABLE_INCREMENTAL_BACKUP = '--disable-incremental-backups'
HBASE_BACKUP_STARTTIME = '--start-time'
HBASE_BACKUP_CONSISTENT = '--consistent'
HBASE_BACKUP_STEP_NAME = 'Backup HBase'
HBASE_RESTORE_STEP_NAME = 'Restore HBase'
HBASE_SCHEDULE_BACKUP_STEP_NAME = 'Modify Backup Schedule'

IMPALA_INSTALL_PATH = '/libs/impala/setup-impala'

# Step
HADOOP_STREAMING_PATH = '/home/hadoop/contrib/streaming/hadoop-streaming.jar'
HADOOP_STREAMING_COMMAND = 'hadoop-streaming'

CUSTOM_JAR = 'custom_jar'
HIVE = 'hive'
PIG = 'pig'
IMPALA = 'impala'
STREAMING = 'streaming'
GANGLIA = 'ganglia'
HBASE = 'hbase'
SPARK = 'spark'

DEFAULT_CUSTOM_JAR_STEP_NAME = 'Custom JAR'
DEFAULT_STREAMING_STEP_NAME = 'Streaming program'
DEFAULT_HIVE_STEP_NAME = 'Hive program'
DEFAULT_PIG_STEP_NAME = 'Pig program'
DEFAULT_IMPALA_STEP_NAME = 'Impala program'
DEFAULT_SPARK_STEP_NAME = 'Spark application'

ARGS = '--args'
RUN_HIVE_SCRIPT = '--run-hive-script'
HIVE_VERSIONS = '--hive-versions'
HIVE_STEP_CONFIG = 'HiveStepConfig'
RUN_PIG_SCRIPT = '--run-pig-script'
PIG_VERSIONS = '--pig-versions'
PIG_STEP_CONFIG = 'PigStepConfig'
RUN_IMPALA_SCRIPT = '--run-impala-script'
SPARK_SUBMIT_PATH = '/home/hadoop/spark/bin/spark-submit'
SPARK_SUBMIT_COMMAND = 'spark-submit'
IMPALA_STEP_CONFIG = 'ImpalaStepConfig'
SPARK_STEP_CONFIG = 'SparkStepConfig'
STREAMING_STEP_CONFIG = 'StreamingStepConfig'
CUSTOM_JAR_STEP_CONFIG = 'CustomJARStepConfig'

INSTALL_PIG_ARG = '--install-pig'
INSTALL_PIG_NAME = 'Install Pig'
INSTALL_HIVE_ARG = '--install-hive'
INSTALL_HIVE_NAME = 'Install Hive'
HIVE_SITE_KEY = '--hive-site'
INSTALL_HIVE_SITE_ARG = '--install-hive-site'
INSTALL_HIVE_SITE_NAME = 'Install Hive Site Configuration'
BASE_PATH_ARG = '--base-path'
INSTALL_GANGLIA_NAME = 'Install Ganglia'
INSTALL_HBASE_NAME = 'Install HBase'
START_HBASE_NAME = 'Start HBase'
INSTALL_IMPALA_NAME = 'Install Impala'
IMPALA_VERSION = '--impala-version'
IMPALA_CONF = '--impala-conf'

FULL = 'full'
INCREMENTAL = 'incremental'

MINUTES = 'minutes'
HOURS = 'hours'
DAYS = 'days'
NOW = 'now'

TRUE = 'true'
FALSE = 'false'

EC2 = 'ec2'
EMR = 'elasticmapreduce'
APPLICATION_AUTOSCALING = 'application-autoscaling'
LATEST = 'latest'

APPLICATIONS = ["HIVE", "PIG", "HBASE", "GANGLIA", "IMPALA", "SPARK", "MAPR",
                "MAPR_M3", "MAPR_M5", "MAPR_M7"]

SSH_USER = 'hadoop'
STARTING_STATES = ['STARTING', 'BOOTSTRAPPING']
TERMINATED_STATES = ['TERMINATED', 'TERMINATING', 'TERMINATED_WITH_ERRORS']

# list-clusters
LIST_CLUSTERS_ACTIVE_STATES = ['STARTING', 'BOOTSTRAPPING', 'RUNNING',
                               'WAITING', 'TERMINATING']
LIST_CLUSTERS_TERMINATED_STATES = ['TERMINATED']
LIST_CLUSTERS_FAILED_STATES = ['TERMINATED_WITH_ERRORS']

INSTANCE_FLEET_TYPE = 'INSTANCE_FLEET'