applicationutils.py

# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

from awscli.customizations.emr import constants
from awscli.customizations.emr import emrutils
from awscli.customizations.emr import exceptions


def build_applications(region,
                       parsed_applications, ami_version=None):
    app_list = []
    step_list = []
    ba_list = []

    for app_config in parsed_applications:
        app_name = app_config['Name'].lower()

        if app_name == constants.HIVE:
            hive_version = constants.LATEST
            step_list.append(
                _build_install_hive_step(region=region))
            args = app_config.get('Args')
            if args is not None:
                hive_site_path = _find_matching_arg(
                    key=constants.HIVE_SITE_KEY, args_list=args)
                if hive_site_path is not None:
                    step_list.append(
                        _build_install_hive_site_step(
                            region=region,
                            hive_site_path=hive_site_path))
        elif app_name == constants.PIG:
            pig_version = constants.LATEST
            step_list.append(
                _build_pig_install_step(
                    region=region))
        elif app_name == constants.GANGLIA:
            ba_list.append(
                _build_ganglia_install_bootstrap_action(
                    region=region))
        elif app_name == constants.HBASE:
            ba_list.append(
                _build_hbase_install_bootstrap_action(
                    region=region))
            if ami_version >= '3.0':
                step_list.append(
                    _build_hbase_install_step(
                        constants.HBASE_PATH_HADOOP2_INSTALL_JAR))
            elif ami_version >= '2.1':
                step_list.append(
                    _build_hbase_install_step(
                        constants.HBASE_PATH_HADOOP1_INSTALL_JAR))
            else:
                raise ValueError('aws: error: AMI version ' + ami_version +
                                 'is not compatible with HBase.')
        elif app_name == constants.IMPALA:
            ba_list.append(
                _build_impala_install_bootstrap_action(
                    region=region,
                    args=app_config.get('Args')))
        else:
            app_list.append(
                _build_supported_product(
                    app_config['Name'], app_config.get('Args')))

    return app_list, ba_list, step_list


def _build_supported_product(name, args):
    if args is None:
        args = []
    config = {'Name': name.lower(), 'Args': args}
    return config


def _build_ganglia_install_bootstrap_action(region):
    return emrutils.build_bootstrap_action(
        name=constants.INSTALL_GANGLIA_NAME,
        path=emrutils.build_s3_link(
            relative_path=constants.GANGLIA_INSTALL_BA_PATH,
            region=region))


def _build_hbase_install_bootstrap_action(region):
    return emrutils.build_bootstrap_action(
        name=constants.INSTALL_HBASE_NAME,
        path=emrutils.build_s3_link(
            relative_path=constants.HBASE_INSTALL_BA_PATH,
            region=region))


def _build_hbase_install_step(jar):
    return emrutils.build_step(
        jar=jar,
        name=constants.START_HBASE_NAME,
        action_on_failure=constants.TERMINATE_CLUSTER,
        args=constants.HBASE_INSTALL_ARG)


def _build_impala_install_bootstrap_action(region, args=None):
    args_list = [
        constants.BASE_PATH_ARG,
        emrutils.build_s3_link(region=region),
        constants.IMPALA_VERSION,
        constants.LATEST]
    if args is not None:
        args_list.append(constants.IMPALA_CONF)
        args_list.append(','.join(args))
    return emrutils.build_bootstrap_action(
        name=constants.INSTALL_IMPALA_NAME,
        path=emrutils.build_s3_link(
            relative_path=constants.IMPALA_INSTALL_PATH,
            region=region),
        args=args_list)


def _build_install_hive_step(region,
                             action_on_failure=constants.TERMINATE_CLUSTER):
    step_args = [
        emrutils.build_s3_link(constants.HIVE_SCRIPT_PATH, region),
        constants.INSTALL_HIVE_ARG,
        constants.BASE_PATH_ARG,
        emrutils.build_s3_link(constants.HIVE_BASE_PATH, region),
        constants.HIVE_VERSIONS,
        constants.LATEST]
    step = emrutils.build_step(
        name=constants.INSTALL_HIVE_NAME,
        action_on_failure=action_on_failure,
        jar=emrutils.build_s3_link(constants.SCRIPT_RUNNER_PATH, region),
        args=step_args)
    return step


def _build_install_hive_site_step(region, hive_site_path,
                                  action_on_failure=constants.CANCEL_AND_WAIT):
    step_args = [
        emrutils.build_s3_link(constants.HIVE_SCRIPT_PATH, region),
        constants.BASE_PATH_ARG,
        emrutils.build_s3_link(constants.HIVE_BASE_PATH),
        constants.INSTALL_HIVE_SITE_ARG,
        hive_site_path,
        constants.HIVE_VERSIONS,
        constants.LATEST]
    step = emrutils.build_step(
        name=constants.INSTALL_HIVE_SITE_NAME,
        action_on_failure=action_on_failure,
        jar=emrutils.build_s3_link(constants.SCRIPT_RUNNER_PATH, region),
        args=step_args)
    return step


def _build_pig_install_step(region,
                            action_on_failure=constants.TERMINATE_CLUSTER):
    step_args = [
        emrutils.build_s3_link(constants.PIG_SCRIPT_PATH, region),
        constants.INSTALL_PIG_ARG,
        constants.BASE_PATH_ARG,
        emrutils.build_s3_link(constants.PIG_BASE_PATH, region),
        constants.PIG_VERSIONS,
        constants.LATEST]
    step = emrutils.build_step(
        name=constants.INSTALL_PIG_NAME,
        action_on_failure=action_on_failure,
        jar=emrutils.build_s3_link(constants.SCRIPT_RUNNER_PATH, region),
        args=step_args)
    return step


def _find_matching_arg(key, args_list):
    for arg in args_list:
        if key in arg:
            return arg

    return None