Skip to content

Instantly share code, notes, and snippets.

@abajwa-hw

abajwa-hw/deploy_hdf32.sh

Last active September 8, 2018 17:01
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save abajwa-hw/231bd92e790bac680c40a3dcd7c5a2c6 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save abajwa-hw/231bd92e790bac680c40a3dcd7c5a2c6 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. abajwa-hw revised this gist Sep 8, 2018. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion deploy_hdf32.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -6,7 +6,7 @@
    export ambari_password=${ambari_password:-StrongPassword}
    export db_password=${db_password:-StrongPassword}
    export nifi_password=${nifi_password:-StrongPassword}
    export ambari_services="ZOOKEEPER STREAMLINE NIFI KAFKA STORM REGISTRY NIFI_REGISTRY AMBARI_METRICS"
    export ambari_services="ZOOKEEPER STREAMLINE NIFI KAFKA STORM REGISTRY NIFI_REGISTRY KNOX AMBARI_METRICS"
    export cluster_name=${cluster_name:-hdf}
    export create_image=${create_image:-false}
    export host_count=${host_count:-1}
  2. abajwa-hw revised this gist Sep 7, 2018. 1 changed file with 3 additions and 0 deletions.
    3 changes: 3 additions & 0 deletions deploy_hdf32.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -101,6 +101,9 @@ cat << EOF > configuration-custom.json
    "ams-grafana-env": {
    "metrics_grafana_password": "${ambari_password}"
    },
    "kafka-broker": {
    "offsets.topic.replication.factor": "1"
    },
    "streamline-common": {
    "jar.storage.type": "local",
    "streamline.storage.type": "mysql",
  3. abajwa-hw revised this gist Sep 7, 2018. 1 changed file with 3 additions and 0 deletions.
    3 changes: 3 additions & 0 deletions deploy_hdf32.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -118,6 +118,9 @@ cat << EOF > configuration-custom.json
    "nifi-registry-ambari-config": {
    "nifi.registry.security.encrypt.configuration.password": "${nifi_password}"
    },
    "nifi-registry-properties": {
    "nifi.registry.db.password": "${nifi_password}"
    },
    "nifi-ambari-config": {
    "nifi.security.encrypt.configuration.password": "${nifi_password}"
    }
  4. abajwa-hw revised this gist Sep 7, 2018. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion deploy_hdf32.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -1,7 +1,7 @@
    #!/usr/bin/env bash
    # Launch Centos/RHEL 7 Vm with at least 4 cores / 16Gb mem / 60Gb disk
    # Then run:
    # curl -sSL https://gist.github.com/abajwa-hw/b7c027d9eea9fbd2a2319a21a955df1f/raw | sudo -E sh
    # curl -sSL https://gist.github.com/abajwa-hw/231bd92e790bac680c40a3dcd7c5a2c6/raw | sudo -E sh

    export ambari_password=${ambari_password:-StrongPassword}
    export db_password=${db_password:-StrongPassword}
  5. abajwa-hw created this gist Sep 7, 2018.
    150 changes: 150 additions & 0 deletions deploy_hdf32.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,150 @@
    #!/usr/bin/env bash
    # Launch Centos/RHEL 7 Vm with at least 4 cores / 16Gb mem / 60Gb disk
    # Then run:
    # curl -sSL https://gist.github.com/abajwa-hw/b7c027d9eea9fbd2a2319a21a955df1f/raw | sudo -E sh

    export ambari_password=${ambari_password:-StrongPassword}
    export db_password=${db_password:-StrongPassword}
    export nifi_password=${nifi_password:-StrongPassword}
    export ambari_services="ZOOKEEPER STREAMLINE NIFI KAFKA STORM REGISTRY NIFI_REGISTRY AMBARI_METRICS"
    export cluster_name=${cluster_name:-hdf}
    export create_image=${create_image:-false}
    export host_count=${host_count:-1}

    export ambari_version=2.7.0.0
    export mpack_url="http://public-repo-1.hortonworks.com/HDF/centos7/3.x/updates/3.2.0.0/tars/hdf_ambari_mp/hdf-ambari-mpack-3.2.0.0-520.tar.gz"
    export hdf_vdf="http://s3.amazonaws.com/public-repo-1.hortonworks.com/HDF/centos7/3.x/updates/3.2.0.0/HDF-3.2.0.0-520.xml"

    export ambari_stack_version=3.2
    export ambari_stack_name=HDF

    #service user for Ambari to start services on boot
    export service_user="demokitadmin"
    export service_password="BadPass#1"


    if [ "${create_image}" = true ]; then
    echo "updating /etc/hosts with demo.hortonworks.com entry pointing to VMs ip, hostname..."
    curl -sSL https://gist.github.com/abajwa-hw/9d7d06b8d0abf705ae311393d2ecdeec/raw | sudo -E sh
    sleep 5
    fi

    export host=$(hostname -f)
    echo "Hostname is: ${host}"


    echo Installing Packages...
    sudo yum localinstall -y https://dev.mysql.com/get/mysql57-community-release-el7-8.noarch.rpm
    sudo yum install -y git python-argparse epel-release mysql-connector-java* mysql-community-server nc

    # MySQL Setup to keep the new services separate from the originals
    echo Database setup...
    sudo systemctl enable mysqld.service
    sudo systemctl start mysqld.service
    #extract system generated Mysql password
    oldpass=$( grep 'temporary.*root@localhost' /var/log/mysqld.log | tail -n 1 | sed 's/.*root@localhost: //' )
    #create sql file that
    # 1. reset Mysql password to temp value and create druid/superset/registry/streamline schemas and users
    # 2. sets passwords for druid/superset/registry/streamline users to ${db_password}
    cat << EOF > mysql-setup.sql
    ALTER USER 'root'@'localhost' IDENTIFIED BY 'Secur1ty!';
    uninstall plugin validate_password;
    CREATE DATABASE registry DEFAULT CHARACTER SET utf8; CREATE DATABASE streamline DEFAULT CHARACTER SET utf8;
    CREATE USER 'registry'@'%' IDENTIFIED BY '${db_password}'; CREATE USER 'streamline'@'%' IDENTIFIED BY '${db_password}';
    GRANT ALL PRIVILEGES ON registry.* TO 'registry'@'%' WITH GRANT OPTION ; GRANT ALL PRIVILEGES ON streamline.* TO 'streamline'@'%' WITH GRANT OPTION ;
    commit;
    EOF
    #execute sql file
    mysql -h localhost -u root -p"$oldpass" --connect-expired-password < mysql-setup.sql
    #change Mysql password to ${db_password}
    mysqladmin -u root -p'Secur1ty!' password ${db_password}
    #test password and confirm dbs created
    mysql -u root -p${db_password} -e 'show databases;'
    # Install Ambari
    echo Installing Ambari

    export install_ambari_server=true
    #curl -sSL https://raw.githubusercontent.com/seanorama/ambari-bootstrap/master/ambari-bootstrap.sh | sudo -E sh
    curl -sSL https://raw.githubusercontent.com/abajwa-hw/ambari-bootstrap/master/ambari-bootstrap.sh | sudo -E sh
    sleep 15
    sudo ambari-server setup --jdbc-db=mysql --jdbc-driver=/usr/share/java/mysql-connector-java.jar
    sudo ambari-server install-mpack --verbose --mpack=${mpack_url}

    # Hack to fix a current bug in Ambari Blueprints
    sudo sed -i.bak "s/\(^ total_sinks_count = \)0$/\11/" /var/lib/ambari-server/resources/stacks/HDP/2.0.6/services/stack_advisor.py

    echo "Creating Storm View..."
    curl -u admin:admin -H "X-Requested-By:ambari" -X POST -d '{"ViewInstanceInfo":{"instance_name":"Storm_View","label":"Storm View","visible":true,"icon_path":"","icon64_path":"","description":"storm view","properties":{"storm.host":"'${host}'","storm.port":"8744","storm.sslEnabled":"false"},"cluster_type":"NONE"}}' http://${host}:8080/api/v1/views/Storm_Monitoring/versions/0.1.0/instances/Storm_View

    #create demokitadmin user
    curl -iv -u admin:admin -H "X-Requested-By: blah" -X POST -d "{\"Users/user_name\":\"${service_user}\",\"Users/password\":\"${service_password}\",\"Users/active\":\"true\",\"Users/admin\":\"true\"}" http://localhost:8080/api/v1/users

    echo "Updating admin password..."
    curl -iv -u admin:admin -H "X-Requested-By: blah" -X PUT -d "{ \"Users\": { \"user_name\": \"admin\", \"old_password\": \"admin\", \"password\": \"${ambari_password}\" }}" http://localhost:8080/api/v1/users/admin


    sudo ambari-server restart
    while ! echo exit | nc ${host} 8080; do echo "waiting for Ambari to be fully up..."; sleep 10; done


    echo "Deploying HDP and HDF services..."
    curl -ssLO https://github.com/seanorama/ambari-bootstrap/archive/master.zip
    unzip -q master.zip -d /tmp


    echo "downloading Blueprint configs template..."
    cd /tmp/ambari-bootstrap-master/deploy

    cat << EOF > configuration-custom.json
    {
    "configurations": {
    "ams-grafana-env": {
    "metrics_grafana_password": "${ambari_password}"
    },
    "streamline-common": {
    "jar.storage.type": "local",
    "streamline.storage.type": "mysql",
    "streamline.storage.connector.connectURI": "jdbc:mysql://$(hostname -f):3306/streamline",
    "registry.url" : "http://localhost:7788/api/v1",
    "streamline.dashboard.url" : "http://localhost:9089",
    "streamline.storage.connector.password": "${db_password}"
    },
    "registry-common": {
    "jar.storage.type": "local",
    "registry.storage.connector.connectURI": "jdbc:mysql://$(hostname -f):3306/registry",
    "registry.storage.type": "mysql",
    "registry.storage.connector.password": "${db_password}"
    },
    "nifi-registry-ambari-config": {
    "nifi.registry.security.encrypt.configuration.password": "${nifi_password}"
    },
    "nifi-ambari-config": {
    "nifi.security.encrypt.configuration.password": "${nifi_password}"
    }
    }
    }
    EOF

    #sed -i.bak "s/\[security\]/\[security\]\nforce_https_protocol=PROTOCOL_TLSv1_2/" /etc/ambari-agent/conf/ambari-agent.ini
    #bash -c "nohup ambari-agent restart" || true

    echo "Waiting for 30s before deploying cluster with services: ${ambari_services}"
    sleep 30
    sudo -E /tmp/ambari-bootstrap-master/deploy/deploy-recommended-cluster.bash

    echo Now open your browser to http://$(curl -s icanhazptr.com):8080 and login as admin/${ambari_password} to observe the cluster install

    echo "Waiting for cluster to be installed..."
    sleep 5

    ambari_pass="${ambari_password}" source /tmp/ambari-bootstrap-master/extras/ambari_functions.sh
    ambari_configs
    ambari_wait_request_complete 1


    if [ "${create_image}" = true ]; then
    echo "Setting up auto start of services on boot"
    curl -sSL https://gist.github.com/abajwa-hw/408134e032c05d5ff7e592cd0770d702/raw | sudo -E sh
    fi

    echo "Done!"