Create EMR

Use shell to setup emr

 

project_name=$1

create emr cluster

cluster_id_json=`aws emr create-cluster \
--region us-west-2 \
--name "${project_name}" \
--release-label emr-5.13.0 \
--instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m4.xlarge InstanceGroupType=CORE,InstanceCount=1,InstanceType=m4.xlarge \
--ec2-attributes KeyName=dpppkey,InstanceProfile=DataPipelineDefaultResourceRole,SubnetId=subnet-e2f2dfa4 \
--service-role DataPipelineDefaultRole \
--applications Name=Spark Name=hive`
 
sleep 2m
 
id_var=$(echo ${cluster_id_json}|grep -o  -E "ClusterId.*")
id_var1=${id_var:13}
cluster_id=${id_var1%%\"*}
 
echo cluster_id : $cluster_id

 

get cluster info

sleep 2m
cluster_info=`aws --region us-west-2 emr describe-cluster --cluster-id $cluster_id`
echo cluster_info : $cluster_info
echo ==================================get master id ======================================
sleep 2m
master_var=$(echo ${cluster_info}|grep -o -E "MASTER.*" | grep -o  -E "Id(.*?)")
master_var1=${master_var:6}
master_id=${master_var1%%\"*}
echo master_id : $master_id

 

get instance info

sleep 2m
instance_group_info=`aws emr --region us-west-2 list-instances --cluster-id $cluster_id`
echo instance_group_info : $instance_group_info
echo ==================================get master ip =======================================
sleep 2m
ip_var=$(echo ${instance_group_info}|grep -o -E "$master_id.*" | grep -o  -E "PrivateIpAddress(.*?)")
ip_var1=${ip_var:20}
echo $ip_var1
private_ip_address=${ip_var1%%\"*}
echo private_ip_address : $private_ip_address
echo project_name : $project_name

 

update rundeck xml file

line=`grep -n "name=\"${project_name}\"" /data/01/home/tnuser/rundeck/projects/trips/etc/resources.xml | cut -d : -f 1`
if [ $line -gt "0" ];then
echo ${project_name} is exist in /data/01/home/tnuser/rundeck/projects/trips/etc/resources.xml
sed -i "${line}d" /data/01/home/tnuser/rundeck/projects/trips/etc/resources.xml
fi

sed "2 a<node name=\"${project_name}\" description=\"Rundeck Client Node\" tags=\"\" hostname=\"${private_ip_address}\" osArch=\"amd64\" osFamily=\"unix\" osName=\"Linux\" osVersion=\"3.10.0-693.11.6.el7.x86_64\" username=\"hadoop\"/>" -i /data/01/home/tnuser/rundeck/projects/trips/etc/resources.xml
status_num=$(echo ${instance_group_info}|grep -c "State")
running_num=$(echo ${instance_group_info}|grep -c "RUNNING")

if [ $status_num -eq $running_num ];then
echo emr start success !!
fi