Azure vm 扩展脚本自动部署Elasticsearch集群

提示：扫一扫查出行
【扫一扫了解最新限行尾号】
复制提示

一、完整过程比较长，我仅给出Azure vm extension script 一键部署Elasticsearch集群的安装脚本，有需要的同学，可以邮件我，我给你完整的ARM Template

如果你不是用Azure VM ，也没关系，我的bash脚本都是一个个的函数，你可以自己改造成自己的Linux安装脚本。

二、此脚本的诞生是为了解决两个问题：

ELK在线安装有时候会异常缓慢，导致整个在线安装脚本奔溃
一个一个vm节点部署Elasticsearch集群比较繁琐，此处实现一键自动生成集群

大致的思路是先离线下载好安装包，存放到自己的容易下载的地方，我此处用的是Azure blob storage，只要达到一个目的：你能通过一个链接，能快速，方便地访问下载到Elasticsearch的安装包即可。利用Azure VM extension，调用我给出的脚本，自行下载和安装。

三、达到的目的：

运行成功后，自动创建多台虚拟机，并且虚拟机ES之间组成集群
ES集群得到初步优化，vm system在适配ES上也得到初步优化

四、安装脚本：

#!/bin/bash # Copyright (c) Microsoft. All rights reserved. # Licensed under ELASTIC LICENSE ## # Author : Wenbo Yang July/16/2018 ## # Reference: https://github.com/Azure/azure-diagnostics-tools/blob/master/ELK-Semantic-Logging/ELK/AzureRM/logstash-on-ubuntu/logstash-install-ubuntu.sh # Reference: https://github.com/Azure/azure-quickstart-templates/blob/master/elasticsearch/scripts/elasticsearch-ubuntu-install.sh # Reference: https://github.com/Azure/azure-quickstart-templates/blob/master/elasticsearch-vmss/install-elasticsearch.sh # Reference: https://github.com/Azure/azure-diagnostics-tools/blob/master/ES-MultiNode/es-ubuntu-install.sh ### Log method to control/redirect log output log() { echo "$1" logger "$1" }if [ "${UID}" -ne 0 ]; then log "Script executed without root permissions" echo "You must be root to run this program." >&2 exit 3 fi# TEMP FIX - Re-evaluate and remove when possible # This is an interim fix for hostname resolution in current VM grep -q "${HOSTNAME}" /etc/hosts if [ $? == 0 ] then echo "${HOSTNAME} found in /etc/hosts" else echo "${HOSTNAME} not found in /etc/hosts" # Append it to the hosts file if not there echo "127.0.0.1 ${HOSTNAME}" >> /etc/hosts log "hostname ${HOSTNAME} added to /etc/hosts" fi#Loop through options passed while getopts :p:c:m:e:k:h optname; do log "Option $optname set with value ${OPTARG}" case $optname in p) #set cluster name FIRSTPRIVATEIP=${OPTARG} ;; c) #set cluster name NODECOUNT=${OPTARG} ;; e) #set master mode ES_DOWNLOAD_URL=${OPTARG} ;; k) #set master mode KIBANA_DOWNLOAD_URL=${OPTARG} ;; h) #show help help exit 2 ;; \?) #unrecognized option - show help echo -e \\n"Option -${BOLD}$OPTARG${NORM} not allowed." help exit 2 ;; esac done# Usage: get_discovery_endpoints start_address node_count # Example: get_discovery_endpoints 10.0.1.4 3 # (returns ["10.0.1.4", "10.0.1.5", "10.0.1.6"] get_discovery_endpoints() { declare start_address=$1 declare address_prefix=${start_address%.*} # Everything up to last dot (not including) declare -i address_suffix_start=${start_address##*.} # Last part of the address, interpreted as a number declare retval='[' declare -i i declare -i suffix for (( i=0; i<$2; ++i )); do suffix=$(( address_suffix_start + i )) retval+="\"${address_prefix}.${suffix}\", " done retval=${retval:0:-2} # Remove last comma and space retval+=']' echo $retval } # Install Oracle Java install_java() { log "Installing Java" add-apt-repository -y ppa:webupd8team/java apt-get -y update > /dev/null echo debconf shared/accepted-oracle-license-v1-1 select true | sudo debconf-set-selections echo debconf shared/accepted-oracle-license-v1-1 seen true | sudo debconf-set-selections apt-get -y install oracle-java8-installer > /dev/null java -version if [ $? -ne 0 ]; then log "Java installation failed" exit 1 fi }# Install ES with Debian Package manually install_es() { log "Installing Elaticsearch" sudo wget -q "$ES_DOWNLOAD_URL" -O elasticsearch.deb sudo dpkg -i elasticsearch.deb }# Install Kibana with Debina Package manually install_kibana() { log "Installing Kibana" sudo wget -q "$KIBANA_DOWNLOAD_URL" -O kibana.deb sudo dpkg -i kibana.deb }# Configure elasticsearch configure_es() { log "Update configuration" mv /etc/elasticsearch/elasticsearch.yml /etc/elasticsearch/elasticsearch.bak echo "cluster.name: elasticsearch" >> /etc/elasticsearch/elasticsearch.yml echo "node.name: ${HOSTNAME}" >> /etc/elasticsearch/elasticsearch.yml declare -i minimum_master_nodes=$((($NODECOUNT / 2) + 1)) echo "discovery.zen.minimum_master_nodes: 2" >> /etc/elasticsearch/elasticsearch.yml discovery_endpoints=$(get_discovery_endpoints $FIRSTPRIVATEIP $NODECOUNT) echo $discovery_endpoints echo "discovery.zen.ping.unicast.hosts: $discovery_endpoints" >> /etc/elasticsearch/elasticsearch.yml IPADDRESS=$(ip route get 8.8.8.8 | awk 'NR==1 {print $NF}') echo "network.host: ${IPADDRESS}" >> /etc/elasticsearch/elasticsearch.yml echo "http.port: 9200" >> /etc/elasticsearch/elasticsearch.yml echo "bootstrap.memory_lock: true" >> /etc/elasticsearch/elasticsearch.yml echo "node.master: true" >> /etc/elasticsearch/elasticsearch.yml echo "node.data: true" >> /etc/elasticsearch/elasticsearch.yml echo "path.data: /var/lib/elasticsearch" >> /etc/elasticsearch/elasticsearch.yml echo "path.logs: /var/log/elasticsearch" >> /etc/elasticsearch/elasticsearch.yml sudo update-rc.d elasticsearch defaults 95 10 service elasticsearch start #sudo systemctl stop elasticsearch.service sleep 30 if [ <code>systemctl is-failed elasticsearch.service</code> == 'failed' ]; then log "Elasticsearch unit failed to start" exit 1 fi }# Configure kibana configure_kibana() { IPADDRESS=$(ip route get 8.8.8.8 | awk 'NR==1 {print $NF}') echo "server.host: \"${IPADDRESS}\"" >> /etc/kibana/kibana.yml echo "elasticsearch.url: \"http://${IPADDRESS}:9200\"" >> /etc/kibana/kibana.yml sudo update-rc.d kibana defaults 95 10 service kibana start #sudo systemctl stop kibana.service sleep 10 if [ <code>systemctl is-failed kibana.service</code> == 'failed' ]; then log "Kibana unit failed to start" exit 1 fi }configure_system() { echo "options timeout:1 attempts:5" >> /etc/resolvconf/resolv.conf.d/head resolvconf -u ES_HEAP=<code>free -m |grep Mem | awk '{if ($2/2 >31744) print 31744;else printf "%.0f", $2/2;}'</code> echo "ES_JAVA_OPTS=\"-Xms${ES_HEAP}m -Xmx${ES_HEAP}m\"" >> /etc/default/elasticsearch echo "JAVA_HOME=$JAVA_HOME" >> /etc/default/elasticsearch echo 'MAX_OPEN_FILES=65536' >> /etc/default/elasticsearch echo 'MAX_LOCKED_MEMORY=unlimited' >> /etc/default/elasticsearch #https://www.elastic.co/guide/en/elasticsearch/reference/current/setting-system-settings.html#systemd mkdir -p /etc/systemd/system/elasticsearch.service.d touch /etc/systemd/system/elasticsearch.service.d/override.conf echo '[Service]' >> /etc/systemd/system/elasticsearch.service.d/override.conf echo 'LimitMEMLOCK=infinity' >> /etc/systemd/system/elasticsearch.service.d/override.conf }log " ---------------begin------------------- " install_java log "Master Node install elasticsearch and kibana" log "Install configure and start elasticsearch" install_es configure_system configure_es log "Install configure and start kibana" install_kibana configure_kibana