elk7.2.0集群部署

 

国内elk快速下载源地址:
https://www.newbe.pro/Mirrors/Mirrors-Elasticsearch/

在地址中查找对应的版本下载即可。

#filebeat部署
filebeat-7.2.1-linux-x86_64.tar.gz
tar zxvf filebeat-7.2.1-linux-x86_64.tar.gz
cd filebeat-7.2.1-linux-x86_64
修改filebeat.yml文件。

vim filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /mnt/disk0/openresty-1.13.6.1/nginx/logs/yumc-api.log
  json.keys_under_root: true
  json.overwrite_keys: true

setup.kibana:
  host: "172.25.138.16:5601"

output.elasticsearch:
  hosts: ["172.25.138.16:9200","172.25.138.17:9200","172.25.138.18:9200"]
  index: "operation_platform_aimenu-%{[beat.version]}-%{+YYYY-MM-dd}"
setup.template.name: "operation_platform_aimenu"
setup.template.pattern: "operation_platform_aimenu-*"
setup.template.enabled: false
setup.template.overwrite: true
setup.ilm.enabled: auto
setup.ilm.rollover_alias: "operation_platform_aimenu"
setup.ilm.pattern: "{now/d}"
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false

创建启动脚本  
nohup ./filebeat -e -c filebeat.yml > nohup.out 2>&1 &

es部署
节点1
cd elasticsearch-7.2.1/config
[root@mw-es01 config]# egrep -v "#|^$" jvm.options
-Xms4G
-Xmx4G
-XX:+UseG1GC
-Des.networkaddress.cache.ttl=60
-Des.networkaddress.cache.negative.ttl=10
-XX:+AlwaysPreTouch
-Xss1m
-Djava.awt.headless=true
-Dfile.encoding=UTF-8
-Djna.nosys=true
-XX:-OmitStackTraceInFastThrow
-Dio.netty.noUnsafe=true
-Dio.netty.noKeySetOptimization=true
-Dio.netty.recycler.maxCapacityPerThread=0
-Dlog4j.shutdownHookEnabled=false
-Dlog4j2.disable.jmx=true
-Djava.io.tmpdir=${ES_TMPDIR}
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=data
-XX:ErrorFile=logs/hs_err_pid%p.log
8:-XX:+PrintGCDetails
8:-XX:+PrintGCDateStamps
8:-XX:+PrintTenuringDistribution
8:-XX:+PrintGCApplicationStoppedTime
8:-Xloggc:logs/gc.log
8:-XX:+UseGCLogFileRotation
8:-XX:NumberOfGCLogFiles=32
8:-XX:GCLogFileSize=64m
9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m
9-:-Djava.locale.providers=COMPAT

[root@mw-es01 config]# egrep -v "#|^$" elasticsearch.yml
cluster.name: es_cluster_log
node.master: true
node.data: true
node.ingest: true
path.data: /mnt/disk0/elasticsearch-7.2.1/data
path.logs: /mnt/disk0/elasticsearch-7.2.1/logs
network.host: 0.0.0.0
http.port: 9200
discovery.seed_hosts: ["172.25.138.16","172.25.138.17","172.25.138.18",]
cluster.initial_master_nodes: ["172.25.138.16","172.25.138.17","172.25.138.18",]
http.cors.enabled: true
http.cors.allow-origin: "*"
http.compression: true
http.compression_level: 3
path.repo: /tmp/es-backups

服务启动：
/bin/elasticsearch -d

节点2
cd elasticsearch-7.2.1/config
[root@mw-es02 config]# egrep -v "#|^$" jvm.options
-Xms4G
-Xmx4G
-XX:+UseG1GC
-Des.networkaddress.cache.ttl=60
-Des.networkaddress.cache.negative.ttl=10
-XX:+AlwaysPreTouch
-Xss1m
-Djava.awt.headless=true
-Dfile.encoding=UTF-8
-Djna.nosys=true
-XX:-OmitStackTraceInFastThrow
-Dio.netty.noUnsafe=true
-Dio.netty.noKeySetOptimization=true
-Dio.netty.recycler.maxCapacityPerThread=0
-Dlog4j.shutdownHookEnabled=false
-Dlog4j2.disable.jmx=true
-Djava.io.tmpdir=${ES_TMPDIR}
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=data
-XX:ErrorFile=logs/hs_err_pid%p.log
8:-XX:+PrintGCDetails
8:-XX:+PrintGCDateStamps
8:-XX:+PrintTenuringDistribution
8:-XX:+PrintGCApplicationStoppedTime
8:-Xloggc:logs/gc.log
8:-XX:+UseGCLogFileRotation
8:-XX:NumberOfGCLogFiles=32
8:-XX:GCLogFileSize=64m
9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m
9-:-Djava.locale.providers=COMPAT
[root@mw-es02 config]# egrep -v "#|^$" elasticsearch.yml
cluster.name: es_cluster_log
node.master: true
node.data: true
node.ingest: true
path.data: /mnt/disk0/elasticsearch-7.2.1/data
path.logs: /mnt/disk0/elasticsearch-7.2.1/logs
network.host: 0.0.0.0
http.port: 9200
discovery.seed_hosts: ["172.25.138.16","172.25.138.17","172.25.138.18",]
cluster.initial_master_nodes: ["172.25.138.16","172.25.138.17","172.25.138.18",]
http.cors.enabled: true
http.cors.allow-origin: "*"
http.compression: true
http.compression_level: 3
path.repo: /tmp/es-backups

服务启动：
/bin/elasticsearch -d

节点3
cd elasticsearch-7.2.1/config
[root@mw-es03 elasticsearch-7.2.1]# egrep -v "#|^$" jvm.options
grep: jvm.options: No such file or directory
[root@mw-es03 elasticsearch-7.2.1]# cd config/
[root@mw-es03 config]# egrep -v "#|^$" jvm.options
-Xms4G
-Xmx4G
-XX:+UseG1GC
-Des.networkaddress.cache.ttl=60
-Des.networkaddress.cache.negative.ttl=10
-XX:+AlwaysPreTouch
-Xss1m
-Djava.awt.headless=true
-Dfile.encoding=UTF-8
-Djna.nosys=true
-XX:-OmitStackTraceInFastThrow
-Dio.netty.noUnsafe=true
-Dio.netty.noKeySetOptimization=true
-Dio.netty.recycler.maxCapacityPerThread=0
-Dlog4j.shutdownHookEnabled=false
-Dlog4j2.disable.jmx=true
-Djava.io.tmpdir=${ES_TMPDIR}
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=data
-XX:ErrorFile=logs/hs_err_pid%p.log
8:-XX:+PrintGCDetails
8:-XX:+PrintGCDateStamps
8:-XX:+PrintTenuringDistribution
8:-XX:+PrintGCApplicationStoppedTime
8:-Xloggc:logs/gc.log
8:-XX:+UseGCLogFileRotation
8:-XX:NumberOfGCLogFiles=32
8:-XX:GCLogFileSize=64m
9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m
9-:-Djava.locale.providers=COMPAT
[root@mw-es03 config]# egrep -v "#|^$" elasticsearch.yml
cluster.name: es_cluster_log
node.master: true
node.data: true
node.ingest: true
path.data: /mnt/disk0/elasticsearch-7.2.1/data
path.logs: /mnt/disk0/elasticsearch-7.2.1/logs
network.host: 0.0.0.0
http.port: 9200
discovery.seed_hosts: ["172.25.138.16","172.25.138.17","172.25.138.18",]
cluster.initial_master_nodes: ["172.25.138.16","172.25.138.17","172.25.138.18",]
http.cors.enabled: true
http.cors.allow-origin: "*"
http.compression: true
http.compression_level: 3
path.repo: /tmp/es-backups
服务启动：
/bin/elasticsearch -d


kibana-7.2.1-linux-x86_64.tar.gz
kibana部署
tar zxvf kibana-7.2.1-linux-x86_64.tar.gz
cd kibana-7.2.1-linux-x86_64/config
vim kibana.yml
server.port: 5601
server.host: "172.25.138.16"
elasticsearch.hosts: ["http://172.25.138.16:9200","http://172.25.138.17:9200","http://172.25.138.18:9200"]
logging.dest: /mnt/disk0/kibana-7.2.1-linux-x86_64/logs/kibana.log

只需简单的配置即可。
启动服务：
nohup ./kibana --allow-root > nohup.out 2>&1 &

 

单节点多实例部署 配置文件
[root@kunpeng7 elasticsearch-5.6.4]# ls
172.25.203.40.mnt_disk2  data         esnode2      esnode3.pid  java_pid17474.hprof  java_pid48652.hprof  logs        plugins
bin                      esnode1      esnode2.pid  esnode5      java_pid459.hprof    lib                  modules     README.textile
config                   esnode1.pid  esnode3      esnode5.pid  java_pid46407.hprof  LICENSE.txt          NOTICE.txt
[root@kunpeng7 elasticsearch-5.6.4]# cd esnode3
[root@kunpeng7 esnode3]# ls
elasticsearch.yml  jvm.options  log4j2.properties  scripts

[root@kunpeng7 esnode3]# egrep -v "#|^$" elasticsearch.yml
cluster.name: prophet-es
node.name: hdp1
node.master: true
node.data: true
path.data: /mnt/disk2/es/elasticsearch-5.6.4/data/es3
path.logs: /mnt/disk2/es/elasticsearch-5.6.4/logs/es3
network.host: 172.25.203.40
http.port: 9202
transport.tcp.port: 9302
discovery.zen.ping.unicast.hosts: 172.25.203.40:9300,172.25.203.40:9301,172.25.203.40:9302,172.25.203.41:9300,172.25.203.41:9301,172.25.203.41:9302
discovery.zen.fd.ping_timeout: 180s
discovery.zen.fd.ping_retries: 8
discovery.zen.fd.ping_interval: 30s
discovery.zen.ping_timeout: 120s
node.max_local_storage_nodes: 3

[root@kunpeng7 esnode3]# egrep -v "#|^$" jvm.options
-Xms30g            #物理机一半的内存
-Xmx30g
-XX:+UseG1GC
-XX:MaxGCPauseMillis=200
-XX:+AlwaysPreTouch
-server
-Xss1m
-Djava.awt.headless=true
-Dfile.encoding=UTF-8
-Djna.nosys=true
-Djdk.io.permissionsUseCanonicalPath=true
-Dio.netty.noUnsafe=true
-Dio.netty.noKeySetOptimization=true
-Dio.netty.recycler.maxCapacityPerThread=0
-Dlog4j.shutdownHookEnabled=false
-Dlog4j2.disable.jmx=true
-Dlog4j.skipJansi=true
-XX:+HeapDumpOnOutOfMemoryError

[root@kunpeng7 esnode3]# egrep -v "#|^$" log4j2.properties
status = error
logger.action.name = org.elasticsearch.action
logger.action.level = debug
appender.console.type = Console
appender.console.name = console
appender.console.layout.type = PatternLayout
appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
appender.rolling.type = RollingFile
appender.rolling.name = rolling
appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
appender.rolling.layout.type = PatternLayout
appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}.log
appender.rolling.policies.type = Policies
appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
appender.rolling.policies.time.interval = 1
appender.rolling.policies.time.modulate = true
rootLogger.level = info
rootLogger.appenderRef.console.ref = console
rootLogger.appenderRef.rolling.ref = rolling
appender.deprecation_rolling.type = RollingFile
appender.deprecation_rolling.name = deprecation_rolling
appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
appender.deprecation_rolling.layout.type = PatternLayout
appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.log.gz
appender.deprecation_rolling.policies.type = Policies
appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy
appender.deprecation_rolling.policies.size.size = 1GB
appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy
appender.deprecation_rolling.strategy.max = 4
logger.deprecation.name = org.elasticsearch.deprecation
logger.deprecation.level = warn
logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling
logger.deprecation.additivity = false
appender.index_search_slowlog_rolling.type = RollingFile
appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog.log
appender.index_search_slowlog_rolling.layout.type = PatternLayout
appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog-%d{yyyy-MM-dd}.log
appender.index_search_slowlog_rolling.policies.type = Policies
appender.index_search_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
appender.index_search_slowlog_rolling.policies.time.interval = 1
appender.index_search_slowlog_rolling.policies.time.modulate = true
logger.index_search_slowlog_rolling.name = index.search.slowlog
logger.index_search_slowlog_rolling.level = trace
logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling
logger.index_search_slowlog_rolling.additivity = false
appender.index_indexing_slowlog_rolling.type = RollingFile
appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling
appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog.log
appender.index_indexing_slowlog_rolling.layout.type = PatternLayout
appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog-%d{yyyy-MM-dd}.log
appender.index_indexing_slowlog_rolling.policies.type = Policies
appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
appender.index_indexing_slowlog_rolling.policies.time.interval = 1
appender.index_indexing_slowlog_rolling.policies.time.modulate = true
logger.index_indexing_slowlog.name = index.indexing.slowlog.index
logger.index_indexing_slowlog.level = trace
logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling
logger.index_indexing_slowlog.additivity = false

#启动方式
./bin/elasticsearch -d -Epath.conf=./esnode3 -p ./esnode3.pid