elasticsearch、logstash、kibana配置与安装

一、ElasticSearch下载与配置启动

1. ElasticSearch下载

[root@iZ2zeapnvuohe8p14289u6Z /]# cd /usr/local
[root@iZ2zeapnvuohe8p14289u6Z /]# mkdir soft
[root@iZ2zeapnvuohe8p14289u6Z /]# cd soft
[root@iZ2zeapnvuohe8p14289u6Z /]# mkdir elk
[root@iZ2zeapnvuohe8p14289u6Z /]# wget  https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.2.2.tar.gz
[root@iZ2zeapnvuohe8p14289u6Z /]# tar -zxvf elasticsearch-6.2.2.tar.gz
[root@iZ2zeapnvuohe8p14289u6Z /]# cd elasticsearch-6.2.2
[root@iZ2zeapnvuohe8p14289u6Z /]# ll

   

2.启动ElasticSearch

#注:如果服务器内存小的话，启动会报如下错误:

[root@iZ2zeapnvuohe8p14289u6Z elasticsearch-6.2.2]#  cd bin
[root@iZ2zeapnvuohe8p14289u6Z bin]# ./elasticsearch

   

 #注：启动elasticsearch会出现如下情况，不能用root用户启动！

[root@iZ2zeapnvuohe8p14289u6Z bin]#  ./elasticsearch

 [2019-12-02T20:14:04,870][WARN ][o.e.b.ElasticsearchUncaughtExceptionHandler] [] uncaught exception in thread [main]org.elasticsearch.bootstrap.StartupException: java.lang.RuntimeException: can not run elasticsearch as root at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:125) ~[elasticsearch-6.2.2.jar:6.2.2] at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:112) ~[elasticsearch-6.2.2.jar:6.2.2] at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:86) ~[elasticsearch-6.2.2.jar:6.2.2] at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:124) ~[elasticsearch-cli-6.2.2.jar:6.2.2]

#注：解决办法

 [root@iZ2zeapnvuohe8p14289u6Z ]# cd /usr/local/soft/elk 
 # 赋予所有权限用户
 [root@iZ2zeapnvuohe8p14289u6Z ]# chmod -R 777 ./
 #添加用户：useradd -m 用户名  然后设置密码  passwd 用户名
 [root@iZ2zeapnvuohe8p14289u6Z ]# su - elk #切换刚才新增的用户
 #第一种启动方式(不建议使用)
 [root@iZ2zeapnvuohe8p14289u6Z bin]#  ./elasticsearch
 #第二种启动方式(推荐)
 [root@iZ2zeapnvuohe8p14289u6Z bin]#  nohup ./elasticsearch > /dev/null 2>&1 & 

3.ElasticSearch配置文件

[root@iZ2zeapnvuohe8p14289u6Z ]# cd /usr/local/soft/elk/elasticsearch-6.2.2/config
[root@iZ2zeapnvuohe8p14289u6Z ]# vim elasticsearch.yml

#修改内容如下:

cluster.name: zkxh-application #集群名称，同一个网段自动加入
node.name: zkxh-node-1 #节点名称
network.host: 0.0.0.0
transport.tcp.port: 9099
http.port: 9200 #http端口
discovery.zen.ping.unicast.hosts: ["123.56.85.94:9099"]
#修改其它配置
[root@iZ2zeapnvuohe8p14289u6Z ]# vim /etc/sysctl.conf
#设置fs.file-max=655350

    

[root@iZ2zeapnvuohe8p14289u6Z ]# vim /etc/security/limits.conf
#新增：
* soft nofile 655350
* hard nofile 655350

     

 4.常用搜索语句

#集群健康检查

http://localhost:9200/_cat/health?v
http://localhost:9200/_cluster/health（推荐）
#查询节点列表
http://localhost:9200/_cat/nodes?v
#查看所有索引
http://localhost:9200/_cat/indices?v
#新增索引
curl -XPUT 'localhost:9200/blog_test?pretty'
curl -XPUT 'localhost:9200/zkxh?pretty'
#删除索引
curl -XDELETE 'localhost:9200/blog_test?pretty'
#美化推荐:在后面加?pretty

二、Logstash插件下载与配置启动

1. Logstash 插件下载

[root@iZ2zeapnvuohe8p14289u6Z ]# cd /usr/local/soft/elk
#下载logstash插件
[root@iZ2zeapnvuohe8p14289u6Z elk]# wget https://artifacts.elastic.co/downloads/logstash/logstash-6.2.2.tar.gz
[root@iZ2zeapnvuohe8p14289u6Z elk]# tar -zxvf logstash-6.2.2.tar.gz
[root@iZ2zeapnvuohe8p14289u6Z elk]# cd logstash-6.2.2
[root@iZ2zeapnvuohe8p14289u6Z logstash-6.2.2]# ll

   

[root@iZ2zeapnvuohe8p14289u6Z logstash-6.2.2]# cd config
[root@iZ2zeapnvuohe8p14289u6Z config]#vim logstash.yml

   

 #配置讲解:

 https://www.elastic.co/guide/en/logstash/6.2/logstash-settings-file.html

 logstash.yml  修改 pipeline.workers，根据CPU核数增加1到2即可

 jvm.options   修改 xms和xmx为相同，一般是系统内存三份之二

 2.测试启动Logstash

[root@iZ2zeapnvuohe8p14289u6Z ]# cd /usr/local/soft/elk/logstash-6.2.2/bin
[root@iZ2zeapnvuohe8p14289u6Z logstash-6.2.2]# ./logstash -e 'input {stdin {}} output {stdout {}}'

   

 3.Logstash 读取数据库日志信息到elasticsearch

[root@iZ2zeapnvuohe8p14289u6Z ]# cd /usr/local/soft/elk/logstash-6.2.2/etc
[root@iZ2zeapnvuohe8p14289u6Z ]# rz #选择MySQL驱动包( mysql-connector-java-8.0.13.jar)

   

 #编写elkmysql.conf配置文件，内容如下:

#注：mysql-connector-java-8.0.13.jar 文件位置找到相对应

 三、kibana插件下载与配置启动

1.kibana 下载

[root@iZ2zeapnvuohe8p14289u6Z ]# cd /usr/local/soft/elk
[root@iZ2zeapnvuohe8p14289u6Z elk]# wget https://artifacts.elastic.co/downloads/kibana/kibana-6.2.2-linux-x86_64.tar.gz
[root@iZ2zeapnvuohe8p14289u6Z elk]# tar -zxvf kibana-6.2.2-linux-x86_64.tar.gz
[root@iZ2zeapnvuohe8p14289u6Z elk]# cd kibana-6.2.2-linux-x86_64
[root@iZ2zeapnvuohe8p14289u6Z kibana-6.2.2-linux-x86_64]# ll

   

  2.修改配置

[root@iZ2zeapnvuohe8p14289u6Z kibana-6.2.2-linux-x86_64]# cd config
[root@iZ2zeapnvuohe8p14289u6Z config]# vim kibana.yml

   

#kibana.yml常见配置项
elasticsearch.pingTimeout    #日常用的ping
elasticsearch.requestTimeout   #读取es的超时时间
elasticsearch.url #es主机地址
elasticsearch.username #es鉴权的用户名
elasticsearch.password #es鉴权的密码
server.host: "0.0.0.0"          #开放端口

3.正式启动Logstash

[root@iZ2zeapnvuohe8p14289u6Z config]# cd /usr/local/soft/elk/kibana-6.2.2-linux-x86_64/bin
#启动
[root@iZ2zeapnvuohe8p14289u6Z config]# nohup ./kibana > /dev/null 2>&1 &
#停止
[root@iZ2zeapnvuohe8p14289u6Z config]# ps aux | grep kibana
[root@iZ2zeapnvuohe8p14289u6Z config]# kill -9 “进程ID”

4.总结

#注：elasticsearch、logstash、kibana的端口号都要开放出来
#常见问题解决
1、JVM内存溢出导致的 ES或者Logstash服务启不来，报错 insufficient memory
 解决：升级机器的内存和CPU；或者改elasticSeach和logstash的JVM.option，最大堆内存xmx和初始堆内存xms
2、ES启动报错
seccomp unavailable: CONFIG_SECCOMP not compiled into kernel, CONFIG_SECCOMP and CONFIG_SECCOMP_FILTER are needed
#修改elasticsearch.yml 添加一下内容
bootstrap.memory_lock: false   为了避免内存和磁盘之间的swap
bootstrap.system_call_filter: false