elk+zookeeper+kafka单机版部署

一、elk+zookeeper+kafka

1、软件目录
mkdir -p /nulige/tools && cd /nulige/tools
jdk-8u25-linux-x64.tar.gz
elasticsearch-6.6.1.tar.gz
kibana-6.6.1-linux-x86_64.tar.gz
logstash-6.6.1.tar.gz
zookeeper-3.4.13.tar.gz
kafka_2.12-2.1.0.tgz
filebeat-6.6.1-linux-x86_64.tar.gz

二、部署elk
2.1、install java
yum install java-1.8.0-openjdk -y

#验证环境变量
java -version

2.2、Elasticsearch二进制安装和启动
cd /nulige/tools
tar xf elasticsearch-6.6.1.tar.gz
mv elasticsearch-6.6.1 /usr/local/
mkdir -p /usr/local/elasticsearch-6.6.1/data
cp /usr/local/elasticsearch-6.6.1/config/elasticsearch.yml /usr/local/elasticsearch-6.6.1/config/elasticsearch.yml.bak
vi /usr/local/elasticsearch-6.6.1/config/elasticsearch.yml
path.data: /usr/local/elasticsearch-6.6.1/data
path.logs: /usr/local/elasticsearch-6.6.1/logs
network.host: 127.0.0.1
http.port: 9200
indices.query.bool.max_clause_count: 10000 #增加消息最大数，默认1024

#Elasticsearch的启动，得用普通用户启动
useradd -s /sbin/nologin elk
chown -R elk:elk /usr/local/elasticsearch-6.6.1/
su - elk -s /bin/bash
/usr/local/elasticsearch-6.6.1/bin/elasticsearch -d

[root@a1 config]$ netstat -lntup|grep java
tcp 0 0 127.0.0.1:9200 0.0.0.0:* LISTEN 23123/java
tcp 0 0 127.0.0.1:9300 0.0.0.0:* LISTEN 11773/java

#观察日志
[elk@a1 local]$ tail -f /usr/local/elasticsearch-6.6.1/logs/elasticsearch.log

#查看是否可以访问
[root@a1 vhosts]# curl 127.0.0.1:9200

#调优方法：物理内存的一半,默认1G
#JVM的内存限制更改jvm.options
-Xms128M
-Xmx128M

3.3、Kibana二进制安装和启动
cd /nulige/tools
tar xf kibana-6.6.1-linux-x86_64.tar.gz
mv kibana-6.6.1-linux-x86_64 /usr/local/kibana-6.6.1

[root@a1 config]# vi /usr/local/kibana-6.6.1/config/kibana.yml
server.port: 5601
server.host: "0.0.0.0"

#启动服务
nohup /usr/local/kibana-6.6.1/bin/kibana >/tmp/kibana.log &

#访问
http://120.79.171.145:5601

2.4、nginx
yum install nginx -y

#修改配置文件
[root@a1 conf]# cat nginx.conf
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';

log_format json '{"@timestamp":"$time_iso8601",'
'"clientip":"$remote_addr",'
'"status":$status,'
'"bodysize":$body_bytes_sent,'
'"referer":"$http_referer",'
'"ua":"$http_user_agent",'
'"handletime":$request_time,'
'"url":"$uri"}';

access_log /roobo/logs/nginx/access.json.log json;

#重启服务
/etc/init.d/nginx reload

3、Logstash
cd /nulige/tools
tar xf logstash-6.6.1.tar.gz
mv logstash-6.6.1 /usr/local/
cp /usr/local/logstash-6.6.1/config/logstash.conf /usr/local/logstash-6.6.1/config/logstash.conf.bak
vi /usr/local/logstash-6.6.1/config/logstash.conf
input {
kafka {
bootstrap_servers => "10.7.19.129:9092"
topics => ["nignx_access"]
group_id => "nginx_access"
codec => "json"
}
}

output{
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "nginx_access-%{+YYYY.MM.dd}"
}
}

#Logstash的JVM配置文件更新/usr/local/logstash-6.6.0/config/jvm.options
-Xms200M
-Xmx200M

#加速logstash启动，需要安装haveged
yum install haveged -y;
chkconfig --add haveged
chkconfig --level 35 haveged on
chkconfig --list |grep haveged

#启动服务
nohup /usr/local/logstash-6.6.1/bin/logstash -f /usr/local/logstash-6.6.1/config/logstash.conf >/tmp/logstash.log &

4、Zookeeper+kafka
cd /nulige/tools
#解压zookeeper
tar xf zookeeper-3.4.13.tar.gz
mv zookeeper-3.4.13 /usr/local/
#修改配置文件：
cp /usr/local/zookeeper-3.4.13/conf/zoo_sample.cfg zoo.cfg
dataDir=/usr/local/zookeeper-3.4.13/data
clientPortAddress=10.7.19.129
clientPort=2181

#启动：/usr/local/zookeeper-3.4.13/bin/zkServer.sh start

#查看进程
[root@a1 conf]# netstat -lntup|grep java
tcp 0 0 0.0.0.0:2181 0.0.0.0:* LISTEN 22256/java

#解压kafka
tar xf kafka_2.12-2.1.0.tgz
mv kafka_2.12-2.1.0 /usr/local/kafka_2.12

cat /usr/local/kafka_2.12/config/server.properties
listeners=PLAINTEXT://10.7.19.129:9092
zookeeper.connect=10.7.19.129:2181

nohup /usr/local/kafka_2.11/bin/kafka-server-start.sh /usr/local/kafka_2.11/config/server.properties > /tmp/kafka.log 2>&1

cd /usr/local/kafka_2.12/bin
./kafka-consumer-groups.sh --bootstrap-server 10.7.19.129:9092 --list
./kafka-consumer-groups.sh --bootstrap-server 10.7.19.129:9092 --group nginx_access --describe

6、Filebeat
cd /nulige/tools
tar xf filebeat-6.6.1-linux-x86_64.tar.gz
mv filebeat-6.6.1-linux-x86_64 /usr/local/filebeat-6.6.1

[root@a1 filebeat-6.6.1]# cd /usr/local/filebeat-6.6.1
[root@a1 filebeat-6.6.1]# cat filebeat.yml
filebeat.inputs:
- type: log
tail_files: true
backoff: "1s"
paths:
- /roobo/logs/nginx/access.json.log

output.kafka:
enabled: true
hosts: ["10.7.19.129:9092"]
topic: nignx_access

nohup /usr/local/filebeat-6.6.1/filebeat -e -c /usr/local/filebeat-6.6.1/filebeat.yml&>/tmp/filebeat.log &
cat /tmp/filebeat.log