一、kafka+ELK日志分析

架构图

 

 

案例环境:

主机名	操作系统	ip地址	部署服务
elk-84	centos7.x	192.168.0.84	es,kibana,zk-kafka
elk-85	centos7.x	192.168.0.85	logstash,zk-kafka
elk-86	centos7.x	192.168.0.86	logstash,zk-kafka

 

 

 

 

一、部署elk（三台机器）

第一步:

环境初始化

Elk-84
vim /etc/hosts
192.168.0.84	elk-84
192.168.0.85	elk-85
192.168.0.86	elk-86

ssh-keygen
ssh-copy-id -i root@192.168.0.85
ssh-copy-id -i root@192.168.0.86
scp /etc/hosts 192.168.0.85:/etc/
scp /etc/hosts 192.168.0.86:/etc/

 

三台机器安装java
yum -y install java
java -version
openjdk version "1.8.0_292"
OpenJDK Runtime Environment (build 1.8.0_292-b10)
OpenJDK 64-Bit Server VM (build 25.292-b10, mixed mode)
 
创建安装elk的目录
mkdir /home/elk
 
调整系统文件描述符的软硬限制
vim /etc/security/limits.conf 
# 末尾添加 
# 打开文件的软限制，ES要求系统文件描述符大于65535 
* soft nofile 655360 
# 打开文件的硬限制
 * hard nofile 655360 
# 用户可用进程数软限制
 * soft nproc 2048 
# 用户可用进程数硬限制
 * hard nproc 4096 
# JVM能够使用最大线程数
echo "vm.max_map_count=655360" >> /etc/sysctl.conf 
sysctl -p
 
配置Elasticsearch服务环境
useradd es
mkdir -p /opt/data/{data,logs} # 日志及数据存放目录 
cd /opt
chown -R es:es data data # 使用es用户启动时，权限不对也会报错

 

第二步: 

部署es集群

Elk-84操作
从elastic 的官网 elastic.co/downloads/elasticsearch 获取最新版本的 Elasticsearch
拉取elasticsearch-7.12.1-linux-x86_64.tar.gz到/home/elk下
[root@elk-84 elk]# tar -zxf elasticsearch-7.12.1-linux-x86_64.tar.gz
[root@elk-84 opt]# vim elk/elasticsearch-7.12.1/config/elasticsearch.yml
cluster.name: my-elk	#集群名称
node.name: elk-84		#本机名
path.data: /opt/data/data
path.logs: /opt/data/logs
network.host: 192.168.0.84	#本机ip
http.port: 9200
discovery.seed_hosts: ["elk-84", "elk-85", "elk-86"]
cluster.initial_master_nodes: ["elk-84"]		#集群master ip
保存退出
[root@elk-84 opt]# scp /home/elk/elasticsearch-7.12.1-linux-x86_64.tar.gz 192.168.0.85:/home/elk/


Elk-85
[root@elk-85 elk]# tar -zxf elasticsearch-7.12.1-linux-x86_64.tar.gz

Elk-86
[root@elk-86 elk]# tar -zxf elasticsearch-7.12.1-linux-x86_64.tar.gz

Elk-84
[root@elk-84 opt]# scp /home/elk/elasticsearch-7.12.1/config/elasticsearch.yml 192.168.0.85:/home/elk/elasticsearch-7.12.1/config/
[root@elk-84 opt]# scp /home/elk/elasticsearch-7.12.1/config/elasticsearch.yml 192.168.0.86:/home/elk/elasticsearch-7.12.1/config/


Elk-85
[root@elk-85 opt]# vim elk/elasticsearch-7.12.1/config/elasticsearch.yml
node.name: elk-85		#本机名
path.data: /opt/data/data
path.logs: /opt/data/logs
network.host: 192.168.0.85	#本机ip

Elk-86
[root@elk-86 opt]# vim elk/elasticsearch-7.12.1/config/elasticsearch.yml
node.name: elk-86		#本机名
path.data: /opt/data/data
path.logs: /opt/data/logs
network.host: 192.168.0.86	#本机ip

三台启动elasticsearch
[root@elk-84 opt]# screen -R elasticsearch
[root@elk-84 opt]# su es
[es@elk-84 opt]$ /home/elk/elasticsearch-7.12.1/bin/elasticsearch
此时是阻塞状态 
启动没有报错ctrl+a+d后台运行
ss -anput | grep 9200  #查看端口

 

验证es集群：

验证：
curl http://192.168.0.84:9200/
{
  "name" : "elk-84",
  "cluster_name" : "my-elk",
  "cluster_uuid" : "Y7Q4FMSWS-uXRm0ifiOVTA",
  "version" : {
    "number" : "7.12.1",
    "build_flavor" : "default",
    "build_type" : "tar",
    "build_hash" : "3186837139b9c6b6d23c3200870651f10d3343b7",
    "build_date" : "2021-04-20T20:56:39.040728659Z",
    "build_snapshot" : false,
    "lucene_version" : "8.8.0",
    "minimum_wire_compatibility_version" : "6.8.0",
    "minimum_index_compatibility_version" : "6.0.0-beta1"
  },
  "tagline" : "You Know, for Search"
}
 
 curl http://192.168.0.85:9200/
{
  "name" : "elk-85",
  "cluster_name" : "my-elk",
  "cluster_uuid" : "Y7Q4FMSWS-uXRm0ifiOVTA",
  "version" : {
    "number" : "7.12.1",
    "build_flavor" : "default",
    "build_type" : "tar",
    "build_hash" : "3186837139b9c6b6d23c3200870651f10d3343b7",
    "build_date" : "2021-04-20T20:56:39.040728659Z",
    "build_snapshot" : false,
    "lucene_version" : "8.8.0",
    "minimum_wire_compatibility_version" : "6.8.0",
    "minimum_index_compatibility_version" : "6.0.0-beta1"
  },
  "tagline" : "You Know, for Search"
}
 
curl http://192.168.0.86:9200/
{
  "name" : "elk-86",
  "cluster_name" : "my-elk",
  "cluster_uuid" : "Y7Q4FMSWS-uXRm0ifiOVTA",
  "version" : {
    "number" : "7.12.1",
    "build_flavor" : "default",
    "build_type" : "tar",
    "build_hash" : "3186837139b9c6b6d23c3200870651f10d3343b7",
    "build_date" : "2021-04-20T20:56:39.040728659Z",
    "build_snapshot" : false,
    "lucene_version" : "8.8.0",
    "minimum_wire_compatibility_version" : "6.8.0",
    "minimum_index_compatibility_version" : "6.0.0-beta1"
  },
  "tagline" : "You Know, for Search"
}
查看集群状态
http://192.168.0.84:9200/_cluster/health?pretty
{
  "cluster_name" : "my-elk",
  "status" : "green",
  "timed_out" : false,
  "number_of_nodes" : 3,
  "number_of_data_nodes" : 3,
  "active_primary_shards" : 9,
  "active_shards" : 18,
  "relocating_shards" : 0,
  "initializing_shards" : 0,
  "unassigned_shards" : 0,
  "delayed_unassigned_shards" : 0,
  "number_of_pending_tasks" : 0,
  "number_of_in_flight_fetch" : 0,
  "task_max_waiting_in_queue_millis" : 0,
  "active_shards_percent_as_number" : 100.0
}

　　

第二步:

部署logstash

Elk-85
[root@elk-85 elk]# tar -zxf logstash-7.12.1-linux-x86_64.tar.gz
[root@elk-85 elk]# vim /home/elk/logstash-Info/logInfo.conf 
input {
    kafka {
        bootstrap_servers => "192.168.0.84:9092,192.168.0.85:9092,192.168.0.86:9092"
        topics => ["logInfo"]
        auto_offset_reset => "earliest"
        codec => json
    }
}

filter {
    ruby {
     code => "event.set('index_day', event.get('@timestamp').time.localtime.strftime('%Y-%m-%d'))"
    }
    mutate {
      rename => { "[host][name]" => "host" }
    }
}
output {
    elasticsearch  {
        hosts => [ "192.168.0.84:9200","192.168.0.85:9200","192.168.0.86:9200" ]
        index => "mylog-loginfo-%{index_day}"
        user => "elastic"
        password => "主机密码"         # 这里填服务密码
    }
}
[root@elk-85 opt]# vim /home/elk/logstash-Info/logInfo.sh
./bin/logstash -f ./logInfo.conf
[root@elk-85 opt]# vim /home/elk/logstash-Info-7.12.1/logInfo.conf 
input {
    kafka {
        bootstrap_servers => "192.168.0.84:9092,192.168.0.85:9092,192.168.0.86:9092"
        topics => ["logInfo"]
        auto_offset_reset => "earliest"
        codec => json
    }
}

filter {
    ruby {
     code => "event.set('index_day', event.get('@timestamp').time.localtime.strftime('%Y-%m-%d'))"
    }
    mutate {
      rename => { "[host][name]" => "host" }
    }
}
output {
    elasticsearch  {
        hosts => [ "192.168.0.84:9200","192.168.0.85:9200","192.168.0.86:9200" ]
        index => "mylog-loginfo-%{index_day}"
    }
}
[root@elk-85 opt]# vim /home/elk/logstash-Info-7.12.1/logInfo.sh
./bin/logstash -f ./logInfo.conf

 

Elk-86
[root@elk-86 elk]# tar -zxf logstash-7.12.1-linux-x86_64.tar.gz
[root@elk-86 elk]# vim /home/elk/logstash-traceInfo/traceInfo.conf
input {
    kafka {
        bootstrap_servers => "192.168.0.84:9092,192.168.0.85:9092,192.168.0.86:9092"
        topics => ["traceInfo"]
        auto_offset_reset => "earliest"
        codec => json
    }
}
filter {
    ruby {
     code => "event.set('index_day', event.get('@timestamp').time.localtime.strftime('%Y-%m-%d'))"
    }
}
output {
    elasticsearch  {
        hosts => [ "192.168.0.84:9200","192.168.0.85:9200","192.168.0.86:9200" ]
        index => "mylog-traceinfo-%{index_day}"
    }
}
[root@elk-86 elk]# vim /home/elk/logstash-traceInfo/traceInfo.sh
./bin/logstash -f ./traceInfo.conf
[root@elk-86 elk]# vim /home/elk/logstash-traceInfo-7.12.1/traceInfo.conf
input {
    kafka {
        bootstrap_servers => "192.168.0.84:9092,192.168.0.85:9092,192.168.0.86:9092"
        topics => ["traceInfo"]
        auto_offset_reset => "earliest"
        codec => json
    }
}
filter {
    ruby {
     code => "event.set('index_day', event.get('@timestamp').time.localtime.strftime('%Y-%m-%d'))"
    }
}
output {
    elasticsearch  {
        hosts => [ "192.168.0.84:9200","192.168.0.85:9200","192.168.0.86:9200" ]
        index => "mylog-traceinfo-%{index_day}"
    }
}
[root@elk-86 elk]# vim /home/elk/logstash-traceInfo-7.12.1/traceInfo.sh
./bin/logstash -f ./traceInfo.conf

　　

两台启动logstash
[root@elk-85 opt]# screen -R logstash
[root@elk-85 opt]# cd  /home/elk/logstash-Info/
[root@elk-85 opt]# sh logInfo.sh
[root@elk-85 opt]# cd /home/elk/logstash-Info-7.12.1/
[root@elk-85 opt]# sh logInfo.sh
[root@elk-86 opt]# screen -R logstash
[root@elk-86 opt]# cd  /home/elk/logstash-traceInfo/
[root@elk-86 opt]# sh traceInfo.sh
[root@elk-86 opt]# cd /home/elk/logstash-traceInfo-7.12.1/
[root@elk-86 opt]# sh traceInfo.sh
阻塞状态 
启动没有报错ctrl+a+d后台运行
ss -anput | grep 9300 #查看端口

 

三、部署kibana

安装kibana

Elk-84
[root@elk-84 elk]# tar -zxf kibana-7.12.1-linux-x86_64.tar.gz 
[root@elk-84 elk]# vim kibana-7.12.1-linux-x86_64/config/kibana.yml
server.port: 5601
server.host: "192.168.0.84"
server.name: "elk-84"
elasticsearch.hosts: ["http://elk-84:9200","http://elk-85:9200","http://elk-86:9200"]
elasticsearch.logQueries: true
logging.verbose: true
i18n.locale: "zh-CN"
[root@elk-84 elk]# screen -R kibana
[root@elk-84 elk]# su es
[es@elk-84 elk]$ /home/elk/kibana-7.12.1-linux-x86_64/bin/kibana
阻塞状态 
启动没有报错ctrl+a+d后台运行
ss -anput | grep 5601 #查看端口
如果有版本报错：（执行以下赋权）
[root@elk-80 elk]# sudo chown -R es:es /home/elk/