Linux操作文档——ELK日志分析系统

文章目录

软件说明参考文档
Elasticsearch数据库，存储数据官方参考文档
logstash日志收集，过滤数据官方参考文档
kibana分析，过滤，展示官方参考文档
filebeat收集日志，传输到ES或logstash官方参考文档

一、部署Elasticsearch软件

1、安装 Elasticsearch 软件

[root@node-1 ~]# mkdir /data/soft
[root@node-1 ~]# cd /data/soft/
[root@node-1 soft]# wget https://mirrors.tuna.tsinghua.edu.cn/elasticstack/6.x/yum/6.6.0/elasticsearch-6.6.0.rpm
[root@node-1 soft]# wget https://mirrors.tuna.tsinghua.edu.cn/elasticstack/6.x/yum/6.6.0/filebeat-6.6.0-x86_64.rpm
[root@node-1 soft]# wget https://mirrors.tuna.tsinghua.edu.cn/elasticstack/6.x/yum/6.6.0/kibana-6.6.0-x86_64.rpm
[root@node-1 soft]# wget https://mirrors.tuna.tsinghua.edu.cn/elasticstack/6.x/yum/6.6.0/logstash-6.6.0.rpm
[root@node-1 ~]# yum -y install elasticsearch-6.6.0.rpm 
[root@node-1 ~]# vim /etc/elasticsearch/elasticsearch.yml
node.name: node-1             //群集中本机节点名
path.data: /data/elasticsearch             //数据目录
path.logs: /var/log/elasticsearch             //日志目录
bootstrap.memory_lock: true             //锁定内存
network.host: 192.168.1.10,127.0.0.1             //监听的ip地址
http.port: 9200             //端口号
[root@node-1 ~]# mkdir -p /data/elasticsearch              //创建数据目录
[root@node-1 ~]# chown -R elasticsearch.elasticsearch /data/elasticsearch/             //修改权限
[root@node-1 ~]# vim /etc/elasticsearch/jvm.options             //分配锁定内存
-Xms1g             //分配最小内存	
-Xmx1g             //分配最大内存,官方推荐为物理内存的一半，但最大为32G
[root@node-1 ~]# systemctl edit elasticsearch             //修改锁定内存后，无法重启，解决方法
[Service]
LimitMEMLOCK=infinity             //F2保存退出
[root@node-1 ~]# systemctl daemon-reload
[root@node-1 ~]# systemctl restart elasticsearch

2、添加elasticsearch群集

[root@node-2 ~]# vim /etc/elasticsearch/elasticsearch.yml
node.name: node-1
path.data: /data/elasticsearch
path.logs: /var/log/elasticsearch
bootstrap.memory_lock: true
network.host: 192.168.1.20,127.0.0.1
http.port: 9200
discovery.zen.ping.unicast.hosts: ["192.168.1.10", "192.168.1.20"]             //主节点，工作节点
discovery.zen.minimum_master_nodes: 2             //添加的值=节点数/2 + 1

3、常见群集管理监控命令

[root@node-1 ~]# curl -XPUT '192.168.1.10:9200/vipinfo/users/1?pretty&pretty' -H 'Content-Type: application/json' -d '{"name": "guofucheng","age": "45","job": "mingxing"}'             //创建索引
[root@node-1 ~]# curl -XGET '192.168.1.10:9200/_cat/indices?pretty'             //查看索引信息
[root@node-1 ~]# curl -XGET '192.168.1.10:9200/_cluster/health?pretty'             //查看群集健康状态
[root@node-1 ~]# curl -XGET '192.168.1.10:9200/_cat/nodes?human&pretty'             //统计群集节点
[root@node-1 ~]# curl -XGET '192.168.1.10:9200/_nodes/_all/info/jvm.process?human&pretty'             //查看群集所有节点详细信息

二、安装Elasticsearch-head插件

1、本机安装

[root@node-1 ~]# git clone https://github.com/mobz/elasticsearch-head.git
[root@node-1 ~]# yum -y install epel-release
[root@node-1 ~]# yum -y install nodejs npm
[root@node-1 ~]# cd elasticsearch-head/
[root@node-1 elasticsearch-head]# cnpm install
[root@node-1 elasticsearch-head]# cd _site/
[root@node-1 _site]# vim app.js 
#原代码为this.base_uri = this.config.base_uri;
this.base_uri = this.config.base_uri || this.prefs.get("app-base_uri") || "http://192.168.1.10:9200";
[root@node-1 ~]# vim /etc/elasticsearch/elasticsearch.yml 
http.cors.enabled: true
http.cors.allow-origin: "*"
[root@node-1 ~]# cd elasticsearch-head/
[root@node-1 elasticsearch-head]# node_modules/grunt/bin/grunt server &

2、浏览器插件安装

下载es-head插件，https://github.com/mobz/elasticsearch-head
下载后，解压，复制crx目录下es-head.crx到桌面
改名es-head.crx为es-head.crx.zip
解压es-head.crx.zip到es-head.crx目录，把目录es-head.crx，上传到谷歌浏览器开发工具–扩展程序里

三、构建es+kibana+filebeat架构（小型）

1、安装kibana

[root@node-1 ~]# cd /data/soft
[root@node-1 soft]# rpm -ivh kibana-6.6.0-x86_64.rpm 
[root@node-1 ~]# vim /etc/kibana/kibana.yml
server.port: 5601
server.host: "192.168.1.10"
server.name: "node-1"             //所在主机的主机名
elasticsearch.hosts: ["http://192.168.1.10:9200"]             //es服务器的ip，便于接收日志数据
[root@node-1 ~]# systemctl start kibana

2、安装filebeat

[root@node-1 ~]# cd /data/soft
[root@node-1 soft]# rpm -ivh filebeat-6.6.0-x86_64.rpm 
[root@node-1 soft]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log

output.elasticsearch:
  hosts: ["192.168.1.10:9200"]
[root@node-1 soft]# systemctl start filebeat

3、修改日志格式为json

[root@node-1 ~]# yum -y install epel-release
[root@node-1 ~]# yum -y install nginx httpd-tools             //安装nginx，httpd-tools
[root@node-1 ~]# systemctl start nginx
[root@node-1 ~]# vim /etc/nginx/nginx.conf
http {             //添加在http {}内
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';
    log_format log_json '{ "@timestamp": "$time_local", '
    '"remote_addr": "$remote_addr", '
    '"referer": "$http_referer", '
    '"request": "$request", '
    '"status": $status, '
    '"bytes": $body_bytes_sent, '
    '"agent": "$http_user_agent", '
    '"x_forwarded": "$http_x_forwarded_for", '
    '"up_addr": "$upstream_addr",'
    '"up_host": "$upstream_http_host",'
    '"up_resp_time": "$upstream_response_time",'
    '"request_time": "$request_time"'
    ' }';
    access_log  /var/log/nginx/access.log  log_json;
[root@node-1 ~]# vim /etc/filebeat/filebeat.yml             //配置access.log和error.log分开
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true
  tags: ["access"]

- type: log
  enabled: true
  paths:
    - /var/log/nginx/error.log
  tags: ["error"]

output.elasticsearch:
  # Array of hosts to connect to.
  hosts: ["192.168.1.10:9200"]
  #index: "nginx-%{[beat.version]}-%{+yyyy.MM}"
  indices:
    - index: "nginx-access-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        tags: "access"
    - index: "nginx-error-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        tags: "error"
  setup.template.name: "nginx"
  setup.template.patten: "nginx-*"
  setup.template.enabled: false
  setup.template.overwrite: true
[root@node-1 ~]# ab -n 100 -c 20 http://192.168.1.10/             //使用ab压力测试工具测试访问
[root@node-1 ~]# tail -1 /var/log/nginx/access.log 
{ "@timestamp": "05/Aug/2020:21:18:15 +0800", "remote_addr": "192.168.1.10", "referer": "-", "request": "GET / HTTP/1.0", "status": 200, "bytes": 4833, "agent": "ApacheBench/2.3", "x_forwarded": "-", "up_addr": "-","up_host": "-","up_resp_time": "-","request_time": "0.000" }

四、构建filebeat+redis+logstash+es+kibana架构（大型）

1、安装redis

[root@node-1 ~]# mkdir -p /opt/redis_cluster/redis_6379/{conf,logs,pid}
[root@node-1 ~]# cd /data/soft/
[root@node-1 soft]# wget http://download.redis.io/releases/redis-5.0.7.tar.gz
[root@node-1 soft]# tar xf redis-5.0.7.tar.gz -C /opt/redis_cluster/
[root@node-1 soft]# ln -s /opt/redis_cluster/redis-5.0.7  /opt/redis_cluster/redis
[root@node-1 soft]# cd /opt/redis_cluster/redis
[root@node-1 redis]# make && make install 
[root@node-1 redis]# vim /opt/redis_cluster/redis_6379/conf/6379.conf
bind 127.0.0.1 192.168.1.10
port 6379
daemonize yes
pidfile /opt/redis_cluster/redis_6379/pid/redis_6379.pid
logfile /opt/redis_cluster/redis_6379/logs/redis_6379.log
databases 16
dbfilename redis.rdb
dir /opt/redis_cluster/redis_6379
[root@node-1 redis]# redis-server /opt/redis_cluster/redis_6379/conf/6379.conf

2、修改filebeat配置文件，output给redis

[root@node-1 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true
  tags: ["access"]

- type: log
  enabled: true
  paths:
    - /var/log/nginx/error.log
  tags: ["error"]

setup.template.settings:
  index.number_of_shards: 3

setup.kibana:

output.redis:
  hosts: ["192.168.1.10"]
  key: "filebeat"
  db: 0
  timeout: 5
[root@node-1 ~]# systemctl restart filebeat
[root@node-1 ~]# ab -n 100 -c 20 http://192.168.1.10/             //使用ab压力测试工具测试访问
[root@node-1 ~]# redis-cli              //登录
127.0.0.1:6379> keys *             //列出所有键
1) "filebeat"
127.0.0.1:6379> type filebeat              //filebeat为键值名
list
127.0.0.1:6379> LLEN filebeat             //查看list长度
(integer) 100
127.0.0.1:6379> LRANGE filebeat 0 -1             //查看list所有内容
  1) "{\"@timestamp\":\"2020-08-05T13:53:57.104Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"doc\",\"version\":\"6.6.0\"},\"up_resp_time\":\"-\",\"source\":\"/var/log/nginx/access.log\",\"error\":{\"message\":\"@timestamp not overwritten (parse error on 05/Aug/2020:21:53:49 +0800)\",\"type\":\"json\"},\"input\":{\"type\":\"log\"},\"host\":{\"name\":\"node-1\"},\"agent\":\"ApacheBench/2.3\",\"up_host\":\"-\",\"remote_addr\":\"192.168.1.10\",\"bytes\":4833,\"prospector\":{\"type\":\"log\"},\"log\":{\"file\":{\"path\":\"/var/log/nginx/access.log\"}},\"request\":\"GET / HTTP/1.0\",\"request_time\":\"0.000\",\"tags\":[\"access\"],\"beat\":{\"version\":\"6.6.0\",\"name\":\"node-1\",\"hostname\":\"node-1\"},\"up_addr\":\"-\",\"x_forwarded\":\"-\",\"offset\":82800,\"referer\":\"-\",\"status\":200}"
......

3、安装logstash，收集redis的日志，提交给es

[root@node-1 ~]# cd /data/soft/
[root@node-1 soft]# rpm -ivh logstash-6.6.0.rpm 
[root@node-1 ~]# vim /etc/logstash/conf.d/redis.conf             //实现access和error日志分离
input {
  redis {
    host => "192.168.1.10"
    port => "6379"
    db => "0"
    key => "filebeat"
    data_type => "list"
  }
}

filter {
  mutate {
    convert => ["upstream_time","float"]
    convert => ["request_time","float"]
  }
}

output {
  stdout {}
   if "access" in [tags] {
    elasticsearch {
      hosts => ["http://192.168.1.10:9200"]
      index => "nginx_access-%{+YYYY.MM.dd}"
      manage_template => false
    }
   }
   if "error" in [tags] {
    elasticsearch {
      hosts => ["http://192.168.1.10:9200"]
      index => "nginx_error-%{+YYYY.MM.dd}"
      manage_template => false
    }
   }
}
[root@node-1 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/redis.conf             //启动logstash

五、kibana使用

访问：http://IP:5601

posted @ 2020-03-02 13:39  高中僧  阅读(35)  评论(0)    收藏  举报