elk
1在elk101中安装elasticsearch
[root@elk101 ~]# ls anaconda-ks.cfg elasticsearch-7.17.3-x86_64.rpm filebeat-7.17.3-x86_64.rpm kibana-7.17.3-x86_64.rpm [root@elk101 ~]# scp elasticsearch-7.17.3-x86_64.rpm elk102:/root elasticsearch-7.17.3-x86_64.rpm 100% 297MB 172.9MB/s 00:01 [root@elk101 ~]# scp elasticsearch-7.17.3-x86_64.rpm elk103:/root elasticsearch-7.17.3-x86_64.rpm 100% 297MB 176.4MB/s 00:01 [root@elk101 ~]# yum -y localinstall elasticsearch-7.17.3-x86_64.rpm Repository extras is listed more than once in the configuration 上次元数据过期检查：1:48:06 前，执行于 2023年03月16日 星期四 15时14分25秒。 依赖关系解决。 =========================================================================================================================================== 软件包 架构 版本 仓库 大小 =========================================================================================================================================== 安装: elasticsearch x86_64 7.17.3-1 @commandline 297 M 事务概要 =========================================================================================================================================== 安装 1 软件包 总计：297 M 安装大小：494 M 下载软件包： 运行事务检查 事务检查成功。 运行事务测试 事务测试成功。 运行事务 准备中 : 1/1 运行脚本: elasticsearch-7.17.3-1.x86_64 1/1 Creating elasticsearch group... OK Creating elasticsearch user... OK 安装 : elasticsearch-7.17.3-1.x86_64 1/1 运行脚本: elasticsearch-7.17.3-1.x86_64 1/1 ### NOT starting on installation, please execute the following statements to configure elasticsearch service to start automatically using systemd sudo systemctl daemon-reload sudo systemctl enable elasticsearch.service ### You can start elasticsearch service by executing sudo systemctl start elasticsearch.service Created elasticsearch keystore in /etc/elasticsearch/elasticsearch.keystore [/usr/lib/tmpfiles.d/elasticsearch.conf:1] Line references path below legacy directory /var/run/, updating /var/run/elasticsearch → /run/elasticsearch; please update the tmpfiles.d/ drop-in file accordingly. 验证 : elasticsearch-7.17.3-1.x86_64 1/1 已安装: elasticsearch-7.17.3-1.x86_64 完毕！ [root@elk101 ~]# systemctl start elasticsearch [root@elk101 ~]# curl 127.0.0.1:9200 { "name" : "elk101", "cluster_name" : "elasticsearch", "cluster_uuid" : "qVkInulZTPG9B-l0tglE4w", "version" : { "number" : "7.17.3", "build_flavor" : "default", "build_type" : "rpm", "build_hash" : "5ad023604c8d7416c9eb6c0eadb62b14e766caff", "build_date" : "2022-04-19T08:11:19.070913226Z", "build_snapshot" : false, "lucene_version" : "8.11.1", "minimum_wire_compatibility_version" : "6.8.0", "minimum_index_compatibility_version" : "6.0.0-beta1" }, "tagline" : "You Know, for Search" } [root@elk101 ~]# vim /etc/elasticsearch/elasticsearch.yml [root@elk101 ~]# ls anaconda-ks.cfg elasticsearch-7.17.3-x86_64.rpm filebeat-7.17.3-x86_64.rpm kibana-7.17.3-x86_64.rpm [root@elk101 ~]# egrep -v "^#|^$" /etc/elasticsearch/elasticsearch.yml cluster.name: tyjs09-elk node.name: elk101 #这里写节点的名称 path.data: /var/lib/elasticsearch path.logs: /var/log/elasticsearch network.host: 0.0.0.0 discovery.seed_hosts: ["10.0.0.101", "10.0.0.102", "10.0.0.103"]
cluster.initial_master_nodes: ["elk101","elk102","elk103"] [root@elk101 ~]# ll /var/log/elasticsearch/ 总用量 212 -rw-r--r--. 1 elasticsearch elasticsearch 0 3月 16 17:02 elasticsearch_audit.json -rw-r--r--. 1 elasticsearch elasticsearch 797 3月 16 17:03 elasticsearch_deprecation.json -rw-r--r--. 1 elasticsearch elasticsearch 509 3月 16 17:03 elasticsearch_deprecation.log -rw-r--r--. 1 elasticsearch elasticsearch 0 3月 16 17:02 elasticsearch_index_indexing_slowlog.json -rw-r--r--. 1 elasticsearch elasticsearch 0 3月 16 17:02 elasticsearch_index_indexing_slowlog.log -rw-r--r--. 1 elasticsearch elasticsearch 0 3月 16 17:02 elasticsearch_index_search_slowlog.json -rw-r--r--. 1 elasticsearch elasticsearch 0 3月 16 17:02 elasticsearch_index_search_slowlog.log -rw-r--r--. 1 elasticsearch elasticsearch 24239 3月 16 17:03 elasticsearch.log -rw-r--r--. 1 elasticsearch elasticsearch 49125 3月 16 17:03 elasticsearch_server.json -rw-r--r--. 1 elasticsearch elasticsearch 78228 3月 16 18:02 gc.log -rw-r--r--. 1 elasticsearch elasticsearch 2114 3月 16 17:02 gc.log.00 [root@elk101 ~]# systemctl restart elasticsearch [root@elk101 ~]# ll /var/log/elasticsearch/ 总用量 268 -rw-r--r--. 1 elasticsearch elasticsearch 0 3月 16 17:02 elasticsearch_audit.json -rw-r--r--. 1 elasticsearch elasticsearch 797 3月 16 17:03 elasticsearch_deprecation.json -rw-r--r--. 1 elasticsearch elasticsearch 509 3月 16 17:03 elasticsearch_deprecation.log -rw-r--r--. 1 elasticsearch elasticsearch 0 3月 16 17:02 elasticsearch_index_indexing_slowlog.json -rw-r--r--. 1 elasticsearch elasticsearch 0 3月 16 17:02 elasticsearch_index_indexing_slowlog.log -rw-r--r--. 1 elasticsearch elasticsearch 0 3月 16 17:02 elasticsearch_index_search_slowlog.json -rw-r--r--. 1 elasticsearch elasticsearch 0 3月 16 17:02 elasticsearch_index_search_slowlog.log -rw-r--r--. 1 elasticsearch elasticsearch 25759 3月 16 18:03 elasticsearch.log -rw-r--r--. 1 elasticsearch elasticsearch 52848 3月 16 18:03 elasticsearch_server.json -rw-r--r--. 1 elasticsearch elasticsearch 38133 3月 16 18:03 gc.log -rw-r--r--. 1 elasticsearch elasticsearch 2114 3月 16 17:02 gc.log.00 -rw-r--r--. 1 elasticsearch elasticsearch 78959 3月 16 18:03 gc.log.01 -rw-r--r--. 1 elasticsearch elasticsearch 2114 3月 16 18:03 gc.log.02 -rw-r--r--. 1 elasticsearch elasticsearch 0 3月 16 18:03 tyjs09-elk_audit.json -rw-r--r--. 1 elasticsearch elasticsearch 794 3月 16 18:03 tyjs09-elk_deprecation.json -rw-r--r--. 1 elasticsearch elasticsearch 509 3月 16 18:03 tyjs09-elk_deprecation.log -rw-r--r--. 1 elasticsearch elasticsearch 0 3月 16 18:03 tyjs09-elk_index_indexing_slowlog.json -rw-r--r--. 1 elasticsearch elasticsearch 0 3月 16 18:03 tyjs09-elk_index_indexing_slowlog.log -rw-r--r--. 1 elasticsearch elasticsearch 0 3月 16 18:03 tyjs09-elk_index_search_slowlog.json -rw-r--r--. 1 elasticsearch elasticsearch 0 3月 16 18:03 tyjs09-elk_index_search_slowlog.log -rw-r--r--. 1 elasticsearch elasticsearch 14725 3月 16 18:03 tyjs09-elk.log -rw-r--r--. 1 elasticsearch elasticsearch 28244 3月 16 18:03 tyjs09-elk_server.json [root@elk101 ~]#
2分别在elk102和elk103中安装es
#1把elk101节点的配置文件传给其它两台机器
[root@elk101 ~]# scp /etc/elasticsearch/elasticsearch.yml elk102:/etc/elasticsearch/
elasticsearch.yml 100% 3443 1.7MB/s 00:00
[root@elk101 ~]# scp /etc/elasticsearch/elasticsearch.yml elk103:/etc/elasticsearch/
elasticsearch.yml 100% 3443 2.0MB/s 00:00
#2修改elk102机器上的配置文件
[root@elk102 ~]# egrep -v "^#|^$" /etc/elasticsearch/elasticsearch.yml
cluster.name: tyjs09-elk
node.name: elk102
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
network.host: 0.0.0.0
discovery.seed_hosts: ["10.0.0.101", "10.0.0.102", "10.0.0.103"]
cluster.initial_master_nodes: ["elk101", "elk102", "elk103"]
#3修改elk103机器上的配置文件
[root@elk103 ~]# egrep -v "^#|^$" /etc/elasticsearch/elasticsearch.yml
cluster.name: tyjs09-elk
node.name: elk103
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
network.host: 0.0.0.0
discovery.seed_hosts: ["10.0.0.101", "10.0.0.102", "10.0.0.103"]
cluster.initial_master_nodes: ["elk101", "elk102", "elk103"]
#4停掉elk101上正在运行的es并清理临时数据目录以及之前单机部署的es数据和日志
[root@elk101 ~]# systemctl stop elasticsearch
[root@elk101 ~]# rm -rf /var/lib/elasticsearch/*
[root@elk101 ~]# rm -rf /var/log/elasticsearch/*
[root@elk101 ~]# rm -rf /tmp/*
#5顺便也清理一下elk102和elk103上的临时数据目录
[root@elk102 ~]# rm -rf /tmp/*
[root@elk103 ~]# rm -rf /tmp/*
#6启动所有节点
[root@elk101 ~]# systemctl start elasticsearch
[root@elk102 ~]# systemctl start elasticsearch
[root@elk103 ~]# systemctl start elasticsearch
#7检查集群是否正常
[root@elk101 ~]# ll /var/log/elasticsearch/
总用量 136
-rw-r--r--. 1 elasticsearch elasticsearch 56699 3月 16 18:43 gc.log
-rw-r--r--. 1 elasticsearch elasticsearch 2114 3月 16 18:39 gc.log.00
-rw-r--r--. 1 elasticsearch elasticsearch 0 3月 16 18:39 tyjs09-elk_audit.json
-rw-r--r--. 1 elasticsearch elasticsearch 0 3月 16 18:39 tyjs09-elk_deprecation.json
-rw-r--r--. 1 elasticsearch elasticsearch 0 3月 16 18:39 tyjs09-elk_deprecation.log
-rw-r--r--. 1 elasticsearch elasticsearch 0 3月 16 18:39 tyjs09-elk_index_indexing_slowlog.json
-rw-r--r--. 1 elasticsearch elasticsearch 0 3月 16 18:39 tyjs09-elk_index_indexing_slowlog.log
-rw-r--r--. 1 elasticsearch elasticsearch 0 3月 16 18:39 tyjs09-elk_index_search_slowlog.json
-rw-r--r--. 1 elasticsearch elasticsearch 0 3月 16 18:39 tyjs09-elk_index_search_slowlog.log
-rw-r--r--. 1 elasticsearch elasticsearch 30324 3月 16 18:43 tyjs09-elk.log
-rw-r--r--. 1 elasticsearch elasticsearch 43613 3月 16 18:43 tyjs09-elk_server.json
[root@elk101 ~]# tail -2f /var/log/elasticsearch/tyjs09-elk.log
[2023-03-16T18:43:48,102][WARN ][o.e.c.c.ClusterFormationFailureHelper] [elk101] master not discovered yet, this node has not previously joined a bootstrapped (v7+) cluster, and [cluster.initial_master_nodes] is empty on this node: have discovered [{elk101}{SjBn6fZKQ8i1lQaYaWWW6Q}{UEd9psd8SI66nnR1uGLivg}{10.0.0.101}{10.0.0.101:9300}{cdfhilmrstw}, {elk102}{Qqw592LGRRyt3EwH8SR0Xg}{oKWGPhsgQiORvc1v5J15VQ}{10.0.0.102}{10.0.0.102:9300}{cdfhilmrstw}, {elk103}{U1E-_8loQY2tFXi-3m4SKA}{SZRF9fkvRaa_yMQNS_N7NA}{10.0.0.103}{10.0.0.103:9300}{cdfhilmrstw}]; discovery will continue using [10.0.0.102:9300, 10.0.0.103:9300] from hosts providers and [{elk101}{SjBn6fZKQ8i1lQaYaWWW6Q}{UEd9psd8SI66nnR1uGLivg}{10.0.0.101}{10.0.0.101:9300}{cdfhilmrstw}] from last-known cluster state; node term 0, last-accepted version 0 in term 0
[2023-03-16T18:43:58,104][WARN ][o.e.c.c.ClusterFormationFailureHelper] [elk101] master not discovered yet, this node has not previously joined a bootstrapped (v7+) cluster, and [cluster.initial_master_nodes] is empty on this node: have discovered [{elk101}{SjBn6fZKQ8i1lQaYaWWW6Q}{UEd9psd8SI66nnR1uGLivg}{10.0.0.101}{10.0.0.101:9300}{cdfhilmrstw}, {elk102}{Qqw592LGRRyt3EwH8SR0Xg}{oKWGPhsgQiORvc1v5J15VQ}{10.0.0.102}{10.0.0.102:9300}{cdfhilmrstw}, {elk103}{U1E-_8loQY2tFXi-3m4SKA}{SZRF9fkvRaa_yMQNS_N7NA}{10.0.0.103}{10.0.0.103:9300}{cdfhilmrstw}]; discovery will continue using [10.0.0.102:9300, 10.0.0.103:9300] from hosts providers and [{elk101}{SjBn6fZKQ8i1lQaYaWWW6Q}{UEd9psd8SI66nnR1uGLivg}{10.0.0.101}{10.0.0.101:9300}{cdfhilmrstw}] from last-known cluster state; node term 0, last-accepted version 0 in term 0
[2023-03-16T18:44:08,109][WARN ][o.e.c.c.ClusterFormationFailureHelper] [elk101] master not discovered yet, this node has not previously joined a bootstrapped (v7+) cluster, and [cluster.initial_master_nodes] is empty on this node: have discovered [{elk101}{SjBn6fZKQ8i1lQaYaWWW6Q}{UEd9psd8SI66nnR1uGLivg}{10.0.0.101}{10.0.0.101:9300}{cdfhilmrstw}, {elk102}{Qqw592LGRRyt3EwH8SR0Xg}{oKWGPhsgQiORvc1v5J15VQ}{10.0.0.102}{10.0.0.102:9300}{cdfhilmrstw}, {elk103}{U1E-_8loQY2tFXi-3m4SKA}{SZRF9fkvRaa_yMQNS_N7NA}{10.0.0.103}{10.0.0.103:9300}{cdfhilmrstw}]; discovery will continue using [10.0.0.102:9300, 10.0.0.103:9300] from hosts providers and [{elk101}{SjBn6fZKQ8i1lQaYaWWW6Q}{UEd9psd8SI66nnR1uGLivg}{10.0.0.101}{10.0.0.101:9300}{cdfhilmrstw}] from last-known cluster state; node term 0, last-accepted version 0 in term 0
^Z
[1]+ 已停止 tail -2f /var/log/elasticsearch/tyjs09-elk.log
[root@elk101 ~]# ss -ntl
State Recv-Q Send-Q Local Address:Port Peer Address:Port Process
LISTEN 0 128 0.0.0.0:22 0.0.0.0:*
LISTEN 0 128 *:9300 *:*
LISTEN 0 128 [::]:22 [::]:*
LISTEN 0 128 *:9200 *:*
[root@elk101 ~]# curl 10.0.0.103:9200
{
"name" : "elk103",
"cluster_name" : "tyjs09-elk",
"cluster_uuid" : "_na_",
"version" : {
"number" : "7.17.3",
"build_flavor" : "default",
"build_type" : "rpm",
"build_hash" : "5ad023604c8d7416c9eb6c0eadb62b14e766caff",
"build_date" : "2022-04-19T08:11:19.070913226Z",
"build_snapshot" : false,
"lucene_version" : "8.11.1",
"minimum_wire_compatibility_version" : "6.8.0",
"minimum_index_compatibility_version" : "6.0.0-beta1"
},
"tagline" : "You Know, for Search"
}
[root@elk101 ~]#
#8验证服务是否可用
[root@elk101 ~]# curl 10.0.0.101:9200/_cat/nodes
10.0.0.102 13 97 50 0.99 0.33 0.16 cdfhilmrstw * elk102
10.0.0.103 6 97 54 0.89 0.26 0.18 cdfhilmrstw - elk103
10.0.0.101 16 94 49 1.29 0.38 0.18 cdfhilmrstw - elk101
[root@elk101 ~]# curl 10.0.0.102:9200/_cat/nodes
10.0.0.101 35 96 19 1.16 0.38 0.19 cdfhilmrstw - elk101
10.0.0.102 19 96 8 0.84 0.32 0.16 cdfhilmrstw * elk102
10.0.0.103 18 96 17 0.75 0.26 0.18 cdfhilmrstw - elk103
[root@elk101 ~]# curl 10.0.0.103:9200/_cat/nodes
10.0.0.101 35 96 1 1.07 0.38 0.18 cdfhilmrstw - elk101
10.0.0.102 19 96 2 0.77 0.31 0.16 cdfhilmrstw * elk102
10.0.0.103 18 96 1 0.69 0.25 0.18 cdfhilmrstw - elk103
2在elk103中安装kibana
[root@elk103 ~]# ls anaconda-ks.cfg elasticsearch-7.17.3-x86_64.rpm kibana-7.17.3-x86_64.rpm [root@elk103 ~]# yum -y localinstall kibana-7.17.3-x86_64.rpm Repository extras is listed more than once in the configuration 上次元数据过期检查：3:44:46 前，执行于 2023年03月16日 星期四 15时24分44秒。 依赖关系解决。 =========================================================================================================================================== 软件包 架构 版本 仓库 大小 =========================================================================================================================================== 安装: kibana x86_64 7.17.3-1 @commandline 256 M 事务概要 =========================================================================================================================================== 安装 1 软件包 总计：256 M 安装大小：646 M 下载软件包： 运行事务检查 事务检查成功。 运行事务测试 事务测试成功。 运行事务 准备中 : 1/1 运行脚本: kibana-7.17.3-1.x86_64 1/1 安装 : kibana-7.17.3-1.x86_64 1/1 运行脚本: kibana-7.17.3-1.x86_64 1/1 Creating kibana group... OK Creating kibana user... OK Created Kibana keystore in /etc/kibana/kibana.keystore [/usr/lib/tmpfiles.d/elasticsearch.conf:1] Line references path below legacy directory /var/run/, updating /var/run/elasticsearch → /run/elasticsearch; please update the tmpfiles.d/ drop-in file accordingly. 验证 : kibana-7.17.3-1.x86_64 1/1 已安装: kibana-7.17.3-1.x86_64 完毕！ [root@elk103 ~]# vim /etc/kibana/kibana.yml [root@elk103 ~]# egrep -v '^$|^#' /etc/kibana/kibana.yml server.port: 5601 server.host: "0.0.0.0" server.name: "10.0.0.103" elasticsearch.hosts: ["http://10.0.0.101:9200","http://10.0.0.102:9200","http://10.0.0.103:9200"] i18n.locale: "zh-CN" [root@elk103 ~]# [root@elk103 ~]# systemctl enable --now kibana [root@elk103 ~]# ss -ntl State Recv-Q Send-Q Local Address:Port Peer Address:Port Process LISTEN 0 128 0.0.0.0:5601 0.0.0.0:* LISTEN 0 128 0.0.0.0:22 0.0.0.0:* LISTEN 0 128 *:9200 *:* LISTEN 0 128 *:9300 *:* LISTEN 0 128 [::]:22 [::]:* [root@elk103 ~]# [root@elk103 ~]# curl http://10.0.0.103:5601
3在elk102中部署filebeat
[root@elk102 ~]# ls anaconda-ks.cfg elasticsearch-7.17.3-x86_64.rpm elasticsearch.yml filebeat-7.17.3-x86_64.rpm [root@elk102 ~]# yum -y localinstall filebeat-7.17.3-x86_64.rpm Repository extras is listed more than once in the configuration 上次元数据过期检查：1:48:05 前，执行于 2023年03月16日 星期四 18时22分38秒。 依赖关系解决。 =========================================================================================================================================== 软件包 架构 版本 仓库 大小 =========================================================================================================================================== 安装: filebeat x86_64 7.17.3-1 @commandline 34 M 事务概要 =========================================================================================================================================== 安装 1 软件包 总计：34 M 安装大小：138 M 下载软件包： 运行事务检查 事务检查成功。 运行事务测试 事务测试成功。 运行事务 准备中 : 1/1 安装 : filebeat-7.17.3-1.x86_64 1/1 运行脚本: filebeat-7.17.3-1.x86_64 1/1 验证 : filebeat-7.17.3-1.x86_64 1/1 已安装: filebeat-7.17.3-1.x86_64 完毕！ [root@elk102 ~]# mkdir /etc/filebeat/config [root@elk102 ~]# cat > /etc/filebeat/config/01-stdin-to-console.yml <<'EOF' > # 指定输入类型 > filebeat.inputs: > # 指定输入的类型为 stdin ，表示标准输入 > - type: stdin > # 指定输出类型 > output.console: > # 打印漂亮的格式 > pretty: true > EOF [root@elk102 ~]# filebeat -e -c /etc/filebeat/config/01-stdin-to-console.yml 2023-03-16T20:12:14.245+0800 INFO instance/beat.go:685 Home path: [/usr/share/filebeat] Config path: [/etc/filebeat] Data path: [/var/lib/filebeat] Logs path: [/var/log/filebeat] Hostfs Path: [/] 2023-03-16T20:12:14.246+0800 INFO instance/beat.go:693 Beat ID: b2e99cfa-205d-4233-b612-de1083bee73d 2023-03-16T20:12:14.247+0800 INFO [seccomp] seccomp/seccomp.go:124 Syscall filter successfully installed 2023-03-16T20:12:14.247+0800 INFO [beat] instance/beat.go:1039 Beat info {"system_info": {"beat": {"path": {"config": "/etc/filebeat", "data": "/var/lib/filebeat", "home": "/usr/share/filebeat", "logs": "/var/log/filebeat"}, "type": "filebeat", "uuid": "b2e99cfa-205d-4233-b612-de1083bee73d"}}} 2023-03-16T20:12:14.247+0800 INFO [beat] instance/beat.go:1048 Build info {"system_info": {"build": {"commit": "1993ee88a11cb34f61a1fb45c7c3cf50533682cb", "libbeat": "7.17.3", "time": "2022-04-19T09:27:20.000Z", "version": "7.17.3"}}} 2023-03-16T20:12:14.247+0800 INFO [beat] instance/beat.go:1051 Go runtime info {"system_info": {"go": {"os":"linux","arch":"amd64","max_procs":2,"version":"go1.17.8"}}} 2023-03-16T20:12:14.248+0800 INFO [beat] instance/beat.go:1055 Host info {"system_info": {"host": {"architecture":"x86_64","boot_time":"2023-03-16T14:39:37+08:00","containerized":false,"name":"elk102","ip":["127.0.0.1/8","::1/128","10.0.0.102/24","fe80::20c:29ff:fef0:dd98/64"],"kernel_version":"4.18.0-425.3.1.el8.x86_64","mac":["00:0c:29:f0:dd:98"],"os":{"type":"linux","family":"","platform":"rocky","name":"Rocky Linux","version":"8.7 (Green Obsidian)","major":8,"minor":7,"patch":0},"timezone":"CST","timezone_offset_sec":28800,"id":"4b743c6f50e2489cbc623d4c36599108"}}} 2023-03-16T20:12:14.248+0800 INFO [beat] instance/beat.go:1084 Process info {"system_info": {"process": {"capabilities": {"inheritable":null,"permitted":["chown","dac_override","dac_read_search","fowner","fsetid","kill","setgid","setuid","setpcap","linux_immutable","net_bind_service","net_broadcast","net_admin","net_raw","ipc_lock","ipc_owner","sys_module","sys_rawio","sys_chroot","sys_ptrace","sys_pacct","sys_admin","sys_boot","sys_nice","sys_resource","sys_time","sys_tty_config","mknod","lease","audit_write","audit_control","setfcap","mac_override","mac_admin","syslog","wake_alarm","block_suspend","audit_read","38","39","40"],"effective":["chown","dac_override","dac_read_search","fowner","fsetid","kill","setgid","setuid","setpcap","linux_immutable","net_bind_service","net_broadcast","net_admin","net_raw","ipc_lock","ipc_owner","sys_module","sys_rawio","sys_chroot","sys_ptrace","sys_pacct","sys_admin","sys_boot","sys_nice","sys_resource","sys_time","sys_tty_config","mknod","lease","audit_write","audit_control","setfcap","mac_override","mac_admin","syslog","wake_alarm","block_suspend","audit_read","38","39","40"],"bounding":["chown","dac_override","dac_read_search","fowner","fsetid","kill","setgid","setuid","setpcap","linux_immutable","net_bind_service","net_broadcast","net_admin","net_raw","ipc_lock","ipc_owner","sys_module","sys_rawio","sys_chroot","sys_ptrace","sys_pacct","sys_admin","sys_boot","sys_nice","sys_resource","sys_time","sys_tty_config","mknod","lease","audit_write","audit_control","setfcap","mac_override","mac_admin","syslog","wake_alarm","block_suspend","audit_read","38","39","40"],"ambient":null}, "cwd": "/root", "exe": "/usr/share/filebeat/bin/filebeat", "name": "filebeat", "pid": 12241, "ppid": 12183, "seccomp": {"mode":"filter","no_new_privs":true}, "start_time": "2023-03-16T20:12:13.310+0800"}}} 2023-03-16T20:12:14.248+0800 INFO instance/beat.go:328 Setup Beat: filebeat; Version: 7.17.3 2023-03-16T20:12:14.248+0800 INFO [publisher] pipeline/module.go:113 Beat name: elk102 2023-03-16T20:12:14.249+0800 WARN beater/filebeat.go:202 Filebeat is unable to load the ingest pipelines for the configured modules because the Elasticsearch output is not configured/enabled. If you have already loaded the ingest pipelines or are using Logstash pipelines, you can ignore this warning. 2023-03-16T20:12:14.249+0800 INFO [monitoring] log/log.go:142 Starting metrics logging every 30s 2023-03-16T20:12:14.250+0800 INFO instance/beat.go:492 filebeat start running. 2023-03-16T20:12:14.251+0800 INFO memlog/store.go:119 Loading data file of '/var/lib/filebeat/registry/filebeat' succeeded. Active transaction id=0 2023-03-16T20:12:14.251+0800 INFO memlog/store.go:124 Finished loading transaction log file for '/var/lib/filebeat/registry/filebeat'. Active transaction id=0 2023-03-16T20:12:14.251+0800 WARN beater/filebeat.go:411 Filebeat is unable to load the ingest pipelines for the configured modules because the Elasticsearch output is not configured/enabled. If you have already loaded the ingest pipelines or are using Logstash pipelines, you can ignore this warning. 2023-03-16T20:12:14.251+0800 INFO [registrar] registrar/registrar.go:109 States Loaded from registrar: 0 2023-03-16T20:12:14.251+0800 INFO [crawler] beater/crawler.go:71 Loading Inputs: 1 2023-03-16T20:12:14.251+0800 INFO [crawler] beater/crawler.go:117 starting input, keys present on the config: [filebeat.inputs.0.type] 2023-03-16T20:12:14.251+0800 INFO [crawler] beater/crawler.go:148 Starting input (ID: 16876905907669988323) 2023-03-16T20:12:14.251+0800 INFO [crawler] beater/crawler.go:106 Loading and starting Inputs completed. Enabled inputs: 1 2023-03-16T20:12:14.251+0800 INFO [stdin.harvester] log/harvester.go:309 Harvester started for paths: [] {"harvester_id": "049abf4a-2fe0-4b30-8fc6-60d752252c06"} hello { "@timestamp": "2023-03-16T12:12:21.964Z", "@metadata": { "beat": "filebeat", "type": "_doc", "version": "7.17.3" }, "message": "hello", "input": { "type": "stdin" }, "ecs": { "version": "1.12.0" }, "host": { "name": "elk102" }, "agent": { "hostname": "elk102", "ephemeral_id": "006788ce-23b5-46b7-aa99-d46bfa7f926b", "id": "b2e99cfa-205d-4233-b612-de1083bee73d", "name": "elk102", "type": "filebeat", "version": "7.17.3" }, "log": { "file": { "path": "" }, "offset": 0 } } 2023-03-16T20:12:22.971+0800 ERROR file/states.go:125 State for should have been dropped, but couldn't as state is not finished. ^Z [1]+ 已停止 filebeat -e -c /etc/filebeat/config/01-stdin-to-console.yml [root@elk102 ~]#
4filebeat输出日志到屏幕测试案例
#在第一个elk102窗口中执行 [root@elk102 ~]# rm -rf /var/lib/filebeat/* [root@elk102 ~]# mkdir ~/config [root@elk102 ~]# vim ~/config/04-log-to-console.yml [root@elk102 ~]# cat ~/config/04-log-to-console.yml #输入 filebeat.inputs: - type: log enabled: true #是否启用 paths: - /tmp/test.log #数据路径 tags: ["tyjs09-linux80","容器运维"] #给此输入打个标记 fields: school: "北京市昌平区沙河镇" class: "linux80" - type: log enabled: true #是否启用 paths: - /tmp/test.log #数据路径 tags: ["13800130888"] #给此输入打个标记 fields: name: "张三" phone: "13800130888" adds: "北京市朝阳区酒仙桥北路甲10" #fields_under_root: true #是否设为顶级字段，意思就是把fields里的所有的value值都放入到input中 #输出 output.console: pretty: true [root@elk102 ~]# filebeat -e -c ~/config/04-log-to-console.yml #2再新开一个elk102窗口执行 [root@elk102 ~]# echo " hahahaha" > /tmp/test.log #3再次回到第一个elk102窗口观察 [root@elk102 ~]# filebeat -e -c ~/config/04-log-to-console.yml 2023-03-16T20:24:46.721+0800 INFO instance/beat.go:685 Home path: [/usr/share/filebeat] Config path: [/etc/filebeat] Data path: [/var/lib/filebeat] Logs path: [/var/log/filebeat] Hostfs Path: [/] 2023-03-16T20:24:46.724+0800 INFO instance/beat.go:693 Beat ID: 68ebae66-7e3c-4f9a-9670-cc5c83a08505 2023-03-16T20:24:46.725+0800 INFO [seccomp] seccomp/seccomp.go:124 Syscall filter successfully installed 2023-03-16T20:24:46.725+0800 INFO [beat] instance/beat.go:1039 Beat info {"system_info": {"beat": {"path": {"config": "/etc/filebeat", "data": "/var/lib/filebeat", "home": "/usr/share/filebeat", "logs": "/var/log/filebeat"}, "type": "filebeat", "uuid": "68ebae66-7e3c-4f9a-9670-cc5c83a08505"}}} 2023-03-16T20:24:46.725+0800 INFO [beat] instance/beat.go:1048 Build info {"system_info": {"build": {"commit": "1993ee88a11cb34f61a1fb45c7c3cf50533682cb", "libbeat": "7.17.3", "time": "2022-04-19T09:27:20.000Z", "version": "7.17.3"}}} 2023-03-16T20:24:46.725+0800 INFO [beat] instance/beat.go:1051 Go runtime info {"system_info": {"go": {"os":"linux","arch":"amd64","max_procs":2,"version":"go1.17.8"}}} 2023-03-16T20:24:46.726+0800 INFO [beat] instance/beat.go:1055 Host info {"system_info": {"host": {"architecture":"x86_64","boot_time":"2023-03-16T14:39:37+08:00","containerized":false,"name":"elk102","ip":["127.0.0.1/8","::1/128","10.0.0.102/24","fe80::20c:29ff:fef0:dd98/64"],"kernel_version":"4.18.0-425.3.1.el8.x86_64","mac":["00:0c:29:f0:dd:98"],"os":{"type":"linux","family":"","platform":"rocky","name":"Rocky Linux","version":"8.7 (Green Obsidian)","major":8,"minor":7,"patch":0},"timezone":"CST","timezone_offset_sec":28800,"id":"4b743c6f50e2489cbc623d4c36599108"}}} 2023-03-16T20:24:46.727+0800 INFO [beat] instance/beat.go:1084 Process info {"system_info": {"process": {"capabilities": {"inheritable":null,"permitted":["chown","dac_override","dac_read_search","fowner","fsetid","kill","setgid","setuid","setpcap","linux_immutable","net_bind_service","net_broadcast","net_admin","net_raw","ipc_lock","ipc_owner","sys_module","sys_rawio","sys_chroot","sys_ptrace","sys_pacct","sys_admin","sys_boot","sys_nice","sys_resource","sys_time","sys_tty_config","mknod","lease","audit_write","audit_control","setfcap","mac_override","mac_admin","syslog","wake_alarm","block_suspend","audit_read","38","39","40"],"effective":["chown","dac_override","dac_read_search","fowner","fsetid","kill","setgid","setuid","setpcap","linux_immutable","net_bind_service","net_broadcast","net_admin","net_raw","ipc_lock","ipc_owner","sys_module","sys_rawio","sys_chroot","sys_ptrace","sys_pacct","sys_admin","sys_boot","sys_nice","sys_resource","sys_time","sys_tty_config","mknod","lease","audit_write","audit_control","setfcap","mac_override","mac_admin","syslog","wake_alarm","block_suspend","audit_read","38","39","40"],"bounding":["chown","dac_override","dac_read_search","fowner","fsetid","kill","setgid","setuid","setpcap","linux_immutable","net_bind_service","net_broadcast","net_admin","net_raw","ipc_lock","ipc_owner","sys_module","sys_rawio","sys_chroot","sys_ptrace","sys_pacct","sys_admin","sys_boot","sys_nice","sys_resource","sys_time","sys_tty_config","mknod","lease","audit_write","audit_control","setfcap","mac_override","mac_admin","syslog","wake_alarm","block_suspend","audit_read","38","39","40"],"ambient":null}, "cwd": "/root", "exe": "/usr/share/filebeat/bin/filebeat", "name": "filebeat", "pid": 12306, "ppid": 12272, "seccomp": {"mode":"filter","no_new_privs":true}, "start_time": "2023-03-16T20:24:45.810+0800"}}} 2023-03-16T20:24:46.727+0800 INFO instance/beat.go:328 Setup Beat: filebeat; Version: 7.17.3 2023-03-16T20:24:46.727+0800 INFO [publisher] pipeline/module.go:113 Beat name: elk102 2023-03-16T20:24:46.728+0800 WARN beater/filebeat.go:202 Filebeat is unable to load the ingest pipelines for the configured modules because the Elasticsearch output is not configured/enabled. If you have already loaded the ingest pipelines or are using Logstash pipelines, you can ignore this warning. 2023-03-16T20:24:46.728+0800 INFO [monitoring] log/log.go:142 Starting metrics logging every 30s 2023-03-16T20:24:46.728+0800 INFO instance/beat.go:492 filebeat start running. 2023-03-16T20:24:46.729+0800 INFO memlog/store.go:119 Loading data file of '/var/lib/filebeat/registry/filebeat' succeeded. Active transaction id=0 2023-03-16T20:24:46.729+0800 INFO memlog/store.go:124 Finished loading transaction log file for '/var/lib/filebeat/registry/filebeat'. Active transaction id=0 2023-03-16T20:24:46.730+0800 WARN beater/filebeat.go:411 Filebeat is unable to load the ingest pipelines for the configured modules because the Elasticsearch output is not configured/enabled. If you have already loaded the ingest pipelines or are using Logstash pipelines, you can ignore this warning. 2023-03-16T20:24:46.730+0800 INFO [registrar] registrar/registrar.go:109 States Loaded from registrar: 0 2023-03-16T20:24:46.730+0800 INFO [crawler] beater/crawler.go:71 Loading Inputs: 2 2023-03-16T20:24:46.730+0800 INFO [crawler] beater/crawler.go:117 starting input, keys present on the config: [filebeat.inputs.0.enabled filebeat.inputs.0.fields.class filebeat.inputs.0.fields.school filebeat.inputs.0.paths.0 filebeat.inputs.0.tags.0 filebeat.inputs.0.tags.1 filebeat.inputs.0.type] 2023-03-16T20:24:46.730+0800 WARN [cfgwarn] log/input.go:89 DEPRECATED: Log input. Use Filestream input instead. 2023-03-16T20:24:46.730+0800 INFO [input] log/input.go:171 Configured paths: [/tmp/test.log] {"input_id": "294d18c6-e4b4-46b3-bcaa-5d2b1ae9de1b"} 2023-03-16T20:24:46.730+0800 INFO [crawler] beater/crawler.go:148 Starting input (ID: 6048882312929343489) 2023-03-16T20:24:46.730+0800 INFO [crawler] beater/crawler.go:117 starting input, keys present on the config: [filebeat.inputs.1.enabled filebeat.inputs.1.fields.adds filebeat.inputs.1.fields.name filebeat.inputs.1.fields.phone filebeat.inputs.1.paths.0 filebeat.inputs.1.tags.0 filebeat.inputs.1.type] 2023-03-16T20:24:46.730+0800 INFO [input] log/input.go:171 Configured paths: [/tmp/test.log] {"input_id": "5faf9526-4a90-4203-80eb-ed377f408427"} 2023-03-16T20:24:46.730+0800 INFO [crawler] beater/crawler.go:148 Starting input (ID: 18090793055986617323) 2023-03-16T20:24:46.730+0800 INFO [crawler] beater/crawler.go:106 Loading and starting Inputs completed. Enabled inputs: 2 hello 2023-03-16T20:25:16.744+0800 INFO [monitoring] log/log.go:184 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cgroup":{"memory":{"id":"session-15.scope","mem":{"limit":{"bytes":9223372036854771712},"usage":{"bytes":58720256}}}},"cpu":{"system":{"ticks":100,"time":{"ms":104}},"total":{"ticks":200,"time":{"ms":212},"value":200},"user":{"ticks":100,"time":{"ms":108}}},"handles":{"limit":{"hard":262144,"soft":1024},"open":12},"info":{"ephemeral_id":"1dd971b6-c108-468a-9ee8-958cbfd98ded","uptime":{"ms":30080},"version":"7.17.3"},"memstats":{"gc_next":20467872,"memory_alloc":16529800,"memory_sys":37307400,"memory_total":55555992,"rss":128761856},"runtime":{"goroutines":32}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"active":0},"type":"console"},"pipeline":{"clients":2,"events":{"active":0},"queue":{"max_events":4096}}},"registrar":{"states":{"current":0}},"system":{"cpu":{"cores":2},"load":{"1":0.24,"15":0.26,"5":0.26,"norm":{"1":0.12,"15":0.13,"5":0.13}}}}}} 2023-03-16T20:25:46.749+0800 INFO [monitoring] log/log.go:184 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cgroup":{"memory":{"mem":{"usage":{"bytes":-2015232}}}},"cpu":{"system":{"ticks":160,"time":{"ms":61}},"total":{"ticks":290,"time":{"ms":87},"value":290},"user":{"ticks":130,"time":{"ms":26}}},"handles":{"limit":{"hard":262144,"soft":1024},"open":12},"info":{"ephemeral_id":"1dd971b6-c108-468a-9ee8-958cbfd98ded","uptime":{"ms":60076},"version":"7.17.3"},"memstats":{"gc_next":18985744,"memory_alloc":9689016,"memory_total":56245272,"rss":124735488},"runtime":{"goroutines":32}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"active":0}},"pipeline":{"clients":2,"events":{"active":0}}},"registrar":{"states":{"current":0}},"system":{"load":{"1":0.15,"15":0.25,"5":0.23,"norm":{"1":0.075,"15":0.125,"5":0.115}}}}}} 2023-03-16T20:26:16.739+0800 INFO [monitoring] log/log.go:184 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cgroup":{"memory":{"mem":{"usage":{"bytes":12288}}}},"cpu":{"system":{"ticks":170,"time":{"ms":11}},"total":{"ticks":300,"time":{"ms":11},"value":300},"user":{"ticks":130}},"handles":{"limit":{"hard":262144,"soft":1024},"open":12},"info":{"ephemeral_id":"1dd971b6-c108-468a-9ee8-958cbfd98ded","uptime":{"ms":90074},"version":"7.17.3"},"memstats":{"gc_next":18985744,"memory_alloc":10176072,"memory_total":56732328,"rss":124735488},"runtime":{"goroutines":32}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"active":0}},"pipeline":{"clients":2,"events":{"active":0}}},"registrar":{"states":{"current":0}},"system":{"load":{"1":0.16,"15":0.25,"5":0.22,"norm":{"1":0.08,"15":0.125,"5":0.11}}}}}} 2023-03-16T20:26:46.738+0800 INFO [monitoring] log/log.go:184 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cgroup":{"memory":{"mem":{"usage":{"bytes":36864}}}},"cpu":{"system":{"ticks":190,"time":{"ms":17}},"total":{"ticks":320,"time":{"ms":17},"value":320},"user":{"ticks":130}},"handles":{"limit":{"hard":262144,"soft":1024},"open":12},"info":{"ephemeral_id":"1dd971b6-c108-468a-9ee8-958cbfd98ded","uptime":{"ms":120075},"version":"7.17.3"},"memstats":{"gc_next":18985744,"memory_alloc":11191552,"memory_total":57747808,"rss":124735488},"runtime":{"goroutines":32}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"active":0}},"pipeline":{"clients":2,"events":{"active":0}}},"registrar":{"states":{"current":0}},"system":{"load":{"1":0.16,"15":0.25,"5":0.22,"norm":{"1":0.08,"15":0.125,"5":0.11}}}}}} 2023-03-16T20:26:46.810+0800 INFO [input.harvester] log/harvester.go:309 Harvester started for paths: [/tmp/test.log] {"input_id": "294d18c6-e4b4-46b3-bcaa-5d2b1ae9de1b", "source": "/tmp/test.log", "state_id": "native::33641184-64768", "finished": false, "os_id": "33641184-64768", "harvester_id": "1ae90814-f6f8-49ef-a375-65636fa27f0e"} 2023-03-16T20:26:46.810+0800 INFO [input.harvester] log/harvester.go:309 Harvester started for paths: [/tmp/test.log] {"input_id": "5faf9526-4a90-4203-80eb-ed377f408427", "source": "/tmp/test.log", "state_id": "native::33641184-64768", "finished": false, "os_id": "33641184-64768", "harvester_id": "27b7e44e-9a9c-4d45-892f-2ca28d9f0ca3"} 2023-03-16T20:27:16.734+0800 INFO [monitoring] log/log.go:184 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cgroup":{"memory":{"mem":{"usage":{"bytes":61440}}}},"cpu":{"system":{"ticks":200,"time":{"ms":10}},"total":{"ticks":330,"time":{"ms":10},"value":330},"user":{"ticks":130}},"handles":{"limit":{"hard":262144,"soft":1024},"open":14},"info":{"ephemeral_id":"1dd971b6-c108-468a-9ee8-958cbfd98ded","uptime":{"ms":150072},"version":"7.17.3"},"memstats":{"gc_next":18985744,"memory_alloc":11739520,"memory_total":58295776,"rss":124735488},"runtime":{"goroutines":42}},"filebeat":{"events":{"added":2,"done":2},"harvester":{"open_files":2,"running":2,"started":2}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"active":0}},"pipeline":{"clients":2,"events":{"active":0,"filtered":2,"total":2}}},"registrar":{"states":{"current":1,"update":2},"writes":{"success":2,"total":2}},"system":{"load":{"1":0.73,"15":0.29,"5":0.34,"norm":{"1":0.365,"15":0.145,"5":0.17}}}}}} { "@timestamp": "2023-03-16T12:27:21.863Z", "@metadata": { "beat": "filebeat", "type": "_doc", "version": "7.17.3" }, "ecs": { "version": "1.12.0" }, "host": { "name": "elk102" }, "agent": { "id": "68ebae66-7e3c-4f9a-9670-cc5c83a08505", "name": "elk102", "type": "filebeat", "version": "7.17.3", "hostname": "elk102", "ephemeral_id": "1dd971b6-c108-468a-9ee8-958cbfd98ded" }, "log": { "offset": 0, "file": { "path": "/tmp/test.log" } }, "message": " hahahaha", "tags": [ "tyjs09-linux80", "容器运维" ], "input": { "type": "log" }, "fields": { "class": "linux80", "school": "北京市昌平区沙河镇" } } { "@timestamp": "2023-03-16T12:27:21.863Z", "@metadata": { "beat": "filebeat", "type": "_doc", "version": "7.17.3" }, "input": { "type": "log" }, "fields": { "adds": "北京市朝阳区酒仙桥北路甲10", "name": "张三", "phone": "13800130888" }, "ecs": { "version": "1.12.0" }, "host": { "name": "elk102" }, "agent": { "name": "elk102", "type": "filebeat", "version": "7.17.3", "hostname": "elk102", "ephemeral_id": "1dd971b6-c108-468a-9ee8-958cbfd98ded", "id": "68ebae66-7e3c-4f9a-9670-cc5c83a08505" }, "log": { "offset": 0, "file": { "path": "/tmp/test.log" } }, "message": " hahahaha", "tags": [ "13800130888" ] } ^Z [1]+ 已停止 filebeat -e -c ~/config/04-log-to-console.yml [root@elk102 ~]#
5filebeat输出日志到es测试案例
[root@elk102 ~]# rm -rf /var/lib/filebeat/* [root@elk102 ~]# cp ~/config/04-log-to-console.yml ~/config/05-log-to-console.yml [root@elk102 ~]# vim ~/config/05-log-to-console.yml [root@elk102 ~]# cat ~/config/05-log-to-console.yml #输入 filebeat.inputs: - type: log enabled: true #是否启用 paths: - /tmp/test.log #数据路径 tags: ["tyjs09-linux80","容器运维"] #给此输入打个标记 fields: school: "北京市昌平区沙河镇" class: "linux80" - type: log enabled: true #是否启用 paths: - /tmp/test.log #数据路径 tags: ["13800130888"] #给此输入打个标记 fields: name: "张三" phone: "13800130888" adds: "北京市朝阳区酒仙桥北路甲10" #fields_under_root: true #是否设为顶级字段，意思就是把fields里的所有的value值都放入到input中 #输出 #output.console: # pretty: true output.elasticsearch: hosts: ["http://10.0.0.101:9200","http://10.0.0.102:9200","http://10.0.0.103:9200"] [root@elk102 ~]# filebeat -e -c ~/config/05-log-to-console.yml #4打开http://10.0.0.103:5601/app/home#/ 自己浏览--点左上角主菜单--Stack Management--索引管理--此时你会看到索引名称filebeat-7.17.3-2023.03.16-000001--回到Stack Management页面选择索引模式--创建索引模式--索引名称输入filebeat-7.17.3-2023.03*表示要取3月份的所有数据--时间戳选择@timestamp即可--创建索引模式 #5在kibana的发现页面中查看数据 主菜单--选择discover--选择索引--选择时间我选择的是本周因为选择当天没有出现数据，之后你会看到左侧边栏会出现一些可用字段，右侧大区域你会看到很多抓取的日志，展开日志后选择jaon格式，然后你会看到你前面测试配置文件内定义的数据--点击左侧边栏message字段然后点击+号后你会看到字段的内容 #6往日志文件存储文件追加内容再次观察kibana [root@elk102 ~]# cat /tmp/test.log 111 222 [root@elk102 ~]# echo AAA >> /tmp/test.log [root@elk102 ~]# cat /tmp/test.log 111 222 AAA kibana操作：主菜单--discover--点击message旁边的加号查看消息
