ElasticSearch 5.5.0之 logstash-02

1.安装java环境，logstash需要java 1.8.0_73以上版本

yum list java*
yum install java-1.8.0-openjdk -y
java -version

 2.logstash安装

# yum 源安装，方式一
rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch
cat > /etc/yum.repos.d/logstash.repo <<EOF
[logstash-5.x]
name=Elastic repository for 5.x packages
baseurl=https://artifacts.elastic.co/packages/5.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md
EOF

yum clean all
yum install logstash


# 先下载rpm直接安装，方式二 （推荐）
wget https://artifacts.elastic.co/downloads/logstash/logstash-5.5.0.rpm
rz -y logstash-5.5.0.rpm
yum localinstall logstash-5.5.0.rpm 

 3.建立logstash文件夹及权限

mkdir /etc/logstash/logs -p
mkdir /etc/logstash/data -p
chown -R logstash.logstash /etc/logstash/logs
chown -R logstash.logstash /etc/logstash/data

 4.修改logstash配置文件

vim /etc/logstash/logstash.yml
grep -v "^#" /etc/logstash/logstash.yml 
path.data: /etc/logstash/data
path.config: /etc/logstash/conf.d
path.logs: /etc/logstash/logs

 5.测试logstash

# 测试logstash
/usr/share/logstash/bin/logstash -e 'input { stdin { } } output { stdout {} }'
/usr/share/logstash/bin/logstash -e 'input { stdin { } } output { stdout{ codec => rubydebug } }'

# 输出到elasticsearch及屏幕打印
/usr/share/logstash/bin/logstash -e 'input { stdin { } } output { elasticsearch { hosts => ["192.168.1.22:9200"] } stdout{ codec => rubydebug } }'

 6.启动，重载logstash服务器

# 启动测试
/usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logs.conf

重载服务
/usr/share/logstash/bin/logstash -t -f /etc/logstash/conf.d/logs.conf --path.settings /etc/logstash/ --config.reload.automatic

 7.logstash启动脚本帮助及生成

cd /usr/share/logstash/bin/
./system-install --help

# centos 7生成启动脚本（方式一）
[root@logstash system]# cd /usr/share/logstash/bin/
[root@logstash bin]# ./system-install /etc/logstash/startup.options systemd
Using provided startup.options file: /etc/logstash/startup.options
Manually creating startup for specified platform: systemd
Successfully created system startup script for Logstash

启动Logstash服务
systemctl enable logstas
systemctl start logstash
systemctl stop logstash
systemctl restart logstash
systemctl status logstash


# centos 6生成启动脚本（方式二）
[root@logstash system]# cd /usr/share/logstash/bin/
[root@logstash bin]# ./system-install /etc/logstash/startup.options sysv

/etc/init.d/logstash start
/etc/init.d/logstash stop

 8.centos7 开放端口

yum install firewalld -y
firewall-cmd --permanent --add-port={9200/tcp,9300/tcp}
firewall-cmd --permanent --add-port=5601/tcp
firewall-cmd --reload
firewall-cmd -list-all

端口转发：
firewall-cmd --parmanent --add-forward-port=port=80:proto=tcp:toport=5601
firewall-cmd --reload

 9.grok规则目录

# logstash日志清洗grok规则
/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-4.1.1/patterns
\s        表示一个空格
\s\s        二个空格
\n        换行符
(?m)        打开多行模式开关

 

 

https://github.com/logstash-plugins/logstash-patterns-core/blob/master/patterns/grok-patterns
https://github.com/elastic/logstash/blob/v1.4.2/patterns/grok-patterns
https://github.com/logstash-plugins/logstash-patterns-core/tree/master/patterns
https://www.elastic.co/blog/do-you-grok-grok
http://blog.csdn.net/loophome/article/details/52353869