elk-（配置）
https://www.dgstack.cn/archives/2858.html
配置：https://wsgzao.github.io/post/elk/
优化：https://www.cnblogs.com/along21/p/8613115.html
实例：
###/etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
enabled: true
paths:
- /home/loguser/logs/gamedb/*.log
fields:
hostname: 172.31.93.185
logindex: gamedb_savedata
exclude_lines: ['^$']
multiline:
pattern: '^20\d{2}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}'
negate: true
match: after
max_lines: 20
timeout: 5s
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
setup.template.settings:
index.number_of_shards: 3
output.kafka:
hosts: ["172.31.71.159:19092","172.31.71.160:19092","172.31.71.161:19092"]
topic: "product_release"
或者
filebeat.inputs:
- type: log
enabled: true
paths:
- /home/loguser/logs/tsg_logs/*/*.log
fields:
hostname: 172.31.93.157
logindex: 三公
exclude_lines: ['^$']
multiline:
pattern: '^\[([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)'
negate: true
match: after
max_lines: 20
timeout: 5s
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
setup.template.settings:
index.number_of_shards: 3
output.kafka:
hosts: ["172.31.71.159:19092","172.31.71.160:19092","172.31.71.161:19092"]
topic: "product_release"
###/home/kafka/kafka_2.11-2.2.0/config/server.properties
broker.id=0
port=19092
host.name=172.31.71.159
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/home/kafka/kafkalogs/
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
message.max.byte=5242880
default.replication.factor=2
replica.fetch.max.bytes=5242880
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=172.31.71.159:12181,172.31.71.160:12181,172.31.71.161:12181
zookeeper.connection.timeout.ms=6000
group.initial.rebalance.delay.ms=0
###/home/zookeeper/zookeeper-3.4.14/conf/zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/home/zookeeper/zkdata
dataLogDir=/home/zookeeper/zkdatalog
clientPort=12181
server.1=172.31.71.159:12888:13888
server.2=172.31.71.160:12888:13888
server.3=172.31.71.161:12888:13888
###/etc/logstash/conf.d/product.conf
input {
kafka {
codec => "json"
topics_pattern => "product_release"
bootstrap_servers => "172.31.71.159:19092,172.31.71.160:19092,172.31.71.161:19092"
}
}
filter{
if [fields][logindex] == "gamedb_savedata"
{
grok{
match => ["message", "%{GAME_DATA_DATEST:log_timestamp} %{LOGLEVEL:level}(?:\s)? %{GREEDYDATA:infomations}"]
}
mutate{
gsub =>["log_timestamp","T"," "]
}
date{
match => ["log_timestamp", "yyyy-MM-dd HH:mm:ss.SSS"]
target => "@timestamp"
}
}
else if [fields][logindex] == "路由" or [fields][logindex] == "网关"
{
grok{
match => ["message", "\[%{JAVA_DATEST:log_timestamp}\] \[%{LOGLEVEL:level}(?:\s)?\] %{GREEDYDATA:infomations}"]
}
date{
match => ["log_timestamp", "yyyy-MM-dd HH:mm:ss.SSS"]
target => "@timestamp"
}
}
else if [fields][logindex] == "定时任务"
{
grok{
match => ["message", "%{JAVA_DATEST:log_timestamp} %{LOGLEVEL:level}(?:\s)?\ \[%{JAVAFILE:filename}\](?:\:)? %{GREEDYDATA:infomations}"]
}
date{
match => ["log_timestamp", "yyyy-MM-dd HH:mm:ss,SSS"]
target => "@timestamp"
}
}
else if [fields][game] and [fields][game] == 'account'
{
grok{
match => ["message", "%{JAVA_DATEST:log_timestamp} \[%{JAVAFILE:filename}\](?:\:)? %{LOGLEVEL:level}(?:\s)? %{GREEDYDATA:infomations}"]
}
date{
match => ["log_timestamp", "yyyy-MM-dd HH:mm:ss.SSS"]
target => "@timestamp"
}
}
else
{
grok{
match => ["message", "\[%{LOGLEVEL:level}(?:\s)?\] %{JAVA_DATEST:log_timestamp} \[%{JAVAFILE:filename}\](?:\:)? %{GREEDYDATA:infomations}"]
}
date{
match => ["log_timestamp", "yyyy-MM-dd HH:mm:ss,SSS"]
target => "@timestamp"
}
}
if "_grokparsefailure" in [tags] {
drop {}
}
mutate {
rename => ["source", "日志目录"]
rename => ["infomations", "日志信息"]
rename => ["[fields][hostname]", "服务器IP"]
rename => ["level", "日志等级"]
gsub =>["日志目录","/home/loguser/logs/",""]
remove_field =>["message","host","@version","offset","beat","log","prospector","id","input","_index","_type","_score"]
}
}
output {
elasticsearch {
hosts => ["172.31.71.156:9200","172.31.71.157:9200","172.31.71.158:9200","172.31.71.162:9200"]
index => "%{[fields][logindex]}_%{+YYYY.MM.dd}"
}
if [日志等级] == "error" and ([fields][logindex] == "网关" or [fields][logindex] == "路由")
{
exec {
command => "/etc/logstash/conf.d/telegram_gate_route_message.py %{服务器IP} %{日志目录} %{log_timestamp} %{[fields][logindex]} %{日志等级} '%{日志信息}'"
}
}
if [日志等级] == "ERROR" and [日志信息] !~ "^netty,通道异常:" and [日志信息] !~ "netty,通道断开" and [日志信息] !~ "房间输赢:useid:\d+,redis储存成功 Error:.*" and [日志信息] !~ "同步DB数据超时" and [日志信息] !~ "回复消息未被处理,回复时间过长已做超时处理,或没有写回调函数,msgType:\d+" and [日志信息] !~ ".*org.springframework.web.HttpRequestMethodNotSupportedException: Request method 'GET' not supported.*"
{
exec {
command => "/etc/logstash/conf.d/telegram_send_message.py %{服务器IP} %{日志目录} %{log_timestamp} %{[fields][logindex]} %{日志等级} '%{日志信息}'"
}
}
#stdout{
# codec => rubydebug
#}
}
####telegram_gate_route_message.py
#!/usr/bin/python
#-*- coding: utf-8 -*-*
import telegram
import sys
from tasks import send_message
ip = sys.argv[1]
source = sys.argv[2]
log_timestamp = sys.argv[3] + " " + sys.argv[4]
index = sys.argv[5]
level = sys.argv[6]
infomations = sys.argv[7].strip()
ip=ip.replace('-','.')
ip="告警主机："+ip
source="日志路径："+source
log_timestamp="告警时间："+log_timestamp
index="日志索引："+index
level="日志等级："+level
infomations="日志信息："+infomations
content=ip+'\n'+source+'\n'+log_timestamp+'\n'+index+'\n'+level+'\n'+infomations
bot = telegram.Bot(token='858445613:AAEZ_c1uMsUyYA0II8mwXNYrywxDxZ5zh4A')
bot.send_message(chat_id='-339121990', text=content)
###tasks.py
#!/usr/bin/python
#-*- coding: utf-8 -*-*
import telegram
from celery import Celery
from celery import platforms
platforms.C_FORCE_ROOT = True
celery = Celery('tasks', broker='redis://127.0.0.1:6379/0', backend='redis://127.0.0.01:6379/0')
@celery.task
def send_message(content):
bot = telegram.Bot(token='858445613:AAEZ_c1uMsUyYA0II8mwXNYrywxDxZ5zh4A')
bot.send_message(chat_id='-391035231', text=content)
###telegram_send_message.py
#!/usr/bin/python
#-*- coding: utf-8 -*-*
import telegram
import sys
#from tasks import send_message
ip = sys.argv[1]
source = sys.argv[2]
log_timestamp = sys.argv[3] + " " + sys.argv[4]
index = sys.argv[5]
level = sys.argv[6]
infomations = sys.argv[7].strip()
ip=ip.replace('-','.')
ip="告警主机："+ip
source="日志路径："+source
log_timestamp="告警时间："+log_timestamp
index="日志索引："+index
level="日志等级："+level
infomations="日志信息："+infomations
content=ip+'\n'+source+'\n'+log_timestamp+'\n'+index+'\n'+level+'\n'+infomations
#send_message.delay(content)
bot = telegram.Bot(token='858445613:AAEZ_c1uMsUyYA0II8mwXNYrywxDxZ5zh4A')
bot.send_message(chat_id='-391035231', text=content)
###send_message.sh
#!/usr/bin/bash
cd /etc/logstash/conf.d
/usr/bin/celery -A tasks worker --loglevel=info
###/usr/bin/celery
!/usr/bin/python2
# -*- coding: utf-8 -*-
import re
import sys
from celery.__main__ import main
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(main())
### logstash.sh
#!/usr/bin/bash
cd /etc/logstash/conf.d
/usr/share/logstash/bin/logstash -f product.conf
###elasticsearch.yml
cluster.name: lygj_es
node.name: node-1
node.master: true
node.data: true
path.data: /home/elasticsearch/data
path.logs: /home/elasticsearch/log
network.host: 172.31.71.156
discovery.zen.ping.unicast.hosts: ["172.31.71.156", "172.31.71.157","172.31.71.158","172.31.71.162"]
discovery.zen.minimum_master_nodes: 2
http.cors.enabled: true
http.cors.allow-origin: "*"
https://blog.51cto.com/13527416/2117141
filebeat.inputs:
- type: log
enabled: true
paths:
- /home/loguser/logs/account-service/*.log
fields:
hostname: 172.20.103.107
logindex: account-server
game: account
exclude_lines: ['^$']
multiline:
pattern: '^20\d{2}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}'
negate: true
match: after
max_lines: 40
timeout: 5s
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
setup.template.settings:
index.number_of_shards: 3
output.redis:
hosts: ["172.20.103.153"]
key: "filebeat"
db: 1
timeout: 5
