第17周

1、Kibana地图显示用户IP所在城市
2、RabbitMQ集群部署
3、Zookeeper集群部署

#============================================================================

1 Kibana地图显示用户IP所在城市

#注意事项：
 1、日志必须是json格式
 2、添加Geoip地址库
 3、es-index名称以logstash开头
 4、logstash才支持，filebeat不支持，在logstash提取主机上配置

#========================================================
#地址库文件
 http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz

# cd /etc/logstash
#传地址库文件
# tar xf GeoLite2-City.tar.gz

#配置logstash提取主机
# vi /etc/logstash/conf.d/geoip-redis-to-es.conf
input {
  redis {
    host => "10.0.0.67"                        #redis-server地址
    port => "6379"
    password => "root"
    key => "message"
    data_type => "list"
    db => "0"
    timeout => "10"
  }
  redis {
    host => "10.0.0.67"                        #redis-server地址
    port => "6379"
    password => "root"
    key => "nginx-access"
    data_type => "list"
    db => "0"
    timeout => "10"
    codec => "json"                #访问日志json格式
  }
  redis {
    host => "10.0.0.67"                        #redis-server地址
    port => "6379"
    password => "root"
    key => "nginx-error"
    data_type => "list"
    db => "0"
    timeout => "10"
  }
  redis {
    host => "10.0.0.67"                        #redis-server地址
    port => "6379"
    password => "root"
    key => "message"
    data_type => "list"
    db => "1"
    timeout => "10"
  }
  redis {
    host => "10.0.0.67"                        #redis-server地址
    port => "6379"
    password => "root"
    key => "tomcat-access"
    data_type => "list"
    db => "1"
    timeout => "10"
    codec => "json"                #访问日志json格式
  }
}

filter {
  if [fields][type] == "nginx-access" {
    geoip {
      source => "clientip"
      target => "geoip"
      database => "/etc/logstash/GeoLite2-City/GeoLite2-City.mmdb"
      add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
      add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
    }
    mutate {
      convert => [ "[geoip][coordinates]", "float" ]
    }
  }
}

output {
  if [fields][type] == "message" {
    elasticsearch {
      hosts => ["10.0.0.248:9200"]                        #es集群vip
      index => "rs-message-0.47-%{+YYYY.MM.dd}"
    }
  }
  if [fields][type] == "nginx-access" {
    elasticsearch {
      hosts => ["10.0.0.248:9200"]                        #es集群vip
      index => "logstash-rs-nginx_access-0.47-%{+YYYY.MM.dd}"
    }
  }
  if [fields][type] == "nginx-error" {
    elasticsearch {
      hosts => ["10.0.0.248:9200"]                        #es集群vip
      index => "rs-nginx_error-0.47-%{+YYYY.ww}"
    }
  }
  if [type] == "message" {
    elasticsearch {
      hosts => ["10.0.0.248:9200"]                        #es集群vip
      index => "rs-message-0.57-%{+YYYY.MM.dd}"
    }
  }
  if [type] == "tomcat_access" {
    elasticsearch {
      hosts => ["10.0.0.248:9200"]                        #es集群vip
      index => "rs-tomcat_access-0.57-%{+YYYY.MM.dd}"
    }
  }
}

#配置文件语法检测
# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/geoip-redis-to-es.conf -t

#以服务方式启动logstash并设置开机自启
# systemctl enable --now logstash

#查看logstash日志输出
# tail -f /var/log/logstash/logstash-plain.log

#kibana创建索引模式

#kibana创建图形
#可视化->创建可视化->Maps->添加图层->文档->索引模式：logstash-rs-nginx_access-0.47-*->添加图层->保存

2 RabbitMQ集群部署

#各节点添加主机解析
# vi /etc/hosts
 10.0.0.7 rabbitmq-1 rabbitmq-1.testou.com
 10.0.0.27 rabbitmq-2 rabbitmq-2.testou.com
 10.0.0.37 rabbitmq-3 rabbitmq-3.testou.com

#rpm安装各rabbitmq节点，先起一个节点
#各节点传包
#编译erlang-23.1
# tar xf /usr/local/src/otp_src_23.1.tar.gz -C /usr/local
# cd /usr/local/otp
# yum -y install gcc gcc-c++ make ncurses-devel wxGTK-devel openssl-devel unixODBC-devel socat
# ./configure --prefix=/usr/local/erlang_23.1
# make -j 4
# make install
# ln -s /usr/local/erlang_23.1 /usr/local/erlang
# ln -s /usr/local/erlang/bin/* /usr/bin/
# rm -rf /usr/local/otp /usr/local/src/otp_src_23.1.tar.gz
#rpm安装rabbitmq-3.8.3
# rpm -ivh --nodeps /usr/local/src/rabbitmq-server-3.8.3-1.el7.noarch.rpm
#插件管理，插件预安装，只须启用
# rabbitmq-plugins list        #查看插件列表
# rabbitmq-plugins enable rabbitmq_management    #打开web管理端，监听tcp_15672
#更改配置文件，允许guest用户远程登录（3.3.0版本开始，guest用户只允许localhost登录）
# sed -ri '/loopback_users.*guest/s#(.*\[).*(\].*)#\1\2#' /usr/lib/rabbitmq/lib/rabbitmq_server-3.8.3/ebin/rabbit.app
#启动rabbitmq一节点
# systemctl enable --now rabbitmq-server

#同步cookie文件            #集群各节点cookie文件必须一致
# scp /var/lib/rabbitmq/.erlang.cookie root@10.0.0.27:/var/lib/rabbitmq/.erlang.cookie
# scp /var/lib/rabbitmq/.erlang.cookie root@10.0.0.37:/var/lib/rabbitmq/.erlang.cookie

#启动剩余rabbitmq节点
# systemctl enable --now rabbitmq-server

#查看集群状态
# rabbitmqctl cluster_status

#在节点一和节点二，以节点三为目标服务器，加入集群，加之前节点一和节点二不能有任何数据
#停止app服务
# rabbitmqctl stop_app
#清空元数据
# rabbitmqctl reset
#添加节点
# rabbitmqctl join_cluster rabbit@rabbitmq-3 --ram        #rabbit为协议，rabbitmq-3为对方短域名，--ram为内存节点
#开启app服务
# rabbitmqctl start_app

#设置集群模式为镜像模式，只需在其中一个节点设置
# rabbitmqctl set_policy ha-all "*" '{"ha-mode":"all"}'

#在节点一新建账号，在其他节点验证
# rabbitmqctl add_user alice 123456
# rabbitmqctl set_permissions alice ".*" ".*" ".*"
#浏览器访问节点二
 http://10.0.0.7:15672，默认用户名密码guest

#查看集群状态
# rabbitmqctl cluster_status

3 Zookeeper集群部署

#各节点装jdk，1.8版本

#各节点传包
#各节点解包
# tar xf /usr/local/src/zookeeper-3.4.14.tar.gz -C /usr/local
# ln -s /usr/local/zookeeper-3.4.14 /usr/local/zookeeper

#各节点拷备配置文件
# cp /usr/local/zookeeper/conf/zoo_sample.cfg /usr/local/zookeeper/conf/zoo.cfg
#各节点修改配置文件（配置项后不可有注释信息，否则无法启动）
# vi /usr/local/zookeeper/conf/zoo.cfg
 tickTime=2000        #单次存活探测时间，单次2000ms
 initLimit=10        #集群初始化时存活探测频率，10次
 syncLimit=5        #集群初始化后存活探测频率，5次
 dataDir=/usr/local/zookeeper/data    #数据目录
 clientPort=2181    #客户端连接zookeeper端口
 maxClientCnxns=128        #单个IP地址可以发起的并发连接数
 autopurge.snapRetainCount=3        #3.4.0新功能，保留最新快照数据的个数
 autopurge.purgeInterval=24            #快照数据清理频率，单位：小时
server.1=10.0.0.7:2888:3888        #集群成员发现，类似定义单播
server.2=10.0.0.27:2888:3888    #server_id：同一集群内必须唯一，2888为数据同步端口，3888为leader选举端口
server.3=10.0.0.37:2888:3888

#各节点创建数据目录
# mkdir /usr/local/zookeeper/data

#各节点生成server_id
# echo 1 > /usr/local/zookeeper/data/myid    #节点1执行
# echo 2 > /usr/local/zookeeper/data/myid    #节点2执行
# echo 3 > /usr/local/zookeeper/data/myid    #节点3执行

#各节点设置开机自启并立即启动，20s内启动全部节点，因为配置文件定义20s认为超时
# /usr/local/zookeeper/bin/zkServer.sh start
# echo '/usr/local/zookeeper/bin/zkServer.sh start' >> /etc/rc.local
# [ -x /etc/rc.d/rc.local ] || chmod +x /etc/rc.d/rc.local

#查看集群节点状态
# /usr/local/zookeeper/bin/zkServer.sh status
 ZooKeeper JMX enabled by default
 Using config: /usr/local/zookeeper/bin/../conf/zoo.cfg
 Mode: leader            #leader节点
# /usr/local/zookeeper/bin/zkServer.sh status
 ZooKeeper JMX enabled by default
 Using config: /usr/local/zookeeper/bin/../conf/zoo.cfg
 Mode: follower            #follower节点

#命令行方式向集群节点写入数据，在其他节点验证是否同步
# /usr/local/zookeeper/bin/zkCli.sh -server 10.0.0.27:2181
 create /app1 "2020-03-23 INFO start nginx"
 get /app1
 delete /app1