docker swarm 部署logstash通过动态扩容消费海量数据

背景

海量数据需要接入logstash消费到ES、Kafka等, 如并发很高的nginx日志

解决方案

• docker swarm部署logstash集群
• 通过docker config增加pipline、templates

docker-compose文件

• docker-compose-logstash.yml

version: '3.9'
services:
  logstash:
    image: logstash:7.10.1
    environment:
      - LS_JAVA_OPTS=-Xmx2g -Xms2g
    user: root
    ports:
      - "4515:4515/udp"
    networks:
      - logstash_distributed
    configs:
      - source: logstash.conf
        target: /usr/share/logstash/pipeline/logstash.conf
      - source: security_template.json
        target: /etc/logstash/templates/security_template.json

    deploy:
      mode: replicated
      replicas: 3
      resources:
        limits: # 资源使用上限
          cpus: "2"
          memory: 4G
        reservations: # 随时可以使用的资源
          cpus: "1"
          memory: 2G

configs:
  logstash.conf:
    file: ./piplines/logstash.conf
  security_template.json:
    file: ./templates/security_template.json

networks:
  logstash_distributed:
    external: true

创建专有网路

docker network create --driver overlay logstash_distributed

部署

docker stack deploy -c docker-compose-logstash.yml logstash

验证

[root@zhongan-soc logstash]# docker service ls
ID             NAME                MODE         REPLICAS   IMAGE             PORTS
pgjboyvloo9v   logstash_logstash   replicated   3/3      logstash:7.10.1   *:4515-4516->4515-4516/udp
[root@zhongan-soc logstash]# docker service ps logstash_logstash 
ID             NAME                   IMAGE             NODE                DESIRED STATE   CURRENT STATE          ERROR     PORTS
cevswc8z6lvt   logstash_logstash.1    logstash:7.10.1   zhongan-soc-core3   Running         Running 16 hours ago             
3gpfjuhwxpad   logstash_logstash.2    logstash:7.10.1   zhongan-soc-core    Running         Running 16 hours ago             
t1iftpc9vnz7   logstash_logstash.3    logstash:7.10.1   zhongan-soc-core2   Running         Running 16 hours ago