K8S ilogtail + kafka + logstash + es + kbana 收集日志(一) 部署 kafka 和 zookeeper

环境说明

1. k8s 版本 v1.24.1
2. 容器运行时 containerd://1.6.8

参考资料
GitHub:
https://github.com/alibaba/ilogtail

社区版用户手册：
https://ilogtail.gitbook.io/ilogtail-docs

使用DaemonSet模式采集K8s容器日志:
https://zhuanlan.zhihu.com/p/544079650

如何将业务日志采集到Kafka:
https://zhuanlan.zhihu.com/p/544081167

采集输出在标准错误流的Java异常堆栈（多行日志）。
https://ilogtail.gitbook.io/ilogtail-docs/data-pipeline/input/input-docker-stdout

logstash kafka 插件
https://www.elastic.co/guide/en/logstash/7.4/plugins-inputs-kafka.html

网上个人文章
https://blog.csdn.net/weixin_41934601/article/details/125865098

注意： 这里使用了 Deployment 来安装 kafka 和 zookeeper 需要使用到静态的pvc 才可以保存数据

安装 zookeeper

mkdir -p /data/yaml/kube-system/zookeeper
cd /data/yaml/kube-system/zookeeper

cat nfs.yaml 
apiVersion: v1
kind: PersistentVolume
metadata:
  name: zookeeper-data
  namespace: kube-system
  labels:
    app: zookeeper-data
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteMany    
  persistentVolumeReclaimPolicy: Retain 
  storageClassName: nfs  
  nfs:
    path: /data/nfsdata/kube-system/zookeeper
    server: 172.16.16.150
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  namespace: kube-system
  name: zookeeper-data
spec:
  accessModes:
    - ReadWriteMany
  volumeMode: Filesystem
  resources:
    requests:
      storage: 10Gi
  storageClassName: nfs

kubectl apply -f nfs.yaml

cat deployment.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: zookeeper
  namespace: kube-system
  labels:
    app: zookeeper
spec:
  replicas: 1
  minReadySeconds: 100
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 0
  selector:
    matchLabels:
      app: zookeeper
  template:
    metadata:
      labels:
        app: zookeeper
    spec:
      containers:
      - name: zookeeper
        image: bitnami/zookeeper:latest
        imagePullPolicy: IfNotPresent
        env:
          - name: ALLOW_ANONYMOUS_LOGIN
            value: 'yes'
        volumeMounts:
          - name: zookeeper-data
            mountPath: /bitnami/zookeeper
      volumes:
        - name: zookeeper-data
          persistentVolumeClaim:
            claimName: zookeeper-data

kubectl apply -f deployment.yaml 

cat svc.yaml 
apiVersion: v1
kind: Service
metadata:
  name: zookeeper
  namespace: kube-system
  labels:
    app: zookeeper
spec:
  ports:
  - port: 2181
    name: zookeeper-client
  selector:
    app: zookeeper
  clusterIP: None

kubectl apply -f svc.yaml

安装 kafka

mkdir -p /data/yaml/kube-system/kafka
cd /data/yaml/kube-system/zkafka

cat nfs.yaml 
apiVersion: v1
kind: PersistentVolume
metadata:
  name: kafka-data
  namespace: kube-system
  labels:
    app: kafka-data
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteMany    
  persistentVolumeReclaimPolicy: Retain 
  storageClassName: nfs  
  nfs:
    path: /data/nfsdata/kube-system/kafka
    server: 172.16.16.150
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  namespace: kube-system
  name: kafka-data
spec:
  accessModes:
    - ReadWriteMany
  volumeMode: Filesystem
  resources:
    requests:
      storage: 10Gi
  storageClassName: nfs

kubectl apply -f nfs.yaml

cat deployment.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: kafka
  namespace: kube-system
  labels:
    app: kafka
spec:
  replicas: 1
  minReadySeconds: 100
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 0
  selector:
    matchLabels:
      app: kafka
  template:
    metadata:
      labels:
        app: kafka
    spec:
      containers:
      - name: kafka
        image: bitnami/kafka:latest
        imagePullPolicy: IfNotPresent
        env:
          - name: KAFKA_BROKER_ID
            value: '1'
          - name: KAFKA_CFG_LISTENERS
            value: 'PLAINTEXT://:9092'
          - name: KAFKA_CFG_ADVERTISED_LISTENERS
            value: 'PLAINTEXT://172.16.16.108:30092'
          - name: KAFKA_CFG_ZOOKEEPER_CONNECT
            value: 'zookeeper:2181'
          - name: ALLOW_PLAINTEXT_LISTENER
            value: 'yes'
        securityContext:
          runAsUser: 0
        volumeMounts:
          - name: kafka-data
            mountPath: /bitnami
      volumes:
        - name: kafka-data
          persistentVolumeClaim:
            claimName: kafka-data

kubectl apply -f deployment.yaml 

cat svc.yaml 
---
apiVersion: v1
kind: Service
metadata:
  name: kafka
  namespace: kube-system
  labels:
    app: kafka
spec:
  type: NodePort
  ports:
  - port: 9092
    targetPort: 9092
    nodePort: 30092
    name: kafka
  selector:
    app: kafka

kubectl apply -f svc.yaml 

测试

# 开启一个终端，进入 kafka 容器
kafka-console-producer.sh --broker-list localhost:9092 --topic topic_test
# 输入信息

# 开启另外一个终端，进入 kafka 容器
kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic topic_test --from-beginning
# 查看是否正常显示