Filbeat采集nginx-ingress日志

 

　　一、创建configmap配置文件

　　注：filebeat6以上版本需要将prospectors改为inputs，paths下指定的nginx-ingress日志路径匹配模式以及hosts指定的kafka地址需要根据实际修改，document_type和topic需要是kafka中存在的，不要在没有nginx-ingress容器的node上部署filebeat会导致无法读取ingress日志文件

[root@dsbx-mysql02 ~]# cat >> filebeat-config.yaml << EOF
apiVersion: v1
data:
  filebeat.yml: |
    filebeat.prospectors:
    - input_type: log
      paths:
        - "/var/log/containers/nginx-ingress-ingress-nginx-controller-*_ingress-nginx_controller-*.log"
      symlinks: true
      json.message_key: log
      json.keys_under_root: true
      json.add_error_key: true
      multiline.pattern: '^\s'
      multiline.match: after
      document_type: k8s-logs
      fields:
        POD_NAME: 'nginx-ingress'
      fields_under_root: true
      exclude_lines: ['\.(xml|gif|jpg|jpeg|png|bmp|swf|woff|woff2|ttf|js|css|svg|ico)']


    output.kafka:
      hosts: ["kafka-service:9092"]
      topic: "k8s-logs"
      required_acks: 1
kind: ConfigMap
metadata:
  name: filebeat-configmap
  namespace: middleware
EOF

 

　　二、创建configmap

[root@develop-k8s-worker02 ~]# kubectl apply -f filebeat-config.yaml

 

　　三、创建filebeat的daemonset配置文件

[root@localhost ~]# cat >> filebeat.yaml <<EOF
apiVersion: apps/v1
kind: DaemonSet
metadata:
  labels:
    app: filebeat
  name: filebeat
  namespace: middleware
spec:
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: filebeat
  template:
    metadata:
      labels:
        app: filebeat
    spec:
      containers:
      - image: elastic/filebeat:5.6.14
        imagePullPolicy: Always
        name: filebeat
        resources:
          limits:
            cpu: 200m
            memory: 200Mi
          requests:
            cpu: 100m
            memory: 100Mi
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /etc/filebeat/
          name: config
          readOnly: true
        - mountPath: /var/log/containers
          name: varlog
        - mountPath: /var/log/pods
          name: varlogpods
          readOnly: true
        - mountPath: /var/lib/docker/containers
          name: varlibdockercontainers
        - mountPath: /var/host/log
          name: volume-1621230243442
      dnsPolicy: ClusterFirst
      imagePullSecrets:
      - name: test
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      terminationGracePeriodSeconds: 30
      tolerations:
      - effect: NoSchedule
        key: node-role.kubernetes.io/master
        operator: Exists
      volumes:
      - configMap:
          defaultMode: 420
          items:
          - key: filebeat.yml
            path: filebeat.yml
          name: filebeat-configmap
        name: config
      - hostPath:
          path: /var/log/containers
          type: ""
        name: varlog
      - hostPath:
          path: /var/log/pods
          type: ""
        name: varlogpods
      - hostPath:
          path: /var/lib/docker/containers
          type: ""
        name: varlibdockercontainers
      - hostPath:
          path: /var/log
          type: ""
        name: volume-1621230243442
  updateStrategy:
    rollingUpdate:
      maxUnavailable: 1
    type: RollingUpdate
EOF

 

　　四、创建filebeat的pod

[root@localhost ~]# kubectl apply -f filebeat.yaml