K8S ilogtail + kafka + logstash + es + kbana 收集日志(二) 部署 ilogtail

mkdir -p /data/yaml/ilogtail
cd /data/yaml/ilogtail

# 部署 命名空间
cat ilogtail-ns.yaml 
apiVersion: v1
kind: Namespace
metadata:
  name: ilogtail

kubectl apply -f ilogtail-ns.yaml 

# 配置 ilogtail 采集，配置修改后要重启 pod
cat ilogtail-user-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: ilogtail-user-cm
  namespace: ilogtail
data:
  java_service.yaml: |
    enable: true
    inputs:
      - Type: service_docker_stdout
        Stderr: true
        Stdout: true
        K8sNamespaceRegex: "^(default)$"                   # 收集 default 命名空间下的日志
        BeginLineCheckLength: 15
        BeginLineRegex: "\\d+-\\d+-\\d+.*"                 # 匹配的开始行的日志格式
        ExcludeLabel:
          io.kubernetes.container.name: "tomcat"           # 不收集 container 名为 tomcat 的日志
    flushers:
      - Type: flusher_kafka
        Brokers:
          - 172.16.16.108:30092        # kafka 的地址，这里要写 nodeport 地址，与 kafka 生产者和消费者有关
        Topic: infrastructure

kubectl apply -f ilogtail-user-configmap.yaml

# 部署 DaemonSet
cat ilogtail-deployment.yaml 
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: ilogtail-ds
  namespace: ilogtail
  labels:
    k8s-app: logtail-ds
spec:
  selector:
    matchLabels:
      k8s-app: logtail-ds
  template:
    metadata:
      labels:
        k8s-app: logtail-ds
    spec:
      tolerations:
      - key: node-role.kubernetes.io/master
        effect: NoSchedule
      containers:
      - name: logtail
        env:
          - name: cpu_usage_limit
            value: "1"
          - name: mem_usage_limit
            value: "512"
        image: >-
          sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/ilogtail-community-edition/ilogtail:latest
        imagePullPolicy: IfNotPresent
        resources:
          limits:
            cpu: 1000m
            memory: 1Gi
          requests:
            cpu: 400m
            memory: 384Mi
        volumeMounts:
          - mountPath: /var/run
            name: run
          - mountPath: /logtail_host
            mountPropagation: HostToContainer
            name: root
            readOnly: true
          - mountPath: /usr/local/ilogtail/checkpoint
            name: checkpoint
          - mountPath: /usr/local/ilogtail/user_yaml_config.d
            name: user-config
            readOnly: true
      dnsPolicy: ClusterFirst
      hostNetwork: true
      volumes:
        - hostPath:
            path: /var/run
            type: Directory
          name: run
        - hostPath:
            path: /
            type: Directory
          name: root
        - hostPath:
            path: /etc/ilogtail-ilogtail-ds/checkpoint
            type: DirectoryOrCreate
          name: checkpoint
        - configMap:
            defaultMode: 420
            name: ilogtail-user-cm
          name: user-config

kubectl apply -f ilogtail-deployment.yaml

检查

kubectl -n ilogtail get pod