filebeat往pulsar发送数据

1.github地址

https://github.com/streamnative/pulsar-beat-output

 

2.下载源码，本地编译

Dockerfile

# 原仓库地址的dockerfile里写的是golang:1.12.4，但这个版本好像有问题。readme里面写的是golang:1.17，故用这个版本
FROM golang:1.17

# Apache pulsar go client dependency package
# 这几步如果网络不好会执行失败，多试几次即可
RUN wget https://archive.apache.org/dist/pulsar/pulsar-2.3.0/DEB/apache-pulsar-client.deb
RUN wget https://archive.apache.org/dist/pulsar/pulsar-2.3.0/DEB/apache-pulsar-client-dev.deb
RUN dpkg -i apache-pulsar-client.deb
RUN dpkg -i apache-pulsar-client-dev.deb

# 设置一个国内的代理
ENV GO111MODULE=on \
    GOPROXY="https://goproxy.cn,direct"

WORKDIR /apps/pulsar-beat-output
COPY pulsar-beat-output /apps/pulsar-beat-output

# 这里只编译filebeat to pulsar的
RUN go build -o filebeat /apps/pulsar-beat-output/filebeat/filebeat.go
# 把编译好的二进制文件复制到test_module目录下，这里是因为看到github的readme写要./filebeat modules enable system,但我觉得可以不这么做，直接执行./filebeat -c filebeat.yml -e就行
RUN cp /apps/pulsar-beat-output/filebeat/filebeat /apps/pulsar-beat-output/test_module
RUN cp /apps/pulsar-beat-output/filebeat.yml /apps/pulsar-beat-output/test_module

# 这里其实用个其他比较小的基础镜像可能也可以（没测试过，如果用alpine基础镜像，会有动态库的问题），因为已经编译成可执行的二进制文件了，加上一个赋予可执行权限的命令即可（有go环境可以不赋这个权限），RUN chmod +x filebeat
FROM golang:bullseye
COPY --from=0 /apps/pulsar-beat-output /apps/pulsar-beat-output
WORKDIR /apps/pulsar-beat-output/test_module
CMD /bin/bash

 

3.运行

在/apps/pulsar-beat-output/test_module目录下执行

./filebeat -c filebeat.yml -e

 

4.k8s部署

apiVersion: v1
data:
  filebeat.yml: |-
    filebeat.inputs: # 具体配置规则看https://www.elastic.co/guide/en/beats/filebeat/current/index.html
    - type: filestream
      paths:
        - /var/log/*.log
      processors:
        - include_fields:
            fields: ["message"]
  
    #output.console: #只是配置控制台输出的，可用来测试
    #  pretty: true

    output.pulsar:
      url: "pulsar://pulsar-broker.paas:6650" #pulsar地址
      topic: persistent://filebeat/log
      #name: log
kind: ConfigMap
metadata:
  name: filebeat-config
  namespace: paas
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: filebeat-ds
  namespace: paas
spec:
  selector:
    matchLabels:
      k8s-app: filebeat
  template:
    metadata:
      labels:
        k8s-app: filebeat
    spec:
      containers:
      - args:
        - ./filebeat
        - -c
        - /etc/filebeat.yml
        - -e
        env:
        - name: NODE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
        image: filebeat-pulsar:0.4.1
        imagePullPolicy: Always
        name: filebeat-container
        resources:
          limits:
            memory: 500Mi
          requests:
            cpu: 100m
            memory: 100Mi
        securityContext:
          runAsUser: 0
        volumeMounts:
        - mountPath: /etc/filebeat.yml
          name: filebeat-config
          readOnly: true
          subPath: filebeat.yml
        - mountPath: /usr/share/filebeat/data
          name: data
        - mountPath: /var/log #把需要采集的目录挂载到容器里面
          name: varlog
          readOnly: true
      dnsPolicy: ClusterFirstWithHostNet
      hostNetwork: true
      terminationGracePeriodSeconds: 30
      volumes:
      - configMap:
          defaultMode: 416
          name: filebeat-config
        name: filebeat-config
      - hostPath:
          path: /var/log
        name: varlog
      - hostPath:
          path: /var/lib/filebeat-data
          type: DirectoryOrCreate
        name: data