K8S 搭建 rabbitmq 3.7 集群(自动创建集群) - 20230104

mkdir -p /data/yaml/public-service/rabbitmq-cluster

cd /data/yaml/public-service/rabbitmq-cluster

创建 namespace

cat >> ns.yaml << EOF 
apiVersion: v1
kind: Namespace
metadata:
  name: public-service
EOF

kubectl apply -f ns.yaml 

创建 rbac

cat >> rbac.yaml << EOF 
apiVersion: v1
kind: ServiceAccount
metadata:
  name: rmq-cluster
  namespace: public-service
  
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: rmq-cluster
  namespace: public-service
rules:
  - apiGroups:
      - ""
    resources:
      - endpoints
    verbs:
      - get

---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: rmq-cluster
  namespace: public-service
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: rmq-cluster
subjects:
- kind: ServiceAccount
  name: rmq-cluster
  namespace: public-service
EOF

kubectl apply -f rbac.yaml 

创建 ConfigMap

cat >> cm.yaml << EOF 
apiVersion: v1
kind: ConfigMap
metadata:
  name: rmq-cluster-config
  namespace: public-service
  labels:
    addonmanager.kubernetes.io/mode: Reconcile
data:
    enabled_plugins: |
      [rabbitmq_management,rabbitmq_peer_discovery_k8s,rabbitmq_delayed_message_exchange].
    rabbitmq.conf: |
      loopback_users.guest = false

      ## Clustering
      cluster_formation.peer_discovery_backend = rabbit_peer_discovery_k8s
      cluster_formation.k8s.host = kubernetes.default.svc.cluster.local
      cluster_formation.k8s.address_type = hostname
      #################################################
      # public-service is rabbitmq-cluster's namespace#
      #################################################
      cluster_formation.k8s.hostname_suffix = .rmq-cluster.public-service.svc.cluster.local
      cluster_formation.node_cleanup.interval = 10
      cluster_formation.node_cleanup.only_log_warning = true
      cluster_partition_handling = autoheal
      ## queue master locator
      queue_master_locator=min-masters
      ## 这里设置用户名和密码
      default_user = admin
      default_pass = admin123
EOF

kubectl apply -f cm.yaml

创建 Secret

cat >> secret.yaml << EOF 
apiVersion: v1
kind: Secret
metadata:
  name: rmq-cluster-secret
  namespace: public-service
stringData:
  cookie: ERLANG_COOKIE
type: Opaque
EOF

kubectl apply -f secret.yaml 

创建 StatefulSet

cat >> sts.yaml << -'EOF' 
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: rmq-cluster
  namespace: public-service
  labels:
    app: rmq-cluster
spec:
  replicas: 3
  selector:
    matchLabels:
      app: rmq-cluster
  serviceName: rmq-cluster
  template:
    metadata:
      labels:
        app: rmq-cluster
    spec:
      serviceAccountName: rmq-cluster
      terminationGracePeriodSeconds: 30
      containers:
      - name: rabbitmq
        #image: rabbitmq:3.7-management
        image: harbor.junengcloud.com/rabbitmq/rabbitmq:3.7.17-alpine-delayed_message_exchange
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 15672
          name: http
          protocol: TCP
        - containerPort: 5672
          name: amqp
          protocol: TCP
        command:
        - sh
        args:
        - -c
        - cp -v /etc/rabbitmq/rabbitmq.conf ${RABBITMQ_CONFIG_FILE}; exec docker-entrypoint.sh
          rabbitmq-server
        env:
        - name: RABBITMQ_ERLANG_COOKIE
          valueFrom:
            secretKeyRef:
              key: cookie
              name: rmq-cluster-secret
        - name: K8S_SERVICE_NAME
          value: rmq-cluster
        - name: POD_IP
          valueFrom:
            fieldRef:
              fieldPath: status.podIP
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: RABBITMQ_USE_LONGNAME
          value: "true"
        - name: RABBITMQ_NODENAME
          value: rabbit@$(POD_NAME).rmq-cluster.$(POD_NAMESPACE).svc.cluster.local
        - name: RABBITMQ_CONFIG_FILE
          value: /var/lib/rabbitmq/rabbitmq.conf
        livenessProbe:
          exec:
            command:
            - rabbitmqctl
            - status
          initialDelaySeconds: 30
          timeoutSeconds: 10
        readinessProbe:
          exec:
            command:
            - rabbitmqctl
            - status
          initialDelaySeconds: 10
          timeoutSeconds: 10
        volumeMounts:
        - name: config-volume
          mountPath: /etc/rabbitmq
          readOnly: false
        - name: rabbitmq-storage
          mountPath: /var/lib/rabbitmq
          readOnly: false
      volumes:
      - name: config-volume
        configMap:
          items:
          - key: rabbitmq.conf
            path: rabbitmq.conf
          - key: enabled_plugins
            path: enabled_plugins
          name: rmq-cluster-config
  volumeClaimTemplates:
    - metadata:
        name: rabbitmq-storage
      spec:
        accessModes: [ "ReadWriteOnce" ]
        storageClassName: "csi-rbd-sc"
        resources:
          requests:
            storage: 2Gi
-EOF

kubectl apply -f sts.yaml 

创建

cat >> svc.yaml << EOF 
apiVersion: v1
kind: Service
metadata:
  name: rmq-cluster
  namespace: public-service
  labels:
    app: rmq-cluster
spec:
  selector:
    app: rmq-cluster
  ports:
  - name: http
    port: 15672
    protocol: TCP
    targetPort: 15672
  - name: amqp
    port: 5672
    protocol: TCP
    targetPort: 5672
  type: ClusterIP
EOF

cat >> svc-nodeport.yaml << EOF 
kind: Service
apiVersion: v1
metadata:
  name: rmq-cluster-nodeport
  namespace: public-service
spec:
  type: NodePort
  ports:
   - name: http
     protocol: TCP
     port: 15672
     targetPort: 15672 
     nodePort: 30072
   - name: amqp
     protocol: TCP
     port: 5672
     targetPort: 5672
     nodePort: 30073
  selector:
    app: rmq-cluster
EOF

kubectl apply -f svc.yaml 
kubectl apply -f svc-nodeport.yaml 

访问 rabbitmq_management 接口，用户名和密码： admin/admin123

集群需要配置镜像模式，参考 https://www.cnblogs.com/klvchen/p/14029336.html