k8s部署redis集群模式

创建namespace

kubectl create namespace redis

创建配置文件

vim redis-cm.yaml

apiVersion: v1

kind: ConfigMap

metadata:  

  name: redis-cluster

  namespace: redis

data:

  fix-ip.sh: |

    #!/bin/sh  

    CLUSTER_CONFIG="/data/nodes.conf" 

    if [ -f ${CLUSTER_CONFIG} ]; then  

      if [ -z "${POD_IP}" ]; then

        echo "Unable to determine Pod IP address!"  

        exit 1 

      fi

      echo "Updating my IP to ${POD_IP} in ${CLUSTER_CONFIG}"  

      sed -i.bak -e '/myself/ s/[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}/'${POD_IP}'/' ${CLUSTER_CONFIG} 

    fi

    exec "$@"   

  redis.conf: |

    bind 0.0.0.0

    protected-mode yes

    port 6379  

    tcp-backlog 2048  

    timeout 0  

    tcp-keepalive 300 

    daemonize no

    supervised no

    pidfile /var/run/redis.pid

    loglevel notice

    logfile /data/redis.log

    databases 16

    always-show-logo yes

    stop-writes-on-bgsave-error yes 

    rdbcompression yes

    rdbchecksum yes

    dbfilename dump.rdb

    dir /data

    masterauth zwzk@2024

    replica-serve-stale-data yes 

    replica-read-only no

    repl-diskless-sync no

    repl-diskless-sync-delay 5   

    repl-disable-tcp-nodelay no

    replica-priority 100  

    requirepass zwzk@2024

    maxclients 32768  

    maxmemory-policy allkeys-lru 

    lazyfree-lazy-eviction no

    lazyfree-lazy-expire no

    lazyfree-lazy-server-del no

    replica-lazy-flush no

    appendonly yes

    appendfilename "appendonly.aof"   

    appendfsync everysec

    no-appendfsync-on-rewrite no 

    auto-aof-rewrite-percentage 100 

    auto-aof-rewrite-min-size 64mb 

    aof-load-truncated yes

    aof-use-rdb-preamble yes

    lua-time-limit 5000

    cluster-enabled yes

    cluster-config-file /data/nodes.conf 

    cluster-node-timeout 15000 

    slowlog-log-slower-than 10000  

    slowlog-max-len 128 

    latency-monitor-threshold 0  

    notify-keyspace-events ""  

    hash-max-ziplist-entries 512  

    hash-max-ziplist-value 64

    list-max-ziplist-size -2

    list-compress-depth 0

    set-max-intset-entries 512 

    zset-max-ziplist-entries 128 

    zset-max-ziplist-value 64  

    hll-sparse-max-bytes 3000  

    stream-node-max-bytes 4096  

    stream-node-max-entries 100

    activerehashing yes

    client-output-buffer-limit normal 0 0 0  

    client-output-buffer-limit replica 256mb 64mb 60   

    client-output-buffer-limit pubsub 32mb 8mb 60  

    hz 10   

    dynamic-hz yes

    aof-rewrite-incremental-fsync yes 

    rdb-save-incremental-fsync yes

运行cm

kubectl apply -f redis-cm.yaml

创建redis pod

vim redis-statefulset.yaml

apiVersion: apps/v1

kind: StatefulSet

metadata: 

  namespace: redis

  name: redis-cluster

spec:  

  serviceName: redis-cluster

  replicas: 6 

  selector:  

    matchLabels: 

      app: redis-cluster

  template: 

    metadata:   

      labels: 

        app: redis-cluster

    spec:                                                                                                                              

      containers: 

      - name: redis

        image: redis:6.2

        ports: 

        - containerPort: 6379 

          name: client

        - containerPort: 16379 

          name: gossip

        command: ["/etc/redis/fix-ip.sh", "redis-server", "/etc/redis/redis.conf"]  

        env:  

        - name: POD_IP

          valueFrom: 

            fieldRef: 

              fieldPath: status.podIP 

        volumeMounts:  

        - name: conf

          mountPath: /etc/redis/  

          readOnly: false  

        - name: data

          mountPath: /data

          readOnly: false  

      volumes:  

      - name: conf

        configMap: 

          name: redis-cluster 

          defaultMode: 0755

  volumeClaimTemplates:  

  - metadata:

      name: data

      #annotations: 

      #volume.beta.kubernetes.io/storage-class: "redis-nfs-storage"    

    spec:

      storageClassName: "managed-nfs-storage"    

      accessModes:

        - ReadWriteMany  

      resources: 

        requests: 

          storage: 2Gi

运行redis pod

kubectl apply -f redis-statefulset.yaml

创建redis的svc

vim redis-svc.yaml

apiVersion: v1

kind: Service

metadata: 

  namespace: redis

  name: redis-cluster

spec:  

  type: NodePort

  ports: 

  - port: 6379  

    targetPort: 6379  

    nodePort: 32379   

    name: client

  - port: 16379 

    targetPort: 16379  

    nodePort: 32380 

    name: gossip

  selector: 

    app: redis-cluster

运行svc

kubectl apply -f redis-svc.yaml

查看pv和pvc

kubectl get pvc -A

 

kubectl get pv -A

初始化redis-cluster集群（三主三从）

进入redis-0 的pod

kubectl exec -it -n redis redis-cluster-0 /bin/bash

 

执行初始化命令

redis-cli -a zwzk@2024 --cluster create 10.202.1.12:6379 10.202.2.10:6379 10.202.0.3:6379 10.202.2.11:6379 10.202.1.13:6379 10.202.0.4:6379 --cluster-replicas 1

 

出现下图提示则为成功初始化集群

 

注：集群某个pod挂掉后重启，还会占用原来的pv和pvc。pod ip会变，我们配置里写了脚本，他会自动把改变后的ip写入到nodes.conf中，redis 集群修复时会将该 ip 同步到其它 pod节点的/data/nodes.conf文件中，从而保证整个 redis 集群的可用性。

 

注：k8s部署的redis集群无法使用公网ip去连接，会出现重定向内网ip的问题，所以在本项目中无法在本地调用测试环境的redis集群，只能把程序跑在服务器里后去连接redis代码里连接redis的命名规范为：redis-cluster-0.redis-cluster.redis-cluster.svc.cluster.local:6379

 

Pod名称.svc名称.命名空间.k8s域名后缀