Kubernetes 实战-基于StatefulSet控制器运行Redis Cluster（二）

配置存储环境

先安装NFS服务
[root@xianchaomaster1 redis-cluster]# mkdir -pv /data/k8sdata/xks/{redis0.redis1,redis2,redis3,redis4,redis5}
mkdir: created directory ‘/data/k8sdata/xks/redis0.redis1’
mkdir: created directory ‘/data/k8sdata/xks/redis2’
mkdir: created directory ‘/data/k8sdata/xks/redis3’
mkdir: created directory ‘/data/k8sdata/xks/redis4’
mkdir: created directory ‘/data/k8sdata/xks/redis5’
[root@xianchaomaster1 redis-cluster]# cat /etc/exports
/data/k8sdata *(rw,no_root_squash)
[root@xianchaomaster1 redis-cluster]# exportfs -arv
exporting *:/data/k8sdata

编辑PV文件

vim redis-cluster-pv.yaml

apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-cluster-pv0
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.40.180
    path: /data/k8sdata/xks/redis0 

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-cluster-pv1
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.40.180
    path: /data/k8sdata/xks/redis1 

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-cluster-pv2
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.40.180
    path: /data/k8sdata/xks/redis2 

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-cluster-pv3
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.40.180
    path: /data/k8sdata/xks/redis3 

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-cluster-pv4
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.40.180
    path: /data/k8sdata/xks/redis4 

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-cluster-pv5
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.40.180
    path: /data/k8sdata/xks/redis5
    
[root@xianchaomaster1 pv]# kubectl apply -f redis-cluster-pv.yaml
persistentvolume/redis-cluster-pv0 created
persistentvolume/redis-cluster-pv1 created
persistentvolume/redis-cluster-pv2 created
persistentvolume/redis-cluster-pv3 created
persistentvolume/redis-cluster-pv4 created
persistentvolume/redis-cluster-pv5 created
[root@xianchaomaster1 pv]# kubectl get pv
NAME                 CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS      CLAIM                                                                                       STORAGECLASS   REASON   AGE
redis-cluster-pv0    5Gi        RWO            Retain           Available                                                                                                                       4s
redis-cluster-pv1    5Gi        RWO            Retain           Available                                                                                                                       4s
redis-cluster-pv2    5Gi        RWO            Retain           Available                                                                                                                       4s
redis-cluster-pv3    5Gi        RWO            Retain           Available                                                                                                                       4s
redis-cluster-pv4    5Gi        RWO            Retain           Available                                                                                                                       4s
redis-cluster-pv5    5Gi        RWO            Retain           Available                                                                                                                       4s
redis-datadir-pv-1   10Gi       RWO            Retain           Bound       default/redis-datadir-pvc-                                                              1                           23h
[root@xianchaomaster1 pv]# kubectl get pv
NAME                 CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS      CLAIM                         STORAGECLASS   REASON   AGE
redis-cluster-pv0    5Gi        RWO            Retain           Available                                                         13s
redis-cluster-pv1    5Gi        RWO            Retain           Available                                                         13s
redis-cluster-pv2    5Gi        RWO            Retain           Available                                                         13s
redis-cluster-pv3    5Gi        RWO            Retain           Available                                                         13s
redis-cluster-pv4    5Gi        RWO            Retain           Available                                                         13s
redis-cluster-pv5    5Gi        RWO            Retain           Available                                                         13s
redis-datadir-pv-1   10Gi       RWO            Retain           Bound       default/redis-datadir-pvc-1                           23h

配置redis.conf

vim redis.conf

appendonly yes
cluster-enabled yes
cluster-config-file /var/lib/redis/nodes.conf
cluster-node-timeout 5000
dir /var/lib/redis
port 6379

[root@xianchaomaster1 redis-cluster]# kubectl create configmap redis-conf --from-file=redis.conf
configmap/redis-conf created
You have new mail in /var/spool/mail/root
[root@xianchaomaster1 redis-cluster]# kubectl get configmaps
NAME                 DATA   AGE
istio-ca-root-cert   1      35h
kube-root-ca.crt     1      2d21h
redis-conf           1      15s
[root@xianchaomaster1 redis-cluster]# kubectl describe configmaps redis-conf
Name:         redis-conf
Namespace:    default
Labels:       <none>
Annotations:  <none>

Data
====
redis.conf:
----
appendonly yes
cluster-enabled yes
cluster-config-file /var/lib/redis/nodes.conf
cluster-node-timeout 5000
dir /var/lib/redis
port 6379

Events:  <none>

创建redis-cluster集群

root@xianchaomaster1 redis-cluster]# cat redis.yaml
apiVersion: v1
kind: Service
metadata:
  name: redis
  labels:
    app: redis
spec:
  selector:
    app: redis
    appCluster: redis-cluster
  ports:
  - name: redis
    port: 6379
  clusterIP: None

---
apiVersion: v1
kind: Service
metadata:
  name: redis-access
  labels:
    app: redis
spec:
  selector:
    app: redis
    appCluster: redis-cluster
  ports:
  - name: redis-access
    protocol: TCP
    port: 6379
    targetPort: 6379

---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis
spec:
  serviceName: redis
  replicas: 6
  selector:
    matchLabels:
      app: redis
      appCluster: redis-cluster
  template:
    metadata:
      labels:
        app: redis
        appCluster: redis-cluster
    spec:
      terminationGracePeriodSeconds: 20
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - redis
              topologyKey: kubernetes.io/hostname
      containers:
      - name: redis
        image: redis:4.0.14
        command:
          - "redis-server"
        args:
          - "/etc/redis/redis.conf"
          - "--protected-mode"
          - "no"
        resources:
          requests:
            cpu: "500m"
            memory: "500Mi"
        ports:
        - containerPort: 6379
          name: redis
          protocol: TCP
        - containerPort: 16379
          name: cluster
          protocol: TCP
        volumeMounts:
        - name: conf
          mountPath: /etc/redis
        - name: data
          mountPath: /var/lib/redis
      volumes:
      - name: conf
        configMap:
          name: redis-conf
          items:
          - key: redis.conf
            path: redis.conf
  volumeClaimTemplates:
  - metadata:
      name: data
    spec:
      accessModes: [ "ReadWriteOnce" ]
      resources:
        requests:
          storage: 5Gi
          
[root@xianchaomaster1 redis-cluster]# kubectl apply -f redis.yaml
service/redis created
service/redis-access created
statefulset.apps/redis created

#redis-0 ~ redis-5
[root@xianchaomaster1 redis-cluster]# kubectl get pods
NAME                                   READY   STATUS    RESTARTS   AGE
deploy-devops-redis-7d6c886cc6-8rwvk   2/2     Running   0          23h
details-v1-65bbfd4f58-c6gmz            2/2     Running   0          35h
nginx-hpa-fb74696c-ksln6               1/1     Running   0          2d1h
php-apache-576f5cdcf6-68qnf            1/1     Running   0          2d2h
productpage-v1-6b746f74dc-7drn6        2/2     Running   0          35h
ratings-v1-b45758b-268w7               2/2     Running   0          35h
redis-0                                2/2     Running   0          42s
redis-1                                2/2     Running   0          38s
redis-2                                2/2     Running   0          34s
redis-3                                2/2     Running   0          30s
redis-4                                2/2     Running   0          27s
redis-5                                2/2     Running   0          23s
reviews-v1-74894b48c8-kqwqd            2/2     Running   0          35h
reviews-v2-f649764d-b68bb              2/2     Running   0          35h
reviews-v3-6c675c6774-x5g67            2/2     Running   0          35h
sleep-557747455f-x4tj8                 2/2     Running   0          29h
v1                                     1/1     Running   1          2d2h

#验证PVC状态
[root@xianchaomaster1 redis-cluster]# kubectl get pvc
NAME                  STATUS   VOLUME               CAPACITY   ACCESS MODES   STORAGECLASS   AGE
data-redis-0          Bound    redis-cluster-pv4    5Gi        RWO                           7m32s
data-redis-1          Bound    redis-cluster-pv0    5Gi        RWO                           7m9s
data-redis-2          Bound    redis-cluster-pv3    5Gi        RWO                           2m34s
data-redis-3          Bound    redis-cluster-pv5    5Gi        RWO                           2m29s
data-redis-4          Bound    redis-cluster-pv1    5Gi        RWO                           2m26s
data-redis-5          Bound    redis-cluster-pv2    5Gi        RWO                           2m22s
redis-datadir-pvc-1   Bound    redis-datadir-pv-1   10Gi       RWO                           23h

redis.yaml

#kubectl get pod -o wide 
redis-0                                2/2     Running   0          29m    10.244.121.27   xianchaonode1   <none>           <none>
redis-1                                2/2     Running   0          28m    10.244.121.28   xianchaonode1   <none>           <none>
redis-2                                2/2     Running   0          28m    10.244.102.84   xianchaonode2   <none>           <none>
redis-3                                2/2     Running   0          28m    10.244.102.85   xianchaonode2   <none>           <none>
redis-4                                2/2     Running   0          28m    10.244.121.29   xianchaonode1   <none>           <none>
redis-5                                2/2     Running   0          28m    10.244.102.86   xianchaonode2   <none>           <none>

 
 apt install python2.7 python3-pip redis-tools dnsutils iputils-ping net-tools
 
 pip install --upgrade pip
 
 pip install redis-trib==0.5.1
 

root@ubuntulastest:/# nslookup redis-0.redis.default.svc.cluster.local
Server:         10.96.0.10
Address:        10.96.0.10#53

Name:   redis-0.redis.default.svc.cluster.local
Address: 10.244.121.27

 #创建集群
 redis-trib.py create \
 `dig +short redis-0.default.svc.cluster.local`:6379 \
 `dig +short redis-1.default.svc.cluster.local`:6379 \
 `dig +short redis-2.default.svc.cluster.local`:6379 
或者
 redis-trib.py create \
 10.244.121.27:6379 \
 10.244.121.28:6379 \
 10.244.102.84:6379 
 
 #将redis-3加入redis-0
 redis-trib.py replicate \
 --master-addr `dig +short redis-0.redis.default.svc.cluster.local`:6379 \
 --slave-addr `dig +short redis-3.redis.default.svc.cluster.local`:6379 
 
  #将redis-4加入redis-1
 redis-trib.py replicate \
 --master-addr `dig +short redis-1.redis.default.svc.cluster.local`:6379 \
 --slave-addr `dig +short redis-4.redis.default.svc.cluster.local`:6379
 
 
  #将redis-5加入redis-2
 redis-trib.py replicate \
 --master-addr `dig +short redis-2.redis.default.svc.cluster.local`:6379 \
 --slave-addr `dig +short redis-5.redis.default.svc.cluster.local`:6379
 
 
 验证redis cluster状态
 kubectl exec -it redis-0 bash