k8s下部署redis集群模式
环境:
OS:Centos 7
k8s:1.28
redis:5.0.4
1.使用nfs做持久化存储(使用k8s的一个节点做为nfs服务器)
[root@master redis]# exportfs -v
/k8s <world>(sync,wdelay,hide,no_subtree_check,sec=sys,rw,secure,no_root_squash,no_all_squash)
[root@master k8s]# showmount -e 192.168.1.102
Export list for 192.168.1.102:
/k8s *
2.为需要持久化的服务创建子目录(必须创建)
mkdir -p /k8s/redis/pv0
mkdir -p /k8s/redis/pv1
mkdir -p /k8s/redis/pv2
mkdir -p /k8s/redis/pv3
mkdir -p /k8s/redis/pv4
mkdir -p /k8s/redis/pv5
3.创建名称空间
kubectl create namespace ns-redis
4.创建PV
[root@master redis_cluster]# more 1-redis-pv.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: redis-cluster-pv-0
namespace: ns-redis
spec:
capacity:
storage: 1Gi
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Recycle
storageClassName: redis-cluster
nfs:
path: /k8s/redis/pv0
server: 192.168.1.102
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: redis-cluster-pv-1
namespace: ns-redis
spec:
capacity:
storage: 1Gi
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Recycle
storageClassName: redis-cluster
nfs:
path: /k8s/redis/pv1
server: 192.168.1.102
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: redis-cluster-pv-2
namespace: ns-redis
spec:
capacity:
storage: 1Gi
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Recycle
storageClassName: redis-cluster
nfs:
path: /k8s/redis/pv2
server: 192.168.1.102
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: redis-cluster-pv-3
namespace: ns-redis
spec:
capacity:
storage: 1Gi
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Recycle
storageClassName: redis-cluster
nfs:
path: /k8s/redis/pv3
server: 192.168.1.102
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: redis-cluster-pv-4
namespace: ns-redis
spec:
capacity:
storage: 1Gi
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Recycle
storageClassName: redis-cluster
nfs:
path: /k8s/redis/pv4
server: 192.168.1.102
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: redis-cluster-pv-5
namespace: ns-redis
spec:
capacity:
storage: 1Gi
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Recycle
storageClassName: redis-cluster
nfs:
path: /k8s/redis/pv5
server: 192.168.1.102
应用
kubectl apply -f 1-redis-pv.yaml
5.创建 ConfigMap
vi 2-configmap.yaml
[root@master redis_cluster]# more 2-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: redis-cluster
namespace: ns-redis
data:
update-node.sh: |
#!/bin/sh
REDIS_NODES="/data/nodes.conf"
sed -i -e "/myself/ s/[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}/${POD_IP}/" ${REDIS_NODES}
exec "$@"
redis.conf: |+
cluster-enabled yes
cluster-require-full-coverage no
cluster-node-timeout 15000
cluster-config-file /data/nodes
cluster-migration-barrier 1
Maxmemory 3GB
port 6379
appendonly yes
protected-mode no
masterauth "hxl123"
requirepass "hxl123"
kubectl apply -f 2-configmap.yaml
6.创建StatefulSet
[root@master redis_cluster]# more 3-sts.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: redis-cluster
namespace: ns-redis
spec:
serviceName: redis-cluster
replicas: 6
selector: # 标签选择器
matchLabels:
app: redis-cluster
template: # pod创建模板
metadata:
labels:
app: redis-cluster
spec:
containers:
- name: redis
image: registry.cn-shenzhen.aliyuncs.com/hxlk8s/redis:5.0.14
ports:
- containerPort: 6379
name: client
- containerPort: 16379
name: gossip
command: ["/conf/update-node.sh", "redis-server", "/conf/redis.conf"]
env:
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
volumeMounts:
- name: conf
mountPath: /conf
readOnly: false
- name: data
mountPath: /data
readOnly: false
volumes:
- name: conf
configMap:
name: redis-cluster
defaultMode: 0755
volumeClaimTemplates:
- metadata:
name: data
namespace: ns-redis
spec:
accessModes: [ "ReadWriteOnce" ]
storageClassName: "redis-cluster"
resources:
requests:
storage: 1Gi
kubectl apply -f 3-sts.yaml
这个时候会启动6个redis pod
[root@master redis_cluster]# kubectl get pods -n ns-redis
NAME READY STATUS RESTARTS AGE
redis-cluster-0 1/1 Running 0 19s
redis-cluster-1 1/1 Running 0 17s
redis-cluster-2 1/1 Running 0 15s
redis-cluster-3 1/1 Running 0 11s
redis-cluster-4 1/1 Running 0 9s
redis-cluster-5 1/1 Running 0 5s
同时会自动创建6个pvc
[root@master redis_cluster]# kubectl get pvc -n ns-redis -owide
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE VOLUMEMODE
data-redis-cluster-0 Bound redis-cluster-pv-1 1Gi RWO redis-cluster 77s Filesystem
data-redis-cluster-1 Bound redis-cluster-pv-2 1Gi RWO redis-cluster 75s Filesystem
data-redis-cluster-2 Bound redis-cluster-pv-3 1Gi RWO redis-cluster 73s Filesystem
data-redis-cluster-3 Bound redis-cluster-pv-0 1Gi RWO redis-cluster 69s Filesystem
data-redis-cluster-4 Bound redis-cluster-pv-4 1Gi RWO redis-cluster 67s Filesystem
data-redis-cluster-5 Bound redis-cluster-pv-5 1Gi RWO redis-cluster 63s Filesystem
7.创建headless service
[root@master redis_cluster]# more 4-headless-service.yml
apiVersion: v1
kind: Service
metadata:
name: redis-cluster
namespace: ns-redis
spec:
clusterIP: None
ports:
- port: 6379
name: client
- port: 16379
name: cluster
selector:
app: redis-cluster
kubectl apply -f 4-headless-service.yml
[root@master redis_cluster]# kubectl get svc -n ns-redis
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
redis-cluster ClusterIP None <none> 6379/TCP,16379/TCP 6s
8.redis集群初始化
kubectl -n ns-redis exec -it redis-cluster-0 -- redis-cli --cluster create $(kubectl get pods -n ns-redis -l app=redis-cluster -o jsonpath='{range.items[*]}{.status.podIP}:6379 ' | sed 's/ :6379//g') --cluster-replicas 1 -a hxl123
[root@master redis_cluster]# kubectl -n ns-redis exec -it redis-cluster-0 -- redis-cli --cluster create $(kubectl get pods -n ns-redis -l app=redis-cluster -o jsonpath='{range.items[*]}{.status.podIP}:6379 ' | sed 's/ :6379//g') --cluster-replicas 1 -a hxl123
>>> Performing hash slots allocation on 6 nodes...
Master[0] -> Slots 0 - 5460
Master[1] -> Slots 5461 - 10922
Master[2] -> Slots 10923 - 16383
Adding replica 10.244.104.11:6379 to 10.244.104.5:6379
Adding replica 10.244.166.144:6379 to 10.244.166.143:6379
Adding replica 10.244.166.139:6379 to 10.244.104.19:6379
M: bae4fb50e955a0c23a40dd9d2099225d637b1cd4 10.244.104.5:6379
slots:[0-5460] (5461 slots) master
M: 7dc7d278e6117945eed010ee734f3da5ab36718d 10.244.166.143:6379
slots:[5461-10922] (5462 slots) master
M: 7172479fce9ef21da5d9aa0433a29f5231672ac4 10.244.104.19:6379
slots:[10923-16383] (5461 slots) master
S: 47215721e27629c0114ca835b31795b164f8dae6 10.244.166.139:6379
replicates 7172479fce9ef21da5d9aa0433a29f5231672ac4
S: e574628d1413ea583a3e1473af0873d55cbc2603 10.244.104.11:6379
replicates bae4fb50e955a0c23a40dd9d2099225d637b1cd4
S: c0eab0272562989e4534eaaa640c153aab586f23 10.244.166.144:6379
replicates 7dc7d278e6117945eed010ee734f3da5ab36718d
Can I set the above configuration? (type 'yes' to accept): yes
>>> Nodes configuration updated
>>> Assign a different config epoch to each node
>>> Sending CLUSTER MEET messages to join the cluster
Waiting for the cluster to join
.
>>> Performing Cluster Check (using node 10.244.104.5:6379)
M: bae4fb50e955a0c23a40dd9d2099225d637b1cd4 10.244.104.5:6379
slots:[0-5460] (5461 slots) master
1 additional replica(s)
M: 7172479fce9ef21da5d9aa0433a29f5231672ac4 10.244.104.19:6379
slots:[10923-16383] (5461 slots) master
1 additional replica(s)
S: 47215721e27629c0114ca835b31795b164f8dae6 10.244.166.139:6379
slots: (0 slots) slave
replicates 7172479fce9ef21da5d9aa0433a29f5231672ac4
S: c0eab0272562989e4534eaaa640c153aab586f23 10.244.166.144:6379
slots: (0 slots) slave
replicates 7dc7d278e6117945eed010ee734f3da5ab36718d
S: e574628d1413ea583a3e1473af0873d55cbc2603 10.244.104.11:6379
slots: (0 slots) slave
replicates bae4fb50e955a0c23a40dd9d2099225d637b1cd4
M: 7dc7d278e6117945eed010ee734f3da5ab36718d 10.244.166.143:6379
slots:[5461-10922] (5462 slots) master
1 additional replica(s)
[OK] All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
[OK] All 16384 slots covered.
9.验证集群
[root@master redis_cluster]# kubectl -n ns-redis exec -it redis-cluster-0 -- redis-cli -a hxl123 cluster info
cluster_state:ok
cluster_slots_assigned:16384
cluster_slots_ok:16384
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:6
cluster_size:3
cluster_current_epoch:6
cluster_my_epoch:1
cluster_stats_messages_ping_sent:124
cluster_stats_messages_pong_sent:133
cluster_stats_messages_sent:257
cluster_stats_messages_ping_received:128
cluster_stats_messages_pong_received:124
cluster_stats_messages_meet_received:5
cluster_stats_messages_received:257
[root@master redis_cluster]# kubectl -n ns-redis exec -it redis-cluster-0 -- redis-cli cluster nodes
7172479fce9ef21da5d9aa0433a29f5231672ac4 10.244.104.19:6379@16379 master - 0 1741680774061 3 connected 10923-16383
bae4fb50e955a0c23a40dd9d2099225d637b1cd4 10.244.104.5:6379@16379 myself,master - 0 1741680771000 1 connected 0-5460
47215721e27629c0114ca835b31795b164f8dae6 10.244.166.139:6379@16379 slave 7172479fce9ef21da5d9aa0433a29f5231672ac4 0 1741680772053 4 connected
c0eab0272562989e4534eaaa640c153aab586f23 10.244.166.144:6379@16379 slave 7dc7d278e6117945eed010ee734f3da5ab36718d 0 1741680773057 6 connected
e574628d1413ea583a3e1473af0873d55cbc2603 10.244.104.11:6379@16379 slave bae4fb50e955a0c23a40dd9d2099225d637b1cd4 0 1741680772000 5 connected
7dc7d278e6117945eed010ee734f3da5ab36718d 10.244.166.143:6379@16379 master - 0 1741680771000 2 connected 5461-10922
遗留问题:
外部如何访问集群
浙公网安备 33010602011771号