10--基于kubernetes的redis-cluster部署
前言：这一章作为操作手册记录，比较偏向于实战内容，内部源与镜像均使用国内地址代替，保证安装的可行性。
1、基础环境
| 角色 | IP |
|---|---|
| k8s-master1 | 192.168.188.101 |
| k8s-node1 | 192.168.188.102 |
| k8s-node2 | 192.168.188.103 |
| nfs服务器 | 192.168.188.200 |
需要部署集群为redis-cluster集群，规模为三主三从，使用到的镜像下文中有详细说明
2、部署实操
2.1、准备nfs存储空间
[root@localhost ~]# mkdir /data
[root@localhost ~]# umount /root/nfs
[root@localhost ~]# mount /dev/sdb /data
[root@localhost ~]# df -Th
文件系统 类型 容量 已用 可用 已用% 挂载点
devtmpfs devtmpfs 475M 0 475M 0% /dev
tmpfs tmpfs 487M 0 487M 0% /dev/shm
tmpfs tmpfs 487M 7.7M 479M 2% /run
tmpfs tmpfs 487M 0 487M 0% /sys/fs/cgroup
/dev/mapper/centos-root xfs 17G 4.8G 13G 28% /
/dev/sda1 xfs 1014M 190M 825M 19% /boot
tmpfs tmpfs 98M 0 98M 0% /run/user/0
/dev/sdb ext4 99G 259M 94G 1% /data
[root@localhost ~]# chmod -R 777 /data
[root@localhost ~]# vim /etc/exports
[root@localhost ~]# cat /etc/exports
/data *(rw,no_root_squash,no_all_squash,sync)
[root@localhost ~]# systemctl restart rpcbind nfs
使用node节点测试nfs挂载情况
[root@k8s-node1 ~]# mount -t nfs 192.168.188.200:/data /test
[root@k8s-node1 ~]# touch /test/1.txt
[root@k8s-node1 ~]# ls /test/
1.txt回到nfs节点查看文件存在
[root@localhost ~]# ls /data/
1.txtnfs配置成功，回到node节点卸载nfs
[root@k8s-node1 ~]# umount /test
[root@k8s-node1 ~]# df -Th | grep "/test"
[root@k8s-node1 ~]#
2.2、配置动态存储
安装配置helm工具（k8s包管理工具）
官网文档：Helm | Docs
百度网盘链接：
链接: https://pan.baidu.com/s/16IxIYH2uWcGPCmEtxVs2Vw?pwd=erwg 提取码: erwg
[root@k8s-master1 ~]# wget https://get.helm.sh/helm-v3.10.1-linux-amd64.tar.gz
[root@k8s-master1 ~]# tar -xvzf helm-v3.10.1-linux-amd64.tar.gz
linux-amd64/
linux-amd64/helm
linux-amd64/LICENSE
linux-amd64/README.md
[root@k8s-master1 ~]# cp linux-amd64/helm /usr/bin/
[root@k8s-master1 ~]# helm version
version.BuildInfo{Version:"v3.10.1", GitCommit:"9f88ccb6aee40b9a0535fcc7efea6055e1ef72c9", GitTreeState:"clean", GoVersion:"go1.18.7"}
# 为helm添加一个名为stable的源
[root@k8s-master1 ~]# helm repo add stable http://mirror.azure.cn/kubernetes/charts
"stable" has been added to your repositories
[root@k8s-master1 ~]# helm repo list
NAME URL
stable http://mirror.azure.cn/kubernetes/chart使用helm制作动态存储storageclass与nfs的共享目录相关联
[root@k8s-master1 ~]# helm install nfs-redis stable/nfs-client-provisioner --set nfs.server=192.168.188.200 --set nfs.path=/data
WARNING: This chart is deprecated
NAME: nfs-redis
LAST DEPLOYED: Wed Sep 4 17:16:32 2024
NAMESPACE: default
STATUS: deployed
REVISION: 1
TEST SUITE: None
[root@k8s-master1 ~]# kubectl get pod
NAME READY STATUS RESTARTS AGE
nfs-redis-nfs-client-provisioner-56f8995966-zc6b9 1/1 Running 0 75s
[root@k8s-master1 ~]# helm list
NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION
nfs-redis default 1 2024-09-04 17:16:32.561824151 +0800 CST deployed nfs-client-provisioner-1.2.11 3.1.0
[root@k8s-master1 ~]# kubectl get sc
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
nfs-client cluster.local/nfs-redis-nfs-client-provisioner Delete Immediate true 2m25s
# 这里自动进行提权2.3、redis配置文件（ConfigMap）
这里使用指定文件创建（可以通过yaml方式创建，yaml方式可以热更新）
[root@k8s-master1 ~]# mkdir redis-ha
[root@k8s-master1 ~]# cd redis-ha/
[root@k8s-master1 redis-ha]# vim redis.conf
[root@k8s-master1 redis-ha]# cat redis.conf
appendonly yes
cluster-enabled yes
cluster-config-file /var/lib/redis/nodes.conf
cluster-node-timeout 5000
dir /var/lib/redis
port 6379
[root@k8s-master1 redis-ha]# kubectl create configmap redis-conf --from-file=redis.conf
configmap/redis-conf created
[root@k8s-master1 redis-ha]# kubectl get cm
NAME DATA AGE
redis-conf 1 108s2.4、创建Headless service
给StatefulSet准备Headless service
[root@k8s-master1 redis-ha]# vim headless-service.yaml
[root@k8s-master1 redis-ha]# cat headless-service.yaml
---
apiVersion: v1
kind: Service
metadata:
name: redis-service
labels:
app: redis
spec:
ports:
- name: redis-port
port: 6379
clusterIP: None
selector:
app: redis
appCluster: redis-cluster
[root@k8s-master1 redis-ha]# kubectl apply -f headless-service.yaml
service/redis-service created
[root@k8s-master1 redis-ha]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 20d
redis-service ClusterIP None <none> 6379/TCP 73s2.5、StatefulSet运行redis实例
利用StatefulSet创建Redis 集群节点
[root@k8s-master1 redis-ha]# vim statefulset.yaml
[root@k8s-master1 redis-ha]# cat statefulset.yaml
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: redis-app
spec:
serviceName: "redis-service"
replicas: 6
selector:
matchLabels:
app: redis
appCluster: redis-cluster
template:
metadata:
labels:
app: redis
appCluster: redis-cluster
spec:
terminationGracePeriodSeconds: 20
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: app
operator: In
values:
- redis
topologyKey: kubernetes.io/hostname
containers:
- name: redis
image: registry.cn-chengdu.aliyuncs.com/liumuquan_app/redis:6-alpine3.12
command:
- "redis-server"
args:
- "/etc/redis/redis.conf"
- "--protected-mode"
- "no"
resources:
requests:
cpu: "100m"
memory: "100Mi"
ports:
- name: redis
containerPort: 6379
protocol: "TCP"
- name: cluster
containerPort: 16379
protocol: "TCP"
volumeMounts:
- name: "redis-conf"
mountPath: "/etc/redis"
- name: "redis-data"
mountPath: "/var/lib/redis"
volumes:
- name: "redis-conf"
configMap:
name: "redis-conf"
items:
- key: "redis.conf"
path: "redis.conf"
volumeClaimTemplates:
- metadata:
name: redis-data
spec:
accessModes: [ "ReadWriteMany" ]
storageClassName: "nfs-client"
resources:
requests:
storage: 200M
[root@k8s-master1 redis-ha]# kubectl apply -f statefulset.yaml
[root@k8s-master1 redis-ha]# kubectl get pod
NAME READY STATUS RESTARTS AGE
nfs-redis-nfs-client-provisioner-56f8995966-zc6b9 1/1 Running 0 56m
redis-app-0 1/1 Running 0 103s
redis-app-1 1/1 Running 0 99s
redis-app-2 1/1 Running 0 95s
redis-app-3 1/1 Running 0 91s
redis-app-4 1/1 Running 0 87s
redis-app-5 1/1 Running 0 83s
[root@k8s-master1 redis-ha]# kubectl get sts
NAME READY AGE
redis-app 6/6 5m52s此时新建的redis-pod应该被分配了集群内唯一的网络域名
2.6、验证唯一访问标识可用性
如上所述，总共创建了6个Redis节点（Pod），其中3个作为master，另3个作为master的slave。Redis配置通过volume将redis-conf ConfigMap挂载到容器的/etc/redis/redis.conf；数据存储路径通过volumeClaimTemplates声明（即PVC），绑定到预先创建的PV上。
关键概念为Affinity。有关详情请参阅官方文档。podAntiAffinity确保某个Pod不会与指定的Pod在同一拓扑域内，增强服务稳定性。PreferredDuringSchedulingIgnoredDuringExecution指调度期间尽量满足亲和性或反亲和性规则，若无法满足，Pod可能仍被调度到相应节点，运行时系统不再检查这些规则。
在此配置中，matchExpressions要求Redis Pod尽量避免调度到已包含Redis的节点上。然而，因只有三个节点而有六个副本，Pod将被调度到所有节点。根据PreferredDuringSchedulingIgnoredDuringExecution，这些Pod会被调度到所有可用节点上。
根据StatefulSet规则，Redis的6个Pod将依次命名为$(statefulset名称)-$(序号)，如图所示。Pods按{0…N-1}的顺序创建，并且只有在redis-app-0处于Running状态后，redis-app-1才会启动。每个Pod还会获得集群内的DNS域名，格式为$(podname).$(service name).$(namespace).svc.cluster.local。
这里新建一个pod验证域名访问，使用这个镜像原因是镜像体积小且有ping功能
[root@k8s-master1 redis-ha]# vim busybox.yaml
[root@k8s-master1 redis-ha]# cat busybox.yaml
---
apiVersion: v1
kind: Pod
metadata:
name: busybox
spec:
containers:
- name: busybox
image: registry.cn-chengdu.aliyuncs.com/liumuquan_app/busybox:latest
stdin: true
tty: true
[root@k8s-master1 redis-ha]# kubectl apply -f busybox.yaml
pod/busybox created
[root@k8s-master1 redis-ha]# kubectl get pod
NAME READY STATUS RESTARTS AGE
busybox 1/1 Running 0 8s
nfs-redis-nfs-client-provisioner-56f8995966-zc6b9 1/1 Running 0 79m
redis-app-0 1/1 Running 0 24m
redis-app-1 1/1 Running 0 24m
redis-app-2 1/1 Running 0 24m
redis-app-3 1/1 Running 0 24m
redis-app-4 1/1 Running 0 24m
redis-app-5 1/1 Running 0 24m
[root@k8s-master1 redis-ha]# kubectl exec -it busybox /bin/sh
/ # ping redis-app-1.redis-service.default.svc.cluster.local
PING redis-app-1.redis-service.default.svc.cluster.local (10.244.1.52): 56 data bytes
64 bytes from 10.244.1.52: seq=0 ttl=62 time=0.653 ms
64 bytes from 10.244.1.52: seq=1 ttl=62 time=0.526 ms
64 bytes from 10.244.1.52: seq=2 ttl=62 time=0.391 ms
64 bytes from 10.244.1.52: seq=3 ttl=62 time=0.466 ms
^C
--- redis-app-1.redis-service.default.svc.cluster.local ping statistics ---
4 packets transmitted, 4 packets received, 0% packet loss
round-trip min/avg/max = 0.391/0.509/0.653 ms
/ # exec attach failed: error on attach stdin: read escape sequence
command terminated with exit code 126
[root@k8s-master1 redis-ha]# kubectl get pod -o wide | grep redis-app-1
redis-app-1 1/1 Running 0 28m 10.244.1.52 k8s-node1 <none> <none>
[root@k8s-master1 redis-ha]#
ping域名解析出来的ip与redis-app-1的ip相同，证明唯一访问标识可用
查看nfs服务器的共享目录可以看到6个文件夹
2.7、 集群初始化
在创建了6个Redis Pod后，需要使用Redis-trib工具进行集群初始化，该工具没有centos版本。为了简化初始化过程，于是在Kubernetes上创建一个额外的Ubuntu Pod，用于管理和控制集群服务。这样可以避免将初始化逻辑写入StatefulSet中，因为那样做会非常复杂且低效。在这个Ubuntu容器中，可以安装Redis-trib并使用它来初始化Redis集群。
[root@k8s-master1 redis-ha]# kubectl run -it ubuntu --image=registry.cn-chengdu.aliyuncs.com/liumuquan_app/ubuntu:artful-20170619 --restart=Never /bin/bash
在ubuntu容器内执行如下命令，更换阿里源
cat > /etc/apt/sources.list << EOF
deb http://mirrors.aliyun.com/ubuntu/ bionic main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ bionic-security main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-security main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ bionic-updates main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-updates main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ bionic-proposed main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-proposed main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ bionic-backports main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-backports main restricted universe multiverse
EOF
root@ubuntu:/# apt-get update
。。。。。。
root@ubuntu:/# echo $?
0
root@ubuntu:/#
安装所需程序
root@ubuntu:/# apt-get install -y vim wget python2.7 python-pip redis-tools dnsutils
# 上述程序安装完成后安装redis-trib
root@ubuntu:/# pip install redis-trib==0.5.1
root@ubuntu:/# echo $?
0
# 这个下载很慢需要检查下是否下载完成
初始化集群
# 创建master集群
root@ubuntu:/# redis-trib.py create \
> `dig +short redis-app-0.redis-service.default.svc.cluster.local`:6379 \
> `dig +short redis-app-1.redis-service.default.svc.cluster.local`:6379 \
> `dig +short redis-app-2.redis-service.default.svc.cluster.local`:6379
Redis-trib 0.5.1 Copyright (c) HunanTV Platform developers
INFO:root:Instance at 10.244.2.37:6379 checked
INFO:root:Instance at 10.244.2.36:6379 checked
INFO:root:Instance at 10.244.1.52:6379 checked
INFO:root:Add 5462 slots to 10.244.2.37:6379
INFO:root:Add 5461 slots to 10.244.2.36:6379
INFO:root:Add 5461 slots to 10.244.1.52:6379
# 为每个Master添加Slave
root@ubuntu:/# redis-trib.py replicate \
> --master-addr `dig +short redis-app-0.redis-service.default.svc.cluster.local`:6379 \
> --slave-addr `dig +short redis-app-3.redis-service.default.svc.cluster.local`:6379
Redis-trib 0.5.1 Copyright (c) HunanTV Platform developers
INFO:root:Instance at 10.244.1.53:6379 has joined 10.244.2.36:6379; now set replica
INFO:root:Instance at 10.244.1.53:6379 set as replica to 066f5d75830577b9c336f5b1b342ca6a410dd993
root@ubuntu:/# echo $?
0
root@ubuntu:/# redis-trib.py replicate \
> --master-addr `dig +short redis-app-1.redis-service.default.svc.cluster.local`:6379 \
> --slave-addr `dig +short redis-app-4.redis-service.default.svc.cluster.local`:6379
Redis-trib 0.5.1 Copyright (c) HunanTV Platform developers
INFO:root:Instance at 10.244.2.38:6379 has joined 10.244.1.52:6379; now set replica
INFO:root:Instance at 10.244.2.38:6379 set as replica to d8e975aca09b0eecd0b84120e7e1992c1db3db66
root@ubuntu:/# echo $?
0
root@ubuntu:/# redis-trib.py replicate \
> --master-addr `dig +short redis-app-2.redis-service.default.svc.cluster.local`:6379 \
> --slave-addr `dig +short redis-app-5.redis-service.default.svc.cluster.local`:6379
Redis-trib 0.5.1 Copyright (c) HunanTV Platform developers
INFO:root:Instance at 10.244.1.54:6379 has joined 10.244.2.37:6379; now set replica
INFO:root:Instance at 10.244.1.54:6379 set as replica to 931f10a16b1ac4220532bf321013a683e44dfed5
root@ubuntu:/# echo $?
0
root@ubuntu:/#至此基于kubernetes的redis-cluster部署完成
3、集群测试
3.1、查看集群信息
退出Ubuntu pod，进入任意一个redis pod
[root@k8s-master1 redis-ha]# kubectl exec -it redis-app-2 /bin/sh
/data # /usr/local/bin/redis-cli -c
127.0.0.1:6379> ping
PONG
127.0.0.1:6379> cluster nodes
f0580ea45cc0ecd56a7ce1cb6a2d9e6dc0e75112 10.244.1.53:6379@16379 slave 066f5d75830577b9c336f5b1b342ca6a410dd993 0 1725451855283 2 connected
931f10a16b1ac4220532bf321013a683e44dfed5 10.244.2.37:6379@16379 myself,master - 0 1725451853000 1 connected 0-5461
94ca16b2b79b397825646879c84b31315c7bf3a0 10.244.1.54:6379@16379 slave 931f10a16b1ac4220532bf321013a683e44dfed5 0 1725451856288 1 connected
b2fcbf9f5ee22379328755b173ab3411de26410d 10.244.2.38:6379@16379 slave d8e975aca09b0eecd0b84120e7e1992c1db3db66 0 1725451855000 3 connected
066f5d75830577b9c336f5b1b342ca6a410dd993 10.244.2.36:6379@16379 master - 0 1725451855000 2 connected 5462-10922
d8e975aca09b0eecd0b84120e7e1992c1db3db66 10.244.1.52:6379@16379 master - 0 1725451855786 3 connected 10923-16383
# 这里可以看到集群内主从对应关系和hash槽点分布查看节点信息
127.0.0.1:6379> cluster info
3.2、 创建用于访问Service
上面创建了用于实现StatefulSet的Headless Service，但该Service没有Cluster IP，因此不能用于外界访问。所以还需要创建一个Service，专用于为Redis集群提供访问和负载均衡，这时使用上文中创建的标签即可：
退出redis pod，创建Service
[root@k8s-master1 redis-ha]# vim redis-access-service.yaml
[root@k8s-master1 redis-ha]# cat redis-access-service.yaml
---
apiVersion: v1
kind: Service
metadata:
name: redis-access-service
labels:
app: redis
spec:
ports:
- name: redis-port
protocol: "TCP"
port: 6379
targetPort: 6379
selector:
app: redis
appCluster: redis-cluster
#如上，该Service名称为 redis-access-service，在K8S集群中暴露6379端口
#并会对labels name为app: redis或appCluster: redis-cluster的pod进行负载均衡。
[root@k8s-master1 redis-ha]# kubectl apply -f redis-access-service.yaml
service/redis-access-service created
[root@k8s-master1 redis-ha]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 20d
redis-access-service ClusterIP 10.104.69.17 <none> 6379/TCP 7s
redis-service ClusterIP None <none> 6379/TCP 161m
[root@k8s-master1 redis-ha]# kubectl get endpoints
NAME ENDPOINTS AGE
cluster.local-nfs-redis-nfs-client-provisioner <none> 3h5m
kubernetes 192.168.188.101:6443 20d
redis-access-service 10.244.1.52:6379,10.244.1.53:6379,10.244.1.54:6379 + 3 more... 54s
redis-service 10.244.1.52:6379,10.244.1.53:6379,10.244.1.54:6379 + 3 more... 162m3.3、测试主从切换
现在0，1，2为master节点3，4，5为slave节点，此时尝试删除一个master节点，查看是否能够完成主从切换。
手动删除redis-app-0
[root@k8s-master1 redis-ha]# kubectl delete pod redis-app-0
pod "redis-app-0" deleted
[root@k8s-master1 redis-ha]# kubectl exec -it redis-app-0 /bin/sh
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
/data # /usr/local/bin/redis-cli -c
127.0.0.1:6379> role
1) "slave"
2) "10.244.1.53"
3) (integer) 6379
4) "connected"
5) (integer) 742
[root@k8s-master1 redis-ha]# kubectl exec -it redis-app-3 /bin/sh
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
/data # /usr/local/bin/redis-cli -c
127.0.0.1:6379> role
1) "master"
2) (integer) 882
3) 1) 1) "10.244.2.41"
2) "6379"
3) "882"
127.0.0.1:6379>
主从切换功能正常