当前位置：首页 > article >正文

10--基于kubernetes的redis-cluster部署

article 2024/11/14 9:42:00

前言：这一章作为操作手册记录，比较偏向于实战内容，内部源与镜像均使用国内地址代替，保证安装的可行性。

1、基础环境

角色	IP
k8s-master1	192.168.188.101
k8s-node1	192.168.188.102
k8s-node2	192.168.188.103
nfs服务器	192.168.188.200

需要部署集群为redis-cluster集群，规模为三主三从，使用到的镜像下文中有详细说明

2、部署实操

2.1、准备nfs存储空间

[root@localhost ~]# mkdir /data
[root@localhost ~]# umount /root/nfs
[root@localhost ~]# mount /dev/sdb /data
[root@localhost ~]# df -Th
文件系统                类型      容量  已用  可用 已用% 挂载点
devtmpfs                devtmpfs  475M     0  475M    0% /dev
tmpfs                   tmpfs     487M     0  487M    0% /dev/shm
tmpfs                   tmpfs     487M  7.7M  479M    2% /run
tmpfs                   tmpfs     487M     0  487M    0% /sys/fs/cgroup
/dev/mapper/centos-root xfs        17G  4.8G   13G   28% /
/dev/sda1               xfs      1014M  190M  825M   19% /boot
tmpfs                   tmpfs      98M     0   98M    0% /run/user/0
/dev/sdb                ext4       99G  259M   94G    1% /data
[root@localhost ~]# chmod -R 777 /data
[root@localhost ~]# vim /etc/exports
[root@localhost ~]# cat /etc/exports
/data *(rw,no_root_squash,no_all_squash,sync)
[root@localhost ~]# systemctl restart rpcbind nfs

使用node节点测试nfs挂载情况

[root@k8s-node1 ~]# mount -t nfs 192.168.188.200:/data /test
[root@k8s-node1 ~]# touch /test/1.txt
[root@k8s-node1 ~]# ls /test/
1.txt

回到nfs节点查看文件存在

[root@localhost ~]# ls /data/
1.txt

nfs配置成功，回到node节点卸载nfs

[root@k8s-node1 ~]# umount /test
[root@k8s-node1 ~]# df -Th | grep "/test"
[root@k8s-node1 ~]#

2.2、配置动态存储

安装配置helm工具（k8s包管理工具）

官网文档：Helm | Docs

百度网盘链接：
链接: https://pan.baidu.com/s/16IxIYH2uWcGPCmEtxVs2Vw?pwd=erwg 提取码: erwg

[root@k8s-master1 ~]# wget https://get.helm.sh/helm-v3.10.1-linux-amd64.tar.gz
[root@k8s-master1 ~]# tar -xvzf helm-v3.10.1-linux-amd64.tar.gz 
linux-amd64/
linux-amd64/helm
linux-amd64/LICENSE
linux-amd64/README.md
[root@k8s-master1 ~]# cp linux-amd64/helm  /usr/bin/
[root@k8s-master1 ~]# helm version
version.BuildInfo{Version:"v3.10.1", GitCommit:"9f88ccb6aee40b9a0535fcc7efea6055e1ef72c9", GitTreeState:"clean", GoVersion:"go1.18.7"}

    #    为helm添加一个名为stable的源
[root@k8s-master1 ~]# helm repo add stable http://mirror.azure.cn/kubernetes/charts
"stable" has been added to your repositories
[root@k8s-master1 ~]# helm repo list
NAME  	URL                                     
stable	http://mirror.azure.cn/kubernetes/chart

使用helm制作动态存储storageclass与nfs的共享目录相关联

[root@k8s-master1 ~]# helm install nfs-redis stable/nfs-client-provisioner --set nfs.server=192.168.188.200 --set nfs.path=/data
WARNING: This chart is deprecated
NAME: nfs-redis
LAST DEPLOYED: Wed Sep  4 17:16:32 2024
NAMESPACE: default
STATUS: deployed
REVISION: 1
TEST SUITE: None
[root@k8s-master1 ~]# kubectl get pod
NAME                                                READY   STATUS    RESTARTS   AGE
nfs-redis-nfs-client-provisioner-56f8995966-zc6b9   1/1     Running   0          75s
[root@k8s-master1 ~]# helm list
NAME     	NAMESPACE	REVISION	UPDATED                                	STATUS  	CHART                        	APP VERSION
nfs-redis	default  	1       	2024-09-04 17:16:32.561824151 +0800 CST	deployed	nfs-client-provisioner-1.2.11	3.1.0      
[root@k8s-master1 ~]# kubectl get sc
NAME         PROVISIONER                                      RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
nfs-client   cluster.local/nfs-redis-nfs-client-provisioner   Delete          Immediate           true                   2m25s
    #    这里自动进行提权

2.3、redis配置文件（ConfigMap）

这里使用指定文件创建（可以通过yaml方式创建，yaml方式可以热更新）

[root@k8s-master1 ~]# mkdir redis-ha
[root@k8s-master1 ~]# cd redis-ha/
[root@k8s-master1 redis-ha]# vim redis.conf
[root@k8s-master1 redis-ha]# cat redis.conf
appendonly yes
cluster-enabled yes
cluster-config-file /var/lib/redis/nodes.conf
cluster-node-timeout 5000
dir /var/lib/redis
port 6379
[root@k8s-master1 redis-ha]# kubectl create configmap redis-conf --from-file=redis.conf
configmap/redis-conf created
[root@k8s-master1 redis-ha]# kubectl get cm
NAME         DATA   AGE
redis-conf   1      108s

2.4、创建Headless service

给StatefulSet准备Headless service

[root@k8s-master1 redis-ha]# vim headless-service.yaml
[root@k8s-master1 redis-ha]# cat headless-service.yaml
---
apiVersion: v1
kind: Service
metadata:
  name: redis-service
  labels:
    app: redis
spec:
  ports:
  - name: redis-port
    port: 6379
  clusterIP: None
  selector:
    app: redis
    appCluster: redis-cluster
[root@k8s-master1 redis-ha]# kubectl apply -f headless-service.yaml 
service/redis-service created
[root@k8s-master1 redis-ha]# kubectl get svc
NAME            TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)    AGE
kubernetes      ClusterIP   10.96.0.1    <none>        443/TCP    20d
redis-service   ClusterIP   None         <none>        6379/TCP   73s

2.5、StatefulSet运行redis实例

利用StatefulSet创建Redis 集群节点

[root@k8s-master1 redis-ha]# vim statefulset.yaml
[root@k8s-master1 redis-ha]# cat statefulset.yaml
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis-app
spec:
  serviceName: "redis-service"
  replicas: 6
  selector:
    matchLabels:
      app: redis
      appCluster: redis-cluster
  template:
    metadata:
      labels:
        app: redis
        appCluster: redis-cluster
    spec:
      terminationGracePeriodSeconds: 20
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - redis
              topologyKey: kubernetes.io/hostname
      containers:
      - name: redis
        image: registry.cn-chengdu.aliyuncs.com/liumuquan_app/redis:6-alpine3.12
        command:
          - "redis-server"
        args:
          - "/etc/redis/redis.conf"
          - "--protected-mode"
          - "no"
        resources:
          requests:
            cpu: "100m"
            memory: "100Mi"
        ports:
            - name: redis
              containerPort: 6379
              protocol: "TCP"
            - name: cluster
              containerPort: 16379
              protocol: "TCP"
        volumeMounts:
          - name: "redis-conf"
            mountPath: "/etc/redis"
          - name: "redis-data"
            mountPath: "/var/lib/redis"
      volumes:
      - name: "redis-conf"
        configMap:
          name: "redis-conf"
          items:
            - key: "redis.conf"
              path: "redis.conf"
  volumeClaimTemplates:
  - metadata:
      name: redis-data
    spec:
      accessModes: [ "ReadWriteMany" ]
      storageClassName: "nfs-client"
      resources:
        requests:
          storage: 200M

[root@k8s-master1 redis-ha]# kubectl apply -f statefulset.yaml
[root@k8s-master1 redis-ha]# kubectl get pod
NAME                                                READY   STATUS    RESTARTS   AGE
nfs-redis-nfs-client-provisioner-56f8995966-zc6b9   1/1     Running   0          56m
redis-app-0                                         1/1     Running   0          103s
redis-app-1                                         1/1     Running   0          99s
redis-app-2                                         1/1     Running   0          95s
redis-app-3                                         1/1     Running   0          91s
redis-app-4                                         1/1     Running   0          87s
redis-app-5                                         1/1     Running   0          83s
[root@k8s-master1 redis-ha]# kubectl get sts
NAME        READY   AGE
redis-app   6/6     5m52s

此时新建的redis-pod应该被分配了集群内唯一的网络域名

2.6、验证唯一访问标识可用性

如上所述，总共创建了6个Redis节点（Pod），其中3个作为master，另3个作为master的slave。Redis配置通过volume将redis-conf ConfigMap挂载到容器的/etc/redis/redis.conf；数据存储路径通过volumeClaimTemplates声明（即PVC），绑定到预先创建的PV上。

关键概念为Affinity。有关详情请参阅官方文档。podAntiAffinity确保某个Pod不会与指定的Pod在同一拓扑域内，增强服务稳定性。PreferredDuringSchedulingIgnoredDuringExecution指调度期间尽量满足亲和性或反亲和性规则，若无法满足，Pod可能仍被调度到相应节点，运行时系统不再检查这些规则。

在此配置中，matchExpressions要求Redis Pod尽量避免调度到已包含Redis的节点上。然而，因只有三个节点而有六个副本，Pod将被调度到所有节点。根据PreferredDuringSchedulingIgnoredDuringExecution，这些Pod会被调度到所有可用节点上。

根据StatefulSet规则，Redis的6个Pod将依次命名为$(statefulset名称)-$(序号)，如图所示。Pods按{0…N-1}的顺序创建，并且只有在redis-app-0处于Running状态后，redis-app-1才会启动。每个Pod还会获得集群内的DNS域名，格式为$(podname).$(service name).$(namespace).svc.cluster.local。

这里新建一个pod验证域名访问，使用这个镜像原因是镜像体积小且有ping功能

[root@k8s-master1 redis-ha]# vim busybox.yaml
[root@k8s-master1 redis-ha]# cat busybox.yaml
---
apiVersion: v1
kind: Pod
metadata:
  name: busybox
spec:
  containers:
    - name: busybox
      image: registry.cn-chengdu.aliyuncs.com/liumuquan_app/busybox:latest
      stdin: true
      tty: true

[root@k8s-master1 redis-ha]# kubectl apply -f busybox.yaml 
pod/busybox created
[root@k8s-master1 redis-ha]# kubectl get pod
NAME                                                READY   STATUS    RESTARTS   AGE
busybox                                             1/1     Running   0          8s
nfs-redis-nfs-client-provisioner-56f8995966-zc6b9   1/1     Running   0          79m
redis-app-0                                         1/1     Running   0          24m
redis-app-1                                         1/1     Running   0          24m
redis-app-2                                         1/1     Running   0          24m
redis-app-3                                         1/1     Running   0          24m
redis-app-4                                         1/1     Running   0          24m
redis-app-5                                         1/1     Running   0          24m
[root@k8s-master1 redis-ha]# kubectl exec -it busybox /bin/sh

/ # ping redis-app-1.redis-service.default.svc.cluster.local
PING redis-app-1.redis-service.default.svc.cluster.local (10.244.1.52): 56 data bytes
64 bytes from 10.244.1.52: seq=0 ttl=62 time=0.653 ms
64 bytes from 10.244.1.52: seq=1 ttl=62 time=0.526 ms
64 bytes from 10.244.1.52: seq=2 ttl=62 time=0.391 ms
64 bytes from 10.244.1.52: seq=3 ttl=62 time=0.466 ms
^C
--- redis-app-1.redis-service.default.svc.cluster.local ping statistics ---
4 packets transmitted, 4 packets received, 0% packet loss
round-trip min/avg/max = 0.391/0.509/0.653 ms
/ # exec attach failed: error on attach stdin: read escape sequence
command terminated with exit code 126

[root@k8s-master1 redis-ha]# kubectl get pod -o wide | grep redis-app-1
redis-app-1                                         1/1     Running   0          28m     10.244.1.52   k8s-node1   <none>           <none>
[root@k8s-master1 redis-ha]#

ping域名解析出来的ip与redis-app-1的ip相同，证明唯一访问标识可用

查看nfs服务器的共享目录可以看到6个文件夹

2.7、集群初始化

在创建了6个Redis Pod后，需要使用Redis-trib工具进行集群初始化，该工具没有centos版本。为了简化初始化过程，于是在Kubernetes上创建一个额外的Ubuntu Pod，用于管理和控制集群服务。这样可以避免将初始化逻辑写入StatefulSet中，因为那样做会非常复杂且低效。在这个Ubuntu容器中，可以安装Redis-trib并使用它来初始化Redis集群。

[root@k8s-master1 redis-ha]# kubectl run -it ubuntu --image=registry.cn-chengdu.aliyuncs.com/liumuquan_app/ubuntu:artful-20170619 --restart=Never /bin/bash

在ubuntu容器内执行如下命令，更换阿里源
cat > /etc/apt/sources.list << EOF
deb http://mirrors.aliyun.com/ubuntu/ bionic main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic main restricted universe multiverse

deb http://mirrors.aliyun.com/ubuntu/ bionic-security main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-security main restricted universe multiverse

deb http://mirrors.aliyun.com/ubuntu/ bionic-updates main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-updates main restricted universe multiverse

deb http://mirrors.aliyun.com/ubuntu/ bionic-proposed main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-proposed main restricted universe multiverse
 
deb http://mirrors.aliyun.com/ubuntu/ bionic-backports main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-backports main restricted universe multiverse
EOF

root@ubuntu:/# apt-get update
            。。。。。。

root@ubuntu:/# echo $?
0
root@ubuntu:/#

安装所需程序

root@ubuntu:/# apt-get install -y vim wget python2.7 python-pip redis-tools dnsutils

    #    上述程序安装完成后安装redis-trib
root@ubuntu:/# pip install redis-trib==0.5.1
root@ubuntu:/# echo $?
0
    #    这个下载很慢需要检查下是否下载完成

初始化集群

    #    创建master集群
root@ubuntu:/# redis-trib.py create \
> `dig +short redis-app-0.redis-service.default.svc.cluster.local`:6379 \
> `dig +short redis-app-1.redis-service.default.svc.cluster.local`:6379 \
> `dig +short redis-app-2.redis-service.default.svc.cluster.local`:6379
Redis-trib 0.5.1 Copyright (c) HunanTV Platform developers
INFO:root:Instance at 10.244.2.37:6379 checked
INFO:root:Instance at 10.244.2.36:6379 checked
INFO:root:Instance at 10.244.1.52:6379 checked
INFO:root:Add 5462 slots to 10.244.2.37:6379
INFO:root:Add 5461 slots to 10.244.2.36:6379
INFO:root:Add 5461 slots to 10.244.1.52:6379

    #    为每个Master添加Slave
root@ubuntu:/# redis-trib.py replicate \
>   --master-addr `dig +short redis-app-0.redis-service.default.svc.cluster.local`:6379 \
>   --slave-addr `dig +short redis-app-3.redis-service.default.svc.cluster.local`:6379
Redis-trib 0.5.1 Copyright (c) HunanTV Platform developers
INFO:root:Instance at 10.244.1.53:6379 has joined 10.244.2.36:6379; now set replica
INFO:root:Instance at 10.244.1.53:6379 set as replica to 066f5d75830577b9c336f5b1b342ca6a410dd993
root@ubuntu:/# echo $?
0
root@ubuntu:/# redis-trib.py replicate \
>   --master-addr `dig +short redis-app-1.redis-service.default.svc.cluster.local`:6379 \
>   --slave-addr `dig +short redis-app-4.redis-service.default.svc.cluster.local`:6379
Redis-trib 0.5.1 Copyright (c) HunanTV Platform developers
INFO:root:Instance at 10.244.2.38:6379 has joined 10.244.1.52:6379; now set replica
INFO:root:Instance at 10.244.2.38:6379 set as replica to d8e975aca09b0eecd0b84120e7e1992c1db3db66
root@ubuntu:/# echo $?
0
root@ubuntu:/# redis-trib.py replicate \
>   --master-addr `dig +short redis-app-2.redis-service.default.svc.cluster.local`:6379 \
>   --slave-addr `dig +short redis-app-5.redis-service.default.svc.cluster.local`:6379
Redis-trib 0.5.1 Copyright (c) HunanTV Platform developers
INFO:root:Instance at 10.244.1.54:6379 has joined 10.244.2.37:6379; now set replica
INFO:root:Instance at 10.244.1.54:6379 set as replica to 931f10a16b1ac4220532bf321013a683e44dfed5
root@ubuntu:/# echo $?
0
root@ubuntu:/#

至此基于kubernetes的redis-cluster部署完成

3、集群测试

3.1、查看集群信息

退出Ubuntu pod，进入任意一个redis pod

[root@k8s-master1 redis-ha]# kubectl exec -it redis-app-2 /bin/sh
/data # /usr/local/bin/redis-cli -c
127.0.0.1:6379> ping
PONG
127.0.0.1:6379> cluster nodes
f0580ea45cc0ecd56a7ce1cb6a2d9e6dc0e75112 10.244.1.53:6379@16379 slave 066f5d75830577b9c336f5b1b342ca6a410dd993 0 1725451855283 2 connected
931f10a16b1ac4220532bf321013a683e44dfed5 10.244.2.37:6379@16379 myself,master - 0 1725451853000 1 connected 0-5461
94ca16b2b79b397825646879c84b31315c7bf3a0 10.244.1.54:6379@16379 slave 931f10a16b1ac4220532bf321013a683e44dfed5 0 1725451856288 1 connected
b2fcbf9f5ee22379328755b173ab3411de26410d 10.244.2.38:6379@16379 slave d8e975aca09b0eecd0b84120e7e1992c1db3db66 0 1725451855000 3 connected
066f5d75830577b9c336f5b1b342ca6a410dd993 10.244.2.36:6379@16379 master - 0 1725451855000 2 connected 5462-10922
d8e975aca09b0eecd0b84120e7e1992c1db3db66 10.244.1.52:6379@16379 master - 0 1725451855786 3 connected 10923-16383

    #    这里可以看到集群内主从对应关系和hash槽点分布

查看节点信息

127.0.0.1:6379> cluster info

3.2、创建用于访问Service

上面创建了用于实现StatefulSet的Headless Service，但该Service没有Cluster IP，因此不能用于外界访问。所以还需要创建一个Service，专用于为Redis集群提供访问和负载均衡，这时使用上文中创建的标签即可：

退出redis pod，创建Service

[root@k8s-master1 redis-ha]# vim redis-access-service.yaml
[root@k8s-master1 redis-ha]# cat redis-access-service.yaml
---
apiVersion: v1
kind: Service
metadata:
  name: redis-access-service
  labels:
    app: redis
spec:
  ports:
  - name: redis-port
    protocol: "TCP"
    port: 6379
    targetPort: 6379
  selector:
    app: redis
    appCluster: redis-cluster


#如上，该Service名称为 redis-access-service，在K8S集群中暴露6379端口
#并会对labels name为app: redis或appCluster: redis-cluster的pod进行负载均衡。

[root@k8s-master1 redis-ha]# kubectl apply -f redis-access-service.yaml 
service/redis-access-service created
[root@k8s-master1 redis-ha]# kubectl get svc
NAME                   TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)    AGE
kubernetes             ClusterIP   10.96.0.1      <none>        443/TCP    20d
redis-access-service   ClusterIP   10.104.69.17   <none>        6379/TCP   7s
redis-service          ClusterIP   None           <none>        6379/TCP   161m
[root@k8s-master1 redis-ha]# kubectl get endpoints
NAME                                             ENDPOINTS                                                        AGE
cluster.local-nfs-redis-nfs-client-provisioner   <none>                                                           3h5m
kubernetes                                       192.168.188.101:6443                                             20d
redis-access-service                             10.244.1.52:6379,10.244.1.53:6379,10.244.1.54:6379 + 3 more...   54s
redis-service                                    10.244.1.52:6379,10.244.1.53:6379,10.244.1.54:6379 + 3 more...   162m

3.3、测试主从切换

现在0，1，2为master节点3，4，5为slave节点，此时尝试删除一个master节点，查看是否能够完成主从切换。

手动删除redis-app-0

[root@k8s-master1 redis-ha]# kubectl delete pod redis-app-0
pod "redis-app-0" deleted
[root@k8s-master1 redis-ha]# kubectl exec -it redis-app-0 /bin/sh
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
/data # /usr/local/bin/redis-cli -c
127.0.0.1:6379> role
1) "slave"
2) "10.244.1.53"
3) (integer) 6379
4) "connected"
5) (integer) 742



[root@k8s-master1 redis-ha]# kubectl exec -it redis-app-3 /bin/sh
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
/data # /usr/local/bin/redis-cli -c
127.0.0.1:6379> role
1) "master"
2) (integer) 882
3) 1) 1) "10.244.2.41"
      2) "6379"
      3) "882"
127.0.0.1:6379>

主从切换功能正常

查看全文

http://www.kler.cn/a/292896.html