当前位置：首页 > article >正文

k8s基于nfs创建storageClass

article 2024/11/19 19:32:59

首先安装nfs

#服务端安装
yum install -y nfs-utils rpcbind
#客户端安装
yum install -y nfs-utils
#启动服务 并设置开启启动
systemctl start rpcbind && systemctl enable rpcbind
systemctl start nfs && systemctl enable nfs
#创建共享目录
mkdir -p /root/data/nfs
#设置权限
chmod 666 /root/data/nfs
#修改 NFS 配置文件
vim /etc/exports
/root/data/nfs 192.168.140.0/24(rw,sync,insecure,no_subtree_check,no_root_squash)
#载入配置
exportfs -rv
#查看 NFS 服务项 rpc 服务器注册的端口列表
rpcinfo -p localhost
#服务端看下是否正确加载了设置的 /etc/exports 配置
showmount -e localhost
#关闭防火墙
systemctl stop firewalld.service
#设置开机自动挂载
echo "192.168.140.129:/root/data/nfs /share nfs defaults 0 0" >> /etc/fstab
#客户端查看能否正常挂载 192.168.140.129 是nfs服务端ip
showmount -e 192.168.140.129
#客户端执行绑定目录注意：一定挂载根目录例如： /share
mount 192.168.140.129:/root/data/nfs /share
#客户端在/share里写入文件测试

搭建好nfs 下一步以静态方式通过手动创建pv pvc来实现nfs存储
注意：首先要在各节点安装nfs客户端
确保 Kubernetes 节点安装了 NFS 客户端
要确保 NFS 能在 Kubernetes 中正常挂载，所有工作节点（worker nodes）必须安装 NFS 客户端工具包。NFS 的挂载操作是由 Kubernetes 节点来处理的，而不是容器内的程序。你可以通过以下命令在节点上安装 NFS 工具。
否则会报如下异常：

 Warning  FailedMount  33s                  kubelet            Unable to attach or mount volumes: unmounted volumes=[nfs-storage], unattached volumes=[nfs-storage kube-api-access-cpsjc]: timed out waiting for the condition
  Warning  FailedMount  27s (x9 over 2m35s)  kubelet            MountVolume.SetUp failed for volume "nfs-pv" : mount failed: exit status 32
Mounting command: mount
Mounting arguments: -t nfs 192.168.140.129:/root/data/nfs /var/lib/kubelet/pods/107290f4-9c5d-40c6-b42a-b5533c76dc57/volumes/kubernetes.io~nfs/nfs-pv
Output: mount: 文件系统类型错误、选项错误、192.168.140.129:/root/data/nfs 上有坏超级块、
       缺少代码页或助手程序，或其他错误
       (对某些文件系统(如 nfs、cifs) 您可能需要
       一款 /sbin/mount.<类型> 助手程序)

       有些情况下在 syslog 中可以找到一些有用信息- 请尝试
       dmesg | tail  这样的命令看看。

部署yaml静态使用nfs示例

#测试pod使用nfs存储
apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-pv
  namespace: storagedemo
spec:
  capacity:
    storage: 1Gi   # 你想要的存储大小
  accessModes:
    - ReadWriteMany  # 允许多个节点读写
  nfs:
    path: /root/data/nfs   # NFS 共享的路径
    server: 192.168.140.129  # NFS 服务器地址
  persistentVolumeReclaimPolicy: Retain

---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: nfs-pvc
  namespace: storagedemo
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 1Gi

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nfsdemo-v1
  namespace: storagedemo
  labels:
    app: nfsdemo
    version: v1
spec:
  replicas: 3
  selector:
    matchLabels:
      app: nfsdemo
      version: v1
  template:
    metadata:
      labels:
        app: nfsdemo
        version: v1
    spec:
      containers:
      - name: nfsdemo
        image: 192.168.140.135:5000/nfsdemo:v1
        ports:
        - containerPort: 6085
        volumeMounts:
        - name: nfs-storage
          mountPath: /root/data  # 容器内的挂载路径
      volumes:
      - name: nfs-storage
        persistentVolumeClaim:
          claimName: nfs-pvc
---
apiVersion: v1
kind: Service
metadata:
  name: nfsdemo-service
  namespace: storagedemo
spec:
  type: NodePort  # 设置为 NodePort 类型
  selector:
    app: nfsdemo  # 与 Deployment 中的 Pod 匹配的标签
  ports:
  - port: 6085          # 服务的端口
    targetPort: 6085    # Pod 中容器的端口

使用storageClass动态创建pv
注意：需要安装nfs的provisioner
本例使用provisioner镜像：nfs-subdir-external-provisioner:latest
还需要配置一系列权限 provisioner 需要许多集群权限否则创建不成功pv报错
处理报错的方法查看provisioner的pod日志一个个修改即可
本例yaml配置

#权限设置
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  namespace: storagedemo
  name: endpoint-reader
rules:
- apiGroups: [""]
  resources: ["persistentvolumes"]
  verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
  resources: ["persistentvolumeclaims"]
  verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
  resources: ["endpoints"]
  verbs: ["get", "list", "create", "update", "delete"]  
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: nfs-provisioner-storageclass-role
rules:
- apiGroups: ["storage.k8s.io"]
  resources: ["storageclasses"]
  verbs: ["get", "list", "watch"]
- apiGroups: [""]
  resources: ["persistentvolumes"]
  verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
  resources: ["persistentvolumeclaims"]
  verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
  resources: ["endpoints"]
  verbs: ["get", "list", "create", "update", "delete"]  
- apiGroups: [""]
  resources: ["events"]
  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] 
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: nfs-provisioner-storageclass-binding
subjects:
- kind: ServiceAccount
  name: default
  namespace: storagedemo
roleRef:
  kind: ClusterRole
  name: nfs-provisioner-storageclass-role
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  namespace: storagedemo
  name: endpoint-reader-binding
subjects:
- kind: ServiceAccount
  name: default
  namespace: storagedemo
roleRef:
  kind: Role
  name: endpoint-reader
  apiGroup: rbac.authorization.k8s.io

---

#先创建nfs的provisioner
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nfs-client-provisioner
  namespace: storagedemo
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nfs-client-provisioner
  template:
    metadata:
      labels:
        app: nfs-client-provisioner
    spec:
      containers:
      - name: nfs-client-provisioner
        image: 192.168.140.135:5000/nfs-subdir-external-provisioner:latest
        volumeMounts:
        - name: nfs-client-root
          mountPath: /persistentvolumes
        env:
        - name: PROVISIONER_NAME
          value: "example.com/nfs"  # 存储类名称
        - name: NFS_SERVER
          value: "192.168.140.129"  # NFS 服务器 IP
        - name: NFS_PATH
          value: "/root/data/nfs"        # NFS 共享路径
      volumes:
      - name: nfs-client-root
        nfs:
          server: 192.168.140.129 # NFS 服务器 IP
          path: /root/data/nfs # NFS 共享路径
---
#创建storageClass
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: nfs-storage-class
  namespace: storagedemo
provisioner: example.com/nfs # 存储类名称与provisioner的存储类名称对应
parameters:
  archiveOnDelete: "false"  # 当 PVC 删除时是否保留 PV

---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: nfs-pvc
  namespace: storagedemo
spec:
  storageClassName: nfs-storage-class
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 1Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nfsdemo-v1
  namespace: storagedemo
  labels:
    app: nfsdemo
    version: v1
spec:
  replicas: 3
  selector:
    matchLabels:
      app: nfsdemo
      version: v1
  template:
    metadata:
      labels:
        app: nfsdemo
        version: v1
    spec:
      containers:
      - name: nfsdemo
        image: 192.168.140.135:5000/nfsdemo:v1
        ports:
        - containerPort: 6085
        volumeMounts:
        - name: nfs-storage
          mountPath: /root/data  # 容器内的挂载路径
      volumes:
      - name: nfs-storage
        persistentVolumeClaim:
          claimName: nfs-pvc
---
apiVersion: v1
kind: Service
metadata:
  name: nfsdemo-service
  namespace: storagedemo
spec:
  type: NodePort  # 设置为 NodePort 类型
  selector:
    app: nfsdemo  # 与 Deployment 中的 Pod 匹配的标签
  ports:
  - port: 6085          # 服务的端口
    targetPort: 6085    # Pod 中容器的端口