Ceph集群2025（Squid版）快速对接K8S cephFS文件存储

ceph的块存储太简单了。所以不做演示

查看集群

创建一个 CephFS 文件系统
# ceph fs volume create cephfs01
需要创建一个子卷

# ceph fs subvolume create cephfs01 my-subvol
-----------------

#以下全部自动创建好
# ceph fs ls       
name: cephfs01, metadata pool: cephfs.cephfs01.meta, data pools: [cephfs.cephfs01.data ]

# ceph osd pool ls
.mgr
cephfs.cephfs01.meta
cephfs.cephfs01.data

最后配置K8S yaml即可

helm repo add ceph-csi https://ceph.github.io/csi-charts
helm install -n ceph-csi ceph-csi-cephfs ceph-csi-cephfs-3.13.1.tgz -f values.yaml

以下我的yaml参考

# egrep -v "^[[:space:]]*#|^$" values.yaml
---
rbac:
  create: true
  leastPrivileges: true
serviceAccounts:
  nodeplugin:
    create: true
    name:
  provisioner:
    create: true
    name:
csiConfig:
   - clusterID: "51edffbe-03fd-11f0-8abe-000c29f6d983"
     monitors:
       - "172.16.8.100:6789"
       - "172.16.8.102:6789"
       - "172.16.8.103:6789"
     cephFS:
       subvolumeGroup: "myfsg"  ----你的子卷
encryptionKMSConfig: {}
commonLabels: {}
logLevel: 5
sidecarLogLevel: 1
logSlowOperationInterval: 30s
CSIDriver:
  fsGroupPolicy: "File"
  seLinuxMount: false
nodeplugin:
  name: nodeplugin
  updateStrategy: RollingUpdate
  priorityClassName: system-node-critical
  httpMetrics:
    enabled: true
    containerPort: 8081
    service:
      enabled: true
      servicePort: 8080
      type: ClusterIP
      annotations: {}
      clusterIP: ""
      externalIPs: []
      loadBalancerIP: ""
      loadBalancerSourceRanges: []
  imagePullSecrets: []
  profiling:
    enabled: false
  registrar:
    image:
      repository: registry.aliyuncs.com/google_containers/csi-node-driver-registrar
      tag: v2.13.0
      pullPolicy: IfNotPresent
    resources: {}
  plugin:
    image:
      repository: quay.io/cephcsi/cephcsi
      tag: canary
      pullPolicy: IfNotPresent
    resources: {}
  nodeSelector: {}
  tolerations: []
  affinity: {}
  annotations: {}
  podSecurityContext: {}
  podAnnotations: {}
  kernelmountoptions: ""
  fusemountoptions: ""
provisioner:
  name: provisioner
  replicaCount: 1
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 50%
  timeout: 60s
  priorityClassName: system-cluster-critical
  enableHostNetwork: false
  httpMetrics:
    enabled: true
    containerPort: 8081
    service:
      enabled: true
      servicePort: 8080
      type: ClusterIP
      annotations: {}
      clusterIP: ""
      externalIPs: []
      loadBalancerIP: ""
      loadBalancerSourceRanges: []
  imagePullSecrets: []
  profiling:
    enabled: false
  provisioner:
    image:
      repository: registry.aliyuncs.com/google_containers/csi-provisioner
      tag: v5.1.0
      pullPolicy: IfNotPresent
    resources: {}
    args:
    extraArgs: []
  setmetadata: true
  resizer:
    name: resizer
    enabled: true
    image:
      repository: registry.aliyuncs.com/google_containers/csi-resizer
      tag: v1.13.1
      pullPolicy: IfNotPresent
    resources: {}
    args:
    extraArgs: []
  snapshotter:
    image:
      repository: registry.aliyuncs.com/google_containers/csi-snapshotter
      tag: v8.2.0
      pullPolicy: IfNotPresent
    resources: {}
    args:
      enableVolumeGroupSnapshots: false
    extraArgs: []
  nodeSelector: {}
  tolerations: []
  affinity: {}
  annotations: {}
  podSecurityContext: {}
  podAnnotations: {}
selinuxMount: false
storageClass:
  create: true
  name: csi-cephfs-sc
  annotations: {}
  clusterID: 51edffbe-03fd-11f0-8abe-000c29f6d983
  fsName: cephfs01   ----------------------你的名字
  pool: "cephfs.cephfs01.data"  
  fuseMountOptions: ""
  kernelMountOptions: ""
  mounter: ""
  volumeNamePrefix: ""
  encrypted: ""
  encryptionKMSID: ""
  provisionerSecret: csi-cephfs-secret
  provisionerSecretNamespace: ""
  controllerExpandSecret: csi-cephfs-secret
  controllerExpandSecretNamespace: ""
  nodeStageSecret: csi-cephfs-secret
  nodeStageSecretNamespace: ""
  reclaimPolicy: Delete
  allowVolumeExpansion: true
  mountOptions:
     - _netdev  ------------------重点在这里cephfs必须使用这个参数 块存储 设置为discard 启用 TRIM 功能 文件存储中无效或冗余，可能导致挂载参数冲突（如 CephFS 报错 Invalid argument）
secret:
  create: true
  name: csi-cephfs-secret
  annotations: {}
  adminID: admin  ###----------这里建议全部使用admin
  adminKey: AQDDddln7+UvHRAAwZVyl1UEJOpY3fWG2Oev+A==
  userID: "admin"
  userKey: "AQDDddln7+UvHRAAwZVyl1UEJOpY3fWG2Oev+A=="
cephconf: |
  [global]
    auth_cluster_required = cephx
    auth_service_required = cephx
    auth_client_required = cephx
extraDeploy: []
provisionerSocketFile: csi-provisioner.sock
pluginSocketFile: csi.sock
kubeletDir: /var/lib/kubelet
driverName: cephfs.csi.ceph.com
configMapName: ceph-csi-config
externallyManagedConfigmap: false
cephConfConfigMapName: ceph-config
kmsConfigMapName: ceph-csi-encryption-kms-config

下面测试

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: csi-cephfs-pvc
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 1Gi
  storageClassName: csi-cephfs-sc
---
apiVersion: v1
kind: Pod
metadata:
  name: fs-pod
spec:
  containers:
    - name: web-server
      image: docker.1ms.run/nginx:alpine
      volumeMounts:
        - name: mypvc
          mountPath: /usr/share/nginx/html
  volumes:
    - name: mypvc
      persistentVolumeClaim:
        claimName: csi-cephfs-pvc
        readOnly: false

默认是开启内核挂载模式的