在k8s中部署一个可外部访问的Redis Sentinel

1.前提条件：

1.部署了multus
想要k8s外部能访问k8s内部的redis，redis-server启动时必须使用multus的IP
2.helm客户端安装

2.开始安装

准备3个multus ip

10.10.10.130 10.10.10.131 10.10.10.132

apiVersion: k8s.cni.cncf.io/v1
kind: NetworkAttachmentDefinition
metadata:
  name: net10-130-132
  namespace: default
spec:
  config: |-
    {
      "cniVersion": "0.3.1",
      "name": "net10-130-132",
      "type": "macvlan",
      "master": "ens224",
      "mode": "bridge",
      "ipam": {
        "type": "whereabouts",
        "range": "10.10.10.0/24", 
        "range_start": "10.10.10.130",
        "range_end": "10.10.10.132",
        "routes": [
          { "dst": "10.206.0.0/16", "gw": "10.10.10.1" }
        ]
      }
    }

通过helm部署redis

helm pull bitnami/redis --version=16.13.2
helm install redis -f values.yaml . 

修改values.yaml

global:
  imageRegistry: ""
  imagePullSecrets: []
  storageClass: "vsan-csi" ##设置storage class
  redis:
    password: "password"  ##设置redis密码
kubeVersion: ""
nameOverride: ""
fullnameOverride: ""
commonLabels: {}
commonAnnotations: {}
secretAnnotations: {}
clusterDomain: cluster.local
extraDeploy: []
diagnosticMode:
  enabled: false
  command:
    - sleep
  args:
    - infinity
image:
  registry: docker.io
  repository: bitnami/redis
  tag: 6.2.7-debian-11-r11
  pullPolicy: IfNotPresent
  pullSecrets: []
  debug: false

architecture: replication
auth:
  enabled: true #开启redis密码
  sentinel: false ##关闭sentinel密码
  password: "password"
  existingSecret: ""
  existingSecretPasswordKey: ""
  usePasswordFiles: false

commonConfiguration: |-
  # Enable AOF https://redis.io/topics/persistence#append-only-file
  appendonly yes
  # Disable RDB persistence, AOF persistence already enabled.
  save ""
existingConfigmap: ""

master:
  count: 1
  configuration: ""
  disableCommands:
    - FLUSHDB
    - FLUSHALL
  command: []
  args: []
  preExecCmds: []
  extraFlags: []
  extraEnvVars: []
  extraEnvVarsCM: ""
  extraEnvVarsSecret: ""
  containerPorts:
    redis: 6379
  startupProbe:
    enabled: false
    initialDelaySeconds: 20
    periodSeconds: 5
    timeoutSeconds: 5
    successThreshold: 1
    failureThreshold: 5
  livenessProbe:
    enabled: true
    initialDelaySeconds: 20
    periodSeconds: 5
    timeoutSeconds: 5
    successThreshold: 1
    failureThreshold: 5
  readinessProbe:
    enabled: true
    initialDelaySeconds: 20
    periodSeconds: 5
    timeoutSeconds: 1
    successThreshold: 1
    failureThreshold: 5
  customStartupProbe: {}
  customLivenessProbe: {}
  customReadinessProbe: {}
  resources: ##设置master资源大小
    limits:
      cpu: 1000m
      memory: 4096MiB
    requests: 
      cpu: 1000m
      memory: 4096MiB
  podSecurityContext:
    enabled: true
    fsGroup: 1001
  containerSecurityContext:
    enabled: true
    runAsUser: 1001
  kind: StatefulSet
  schedulerName: ""
  updateStrategy:
    type: RollingUpdate
    rollingUpdate: {}
  priorityClassName: ""
  hostAliases: []
  podLabels: {}
  podAnnotations: {}
  shareProcessNamespace: false
  podAffinityPreset: ""
  podAntiAffinityPreset: soft
  nodeAffinityPreset:
    type: ""
    key: ""
    values: []
  affinity: {}
  nodeSelector: {}
  tolerations: []
  topologySpreadConstraints: []
  dnsPolicy: ""
  dnsConfig: {}
  lifecycleHooks: {}
  extraVolumes: []
  extraVolumeMounts: []
  sidecars: []
  initContainers: []
  persistence:
    enabled: true
    medium: ""
    sizeLimit: ""
    path: /data
    subPath: ""
    storageClass: "vsan-csi"
    accessModes:
      - ReadWriteOnce
    size: 10Gi
    annotations: {}
    selector: {}
    dataSource: {}
    existingClaim: ""
  service:
    type: ClusterIP
    ports:
      redis: 6379
    nodePorts:
      redis: ""
    externalTrafficPolicy: Cluster
    extraPorts: []
    internalTrafficPolicy: Cluster
    clusterIP: ""
    loadBalancerIP: ""
    loadBalancerSourceRanges: []
    annotations: {}
    sessionAffinity: None
    sessionAffinityConfig: {}
  terminationGracePeriodSeconds: 30

replica:
  replicaCount: 3
  configuration: ""
  disableCommands:
    - FLUSHDB
    - FLUSHALL
  command: []
  args: []
  preExecCmds: []
  extraFlags: []
  extraEnvVars: []
  extraEnvVarsCM: ""
  extraEnvVarsSecret: ""
  externalMaster:
    enabled: false
    host: ""
    port: 6379
  containerPorts:
    redis: 6379
  startupProbe:
    enabled: true
    initialDelaySeconds: 10
    periodSeconds: 10
    timeoutSeconds: 5
    successThreshold: 1
    failureThreshold: 22
  livenessProbe:
    enabled: true
    initialDelaySeconds: 20
    periodSeconds: 5
    timeoutSeconds: 5
    successThreshold: 1
    failureThreshold: 5
  readinessProbe:
    enabled: true
    initialDelaySeconds: 20
    periodSeconds: 5
    timeoutSeconds: 1
    successThreshold: 1
    failureThreshold: 5
  customStartupProbe: {}
  customLivenessProbe: {}
  customReadinessProbe: {}
  resources:
    limits: 
      cpu: 250m
      memory: 256Mi
    requests: 
      cpu: 250m
      memory: 256Mi
  podSecurityContext:
    enabled: true
    fsGroup: 1001
  containerSecurityContext:
    enabled: true
    runAsUser: 1001
  schedulerName: ""
  updateStrategy:
    type: RollingUpdate
    rollingUpdate: {}
  priorityClassName: ""
  podManagementPolicy: ""
  hostAliases: []
  podLabels: {}
  podAnnotations: {}
  shareProcessNamespace: false
  podAffinityPreset: ""
  podAntiAffinityPreset: soft
  nodeAffinityPreset:
    type: ""
    key: ""
    values: []
  affinity: {}
  nodeSelector: {}
  tolerations: []
  topologySpreadConstraints: []
  dnsPolicy: ""
  dnsConfig: {}
  lifecycleHooks: {}
  extraVolumes: []
  extraVolumeMounts: []
  sidecars: []
  initContainers: []
  persistence:
    enabled: true
    medium: ""
    sizeLimit: ""
    path: /data
    subPath: ""
    storageClass: "vsan-csi"
    accessModes:
      - ReadWriteOnce
    size: 10Gi
    annotations: {}
    selector: {}
    dataSource: {}
    existingClaim: ""
  service:
    type: ClusterIP
    ports:
      redis: 6379
    nodePorts:
      redis: ""
    externalTrafficPolicy: Cluster
    internalTrafficPolicy: Cluster
    extraPorts: []
    clusterIP: ""
    loadBalancerIP: ""
    loadBalancerSourceRanges: []
    annotations: {}
    sessionAffinity: None
    sessionAffinityConfig: {}
  terminationGracePeriodSeconds: 30
  autoscaling:
    enabled: false
    minReplicas: 1
    maxReplicas: 11
    targetCPU: ""
    targetMemory: ""

sentinel:
  enabled: true
  image:
    registry: docker.io
    repository: bitnami/redis-sentinel
    tag: 6.2.7-debian-11-r12
    pullPolicy: IfNotPresent
    pullSecrets: []
    debug: false
  masterSet: mymaster
  quorum: 2
  getMasterTimeout: 220
  automateClusterRecovery: false
  downAfterMilliseconds: 60000
  failoverTimeout: 18000
  parallelSyncs: 1
  configuration: ""
  command: []
  args: []
  preExecCmds: []
  extraEnvVars: []
  extraEnvVarsCM: ""
  extraEnvVarsSecret: ""
  externalMaster:
    enabled: false
    host: ""
    port: 6379
  containerPorts:
    sentinel: 26379
  startupProbe:
    enabled: true
    initialDelaySeconds: 10
    periodSeconds: 10
    timeoutSeconds: 5
    successThreshold: 1
    failureThreshold: 22
  livenessProbe:
    enabled: true
    initialDelaySeconds: 20
    periodSeconds: 5
    timeoutSeconds: 5
    successThreshold: 1
    failureThreshold: 5
  readinessProbe:
    enabled: true
    initialDelaySeconds: 20
    periodSeconds: 5
    timeoutSeconds: 1
    successThreshold: 1
    failureThreshold: 5
  customStartupProbe: {}
  customLivenessProbe: {}
  customReadinessProbe: {}
  persistence:
    enabled: false
    storageClass: ""
    accessModes:
      - ReadWriteOnce
    size: 100Mi
    annotations: {}
    selector: {}
    dataSource: {}
    medium: ""
  resources:
    limits: 
      cpu: 250m
      memory: 256Mi
    requests: 
      cpu: 250m
      memory: 256Mi
  containerSecurityContext:
    enabled: true
    runAsUser: 1001
  lifecycleHooks: {}
  extraVolumes: []
  extraVolumeMounts: []
  service:
    type: ClusterIP
    ports:
      redis: 6379
      sentinel: 26379
    nodePorts:
      redis: ""
      sentinel: ""
    externalTrafficPolicy: Cluster
    extraPorts: []
    clusterIP: ""
    loadBalancerIP: ""
    loadBalancerSourceRanges: []
    annotations: {}
    sessionAffinity: None
    sessionAffinityConfig: {}
  terminationGracePeriodSeconds: 30

networkPolicy:
  enabled: false
  allowExternal: true
  extraIngress: []
  extraEgress: []
  ingressNSMatchLabels: {}
  ingressNSPodMatchLabels: {}
podSecurityPolicy:
  create: false
  enabled: false
rbac:
  create: false
  rules: []
serviceAccount:
  create: true
  name: ""
  automountServiceAccountToken: true
  annotations: {}
pdb:
  create: false
  minAvailable: 1
  maxUnavailable: ""
tls:
  enabled: false
  authClients: true
  autoGenerated: false
  existingSecret: ""
  certificatesSecret: ""
  certFilename: ""
  certKeyFilename: ""
  certCAFilename: ""
  dhParamsFilename: ""

metrics:
  enabled: true
  image:
    registry: docker.io
    repository: bitnami/redis-exporter
    tag: 1.43.0-debian-11-r4
    pullPolicy: IfNotPresent
    pullSecrets: []
  command: []
  redisTargetHost: "localhost"
  extraArgs: {}
  extraEnvVars: []
  containerSecurityContext:
    enabled: true
    runAsUser: 1001
  extraVolumes: []
  extraVolumeMounts: []
  resources:
    limits: 
      cpu: 250m
      memory: 256Mi
    requests: 
      cpu: 250m
      memory: 256Mi
  podLabels: {}
  podAnnotations:
    prometheus.io/scrape: "true"
    prometheus.io/port: "9121"
  service:
    type: ClusterIP
    port: 9121
    externalTrafficPolicy: Cluster
    extraPorts: []
    loadBalancerIP: ""
    loadBalancerSourceRanges: []
    annotations: {}
  serviceMonitor:
    enabled: true
    namespace: ""
    interval: 30s
    scrapeTimeout: ""
    relabellings: []
    metricRelabelings: []
    honorLabels: false
    additionalLabels: {}
  prometheusRule:
    enabled: true
    namespace: ""
    additionalLabels: {}
    rules:
      - alert: RedisDown
        expr: redis_up{service="{{ template "common.names.fullname" . }}-metrics"} == 0
        for: 2m
        labels:
          severity: error
        annotations:
          summary: Redis® instance {{ "{{ $labels.instance }}" }} down
          description: Redis® instance {{ "{{ $labels.instance }}" }} is down
      - alert: RedisMemoryHigh
        expr: >
          redis_memory_used_bytes{service="{{ template "common.names.fullname" . }}-metrics"} * 100
          /
          redis_memory_max_bytes{service="{{ template "common.names.fullname" . }}-metrics"}
          > 90
        for: 2m
        labels:
          severity: error
        annotations:
          summary: Redis® instance {{ "{{ $labels.instance }}" }} is using too much memory
          description: |
            Redis® instance {{ "{{ $labels.instance }}" }} is using {{ "{{ $value }}" }}% of its available memory.
      - alert: RedisKeyEviction
        expr: |
          increase(redis_evicted_keys_total{service="{{ template "common.names.fullname" . }}-metrics"}[5m]) > 0
        for: 1s
        labels:
          severity: error
        annotations:
          summary: Redis® instance {{ "{{ $labels.instance }}" }} has evicted keys
          description: |
            Redis® instance {{ "{{ $labels.instance }}" }} has evicted {{ "{{ $value }}" }} keys in the last 5 minutes.
volumePermissions:
  enabled: false
  image:
    registry: docker.io
    repository: bitnami/bitnami-shell
    tag: 11-debian-11-r11
    pullPolicy: IfNotPresent
    pullSecrets: []
  resources:
    limits: {}
    requests: {}
  containerSecurityContext:
    runAsUser: 0

sysctl:
  enabled: false
  image:
    registry: docker.io
    repository: bitnami/bitnami-shell
    tag: 11-debian-11-r11
    pullPolicy: IfNotPresent
    pullSecrets: []
  command: []
  mountHostSys: false
  resources:
    limits: {}
    requests: {}

useExternalDNS:
  enabled: false
  suffix: ""
  annotationKey: external-dns.alpha.kubernetes.io/
  additionalAnnotations: {}

修改configMap redis-scripts
将

get_full_hostname() {
    hostname="$1"
    echo "${hostname}.${HEADLESS_SERVICE}"
}

替换为

get_full_hostname() {
    hostname="$1"
    if [[ "${hostname}" =~ 0$ ]]; then
        echo "10.10.10.130"
    fi
    if [[ "${hostname}" =~ 1$ ]]; then
        echo "10.10.10.131"
    fi
    if [[ "${hostname}" =~ 2$ ]]; then
        echo "10.10.10.132"
    fi
}

然后将statefulset的replica从3改为0，

spec:
  template:
    metadata:
      annotations:
        k8s.v1.cni.cncf.io/networks: default/net10-130-132

再从0改为3。

3.测试连接：

测试工具 Another Redis Desktop Manager


连接OK