k8s1.30.0高可用集群部署

负载均衡

nginx负载均衡

两台nginx负载均衡
vim /etc/nginx/nginx.conf

stream {
    upstream kube-apiserver {
        server 192.168.0.11:6443     max_fails=3 fail_timeout=30s;
        #server 192.168.0.12:6443     max_fails=3 fail_timeout=30s;
        #server 192.168.0.13:6443     max_fails=3 fail_timeout=30s;
    }
    server {
        listen 6443;
        proxy_connect_timeout 2s;
        proxy_timeout 900s;
        proxy_pass kube-apiserver;
    }
}

keepalived

nginx检测脚本
vim /data/shell/check_nginx_status.sh

#!/bin/bash
nginx_status=$(ps -ef | grep nginx | grep -v grep | grep -v check | wc -l)
 
if [ $nginx_status -eq 0 ];then
  systemctl stop keepalived.service
fi

master节点keepalived配置（不抢占机制）
vim /etc/keepalived/keepalived.conf

global_defs {
    router_id real-server1
 }
 
vrrp_script chk_nginx {
    script "/data/shell/check_nginx_status.sh"
    interval 2
}
 
vrrp_instance VI_1 {
     state BACKUP
     interface ens32
     virtual_router_id 151
     priority 100
     nopreempt
     advert_int 5
     authentication {
         auth_type  PASS
         auth_pass  1111
     }
     virtual_ipaddress {
       192.168.0.10/24
     }
    
      track_script {                                                                                  
       chk_nginx
    }
 }

backup节点keepalived配置

vim /etc/keepalived/keepalived.conf

global_defs {
    router_id real-server2
 }
 
vrrp_script chk_nginx {
    script "/data/shell/check_nginx_status.sh"
    interval 2
}
 
vrrp_instance VI_1 {
     state BACKUP
     interface ens32
     virtual_router_id 151
     priority 50
     nopreempt
     advert_int 5
     authentication {
         auth_type  PASS
         auth_pass  1111
     }
     virtual_ipaddress {
       192.168.0.10/24
     }
    
      track_script {                                                                                  
       chk_nginx
    }
 }

k8s节点系统设置

vim /etc/modules-load.d/containerd.conf

overlay
br_netfilter

modprobe overlay
modprobe br_netfilter

vim /etc/sysctl.d/k8s.conf

net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
vm.swappiness=0
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720

sysctl --system

ipvsadm

yum install ipset ipvsadm

modprobe br_netfilter
modprobe overlay
modprobe ip_conntrack
modprobe  ip_vs
modprobe  ip_vs_rr
modprobe  ip_vs_wrr
modprobe  ip_vs_sh
modprobe  nf_conntrack
lsmod | grep conntrack
lsmod | grep br_netfilt
lsmod | grep overlay
lsmod |egrep  "ip_vs|nf_conntrack"

cat > /etc/modules-load.d/kubernetes.conf << EOF
# /etc/modules-load.d/kubernetes.conf
br_netfilter
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack_ipv4
ip_tables
EOF

chmod a+x /etc/modules-load.d/kubernetes.conf

containerd

wget https://github.com/containerd/containerd/releases/download/v1.7.23/cri-containerd-cni-1.7.23-linux-amd64.tar.gz
 
tar xvf cri-containerd-cni-1.7.23-linux-amd64.tar.gz -C /
 
wget https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.amd64
cp runc.amd64 /usr/local/sbin/runc

mkdir -p /etc/containerd
containerd config default > /etc/containerd/config.toml

vim /etc/containerd/config.toml

...
SystemdCgroup = true
...
sandbox_image = "registry.cn-beijing.aliyuncs.com/wuxingge123/pause:3.9"
...
      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
          endpoint = ["https://p4oudlho.mirror.aliyuncs.com"]

配置私有仓库

[plugins."io.containerd.grpc.v1.cri".registry]
  [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
    [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
      endpoint = ["https://bqr1dr1n.mirror.aliyuncs.com"]
  [plugins."io.containerd.grpc.v1.cri".registry.configs]
    [plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.k8s.local".tls]
      insecure_skip_verify = true
    [plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.k8s.local".auth]
      username = "admin"
      password = "Harbor12345"

启动containerd

systemctl start containerd.service
systemctl enable containerd.service

k8s

安装

https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.30/rpm/x86_64/

vim /etc/yum.repos.d/k8s.repo

[kubernetes]
name=kubernetes
baseurl=https://pkgs.k8s.io/core:/stable:/v1.30/rpm/
enabled=1
gpgcheck=0

在线安装

yum install kubelet-1.30.0 kubectl-1.30.0 kubeadm-1.30.0

离线安装

yum localinstall kubernetes-cni-1.4.0-150500.1.1.x86_64.rpm 
yum localinstall cri-tools-1.30.0-150500.1.1.x86_64.rpm
yum localinstall kubeadm-1.30.0-150500.1.1.x86_64.rpm kubectl-1.30.0-150500.1.1.x86_64.rpm kubelet-1.30.0-150500.1.1.x86_64.rpm

配置kubelet

vim /etc/sysconfig/kubelet

KUBELET_EXTRA_ARGS="--cgroup-driver=systemd"

systemctl enable kubelet

下载镜像

kubeadm config images list --kubernetes-version=v1.30.0 --image-repository registry.cn-beijing.aliyuncs.com/wuxingge123

kubeadm config images pull --kubernetes-version=v1.30.0 --image-repository registry.cn-beijing.aliyuncs.com/wuxingge123

calico准备

wget https://docs.projectcalico.org/v3.25/manifests/calico.yaml --no-check-certificate

vim calico.yaml

            - name: CALICO_IPV4POOL_CIDR
              value: "10.224.0.0/16"

下载calico镜像

ctr -n k8s.io image pull --all-platforms registry.cn-beijing.aliyuncs.com/wuxingge123/cni:v3.25.0

ctr -n k8s.io image pull --all-platforms registry.cn-beijing.aliyuncs.com/wuxingge123/node:v3.25.0

ctr -n k8s.io image pull --all-platforms registry.cn-beijing.aliyuncs.com/wuxingge123/kube-controllers:v3.25.0

k8s init

生成初始化配置文件

kubeadm config print init-defaults > kubeadm-config.yaml

vim kubeadm-config.yaml

apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 192.168.0.11
  bindPort: 6443
nodeRegistration:
  criSocket: unix:///var/run/containerd/containerd.sock
  imagePullPolicy: IfNotPresent
  taints: null
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: 192.168.0.10:6443
controllerManager: {}
dns: {}
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: swr.cn-jl-1.manageone.cloud.cnpc/gsms-project
kind: ClusterConfiguration
kubernetesVersion: 1.30.0
networking:
  dnsDomain: cluster.local
  podSubnet: 10.224.0.0/16
  serviceSubnet: 10.96.0.0/12
scheduler: {}

初始化集群

kubeadm init --config kubeadm-config.yaml --upload-certs

初始化成功显示以下内容

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of the control-plane node running the following command on each as root:

  kubeadm join 192.168.0.10:6443 --token abcdef.0123456789abcdef \
        --discovery-token-ca-cert-hash sha256:aac70b668d8010aca7af9a27ad9451468fc985dfd8f52a52025d14b180e16464 \
        --control-plane --certificate-key 8ea5c7b87d52438496fce053b1e9788217ffe74ed397d89f6a114e65d06c8826

Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 192.168.0.10:6443 --token abcdef.0123456789abcdef \
        --discovery-token-ca-cert-hash sha256:aac70b668d8010aca7af9a27ad9451468fc985dfd8f52a52025d14b180e16464

配置kubectl客户端

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

设置kubectl命令补全

echo "source <(kubectl completion bash)" >> /etc/profile

部署calico

kubectl apply -f calico.yaml

添加master节点

  kubeadm join 192.168.0.10:6443 --token abcdef.0123456789abcdef \
        --discovery-token-ca-cert-hash sha256:aac70b668d8010aca7af9a27ad9451468fc985dfd8f52a52025d14b180e16464 \
        --control-plane --certificate-key 8ea5c7b87d52438496fce053b1e9788217ffe74ed397d89f6a114e65d06c8826

添加node节点

kubeadm join 192.168.0.10:6443 --token abcdef.0123456789abcdef \
        --discovery-token-ca-cert-hash sha256:aac70b668d8010aca7af9a27ad9451468fc985dfd8f52a52025d14b180e16464

ingress-nginx v1.11.2

kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.11.2/deploy/static/provider/cloud/deploy.yaml

metrics-server

wget https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml