kubeadm部署k8s1.25.3一主二从集群(Containerd)
第一章：K8S集群部署
kubernetes集群规划
| 主机IP | 主机名 | 主机配置 | 角色 |
|---|---|---|---|
| 10.0.0.3 | master1 | 2C/4G | 管理节点 |
| 10.0.0.4 | node1 | 2C/4G | 工作节点 |
| 10.0.0.5 | node2 | 2C/4G | 工作节点 |
集群前期环境准备
#!/bin/bash
echo "——>>> 关闭防火墙与SELinux <<<——"
sleep 3
systemctl disable firewalld --now &> /dev/null
setenforce 0
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
echo "——>>> 创建阿里仓库 <<<——"
sleep 3
mv /etc/yum.repos.d/* /tmp
curl -o /etc/yum.repos.d/centos.repo https://mirrors.aliyun.com/repo/Centos-7.repo
curl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
echo "——>>> 设置时区并同步时间 <<<——"
sleep 3
timedatectl set-timezone Asia/Shanghai
yum install -y chrony
systemctl enable chronyd --now &> /dev/null
sed -i '/^server/s/^/# /' /etc/chrony.conf
sed -i '/^# server 3.centos.pool.ntp.org iburst/a\server ntp1.aliyun.com iburst\nserver ntp2.aliyun.com iburst\nserver ntp3.aliyun.com iburst' /etc/chrony.conf
systemctl restart chronyd
chronyc sources
echo "——>>> 设置系统最大打开文件数 <<<——"
sleep 3
if ! grep "* soft nofile 65535" /etc/security/limits.conf &>/dev/null; then
cat >> /etc/security/limits.conf << EOF
* soft nofile 65535 # 软限制
* hard nofile 65535 # 硬限制
EOF
fi
echo "——>>> 系统内核优化 <<<——"
sleep 3
cat >> /etc/sysctl.conf << EOF
net.ipv4.tcp_syncookies = 1 # 防范SYN洪水攻击，0为关闭
net.ipv4.tcp_max_tw_buckets = 20480 # 此项参数可以控制TIME_WAIT套接字的最大数量，避免Squid服务器被大量的TIME_WAIT套接字拖死
net.ipv4.tcp_max_syn_backlog = 20480 # 表示SYN队列的长度，默认为1024，加大队列长度为8192，可以容纳更多等待连接的网络连接数
net.core.netdev_max_backlog = 262144 # 每个网络接口 接受数据包的速率比内核处理这些包的速率快时，允许发送到队列的数据包的最大数目
net.ipv4.tcp_fin_timeout = 20 # FIN-WAIT-2状态的超时时间，避免内核崩溃
EOF
echo "——>>> 减少SWAP使用 <<<——"
sleep 3
echo "0" > /proc/sys/vm/swappiness
echo "——>>> 安装系统性能分析工具及其他 <<<——"
sleep 3
yum install -y vim net-tools lsof wget lrzsz
Docker环境安装
# 解压离线安装包
[root@master1 ~]# tar xf docker-ce-24.0.6.tar.gz
# 配置本地yum源
[root@master1 ~]# cat > /etc/yum.repos.d/local.repo << EOF
[local]
name=local
baseurl=file:///root/docker-ce-24.0.6
gpgcheck=0
EOF
# 查看docker源
[root@master1 ~]# yum list docker-ce docker-ce-cli containerd.io
...
Available Packages
containerd.io.x86_64 1.6.33-3.1.el7 local
docker-ce.x86_64 3:24.0.6-1.el7 local
docker-ce-cli.x86_64 1:24.0.6-1.el7 local
# 安装Docker
[root@master1 ~]# yum install -y yum-utils device-mapper-persistent-data lvm2
[root@master1 ~]# yum install -y docker-ce docker-ce-cli containerd.io
# 设置Docker开机自启
[root@master1 ~]# systemctl enable docker --now
# 查看Docker版本
[root@master1 ~]# docker --version
Docker version 24.0.6, build ed223bc
配置镜像加速
镜像加速站不稳定变换频繁可以尝试做个代理Centos7配置代理安装最新版Docker并拉取镜像
sudo mkdir -p /etc/docker
sudo tee /etc/docker/daemon.json << 'EOF'
{
"exec-opts": ["native.cgroupdriver=systemd"],
"registry-mirrors": [
"https://docker.m.daocloud.io",
"https://reg-mirror.qiniu.com",
"https://k8s.m.daocloud.io",
"https://elastic.m.daocloud.io",
"https://gcr.m.daocloud.io",
"https://ghcr.m.daocloud.io",
"https://k8s-gcr.m.daocloud.io",
"https://mcr.m.daocloud.io",
"https://nvcr.m.daocloud.io",
"https://quay.m.daocloud.io",
"https://jujucharms.m.daocloud.io",
"https://rocks-canonical.m.daocloud.io",
"https://d3p1s1ji.mirror.aliyuncs.com"
]
}
EOF
sudo systemctl daemon-reload
sudo systemctl restart docker
安装Docker Compose
# 下载二进制文件
[root@master1 ~]# sudo curl -L "https://github.com/docker/compose/releases/download/v2.10.2/docker-compose-linux-x86_64" -o /usr/local/bin/docker-compose
# 给予可执行权限
[root@master1 ~]# sudo chmod +x /usr/local/bin/docker-compose
# 验证安装
[root@master1 ~]# docker-compose --version
Docker Compose version v2.10.2
Containerd环境安装
配置containerd
[root@master1 ~]# tar Czxvf /usr/local/ containerd-1.6.2-linux-amd64.tar.gz
bin/
bin/containerd-shim-runc-v2
bin/containerd-shim
bin/ctr
bin/containerd-shim-runc-v1
bin/containerd
bin/containerd-stress
配置启动项
[root@master1 ~]# cat > /etc/systemd/system/containerd.service << EOF
# Copyright The containerd Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target local-fs.target
[Service]
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/containerd
Type=notify
Delegate=yes
KillMode=process
Restart=always
RestartSec=5
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNPROC=infinity
LimitCORE=infinity
# Comment TasksMax if your systemd version does not supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
OOMScoreAdjust=-999
[Install]
WantedBy=multi-user.target
EOF
启动服务
# 重新加载 systemd 守护进程，以便使配置生效
[root@master1 ~]# systemctl daemon-reload
# 设置containerd开机自启
[root@master1 ~]# systemctl enable containerd --now
配置runc
[root@master1 ~]# install -m 755 runc.amd64 /usr/local/sbin/runc
# 查看权限
[root@master1 ~]# ll /usr/local/sbin/runc
-rwxr-xr-x. 1 root root 10802720 Jul 7 13:33 /usr/local/sbin/runc
配置cni
# 创建目录
[root@master1 ~]# mkdir -p /opt/cni/bin
# 解压二进制包
[root@master1 ~]# tar Czxvf /opt/cni/bin/ cni-plugins-linux-amd64-v1.1.1.tgz
./
./macvlan
./static
./vlan
./portmap
./host-local
./vrf
./bridge
./tuning
./firewall
./host-device
./sbr
./loopback
./dhcp
./ptp
./ipvlan
./bandwidth
# 把cni命令ln到/usr/local/bin目录下
[root@master1 ~]# ln -s /opt/cni/bin/* /usr/local/bin
生成containerd配置文件
[root@master1 ~]# mkdir -p /etc/containerd
[root@master1 ~]# containerd config default > $HOME/config.toml
[root@master1 ~]# cp $HOME/config.toml /etc/containerd/config.toml
配置镜像加速
# 修改 /etc/containerd/config.toml 文件
[root@master1 ~]# sudo sed -i 's#registry.k8s.io/pause:3.8#registry.aliyuncs.com/google_containers/pause:3.8#g' /etc/containerd/config.toml
# 确保 /etc/containerd/config.toml 中的 disabled_plugins 内不存在 cri
[root@master1 ~]# sudo sed -i "s#SystemdCgroup = false#SystemdCgroup = true#g" /etc/containerd/config.toml
# 修改145行为 config_path = "/etc/containerd/certs.d"
[root@master1 ~]# sudo sed -i 's#config_path = ""#config_path = "/etc/containerd/certs.d"#' /etc/containerd/config.toml
# docker hub镜像加速
mkdir -p /etc/containerd/certs.d/docker.io
tee > /etc/containerd/certs.d/docker.io/hosts.toml << 'EOF'
server = "https://docker.io"
[host."https://d3p1s1ji.mirror.aliyuncs.com"]
capabilities = ["pull", "resolve"]
[host."https://docker.m.daocloud.io"]
capabilities = ["pull", "resolve"]
[host."https://reg-mirror.qiniu.com"]
capabilities = ["pull", "resolve"]
EOF
# registry.k8s.io镜像加速
mkdir -p /etc/containerd/certs.d/registry.k8s.io
tee /etc/containerd/certs.d/registry.k8s.io/hosts.toml << 'EOF'
server = "https://registry.k8s.io"
[host."https://k8s.m.daocloud.io"]
capabilities = ["pull", "resolve", "push"]
EOF
# docker.elastic.co镜像加速
mkdir -p /etc/containerd/certs.d/docker.elastic.co
tee /etc/containerd/certs.d/docker.elastic.co/hosts.toml << 'EOF'
server = "https://docker.elastic.co"
[host."https://elastic.m.daocloud.io"]
capabilities = ["pull", "resolve", "push"]
EOF
# gcr.io镜像加速
mkdir -p /etc/containerd/certs.d/gcr.io
tee /etc/containerd/certs.d/gcr.io/hosts.toml << 'EOF'
server = "https://gcr.io"
[host."https://gcr.m.daocloud.io"]
capabilities = ["pull", "resolve", "push"]
EOF
# ghcr.io镜像加速
mkdir -p /etc/containerd/certs.d/ghcr.io
tee /etc/containerd/certs.d/ghcr.io/hosts.toml << 'EOF'
server = "https://ghcr.io"
[host."https://ghcr.m.daocloud.io"]
capabilities = ["pull", "resolve", "push"]
EOF
# k8s.gcr.io镜像加速
mkdir -p /etc/containerd/certs.d/k8s.gcr.io
tee /etc/containerd/certs.d/k8s.gcr.io/hosts.toml << 'EOF'
server = "https://k8s.gcr.io"
[host."https://k8s-gcr.m.daocloud.io"]
capabilities = ["pull", "resolve", "push"]
EOF
# mcr.m.daocloud.io镜像加速
mkdir -p /etc/containerd/certs.d/mcr.microsoft.com
tee /etc/containerd/certs.d/mcr.microsoft.com/hosts.toml << 'EOF'
server = "https://mcr.microsoft.com"
[host."https://mcr.m.daocloud.io"]
capabilities = ["pull", "resolve", "push"]
EOF
# nvcr.io镜像加速
mkdir -p /etc/containerd/certs.d/nvcr.io
tee /etc/containerd/certs.d/nvcr.io/hosts.toml << 'EOF'
server = "https://nvcr.io"
[host."https://nvcr.m.daocloud.io"]
capabilities = ["pull", "resolve", "push"]
EOF
# quay.io镜像加速
mkdir -p /etc/containerd/certs.d/quay.io
tee /etc/containerd/certs.d/quay.io/hosts.toml << 'EOF'
server = "https://quay.io"
[host."https://quay.m.daocloud.io"]
capabilities = ["pull", "resolve", "push"]
EOF
# registry.jujucharms.com镜像加速
mkdir -p /etc/containerd/certs.d/registry.jujucharms.com
tee /etc/containerd/certs.d/registry.jujucharms.com/hosts.toml << 'EOF'
server = "https://registry.jujucharms.com"
[host."https://jujucharms.m.daocloud.io"]
capabilities = ["pull", "resolve", "push"]
EOF
# rocks.canonical.com镜像加速
mkdir -p /etc/containerd/certs.d/rocks.canonical.com
tee /etc/containerd/certs.d/rocks.canonical.com/hosts.toml << 'EOF'
server = "https://rocks.canonical.com"
[host."https://rocks-canonical.m.daocloud.io"]
capabilities = ["pull", "resolve", "push"]
EOF
重启服务
[root@master1 ~]# systemctl daemon-reload
[root@master1 ~]# systemctl restart containerd
ctr拉取镜像测试
# ctr拉取镜像
[root@master1 ~]# ctr image pull --hosts-dir=/etc/containerd/certs.d docker.io/library/nginx:latest
docker.io/library/nginx:latest: resolving |--------------------------------------|
elapsed: 20.9s total: 0.0 B (0.0 B/s)
docker.io/library/nginx:latest: resolved |++++++++++++++++++++++++++++++++++++++|
index-sha256:447a8665cc1dab95b1ca778e162215839ccbb9189104c79d7ec3a81e14577add: exists |++++++++++++++++++++++++++++++++++++++|
manifest-sha256:5f0574409b3add89581b96c68afe9e9c7b284651c3a974b6e8bac46bf95e6b7f: exists |++++++++++++++++++++++++++++++++++++++|
layer-sha256:23fa5a7b99a685258885918c468ded042b95b5a7c56cee758a689f4f7e5971e0: exists |++++++++++++++++++++++++++++++++++++++|
config-sha256:5ef79149e0ec84a7a9f9284c3f91aa3c20608f8391f5445eabe92ef07dbda03c: exists |++++++++++++++++++++++++++++++++++++++|
layer-sha256:e4fff0779e6ddd22366469f08626c3ab1884b5cbe1719b26da238c95f247b305: exists |++++++++++++++++++++++++++++++++++++++|
layer-sha256:2a0cb278fd9f7737ef5ddc52b4198821dd02e87ed204f74d7e491016b96ebe7f: exists |++++++++++++++++++++++++++++++++++++++|
layer-sha256:7045d6c32ae2d3dc002f33beb0c1cdd7f69b2663a9720117ac9b82ec28865e30: exists |++++++++++++++++++++++++++++++++++++++|
layer-sha256:03de31afb03573e0fa679d6777ba3267c2b8ec087cbc0efa46524c1de08f43ec: exists |++++++++++++++++++++++++++++++++++++++|
layer-sha256:0f17be8dcff2e2c27ee6a33c1bacc582e71f76f855c2d69d510f2a93df897303: exists |++++++++++++++++++++++++++++++++++++++|
layer-sha256:14b7e5e8f3946da0f9120dab3b0e05ef24a5ca874ba484327db8b3308a92b532: exists |++++++++++++++++++++++++++++++++++++++|
elapsed: 22.5s total: 0.0 B (0.0 B/s)
unpacking linux/amd64 sha256:447a8665cc1dab95b1ca778e162215839ccbb9189104c79d7ec3a81e14577add...
done: 8.365106ms
# 查看镜像
[root@master1 ~]# ctr i ls
REF TYPE DIGEST SIZE PLATFORMS LABELS
docker.io/library/nginx:latest application/vnd.oci.image.index.v1+json sha256:447a8665cc1dab95b1ca778e162215839ccbb9189104c79d7ec3a81e14577add 67.7 MiB linux/386,linux/amd64,linux/arm/v5,linux/arm/v7,linux/arm64/v8,linux/mips64le,linux/ppc64le,linux/s390x,unknown/unknown -
配置crictl
# 安装工具
[root@master1 ~]# tar xf crictl-v1.25.0-linux-amd64.tar.gz -C /usr/local/bin/
# 生成配置文件
[root@master1 ~]# cat > /etc/crictl.yaml << EOF
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false
EOF
- runtime-endpoint # 指定了容器运行时的sock文件位置
- image-endpoint # 指定了容器镜像使用的sock文件位置
- timeout # 容器运行时或容器镜像服务之间的通信超时时间
- debug # 指定了crictl工具的调试模式,false表示调试模式未启用,true则会在输出中包含更多的调试日志信息，有助于故障排除和问题调试
查看配置是否生效
[root@master1 ~]# crictl info
使用 crictl 拉取测试测试
[root@master1 ~]# crictl pull docker.io/library/nginx:1.20.2
DEBU[0000] get image connection
DEBU[0000] PullImageRequest: &PullImageRequest{Image:&ImageSpec{Image:docker.io/library/nginx:1.20.2,Annotations:map[string]string{},},Auth:nil,SandboxConfig:nil,}
DEBU[0046] PullImageResponse: &PullImageResponse{ImageRef:sha256:0584b370e957bf9d09e10f424859a02ab0fda255103f75b3f8c7d410a4e96ed5,}
Image is up to date for sha256:0584b370e957bf9d09e10f424859a02ab0fda255103f75b3f8c7d410a4e96ed5
# 查看拉取的结果
[root@master1 ~]# crictl images
DEBU[0000] get image connection
DEBU[0000] ListImagesRequest: &ListImagesRequest{Filter:&ImageFilter{Image:&ImageSpec{Image:,Annotations:map[string]string{},},},}
DEBU[0000] ListImagesResponse: &ListImagesResponse{Images:[]*Image{&Image{Id:sha256:0584b370e957bf9d09e10f424859a02ab0fda255103f75b3f8c7d410a4e96ed5,RepoTags:[docker.io/library/nginx:1.20.2],RepoDigests:[docker.io/library/nginx@sha256:38f8c1d9613f3f42e7969c3b1dd5c3277e635d4576713e6453c6193e66270a6d],Size_:56732885,Uid:nil,Username:,Spec:nil,Pinned:false,},},}
IMAGE TAG IMAGE ID SIZE
docker.io/library/nginx 1.20.2 0584b370e957b 56.7MB
配置nerdctl
[root@node1 ~]# tar xf nerdctl-0.21.0-linux-amd64.tar.gz -C /usr/local/bin/
[root@node1 ~]# nerdctl --version
nerdctl version 0.21.0
优化nerdctl
[root@node1 ~]# mkdir -p /etc/nerdctl
[root@node1 ~]# cat > /etc/nerdctl/nerdctl.toml << EOF
namespace = "k8s.io"
insecure_registry = true
cni_path = "/opt/cni/bin/"
EOF
为了测试 insecure_registry = true 设置，需要配置并运行一个不使用 TLS 的镜像仓库。我们可以使用 Docker Registry 镜像来创建一个本地不安全的镜像仓库。
使用 Docker Registry 镜像创建不安全的本地仓库
启动一个不安全的本地 Docker Registry
你可以使用 docker run 命令启动一个不使用 TLS 的本地镜像仓库：
docker run -d -p 5000:5000 --name registry --restart=always registry:2
配置 nerdctl 使用不安全的仓库
namespace = "k8s.io"
insecure_registry = true
cni_path = "/data/kube/bin"
使用 nerdctl 推送和拉取镜像以测试连接
# 拉取一个测试镜像
nerdctl pull nginx:1.20.2
# 给镜像打标签
nerdctl tag nginx:1.20.2 localhost:5000/my-nginx
# 推送镜像到本地不安全仓库
nerdctl push localhost:5000/my-nginx
# 从不安全仓库拉取镜像
nerdctl pull localhost:5000/my-nginx
nerdctl拉取镜像测试
[root@node1 ~]# nerdctl -n k8s.io image pull docker.io/library/nginx:1.20.2
docker.io/library/nginx:1.20.2: resolved |++++++++++++++++++++++++++++++++++++++|
index-sha256:03f3cb0afb7bd5c76e01bfec0ce08803c495348dccce37bcb82c347b4853c00b: done |++++++++++++++++++++++++++++++++++++++|
manifest-sha256:cba27ee29d62dfd6034994162e71c399b08a84b50ab25783eabce64b1907f774: done |++++++++++++++++++++++++++++++++++++++|
config-sha256:50fe74b50e0d0258922495297efbb9ebc3cbd5742103df1ca54dc21c07d24575: done |++++++++++++++++++++++++++++++++++++++|
layer-sha256:c423e1dacb26b544d5623a4a6a137c5a6e03e00048c3a3e074149b660ea78a2d: done |++++++++++++++++++++++++++++++++++++++|
layer-sha256:a2abf6c4d29d43a4bf9fbb769f524d0fb36a2edab49819c1bf3e76f409f953ea: done |++++++++++++++++++++++++++++++++++++++|
layer-sha256:da03644a12939e348735c7b34b6678429795293c69597602c50e9b3fb344973e: done |++++++++++++++++++++++++++++++++++++++|
layer-sha256:dcbfc6badd70b93971be6029156559388b9676386d543c042f8ff92ce83ab9c0: done |++++++++++++++++++++++++++++++++++++++|
layer-sha256:3f7ccff97047fb175bc00671991889f0c8e942a80b2857e9fd662293d275be9e: done |++++++++++++++++++++++++++++++++++++++|
layer-sha256:49e31097680b161295ba1a3963cf0f8516a5e43ac1b99a1dafa1425fc9bec29f: done |++++++++++++++++++++++++++++++++++++++|
elapsed: 30.6s total: 54.1 M (1.8 MiB/s)
[root@node1 ~]# nerdctl -n k8s.io image ls
REPOSITORY TAG IMAGE ID CREATED PLATFORM SIZE BLOB SIZE
nginx 1.20.2 03f3cb0afb7b 25 seconds ago linux/amd64 146.2 MiB 54.1 MiB
sha256 50fe74b50e0d0258922495297efbb9ebc3cbd5742103df1ca54dc21c07d24575 03f3cb0afb7b 25 seconds ago linux/amd64 146.2 MiB 54.1 MiB
k8s集群前期准备
k8s YUM源准备
集群所有节点安装
# 配置yum源
[root@master1 ~]# cat > /etc/yum.repos.d/kubernetes.repo << EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
# 查询版本信息
[root@master1 ~]# yum list kubelet --showduplicates --nogpgcheck
...
kubelet.x86_64 1.25.3-0 kubernetes
...
# 安装指定版本 kubelet kubeadm kubectl
[root@master1 ~]# yum install -y kubelet-1.25.3-0 kubeadm-1.25.3-0 kubectl-1.25.3-0
- kubeadm：用于初始化集群，并配置集群所需的组件并生成对应的安全证书和令牌；
- kubelet：负责与 Master 节点通信，并根据 Master 节点的调度决策来创建、更新和删除 Pod，同时维护 Node 节点上的容器状态；
- kubectl：用于管理k8集群的一个命令行工具；
开启bridge网桥过滤功能
[root@master1 ~]# cat > /etc/sysctl.d/k8s.conf <<EOF
# 启用 IPv4 转发
net.ipv4.ip_forward=1
# 启用桥接的 IPv6 转发
net.bridge.bridge-nf-call-ip6tables=1
# 启用桥接的 IPv4 转发
net.bridge.bridge-nf-call-iptables=1
# 禁用所有接口的 IPv6
net.ipv6.conf.all.disable_ipv6=1
# 禁用默认接口的 IPv6
net.ipv6.conf.default.disable_ipv6=1
# 禁用回环接口的 IPv6
net.ipv6.conf.lo.disable_ipv6=1
# 启用所有接口的 IPv6 转发
net.ipv6.conf.all.forwarding=1
EOF
# 由于开启bridge功能，需要加载br_netfilter模块来允许在bridge设备上的数据包经过iptables防火墙处理
[root@master1 ~]# modprobe br_netfilter && lsmod | grep br_netfilter
# ...会输出以下内容
br_netfilter 22256 0
bridge 151336 1 br_netfilter
# 参数解释：
modprobe //命令可以加载内核模块
br_netfilter //模块模块允许在bridge设备上的数据包经过iptables防火墙处理
# 加载配置文件，使上述配置生效
[root@master1 ~]# sysctl -p /etc/sysctl.d/k8s.conf
配置ipvs功能
在k8s中Service有两种代理模式，一种是基于iptables的，一种是基于ipvs，两者对比ipvs负载均衡算法更加的灵活，且带有健康检查的功能，如果想要使用ipvs模式，需要手动载入ipvs模块。
ipset 和 ipvsadm 是两个与网络管理和负载均衡相关的软件包，在k8s代理模式中，提供多种负载均衡算法，如轮询（Round Robin）、最小连接（Least Connection）和加权最小连接（Weighted Least Connection）等；
[root@master1 ~]# yum install -y ipset ipvsadm
将需要加载的ipvs相关模块写入到文件中
[root@master1 ~]# cat > /etc/sysconfig/modules/ipvs.modules <<EOF
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
EOF
#模块介绍
ip_vs //提供负载均衡的模块,支持多种负载均衡算法,如轮询、最小连接、加权最小连接等
ip_vs_rr //轮询算法的模块（默认算法）
ip_vs_wrr //加权轮询算法的模块,根据后端服务器的权重值转发请求
ip_vs_sh //哈希算法的模块,同一客户端的请求始终被分发到相同的后端服务器,保证会话一致性
nf_conntrack //链接跟踪的模块,用于跟踪一个连接的状态,例如 TCP 握手、数据传输和连接关闭等
加载ipvs模块
[root@master1 ~]# chmod 755 /etc/sysconfig/modules/ipvs.modules
[root@master1 ~]# lsmod | grep -e ip_vs -e nf_conntrack
关闭SWAP分区
为了保证 kubelet 正常工作，k8s强制要求禁用，否则集群初始化失败
# 临时关闭
[root@master1 ~]# swapoff -a
# 永久关闭
[root@master1 ~]# sed -ri 's/.*swap.*/#&/' /etc/fstab
[root@master1 ~]# grep ".*swap.*" /etc/fstab
配置kubelet
启用Cgroup控制组，用于限制进程的资源使用量，如CPU、内存等
[root@master1 ~]# cat > /etc/sysconfig/kubelet << EOF
KUBELET_EXTRA_ARGS="--cgroup-driver=systemd"
EOF
# 设置kubelet为开机自启动即可，集群初始化后自动启动
[root@master1 ~]# systemctl enable kubelet
集群初始化
在master01节点初始化集群，查看集群所需镜像文件，ApiServer、ControllerManager、Schedule、kubeproxy都是以容器的方式运行，kubelet是yum安装
[root@master1 ~]# kubeadm config images list
# ...以下是集群初始化所需的集群组件镜像
remote version is much newer: v1.31.0; falling back to: stable-1.25
registry.k8s.io/kube-apiserver:v1.25.16 ### 集群管理入口
registry.k8s.io/kube-controller-manager:v1.25.16 ### 集群控制器
registry.k8s.io/kube-scheduler:v1.25.16 ### 资源调度
registry.k8s.io/kube-proxy:v1.25.16 ### 集群代理
registry.k8s.io/pause:3.8 ### 集群实现Pod网络隔离、共享和健康检查的关键组件
registry.k8s.io/etcd:3.5.4-0 ### 集群数据库
registry.k8s.io/coredns/coredns:v1.9.3 ### 集群DNS
需要创建集群初始化配置文件
[root@master1 ~]# kubeadm config print init-defaults > kubeadm-config.yaml
配置文件需要修改如下内容
[root@master1 ~]# vim kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 10.0.0.3 # 本机IP地址
bindPort: 6443
nodeRegistration:
criSocket: unix:///var/run/containerd/containerd.sock
imagePullPolicy: IfNotPresent
name: master1 # 本机主机名
taints: null
---
apiServer:
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns: {}
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers # 集群组件镜像仓库地址
kind: ClusterConfiguration
kubernetesVersion: 1.25.0
networking:
dnsDomain: cluster.local
serviceSubnet: 10.96.0.0/12
scheduler: {}
#初始化集群
[root@master1 ~]# kubeadm init --config kubeadm-config.yaml --upload-certs
#选项说明：
--upload-certs //初始化过程将生成证书，并将其上传到etcd存储中，否则节点无法加入集群
初始化成功后，按照提示执行以下命令
[root@master1 ~]# export KUBECONFIG=/etc/kubernetes/admin.conf
[root@master1 ~]# mkdir -p $HOME/.kube
### 集群管理员配置文件
[root@master1 ~]# cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
[root@master1 ~]# chown $(id -u):$(id -g) $HOME/.kube/config
工作节点执行以下命令
[root@node1 ~]# kubeadm join 10.0.0.3:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:7d98bfbe0fc7d0d506d7597b551ef9d7ec8de251c3e5cf5055b0f4247fbebc96
部署Calico网络
[root@master1 ~]# wget --no-check-certificate https://framagit.org/mirrors-github/projectcalico/calico/-/raw/v3.26.4/manifests/calico.yaml
[root@master1 ~]# vim calico.yaml
# 在 - name: CLUSTER_TYPE 下方添加如下内容
- name: CLUSTER_TYPE
value: "k8s,bgp"
# 下方为新增内容
# 如果集群服务器中存在不同的网卡名称，需要在这里将每台服务器所使用的网卡名称全部填写（使用英文逗号分隔），否则网络无法使用，一直报错
# 例如：集群一共存在10台机器，其中有些机器的网卡名称是 ens33，有些是 eth0，有些是 enp9s0f0，则网卡配置为 interface=ens33,eth0,enp9s0f0
- name: IP_AUTODETECTION_METHOD
value: "interface=网卡名称"
# 配置网络
[root@master1 ~]# kubectl apply -f calico.yaml
[root@master1 ~]# ll
total 211192
-rw-r--r--. 1 root root 93779456 Jul 11 10:31 calico-cni_v3.26.4.tar
-rw-r--r--. 1 root root 32987136 Jul 11 10:32 calico-kube-controllers_v3.26.4.tar
-rw-r--r--. 1 root root 89487872 Jul 11 10:31 calico-node_v3.26.4.tar
[root@master1 ~]# for node in node1 node2; do scp * "$node":/root; done
[root@master1 ~]# for i in $(ls); do nerdctl load -i $i; done
验证集群可用性
#查看所有的节点
[root@master1 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
master1 Ready control-plane 33m v1.25.3
node1 Ready <none> 14m v1.25.3
node2 Ready <none> 14m v1.25.3
查看kubernetes集群pod运行情况
[root@master1 ~]# kubectl get all -A
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system pod/calico-kube-controllers-69b5dd6548-64qtd 1/1 Running 0 11m
kube-system pod/calico-node-9qggm 1/1 Running 0 11m
kube-system pod/calico-node-dpcdn 1/1 Running 0 116s
kube-system pod/calico-node-gn7p9 1/1 Running 0 11m
kube-system pod/coredns-7f8cbcb969-9znjq 1/1 Running 0 33m
kube-system pod/coredns-7f8cbcb969-bjx48 1/1 Running 0 33m
kube-system pod/etcd-master1 1/1 Running 0 33m
kube-system pod/kube-apiserver-master1 1/1 Running 0 33m
kube-system pod/kube-controller-manager-master1 1/1 Running 0 33m
kube-system pod/kube-proxy-cssnf 1/1 Running 0 14m
kube-system pod/kube-proxy-cxddc 1/1 Running 0 33m
kube-system pod/kube-proxy-vrv5f 1/1 Running 0 14m
kube-system pod/kube-scheduler-master1 1/1 Running 0 33m
NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
default service/kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 33m
kube-system service/kube-dns ClusterIP 10.96.0.10 <none> 53/UDP,53/TCP,9153/TCP 33m
NAMESPACE NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE
kube-system daemonset.apps/calico-node 3 3 3 3 3 kubernetes.io/os=linux 11m
kube-system daemonset.apps/kube-proxy 3 3 3 3 3 kubernetes.io/os=linux 33m
NAMESPACE NAME READY UP-TO-DATE AVAILABLE AGE
kube-system deployment.apps/calico-kube-controllers 1/1 1 1 11m
kube-system deployment.apps/coredns 2/2 2 2 33m
NAMESPACE NAME DESIRED CURRENT READY AGE
kube-system replicaset.apps/calico-kube-controllers-69b5dd6548 1 1 1 11m
kube-system replicaset.apps/coredns-7f8cbcb969 2 2 2 33m
命令补全
# k8s命令补全
[root@master1 ~]# yum install -y bash-completion
[root@master1 ~]# source /usr/share/bash-completion/bash_completion
[root@master1 ~]# echo "source <(kubectl completion bash)" >> ~/.bashrc
[root@master1 ~]# source ~/.bashrc
# Docker命令补全
# 手动下载并安装Docker的自动补全脚本
[root@master1 ~]# curl -L https://raw.githubusercontent.com/docker/cli/master/contrib/completion/bash/docker > /etc/bash_completion.d/docker
# 加载Docker自动补全脚本
[root@master1 ~]# cat >> ~/.bashrc << EOF
# Load Docker bash completion script
if [ -f /etc/bash_completion.d/docker ]; then
. /etc/bash_completion.d/docker
fi
EOF
# 生效配置
[root@master1 ~]# source ~/.bashrc
启用IPVS模式
在 Kubernetes 中，iptables 和 IPVS 是两种用于实现 Service 的负载均衡和网络转发的技术。它们各自有不同的机制、优势和适用场景。
iptables
iptables 是 Linux 内核中的一个数据包过滤框架，通常用于实现网络地址转换（NAT）、包过滤和负载均衡。
工作原理：
- 基于规则链：
iptables通过规则链的方式逐一检查数据包，规则链中的每条规则指定了数据包匹配的条件和相应的动作（例如转发、丢弃等）。 - DNAT 实现负载均衡：对于 Kubernetes 中的 Service，
iptables通过 DNAT（目的地址转换）将流量转发到后端的 Pod。
优点：
- 成熟稳定：
iptables已经广泛使用多年，功能稳定且适合小规模集群。 - 轻量级：对于流量不大的场景，
iptables的开销较小，适合小规模集群。
缺点：
- 性能问题：在大规模集群中，随着规则数量的增加，
iptables需要逐条检查规则，处理效率降低，导致网络性能瓶颈。 - 可伸缩性较差：
iptables不太适合处理大量并发请求，因为它是按顺序处理规则链，无法很好地进行并行处理。
IPVS (IP Virtual Server)
IPVS 是基于 Linux 虚拟服务器（LVS）实现的网络负载均衡技术，是 iptables 的一种替代方案。它是一个更高级的四层负载均衡器，可以处理大量并发连接。
工作原理：
- 基于哈希表：
IPVS通过哈希表直接查找目标 Pod，而不是逐条遍历规则链，因此在处理大量规则时效率更高。 - 支持多种调度算法：
IPVS提供了多种负载均衡算法，如轮询（Round Robin）、最少连接（Least Connections）等，可以根据需求选择合适的算法。
优点：
- 高性能：
IPVS通过哈希表来处理请求，避免了逐条匹配规则链的开销，在大规模集群中性能更好。 - 灵活的负载均衡算法：支持多种负载均衡算法，能够更好地适应不同的网络流量模式。
- 快速恢复：当后端 Pod 状态发生变化时，
IPVS的反应速度比iptables更快，能更及时地调整流量分配。
缺点：
- 复杂性：相较于
iptables，IPVS的配置和管理更为复杂，可能需要额外的学习和调试。 - 对系统要求较高：由于
IPVS是在内核级别操作，它对内核版本有一定要求，旧版本的 Linux 内核可能不支持。
总结
iptables适合小规模、简单的场景，优点是轻量且易于配置，但在大规模场景中性能可能成为瓶颈。IPVS适合大规模、高性能需求的场景，提供更好的性能和多样化的负载均衡策略，但复杂度较高。
两者的选择应根据具体的集群规模和性能要求来决定。
[root@master1 ~]# kubectl edit configmaps -n kube-system kube-proxy
配置ipvs功能
[root@master1 ~]# yum install -y ipset ipvsadm
将需要加载的ipvs相关模块写入到文件中
[root@master1 ~]# cat > /etc/sysconfig/modules/ipvs.modules <<EOF
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
EOF
#模块介绍
ip_vs //提供负载均衡的模块,支持多种负载均衡算法,如轮询、最小连接、加权最小连接等
ip_vs_rr //轮询算法的模块（默认算法）
ip_vs_wrr //加权轮询算法的模块,根据后端服务器的权重值转发请求
ip_vs_sh //哈希算法的模块,同一客户端的请求始终被分发到相同的后端服务器,保证会话一致性
nf_conntrack //链接跟踪的模块,用于跟踪一个连接的状态,例如 TCP 握手、数据传输和连接关闭等
加载ipvs模块
[root@master1 ~]# chmod 755 /etc/sysconfig/modules/ipvs.modules
[root@master1 ~]# lsmod | grep -e ip_vs -e nf_conntrack
查看原配置信息
# 获取pod信息
[root@master ~]# kubectl get pod -n kube-system -l k8s-app=kube-proxy
NAME READY STATUS RESTARTS AGE
kube-proxy-898wx 1/1 Running 3 (4h39m ago) 2d20h
kube-proxy-zjhq9 1/1 Running 3 (4h39m ago) 2d20h
# 查看默认模式为 iptabels
[root@master ~]# kubectl logs -n kube-system pods/kube-proxy-898wx
I0830 10:21:26.339470 1 node.go:163] Successfully retrieved node IP: 192.168.100.3
I0830 10:21:26.339571 1 server_others.go:138] "Detected node IP" address="192.168.100.3"
I0830 10:21:26.339597 1 server_others.go:578] "Unknown proxy mode, assuming iptables proxy" proxyMode=""
I0830 10:21:26.375037 1 server_others.go:206] "Using iptables Proxier"
I0830 10:21:26.375184 1 server_others.go:213] "kube-proxy running in dual-stack mode" ipFamily=IPv4
I0830 10:21:26.375190 1 server_others.go:214] "Creating dualStackProxier for iptables"
修改kube-proxy模式
[root@master1 ~]# kubectl edit configmaps -n kube-system kube-proxy
...
iptables:
masqueradeAll: false
masqueradeBit: null
minSyncPeriod: 0s
syncPeriod: 0s
ipvs:
excludeCIDRs: null
minSyncPeriod: 0s
scheduler: ""
strictARP: false
syncPeriod: 0s
tcpFinTimeout: 0s
tcpTimeout: 0s
udpTimeout: 0s
kind: KubeProxyConfiguration
metricsBindAddress: ""
mode: "ipvs" ### 添加ipvs
nodePortAddresses: null
删除kube-proxy的pod
# 获取pod的信息
[root@master ~]# kubectl get pod -n kube-system -l k8s-app=kube-proxy
NAME READY STATUS RESTARTS AGE
kube-proxy-898wx 1/1 Running 3 (4h43m ago) 2d20h
kube-proxy-zjhq9 1/1 Running 3 (4h43m ago) 2d20h
# 依次删除pod生效配置
[root@master ~]# kubectl delete pod -n kube-system kube-proxy-898wx
pod "kube-proxy-898wx" deleted
# 删除pod之后 会重新运行一个pod
[root@master ~]# kubectl get pod -n kube-system -l k8s-app=kube-proxy
NAME READY STATUS RESTARTS AGE
kube-proxy-djfmq 1/1 Running 0 2s
kube-proxy-zjhq9 1/1 Running 3 (4h44m ago) 2d20h
# 依次删除pod生效配置
[root@master ~]# kubectl delete pod -n kube-system kube-proxy-zjhq9
pod "kube-proxy-zjhq9" deleted
# 删除pod之后 会重新运行一个pod
[root@master ~]# kubectl get pod -n kube-system -l k8s-app=kube-proxy
NAME READY STATUS RESTARTS AGE
kube-proxy-djfmq 1/1 Running 0 12s
kube-proxy-fxn67 1/1 Running 0 2s
# 完成配置验证IPVS功能
[root@master ~]# kubectl logs -n kube-system pods/kube-proxy-djfmq | grep ipvs
I0830 15:03:02.528040 1 server_others.go:269] "Using ipvs Proxier"
I0830 15:03:02.528056 1 server_others.go:271] "Creating dualStackProxier for ipvs"
# 快速检查和管理 Kubernetes 集群中使用 IPVS 的网络负载均衡配置
[root@master ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 172.17.0.1:30001 rr
TCP 172.18.0.1:30001 rr
TCP 192.168.100.3:30001 rr
TCP 10.96.0.1:443 rr
-> 192.168.100.3:6443 Masq 1 0 0
TCP 10.96.0.10:53 rr
-> 10.244.0.14:53 Masq 1 0 0
-> 10.244.0.15:53 Masq 1 0 0
TCP 10.96.0.10:9153 rr
-> 10.244.0.14:9153 Masq 1 0 0
-> 10.244.0.15:9153 Masq 1 0 0
TCP 10.98.147.132:8000 rr
-> 10.244.0.16:8000 Masq 1 0 0
TCP 10.103.213.149:443 rr
TCP 10.244.0.0:30001 rr
TCP 10.244.0.1:30001 rr
UDP 10.96.0.10:53 rr
-> 10.244.0.14:53 Masq 1 0 0
-> 10.244.0.15:53 Masq 1 0 0
Ln: 以数字格式列出当前的 IPVS 配置，它展示虚拟服务器的 IP 地址、端口、调度算法以及每个虚拟服务器对应的真实服务器的 IP 地址、端口、权重和连接状态。