关于k8s的cilium网络插件踩坑记

cilium状态为CrashLoopBackOff

coredns状态为Pending

使用的系统镜像为centos7.6 内核是3.***      然后用的工具来部署集群，开始以为是调度啥的，然后单节点试也出现CrashLoopBackOff、Pending等状态，更新内核在到5+  集群自己就好了

注意：使用cilium网络插件保证系统内核在5以上，自己网上找个内核更新教程

[root@k8s-vip ~]# uname -r        
5.4.278-1.el7.elrepo.x86_64                #更新后

报错的关键字，方便大数据搜索

[root@k8s-vip ~]# kubectl get pods --namespace kube-system -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES cilium-kpr88 0/1 Init:CrashLoopBackOff 5 (2m34s ago) 5m40s 192.168.0.178 k8s-vip <none> <none> cilium-operator-86666d88cb-gg5k9 1/1 Running 0 5m40s 192.168.0.178 k8s-vip <none> <none> coredns-5d78c9869d-f8hvw 0/1 Pending 0 5m40s <none> <none> <none> <none> coredns-5d78c9869d-j4l65 0/1 Pending 0 5m40s <none> <none> <none> <none> etcd-k8s-vip 1/1 Running 0 5m57s 192.168.0.178 k8s-vip <none> <none> kube-apiserver-k8s-vip 1/1 Running 0 5m57s 192.168.0.178 k8s-vip <none> <none> kube-controller-manager-k8s-vip 1/1 Running 0 5m55s 192.168.0.178 k8s-vip <none> <none> kube-proxy-zkpbq 1/1 Running 0 5m40s 192.168.0.178 k8s-vip <none> <none> kube-scheduler-k8s-vip 1/1 Running 0 5m57s 192.168.0.178 k8s-vip <none> <none> 

[root@k8s-master1 ~]# kubectl get pods -n kube-system NAME READY STATUS RESTARTS AGE cilium-4k62x 0/1 Init:CrashLoopBackOff 8 (4m2s ago) 20m cilium-6jxqk 0/1 Init:CrashLoopBackOff 8 (3m50s ago) 20m cilium-8lfxw 0/1 Init:CrashLoopBackOff 12 (90s ago) 20m cilium-hfssf 0/1 Init:CrashLoopBackOff 8 (3m47s ago) 20m cilium-operator-86666d88cb-nfww9 1/1 Running 0 20m cilium-z6xrg 0/1 Init:CrashLoopBackOff 8 (3m58s ago) 20m coredns-5d78c9869d-22ws4 0/1 Pending 0 21m coredns-5d78c9869d-gz5qq 0/1 Pending 0 21m etcd-k8s-master1 1/1 Running 0 21m etcd-k8s-master2 1/1 Running 0 20m etcd-k8s-master3 1/1 Running 0 20m kube-apiserver-k8s-master1 1/1 Running 0 21m kube-apiserver-k8s-master2 1/1 Running 0 21m kube-apiserver-k8s-master3 1/1 Running 0 20m kube-controller-manager-k8s-master1 1/1 Running 1 (20m ago) 21m kube-controller-manager-k8s-master2 1/1 Running 0 20m kube-controller-manager-k8s-master3 1/1 Running 0 20m kube-proxy-cnh8n 1/1 Running 0 20m kube-proxy-pbchh 1/1 Running 0 20m kube-proxy-qlknc 1/1 Running 0 21m kube-proxy-vfzrx 1/1 Running 0 21m kube-proxy-zqgt2 1/1 Running 0 20m kube-scheduler-k8s-master1 1/1 Running 1 (20m ago) 21m kube-scheduler-k8s-master2 1/1 Running 0 20m kube-scheduler-k8s-master3 1/1 Running 0 20m kube-sealos-lvscare-k8s-node1 1/1 Running 0 19m kube-sealos-lvscare-k8s-node2 1/1 Running 0 19m

#这条是查看cilium的容器详情发现的

Warning BackOff 94s (x94 over 21m) kubelet Back-off restarting failed container mount-cgroup in pod cilium-4k62x_kube-system(b9d7e309-b664-436c-adc0-c65cb64fdd42)

#这条是实时查询cgroup的

[root@k8s-master1 ~]# ls /proc/1/ns/cgroup ls: cannot access /proc/1/ns/cgroup: No such file or directory Oct 25 11:46:30 k8s-master1 kubelet[16627]: E1025 11:46:30.252517 16627 pod_workers.go:1294] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"mount-cgroup\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=mount-cgroup pod=cilium-8lfxw Oct 25 11:46:32 k8s-master1 kubelet[16627]: E1025 11:46:32.516907 16627 kubelet.go:2760] "Container runtime network not ready" networkReady="NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: cni plugin not initialized" Oct 25 11:46:37 k8s-master1 kubelet[16627]: E1025 11:46:37.517899 16627 kubelet.go:2760] "Container runtime network not ready" networkReady="NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: cni plugin not initialized" Oct 25 11:46:42 k8s-master1 kubelet[16627]: E1025 11:46:42.519728 16627 kubelet.go:2760] "Container runtime network not ready" networkReady="NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: cni plugin not initialized"

#这条是containerd的日志信息

Oct 25 14:07:09 k8s-vip containerd[6837]: time="2024-10-25T14:07:09.312759255+08:00" level=warning msg="get container info failed" error="container \"32c367f459cb167292b12ef2c284c17786cca556cd054616b84220bda8583238\" in namespace \"k8s.io\": not found" Oct 25 14:09:58 k8s-vip containerd[6837]: time="2024-10-25T14:09:58.172299056+08:00" level=warning msg="cleaning up after shim disconnected" id=529aad484d8c2d9fd2fc45722f9c3bf50fce38edd5fc32f03a5e8d14879d2664 namespace=k8s.io Oct 25 14:09:58 k8s-vip containerd[6837]: time="2024-10-25T14:09:58.703783378+08:00" level=warning msg="get container info failed" error="container \"0e16c9a5f3dd2faa57d3c7e6e0e1627e26a8aadbbaf24e1f6d196996e6383513\" in namespace \"k8s.io\": not found" Oct 25 14:14:59 k8s-vip containerd[6837]: time="2024-10-25T14:14:59.143294777+08:00" level=warning msg="cleaning up after shim disconnected" id=993eb93eb74b8fcbc31746f4e45f960c5c1cd1a19f47fff65b69c3b062d138f3 namespace=k8s.io Oct 25 14:14:59 k8s-vip containerd[6837]: time="2024-10-25T14:14:59.378391295+08:00" level=error msg="RemoveContainer for \"529aad484d8c2d9fd2fc45722f9c3bf50fce38edd5fc32f03a5e8d14879d2664\" failed" error="failed to set removing state for container \"529aad484d8c2d9fd2fc45722f9c3bf50fce38edd5fc32f03a5e8d14879d2664\": container is already in removing state"

#这是容器接口cni，但原因还是cilium网络

Oct 25 14:16:16 k8s-vip containerd[15865]: time="2024-10-25T14:16:16.946120681+08:00" level=error msg="failed to load cni during init, please check CRI plugin status before setting up network for pods" error="cni config load failed: no network config found in /etc/cni/net.d: cni plugin not initialized: failed to load cni config"