K3S 常见问题

# kine 不支持 --keys-only
ETCDCTL_API=3 etcdctl --endpoints=unix:///var/lib/rancher/k3s/server/kine.sock get /registry/clusterrolebindings/system:kube-dns
# 所有
# ETCDCTL_API=3 etcdctl --endpoints=unix:///var/lib/rancher/k3s/server/kine.sock get / --prefix

nslookup kubernetes.default

# 确保网络正常
ping 1.1.1.1

# 排查 host 路由
route -n

cat /run/flannel/subnet.env

# host
ping 10.42.0.1

# 排查 pod 网络
tcpdump -i cni0 -nn -s0 -v -l host

FLANNEL_NETWORK=10.42.0.0/16
FLANNEL_SUBNET=10.42.0.1/24
FLANNEL_MTU=1450
FLANNEL_IPMASQ=true

iptables -P FORWARD ACCEPT

echo '#!/bin/sh -
sleep ${RANDOM:0:2}m

rc-service k3s restart --ifstarted
rc-service k0scontroller restart --ifstarted
' | sudo tee /etc/periodic/monthly/kube-restart
sudo chmod +x /etc/periodic/monthly/kube-restart

sudo rc-service crond start --ifstopped
sudo rc-update add crond

# 如果通过 get.k3s.io 安装会有该脚本
/usr/local/bin/k3s-killall.sh

# 清理 iptables 规则
iptables-save | grep -v KUBE- | grep -v CNI- | iptables-restore
iptables-save | grep -v 10.4[2,3] | iptables-restore

# 容器镜像清理
# 停止所有 pod，清除停止容器，清除 volume，但保留下载的镜像
docker stop $(docker ps -aq)
docker system prune --volumes -f
docker system prune -f
# 如果不想保留镜像
# docker system prune -a
# 取消所有挂载
sudo umount $(mount -v | grep '/var/lib/kubelet' | awk '{print $3}')
# k3s 和 kubelet 状态
sudo rm -rf /var/lib/rancher
sudo rm -rf /var/lib/kubelet
# kubeconf
sudo rm -f /etc/rancher/k3s/k3s.yaml
# 日志清理
sudo rm -rf /var/log/containers/*
sudo rm -rf /var/log/pods/*
sudo rm -f /var/log/k3s.log

# 如果用了数据库
echo drop table if exists kine | psql -d $DATABASE_URL

# 如果有日志文件，例如 --log k3s-server.log --alsologtostderr
rm k3s{,-server,-agent}.log -f

ip li del cni0
ip li del flannel.1
rm -rf /var/lib/cni/*

E0905 23:03:57.992668   12600 controller.go:114] loading OpenAPI spec for "v1alpha1.tap.linkerd.io" failed with: failed to retrieve openAPI spec, http error: ResponseCode: 503, Body: Error trying to reach service: 'x509: certificate relies on legacy Common Name field, use SANs or temporarily enable Common Name matching with GODEBUG=x509ignoreCN=0', Header: map[Content-Type:[text/plain; charset=utf-8] X-Content-Type-Options:[nosniff]]

# 启动时添加环境变量
GODEBUG=x509ignoreCN=0

service k3s stop
# 如果没有 k3s-killal 可以考虑关闭 k3s 开机自动启动然后重启
k3s-killall

mkdir -p /data/k3s
rsync -aP /var/lib/rancher/k3s/ /data/k3s/

# 修改 data-dir 启动服务
echo 'data-dir: /data/k3s' >> /etc/rancher/k3s/config.yaml
service k3s start

# busybox 的 lsof 不支持过滤路径
apk search lsof
lsof +D /data/k3s | wc -l
# 输出 0 则迁移完成
lsof +D /var/lib/rancher/k3s | wc -l

# 清除
rm -rf /var/lib/rancher/k3s

x509: certificate has expired or is not yet valid: current time 2022-03-28T15:55:27+08:00 is after 2022-02-20T13:44:47Z

# 当前证书 失效时间
curl -v -k https://localhost:6443 -s 2>&1 | grep date
openssl s_client -connect localhost:6443 -showcerts < /dev/null 2>&1 | openssl x509 -noout -enddate

for i in $(ls /var/lib/rancher/k3s/server/tls/*.crt); do
  echo $i
  openssl x509 -enddate -noout -in $i
done

kubectl get secret -n kube-system k3s-serving -o jsonpath='{.data.tls\.crt}' | base64 -d | openssl x509 -noout -text | grep Not

openssl s_client -connect localhost:6443 -showcerts < /dev/null 2>&1 | openssl x509 -noout -startdate -enddate

# 设置为 < 90 天
date -s 20210514
# sudo hwclock -w

service k3s restart

# sudo chronyc -a makestep

# 可以尝试
kubectl --insecure-skip-tls-verify delete secret -n kube-system k3s-serving
rm -f /var/lib/rancher/k3s/server/tls/dynamic-cert.json
# 重启 k3s

kubectl -n kube-system get secrets | grep "coredns"

kubectl -n kube-system delete secret coredns-token-lkt4c
kubectl -n kube-system delete pod coredns-66c464876b-25kl9

k3s etcd-snapshot list
# 尝试恢复
k3s server --cluster-reset --cluster-reset-restore-path=/var/lib/rancher/k3s/server/db/snapshots/XXX

# mv /var/lib/rancher/k3s/server/db/etcd/ etcd.bk
# k3s etcd-snapshot prune --snapshot-retention 10

# openrc
echo 'rc_ulimit="-n 1048576"' >> /etc/rancher/k3s/k3s.env

echo 'cluster-init: true' >> /etc/rancher/k3s/config.yaml

# 迁移后会记录 state.db.migrated
service k3s restart
du -sh /var/lib/rancher/k3s/server/db/etcd

# etcd 支持快照
k3s etcd-snapshot save
k3s etcd-snapshot list
# 快照目录
ls /var/lib/rancher/k3s/server/db/snapshots

# 判断
lsof -f -- /var/lib/rancher/k3s/server/db/state.db-wal

kubelet-arg:
  - config=/etc/rancher/k3s/kubelet.config
#  - 'maxPods=250'
#  - "kube-reserved=cpu=500m,memory=1Gi,ephemeral-storage=2Gi"
#  - "system-reserved=cpu=500m, memory=1Gi,ephemeral-storage=2Gi"
#  - "eviction-hard=memory.available<500Mi,nodefs.available<10%"

--kubelet-arg=config=/etc/rancher/k3s/kubelet.config

apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
maxPods: 250

time="2023-06-04T16:17:14.398351570+08:00" level=info msg="Connect containerd service"
time="2023-06-04T16:17:14.398408099+08:00" level=info msg="using legacy CRI server"
time="2023-06-04T16:17:14.398418537+08:00" level=info msg="using experimental NRI integration - disable nri plugin to prevent this"
time="2023-06-04T16:17:14.398549887+08:00" level=warning msg="failed to load plugin io.containerd.grpc.v1.cri" error="failed to create CRI service: failed to find snapshotter \"zfs\""
time="2023-06-04T16:17:14.398913047+08:00" level=info msg=serving... address=/run/k3s/containerd/containerd.sock.ttrpc
time="2023-06-04T16:17:14.398971211+08:00" level=info msg=serving... address=/run/k3s/containerd/containerd.sock
time="2023-06-04T16:17:14.398989382+08:00" level=info msg="containerd successfully booted in 0.430664s"

k3s ctr plugin ls

grep 'failed to get network' /var/log/k3s.log
ls -s /var/lib/cni/flannel/ # size=0 的文件

k3s ctr container rm $(find /var/lib/cni/flannel/ -size 0 | xargs -n1 basename)

find /var/lib/cni/flannel/ -size 0 -delete
find /var/lib/cni/results/ -size 0 -delete # cached result

iptables -L | wc -l -c

ps -e -o pid= -o args= | sed -e 's/^ *//; s/\s\s*/\t/;' | grep containerd-shim | grep '/run/k3s/containerd' | cut -f1