Skip to main content

containerd

caution
  • 不支持 reload 配置 #4478
  • 不要用 zfs snapshotter
    • 非常慢 - vol 多了后 zfs list,import,mount 全都慢
    • 导致集群启动慢
ctr plugin ls
TYPE                                   ID                       PLATFORMS      STATUS
io.containerd.service.v1               introspection-service    -              ok
io.containerd.event.v1                 exchange                 -              ok
io.containerd.content.v1               content                  -              ok
io.containerd.snapshotter.v1           btrfs                    linux/amd64    skip
io.containerd.snapshotter.v1           native                   linux/amd64    ok
io.containerd.snapshotter.v1           overlayfs                linux/amd64    ok
io.containerd.snapshotter.v1           fuse-overlayfs           linux/amd64    ok
io.containerd.snapshotter.v1           stargz                   linux/amd64    ok
io.containerd.metadata.v1              bolt                     -              ok
io.containerd.service.v1               containers-service       -              ok
io.containerd.service.v1               content-service          -              ok
io.containerd.differ.v1                walking                  linux/amd64    ok
io.containerd.service.v1               diff-service             -              ok
io.containerd.gc.v1                    scheduler                -              ok
io.containerd.service.v1               images-service           -              ok
io.containerd.service.v1               namespaces-service       -              ok
io.containerd.service.v1               snapshots-service        -              ok
io.containerd.runtime.v1               linux                    linux/amd64    ok
io.containerd.runtime.v2               task                     linux/amd64    ok
io.containerd.runtime.v2               shim                     -              ok
io.containerd.monitor.v1               cgroups                  linux/amd64    ok
io.containerd.service.v1               tasks-service            -              ok
io.containerd.grpc.v1                  introspection            -              ok
io.containerd.lease.v1                 manager                  -              ok
io.containerd.nri.v1                   nri                      -              ok
io.containerd.sandbox.store.v1         local                    -              ok
io.containerd.sandbox.controller.v1    local                    -              ok
io.containerd.streaming.v1             manager                  -              ok
io.containerd.transfer.v1              local                    -              ok
io.containerd.internal.v1              restart                  -              ok
io.containerd.grpc.v1                  containers               -              ok
io.containerd.grpc.v1                  content                  -              ok
io.containerd.grpc.v1                  diff                     -              ok
io.containerd.grpc.v1                  events                   -              ok
io.containerd.grpc.v1                  healthcheck              -              ok
io.containerd.grpc.v1                  images                   -              ok
io.containerd.grpc.v1                  leases                   -              ok
io.containerd.grpc.v1                  namespaces               -              ok
io.containerd.internal.v1              opt                      -              ok
io.containerd.grpc.v1                  sandbox-controllers      -              ok
io.containerd.grpc.v1                  sandboxes                -              ok
io.containerd.grpc.v1                  snapshots                -              ok
io.containerd.grpc.v1                  streaming                -              ok
io.containerd.grpc.v1                  tasks                    -              ok
io.containerd.grpc.v1                  transfer                 -              ok
io.containerd.grpc.v1                  version                  -              ok
io.containerd.grpc.v1                  cri                      linux/amd64    error

containerd.toml​

# 获取默认配置
containerd config default > containerd.toml
参考
version=2
root=/var/lib/containerd
state=/run/containerd
plugin_dir=

oom_score = 0
# 可导入配置
imports = ["/etc/containerd/runtime_*.toml", "./debug.toml"]

[grpc]
[ttrpc]
[debug]
address=/run/containerd/debug.sock
uid=0
gid=0
level=info
[metrics]
[plugins]

# registry
# ==========
[plugins."io.containerd.grpc.v1.cri".registry]
# https://docs.docker.com/engine/security/certificates/
# 同 /etc/docker/certs.d/
config_path = "/etc/containerd/certs.d"

[plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
endpoint = ["https://fogjl973.mirror.aliyuncs.com", "https://8x40wsit.mirror.aliyuncs.com", "https://docker.mirrors.ustc.edu.cn", "https://registry-1.docker.io"]

[plugins."io.containerd.grpc.v1.cri".registry.configs."gcr.io".auth]

# 1.4+ 废弃 registry.mirrors, 只有没配置 config_path 时生效
# docker.io 镜像
[plugins.cri.registry.mirrors."docker.io"]
endpoint = ["https://fogjl973.mirror.aliyuncs.com", "https://8x40wsit.mirror.aliyuncs.com", "https://docker.mirrors.ustc.edu.cn", "https://registry-1.docker.io"]

# 自定义授权信息
[plugins."io.containerd.grpc.v1.cri".registry.configs."my.custom.registry".auth]
# 同 .docker/config.json
username = ""
password = ""
auth = ""
identitytoken = ""
# 自定义证书配置
[plugins."io.containerd.grpc.v1.cri".registry.configs."my.custom.registry".tls]
ca_file   = "ca.pem"
cert_file = "cert.pem"
key_file  = "key.pem"
# 或不校验
insecure_skip_verify = true
  • /etc/containerd/certs.d/
    • docker.io/
      • hosts.toml
hosts.toml
server = "https://docker.io"

[host."https://registry-1.docker.io"]
capabilities = ["pull", "resolve"]

zfs​

zfs create -o mountpoint=/var/lib/containerd/io.containerd.snapshotter.v1.zfs main/containerd
# 不必须，但推荐 - 镜像存储到 zfs
zfs create -o mountpoint=/var/lib/containerd/io.containerd.content.v1.content main/containerd-content
/etc/containerd.toml
version=2
[plugins."io.containerd.grpc.v1.cri".containerd]
snapshotter = "zfs"

Notes​

  • io.containerd.content.v1.content
  • io.containerd.metadata
    • 存储镜像元信息
    • io.containerd.metadata.v1.bolt/meta.db
  • io.containerd.grpc.v1.cri
    • 提供 CRI 接口
  • io.containerd.runtime.v1.linux
  • io.containerd.runtime.v2.task
    • k8s.io
  • io.containerd.snapshotter.v1 - 默认 overlayfs - container 运行时快照
    • btrfs
    • native
    • overlayfs
    • zfs
    • aufs
    • devmapper
  • io.containerd.service.v1
  • io.containerd.monitor.v1
    • cgroups
  • tmpmounts

FAQ

映射常用仓库为镜像仓库​

  • 镜像仓库可直接同步上游 - 例如 harbor 可指定 prpject 同步上游
  • docker.io 可使用现有的服务
  • 其他 仓库 没有现有的镜像服务
    • harbor 子域名映射需要特殊配置
  • 假设: harbor 地址 cr.example.com, 添加项目 quay 镜像 quay.io
    • 需要配置域名映射 quay.cr.example.com 到 cr.example.com/quay
    • 配置方式参考 #13579
# 现有容器仓库
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
endpoint = ["https://fogjl973.mirror.aliyuncs.com", "https://8x40wsit.mirror.aliyuncs.com", "https://docker.mirrors.ustc.edu.cn", "https://registry-1.docker.io"]

# harbor 子域名映射
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."quay.io"]
endpoint = ["https://quay.cr.example.com"]

映射常用仓库为单个私有仓库​

  • 假设仓库为 https://registry:5000
  • 将常用仓库映射为单个内部仓库 - 用于 airgap 场景
    • 因为拉取不存在的镜像不能从上游同步，只能拉预先导入的镜像
tip

containerd 不支持 reload 配置，所以需要提前配置好。

# 重定向所有到私有 registry
# 常见: docker.io, gcr.io, k8s.gcr.io, quay.io, ghcr.io
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."*"]
endpoint = ["https://registry:5000"]

[plugins."io.containerd.grpc.v1.cri".registry.configs."docker.io".tls]
insecure_skip_verify = true
[plugins."io.containerd.grpc.v1.cri".registry.configs."gcr.io".tls]
insecure_skip_verify = true
[plugins."io.containerd.grpc.v1.cri".registry.configs."k8s.gcr.io".tls]
insecure_skip_verify = true
[plugins."io.containerd.grpc.v1.cri".registry.configs."quay.io".tls]
insecure_skip_verify = true
[plugins."io.containerd.grpc.v1.cri".registry.configs."ghcr.io".tls]
insecure_skip_verify = true

mount through procfd not a directory​

检查 mount 逻辑，是不把 目录 挂载到了文件上。

mount through procfd: open o_path procfd: open /run/k0s/containerd/io.containerd.runtime.v2.task/k8s.io/promxy/rootfs/etc/promxy/secrets: not a directory

failed to reserve sandbox name: name is reserved for​

最终会自己修复，可能花费 20m-30m 时间。