OpenEBS

# 需要先安装 iscsi
# https://docs.openebs.io/docs/next/prerequisites.html

helm repo add openebs https://openebs.github.io/charts
helm repo update
helm install --namespace openebs --name openebs openebs/openebs

# 直接安装
kubectl apply -f https://openebs.github.io/charts/openebs-operator.yaml

kubectl get csinodes -A

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: openebs-zfspv
allowVolumeExpansion: true
parameters:
  poolname: 'zfspv-pool'
  # zfs,ext2,ext3,ext4,xfs,btrfs
  # 非 zfs 则用 zvol
  # 有些应用在指定的 fs 上运行的更好 - https://github.com/openebs/zfs-localpv/issues/169
  # 不设置默认 ext4
  fstype: 'zfs'
  # 允许多次挂载
  # Deployment 可以不用 Recreate 策略
  shared: 'yes'
  # yes - 即便 zpool 空间不够也会提供 pv
  # no - 会确保保留空间足够
  thinprovision: 'no'
  dedup: 'no'
  # on,off,lzjb,zstd,gzip,gzip-1,gzip-2,gzip-3,gzip-4,gzip-5,gzip-6,gzip-7,gzip-8,gzip-9,zle,lz4
  # on - 默认加密算法
  compression: 'on'
  # zfs 配置
  # recordsize:
  # zvol 配置
  # volblocksize:
provisioner: zfs.csi.openebs.io
allowedTopologies:
  - matchLabelExpressions:
      - key: openebs.io/nodeid
        values:
          - zfspv-node1
          - zfspv-node2

kind: VolumeSnapshotClass
apiVersion: snapshot.storage.k8s.io/v1beta1
metadata:
  name: zfspv-snapclass
  annotations:
    # snapshot.storage.kubernetes.io/is-default-class: "true"
driver: zfs.csi.openebs.io
deletionPolicy: Delete

apiVersion: snapshot.storage.k8s.io/v1beta1
kind: VolumeSnapshot
metadata:
  name: zfspv-snap
spec:
  volumeSnapshotClassName: zfspv-snapclass
  source:
    persistentVolumeClaimName: csi-zfspv

kubectl get volumesnapshot.snapshot
# 查看快照
kubectl get zfssnap -n openebs
zfs list -t all

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: openebs-zfspv
parameters:
  # zfs 参数
  recordsize: '4k'
  # volblocksize: "4k" # zvol 使用该参数
  compression: 'off'
  dedup: 'off'
  fstype: 'zfs' # 其他 fs 会创建为 zvol
  poolname: 'zfspv-pool' # 匹配创建的名字
provisioner: zfs.csi.openebs.io
allowedTopologies:
  - matchLabelExpressions:
      # 限定节点
      - key: kubernetes.io/hostname
        values:
          - zfspv-node1
          - zfspv-node2
      # 自定义 key
      - key: openebs.io/rack
        values:
          - rack1
# 默认使用 zfs scheduler - VolumeWeighted, CapacityWeighted
# 可使用 kubernetes scheduler - pod 节点亲和
volumeBindingMode: WaitForFirstConsumer

kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: csi-zfspv
spec:
  storageClassName: openebs-zfspv
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 4Gi

kind: VolumeSnapshotClass
apiVersion: snapshot.storage.k8s.io/v1beta1
metadata:
  name: zfspv-snapclass
  annotations:
    snapshot.storage.kubernetes.io/is-default-class: 'true'
driver: zfs.csi.openebs.io
deletionPolicy: Delete

apiVersion: snapshot.storage.k8s.io/v1beta1
kind: VolumeSnapshot
metadata:
  name: zfspv-snap
spec:
  volumeSnapshotClassName: zfspv-snapclass
  source:
    persistentVolumeClaimName: csi-zfspv

kubectl get volumesnapshot.snapshot
# 节点上也可以查看快照
zfs list -t all

kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: zfspv-clone
spec:
  storageClassName: openebs-zfspv
  dataSource:
    # clone 自 snapshot
    name: zfspv-snap
    kind: VolumeSnapshot
    apiGroup: snapshot.storage.k8s.io
    # clone 自 volume
    name: zfspv-pvc
    kind: PersistentVolumeClaim
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 4Gi

apiVersion: velero.io/v1
kind: VolumeSnapshotLocation
metadata:
  name: zfspv-full
  namespace: velero
spec:
  provider: openebs.io/zfspv-blockstore
  config:
    bucket: velero
    prefix: zfs
    incrBackupCount: '3' # number of incremental backups we want to have
    namespace: openebs # this is the namespace where ZFS-LocalPV creates all the CRs, passed as OPENEBS_NAMESPACE env in the ZFS-LocalPV deployment
    provider: aws
    region: minio
    s3ForcePathStyle: 'true'
    s3Url: http://minio.velero.svc:9000

# 确保 driver 存在
kubectl get csinodes -oyaml

# 可尝试删除 openebs-zfs-localpv-node 的 Pod
kubectl delete csinodes NODEID
kubectl get pods -A -l role=openebs-zfs -owide