NFS服务器部署

NFS服务器IP: 192.168.77.11

yum install -y nfs-utils
systemctl start nfs-server
mkdir /data/nfs/k8s -p
echo "/data/nfs *(rw,sync,no_subtree_check,no_root_squash)" > /etc/exports
exportfs -r
systemctl reload nfs-server

注：防火墙和selinux配置

[root@nfs ~]# sed -i 's#^SELINUX=.*#SELINUX=disabled#' /etc/sysconfig/selinux
[root@nfs ~]# setenforce 0
[root@nfs ~]# systemctl stop firewalld

注：
K8S Master/Client 需要NFS持久卷均需要安装nfs-utils

[root@k8s-master01 ~]# yum install -y nfs-utils

查看NFS服务器所有共享目录

[root@k8s-master01 ~]# showmount -e 192.168.77.11
Export list for 192.168.77.11:
/data/nfs *

静态持久卷

cat << EOF | kubectl apply -f -
apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-pv
spec:
  capacity:
    storage: 5Gi
  volumeMode: Filesystem
  accessModes:
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Recycle
  storageClassName: nfs-slow
  mountOptions:
    - hard
    - nfsvers=4.1
  nfs:
    path: /data/nfs/k8s
    server: 192.168.77.11
EOF

cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: nfs-pvc
spec:
  accessModes:
    - ReadWriteMany
  volumeMode: Filesystem
  resources:
    requests:
      storage: 2Gi
  storageClassName: nfs-slow
EOF

cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: nfs-pod-test
spec:
  containers:
  - image: nginx:latest
    name: nginx
    volumeMounts:
    - mountPath: /mnt
      name: nfs-volume
  volumes:
  - name: nfs-volume
    persistentVolumeClaim:
      claimName: nfs-pvc
EOF

测试

[root@ha ~]# kubectl get pvc
NAME         STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
nfs-pvc      Bound    nfs-pv                                     5Gi        RWX            nfs-slow       10s

[root@ha ~]# kubectl get pod
NAME           READY   STATUS    RESTARTS   AGE
nfs-pod-test   1/1     Running   0          2m13s

删除

kubectl delete pod nfs-pod-test
kubectl delete pvc nfs-pvc
kubectl delete pv nfs-pv

动态持久卷

nfs-subdir-external-provisioner

下载镜像到私有仓库

export https_proxy="192.168.77.73:7890"
image=registry.k8s.io/sig-storage/nfs-subdir-external-provisioner:v4.0.2
#skopeo copy docker://${image} docker://registry.sundayhk.com/k8s/${image#*/}
skopeo copy docker://${image} docker://harbor.sundayhk.com/k8s/${image}
unset https_proxy

helm安装

helm repo add nfs-subdir-external-provisioner https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/

helm install nfs-subdir-external-provisioner nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \
    --set nfs.server=192.168.77.11 \
    --set nfs.path=/data/nfs/k8s \
    --set image.repository=registry.sundayhk.com/k8s/sig-storage/nfs-subdir-external-provisioner \
    -n nfs-provisioner \
    --create-namespace

[root@k8s-master01 ~]# kubectl get storageclass -n nfs-provisioner
NAME         PROVISIONER                                     RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
nfs-client   cluster.local/nfs-subdir-external-provisioner   Delete          Immediate           true                   3m

[root@k8s-master01 ~]# kubectl get pod -n nfs-provisioner
NAME                                               READY   STATUS    RESTARTS   AGE
nfs-subdir-external-provisioner-757cd7dd45-m8mws   1/1     Running   0          3m

测试

kubectl create -f https://raw.githubusercontent.com/kubernetes-sigs/nfs-subdir-external-provisioner/master/deploy/test-claim.yaml 

# test-claim.yaml
cat <<EOF | kubectl apply -f -
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: test-claim
spec:
  storageClassName: nfs-client
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 1Mi
EOF

kubectl create -f https://raw.githubusercontent.com/kubernetes-sigs/nfs-subdir-external-provisioner/master/deploy/test-pod.yaml

# test-pod.yaml
cat <<EOF | kubectl apply -f -
kind: Pod
apiVersion: v1
metadata:
  name: test-pod
spec:
  containers:
  - name: test-pod
    image: busybox:1.28
    command:
      - "/bin/sh"
    args:
      - "-c"
      - "touch /mnt/SUCCESS && exit 0 || exit 1"
    volumeMounts:
      - name: nfs-pvc
        mountPath: "/mnt"
  restartPolicy: "Never"
  volumes:
    - name: nfs-pvc
      persistentVolumeClaim:
        claimName: test-claim
EOF

[root@harbor ~]# kubectl get pod test-pod
NAME       READY   STATUS      RESTARTS   AGE
test-pod   0/1     Completed   0          100s

[root@harbor rabbitmq]# kubectl get pvc
NAME         STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
test-claim   Bound    pvc-15fe2f96-7fc4-4064-8ab4-19fb9eec1f8c   1Mi        RWX            nfs-client     6m41s

NFS服务器查看

[root@nfs /]# ls -l /data/nfs/k8s/default-test-claim-pvc-15fe2f96-7fc4-4064-8ab4-19fb9eec1f8c/
总用量 0
-rw-r--r-- 1 root root 0 11月 21 03:41 SUCCESS

设置为默认storageclass

kubectl patch storageclass nfs-client -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'

[root@harbor ~]# kubectl get sc
NAME                   PROVISIONER                                     RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
nfs-client (default)   cluster.local/nfs-subdir-external-provisioner   Delete          Immediate           true                   10m

删除

kubectl delete pod test-pod
kubectl delete pvc test-claim