From 1ad86af1ac85ea9827984a6974c6b325f917fb68 Mon Sep 17 00:00:00 2001 From: Jiaxun Song Date: Tue, 17 Nov 2020 00:52:59 +0000 Subject: [PATCH] test: add more test cases for e2e test --- Makefile | 10 ++- deploy/install-driver.sh | 4 +- deploy/uninstall-driver.sh | 4 +- .../{kubernetes/nfs-server => }/README.md | 0 examples/deployment.yaml | 43 ++++++++++++ .../kubernetes/nfs-server/deployment.yaml | 65 ------------------- examples/kubernetes/nginx.yaml | 52 --------------- .../nfs-server => }/nfs-server.yaml | 0 examples/nginx-pod.yaml | 31 +++++++++ examples/pv-nfs-csi.yaml | 21 ++++++ examples/pvc-nfs-csi-dynamic.yaml | 12 ++++ examples/pvc-nfs-csi-static.yaml | 13 ++++ examples/{kubernetes => }/statefulset.yaml | 2 +- .../{kubernetes => }/storageclass-nfs.yaml | 0 test/e2e/e2e_suite_test.go | 35 +++++++++- test/utils/nfs_log.sh | 46 +++++++++++++ test/utils/testutils/testutils.go | 21 ------ 17 files changed, 213 insertions(+), 146 deletions(-) rename examples/{kubernetes/nfs-server => }/README.md (100%) create mode 100644 examples/deployment.yaml delete mode 100644 examples/kubernetes/nfs-server/deployment.yaml delete mode 100644 examples/kubernetes/nginx.yaml rename examples/{kubernetes/nfs-server => }/nfs-server.yaml (100%) create mode 100644 examples/nginx-pod.yaml create mode 100644 examples/pv-nfs-csi.yaml create mode 100644 examples/pvc-nfs-csi-dynamic.yaml create mode 100644 examples/pvc-nfs-csi-static.yaml rename examples/{kubernetes => }/statefulset.yaml (98%) rename examples/{kubernetes => }/storageclass-nfs.yaml (100%) create mode 100644 test/utils/nfs_log.sh delete mode 100644 test/utils/testutils/testutils.go diff --git a/Makefile b/Makefile index a542a615..00ffb229 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,7 @@ push: .PHONY: install-nfs-server install-nfs-server: - kubectl apply -f ./examples/kubernetes/nfs-server/nfs-server.yaml + kubectl apply -f ./examples/nfs-server.yaml .PHONY: install-helm install-helm: @@ -111,4 +111,10 @@ e2e-teardown: .PHONY: e2e-test e2e-test: - go test -v -timeout=0 ./test/e2e ${GINKGO_FLAGS} \ No newline at end of file + go test -v -timeout=0 ./test/e2e ${GINKGO_FLAGS} + +.PHONY: create-example-deployment +create-example-deployment: + kubectl apply -f ./examples/storageclass-nfs.yaml + kubectl apply -f ./examples/deployment.yaml + kubectl apply -f ./examples/statefulset.yaml diff --git a/deploy/install-driver.sh b/deploy/install-driver.sh index eec69e1d..6e0a3ab4 100755 --- a/deploy/install-driver.sh +++ b/deploy/install-driver.sh @@ -21,11 +21,11 @@ if [[ "$#" -gt 0 ]]; then ver="$1" fi -repo="https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/$ver/deploy/kubernetes" +repo="https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/$ver/deploy" if [[ "$#" -gt 1 ]]; then if [[ "$2" == *"local"* ]]; then echo "use local deploy" - repo="./deploy/kubernetes" + repo="./deploy" fi fi diff --git a/deploy/uninstall-driver.sh b/deploy/uninstall-driver.sh index 6b48ab79..0785e132 100755 --- a/deploy/uninstall-driver.sh +++ b/deploy/uninstall-driver.sh @@ -21,11 +21,11 @@ if [[ "$#" -gt 0 ]]; then ver="$1" fi -repo="https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/$ver/deploy/kubernetes" +repo="https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/$ver/deploy" if [[ "$#" -gt 1 ]]; then if [[ "$2" == *"local"* ]]; then echo "use local deploy" - repo="./deploy/kubernetes" + repo="./deploy" fi fi diff --git a/examples/kubernetes/nfs-server/README.md b/examples/README.md similarity index 100% rename from examples/kubernetes/nfs-server/README.md rename to examples/README.md diff --git a/examples/deployment.yaml b/examples/deployment.yaml new file mode 100644 index 00000000..c1006b7b --- /dev/null +++ b/examples/deployment.yaml @@ -0,0 +1,43 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-deployment-nfs +spec: + accessModes: + - ReadWriteMany # In this example, multiple Pods consume the same PVC. + resources: + requests: + storage: 10Gi + storageClassName: nfs-csi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment-nfs-rwm +spec: + replicas: 3 + selector: + matchLabels: + name: deployment-nfs-rwm + template: + metadata: + name: deployment-nfs-rwm + labels: + name: deployment-nfs-rwm + spec: + containers: + containers: + - name: deployment-nfs-rwm + image: mcr.microsoft.com/oss/nginx/nginx:1.17.3-alpine + command: + - "/bin/sh" + - "-c" + - while true; do echo $(hostname) $(date) >> /mnt/nfs/outfile; sleep 1; done + volumeMounts: + - name: nfs + mountPath: "/mnt/nfs" + volumes: + - name: nfs + persistentVolumeClaim: + claimName: pvc-deployment-nfs diff --git a/examples/kubernetes/nfs-server/deployment.yaml b/examples/kubernetes/nfs-server/deployment.yaml deleted file mode 100644 index d377e03e..00000000 --- a/examples/kubernetes/nfs-server/deployment.yaml +++ /dev/null @@ -1,65 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: pv-nfs -spec: - capacity: - storage: 10Gi - accessModes: - - ReadWriteMany - persistentVolumeReclaimPolicy: Retain - mountOptions: - - hard - - nfsvers=4.1 - csi: - driver: nfs.csi.k8s.io - readOnly: false - volumeHandle: unique-volumeid # make sure it's a unique id in the cluster - volumeAttributes: - server: nfs-server.default.svc.cluster.local - share: / ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: nfs -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 10Gi - volumeName: pv-nfs - storageClassName: "" ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nfs-busybox -spec: - replicas: 1 - selector: - matchLabels: - name: nfs-busybox - template: - metadata: - name: nfs-busybox - labels: - name: nfs-busybox - spec: - containers: - - image: busybox - command: - - sh - - -c - - 'while true; do date > /mnt/index.html; hostname >> /mnt/index.html; sleep $(($RANDOM % 5 + 5)); done' - imagePullPolicy: IfNotPresent - name: busybox - volumeMounts: - - name: nfs - mountPath: "/mnt" - volumes: - - name: nfs - persistentVolumeClaim: - claimName: nfs diff --git a/examples/kubernetes/nginx.yaml b/examples/kubernetes/nginx.yaml deleted file mode 100644 index 813b8696..00000000 --- a/examples/kubernetes/nginx.yaml +++ /dev/null @@ -1,52 +0,0 @@ -apiVersion: v1 -kind: PersistentVolume -metadata: - name: data-nfsplugin - labels: - name: data-nfsplugin -spec: - claimRef: - name: data-nfsplugin - namespace: default - accessModes: - - ReadWriteMany - capacity: - storage: 100Gi - csi: - driver: nfs.csi.k8s.io - volumeHandle: data-id - volumeAttributes: - # The nfs server could be a K8s service - # server: nfs-server.default.svc.cluster.local - server: 127.0.0.1 - share: /export ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: data-nfsplugin -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 100Gi ---- -apiVersion: v1 -kind: Pod -metadata: - name: nginx -spec: - containers: - - image: nginx - name: nginx - ports: - - containerPort: 80 - protocol: TCP - volumeMounts: - - mountPath: /var/www - name: data-nfsplugin - volumes: - - name: data-nfsplugin - persistentVolumeClaim: - claimName: data-nfsplugin diff --git a/examples/kubernetes/nfs-server/nfs-server.yaml b/examples/nfs-server.yaml similarity index 100% rename from examples/kubernetes/nfs-server/nfs-server.yaml rename to examples/nfs-server.yaml diff --git a/examples/nginx-pod.yaml b/examples/nginx-pod.yaml new file mode 100644 index 00000000..904f4e0e --- /dev/null +++ b/examples/nginx-pod.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-nginx +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 100Gi + storageClassName: nfs-csi +--- +apiVersion: v1 +kind: Pod +metadata: + name: nginx +spec: + containers: + - image: nginx + name: nginx + ports: + - containerPort: 80 + protocol: TCP + volumeMounts: + - mountPath: /var/www + name: pvc-nginx + volumes: + - name: pvc-nginx + persistentVolumeClaim: + claimName: pvc-nginx diff --git a/examples/pv-nfs-csi.yaml b/examples/pv-nfs-csi.yaml new file mode 100644 index 00000000..80324dbd --- /dev/null +++ b/examples/pv-nfs-csi.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-nfs +spec: + capacity: + storage: 10Gi + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + mountOptions: + - hard + - nfsvers=4.1 + csi: + driver: nfs.csi.k8s.io + readOnly: false + volumeHandle: unique-volumeid # make sure it's a unique id in the cluster + volumeAttributes: + server: nfs-server.default.svc.cluster.local + share: / diff --git a/examples/pvc-nfs-csi-dynamic.yaml b/examples/pvc-nfs-csi-dynamic.yaml new file mode 100644 index 00000000..cea7acb5 --- /dev/null +++ b/examples/pvc-nfs-csi-dynamic.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-nfs-dynamic +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 10Gi + storageClassName: nfs-csi \ No newline at end of file diff --git a/examples/pvc-nfs-csi-static.yaml b/examples/pvc-nfs-csi-static.yaml new file mode 100644 index 00000000..b52186fd --- /dev/null +++ b/examples/pvc-nfs-csi-static.yaml @@ -0,0 +1,13 @@ +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: pvc-nfs-static +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 10Gi + volumeName: pv-nfs + storageClassName: "" diff --git a/examples/kubernetes/statefulset.yaml b/examples/statefulset.yaml similarity index 98% rename from examples/kubernetes/statefulset.yaml rename to examples/statefulset.yaml index 96b53112..99bc8aed 100644 --- a/examples/kubernetes/statefulset.yaml +++ b/examples/statefulset.yaml @@ -7,7 +7,7 @@ metadata: app: nginx spec: serviceName: statefulset-nfs - replicas: 1 + replicas: 3 template: metadata: labels: diff --git a/examples/kubernetes/storageclass-nfs.yaml b/examples/storageclass-nfs.yaml similarity index 100% rename from examples/kubernetes/storageclass-nfs.yaml rename to examples/storageclass-nfs.yaml diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 0d02479b..90d39986 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -10,6 +10,8 @@ import ( "strings" "testing" + "time" + "github.com/kubernetes-csi/csi-driver-nfs/pkg/nfs" "github.com/onsi/ginkgo" "github.com/onsi/gomega" @@ -76,14 +78,45 @@ var _ = ginkgo.BeforeSuite(func() { }) var _ = ginkgo.AfterSuite(func() { + createExampleDeployment := testCmd{ + command: "make", + args: []string{"create-example-deployment"}, + startLog: "create example deployments", + endLog: "example deployments created", + } + execTestCmd([]testCmd{createExampleDeployment}) + // sleep 120s waiting for deployment running complete + time.Sleep(120 * time.Second) + + nfsLog := testCmd{ + command: "bash", + args: []string{"test/utils/nfs_log.sh"}, + startLog: "===================nfs log===================", + endLog: "==================================================", + } + e2eTeardown := testCmd{ command: "make", args: []string{"e2e-teardown"}, startLog: "Uninstalling NFS CSI Driver...", endLog: "NFS Driver uninstalled", } - execTestCmd([]testCmd{e2eTeardown}) + execTestCmd([]testCmd{nfsLog, e2eTeardown}) + // install/uninstall CSI Driver deployment scripts test + installDriver := testCmd{ + command: "bash", + args: []string{"deploy/install-driver.sh", "master", "local"}, + startLog: "===================install CSI Driver deployment scripts test===================", + endLog: "===================================================", + } + uninstallDriver := testCmd{ + command: "bash", + args: []string{"deploy/uninstall-driver.sh", "master", "local"}, + startLog: "===================uninstall CSI Driver deployment scripts test===================", + endLog: "===================================================", + } + execTestCmd([]testCmd{installDriver, uninstallDriver}) }) // handleFlags sets up all flags and parses the command line. diff --git a/test/utils/nfs_log.sh b/test/utils/nfs_log.sh new file mode 100644 index 00000000..9ae8de47 --- /dev/null +++ b/test/utils/nfs_log.sh @@ -0,0 +1,46 @@ +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash + +set -e + +NS=kube-system +CONTAINER=nfs + +echo "print out all nodes status ..." +kubectl get nodes -o wide +echo "======================================================================================" + +echo "print out all default namespace pods status ..." +kubectl get pods -n default -o wide +echo "======================================================================================" + +echo "print out all $NS namespace pods status ..." +kubectl get pods -n${NS} +echo "======================================================================================" + +echo "print out csi-nfs-controller logs ..." +echo "======================================================================================" +LABEL='app=csi-nfs-controller' +kubectl get pods -n${NS} -l${LABEL} \ + | awk 'NR>1 {print $1}' \ + | xargs -I {} kubectl logs {} --prefix -c${CONTAINER} -n${NS} + +echo "print out csi-nfs-node logs ..." +echo "======================================================================================" +LABEL='app=csi-nfs-node' +kubectl get pods -n${NS} -l${LABEL} \ + | awk 'NR>1 {print $1}' \ + | xargs -I {} kubectl logs {} --prefix -c${CONTAINER} -n${NS} diff --git a/test/utils/testutils/testutils.go b/test/utils/testutils/testutils.go deleted file mode 100644 index 38d3705a..00000000 --- a/test/utils/testutils/testutils.go +++ /dev/null @@ -1,21 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package testutil - -import "os" - -func IsRunningInProw() bool { - _, ok := os.LookupEnv("AZURE_CREDENTIALS") - return ok -}