diff --git a/app/nfsplugin/main.go b/app/nfsplugin/main.go new file mode 100644 index 00000000..6b68e9a0 --- /dev/null +++ b/app/nfsplugin/main.go @@ -0,0 +1,70 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/kubernetes-csi/drivers/pkg/nfs" +) + +var ( + endpoint string + nodeID string +) + +func init() { + flag.Set("logtostderr", "true") +} + +func main() { + + flag.CommandLine.Parse([]string{}) + + cmd := &cobra.Command{ + Use: "NFS", + Short: "CSI based NFS driver", + Run: func(cmd *cobra.Command, args []string) { + handle() + }, + } + + cmd.Flags().AddGoFlagSet(flag.CommandLine) + + cmd.PersistentFlags().StringVar(&nodeID, "nodeid", "", "node id") + cmd.MarkPersistentFlagRequired("nodeid") + + cmd.PersistentFlags().StringVar(&endpoint, "endpoint", "", "CSI endpoint") + cmd.MarkPersistentFlagRequired("endpoint") + + cmd.ParseFlags(os.Args[1:]) + if err := cmd.Execute(); err != nil { + fmt.Fprintf(os.Stderr, "%s", err.Error()) + os.Exit(1) + } + + os.Exit(0) +} + +func handle() { + d := nfs.NewDriver(nodeID, endpoint) + d.Run() +} diff --git a/pkg/nfs/README.md b/pkg/nfs/README.md new file mode 100644 index 00000000..744eb270 --- /dev/null +++ b/pkg/nfs/README.md @@ -0,0 +1,69 @@ +# CSI NFS driver + + +## Kubernetes +### Requirements + +The folllowing feature gates and runtime config have to be enabled to deploy the driver + +``` +FEATURE_GATES=CSIPersistentVolume=true,MountPropagation=true +RUNTIME_CONFIG="storage.k8s.io/v1alpha1=true" +``` + +Mountprogpation requries support for privileged containers. So, make sure privileged containers are enabled in the cluster. + +### Example local-up-cluster.sh + +```ALLOW_PRIVILEGED=true FEATURE_GATES=CSIPersistentVolume=true,MountPropagation=true RUNTIME_CONFIG="storage.k8s.io/v1alpha1=true" LOG_LEVEL=5 hack/local-up-cluster.sh``` + +### Deploy + +```kubectl -f deploy/kubernetes create``` + +### Example Nginx application +Please update the NFS Server & share information in nginx.yaml file. + +```kubectl -f examples/kubernetes/nginx.yaml create``` + +## Using CSC tool + +### Build nfsplugin +``` +$ make nfs +``` + +### Start NFS driver +``` +$ sudo ./_output/nfsplugin --endpoint tcp://127.0.0.1:10000 --nodeid CSINode -v=5 +``` + +## Test +Get ```csc``` tool from https://github.com/rexray/gocsi/tree/master/csc + +#### Get plugin info +``` +$ csc identity plugin-info --endpoint tcp://127.0.0.1:10000 +"NFS" "0.1.0" +``` + +#### NodePublish a volume +``` +$ export NFS_SERVER="Your Server IP (Ex: 10.10.10.10)" +$ export NFS_SHARE="Your NFS share" +$ csc node publish --endpoint tcp://127.0.0.1:10000 --target-path /mnt/nfs --attrib server=$NFS_SERVER --attrib share=$NFS_SHARE nfstestvol +nfstestvol +``` + +#### NodeUnpublish a volume +``` +$ csc node unpublish --endpoint tcp://127.0.0.1:10000 --target-path /mnt/nfs nfstestvol +nfstestvol +``` + +#### Get NodeID +``` +$ csc node get-id --endpoint tcp://127.0.0.1:10000 +CSINode +``` + diff --git a/pkg/nfs/deploy/kubernetes/csi-attacher-nfsplugin.yaml b/pkg/nfs/deploy/kubernetes/csi-attacher-nfsplugin.yaml new file mode 100644 index 00000000..c8165fe8 --- /dev/null +++ b/pkg/nfs/deploy/kubernetes/csi-attacher-nfsplugin.yaml @@ -0,0 +1,64 @@ +# This YAML file contains attacher & csi driver API objects that are necessary +# to run external CSI attacher for nfs + +kind: Service +apiVersion: v1 +metadata: + name: csi-attacher-nfsplugin + labels: + app: csi-attacher-nfsplugin +spec: + selector: + app: csi-attacher-nfsplugin + ports: + - name: dummy + port: 12345 + +--- +kind: StatefulSet +apiVersion: apps/v1beta1 +metadata: + name: csi-attacher-nfsplugin +spec: + serviceName: "csi-attacher" + replicas: 1 + template: + metadata: + labels: + app: csi-attacher-nfsplugin + spec: + serviceAccount: csi-attacher + containers: + - name: csi-attacher + image: quay.io/k8scsi/csi-attacher:v0.3.0 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + + - name: nfs + image: quay.io/k8scsi/nfsplugin:v0.3.0 + args : + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + env: + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix://plugin/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /plugin + volumes: + - name: socket-dir + emptyDir: + diff --git a/pkg/nfs/deploy/kubernetes/csi-attacher-rbac.yaml b/pkg/nfs/deploy/kubernetes/csi-attacher-rbac.yaml new file mode 100644 index 00000000..975fdd67 --- /dev/null +++ b/pkg/nfs/deploy/kubernetes/csi-attacher-rbac.yaml @@ -0,0 +1,37 @@ +# This YAML file contains RBAC API objects that are necessary to run external +# CSI attacher for nfs flex adapter + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-attacher + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: external-attacher-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-attacher-role +subjects: + - kind: ServiceAccount + name: csi-attacher + namespace: default +roleRef: + kind: ClusterRole + name: external-attacher-runner + apiGroup: rbac.authorization.k8s.io diff --git a/pkg/nfs/deploy/kubernetes/csi-nodeplugin-nfsplugin.yaml b/pkg/nfs/deploy/kubernetes/csi-nodeplugin-nfsplugin.yaml new file mode 100644 index 00000000..457e6f27 --- /dev/null +++ b/pkg/nfs/deploy/kubernetes/csi-nodeplugin-nfsplugin.yaml @@ -0,0 +1,66 @@ +# This YAML file contains driver-registrar & csi driver nodeplugin API objects +# that are necessary to run CSI nodeplugin for nfs +kind: DaemonSet +apiVersion: apps/v1beta2 +metadata: + name: csi-nodeplugin-nfsplugin +spec: + selector: + matchLabels: + app: csi-nodeplugin-nfsplugin + template: + metadata: + labels: + app: csi-nodeplugin-nfsplugin + spec: + serviceAccount: csi-nodeplugin + hostNetwork: true + containers: + - name: driver-registrar + image: quay.io/k8scsi/driver-registrar:v0.3.0 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /plugin/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: plugin-dir + mountPath: /plugin + - name: nfs + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: quay.io/k8scsi/nfsplugin:v0.3.0 + args : + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + env: + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix://plugin/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: plugin-dir + mountPath: /plugin + - name: pods-mount-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: "Bidirectional" + volumes: + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/csi-nfsplugin + type: DirectoryOrCreate + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet/pods + type: Directory diff --git a/pkg/nfs/deploy/kubernetes/csi-nodeplugin-rbac.yaml b/pkg/nfs/deploy/kubernetes/csi-nodeplugin-rbac.yaml new file mode 100644 index 00000000..530e067b --- /dev/null +++ b/pkg/nfs/deploy/kubernetes/csi-nodeplugin-rbac.yaml @@ -0,0 +1,34 @@ +# This YAML defines all API objects to create RBAC roles for CSI node plugin +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-nodeplugin + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-nodeplugin +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-nodeplugin +subjects: + - kind: ServiceAccount + name: csi-nodeplugin + namespace: default +roleRef: + kind: ClusterRole + name: csi-nodeplugin + apiGroup: rbac.authorization.k8s.io diff --git a/pkg/nfs/dockerfile/Dockerfile b/pkg/nfs/dockerfile/Dockerfile new file mode 100644 index 00000000..8306ca37 --- /dev/null +++ b/pkg/nfs/dockerfile/Dockerfile @@ -0,0 +1,8 @@ +FROM centos:7.4.1708 + +# Copy nfsplugin from build _output directory +COPY nfsplugin /nfsplugin + +RUN yum -y install nfs-utils && yum -y install epel-release && yum -y install jq && yum clean all + +ENTRYPOINT ["/nfsplugin"] diff --git a/pkg/nfs/driver.go b/pkg/nfs/driver.go new file mode 100644 index 00000000..cc801e9f --- /dev/null +++ b/pkg/nfs/driver.go @@ -0,0 +1,78 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nfs + +import ( + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/golang/glog" + + "github.com/kubernetes-csi/drivers/pkg/csi-common" +) + +type driver struct { + csiDriver *csicommon.CSIDriver + endpoint string + + ids *csicommon.DefaultIdentityServer + ns *nodeServer + + cap []*csi.VolumeCapability_AccessMode + cscap []*csi.ControllerServiceCapability +} + +const ( + driverName = "csi-nfsplugin" +) + +var ( + version = "1.0.0-rc2" +) + +func NewDriver(nodeID, endpoint string) *driver { + glog.Infof("Driver: %v version: %v", driverName, version) + + d := &driver{} + + d.endpoint = endpoint + + csiDriver := csicommon.NewCSIDriver(driverName, version, nodeID) + csiDriver.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER}) + // NFS plugin does not support ControllerServiceCapability now. + // If support is added, it should set to appropriate + // ControllerServiceCapability RPC types. + csiDriver.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{csi.ControllerServiceCapability_RPC_UNKNOWN}) + + d.csiDriver = csiDriver + + return d +} + +func NewNodeServer(d *driver) *nodeServer { + return &nodeServer{ + DefaultNodeServer: csicommon.NewDefaultNodeServer(d.csiDriver), + } +} + +func (d *driver) Run() { + s := csicommon.NewNonBlockingGRPCServer() + s.Start(d.endpoint, + csicommon.NewDefaultIdentityServer(d.csiDriver), + // NFS plugin has not implemented ControllerServer. + nil, + NewNodeServer(d)) + s.Wait() +} diff --git a/pkg/nfs/examples/kubernetes/nginx.yaml b/pkg/nfs/examples/kubernetes/nginx.yaml new file mode 100644 index 00000000..8048e9cd --- /dev/null +++ b/pkg/nfs/examples/kubernetes/nginx.yaml @@ -0,0 +1,53 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: data-nfsplugin + labels: + name: data-nfsplugin +spec: + accessModes: + - ReadWriteMany + capacity: + storage: 100Gi + csi: + driver: csi-nfsplugin + volumeHandle: data-id + volumeAttributes: + server: 127.0.0.1 + share: /export +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: data-nfsplugin +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 100Gi + selector: + matchExpressions: + - key: name + operator: In + values: ["data-nfsplugin"] +--- +apiVersion: v1 +kind: Pod +metadata: + name: nginx +spec: + containers: + - image: maersk/nginx + imagePullPolicy: Always + name: nginx + ports: + - containerPort: 80 + protocol: TCP + volumeMounts: + - mountPath: /var/www + name: data-nfsplugin + volumes: + - name: data-nfsplugin + persistentVolumeClaim: + claimName: data-nfsplugin diff --git a/pkg/nfs/nodeserver.go b/pkg/nfs/nodeserver.go new file mode 100644 index 00000000..36ced455 --- /dev/null +++ b/pkg/nfs/nodeserver.go @@ -0,0 +1,109 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nfs + +import ( + "fmt" + "os" + "strings" + + "github.com/container-storage-interface/spec/lib/go/csi" + "golang.org/x/net/context" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "k8s.io/kubernetes/pkg/util/mount" + "k8s.io/kubernetes/pkg/volume/util" + + "github.com/kubernetes-csi/drivers/pkg/csi-common" +) + +type nodeServer struct { + *csicommon.DefaultNodeServer +} + +func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { + targetPath := req.GetTargetPath() + notMnt, err := mount.New("").IsLikelyNotMountPoint(targetPath) + if err != nil { + if os.IsNotExist(err) { + if err := os.MkdirAll(targetPath, 0750); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + notMnt = true + } else { + return nil, status.Error(codes.Internal, err.Error()) + } + } + + if !notMnt { + return &csi.NodePublishVolumeResponse{}, nil + } + + mo := req.GetVolumeCapability().GetMount().GetMountFlags() + if req.GetReadonly() { + mo = append(mo, "ro") + } + + s := req.GetVolumeContext()["server"] + ep := req.GetVolumeContext()["share"] + source := fmt.Sprintf("%s:%s", s, ep) + + mounter := mount.New("") + err = mounter.Mount(source, targetPath, "nfs", mo) + if err != nil { + if os.IsPermission(err) { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + if strings.Contains(err.Error(), "invalid argument") { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + return nil, status.Error(codes.Internal, err.Error()) + } + + return &csi.NodePublishVolumeResponse{}, nil +} + +func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { + targetPath := req.GetTargetPath() + notMnt, err := mount.New("").IsLikelyNotMountPoint(targetPath) + + if err != nil { + if os.IsNotExist(err) { + return nil, status.Error(codes.NotFound, "Targetpath not found") + } else { + return nil, status.Error(codes.Internal, err.Error()) + } + } + if notMnt { + return nil, status.Error(codes.NotFound, "Volume not mounted") + } + + err = util.UnmountPath(req.GetTargetPath(), mount.New("")) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + return &csi.NodeUnpublishVolumeResponse{}, nil +} + +func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { + return &csi.NodeUnstageVolumeResponse{}, nil +} + +func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { + return &csi.NodeStageVolumeResponse{}, nil +}