Merge pull request #430 from wozniakjan/snapshots_with_tarballs

feat: implement snapshots as tarballs

charts/latest/csi-driver-nfs/templates/csi-nfs-controller.yaml

@@ -63,6 +63,22 @@ spec:
           resources: {{- toYaml .Values.controller.resources.csiProvisioner | nindent 12 }}
           securityContext:
             readOnlyRootFilesystem: true
+        {{- if .Values.externalSnapshotter.enabled }}
+        - name: csi-snapshotter
+          image: "{{ .Values.image.csiSnapshotter.repository }}:{{ .Values.image.csiSnapshotter.tag }}"
+          args:
+            - "--v=5"
+            - "--csi-address=$(ADDRESS)"
+            - "--leader-election-namespace={{ .Release.Namespace }}"
+            - "--leader-election"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          imagePullPolicy: {{ .Values.image.csiSnapshotter.pullPolicy }}
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        {{- end }}
         - name: liveness-probe
           image: "{{ .Values.image.livenessProbe.repository }}:{{ .Values.image.livenessProbe.tag }}"
           args:

charts/latest/csi-driver-nfs/templates/csi-snapshot-controller.yaml

@@ -0,0 +1,51 @@
+{{- if .Values.externalSnapshotter.enabled -}}
+# This YAML file shows how to deploy the snapshot controller
+
+# The snapshot controller implements the control loop for CSI snapshot functionality.
+# It should be installed as part of the base Kubernetes distribution in an appropriate
+# namespace for components implementing base system functionality. For installing with
+# Vanilla Kubernetes, kube-system makes sense for the namespace.
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: {{ .Values.externalSnapshotter.name }}
+  namespace: {{ .Release.Namespace }}
+{{ include "nfs.labels" . | indent 2 }}
+    app: {{ .Values.externalSnapshotter.name }}
+{{- with .Values.externalSnapshotter.labels }}
+{{ . | toYaml | indent 4 }}
+{{- end }}
+{{- with .Values.externalSnapshotter.annotations }}
+  annotations:
+{{ . | toYaml | indent 4 }}
+{{- end }}
+spec:
+  replicas: {{ .Values.externalSnapshotter.controller.replicas }}
+  selector:
+    matchLabels:
+      app: {{ .Values.externalSnapshotter.name }}
+  # the snapshot controller won't be marked as ready if the v1 CRDs are unavailable
+  # in #504 the snapshot-controller will exit after around 7.5 seconds if it
+  # can't find the v1 CRDs so this value should be greater than that
+  minReadySeconds: 15
+  strategy:
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: {{ .Values.externalSnapshotter.name }}
+    spec:
+      serviceAccountName: {{ .Values.externalSnapshotter.name }}
+      containers:
+        - name: {{ .Values.externalSnapshotter.name }}
+          image: {{ .Values.image.externalSnapshotter.repository }}:{{ .Values.image.externalSnapshotter.tag }}
+          args:
+            - "--v=5"
+            - "--leader-election=true"
+            - "--leader-election-namespace={{ .Release.Namespace }}"
+          imagePullPolicy: {{ .Values.image.externalSnapshotter.pullPolicy }}
+{{- end -}}

charts/latest/csi-driver-nfs/templates/rbac-csi-nfs.yaml

@@ -32,6 +32,17 @@ rules:
   - apiGroups: ["storage.k8s.io"]
     resources: ["storageclasses"]
     verbs: ["get", "list", "watch"]
+  {{- if .Values.externalSnapshotter.enabled }}
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses", "volumesnapshots"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["get", "update", "patch"]
+  {{- end }}
   - apiGroups: [""]
     resources: ["events"]
     verbs: ["get", "list", "watch", "create", "update", "patch"]

charts/latest/csi-driver-nfs/templates/rbac-snapshot-controller.yaml

@@ -0,0 +1,93 @@
+{{- if .Values.externalSnapshotter.enabled -}}
+# RBAC file for the snapshot controller.
+#
+# The snapshot controller implements the control loop for CSI snapshot functionality.
+# It should be installed as part of the base Kubernetes distribution in an appropriate
+# namespace for components implementing base system functionality. For installing with
+# Vanilla Kubernetes, kube-system makes sense for the namespace.
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Values.externalSnapshotter.name }}
+  namespace: {{ .Release.Namespace }}
+{{ include "nfs.labels" . | indent 2 }}
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ .Values.externalSnapshotter.name }}-runner
+{{ include "nfs.labels" . | indent 2 }}
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots/status"]
+    verbs: ["update", "patch"]
+{{- if .Values.externalSnapshotter.enabledDistributedSnapshotting }}
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+{{- end }}
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ .Values.externalSnapshotter.name }}-role
+{{ include "nfs.labels" . | indent 2 }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.externalSnapshotter.name }}
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: {{ .Values.externalSnapshotter.name }}-runner
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ .Values.externalSnapshotter.name }}-leaderelection
+  namespace: {{ .Release.Namespace }}
+{{ include "nfs.labels" . | indent 2 }}
+rules:
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ .Values.externalSnapshotter.name }}-leaderelection
+  namespace: {{ .Release.Namespace }}
+{{ include "nfs.labels" . | indent 2 }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.externalSnapshotter.name }}
+roleRef:
+  kind: Role
+  name: {{ .Values.externalSnapshotter.name }}-leaderelection
+  apiGroup: rbac.authorization.k8s.io
+{{- end -}}

charts/latest/csi-driver-nfs/templates/snapshot.storage.k8s.io_volumesnapshotclasses.yaml

@@ -0,0 +1,146 @@
+{{- if .Values.externalSnapshotter.enabled -}}
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.8.0
+    api-approved.kubernetes.io: https://github.com/kubernetes-csi/external-snapshotter/pull/665
+  name: volumesnapshotclasses.snapshot.storage.k8s.io
+spec:
+  group: snapshot.storage.k8s.io
+  names:
+    kind: VolumeSnapshotClass
+    listKind: VolumeSnapshotClassList
+    plural: volumesnapshotclasses
+    shortNames: [vsclass, vsclasses]
+    singular: volumesnapshotclass
+  scope: Cluster
+  versions:
+    - additionalPrinterColumns:
+        - jsonPath: .driver
+          name: Driver
+          type: string
+        - description: Determines whether a VolumeSnapshotContent created through
+            the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot
+            is deleted.
+          jsonPath: .deletionPolicy
+          name: DeletionPolicy
+          type: string
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshotClass specifies parameters that a underlying
+            storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass
+            is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses
+            are non-namespaced
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+                of an object. Servers should convert recognized schemas to the latest
+                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            deletionPolicy:
+              description: deletionPolicy determines whether a VolumeSnapshotContent
+                created through the VolumeSnapshotClass should be deleted when its
+                bound VolumeSnapshot is deleted. Supported values are "Retain" and
+                "Delete". "Retain" means that the VolumeSnapshotContent and its physical
+                snapshot on underlying storage system are kept. "Delete" means that
+                the VolumeSnapshotContent and its physical snapshot on underlying
+                storage system are deleted. Required.
+              enum: [Delete, Retain]
+              type: string
+            driver:
+              description: driver is the name of the storage driver that handles this
+                VolumeSnapshotClass. Required.
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource
+                this object represents. Servers may infer this from the endpoint the
+                client submits requests to. Cannot be updated. In CamelCase. More
+                info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            parameters:
+              additionalProperties:
+                type: string
+              description: parameters is a key-value map with storage driver specific
+                parameters for creating snapshots. These values are opaque to Kubernetes.
+              type: object
+          required: [deletionPolicy, driver]
+          type: object
+      served: true
+      storage: true
+      subresources: {}
+    - additionalPrinterColumns:
+        - jsonPath: .driver
+          name: Driver
+          type: string
+        - description: Determines whether a VolumeSnapshotContent created through
+            the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot
+            is deleted.
+          jsonPath: .deletionPolicy
+          name: DeletionPolicy
+          type: string
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1beta1
+      # This indicates the v1beta1 version of the custom resource is deprecated.
+      # API requests to this version receive a warning in the server response.
+      deprecated: true
+      # This overrides the default warning returned to clients making v1beta1 API requests.
+      deprecationWarning: snapshot.storage.k8s.io/v1beta1 VolumeSnapshotClass is deprecated;
+        use snapshot.storage.k8s.io/v1 VolumeSnapshotClass
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshotClass specifies parameters that a underlying
+            storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass
+            is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses
+            are non-namespaced
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+                of an object. Servers should convert recognized schemas to the latest
+                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            deletionPolicy:
+              description: deletionPolicy determines whether a VolumeSnapshotContent
+                created through the VolumeSnapshotClass should be deleted when its
+                bound VolumeSnapshot is deleted. Supported values are "Retain" and
+                "Delete". "Retain" means that the VolumeSnapshotContent and its physical
+                snapshot on underlying storage system are kept. "Delete" means that
+                the VolumeSnapshotContent and its physical snapshot on underlying
+                storage system are deleted. Required.
+              enum: [Delete, Retain]
+              type: string
+            driver:
+              description: driver is the name of the storage driver that handles this
+                VolumeSnapshotClass. Required.
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource
+                this object represents. Servers may infer this from the endpoint the
+                client submits requests to. Cannot be updated. In CamelCase. More
+                info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            parameters:
+              additionalProperties:
+                type: string
+              description: parameters is a key-value map with storage driver specific
+                parameters for creating snapshots. These values are opaque to Kubernetes.
+              type: object
+          required: [deletionPolicy, driver]
+          type: object
+      served: false
+      storage: false
+      subresources: {}
+status:
+  acceptedNames:
+    kind: ''
+    plural: ''
+  conditions: []
+  storedVersions: []
+{{- end -}}

charts/latest/csi-driver-nfs/templates/snapshot.storage.k8s.io_volumesnapshotcontents.yaml

@@ -0,0 +1,473 @@
+{{- if .Values.externalSnapshotter.enabled -}}
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.8.0
+    api-approved.kubernetes.io: https://github.com/kubernetes-csi/external-snapshotter/pull/665
+  name: volumesnapshotcontents.snapshot.storage.k8s.io
+spec:
+  group: snapshot.storage.k8s.io
+  names:
+    kind: VolumeSnapshotContent
+    listKind: VolumeSnapshotContentList
+    plural: volumesnapshotcontents
+    shortNames: [vsc, vscs]
+    singular: volumesnapshotcontent
+  scope: Cluster
+  versions:
+    - additionalPrinterColumns:
+        - description: Indicates if the snapshot is ready to be used to restore a
+            volume.
+          jsonPath: .status.readyToUse
+          name: ReadyToUse
+          type: boolean
+        - description: Represents the complete size of the snapshot in bytes
+          jsonPath: .status.restoreSize
+          name: RestoreSize
+          type: integer
+        - description: Determines whether this VolumeSnapshotContent and its physical
+            snapshot on the underlying storage system should be deleted when its bound
+            VolumeSnapshot is deleted.
+          jsonPath: .spec.deletionPolicy
+          name: DeletionPolicy
+          type: string
+        - description: Name of the CSI driver used to create the physical snapshot
+            on the underlying storage system.
+          jsonPath: .spec.driver
+          name: Driver
+          type: string
+        - description: Name of the VolumeSnapshotClass to which this snapshot belongs.
+          jsonPath: .spec.volumeSnapshotClassName
+          name: VolumeSnapshotClass
+          type: string
+        - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent
+            object is bound.
+          jsonPath: .spec.volumeSnapshotRef.name
+          name: VolumeSnapshot
+          type: string
+        - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent
+            object is bound.
+          jsonPath: .spec.volumeSnapshotRef.namespace
+          name: VolumeSnapshotNamespace
+          type: string
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshotContent represents the actual "on-disk" snapshot
+            object in the underlying storage system
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+                of an object. Servers should convert recognized schemas to the latest
+                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource
+                this object represents. Servers may infer this from the endpoint the
+                client submits requests to. Cannot be updated. In CamelCase. More
+                info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            spec:
+              description: spec defines properties of a VolumeSnapshotContent created
+                by the underlying storage system. Required.
+              properties:
+                deletionPolicy:
+                  description: deletionPolicy determines whether this VolumeSnapshotContent
+                    and its physical snapshot on the underlying storage system should
+                    be deleted when its bound VolumeSnapshot is deleted. Supported
+                    values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent
+                    and its physical snapshot on underlying storage system are kept.
+                    "Delete" means that the VolumeSnapshotContent and its physical
+                    snapshot on underlying storage system are deleted. For dynamically
+                    provisioned snapshots, this field will automatically be filled
+                    in by the CSI snapshotter sidecar with the "DeletionPolicy" field
+                    defined in the corresponding VolumeSnapshotClass. For pre-existing
+                    snapshots, users MUST specify this field when creating the VolumeSnapshotContent
+                    object. Required.
+                  enum: [Delete, Retain]
+                  type: string
+                driver:
+                  description: driver is the name of the CSI driver used to create
+                    the physical snapshot on the underlying storage system. This MUST
+                    be the same as the name returned by the CSI GetPluginName() call
+                    for that driver. Required.
+                  type: string
+                source:
+                  description: source specifies whether the snapshot is (or should
+                    be) dynamically provisioned or already exists, and just requires
+                    a Kubernetes object representation. This field is immutable after
+                    creation. Required.
+                  properties:
+                    snapshotHandle:
+                      description: snapshotHandle specifies the CSI "snapshot_id"
+                        of a pre-existing snapshot on the underlying storage system
+                        for which a Kubernetes object representation was (or should
+                        be) created. This field is immutable.
+                      type: string
+                    volumeHandle:
+                      description: volumeHandle specifies the CSI "volume_id" of the
+                        volume from which a snapshot should be dynamically taken from.
+                        This field is immutable.
+                      type: string
+                  type: object
+                  oneOf:
+                    - required: [snapshotHandle]
+                    - required: [volumeHandle]
+                sourceVolumeMode:
+                  description: SourceVolumeMode is the mode of the volume whose snapshot
+                    is taken. Can be either “Filesystem” or “Block”. If not specified,
+                    it indicates the source volume's mode is unknown. This field is
+                    immutable. This field is an alpha field.
+                  type: string
+                volumeSnapshotClassName:
+                  description: name of the VolumeSnapshotClass from which this snapshot
+                    was (or will be) created. Note that after provisioning, the VolumeSnapshotClass
+                    may be deleted or recreated with different set of values, and
+                    as such, should not be referenced post-snapshot creation.
+                  type: string
+                volumeSnapshotRef:
+                  description: volumeSnapshotRef specifies the VolumeSnapshot object
+                    to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName
+                    field must reference to this VolumeSnapshotContent's name for
+                    the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent
+                    object, name and namespace of the VolumeSnapshot object MUST be
+                    provided for binding to happen. This field is immutable after
+                    creation. Required.
+                  properties:
+                    apiVersion:
+                      description: API version of the referent.
+                      type: string
+                    fieldPath:
+                      description: 'If referring to a piece of an object instead of
+                        an entire object, this string should contain a valid JSON/Go
+                        field access statement, such as desiredState.manifest.containers[2].
+                        For example, if the object reference is to a container within
+                        a pod, this would take on a value like: "spec.containers{name}"
+                        (where "name" refers to the name of the container that triggered
+                        the event) or if no container name is specified "spec.containers[2]"
+                        (container with index 2 in this pod). This syntax is chosen
+                        only to have some well-defined way of referencing a part of
+                        an object. TODO: this design is not final and this field is
+                        subject to change in the future.'
+                      type: string
+                    kind:
+                      description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                      type: string
+                    name:
+                      description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                      type: string
+                    namespace:
+                      description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
+                      type: string
+                    resourceVersion:
+                      description: 'Specific resourceVersion to which this reference
+                        is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
+                      type: string
+                    uid:
+                      description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                      type: string
+                  type: object
+              required: [deletionPolicy, driver, source, volumeSnapshotRef]
+              type: object
+            status:
+              description: status represents the current information of a snapshot.
+              properties:
+                creationTime:
+                  description: creationTime is the timestamp when the point-in-time
+                    snapshot is taken by the underlying storage system. In dynamic
+                    snapshot creation case, this field will be filled in by the CSI
+                    snapshotter sidecar with the "creation_time" value returned from
+                    CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this
+                    field will be filled with the "creation_time" value returned from
+                    the CSI "ListSnapshots" gRPC call if the driver supports it. If
+                    not specified, it indicates the creation time is unknown. The
+                    format of this field is a Unix nanoseconds time encoded as an
+                    int64. On Unix, the command `date +%s%N` returns the current time
+                    in nanoseconds since 1970-01-01 00:00:00 UTC.
+                  format: int64
+                  type: integer
+                error:
+                  description: error is the last observed error during snapshot creation,
+                    if any. Upon success after retry, this error field will be cleared.
+                  properties:
+                    message:
+                      description: 'message is a string detailing the encountered
+                        error during snapshot creation if specified. NOTE: message
+                        may be logged, and it should not contain sensitive information.'
+                      type: string
+                    time:
+                      description: time is the timestamp when the error was encountered.
+                      format: date-time
+                      type: string
+                  type: object
+                readyToUse:
+                  description: readyToUse indicates if a snapshot is ready to be used
+                    to restore a volume. In dynamic snapshot creation case, this field
+                    will be filled in by the CSI snapshotter sidecar with the "ready_to_use"
+                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "ready_to_use" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver
+                    supports it, otherwise, this field will be set to "True". If not
+                    specified, it means the readiness of a snapshot is unknown.
+                  type: boolean
+                restoreSize:
+                  description: restoreSize represents the complete size of the snapshot
+                    in bytes. In dynamic snapshot creation case, this field will be
+                    filled in by the CSI snapshotter sidecar with the "size_bytes"
+                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "size_bytes" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver
+                    supports it. When restoring a volume from this snapshot, the size
+                    of the volume MUST NOT be smaller than the restoreSize if it is
+                    specified, otherwise the restoration will fail. If not specified,
+                    it indicates that the size is unknown.
+                  format: int64
+                  minimum: 0
+                  type: integer
+                snapshotHandle:
+                  description: snapshotHandle is the CSI "snapshot_id" of a snapshot
+                    on the underlying storage system. If not specified, it indicates
+                    that dynamic snapshot creation has either failed or it is still
+                    in progress.
+                  type: string
+              type: object
+          required: [spec]
+          type: object
+      served: true
+      storage: true
+      subresources:
+        status: {}
+    - additionalPrinterColumns:
+        - description: Indicates if the snapshot is ready to be used to restore a
+            volume.
+          jsonPath: .status.readyToUse
+          name: ReadyToUse
+          type: boolean
+        - description: Represents the complete size of the snapshot in bytes
+          jsonPath: .status.restoreSize
+          name: RestoreSize
+          type: integer
+        - description: Determines whether this VolumeSnapshotContent and its physical
+            snapshot on the underlying storage system should be deleted when its bound
+            VolumeSnapshot is deleted.
+          jsonPath: .spec.deletionPolicy
+          name: DeletionPolicy
+          type: string
+        - description: Name of the CSI driver used to create the physical snapshot
+            on the underlying storage system.
+          jsonPath: .spec.driver
+          name: Driver
+          type: string
+        - description: Name of the VolumeSnapshotClass to which this snapshot belongs.
+          jsonPath: .spec.volumeSnapshotClassName
+          name: VolumeSnapshotClass
+          type: string
+        - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent
+            object is bound.
+          jsonPath: .spec.volumeSnapshotRef.name
+          name: VolumeSnapshot
+          type: string
+        - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent
+            object is bound.
+          jsonPath: .spec.volumeSnapshotRef.namespace
+          name: VolumeSnapshotNamespace
+          type: string
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1beta1
+      # This indicates the v1beta1 version of the custom resource is deprecated.
+      # API requests to this version receive a warning in the server response.
+      deprecated: true
+      # This overrides the default warning returned to clients making v1beta1 API requests.
+      deprecationWarning: snapshot.storage.k8s.io/v1beta1 VolumeSnapshotContent is
+        deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotContent
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshotContent represents the actual "on-disk" snapshot
+            object in the underlying storage system
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+                of an object. Servers should convert recognized schemas to the latest
+                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource
+                this object represents. Servers may infer this from the endpoint the
+                client submits requests to. Cannot be updated. In CamelCase. More
+                info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            spec:
+              description: spec defines properties of a VolumeSnapshotContent created
+                by the underlying storage system. Required.
+              properties:
+                deletionPolicy:
+                  description: deletionPolicy determines whether this VolumeSnapshotContent
+                    and its physical snapshot on the underlying storage system should
+                    be deleted when its bound VolumeSnapshot is deleted. Supported
+                    values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent
+                    and its physical snapshot on underlying storage system are kept.
+                    "Delete" means that the VolumeSnapshotContent and its physical
+                    snapshot on underlying storage system are deleted. For dynamically
+                    provisioned snapshots, this field will automatically be filled
+                    in by the CSI snapshotter sidecar with the "DeletionPolicy" field
+                    defined in the corresponding VolumeSnapshotClass. For pre-existing
+                    snapshots, users MUST specify this field when creating the  VolumeSnapshotContent
+                    object. Required.
+                  enum: [Delete, Retain]
+                  type: string
+                driver:
+                  description: driver is the name of the CSI driver used to create
+                    the physical snapshot on the underlying storage system. This MUST
+                    be the same as the name returned by the CSI GetPluginName() call
+                    for that driver. Required.
+                  type: string
+                source:
+                  description: source specifies whether the snapshot is (or should
+                    be) dynamically provisioned or already exists, and just requires
+                    a Kubernetes object representation. This field is immutable after
+                    creation. Required.
+                  properties:
+                    snapshotHandle:
+                      description: snapshotHandle specifies the CSI "snapshot_id"
+                        of a pre-existing snapshot on the underlying storage system
+                        for which a Kubernetes object representation was (or should
+                        be) created. This field is immutable.
+                      type: string
+                    volumeHandle:
+                      description: volumeHandle specifies the CSI "volume_id" of the
+                        volume from which a snapshot should be dynamically taken from.
+                        This field is immutable.
+                      type: string
+                  type: object
+                volumeSnapshotClassName:
+                  description: name of the VolumeSnapshotClass from which this snapshot
+                    was (or will be) created. Note that after provisioning, the VolumeSnapshotClass
+                    may be deleted or recreated with different set of values, and
+                    as such, should not be referenced post-snapshot creation.
+                  type: string
+                volumeSnapshotRef:
+                  description: volumeSnapshotRef specifies the VolumeSnapshot object
+                    to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName
+                    field must reference to this VolumeSnapshotContent's name for
+                    the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent
+                    object, name and namespace of the VolumeSnapshot object MUST be
+                    provided for binding to happen. This field is immutable after
+                    creation. Required.
+                  properties:
+                    apiVersion:
+                      description: API version of the referent.
+                      type: string
+                    fieldPath:
+                      description: 'If referring to a piece of an object instead of
+                        an entire object, this string should contain a valid JSON/Go
+                        field access statement, such as desiredState.manifest.containers[2].
+                        For example, if the object reference is to a container within
+                        a pod, this would take on a value like: "spec.containers{name}"
+                        (where "name" refers to the name of the container that triggered
+                        the event) or if no container name is specified "spec.containers[2]"
+                        (container with index 2 in this pod). This syntax is chosen
+                        only to have some well-defined way of referencing a part of
+                        an object. TODO: this design is not final and this field is
+                        subject to change in the future.'
+                      type: string
+                    kind:
+                      description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                      type: string
+                    name:
+                      description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                      type: string
+                    namespace:
+                      description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
+                      type: string
+                    resourceVersion:
+                      description: 'Specific resourceVersion to which this reference
+                        is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
+                      type: string
+                    uid:
+                      description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                      type: string
+                  type: object
+              required: [deletionPolicy, driver, source, volumeSnapshotRef]
+              type: object
+            status:
+              description: status represents the current information of a snapshot.
+              properties:
+                creationTime:
+                  description: creationTime is the timestamp when the point-in-time
+                    snapshot is taken by the underlying storage system. In dynamic
+                    snapshot creation case, this field will be filled in by the CSI
+                    snapshotter sidecar with the "creation_time" value returned from
+                    CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this
+                    field will be filled with the "creation_time" value returned from
+                    the CSI "ListSnapshots" gRPC call if the driver supports it. If
+                    not specified, it indicates the creation time is unknown. The
+                    format of this field is a Unix nanoseconds time encoded as an
+                    int64. On Unix, the command `date +%s%N` returns the current time
+                    in nanoseconds since 1970-01-01 00:00:00 UTC.
+                  format: int64
+                  type: integer
+                error:
+                  description: error is the last observed error during snapshot creation,
+                    if any. Upon success after retry, this error field will be cleared.
+                  properties:
+                    message:
+                      description: 'message is a string detailing the encountered
+                        error during snapshot creation if specified. NOTE: message
+                        may be logged, and it should not contain sensitive information.'
+                      type: string
+                    time:
+                      description: time is the timestamp when the error was encountered.
+                      format: date-time
+                      type: string
+                  type: object
+                readyToUse:
+                  description: readyToUse indicates if a snapshot is ready to be used
+                    to restore a volume. In dynamic snapshot creation case, this field
+                    will be filled in by the CSI snapshotter sidecar with the "ready_to_use"
+                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "ready_to_use" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver
+                    supports it, otherwise, this field will be set to "True". If not
+                    specified, it means the readiness of a snapshot is unknown.
+                  type: boolean
+                restoreSize:
+                  description: restoreSize represents the complete size of the snapshot
+                    in bytes. In dynamic snapshot creation case, this field will be
+                    filled in by the CSI snapshotter sidecar with the "size_bytes"
+                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "size_bytes" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver
+                    supports it. When restoring a volume from this snapshot, the size
+                    of the volume MUST NOT be smaller than the restoreSize if it is
+                    specified, otherwise the restoration will fail. If not specified,
+                    it indicates that the size is unknown.
+                  format: int64
+                  minimum: 0
+                  type: integer
+                snapshotHandle:
+                  description: snapshotHandle is the CSI "snapshot_id" of a snapshot
+                    on the underlying storage system. If not specified, it indicates
+                    that dynamic snapshot creation has either failed or it is still
+                    in progress.
+                  type: string
+              type: object
+          required: [spec]
+          type: object
+      served: false
+      storage: false
+      subresources:
+        status: {}
+status:
+  acceptedNames:
+    kind: ''
+    plural: ''
+  conditions: []
+  storedVersions: []
+{{- end -}}

charts/latest/csi-driver-nfs/templates/snapshot.storage.k8s.io_volumesnapshots.yaml

@@ -0,0 +1,387 @@
+{{- if .Values.externalSnapshotter.enabled -}}
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.8.0
+    api-approved.kubernetes.io: https://github.com/kubernetes-csi/external-snapshotter/pull/665
+  name: volumesnapshots.snapshot.storage.k8s.io
+spec:
+  group: snapshot.storage.k8s.io
+  names:
+    kind: VolumeSnapshot
+    listKind: VolumeSnapshotList
+    plural: volumesnapshots
+    shortNames: [vs]
+    singular: volumesnapshot
+  scope: Namespaced
+  versions:
+    - additionalPrinterColumns:
+        - description: Indicates if the snapshot is ready to be used to restore a
+            volume.
+          jsonPath: .status.readyToUse
+          name: ReadyToUse
+          type: boolean
+        - description: If a new snapshot needs to be created, this contains the name
+            of the source PVC from which this snapshot was (or will be) created.
+          jsonPath: .spec.source.persistentVolumeClaimName
+          name: SourcePVC
+          type: string
+        - description: If a snapshot already exists, this contains the name of the
+            existing VolumeSnapshotContent object representing the existing snapshot.
+          jsonPath: .spec.source.volumeSnapshotContentName
+          name: SourceSnapshotContent
+          type: string
+        - description: Represents the minimum size of volume required to rehydrate
+            from this snapshot.
+          jsonPath: .status.restoreSize
+          name: RestoreSize
+          type: string
+        - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot.
+          jsonPath: .spec.volumeSnapshotClassName
+          name: SnapshotClass
+          type: string
+        - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot
+            object intends to bind to. Please note that verification of binding actually
+            requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure
+            both are pointing at each other. Binding MUST be verified prior to usage
+            of this object.
+          jsonPath: .status.boundVolumeSnapshotContentName
+          name: SnapshotContent
+          type: string
+        - description: Timestamp when the point-in-time snapshot was taken by the
+            underlying storage system.
+          jsonPath: .status.creationTime
+          name: CreationTime
+          type: date
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshot is a user's request for either creating a point-in-time
+            snapshot of a persistent volume, or binding to a pre-existing snapshot.
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+                of an object. Servers should convert recognized schemas to the latest
+                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource
+                this object represents. Servers may infer this from the endpoint the
+                client submits requests to. Cannot be updated. In CamelCase. More
+                info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            spec:
+              description: 'spec defines the desired characteristics of a snapshot
+                requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots
+                Required.'
+              properties:
+                source:
+                  description: source specifies where a snapshot will be created from.
+                    This field is immutable after creation. Required.
+                  properties:
+                    persistentVolumeClaimName:
+                      description: persistentVolumeClaimName specifies the name of
+                        the PersistentVolumeClaim object representing the volume from
+                        which a snapshot should be created. This PVC is assumed to
+                        be in the same namespace as the VolumeSnapshot object. This
+                        field should be set if the snapshot does not exists, and needs
+                        to be created. This field is immutable.
+                      type: string
+                    volumeSnapshotContentName:
+                      description: volumeSnapshotContentName specifies the name of
+                        a pre-existing VolumeSnapshotContent object representing an
+                        existing volume snapshot. This field should be set if the
+                        snapshot already exists and only needs a representation in
+                        Kubernetes. This field is immutable.
+                      type: string
+                  type: object
+                  oneOf:
+                    - required: [persistentVolumeClaimName]
+                    - required: [volumeSnapshotContentName]
+                volumeSnapshotClassName:
+                  description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass
+                    requested by the VolumeSnapshot. VolumeSnapshotClassName may be
+                    left nil to indicate that the default SnapshotClass should be
+                    used. A given cluster may have multiple default Volume SnapshotClasses:
+                    one default per CSI Driver. If a VolumeSnapshot does not specify
+                    a SnapshotClass, VolumeSnapshotSource will be checked to figure
+                    out what the associated CSI Driver is, and the default VolumeSnapshotClass
+                    associated with that CSI Driver will be used. If more than one
+                    VolumeSnapshotClass exist for a given CSI Driver and more than
+                    one have been marked as default, CreateSnapshot will fail and
+                    generate an event. Empty string is not allowed for this field.'
+                  type: string
+              required: [source]
+              type: object
+            status:
+              description: status represents the current information of a snapshot.
+                Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent
+                objects is successful (by validating that both VolumeSnapshot and
+                VolumeSnapshotContent point at each other) before using this object.
+              properties:
+                boundVolumeSnapshotContentName:
+                  description: 'boundVolumeSnapshotContentName is the name of the
+                    VolumeSnapshotContent object to which this VolumeSnapshot object
+                    intends to bind to. If not specified, it indicates that the VolumeSnapshot
+                    object has not been successfully bound to a VolumeSnapshotContent
+                    object yet. NOTE: To avoid possible security issues, consumers
+                    must verify binding between VolumeSnapshot and VolumeSnapshotContent
+                    objects is successful (by validating that both VolumeSnapshot
+                    and VolumeSnapshotContent point at each other) before using this
+                    object.'
+                  type: string
+                creationTime:
+                  description: creationTime is the timestamp when the point-in-time
+                    snapshot is taken by the underlying storage system. In dynamic
+                    snapshot creation case, this field will be filled in by the snapshot
+                    controller with the "creation_time" value returned from CSI "CreateSnapshot"
+                    gRPC call. For a pre-existing snapshot, this field will be filled
+                    with the "creation_time" value returned from the CSI "ListSnapshots"
+                    gRPC call if the driver supports it. If not specified, it may
+                    indicate that the creation time of the snapshot is unknown.
+                  format: date-time
+                  type: string
+                error:
+                  description: error is the last observed error during snapshot creation,
+                    if any. This field could be helpful to upper level controllers(i.e.,
+                    application controller) to decide whether they should continue
+                    on waiting for the snapshot to be created based on the type of
+                    error reported. The snapshot controller will keep retrying when
+                    an error occurs during the snapshot creation. Upon success, this
+                    error field will be cleared.
+                  properties:
+                    message:
+                      description: 'message is a string detailing the encountered
+                        error during snapshot creation if specified. NOTE: message
+                        may be logged, and it should not contain sensitive information.'
+                      type: string
+                    time:
+                      description: time is the timestamp when the error was encountered.
+                      format: date-time
+                      type: string
+                  type: object
+                readyToUse:
+                  description: readyToUse indicates if the snapshot is ready to be
+                    used to restore a volume. In dynamic snapshot creation case, this
+                    field will be filled in by the snapshot controller with the "ready_to_use"
+                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "ready_to_use" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver
+                    supports it, otherwise, this field will be set to "True". If not
+                    specified, it means the readiness of a snapshot is unknown.
+                  type: boolean
+                restoreSize:
+                  type: string
+                  description: restoreSize represents the minimum size of volume required
+                    to create a volume from this snapshot. In dynamic snapshot creation
+                    case, this field will be filled in by the snapshot controller
+                    with the "size_bytes" value returned from CSI "CreateSnapshot"
+                    gRPC call. For a pre-existing snapshot, this field will be filled
+                    with the "size_bytes" value returned from the CSI "ListSnapshots"
+                    gRPC call if the driver supports it. When restoring a volume from
+                    this snapshot, the size of the volume MUST NOT be smaller than
+                    the restoreSize if it is specified, otherwise the restoration
+                    will fail. If not specified, it indicates that the size is unknown.
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
+              type: object
+          required: [spec]
+          type: object
+      served: true
+      storage: true
+      subresources:
+        status: {}
+    - additionalPrinterColumns:
+        - description: Indicates if the snapshot is ready to be used to restore a
+            volume.
+          jsonPath: .status.readyToUse
+          name: ReadyToUse
+          type: boolean
+        - description: If a new snapshot needs to be created, this contains the name
+            of the source PVC from which this snapshot was (or will be) created.
+          jsonPath: .spec.source.persistentVolumeClaimName
+          name: SourcePVC
+          type: string
+        - description: If a snapshot already exists, this contains the name of the
+            existing VolumeSnapshotContent object representing the existing snapshot.
+          jsonPath: .spec.source.volumeSnapshotContentName
+          name: SourceSnapshotContent
+          type: string
+        - description: Represents the minimum size of volume required to rehydrate
+            from this snapshot.
+          jsonPath: .status.restoreSize
+          name: RestoreSize
+          type: string
+        - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot.
+          jsonPath: .spec.volumeSnapshotClassName
+          name: SnapshotClass
+          type: string
+        - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot
+            object intends to bind to. Please note that verification of binding actually
+            requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure
+            both are pointing at each other. Binding MUST be verified prior to usage
+            of this object.
+          jsonPath: .status.boundVolumeSnapshotContentName
+          name: SnapshotContent
+          type: string
+        - description: Timestamp when the point-in-time snapshot was taken by the
+            underlying storage system.
+          jsonPath: .status.creationTime
+          name: CreationTime
+          type: date
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1beta1
+      # This indicates the v1beta1 version of the custom resource is deprecated.
+      # API requests to this version receive a warning in the server response.
+      deprecated: true
+      # This overrides the default warning returned to clients making v1beta1 API requests.
+      deprecationWarning: snapshot.storage.k8s.io/v1beta1 VolumeSnapshot is deprecated;
+        use snapshot.storage.k8s.io/v1 VolumeSnapshot
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshot is a user's request for either creating a point-in-time
+            snapshot of a persistent volume, or binding to a pre-existing snapshot.
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+                of an object. Servers should convert recognized schemas to the latest
+                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource
+                this object represents. Servers may infer this from the endpoint the
+                client submits requests to. Cannot be updated. In CamelCase. More
+                info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            spec:
+              description: 'spec defines the desired characteristics of a snapshot
+                requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots
+                Required.'
+              properties:
+                source:
+                  description: source specifies where a snapshot will be created from.
+                    This field is immutable after creation. Required.
+                  properties:
+                    persistentVolumeClaimName:
+                      description: persistentVolumeClaimName specifies the name of
+                        the PersistentVolumeClaim object representing the volume from
+                        which a snapshot should be created. This PVC is assumed to
+                        be in the same namespace as the VolumeSnapshot object. This
+                        field should be set if the snapshot does not exists, and needs
+                        to be created. This field is immutable.
+                      type: string
+                    volumeSnapshotContentName:
+                      description: volumeSnapshotContentName specifies the name of
+                        a pre-existing VolumeSnapshotContent object representing an
+                        existing volume snapshot. This field should be set if the
+                        snapshot already exists and only needs a representation in
+                        Kubernetes. This field is immutable.
+                      type: string
+                  type: object
+                volumeSnapshotClassName:
+                  description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass
+                    requested by the VolumeSnapshot. VolumeSnapshotClassName may be
+                    left nil to indicate that the default SnapshotClass should be
+                    used. A given cluster may have multiple default Volume SnapshotClasses:
+                    one default per CSI Driver. If a VolumeSnapshot does not specify
+                    a SnapshotClass, VolumeSnapshotSource will be checked to figure
+                    out what the associated CSI Driver is, and the default VolumeSnapshotClass
+                    associated with that CSI Driver will be used. If more than one
+                    VolumeSnapshotClass exist for a given CSI Driver and more than
+                    one have been marked as default, CreateSnapshot will fail and
+                    generate an event. Empty string is not allowed for this field.'
+                  type: string
+              required: [source]
+              type: object
+            status:
+              description: status represents the current information of a snapshot.
+                Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent
+                objects is successful (by validating that both VolumeSnapshot and
+                VolumeSnapshotContent point at each other) before using this object.
+              properties:
+                boundVolumeSnapshotContentName:
+                  description: 'boundVolumeSnapshotContentName is the name of the
+                    VolumeSnapshotContent object to which this VolumeSnapshot object
+                    intends to bind to. If not specified, it indicates that the VolumeSnapshot
+                    object has not been successfully bound to a VolumeSnapshotContent
+                    object yet. NOTE: To avoid possible security issues, consumers
+                    must verify binding between VolumeSnapshot and VolumeSnapshotContent
+                    objects is successful (by validating that both VolumeSnapshot
+                    and VolumeSnapshotContent point at each other) before using this
+                    object.'
+                  type: string
+                creationTime:
+                  description: creationTime is the timestamp when the point-in-time
+                    snapshot is taken by the underlying storage system. In dynamic
+                    snapshot creation case, this field will be filled in by the snapshot
+                    controller with the "creation_time" value returned from CSI "CreateSnapshot"
+                    gRPC call. For a pre-existing snapshot, this field will be filled
+                    with the "creation_time" value returned from the CSI "ListSnapshots"
+                    gRPC call if the driver supports it. If not specified, it may
+                    indicate that the creation time of the snapshot is unknown.
+                  format: date-time
+                  type: string
+                error:
+                  description: error is the last observed error during snapshot creation,
+                    if any. This field could be helpful to upper level controllers(i.e.,
+                    application controller) to decide whether they should continue
+                    on waiting for the snapshot to be created based on the type of
+                    error reported. The snapshot controller will keep retrying when
+                    an error occurs during the snapshot creation. Upon success, this
+                    error field will be cleared.
+                  properties:
+                    message:
+                      description: 'message is a string detailing the encountered
+                        error during snapshot creation if specified. NOTE: message
+                        may be logged, and it should not contain sensitive information.'
+                      type: string
+                    time:
+                      description: time is the timestamp when the error was encountered.
+                      format: date-time
+                      type: string
+                  type: object
+                readyToUse:
+                  description: readyToUse indicates if the snapshot is ready to be
+                    used to restore a volume. In dynamic snapshot creation case, this
+                    field will be filled in by the snapshot controller with the "ready_to_use"
+                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "ready_to_use" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver
+                    supports it, otherwise, this field will be set to "True". If not
+                    specified, it means the readiness of a snapshot is unknown.
+                  type: boolean
+                restoreSize:
+                  type: string
+                  description: restoreSize represents the minimum size of volume required
+                    to create a volume from this snapshot. In dynamic snapshot creation
+                    case, this field will be filled in by the snapshot controller
+                    with the "size_bytes" value returned from CSI "CreateSnapshot"
+                    gRPC call. For a pre-existing snapshot, this field will be filled
+                    with the "size_bytes" value returned from the CSI "ListSnapshots"
+                    gRPC call if the driver supports it. When restoring a volume from
+                    this snapshot, the size of the volume MUST NOT be smaller than
+                    the restoreSize if it is specified, otherwise the restoration
+                    will fail. If not specified, it indicates that the size is unknown.
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
+              type: object
+          required: [spec]
+          type: object
+      served: false
+      storage: false
+      subresources:
+        status: {}
+status:
+  acceptedNames:
+    kind: ''
+    plural: ''
+  conditions: []
+  storedVersions: []
+{{- end -}}

charts/latest/csi-driver-nfs/values.yaml

@@ -8,6 +8,10 @@ image:
         repository: registry.k8s.io/sig-storage/csi-provisioner
         tag: v3.3.0
         pullPolicy: IfNotPresent
+    csiSnapshotter:
+        repository: registry.k8s.io/sig-storage/csi-snapshotter
+        tag: v6.2.1
+        pullPolicy: IfNotPresent
     livenessProbe:
         repository: registry.k8s.io/sig-storage/livenessprobe
         tag: v2.8.0
@@ -16,6 +20,10 @@ image:
         repository: registry.k8s.io/sig-storage/csi-node-driver-registrar
         tag: v2.6.2
         pullPolicy: IfNotPresent
+    externalSnapshotter:
+        repository: registry.k8s.io/sig-storage/snapshot-controller
+        tag: v6.1.0
+        pullPolicy: IfNotPresent
 
 serviceAccount:
   create: true # When true, service accounts will be created for you. Set to false if you want to use your own.
@@ -110,6 +118,12 @@ node:
         cpu: 10m
         memory: 20Mi
 
+externalSnapshotter:
+  enabled: true
+  name: snapshot-controller
+  controller:
+    replicas: 1
+
 ## Reference to one or more secrets to be used when pulling images
 ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
 ##

deploy/csi-nfs-controller.yaml

@@ -51,6 +51,20 @@ spec:
             requests:
               cpu: 10m
               memory: 20Mi
+        - name: csi-snapshotter
+          image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1
+          args:
+            - "--v=5"
+            - "--csi-address=$(ADDRESS)"
+            - "--leader-election-namespace=kube-system"
+            - "--leader-election"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
         - name: liveness-probe
           image: registry.k8s.io/sig-storage/livenessprobe:v2.8.0
           args:

deploy/csi-snapshot-controller.yaml

@@ -0,0 +1,40 @@
+# This YAML file shows how to deploy the snapshot controller
+
+# The snapshot controller implements the control loop for CSI snapshot functionality.
+# It should be installed as part of the base Kubernetes distribution in an appropriate
+# namespace for components implementing base system functionality. For installing with
+# Vanilla Kubernetes, kube-system makes sense for the namespace.
+
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: snapshot-controller
+  namespace: kube-system
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: snapshot-controller
+  # the snapshot controller won't be marked as ready if the v1 CRDs are unavailable
+  # in #504 the snapshot-controller will exit after around 7.5 seconds if it
+  # can't find the v1 CRDs so this value should be greater than that
+  minReadySeconds: 15
+  strategy:
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: snapshot-controller
+    spec:
+      serviceAccountName: snapshot-controller
+      containers:
+        - name: snapshot-controller
+          image: registry.k8s.io/sig-storage/snapshot-controller:v6.1.0
+          args:
+            - "--v=5"
+            - "--leader-election=true"
+          imagePullPolicy: IfNotPresent

deploy/example/snapshot-nfs-dynamic.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: snapshot.storage.k8s.io/v1
+kind: VolumeSnapshot
+metadata:
+  name: test-nfs-snapshot
+spec:
+  volumeSnapshotClassName: csi-nfs-snapclass
+  source:
+    persistentVolumeClaimName: pvc-nfs-dynamic

deploy/example/snapshotclass-nfs.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: snapshot.storage.k8s.io/v1
+kind: VolumeSnapshotClass
+metadata:
+  name: csi-nfs-snapclass
+driver: nfs.csi.k8s.io
+deletionPolicy: Delete

deploy/install-driver.sh

@@ -16,6 +16,13 @@
 
 set -euo pipefail
 
+# min version of this nfs plugin supporting CSI snapshots
+snap_ver='4.3.0'
+
+# external-snapshotter to install for snapshots to work
+snap_controller_repo='github.com/kubernetes-csi/external-snapshotter/deploy/kubernetes/snapshot-controller?ref=v6.2.1'
+snap_crd_repo='github.com/kubernetes-csi/external-snapshotter/client/config/crd?ref=v6.2.1'
+
 ver="master"
 if [[ "$#" -gt 0 ]]; then
   ver="$1"
@@ -33,6 +40,12 @@ if [ $ver != "master" ]; then
   repo="$repo/$ver"
 fi
 
+# check for min supported version for snapshots or master to install external-snapshotter
+if [[ "$ver" == master || "$(printf '%s\n' "$ver" "$snap_ver" | sort -V | head -n1)" = "$snap_ver" ]]; then
+  kubectl kustomize "$snap_crd_repo" | kubectl apply -f -
+  kubectl -n kube-system kustomize "$snap_controller_repo" | kubectl apply -f -
+fi
+
 echo "Installing NFS CSI driver, version: $ver ..."
 kubectl apply -f $repo/rbac-csi-nfs.yaml
 kubectl apply -f $repo/csi-nfs-driverinfo.yaml

deploy/rbac-csi-nfs.yaml

@@ -26,6 +26,15 @@ rules:
   - apiGroups: ["storage.k8s.io"]
     resources: ["storageclasses"]
     verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses", "volumesnapshots"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["get", "update", "patch"]
   - apiGroups: [""]
     resources: ["events"]
     verbs: ["get", "list", "watch", "create", "update", "patch"]

deploy/rbac-snapshot-controller.yaml

@@ -0,0 +1,82 @@
+# RBAC file for the snapshot controller.
+#
+# The snapshot controller implements the control loop for CSI snapshot functionality.
+# It should be installed as part of the base Kubernetes distribution in an appropriate
+# namespace for components implementing base system functionality. For installing with
+# Vanilla Kubernetes, kube-system makes sense for the namespace.
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: snapshot-controller
+  namespace: kube-system
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: snapshot-controller-runner
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots/status"]
+    verbs: ["update", "patch"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: snapshot-controller-role
+subjects:
+  - kind: ServiceAccount
+    name: snapshot-controller
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: snapshot-controller-runner
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: snapshot-controller-leaderelection
+  namespace: kube-system
+rules:
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: snapshot-controller-leaderelection
+  namespace: kube-system
+subjects:
+  - kind: ServiceAccount
+    name: snapshot-controller
+roleRef:
+  kind: Role
+  name: snapshot-controller-leaderelection
+  apiGroup: rbac.authorization.k8s.io

deploy/snapshot.storage.k8s.io_volumesnapshotclasses.yaml

@@ -0,0 +1,144 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.8.0
+    api-approved.kubernetes.io: https://github.com/kubernetes-csi/external-snapshotter/pull/665
+  name: volumesnapshotclasses.snapshot.storage.k8s.io
+spec:
+  group: snapshot.storage.k8s.io
+  names:
+    kind: VolumeSnapshotClass
+    listKind: VolumeSnapshotClassList
+    plural: volumesnapshotclasses
+    shortNames: [vsclass, vsclasses]
+    singular: volumesnapshotclass
+  scope: Cluster
+  versions:
+    - additionalPrinterColumns:
+        - jsonPath: .driver
+          name: Driver
+          type: string
+        - description: Determines whether a VolumeSnapshotContent created through
+            the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot
+            is deleted.
+          jsonPath: .deletionPolicy
+          name: DeletionPolicy
+          type: string
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshotClass specifies parameters that a underlying
+            storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass
+            is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses
+            are non-namespaced
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+                of an object. Servers should convert recognized schemas to the latest
+                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            deletionPolicy:
+              description: deletionPolicy determines whether a VolumeSnapshotContent
+                created through the VolumeSnapshotClass should be deleted when its
+                bound VolumeSnapshot is deleted. Supported values are "Retain" and
+                "Delete". "Retain" means that the VolumeSnapshotContent and its physical
+                snapshot on underlying storage system are kept. "Delete" means that
+                the VolumeSnapshotContent and its physical snapshot on underlying
+                storage system are deleted. Required.
+              enum: [Delete, Retain]
+              type: string
+            driver:
+              description: driver is the name of the storage driver that handles this
+                VolumeSnapshotClass. Required.
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource
+                this object represents. Servers may infer this from the endpoint the
+                client submits requests to. Cannot be updated. In CamelCase. More
+                info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            parameters:
+              additionalProperties:
+                type: string
+              description: parameters is a key-value map with storage driver specific
+                parameters for creating snapshots. These values are opaque to Kubernetes.
+              type: object
+          required: [deletionPolicy, driver]
+          type: object
+      served: true
+      storage: true
+      subresources: {}
+    - additionalPrinterColumns:
+        - jsonPath: .driver
+          name: Driver
+          type: string
+        - description: Determines whether a VolumeSnapshotContent created through
+            the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot
+            is deleted.
+          jsonPath: .deletionPolicy
+          name: DeletionPolicy
+          type: string
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1beta1
+      # This indicates the v1beta1 version of the custom resource is deprecated.
+      # API requests to this version receive a warning in the server response.
+      deprecated: true
+      # This overrides the default warning returned to clients making v1beta1 API requests.
+      deprecationWarning: snapshot.storage.k8s.io/v1beta1 VolumeSnapshotClass is deprecated;
+        use snapshot.storage.k8s.io/v1 VolumeSnapshotClass
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshotClass specifies parameters that a underlying
+            storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass
+            is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses
+            are non-namespaced
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+                of an object. Servers should convert recognized schemas to the latest
+                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            deletionPolicy:
+              description: deletionPolicy determines whether a VolumeSnapshotContent
+                created through the VolumeSnapshotClass should be deleted when its
+                bound VolumeSnapshot is deleted. Supported values are "Retain" and
+                "Delete". "Retain" means that the VolumeSnapshotContent and its physical
+                snapshot on underlying storage system are kept. "Delete" means that
+                the VolumeSnapshotContent and its physical snapshot on underlying
+                storage system are deleted. Required.
+              enum: [Delete, Retain]
+              type: string
+            driver:
+              description: driver is the name of the storage driver that handles this
+                VolumeSnapshotClass. Required.
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource
+                this object represents. Servers may infer this from the endpoint the
+                client submits requests to. Cannot be updated. In CamelCase. More
+                info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            parameters:
+              additionalProperties:
+                type: string
+              description: parameters is a key-value map with storage driver specific
+                parameters for creating snapshots. These values are opaque to Kubernetes.
+              type: object
+          required: [deletionPolicy, driver]
+          type: object
+      served: false
+      storage: false
+      subresources: {}
+status:
+  acceptedNames:
+    kind: ''
+    plural: ''
+  conditions: []
+  storedVersions: []

deploy/snapshot.storage.k8s.io_volumesnapshotcontents.yaml

@@ -0,0 +1,471 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.8.0
+    api-approved.kubernetes.io: https://github.com/kubernetes-csi/external-snapshotter/pull/665
+  name: volumesnapshotcontents.snapshot.storage.k8s.io
+spec:
+  group: snapshot.storage.k8s.io
+  names:
+    kind: VolumeSnapshotContent
+    listKind: VolumeSnapshotContentList
+    plural: volumesnapshotcontents
+    shortNames: [vsc, vscs]
+    singular: volumesnapshotcontent
+  scope: Cluster
+  versions:
+    - additionalPrinterColumns:
+        - description: Indicates if the snapshot is ready to be used to restore a
+            volume.
+          jsonPath: .status.readyToUse
+          name: ReadyToUse
+          type: boolean
+        - description: Represents the complete size of the snapshot in bytes
+          jsonPath: .status.restoreSize
+          name: RestoreSize
+          type: integer
+        - description: Determines whether this VolumeSnapshotContent and its physical
+            snapshot on the underlying storage system should be deleted when its bound
+            VolumeSnapshot is deleted.
+          jsonPath: .spec.deletionPolicy
+          name: DeletionPolicy
+          type: string
+        - description: Name of the CSI driver used to create the physical snapshot
+            on the underlying storage system.
+          jsonPath: .spec.driver
+          name: Driver
+          type: string
+        - description: Name of the VolumeSnapshotClass to which this snapshot belongs.
+          jsonPath: .spec.volumeSnapshotClassName
+          name: VolumeSnapshotClass
+          type: string
+        - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent
+            object is bound.
+          jsonPath: .spec.volumeSnapshotRef.name
+          name: VolumeSnapshot
+          type: string
+        - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent
+            object is bound.
+          jsonPath: .spec.volumeSnapshotRef.namespace
+          name: VolumeSnapshotNamespace
+          type: string
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshotContent represents the actual "on-disk" snapshot
+            object in the underlying storage system
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+                of an object. Servers should convert recognized schemas to the latest
+                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource
+                this object represents. Servers may infer this from the endpoint the
+                client submits requests to. Cannot be updated. In CamelCase. More
+                info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            spec:
+              description: spec defines properties of a VolumeSnapshotContent created
+                by the underlying storage system. Required.
+              properties:
+                deletionPolicy:
+                  description: deletionPolicy determines whether this VolumeSnapshotContent
+                    and its physical snapshot on the underlying storage system should
+                    be deleted when its bound VolumeSnapshot is deleted. Supported
+                    values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent
+                    and its physical snapshot on underlying storage system are kept.
+                    "Delete" means that the VolumeSnapshotContent and its physical
+                    snapshot on underlying storage system are deleted. For dynamically
+                    provisioned snapshots, this field will automatically be filled
+                    in by the CSI snapshotter sidecar with the "DeletionPolicy" field
+                    defined in the corresponding VolumeSnapshotClass. For pre-existing
+                    snapshots, users MUST specify this field when creating the VolumeSnapshotContent
+                    object. Required.
+                  enum: [Delete, Retain]
+                  type: string
+                driver:
+                  description: driver is the name of the CSI driver used to create
+                    the physical snapshot on the underlying storage system. This MUST
+                    be the same as the name returned by the CSI GetPluginName() call
+                    for that driver. Required.
+                  type: string
+                source:
+                  description: source specifies whether the snapshot is (or should
+                    be) dynamically provisioned or already exists, and just requires
+                    a Kubernetes object representation. This field is immutable after
+                    creation. Required.
+                  properties:
+                    snapshotHandle:
+                      description: snapshotHandle specifies the CSI "snapshot_id"
+                        of a pre-existing snapshot on the underlying storage system
+                        for which a Kubernetes object representation was (or should
+                        be) created. This field is immutable.
+                      type: string
+                    volumeHandle:
+                      description: volumeHandle specifies the CSI "volume_id" of the
+                        volume from which a snapshot should be dynamically taken from.
+                        This field is immutable.
+                      type: string
+                  type: object
+                  oneOf:
+                    - required: [snapshotHandle]
+                    - required: [volumeHandle]
+                sourceVolumeMode:
+                  description: SourceVolumeMode is the mode of the volume whose snapshot
+                    is taken. Can be either “Filesystem” or “Block”. If not specified,
+                    it indicates the source volume's mode is unknown. This field is
+                    immutable. This field is an alpha field.
+                  type: string
+                volumeSnapshotClassName:
+                  description: name of the VolumeSnapshotClass from which this snapshot
+                    was (or will be) created. Note that after provisioning, the VolumeSnapshotClass
+                    may be deleted or recreated with different set of values, and
+                    as such, should not be referenced post-snapshot creation.
+                  type: string
+                volumeSnapshotRef:
+                  description: volumeSnapshotRef specifies the VolumeSnapshot object
+                    to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName
+                    field must reference to this VolumeSnapshotContent's name for
+                    the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent
+                    object, name and namespace of the VolumeSnapshot object MUST be
+                    provided for binding to happen. This field is immutable after
+                    creation. Required.
+                  properties:
+                    apiVersion:
+                      description: API version of the referent.
+                      type: string
+                    fieldPath:
+                      description: 'If referring to a piece of an object instead of
+                        an entire object, this string should contain a valid JSON/Go
+                        field access statement, such as desiredState.manifest.containers[2].
+                        For example, if the object reference is to a container within
+                        a pod, this would take on a value like: "spec.containers{name}"
+                        (where "name" refers to the name of the container that triggered
+                        the event) or if no container name is specified "spec.containers[2]"
+                        (container with index 2 in this pod). This syntax is chosen
+                        only to have some well-defined way of referencing a part of
+                        an object. TODO: this design is not final and this field is
+                        subject to change in the future.'
+                      type: string
+                    kind:
+                      description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                      type: string
+                    name:
+                      description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                      type: string
+                    namespace:
+                      description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
+                      type: string
+                    resourceVersion:
+                      description: 'Specific resourceVersion to which this reference
+                        is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
+                      type: string
+                    uid:
+                      description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                      type: string
+                  type: object
+              required: [deletionPolicy, driver, source, volumeSnapshotRef]
+              type: object
+            status:
+              description: status represents the current information of a snapshot.
+              properties:
+                creationTime:
+                  description: creationTime is the timestamp when the point-in-time
+                    snapshot is taken by the underlying storage system. In dynamic
+                    snapshot creation case, this field will be filled in by the CSI
+                    snapshotter sidecar with the "creation_time" value returned from
+                    CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this
+                    field will be filled with the "creation_time" value returned from
+                    the CSI "ListSnapshots" gRPC call if the driver supports it. If
+                    not specified, it indicates the creation time is unknown. The
+                    format of this field is a Unix nanoseconds time encoded as an
+                    int64. On Unix, the command `date +%s%N` returns the current time
+                    in nanoseconds since 1970-01-01 00:00:00 UTC.
+                  format: int64
+                  type: integer
+                error:
+                  description: error is the last observed error during snapshot creation,
+                    if any. Upon success after retry, this error field will be cleared.
+                  properties:
+                    message:
+                      description: 'message is a string detailing the encountered
+                        error during snapshot creation if specified. NOTE: message
+                        may be logged, and it should not contain sensitive information.'
+                      type: string
+                    time:
+                      description: time is the timestamp when the error was encountered.
+                      format: date-time
+                      type: string
+                  type: object
+                readyToUse:
+                  description: readyToUse indicates if a snapshot is ready to be used
+                    to restore a volume. In dynamic snapshot creation case, this field
+                    will be filled in by the CSI snapshotter sidecar with the "ready_to_use"
+                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "ready_to_use" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver
+                    supports it, otherwise, this field will be set to "True". If not
+                    specified, it means the readiness of a snapshot is unknown.
+                  type: boolean
+                restoreSize:
+                  description: restoreSize represents the complete size of the snapshot
+                    in bytes. In dynamic snapshot creation case, this field will be
+                    filled in by the CSI snapshotter sidecar with the "size_bytes"
+                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "size_bytes" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver
+                    supports it. When restoring a volume from this snapshot, the size
+                    of the volume MUST NOT be smaller than the restoreSize if it is
+                    specified, otherwise the restoration will fail. If not specified,
+                    it indicates that the size is unknown.
+                  format: int64
+                  minimum: 0
+                  type: integer
+                snapshotHandle:
+                  description: snapshotHandle is the CSI "snapshot_id" of a snapshot
+                    on the underlying storage system. If not specified, it indicates
+                    that dynamic snapshot creation has either failed or it is still
+                    in progress.
+                  type: string
+              type: object
+          required: [spec]
+          type: object
+      served: true
+      storage: true
+      subresources:
+        status: {}
+    - additionalPrinterColumns:
+        - description: Indicates if the snapshot is ready to be used to restore a
+            volume.
+          jsonPath: .status.readyToUse
+          name: ReadyToUse
+          type: boolean
+        - description: Represents the complete size of the snapshot in bytes
+          jsonPath: .status.restoreSize
+          name: RestoreSize
+          type: integer
+        - description: Determines whether this VolumeSnapshotContent and its physical
+            snapshot on the underlying storage system should be deleted when its bound
+            VolumeSnapshot is deleted.
+          jsonPath: .spec.deletionPolicy
+          name: DeletionPolicy
+          type: string
+        - description: Name of the CSI driver used to create the physical snapshot
+            on the underlying storage system.
+          jsonPath: .spec.driver
+          name: Driver
+          type: string
+        - description: Name of the VolumeSnapshotClass to which this snapshot belongs.
+          jsonPath: .spec.volumeSnapshotClassName
+          name: VolumeSnapshotClass
+          type: string
+        - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent
+            object is bound.
+          jsonPath: .spec.volumeSnapshotRef.name
+          name: VolumeSnapshot
+          type: string
+        - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent
+            object is bound.
+          jsonPath: .spec.volumeSnapshotRef.namespace
+          name: VolumeSnapshotNamespace
+          type: string
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1beta1
+      # This indicates the v1beta1 version of the custom resource is deprecated.
+      # API requests to this version receive a warning in the server response.
+      deprecated: true
+      # This overrides the default warning returned to clients making v1beta1 API requests.
+      deprecationWarning: snapshot.storage.k8s.io/v1beta1 VolumeSnapshotContent is
+        deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotContent
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshotContent represents the actual "on-disk" snapshot
+            object in the underlying storage system
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+                of an object. Servers should convert recognized schemas to the latest
+                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource
+                this object represents. Servers may infer this from the endpoint the
+                client submits requests to. Cannot be updated. In CamelCase. More
+                info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            spec:
+              description: spec defines properties of a VolumeSnapshotContent created
+                by the underlying storage system. Required.
+              properties:
+                deletionPolicy:
+                  description: deletionPolicy determines whether this VolumeSnapshotContent
+                    and its physical snapshot on the underlying storage system should
+                    be deleted when its bound VolumeSnapshot is deleted. Supported
+                    values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent
+                    and its physical snapshot on underlying storage system are kept.
+                    "Delete" means that the VolumeSnapshotContent and its physical
+                    snapshot on underlying storage system are deleted. For dynamically
+                    provisioned snapshots, this field will automatically be filled
+                    in by the CSI snapshotter sidecar with the "DeletionPolicy" field
+                    defined in the corresponding VolumeSnapshotClass. For pre-existing
+                    snapshots, users MUST specify this field when creating the  VolumeSnapshotContent
+                    object. Required.
+                  enum: [Delete, Retain]
+                  type: string
+                driver:
+                  description: driver is the name of the CSI driver used to create
+                    the physical snapshot on the underlying storage system. This MUST
+                    be the same as the name returned by the CSI GetPluginName() call
+                    for that driver. Required.
+                  type: string
+                source:
+                  description: source specifies whether the snapshot is (or should
+                    be) dynamically provisioned or already exists, and just requires
+                    a Kubernetes object representation. This field is immutable after
+                    creation. Required.
+                  properties:
+                    snapshotHandle:
+                      description: snapshotHandle specifies the CSI "snapshot_id"
+                        of a pre-existing snapshot on the underlying storage system
+                        for which a Kubernetes object representation was (or should
+                        be) created. This field is immutable.
+                      type: string
+                    volumeHandle:
+                      description: volumeHandle specifies the CSI "volume_id" of the
+                        volume from which a snapshot should be dynamically taken from.
+                        This field is immutable.
+                      type: string
+                  type: object
+                volumeSnapshotClassName:
+                  description: name of the VolumeSnapshotClass from which this snapshot
+                    was (or will be) created. Note that after provisioning, the VolumeSnapshotClass
+                    may be deleted or recreated with different set of values, and
+                    as such, should not be referenced post-snapshot creation.
+                  type: string
+                volumeSnapshotRef:
+                  description: volumeSnapshotRef specifies the VolumeSnapshot object
+                    to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName
+                    field must reference to this VolumeSnapshotContent's name for
+                    the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent
+                    object, name and namespace of the VolumeSnapshot object MUST be
+                    provided for binding to happen. This field is immutable after
+                    creation. Required.
+                  properties:
+                    apiVersion:
+                      description: API version of the referent.
+                      type: string
+                    fieldPath:
+                      description: 'If referring to a piece of an object instead of
+                        an entire object, this string should contain a valid JSON/Go
+                        field access statement, such as desiredState.manifest.containers[2].
+                        For example, if the object reference is to a container within
+                        a pod, this would take on a value like: "spec.containers{name}"
+                        (where "name" refers to the name of the container that triggered
+                        the event) or if no container name is specified "spec.containers[2]"
+                        (container with index 2 in this pod). This syntax is chosen
+                        only to have some well-defined way of referencing a part of
+                        an object. TODO: this design is not final and this field is
+                        subject to change in the future.'
+                      type: string
+                    kind:
+                      description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                      type: string
+                    name:
+                      description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                      type: string
+                    namespace:
+                      description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
+                      type: string
+                    resourceVersion:
+                      description: 'Specific resourceVersion to which this reference
+                        is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
+                      type: string
+                    uid:
+                      description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                      type: string
+                  type: object
+              required: [deletionPolicy, driver, source, volumeSnapshotRef]
+              type: object
+            status:
+              description: status represents the current information of a snapshot.
+              properties:
+                creationTime:
+                  description: creationTime is the timestamp when the point-in-time
+                    snapshot is taken by the underlying storage system. In dynamic
+                    snapshot creation case, this field will be filled in by the CSI
+                    snapshotter sidecar with the "creation_time" value returned from
+                    CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this
+                    field will be filled with the "creation_time" value returned from
+                    the CSI "ListSnapshots" gRPC call if the driver supports it. If
+                    not specified, it indicates the creation time is unknown. The
+                    format of this field is a Unix nanoseconds time encoded as an
+                    int64. On Unix, the command `date +%s%N` returns the current time
+                    in nanoseconds since 1970-01-01 00:00:00 UTC.
+                  format: int64
+                  type: integer
+                error:
+                  description: error is the last observed error during snapshot creation,
+                    if any. Upon success after retry, this error field will be cleared.
+                  properties:
+                    message:
+                      description: 'message is a string detailing the encountered
+                        error during snapshot creation if specified. NOTE: message
+                        may be logged, and it should not contain sensitive information.'
+                      type: string
+                    time:
+                      description: time is the timestamp when the error was encountered.
+                      format: date-time
+                      type: string
+                  type: object
+                readyToUse:
+                  description: readyToUse indicates if a snapshot is ready to be used
+                    to restore a volume. In dynamic snapshot creation case, this field
+                    will be filled in by the CSI snapshotter sidecar with the "ready_to_use"
+                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "ready_to_use" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver
+                    supports it, otherwise, this field will be set to "True". If not
+                    specified, it means the readiness of a snapshot is unknown.
+                  type: boolean
+                restoreSize:
+                  description: restoreSize represents the complete size of the snapshot
+                    in bytes. In dynamic snapshot creation case, this field will be
+                    filled in by the CSI snapshotter sidecar with the "size_bytes"
+                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "size_bytes" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver
+                    supports it. When restoring a volume from this snapshot, the size
+                    of the volume MUST NOT be smaller than the restoreSize if it is
+                    specified, otherwise the restoration will fail. If not specified,
+                    it indicates that the size is unknown.
+                  format: int64
+                  minimum: 0
+                  type: integer
+                snapshotHandle:
+                  description: snapshotHandle is the CSI "snapshot_id" of a snapshot
+                    on the underlying storage system. If not specified, it indicates
+                    that dynamic snapshot creation has either failed or it is still
+                    in progress.
+                  type: string
+              type: object
+          required: [spec]
+          type: object
+      served: false
+      storage: false
+      subresources:
+        status: {}
+status:
+  acceptedNames:
+    kind: ''
+    plural: ''
+  conditions: []
+  storedVersions: []

deploy/snapshot.storage.k8s.io_volumesnapshots.yaml

@@ -0,0 +1,385 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.8.0
+    api-approved.kubernetes.io: https://github.com/kubernetes-csi/external-snapshotter/pull/665
+  name: volumesnapshots.snapshot.storage.k8s.io
+spec:
+  group: snapshot.storage.k8s.io
+  names:
+    kind: VolumeSnapshot
+    listKind: VolumeSnapshotList
+    plural: volumesnapshots
+    shortNames: [vs]
+    singular: volumesnapshot
+  scope: Namespaced
+  versions:
+    - additionalPrinterColumns:
+        - description: Indicates if the snapshot is ready to be used to restore a
+            volume.
+          jsonPath: .status.readyToUse
+          name: ReadyToUse
+          type: boolean
+        - description: If a new snapshot needs to be created, this contains the name
+            of the source PVC from which this snapshot was (or will be) created.
+          jsonPath: .spec.source.persistentVolumeClaimName
+          name: SourcePVC
+          type: string
+        - description: If a snapshot already exists, this contains the name of the
+            existing VolumeSnapshotContent object representing the existing snapshot.
+          jsonPath: .spec.source.volumeSnapshotContentName
+          name: SourceSnapshotContent
+          type: string
+        - description: Represents the minimum size of volume required to rehydrate
+            from this snapshot.
+          jsonPath: .status.restoreSize
+          name: RestoreSize
+          type: string
+        - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot.
+          jsonPath: .spec.volumeSnapshotClassName
+          name: SnapshotClass
+          type: string
+        - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot
+            object intends to bind to. Please note that verification of binding actually
+            requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure
+            both are pointing at each other. Binding MUST be verified prior to usage
+            of this object.
+          jsonPath: .status.boundVolumeSnapshotContentName
+          name: SnapshotContent
+          type: string
+        - description: Timestamp when the point-in-time snapshot was taken by the
+            underlying storage system.
+          jsonPath: .status.creationTime
+          name: CreationTime
+          type: date
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshot is a user's request for either creating a point-in-time
+            snapshot of a persistent volume, or binding to a pre-existing snapshot.
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+                of an object. Servers should convert recognized schemas to the latest
+                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource
+                this object represents. Servers may infer this from the endpoint the
+                client submits requests to. Cannot be updated. In CamelCase. More
+                info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            spec:
+              description: 'spec defines the desired characteristics of a snapshot
+                requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots
+                Required.'
+              properties:
+                source:
+                  description: source specifies where a snapshot will be created from.
+                    This field is immutable after creation. Required.
+                  properties:
+                    persistentVolumeClaimName:
+                      description: persistentVolumeClaimName specifies the name of
+                        the PersistentVolumeClaim object representing the volume from
+                        which a snapshot should be created. This PVC is assumed to
+                        be in the same namespace as the VolumeSnapshot object. This
+                        field should be set if the snapshot does not exists, and needs
+                        to be created. This field is immutable.
+                      type: string
+                    volumeSnapshotContentName:
+                      description: volumeSnapshotContentName specifies the name of
+                        a pre-existing VolumeSnapshotContent object representing an
+                        existing volume snapshot. This field should be set if the
+                        snapshot already exists and only needs a representation in
+                        Kubernetes. This field is immutable.
+                      type: string
+                  type: object
+                  oneOf:
+                    - required: [persistentVolumeClaimName]
+                    - required: [volumeSnapshotContentName]
+                volumeSnapshotClassName:
+                  description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass
+                    requested by the VolumeSnapshot. VolumeSnapshotClassName may be
+                    left nil to indicate that the default SnapshotClass should be
+                    used. A given cluster may have multiple default Volume SnapshotClasses:
+                    one default per CSI Driver. If a VolumeSnapshot does not specify
+                    a SnapshotClass, VolumeSnapshotSource will be checked to figure
+                    out what the associated CSI Driver is, and the default VolumeSnapshotClass
+                    associated with that CSI Driver will be used. If more than one
+                    VolumeSnapshotClass exist for a given CSI Driver and more than
+                    one have been marked as default, CreateSnapshot will fail and
+                    generate an event. Empty string is not allowed for this field.'
+                  type: string
+              required: [source]
+              type: object
+            status:
+              description: status represents the current information of a snapshot.
+                Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent
+                objects is successful (by validating that both VolumeSnapshot and
+                VolumeSnapshotContent point at each other) before using this object.
+              properties:
+                boundVolumeSnapshotContentName:
+                  description: 'boundVolumeSnapshotContentName is the name of the
+                    VolumeSnapshotContent object to which this VolumeSnapshot object
+                    intends to bind to. If not specified, it indicates that the VolumeSnapshot
+                    object has not been successfully bound to a VolumeSnapshotContent
+                    object yet. NOTE: To avoid possible security issues, consumers
+                    must verify binding between VolumeSnapshot and VolumeSnapshotContent
+                    objects is successful (by validating that both VolumeSnapshot
+                    and VolumeSnapshotContent point at each other) before using this
+                    object.'
+                  type: string
+                creationTime:
+                  description: creationTime is the timestamp when the point-in-time
+                    snapshot is taken by the underlying storage system. In dynamic
+                    snapshot creation case, this field will be filled in by the snapshot
+                    controller with the "creation_time" value returned from CSI "CreateSnapshot"
+                    gRPC call. For a pre-existing snapshot, this field will be filled
+                    with the "creation_time" value returned from the CSI "ListSnapshots"
+                    gRPC call if the driver supports it. If not specified, it may
+                    indicate that the creation time of the snapshot is unknown.
+                  format: date-time
+                  type: string
+                error:
+                  description: error is the last observed error during snapshot creation,
+                    if any. This field could be helpful to upper level controllers(i.e.,
+                    application controller) to decide whether they should continue
+                    on waiting for the snapshot to be created based on the type of
+                    error reported. The snapshot controller will keep retrying when
+                    an error occurs during the snapshot creation. Upon success, this
+                    error field will be cleared.
+                  properties:
+                    message:
+                      description: 'message is a string detailing the encountered
+                        error during snapshot creation if specified. NOTE: message
+                        may be logged, and it should not contain sensitive information.'
+                      type: string
+                    time:
+                      description: time is the timestamp when the error was encountered.
+                      format: date-time
+                      type: string
+                  type: object
+                readyToUse:
+                  description: readyToUse indicates if the snapshot is ready to be
+                    used to restore a volume. In dynamic snapshot creation case, this
+                    field will be filled in by the snapshot controller with the "ready_to_use"
+                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "ready_to_use" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver
+                    supports it, otherwise, this field will be set to "True". If not
+                    specified, it means the readiness of a snapshot is unknown.
+                  type: boolean
+                restoreSize:
+                  type: string
+                  description: restoreSize represents the minimum size of volume required
+                    to create a volume from this snapshot. In dynamic snapshot creation
+                    case, this field will be filled in by the snapshot controller
+                    with the "size_bytes" value returned from CSI "CreateSnapshot"
+                    gRPC call. For a pre-existing snapshot, this field will be filled
+                    with the "size_bytes" value returned from the CSI "ListSnapshots"
+                    gRPC call if the driver supports it. When restoring a volume from
+                    this snapshot, the size of the volume MUST NOT be smaller than
+                    the restoreSize if it is specified, otherwise the restoration
+                    will fail. If not specified, it indicates that the size is unknown.
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
+              type: object
+          required: [spec]
+          type: object
+      served: true
+      storage: true
+      subresources:
+        status: {}
+    - additionalPrinterColumns:
+        - description: Indicates if the snapshot is ready to be used to restore a
+            volume.
+          jsonPath: .status.readyToUse
+          name: ReadyToUse
+          type: boolean
+        - description: If a new snapshot needs to be created, this contains the name
+            of the source PVC from which this snapshot was (or will be) created.
+          jsonPath: .spec.source.persistentVolumeClaimName
+          name: SourcePVC
+          type: string
+        - description: If a snapshot already exists, this contains the name of the
+            existing VolumeSnapshotContent object representing the existing snapshot.
+          jsonPath: .spec.source.volumeSnapshotContentName
+          name: SourceSnapshotContent
+          type: string
+        - description: Represents the minimum size of volume required to rehydrate
+            from this snapshot.
+          jsonPath: .status.restoreSize
+          name: RestoreSize
+          type: string
+        - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot.
+          jsonPath: .spec.volumeSnapshotClassName
+          name: SnapshotClass
+          type: string
+        - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot
+            object intends to bind to. Please note that verification of binding actually
+            requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure
+            both are pointing at each other. Binding MUST be verified prior to usage
+            of this object.
+          jsonPath: .status.boundVolumeSnapshotContentName
+          name: SnapshotContent
+          type: string
+        - description: Timestamp when the point-in-time snapshot was taken by the
+            underlying storage system.
+          jsonPath: .status.creationTime
+          name: CreationTime
+          type: date
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1beta1
+      # This indicates the v1beta1 version of the custom resource is deprecated.
+      # API requests to this version receive a warning in the server response.
+      deprecated: true
+      # This overrides the default warning returned to clients making v1beta1 API requests.
+      deprecationWarning: snapshot.storage.k8s.io/v1beta1 VolumeSnapshot is deprecated;
+        use snapshot.storage.k8s.io/v1 VolumeSnapshot
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshot is a user's request for either creating a point-in-time
+            snapshot of a persistent volume, or binding to a pre-existing snapshot.
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+                of an object. Servers should convert recognized schemas to the latest
+                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource
+                this object represents. Servers may infer this from the endpoint the
+                client submits requests to. Cannot be updated. In CamelCase. More
+                info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            spec:
+              description: 'spec defines the desired characteristics of a snapshot
+                requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots
+                Required.'
+              properties:
+                source:
+                  description: source specifies where a snapshot will be created from.
+                    This field is immutable after creation. Required.
+                  properties:
+                    persistentVolumeClaimName:
+                      description: persistentVolumeClaimName specifies the name of
+                        the PersistentVolumeClaim object representing the volume from
+                        which a snapshot should be created. This PVC is assumed to
+                        be in the same namespace as the VolumeSnapshot object. This
+                        field should be set if the snapshot does not exists, and needs
+                        to be created. This field is immutable.
+                      type: string
+                    volumeSnapshotContentName:
+                      description: volumeSnapshotContentName specifies the name of
+                        a pre-existing VolumeSnapshotContent object representing an
+                        existing volume snapshot. This field should be set if the
+                        snapshot already exists and only needs a representation in
+                        Kubernetes. This field is immutable.
+                      type: string
+                  type: object
+                volumeSnapshotClassName:
+                  description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass
+                    requested by the VolumeSnapshot. VolumeSnapshotClassName may be
+                    left nil to indicate that the default SnapshotClass should be
+                    used. A given cluster may have multiple default Volume SnapshotClasses:
+                    one default per CSI Driver. If a VolumeSnapshot does not specify
+                    a SnapshotClass, VolumeSnapshotSource will be checked to figure
+                    out what the associated CSI Driver is, and the default VolumeSnapshotClass
+                    associated with that CSI Driver will be used. If more than one
+                    VolumeSnapshotClass exist for a given CSI Driver and more than
+                    one have been marked as default, CreateSnapshot will fail and
+                    generate an event. Empty string is not allowed for this field.'
+                  type: string
+              required: [source]
+              type: object
+            status:
+              description: status represents the current information of a snapshot.
+                Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent
+                objects is successful (by validating that both VolumeSnapshot and
+                VolumeSnapshotContent point at each other) before using this object.
+              properties:
+                boundVolumeSnapshotContentName:
+                  description: 'boundVolumeSnapshotContentName is the name of the
+                    VolumeSnapshotContent object to which this VolumeSnapshot object
+                    intends to bind to. If not specified, it indicates that the VolumeSnapshot
+                    object has not been successfully bound to a VolumeSnapshotContent
+                    object yet. NOTE: To avoid possible security issues, consumers
+                    must verify binding between VolumeSnapshot and VolumeSnapshotContent
+                    objects is successful (by validating that both VolumeSnapshot
+                    and VolumeSnapshotContent point at each other) before using this
+                    object.'
+                  type: string
+                creationTime:
+                  description: creationTime is the timestamp when the point-in-time
+                    snapshot is taken by the underlying storage system. In dynamic
+                    snapshot creation case, this field will be filled in by the snapshot
+                    controller with the "creation_time" value returned from CSI "CreateSnapshot"
+                    gRPC call. For a pre-existing snapshot, this field will be filled
+                    with the "creation_time" value returned from the CSI "ListSnapshots"
+                    gRPC call if the driver supports it. If not specified, it may
+                    indicate that the creation time of the snapshot is unknown.
+                  format: date-time
+                  type: string
+                error:
+                  description: error is the last observed error during snapshot creation,
+                    if any. This field could be helpful to upper level controllers(i.e.,
+                    application controller) to decide whether they should continue
+                    on waiting for the snapshot to be created based on the type of
+                    error reported. The snapshot controller will keep retrying when
+                    an error occurs during the snapshot creation. Upon success, this
+                    error field will be cleared.
+                  properties:
+                    message:
+                      description: 'message is a string detailing the encountered
+                        error during snapshot creation if specified. NOTE: message
+                        may be logged, and it should not contain sensitive information.'
+                      type: string
+                    time:
+                      description: time is the timestamp when the error was encountered.
+                      format: date-time
+                      type: string
+                  type: object
+                readyToUse:
+                  description: readyToUse indicates if the snapshot is ready to be
+                    used to restore a volume. In dynamic snapshot creation case, this
+                    field will be filled in by the snapshot controller with the "ready_to_use"
+                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "ready_to_use" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver
+                    supports it, otherwise, this field will be set to "True". If not
+                    specified, it means the readiness of a snapshot is unknown.
+                  type: boolean
+                restoreSize:
+                  type: string
+                  description: restoreSize represents the minimum size of volume required
+                    to create a volume from this snapshot. In dynamic snapshot creation
+                    case, this field will be filled in by the snapshot controller
+                    with the "size_bytes" value returned from CSI "CreateSnapshot"
+                    gRPC call. For a pre-existing snapshot, this field will be filled
+                    with the "size_bytes" value returned from the CSI "ListSnapshots"
+                    gRPC call if the driver supports it. When restoring a volume from
+                    this snapshot, the size of the volume MUST NOT be smaller than
+                    the restoreSize if it is specified, otherwise the restoration
+                    will fail. If not specified, it indicates that the size is unknown.
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
+              type: object
+          required: [spec]
+          type: object
+      served: false
+      storage: false
+      subresources:
+        status: {}
+status:
+  acceptedNames:
+    kind: ''
+    plural: ''
+  conditions: []
+  storedVersions: []

deploy/uninstall-driver.sh

@@ -16,6 +16,10 @@
 
 set -euo pipefail
 
+# external-snapshotter to install for snapshots to work
+snap_controller_repo='github.com/kubernetes-csi/external-snapshotter/deploy/kubernetes/snapshot-controller?ref=v6.2.1'
+snap_crd_repo='github.com/kubernetes-csi/external-snapshotter/client/config/crd?ref=v6.2.1'
+
 ver="master"
 if [[ "$#" -gt 0 ]]; then
   ver="$1"
@@ -34,6 +38,8 @@ if [ $ver != "master" ]; then
 fi
 
 echo "Uninstalling NFS driver, version: $ver ..."
+# this keeps VolumeSnapshot CRDs untouched
+kubectl -n kube-system kustomize "$snap_controller_repo" | kubectl delete --ignore-not-found -f -
 kubectl delete -f $repo/csi-nfs-controller.yaml --ignore-not-found
 kubectl delete -f $repo/csi-nfs-node.yaml --ignore-not-found
 kubectl delete -f $repo/csi-nfs-driverinfo.yaml --ignore-not-found

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/stretchr/testify v1.8.0
 	golang.org/x/net v0.7.0
 	google.golang.org/grpc v1.40.0
+	google.golang.org/protobuf v1.27.1
 	k8s.io/api v0.23.14
 	k8s.io/apimachinery v0.23.14
 	k8s.io/client-go v0.23.14
@@ -82,7 +83,6 @@ require (
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2 // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

hack/verify-helm-chart.sh

@@ -62,12 +62,16 @@ pip install yq --ignore-installed PyYAML
 
 # Extract images from csi-nfs-controller.yaml
 expected_csi_provisioner_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[0].image | head -n 1)"
-expected_liveness_probe_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[1].image | head -n 1)"
+expected_csi_snapshotter_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[1].image | head -n 1)"
-expected_nfs_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[2].image | head -n 1)"
+expected_liveness_probe_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[2].image | head -n 1)"
+expected_nfs_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[3].image | head -n 1)"
 
 csi_provisioner_image="$(get_image_from_helm_chart "csiProvisioner")"
 validate_image "${expected_csi_provisioner_image}" "${csi_provisioner_image}"
 
+csi_snapshotter_image="$(get_image_from_helm_chart "csiSnapshotter")"
+validate_image "${expected_csi_snapshotter_image}" "${csi_snapshotter_image}"
+
 liveness_probe_image="$(get_image_from_helm_chart "livenessProbe")"
 validate_image "${expected_liveness_probe_image}" "${liveness_probe_image}"
 

pkg/nfs/controllerserver.go

@@ -18,6 +18,7 @@ package nfs
 
 import (
 	"fmt"
+	"io/fs"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -29,6 +30,7 @@ import (
 	"golang.org/x/net/context"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/types/known/timestamppb"
 
 	"k8s.io/klog/v2"
 )
@@ -59,6 +61,35 @@ type nfsVolume struct {
 	onDelete string
 }
 
+// nfsSnapshot is an internal representation of a volume snapshot
+// created by the provisioner.
+type nfsSnapshot struct {
+	// Snapshot id.
+	id string
+	// Address of the NFS server.
+	// Matches paramServer.
+	server string
+	// Base directory of the NFS server to create snapshots under
+	// Matches paramShare.
+	baseDir string
+	// Snapshot name.
+	uuid string
+	// Source volume.
+	src string
+}
+
+func (snap nfsSnapshot) archiveSubPath() string {
+	return snap.uuid
+}
+
+func (snap nfsSnapshot) archiveName() string {
+	return fmt.Sprintf("%v.tar.gz", snap.src)
+}
+
+func (snap nfsSnapshot) archivePath() string {
+	return filepath.Join(snap.archiveSubPath(), snap.archiveName())
+}
+
 // Ordering of elements in the CSI volume id.
 // ID is of the form {server}/{baseDir}/{subDir}.
 // TODO: This volume id format limits baseDir and
@@ -74,6 +105,19 @@ const (
 	totalIDElements // Always last
 )
 
+// Ordering of elements in the CSI snapshot id.
+// ID is of the form {server}/{baseDir}/{snapName}/{srcVolumeName}.
+// Adding a new element should always go at the end
+// before totalSnapIDElements
+const (
+	idSnapServer = iota
+	idSnapBaseDir
+	idSnapUUID
+	idSnapArchivePath
+	idSnapArchiveName
+	totalIDSnapElements // Always last
+)
+
 // CreateVolume create a volume
 func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) {
 	name := req.GetName()
@@ -263,11 +307,115 @@ func (cs *ControllerServer) ControllerGetCapabilities(ctx context.Context, req *
 }
 
 func (cs *ControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) {
-	return nil, status.Error(codes.Unimplemented, "")
+	if len(req.GetName()) == 0 {
+		return nil, status.Error(codes.InvalidArgument, "CreateSnapshot name must be provided")
+	}
+	if len(req.GetSourceVolumeId()) == 0 {
+		return nil, status.Error(codes.InvalidArgument, "CreateSnapshot source volume ID must be provided")
+	}
+
+	srcVol, err := getNfsVolFromID(req.GetSourceVolumeId())
+	if err != nil {
+		return nil, status.Errorf(codes.NotFound, "failed to create source volume: %v", err)
+	}
+	snapshot, err := newNFSSnapshot(req.GetName(), req.GetParameters(), srcVol)
+	if err != nil {
+		return nil, status.Errorf(codes.NotFound, "failed to create nfsSnapshot: %v", err)
+	}
+	snapVol := volumeFromSnapshot(snapshot)
+	if err = cs.internalMount(ctx, snapVol, nil, nil); err != nil {
+		return nil, status.Errorf(codes.Internal, "failed to mount snapshot nfs server: %v", err)
+	}
+	defer func() {
+		if err = cs.internalUnmount(ctx, snapVol); err != nil {
+			klog.Warningf("failed to unmount snapshot nfs server: %v", err)
+		}
+	}()
+	snapInternalVolPath := filepath.Join(getInternalVolumePath(cs.Driver.workingMountDir, snapVol), snapshot.archiveSubPath())
+	if err = os.MkdirAll(snapInternalVolPath, 0777); err != nil {
+		return nil, status.Errorf(codes.Internal, "failed to make subdirectory: %v", err)
+	}
+	if err := validateSnapshot(snapInternalVolPath, snapshot); err != nil {
+		return nil, err
+	}
+
+	if err = cs.internalMount(ctx, srcVol, nil, nil); err != nil {
+		return nil, status.Errorf(codes.Internal, "failed to mount src nfs server: %v", err)
+	}
+	defer func() {
+		if err = cs.internalUnmount(ctx, srcVol); err != nil {
+			klog.Warningf("failed to unmount src nfs server: %v", err)
+		}
+	}()
+
+	srcPath := getInternalVolumePath(cs.Driver.workingMountDir, srcVol)
+	dstPath := filepath.Join(snapInternalVolPath, snapshot.archiveName())
+	klog.V(2).Infof("archiving %v -> %v", srcPath, dstPath)
+	out, err := exec.Command("tar", "-C", srcPath, "-czvf", dstPath, ".").CombinedOutput()
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "failed to create archive for snapshot: %v: %v", err, string(out))
+	}
+	klog.V(2).Infof("archived %s -> %s", srcPath, dstPath)
+
+	var snapshotSize int64
+	fi, err := os.Stat(dstPath)
+	if err != nil {
+		klog.Warningf("failed to determine snapshot size: %v", err)
+	} else {
+		snapshotSize = fi.Size()
+	}
+	return &csi.CreateSnapshotResponse{
+		Snapshot: &csi.Snapshot{
+			SnapshotId:     snapshot.id,
+			SourceVolumeId: srcVol.id,
+			SizeBytes:      snapshotSize,
+			CreationTime:   timestamppb.Now(),
+			ReadyToUse:     true,
+		},
+	}, nil
 }
 
 func (cs *ControllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) {
-	return nil, status.Error(codes.Unimplemented, "")
+	if len(req.GetSnapshotId()) == 0 {
+		return nil, status.Error(codes.InvalidArgument, "Snapshot ID is required for deletion")
+	}
+	snap, err := getNfsSnapFromID(req.GetSnapshotId())
+	if err != nil {
+		// An invalid ID should be treated as doesn't exist
+		klog.Warningf("failed to get nfs snapshot for id %v deletion: %v", req.GetSnapshotId(), err)
+		return &csi.DeleteSnapshotResponse{}, nil
+	}
+
+	var volCap *csi.VolumeCapability
+	mountOptions := getMountOptions(req.GetSecrets())
+	if mountOptions != "" {
+		klog.V(2).Infof("DeleteSnapshot: found mountOptions(%s) for snapshot(%s)", mountOptions, req.GetSnapshotId())
+		volCap = &csi.VolumeCapability{
+			AccessType: &csi.VolumeCapability_Mount{
+				Mount: &csi.VolumeCapability_MountVolume{
+					MountFlags: []string{mountOptions},
+				},
+			},
+		}
+	}
+	vol := volumeFromSnapshot(snap)
+	if err = cs.internalMount(ctx, vol, nil, volCap); err != nil {
+		return nil, status.Errorf(codes.Internal, "failed to mount nfs server for snapshot deletion: %v", err)
+	}
+	defer func() {
+		if err = cs.internalUnmount(ctx, vol); err != nil {
+			klog.Warningf("failed to unmount nfs server after snapshot deletion: %v", err)
+		}
+	}()
+
+	// delete snapshot archive
+	internalVolumePath := filepath.Join(getInternalVolumePath(cs.Driver.workingMountDir, vol), snap.archiveSubPath())
+	klog.V(2).Infof("Removing snapshot archive at %v", internalVolumePath)
+	if err = os.RemoveAll(internalVolumePath); err != nil {
+		return nil, status.Errorf(codes.Internal, "failed to delete subdirectory: %v", err.Error())
+	}
+
+	return &csi.DeleteSnapshotResponse{}, nil
 }
 
 func (cs *ControllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) {
@@ -325,6 +473,47 @@ func (cs *ControllerServer) internalUnmount(ctx context.Context, vol *nfsVolume)
 	return err
 }
 
+func (cs *ControllerServer) copyFromSnapshot(ctx context.Context, req *csi.CreateVolumeRequest, dstVol *nfsVolume) error {
+	snap, err := getNfsSnapFromID(req.VolumeContentSource.GetSnapshot().GetSnapshotId())
+	if err != nil {
+		return status.Error(codes.NotFound, err.Error())
+	}
+	snapVol := volumeFromSnapshot(snap)
+
+	var volCap *csi.VolumeCapability
+	if len(req.GetVolumeCapabilities()) > 0 {
+		volCap = req.GetVolumeCapabilities()[0]
+	}
+
+	if err = cs.internalMount(ctx, snapVol, nil, volCap); err != nil {
+		return status.Errorf(codes.Internal, "failed to mount src nfs server for snapshot volume copy: %v", err)
+	}
+	defer func() {
+		if err = cs.internalUnmount(ctx, snapVol); err != nil {
+			klog.Warningf("failed to unmount src nfs server after snapshot volume copy: %v", err)
+		}
+	}()
+	if err = cs.internalMount(ctx, dstVol, nil, volCap); err != nil {
+		return status.Errorf(codes.Internal, "failed to mount dst nfs server for snapshot volume copy: %v", err)
+	}
+	defer func() {
+		if err = cs.internalUnmount(ctx, dstVol); err != nil {
+			klog.Warningf("failed to unmount dst nfs server after snapshot volume copy: %v", err)
+		}
+	}()
+
+	// untar snapshot archive to dst path
+	snapPath := filepath.Join(getInternalVolumePath(cs.Driver.workingMountDir, snapVol), snap.archivePath())
+	dstPath := getInternalVolumePath(cs.Driver.workingMountDir, dstVol)
+	klog.V(2).Infof("copy volume from snapshot %v -> %v", snapPath, dstPath)
+	out, err := exec.Command("tar", "-xzvf", snapPath, "-C", dstPath).CombinedOutput()
+	if err != nil {
+		return status.Errorf(codes.Internal, "failed to copy volume for snapshot: %v: %v", err, string(out))
+	}
+	klog.V(2).Infof("volume copied from snapshot %v -> %v", snapPath, dstPath)
+	return nil
+}
+
 func (cs *ControllerServer) copyFromVolume(ctx context.Context, req *csi.CreateVolumeRequest, dstVol *nfsVolume) error {
 	srcVol, err := getNfsVolFromID(req.GetVolumeContentSource().GetVolume().GetVolumeId())
 	if err != nil {
@@ -340,26 +529,26 @@ func (cs *ControllerServer) copyFromVolume(ctx context.Context, req *csi.CreateV
 		volCap = req.GetVolumeCapabilities()[0]
 	}
 	if err = cs.internalMount(ctx, srcVol, nil, volCap); err != nil {
-		return status.Errorf(codes.Internal, "failed to mount src nfs server: %v", err.Error())
+		return status.Errorf(codes.Internal, "failed to mount src nfs server: %v", err)
 	}
 	defer func() {
 		if err = cs.internalUnmount(ctx, srcVol); err != nil {
-			klog.Warningf("failed to unmount nfs server: %v", err.Error())
+			klog.Warningf("failed to unmount nfs server: %v", err)
 		}
 	}()
 	if err = cs.internalMount(ctx, dstVol, nil, volCap); err != nil {
-		return status.Errorf(codes.Internal, "failed to mount dst nfs server: %v", err.Error())
+		return status.Errorf(codes.Internal, "failed to mount dst nfs server: %v", err)
 	}
 	defer func() {
 		if err = cs.internalUnmount(ctx, dstVol); err != nil {
-			klog.Warningf("failed to unmount dst nfs server: %v", err.Error())
+			klog.Warningf("failed to unmount dst nfs server: %v", err)
 		}
 	}()
 
 	// recursive 'cp' with '-a' to handle symlinks
 	out, err := exec.Command("cp", "-a", srcPath, dstPath).CombinedOutput()
 	if err != nil {
-		return status.Error(codes.Internal, fmt.Sprintf("%v: %v", err, string(out)))
+		return status.Errorf(codes.Internal, "failed to copy volume %v: %v", err, string(out))
 	}
 	klog.V(2).Infof("copied %s -> %s", srcPath, dstPath)
 	return nil
@@ -369,7 +558,7 @@ func (cs *ControllerServer) copyVolume(ctx context.Context, req *csi.CreateVolum
 	vs := req.VolumeContentSource
 	switch vs.Type.(type) {
 	case *csi.VolumeContentSource_Snapshot:
-		return status.Error(codes.Unimplemented, "Currently only volume copy from another volume is supported")
+		return cs.copyFromSnapshot(ctx, req, vol)
 	case *csi.VolumeContentSource_Volume:
 		return cs.copyFromVolume(ctx, req, vol)
 	default:
@@ -377,6 +566,40 @@ func (cs *ControllerServer) copyVolume(ctx context.Context, req *csi.CreateVolum
 	}
 }
 
+// newNFSSnapshot Convert VolumeSnapshot parameters to a nfsSnapshot
+func newNFSSnapshot(name string, params map[string]string, vol *nfsVolume) (*nfsSnapshot, error) {
+	server := vol.server
+	baseDir := vol.baseDir
+	for k, v := range params {
+		switch strings.ToLower(k) {
+		case paramServer:
+			server = v
+		case paramShare:
+			baseDir = v
+		}
+	}
+
+	if server == "" {
+		return nil, fmt.Errorf("%v is a required parameter", paramServer)
+	}
+	snapshot := &nfsSnapshot{
+		server:  server,
+		baseDir: baseDir,
+		uuid:    name,
+	}
+	if vol.subDir != "" {
+		snapshot.src = vol.subDir
+	}
+	if vol.uuid != "" {
+		snapshot.src = vol.uuid
+	}
+	if snapshot.src == "" {
+		return nil, fmt.Errorf("missing required source volume name")
+	}
+	snapshot.id = getSnapshotIDFromNfsSnapshot(snapshot)
+	return snapshot, nil
+}
+
 // newNFSVolume Convert VolumeCreate parameters to an nfsVolume
 func newNFSVolume(name string, size int64, params map[string]string, defaultOnDeletePolicy string) (*nfsVolume, error) {
 	var server, baseDir, subDir, onDelete string
@@ -470,6 +693,17 @@ func getVolumeIDFromNfsVol(vol *nfsVolume) string {
 	return strings.Join(idElements, separator)
 }
 
+// Given a nfsSnapshot, return a CSI snapshot id.
+func getSnapshotIDFromNfsSnapshot(snap *nfsSnapshot) string {
+	idElements := make([]string, totalIDSnapElements)
+	idElements[idSnapServer] = strings.Trim(snap.server, "/")
+	idElements[idSnapBaseDir] = strings.Trim(snap.baseDir, "/")
+	idElements[idSnapUUID] = snap.uuid
+	idElements[idSnapArchivePath] = snap.uuid
+	idElements[idSnapArchiveName] = snap.src
+	return strings.Join(idElements, separator)
+}
+
 // Given a CSI volume id, return a nfsVolume
 // sample volume Id:
 //
@@ -513,6 +747,25 @@ func getNfsVolFromID(id string) (*nfsVolume, error) {
 	}, nil
 }
 
+// Given a CSI snapshot ID, return a nfsSnapshot
+// sample snapshot ID:
+//
+//	nfs-server.default.svc.cluster.local#share#snapshot-016f784f-56f4-44d1-9041-5f59e82dbce1#snapshot-016f784f-56f4-44d1-9041-5f59e82dbce1#pvc-4bcbf944-b6f7-4bd0-b50f-3c3dd00efc64
+func getNfsSnapFromID(id string) (*nfsSnapshot, error) {
+	segments := strings.Split(id, separator)
+	if len(segments) == totalIDSnapElements {
+		return &nfsSnapshot{
+			id:      id,
+			server:  segments[idSnapServer],
+			baseDir: segments[idSnapBaseDir],
+			src:     segments[idSnapArchiveName],
+			uuid:    segments[idSnapUUID],
+		}, nil
+	}
+
+	return &nfsSnapshot{}, fmt.Errorf("failed to create nfsSnapshot from snapshot ID")
+}
+
 // isValidVolumeCapabilities validates the given VolumeCapability array is valid
 func isValidVolumeCapabilities(volCaps []*csi.VolumeCapability) error {
 	if len(volCaps) == 0 {
@@ -525,3 +778,32 @@ func isValidVolumeCapabilities(volCaps []*csi.VolumeCapability) error {
 	}
 	return nil
 }
+
+// Validate snapshot after internal mount
+func validateSnapshot(snapInternalVolPath string, snap *nfsSnapshot) error {
+	return filepath.WalkDir(snapInternalVolPath, func(path string, d fs.DirEntry, err error) error {
+		if path == snapInternalVolPath {
+			// skip root
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+		if d.Name() != snap.archiveName() {
+			// there should be just one archive in the snapshot path and archive name should match
+			return status.Errorf(codes.AlreadyExists, "snapshot with the same name but different source volume ID already exists: found %q, desired %q", d.Name(), snap.archiveName())
+		}
+		return nil
+	})
+}
+
+// Volume for snapshot internal mount/unmount
+func volumeFromSnapshot(snap *nfsSnapshot) *nfsVolume {
+	return &nfsVolume{
+		id:      snap.id,
+		server:  snap.server,
+		baseDir: snap.baseDir,
+		subDir:  snap.baseDir,
+		uuid:    snap.uuid,
+	}
+}

pkg/nfs/controllerserver_test.go

@@ -17,6 +17,8 @@ limitations under the License.
 package nfs
 
 import (
+	"archive/tar"
+	"compress/gzip"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -31,6 +33,7 @@ import (
 	"golang.org/x/net/context"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/types/known/timestamppb"
 	mount "k8s.io/mount-utils"
 )
 
@@ -352,6 +355,13 @@ func TestControllerGetCapabilities(t *testing.T) {
 							},
 						},
 					},
+					{
+						Type: &csi.ControllerServiceCapability_Rpc{
+							Rpc: &csi.ControllerServiceCapability_RPC{
+								Type: csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT,
+							},
+						},
+					},
 				},
 			},
 			expectedErr: nil,
@@ -687,6 +697,8 @@ func TestCopyVolume(t *testing.T) {
 		req       *csi.CreateVolumeRequest
 		dstVol    *nfsVolume
 		expectErr bool
+		prepare   func() error
+		cleanup   func() error
 	}{
 		{
 			desc: "copy volume from valid volume",
@@ -707,6 +719,56 @@ func TestCopyVolume(t *testing.T) {
 				subDir:  "subdir",
 				uuid:    "dst-pv-name",
 			},
+			prepare: func() error { return os.MkdirAll("/tmp/src-pv-name/subdir", 0777) },
+			cleanup: func() error { return os.RemoveAll("/tmp/src-pv-name") },
+		},
+		{
+			desc: "copy volume from valid snapshot",
+			req: &csi.CreateVolumeRequest{
+				Name: "snapshot-name",
+				VolumeContentSource: &csi.VolumeContentSource{
+					Type: &csi.VolumeContentSource_Snapshot{
+						Snapshot: &csi.VolumeContentSource_SnapshotSource{
+							SnapshotId: "nfs-server.default.svc.cluster.local#share#snapshot-name#snapshot-name#src-pv-name",
+						},
+					},
+				},
+			},
+			dstVol: &nfsVolume{
+				id:      "nfs-server.default.svc.cluster.local#share#subdir#dst-pv-name",
+				server:  "//nfs-server.default.svc.cluster.local",
+				baseDir: "share",
+				subDir:  "subdir",
+				uuid:    "dst-pv-name",
+			},
+			prepare: func() error {
+				if err := os.MkdirAll("/tmp/snapshot-name/share/snapshot-name/", 0777); err != nil {
+					return err
+				}
+				file, err := os.Create("/tmp/snapshot-name/share/snapshot-name/src-pv-name.tar.gz")
+				if err != nil {
+					return err
+				}
+				defer file.Close()
+				gzipWriter := gzip.NewWriter(file)
+				defer gzipWriter.Close()
+				tarWriter := tar.NewWriter(gzipWriter)
+				defer tarWriter.Close()
+				body := "test file"
+				hdr := &tar.Header{
+					Name: "test.txt",
+					Mode: 0777,
+					Size: int64(len(body)),
+				}
+				if err := tarWriter.WriteHeader(hdr); err != nil {
+					return err
+				}
+				if _, err := tarWriter.Write([]byte(body)); err != nil {
+					return err
+				}
+				return nil
+			},
+			cleanup: func() error { return os.RemoveAll("/tmp/snapshot-name") },
 		},
 		{
 			desc: "copy volume missing source id",
@@ -747,17 +809,263 @@ func TestCopyVolume(t *testing.T) {
 			},
 			expectErr: true,
 		},
-	}
+		{
-	if err := os.MkdirAll("/tmp/src-pv-name/subdir", 0777); err != nil {
+			desc: "copy volume from broken snapshot",
-		t.Fatalf("Unexpected error when creating srcVolume: %v", err)
+			req: &csi.CreateVolumeRequest{
+				Name: "snapshot-name",
+				VolumeContentSource: &csi.VolumeContentSource{
+					Type: &csi.VolumeContentSource_Snapshot{
+						Snapshot: &csi.VolumeContentSource_SnapshotSource{
+							SnapshotId: "nfs-server.default.svc.cluster.local#share#snapshot-name#snapshot-name#src-pv-name",
+						},
+					},
+				},
+			},
+			dstVol: &nfsVolume{
+				id:      "nfs-server.default.svc.cluster.local#share#subdir#dst-pv-name",
+				server:  "//nfs-server.default.svc.cluster.local",
+				baseDir: "share",
+				subDir:  "subdir",
+				uuid:    "dst-pv-name",
+			},
+			expectErr: true,
+		},
+		{
+			desc: "copy volume from missing snapshot",
+			req: &csi.CreateVolumeRequest{
+				Name: "snapshot-name",
+				VolumeContentSource: &csi.VolumeContentSource{
+					Type: &csi.VolumeContentSource_Snapshot{
+						Snapshot: &csi.VolumeContentSource_SnapshotSource{},
+					},
+				},
+			},
+			dstVol: &nfsVolume{
+				id:      "nfs-server.default.svc.cluster.local#share#subdir#dst-pv-name",
+				server:  "//nfs-server.default.svc.cluster.local",
+				baseDir: "share",
+				subDir:  "subdir",
+				uuid:    "dst-pv-name",
+			},
+			expectErr: true,
+		},
+		{
+			desc: "copy volume from snapshot into missing dst volume",
+			req: &csi.CreateVolumeRequest{
+				Name: "snapshot-name",
+				VolumeContentSource: &csi.VolumeContentSource{
+					Type: &csi.VolumeContentSource_Snapshot{
+						Snapshot: &csi.VolumeContentSource_SnapshotSource{},
+					},
+				},
+			},
+			dstVol: &nfsVolume{
+				server:  "//nfs-server.default.svc.cluster.local",
+				baseDir: "share",
+				subDir:  "subdir",
+				uuid:    "dst-pv-name",
+			},
+			expectErr: true,
+		},
 	}
 	for _, test := range cases {
 		t.Run(test.desc, func(t *testing.T) {
+			if test.prepare != nil {
+				if err := test.prepare(); err != nil {
+					t.Errorf(`[test: %s] prepare failed: "%v"`, test.desc, err)
+				}
+			}
 			cs := initTestController(t)
 			err := cs.copyVolume(context.TODO(), test.req, test.dstVol)
 			if (err == nil) == test.expectErr {
 				t.Errorf(`[test: %s] Error expectation mismatch, expected error: "%v", received: %q`, test.desc, test.expectErr, err)
 			}
+			if test.cleanup != nil {
+				if err := test.cleanup(); err != nil {
+					t.Errorf(`[test: %s] cleanup failed: "%v"`, test.desc, err)
+				}
+			}
 		})
 	}
 }
+
+func TestCreateSnapshot(t *testing.T) {
+	cases := []struct {
+		desc      string
+		req       *csi.CreateSnapshotRequest
+		expResp   *csi.CreateSnapshotResponse
+		expectErr bool
+		prepare   func() error
+		cleanup   func() error
+	}{
+		{
+			desc: "create snapshot with valid request",
+			req: &csi.CreateSnapshotRequest{
+				SourceVolumeId: "nfs-server.default.svc.cluster.local#share#subdir#src-pv-name",
+				Name:           "snapshot-name",
+			},
+			expResp: &csi.CreateSnapshotResponse{
+				Snapshot: &csi.Snapshot{
+					SnapshotId:     "nfs-server.default.svc.cluster.local#share#snapshot-name#snapshot-name#src-pv-name",
+					SourceVolumeId: "nfs-server.default.svc.cluster.local#share#subdir#src-pv-name",
+					ReadyToUse:     true,
+					SizeBytes:      1,                 // doesn't match exact size, just denotes non-zero size expected
+					CreationTime:   timestamppb.Now(), // doesn't match exact timestamp, just denotes non-zero ts expected
+				},
+			},
+			prepare: func() error { return os.MkdirAll("/tmp/src-pv-name/subdir", 0777) },
+			cleanup: func() error { return os.RemoveAll("/tmp/src-pv-name") },
+		},
+		{
+			desc: "create snapshot from nonexisting volume",
+			req: &csi.CreateSnapshotRequest{
+				SourceVolumeId: "nfs-server.default.svc.cluster.local#share#subdir#src-pv-name",
+				Name:           "snapshot-name",
+			},
+			expectErr: true,
+		},
+	}
+	for _, test := range cases {
+		t.Run(test.desc, func(t *testing.T) {
+			if test.prepare != nil {
+				if err := test.prepare(); err != nil {
+					t.Errorf(`[test: %s] prepare failed: "%v"`, test.desc, err)
+				}
+			}
+			cs := initTestController(t)
+			resp, err := cs.CreateSnapshot(context.TODO(), test.req)
+			if (err == nil) == test.expectErr {
+				t.Errorf(`[test: %s] Error expectation mismatch, expected error: "%v", received: %q`, test.desc, test.expectErr, err)
+			}
+			if err := matchCreateSnapshotResponse(test.expResp, resp); err != nil {
+				t.Errorf("[test: %s] failed %q: got resp %+v, expected %+v", test.desc, err, resp, test.expResp)
+			}
+			if test.cleanup != nil {
+				if err := test.cleanup(); err != nil {
+					t.Errorf(`[test: %s] cleanup failed: "%v"`, test.desc, err)
+				}
+			}
+		})
+	}
+}
+
+func TestDeleteSnapshot(t *testing.T) {
+	cases := []struct {
+		desc      string
+		req       *csi.DeleteSnapshotRequest
+		expResp   *csi.DeleteSnapshotResponse
+		expectErr bool
+		prepare   func() error
+		cleanup   func() error
+	}{
+		{
+			desc: "delete valid snapshot",
+			req: &csi.DeleteSnapshotRequest{
+				SnapshotId: "nfs-server.default.svc.cluster.local#share#snapshot-name#snapshot-name#src-pv-name",
+			},
+			expResp: &csi.DeleteSnapshotResponse{},
+			prepare: func() error {
+				if err := os.MkdirAll("/tmp/snapshot-name/snapshot-name/", 0777); err != nil {
+					return err
+				}
+				f, err := os.OpenFile("/tmp/snapshot-name/snapshot-name/src-pv-name.tar.gz", os.O_CREATE, 0777)
+				if err != nil {
+					return err
+				}
+				return f.Close()
+			},
+			cleanup: func() error { return os.RemoveAll("/tmp/snapshot-name") },
+		},
+		{
+			desc: "delete nonexisting snapshot",
+			req: &csi.DeleteSnapshotRequest{
+				SnapshotId: "nfs-server.default.svc.cluster.local#share#snapshot-name#snapshot-name#src-pv-name",
+			},
+			expResp: &csi.DeleteSnapshotResponse{},
+		},
+		{
+			desc: "delete snapshot with improper id",
+			req: &csi.DeleteSnapshotRequest{
+				SnapshotId: "incorrect-snap-id",
+			},
+			expResp: &csi.DeleteSnapshotResponse{},
+		},
+		{
+			desc: "delete valid snapshot with mount options",
+			req: &csi.DeleteSnapshotRequest{
+				SnapshotId: "nfs-server.default.svc.cluster.local#share#snapshot-name#snapshot-name#src-pv-name",
+				Secrets:    map[string]string{"mountoptions": "nfsvers=4.1"},
+			},
+			expResp: &csi.DeleteSnapshotResponse{},
+			prepare: func() error {
+				if err := os.MkdirAll("/tmp/snapshot-name/snapshot-name/", 0777); err != nil {
+					return err
+				}
+				f, err := os.OpenFile("/tmp/snapshot-name/snapshot-name/src-pv-name.tar.gz", os.O_CREATE, 0777)
+				if err != nil {
+					return err
+				}
+				return f.Close()
+			},
+			cleanup: func() error { return os.RemoveAll("/tmp/snapshot-name") },
+		},
+	}
+	for _, test := range cases {
+		t.Run(test.desc, func(t *testing.T) {
+			if test.prepare != nil {
+				if err := test.prepare(); err != nil {
+					t.Errorf(`[test: %s] prepare failed: "%v"`, test.desc, err)
+				}
+			}
+			cs := initTestController(t)
+			resp, err := cs.DeleteSnapshot(context.TODO(), test.req)
+			if (err == nil) == test.expectErr {
+				t.Errorf(`[test: %s] Error expectation mismatch, expected error: "%v", received: %q`, test.desc, test.expectErr, err)
+			}
+			if !reflect.DeepEqual(test.expResp, resp) {
+				t.Errorf("[test: %s] got resp %+v, expected %+v", test.desc, resp, test.expResp)
+			}
+			if test.cleanup != nil {
+				if err := test.cleanup(); err != nil {
+					t.Errorf(`[test: %s] cleanup failed: "%v"`, test.desc, err)
+				}
+			}
+		})
+	}
+}
+
+func matchCreateSnapshotResponse(e, r *csi.CreateSnapshotResponse) error {
+	if e == nil && r == nil {
+		return nil
+	}
+	if e == nil || e.Snapshot == nil {
+		return fmt.Errorf("expected nil response")
+	}
+	if r == nil || r.Snapshot == nil {
+		return fmt.Errorf("unexpected nil response")
+	}
+	es, rs := e.Snapshot, r.Snapshot
+
+	var errs []string
+	// comparing ts and size just for presence, not the exact value
+	if es.CreationTime.IsValid() != rs.CreationTime.IsValid() {
+		errs = append(errs, "CreationTime")
+	}
+	if (es.SizeBytes == 0) != (rs.SizeBytes == 0) {
+		errs = append(errs, "SizeBytes")
+	}
+	// comparing remaining fields for exact match
+	if es.ReadyToUse != rs.ReadyToUse {
+		errs = append(errs, "ReadyToUse")
+	}
+	if es.SnapshotId != rs.SnapshotId {
+		errs = append(errs, "SnapshotId")
+	}
+	if es.SourceVolumeId != rs.SourceVolumeId {
+		errs = append(errs, "SourceVolumeId")
+	}
+	if len(errs) == 0 {
+		return nil
+	}
+	return fmt.Errorf("mismatch CreateSnapshotResponse in fields: %v", strings.Join(errs, ", "))
+}

pkg/nfs/nfs.go

@@ -88,6 +88,7 @@ func NewDriver(options *DriverOptions) *Driver {
 		csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
 		csi.ControllerServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER,
 		csi.ControllerServiceCapability_RPC_CLONE_VOLUME,
+		csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT,
 	})
 
 	n.AddNodeServiceCapabilities([]csi.NodeServiceCapability_RPC_Type{

test/external-e2e/run.sh

@@ -32,8 +32,11 @@ setup_e2e_binaries() {
 
     # test on alternative driver name
     sed -i "s/nfs.csi.k8s.io/$DRIVER.csi.k8s.io/g" deploy/example/storageclass-nfs.yaml
+    sed -i "s/nfs.csi.k8s.io/$DRIVER.csi.k8s.io/g" deploy/example/snapshotclass-nfs.yaml
     # install csi driver
-    mkdir -p /tmp/csi && cp deploy/example/storageclass-nfs.yaml /tmp/csi/storageclass.yaml
+    mkdir -p /tmp/csi
+    cp deploy/example/storageclass-nfs.yaml /tmp/csi/storageclass.yaml
+    cp deploy/example/snapshotclass-nfs.yaml /tmp/csi/snapshotclass.yaml
     make e2e-bootstrap
     make install-nfs-server
 }

test/external-e2e/testdriver.yaml

@@ -3,6 +3,8 @@
 
 StorageClass:
   FromFile: /tmp/csi/storageclass.yaml
+SnapshotClass:
+  FromName: true
 DriverInfo:
   Name: test.csi.k8s.io
   SupportedFsType: {"nfs"}
@@ -13,6 +15,7 @@ DriverInfo:
     RWX: true
     fsGroup: true
     pvcDataSource: true
+    snapshotDataSource: true
 InlineVolumes:
 - Attributes:
     server: nfs-server.default.svc.cluster.local