feat: add csi-snapshotter sidecar and external-snapshotter

$ helm package charts/latest/csi-driver-nfs -d charts/latest/

charts/latest/csi-driver-nfs/templates/csi-nfs-controller.yaml

@@ -63,6 +63,22 @@ spec:
           resources: {{- toYaml .Values.controller.resources.csiProvisioner | nindent 12 }}
           securityContext:
             readOnlyRootFilesystem: true
+        {{- if .Values.externalSnapshotter.enabled }}
+        - name: csi-snapshotter
+          image: "{{ .Values.image.csiSnapshotter.repository }}:{{ .Values.image.csiSnapshotter.tag }}"
+          args:
+            - "--v=5"
+            - "--csi-address=$(ADDRESS)"
+            - "--leader-election-namespace={{ .Release.Namespace }}"
+            - "--leader-election"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          imagePullPolicy: {{ .Values.image.csiSnapshotter.pullPolicy }}
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        {{- end }}
         - name: liveness-probe
           image: "{{ .Values.image.livenessProbe.repository }}:{{ .Values.image.livenessProbe.tag }}"
           args:

charts/latest/csi-driver-nfs/templates/csi-snapshot-controller.yaml

@@ -0,0 +1,51 @@
+{{- if .Values.externalSnapshotter.enabled -}}
+# This YAML file shows how to deploy the snapshot controller
+
+# The snapshot controller implements the control loop for CSI snapshot functionality.
+# It should be installed as part of the base Kubernetes distribution in an appropriate
+# namespace for components implementing base system functionality. For installing with
+# Vanilla Kubernetes, kube-system makes sense for the namespace.
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: {{ .Values.externalSnapshotter.name }}
+  namespace: {{ .Release.Namespace }}
+{{ include "nfs.labels" . | indent 2 }}
+    app: {{ .Values.externalSnapshotter.name }}
+{{- with .Values.externalSnapshotter.labels }}
+{{ . | toYaml | indent 4 }}
+{{- end }}
+{{- with .Values.externalSnapshotter.annotations }}
+  annotations:
+{{ . | toYaml | indent 4 }}
+{{- end }}
+spec:
+  replicas: {{ .Values.externalSnapshotter.controller.replicas }}
+  selector:
+    matchLabels:
+      app: {{ .Values.externalSnapshotter.name }}
+  # the snapshot controller won't be marked as ready if the v1 CRDs are unavailable
+  # in #504 the snapshot-controller will exit after around 7.5 seconds if it
+  # can't find the v1 CRDs so this value should be greater than that
+  minReadySeconds: 15
+  strategy:
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: {{ .Values.externalSnapshotter.name }}
+    spec:
+      serviceAccountName: {{ .Values.externalSnapshotter.name }}
+      containers:
+        - name: {{ .Values.externalSnapshotter.name }}
+          image: {{ .Values.image.externalSnapshotter.repository }}:{{ .Values.image.externalSnapshotter.tag }}
+          args:
+            - "--v=5"
+            - "--leader-election=true"
+            - "--leader-election-namespace={{ .Release.Namespace }}"
+          imagePullPolicy: {{ .Values.image.externalSnapshotter.pullPolicy }}
+{{- end -}}

charts/latest/csi-driver-nfs/templates/rbac-csi-nfs.yaml

@@ -32,6 +32,17 @@ rules:
   - apiGroups: ["storage.k8s.io"]
     resources: ["storageclasses"]
     verbs: ["get", "list", "watch"]
+  {{- if .Values.externalSnapshotter.enabled }}
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses", "volumesnapshots"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["get", "update", "patch"]
+  {{- end }}
   - apiGroups: [""]
     resources: ["events"]
     verbs: ["get", "list", "watch", "create", "update", "patch"]

charts/latest/csi-driver-nfs/templates/rbac-snapshot-controller.yaml

@@ -0,0 +1,93 @@
+{{- if .Values.externalSnapshotter.enabled -}}
+# RBAC file for the snapshot controller.
+#
+# The snapshot controller implements the control loop for CSI snapshot functionality.
+# It should be installed as part of the base Kubernetes distribution in an appropriate
+# namespace for components implementing base system functionality. For installing with
+# Vanilla Kubernetes, kube-system makes sense for the namespace.
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Values.externalSnapshotter.name }}
+  namespace: {{ .Release.Namespace }}
+{{ include "nfs.labels" . | indent 2 }}
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ .Values.externalSnapshotter.name }}-runner
+{{ include "nfs.labels" . | indent 2 }}
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots/status"]
+    verbs: ["update", "patch"]
+{{- if .Values.externalSnapshotter.enabledDistributedSnapshotting }}
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+{{- end }}
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ .Values.externalSnapshotter.name }}-role
+{{ include "nfs.labels" . | indent 2 }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.externalSnapshotter.name }}
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: {{ .Values.externalSnapshotter.name }}-runner
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ .Values.externalSnapshotter.name }}-leaderelection
+  namespace: {{ .Release.Namespace }}
+{{ include "nfs.labels" . | indent 2 }}
+rules:
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ .Values.externalSnapshotter.name }}-leaderelection
+  namespace: {{ .Release.Namespace }}
+{{ include "nfs.labels" . | indent 2 }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.externalSnapshotter.name }}
+roleRef:
+  kind: Role
+  name: {{ .Values.externalSnapshotter.name }}-leaderelection
+  apiGroup: rbac.authorization.k8s.io
+{{- end -}}

charts/latest/csi-driver-nfs/templates/snapshot.storage.k8s.io_volumesnapshotclasses.yaml

@@ -0,0 +1,146 @@
+{{- if .Values.externalSnapshotter.enabled -}}
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.8.0
+    api-approved.kubernetes.io: https://github.com/kubernetes-csi/external-snapshotter/pull/665
+  name: volumesnapshotclasses.snapshot.storage.k8s.io
+spec:
+  group: snapshot.storage.k8s.io
+  names:
+    kind: VolumeSnapshotClass
+    listKind: VolumeSnapshotClassList
+    plural: volumesnapshotclasses
+    shortNames: [vsclass, vsclasses]
+    singular: volumesnapshotclass
+  scope: Cluster
+  versions:
+    - additionalPrinterColumns:
+        - jsonPath: .driver
+          name: Driver
+          type: string
+        - description: Determines whether a VolumeSnapshotContent created through
+            the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot
+            is deleted.
+          jsonPath: .deletionPolicy
+          name: DeletionPolicy
+          type: string
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshotClass specifies parameters that a underlying
+            storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass
+            is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses
+            are non-namespaced
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+                of an object. Servers should convert recognized schemas to the latest
+                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            deletionPolicy:
+              description: deletionPolicy determines whether a VolumeSnapshotContent
+                created through the VolumeSnapshotClass should be deleted when its
+                bound VolumeSnapshot is deleted. Supported values are "Retain" and
+                "Delete". "Retain" means that the VolumeSnapshotContent and its physical
+                snapshot on underlying storage system are kept. "Delete" means that
+                the VolumeSnapshotContent and its physical snapshot on underlying
+                storage system are deleted. Required.
+              enum: [Delete, Retain]
+              type: string
+            driver:
+              description: driver is the name of the storage driver that handles this
+                VolumeSnapshotClass. Required.
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource
+                this object represents. Servers may infer this from the endpoint the
+                client submits requests to. Cannot be updated. In CamelCase. More
+                info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            parameters:
+              additionalProperties:
+                type: string
+              description: parameters is a key-value map with storage driver specific
+                parameters for creating snapshots. These values are opaque to Kubernetes.
+              type: object
+          required: [deletionPolicy, driver]
+          type: object
+      served: true
+      storage: true
+      subresources: {}
+    - additionalPrinterColumns:
+        - jsonPath: .driver
+          name: Driver
+          type: string
+        - description: Determines whether a VolumeSnapshotContent created through
+            the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot
+            is deleted.
+          jsonPath: .deletionPolicy
+          name: DeletionPolicy
+          type: string
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1beta1
+      # This indicates the v1beta1 version of the custom resource is deprecated.
+      # API requests to this version receive a warning in the server response.
+      deprecated: true
+      # This overrides the default warning returned to clients making v1beta1 API requests.
+      deprecationWarning: snapshot.storage.k8s.io/v1beta1 VolumeSnapshotClass is deprecated;
+        use snapshot.storage.k8s.io/v1 VolumeSnapshotClass
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshotClass specifies parameters that a underlying
+            storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass
+            is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses
+            are non-namespaced
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+                of an object. Servers should convert recognized schemas to the latest
+                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            deletionPolicy:
+              description: deletionPolicy determines whether a VolumeSnapshotContent
+                created through the VolumeSnapshotClass should be deleted when its
+                bound VolumeSnapshot is deleted. Supported values are "Retain" and
+                "Delete". "Retain" means that the VolumeSnapshotContent and its physical
+                snapshot on underlying storage system are kept. "Delete" means that
+                the VolumeSnapshotContent and its physical snapshot on underlying
+                storage system are deleted. Required.
+              enum: [Delete, Retain]
+              type: string
+            driver:
+              description: driver is the name of the storage driver that handles this
+                VolumeSnapshotClass. Required.
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource
+                this object represents. Servers may infer this from the endpoint the
+                client submits requests to. Cannot be updated. In CamelCase. More
+                info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            parameters:
+              additionalProperties:
+                type: string
+              description: parameters is a key-value map with storage driver specific
+                parameters for creating snapshots. These values are opaque to Kubernetes.
+              type: object
+          required: [deletionPolicy, driver]
+          type: object
+      served: false
+      storage: false
+      subresources: {}
+status:
+  acceptedNames:
+    kind: ''
+    plural: ''
+  conditions: []
+  storedVersions: []
+{{- end -}}

charts/latest/csi-driver-nfs/templates/snapshot.storage.k8s.io_volumesnapshotcontents.yaml

@@ -0,0 +1,473 @@
+{{- if .Values.externalSnapshotter.enabled -}}
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.8.0
+    api-approved.kubernetes.io: https://github.com/kubernetes-csi/external-snapshotter/pull/665
+  name: volumesnapshotcontents.snapshot.storage.k8s.io
+spec:
+  group: snapshot.storage.k8s.io
+  names:
+    kind: VolumeSnapshotContent
+    listKind: VolumeSnapshotContentList
+    plural: volumesnapshotcontents
+    shortNames: [vsc, vscs]
+    singular: volumesnapshotcontent
+  scope: Cluster
+  versions:
+    - additionalPrinterColumns:
+        - description: Indicates if the snapshot is ready to be used to restore a
+            volume.
+          jsonPath: .status.readyToUse
+          name: ReadyToUse
+          type: boolean
+        - description: Represents the complete size of the snapshot in bytes
+          jsonPath: .status.restoreSize
+          name: RestoreSize
+          type: integer
+        - description: Determines whether this VolumeSnapshotContent and its physical
+            snapshot on the underlying storage system should be deleted when its bound
+            VolumeSnapshot is deleted.
+          jsonPath: .spec.deletionPolicy
+          name: DeletionPolicy
+          type: string
+        - description: Name of the CSI driver used to create the physical snapshot
+            on the underlying storage system.
+          jsonPath: .spec.driver
+          name: Driver
+          type: string
+        - description: Name of the VolumeSnapshotClass to which this snapshot belongs.
+          jsonPath: .spec.volumeSnapshotClassName
+          name: VolumeSnapshotClass
+          type: string
+        - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent
+            object is bound.
+          jsonPath: .spec.volumeSnapshotRef.name
+          name: VolumeSnapshot
+          type: string
+        - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent
+            object is bound.
+          jsonPath: .spec.volumeSnapshotRef.namespace
+          name: VolumeSnapshotNamespace
+          type: string
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshotContent represents the actual "on-disk" snapshot
+            object in the underlying storage system
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+                of an object. Servers should convert recognized schemas to the latest
+                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource
+                this object represents. Servers may infer this from the endpoint the
+                client submits requests to. Cannot be updated. In CamelCase. More
+                info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            spec:
+              description: spec defines properties of a VolumeSnapshotContent created
+                by the underlying storage system. Required.
+              properties:
+                deletionPolicy:
+                  description: deletionPolicy determines whether this VolumeSnapshotContent
+                    and its physical snapshot on the underlying storage system should
+                    be deleted when its bound VolumeSnapshot is deleted. Supported
+                    values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent
+                    and its physical snapshot on underlying storage system are kept.
+                    "Delete" means that the VolumeSnapshotContent and its physical
+                    snapshot on underlying storage system are deleted. For dynamically
+                    provisioned snapshots, this field will automatically be filled
+                    in by the CSI snapshotter sidecar with the "DeletionPolicy" field
+                    defined in the corresponding VolumeSnapshotClass. For pre-existing
+                    snapshots, users MUST specify this field when creating the VolumeSnapshotContent
+                    object. Required.
+                  enum: [Delete, Retain]
+                  type: string
+                driver:
+                  description: driver is the name of the CSI driver used to create
+                    the physical snapshot on the underlying storage system. This MUST
+                    be the same as the name returned by the CSI GetPluginName() call
+                    for that driver. Required.
+                  type: string
+                source:
+                  description: source specifies whether the snapshot is (or should
+                    be) dynamically provisioned or already exists, and just requires
+                    a Kubernetes object representation. This field is immutable after
+                    creation. Required.
+                  properties:
+                    snapshotHandle:
+                      description: snapshotHandle specifies the CSI "snapshot_id"
+                        of a pre-existing snapshot on the underlying storage system
+                        for which a Kubernetes object representation was (or should
+                        be) created. This field is immutable.
+                      type: string
+                    volumeHandle:
+                      description: volumeHandle specifies the CSI "volume_id" of the
+                        volume from which a snapshot should be dynamically taken from.
+                        This field is immutable.
+                      type: string
+                  type: object
+                  oneOf:
+                    - required: [snapshotHandle]
+                    - required: [volumeHandle]
+                sourceVolumeMode:
+                  description: SourceVolumeMode is the mode of the volume whose snapshot
+                    is taken. Can be either “Filesystem” or “Block”. If not specified,
+                    it indicates the source volume's mode is unknown. This field is
+                    immutable. This field is an alpha field.
+                  type: string
+                volumeSnapshotClassName:
+                  description: name of the VolumeSnapshotClass from which this snapshot
+                    was (or will be) created. Note that after provisioning, the VolumeSnapshotClass
+                    may be deleted or recreated with different set of values, and
+                    as such, should not be referenced post-snapshot creation.
+                  type: string
+                volumeSnapshotRef:
+                  description: volumeSnapshotRef specifies the VolumeSnapshot object
+                    to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName
+                    field must reference to this VolumeSnapshotContent's name for
+                    the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent
+                    object, name and namespace of the VolumeSnapshot object MUST be
+                    provided for binding to happen. This field is immutable after
+                    creation. Required.
+                  properties:
+                    apiVersion:
+                      description: API version of the referent.
+                      type: string
+                    fieldPath:
+                      description: 'If referring to a piece of an object instead of
+                        an entire object, this string should contain a valid JSON/Go
+                        field access statement, such as desiredState.manifest.containers[2].
+                        For example, if the object reference is to a container within
+                        a pod, this would take on a value like: "spec.containers{name}"
+                        (where "name" refers to the name of the container that triggered
+                        the event) or if no container name is specified "spec.containers[2]"
+                        (container with index 2 in this pod). This syntax is chosen
+                        only to have some well-defined way of referencing a part of
+                        an object. TODO: this design is not final and this field is
+                        subject to change in the future.'
+                      type: string
+                    kind:
+                      description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                      type: string
+                    name:
+                      description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                      type: string
+                    namespace:
+                      description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
+                      type: string
+                    resourceVersion:
+                      description: 'Specific resourceVersion to which this reference
+                        is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
+                      type: string
+                    uid:
+                      description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                      type: string
+                  type: object
+              required: [deletionPolicy, driver, source, volumeSnapshotRef]
+              type: object
+            status:
+              description: status represents the current information of a snapshot.
+              properties:
+                creationTime:
+                  description: creationTime is the timestamp when the point-in-time
+                    snapshot is taken by the underlying storage system. In dynamic
+                    snapshot creation case, this field will be filled in by the CSI
+                    snapshotter sidecar with the "creation_time" value returned from
+                    CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this
+                    field will be filled with the "creation_time" value returned from
+                    the CSI "ListSnapshots" gRPC call if the driver supports it. If
+                    not specified, it indicates the creation time is unknown. The
+                    format of this field is a Unix nanoseconds time encoded as an
+                    int64. On Unix, the command `date +%s%N` returns the current time
+                    in nanoseconds since 1970-01-01 00:00:00 UTC.
+                  format: int64
+                  type: integer
+                error:
+                  description: error is the last observed error during snapshot creation,
+                    if any. Upon success after retry, this error field will be cleared.
+                  properties:
+                    message:
+                      description: 'message is a string detailing the encountered
+                        error during snapshot creation if specified. NOTE: message
+                        may be logged, and it should not contain sensitive information.'
+                      type: string
+                    time:
+                      description: time is the timestamp when the error was encountered.
+                      format: date-time
+                      type: string
+                  type: object
+                readyToUse:
+                  description: readyToUse indicates if a snapshot is ready to be used
+                    to restore a volume. In dynamic snapshot creation case, this field
+                    will be filled in by the CSI snapshotter sidecar with the "ready_to_use"
+                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "ready_to_use" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver
+                    supports it, otherwise, this field will be set to "True". If not
+                    specified, it means the readiness of a snapshot is unknown.
+                  type: boolean
+                restoreSize:
+                  description: restoreSize represents the complete size of the snapshot
+                    in bytes. In dynamic snapshot creation case, this field will be
+                    filled in by the CSI snapshotter sidecar with the "size_bytes"
+                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "size_bytes" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver
+                    supports it. When restoring a volume from this snapshot, the size
+                    of the volume MUST NOT be smaller than the restoreSize if it is
+                    specified, otherwise the restoration will fail. If not specified,
+                    it indicates that the size is unknown.
+                  format: int64
+                  minimum: 0
+                  type: integer
+                snapshotHandle:
+                  description: snapshotHandle is the CSI "snapshot_id" of a snapshot
+                    on the underlying storage system. If not specified, it indicates
+                    that dynamic snapshot creation has either failed or it is still
+                    in progress.
+                  type: string
+              type: object
+          required: [spec]
+          type: object
+      served: true
+      storage: true
+      subresources:
+        status: {}
+    - additionalPrinterColumns:
+        - description: Indicates if the snapshot is ready to be used to restore a
+            volume.
+          jsonPath: .status.readyToUse
+          name: ReadyToUse
+          type: boolean
+        - description: Represents the complete size of the snapshot in bytes
+          jsonPath: .status.restoreSize
+          name: RestoreSize
+          type: integer
+        - description: Determines whether this VolumeSnapshotContent and its physical
+            snapshot on the underlying storage system should be deleted when its bound
+            VolumeSnapshot is deleted.
+          jsonPath: .spec.deletionPolicy
+          name: DeletionPolicy
+          type: string
+        - description: Name of the CSI driver used to create the physical snapshot
+            on the underlying storage system.
+          jsonPath: .spec.driver
+          name: Driver
+          type: string
+        - description: Name of the VolumeSnapshotClass to which this snapshot belongs.
+          jsonPath: .spec.volumeSnapshotClassName
+          name: VolumeSnapshotClass
+          type: string
+        - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent
+            object is bound.
+          jsonPath: .spec.volumeSnapshotRef.name
+          name: VolumeSnapshot
+          type: string
+        - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent
+            object is bound.
+          jsonPath: .spec.volumeSnapshotRef.namespace
+          name: VolumeSnapshotNamespace
+          type: string
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1beta1
+      # This indicates the v1beta1 version of the custom resource is deprecated.
+      # API requests to this version receive a warning in the server response.
+      deprecated: true
+      # This overrides the default warning returned to clients making v1beta1 API requests.
+      deprecationWarning: snapshot.storage.k8s.io/v1beta1 VolumeSnapshotContent is
+        deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotContent
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshotContent represents the actual "on-disk" snapshot
+            object in the underlying storage system
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+                of an object. Servers should convert recognized schemas to the latest
+                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource
+                this object represents. Servers may infer this from the endpoint the
+                client submits requests to. Cannot be updated. In CamelCase. More
+                info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            spec:
+              description: spec defines properties of a VolumeSnapshotContent created
+                by the underlying storage system. Required.
+              properties:
+                deletionPolicy:
+                  description: deletionPolicy determines whether this VolumeSnapshotContent
+                    and its physical snapshot on the underlying storage system should
+                    be deleted when its bound VolumeSnapshot is deleted. Supported
+                    values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent
+                    and its physical snapshot on underlying storage system are kept.
+                    "Delete" means that the VolumeSnapshotContent and its physical
+                    snapshot on underlying storage system are deleted. For dynamically
+                    provisioned snapshots, this field will automatically be filled
+                    in by the CSI snapshotter sidecar with the "DeletionPolicy" field
+                    defined in the corresponding VolumeSnapshotClass. For pre-existing
+                    snapshots, users MUST specify this field when creating the  VolumeSnapshotContent
+                    object. Required.
+                  enum: [Delete, Retain]
+                  type: string
+                driver:
+                  description: driver is the name of the CSI driver used to create
+                    the physical snapshot on the underlying storage system. This MUST
+                    be the same as the name returned by the CSI GetPluginName() call
+                    for that driver. Required.
+                  type: string
+                source:
+                  description: source specifies whether the snapshot is (or should
+                    be) dynamically provisioned or already exists, and just requires
+                    a Kubernetes object representation. This field is immutable after
+                    creation. Required.
+                  properties:
+                    snapshotHandle:
+                      description: snapshotHandle specifies the CSI "snapshot_id"
+                        of a pre-existing snapshot on the underlying storage system
+                        for which a Kubernetes object representation was (or should
+                        be) created. This field is immutable.
+                      type: string
+                    volumeHandle:
+                      description: volumeHandle specifies the CSI "volume_id" of the
+                        volume from which a snapshot should be dynamically taken from.
+                        This field is immutable.
+                      type: string
+                  type: object
+                volumeSnapshotClassName:
+                  description: name of the VolumeSnapshotClass from which this snapshot
+                    was (or will be) created. Note that after provisioning, the VolumeSnapshotClass
+                    may be deleted or recreated with different set of values, and
+                    as such, should not be referenced post-snapshot creation.
+                  type: string
+                volumeSnapshotRef:
+                  description: volumeSnapshotRef specifies the VolumeSnapshot object
+                    to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName
+                    field must reference to this VolumeSnapshotContent's name for
+                    the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent
+                    object, name and namespace of the VolumeSnapshot object MUST be
+                    provided for binding to happen. This field is immutable after
+                    creation. Required.
+                  properties:
+                    apiVersion:
+                      description: API version of the referent.
+                      type: string
+                    fieldPath:
+                      description: 'If referring to a piece of an object instead of
+                        an entire object, this string should contain a valid JSON/Go
+                        field access statement, such as desiredState.manifest.containers[2].
+                        For example, if the object reference is to a container within
+                        a pod, this would take on a value like: "spec.containers{name}"
+                        (where "name" refers to the name of the container that triggered
+                        the event) or if no container name is specified "spec.containers[2]"
+                        (container with index 2 in this pod). This syntax is chosen
+                        only to have some well-defined way of referencing a part of
+                        an object. TODO: this design is not final and this field is
+                        subject to change in the future.'
+                      type: string
+                    kind:
+                      description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                      type: string
+                    name:
+                      description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                      type: string
+                    namespace:
+                      description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
+                      type: string
+                    resourceVersion:
+                      description: 'Specific resourceVersion to which this reference
+                        is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
+                      type: string
+                    uid:
+                      description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                      type: string
+                  type: object
+              required: [deletionPolicy, driver, source, volumeSnapshotRef]
+              type: object
+            status:
+              description: status represents the current information of a snapshot.
+              properties:
+                creationTime:
+                  description: creationTime is the timestamp when the point-in-time
+                    snapshot is taken by the underlying storage system. In dynamic
+                    snapshot creation case, this field will be filled in by the CSI
+                    snapshotter sidecar with the "creation_time" value returned from
+                    CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this
+                    field will be filled with the "creation_time" value returned from
+                    the CSI "ListSnapshots" gRPC call if the driver supports it. If
+                    not specified, it indicates the creation time is unknown. The
+                    format of this field is a Unix nanoseconds time encoded as an
+                    int64. On Unix, the command `date +%s%N` returns the current time
+                    in nanoseconds since 1970-01-01 00:00:00 UTC.
+                  format: int64
+                  type: integer
+                error:
+                  description: error is the last observed error during snapshot creation,
+                    if any. Upon success after retry, this error field will be cleared.
+                  properties:
+                    message:
+                      description: 'message is a string detailing the encountered
+                        error during snapshot creation if specified. NOTE: message
+                        may be logged, and it should not contain sensitive information.'
+                      type: string
+                    time:
+                      description: time is the timestamp when the error was encountered.
+                      format: date-time
+                      type: string
+                  type: object
+                readyToUse:
+                  description: readyToUse indicates if a snapshot is ready to be used
+                    to restore a volume. In dynamic snapshot creation case, this field
+                    will be filled in by the CSI snapshotter sidecar with the "ready_to_use"
+                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "ready_to_use" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver
+                    supports it, otherwise, this field will be set to "True". If not
+                    specified, it means the readiness of a snapshot is unknown.
+                  type: boolean
+                restoreSize:
+                  description: restoreSize represents the complete size of the snapshot
+                    in bytes. In dynamic snapshot creation case, this field will be
+                    filled in by the CSI snapshotter sidecar with the "size_bytes"
+                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "size_bytes" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver
+                    supports it. When restoring a volume from this snapshot, the size
+                    of the volume MUST NOT be smaller than the restoreSize if it is
+                    specified, otherwise the restoration will fail. If not specified,
+                    it indicates that the size is unknown.
+                  format: int64
+                  minimum: 0
+                  type: integer
+                snapshotHandle:
+                  description: snapshotHandle is the CSI "snapshot_id" of a snapshot
+                    on the underlying storage system. If not specified, it indicates
+                    that dynamic snapshot creation has either failed or it is still
+                    in progress.
+                  type: string
+              type: object
+          required: [spec]
+          type: object
+      served: false
+      storage: false
+      subresources:
+        status: {}
+status:
+  acceptedNames:
+    kind: ''
+    plural: ''
+  conditions: []
+  storedVersions: []
+{{- end -}}

charts/latest/csi-driver-nfs/templates/snapshot.storage.k8s.io_volumesnapshots.yaml

@@ -0,0 +1,387 @@
+{{- if .Values.externalSnapshotter.enabled -}}
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.8.0
+    api-approved.kubernetes.io: https://github.com/kubernetes-csi/external-snapshotter/pull/665
+  name: volumesnapshots.snapshot.storage.k8s.io
+spec:
+  group: snapshot.storage.k8s.io
+  names:
+    kind: VolumeSnapshot
+    listKind: VolumeSnapshotList
+    plural: volumesnapshots
+    shortNames: [vs]
+    singular: volumesnapshot
+  scope: Namespaced
+  versions:
+    - additionalPrinterColumns:
+        - description: Indicates if the snapshot is ready to be used to restore a
+            volume.
+          jsonPath: .status.readyToUse
+          name: ReadyToUse
+          type: boolean
+        - description: If a new snapshot needs to be created, this contains the name
+            of the source PVC from which this snapshot was (or will be) created.
+          jsonPath: .spec.source.persistentVolumeClaimName
+          name: SourcePVC
+          type: string
+        - description: If a snapshot already exists, this contains the name of the
+            existing VolumeSnapshotContent object representing the existing snapshot.
+          jsonPath: .spec.source.volumeSnapshotContentName
+          name: SourceSnapshotContent
+          type: string
+        - description: Represents the minimum size of volume required to rehydrate
+            from this snapshot.
+          jsonPath: .status.restoreSize
+          name: RestoreSize
+          type: string
+        - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot.
+          jsonPath: .spec.volumeSnapshotClassName
+          name: SnapshotClass
+          type: string
+        - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot
+            object intends to bind to. Please note that verification of binding actually
+            requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure
+            both are pointing at each other. Binding MUST be verified prior to usage
+            of this object.
+          jsonPath: .status.boundVolumeSnapshotContentName
+          name: SnapshotContent
+          type: string
+        - description: Timestamp when the point-in-time snapshot was taken by the
+            underlying storage system.
+          jsonPath: .status.creationTime
+          name: CreationTime
+          type: date
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshot is a user's request for either creating a point-in-time
+            snapshot of a persistent volume, or binding to a pre-existing snapshot.
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+                of an object. Servers should convert recognized schemas to the latest
+                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource
+                this object represents. Servers may infer this from the endpoint the
+                client submits requests to. Cannot be updated. In CamelCase. More
+                info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            spec:
+              description: 'spec defines the desired characteristics of a snapshot
+                requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots
+                Required.'
+              properties:
+                source:
+                  description: source specifies where a snapshot will be created from.
+                    This field is immutable after creation. Required.
+                  properties:
+                    persistentVolumeClaimName:
+                      description: persistentVolumeClaimName specifies the name of
+                        the PersistentVolumeClaim object representing the volume from
+                        which a snapshot should be created. This PVC is assumed to
+                        be in the same namespace as the VolumeSnapshot object. This
+                        field should be set if the snapshot does not exists, and needs
+                        to be created. This field is immutable.
+                      type: string
+                    volumeSnapshotContentName:
+                      description: volumeSnapshotContentName specifies the name of
+                        a pre-existing VolumeSnapshotContent object representing an
+                        existing volume snapshot. This field should be set if the
+                        snapshot already exists and only needs a representation in
+                        Kubernetes. This field is immutable.
+                      type: string
+                  type: object
+                  oneOf:
+                    - required: [persistentVolumeClaimName]
+                    - required: [volumeSnapshotContentName]
+                volumeSnapshotClassName:
+                  description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass
+                    requested by the VolumeSnapshot. VolumeSnapshotClassName may be
+                    left nil to indicate that the default SnapshotClass should be
+                    used. A given cluster may have multiple default Volume SnapshotClasses:
+                    one default per CSI Driver. If a VolumeSnapshot does not specify
+                    a SnapshotClass, VolumeSnapshotSource will be checked to figure
+                    out what the associated CSI Driver is, and the default VolumeSnapshotClass
+                    associated with that CSI Driver will be used. If more than one
+                    VolumeSnapshotClass exist for a given CSI Driver and more than
+                    one have been marked as default, CreateSnapshot will fail and
+                    generate an event. Empty string is not allowed for this field.'
+                  type: string
+              required: [source]
+              type: object
+            status:
+              description: status represents the current information of a snapshot.
+                Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent
+                objects is successful (by validating that both VolumeSnapshot and
+                VolumeSnapshotContent point at each other) before using this object.
+              properties:
+                boundVolumeSnapshotContentName:
+                  description: 'boundVolumeSnapshotContentName is the name of the
+                    VolumeSnapshotContent object to which this VolumeSnapshot object
+                    intends to bind to. If not specified, it indicates that the VolumeSnapshot
+                    object has not been successfully bound to a VolumeSnapshotContent
+                    object yet. NOTE: To avoid possible security issues, consumers
+                    must verify binding between VolumeSnapshot and VolumeSnapshotContent
+                    objects is successful (by validating that both VolumeSnapshot
+                    and VolumeSnapshotContent point at each other) before using this
+                    object.'
+                  type: string
+                creationTime:
+                  description: creationTime is the timestamp when the point-in-time
+                    snapshot is taken by the underlying storage system. In dynamic
+                    snapshot creation case, this field will be filled in by the snapshot
+                    controller with the "creation_time" value returned from CSI "CreateSnapshot"
+                    gRPC call. For a pre-existing snapshot, this field will be filled
+                    with the "creation_time" value returned from the CSI "ListSnapshots"
+                    gRPC call if the driver supports it. If not specified, it may
+                    indicate that the creation time of the snapshot is unknown.
+                  format: date-time
+                  type: string
+                error:
+                  description: error is the last observed error during snapshot creation,
+                    if any. This field could be helpful to upper level controllers(i.e.,
+                    application controller) to decide whether they should continue
+                    on waiting for the snapshot to be created based on the type of
+                    error reported. The snapshot controller will keep retrying when
+                    an error occurs during the snapshot creation. Upon success, this
+                    error field will be cleared.
+                  properties:
+                    message:
+                      description: 'message is a string detailing the encountered
+                        error during snapshot creation if specified. NOTE: message
+                        may be logged, and it should not contain sensitive information.'
+                      type: string
+                    time:
+                      description: time is the timestamp when the error was encountered.
+                      format: date-time
+                      type: string
+                  type: object
+                readyToUse:
+                  description: readyToUse indicates if the snapshot is ready to be
+                    used to restore a volume. In dynamic snapshot creation case, this
+                    field will be filled in by the snapshot controller with the "ready_to_use"
+                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "ready_to_use" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver
+                    supports it, otherwise, this field will be set to "True". If not
+                    specified, it means the readiness of a snapshot is unknown.
+                  type: boolean
+                restoreSize:
+                  type: string
+                  description: restoreSize represents the minimum size of volume required
+                    to create a volume from this snapshot. In dynamic snapshot creation
+                    case, this field will be filled in by the snapshot controller
+                    with the "size_bytes" value returned from CSI "CreateSnapshot"
+                    gRPC call. For a pre-existing snapshot, this field will be filled
+                    with the "size_bytes" value returned from the CSI "ListSnapshots"
+                    gRPC call if the driver supports it. When restoring a volume from
+                    this snapshot, the size of the volume MUST NOT be smaller than
+                    the restoreSize if it is specified, otherwise the restoration
+                    will fail. If not specified, it indicates that the size is unknown.
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
+              type: object
+          required: [spec]
+          type: object
+      served: true
+      storage: true
+      subresources:
+        status: {}
+    - additionalPrinterColumns:
+        - description: Indicates if the snapshot is ready to be used to restore a
+            volume.
+          jsonPath: .status.readyToUse
+          name: ReadyToUse
+          type: boolean
+        - description: If a new snapshot needs to be created, this contains the name
+            of the source PVC from which this snapshot was (or will be) created.
+          jsonPath: .spec.source.persistentVolumeClaimName
+          name: SourcePVC
+          type: string
+        - description: If a snapshot already exists, this contains the name of the
+            existing VolumeSnapshotContent object representing the existing snapshot.
+          jsonPath: .spec.source.volumeSnapshotContentName
+          name: SourceSnapshotContent
+          type: string
+        - description: Represents the minimum size of volume required to rehydrate
+            from this snapshot.
+          jsonPath: .status.restoreSize
+          name: RestoreSize
+          type: string
+        - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot.
+          jsonPath: .spec.volumeSnapshotClassName
+          name: SnapshotClass
+          type: string
+        - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot
+            object intends to bind to. Please note that verification of binding actually
+            requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure
+            both are pointing at each other. Binding MUST be verified prior to usage
+            of this object.
+          jsonPath: .status.boundVolumeSnapshotContentName
+          name: SnapshotContent
+          type: string
+        - description: Timestamp when the point-in-time snapshot was taken by the
+            underlying storage system.
+          jsonPath: .status.creationTime
+          name: CreationTime
+          type: date
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1beta1
+      # This indicates the v1beta1 version of the custom resource is deprecated.
+      # API requests to this version receive a warning in the server response.
+      deprecated: true
+      # This overrides the default warning returned to clients making v1beta1 API requests.
+      deprecationWarning: snapshot.storage.k8s.io/v1beta1 VolumeSnapshot is deprecated;
+        use snapshot.storage.k8s.io/v1 VolumeSnapshot
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshot is a user's request for either creating a point-in-time
+            snapshot of a persistent volume, or binding to a pre-existing snapshot.
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+                of an object. Servers should convert recognized schemas to the latest
+                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource
+                this object represents. Servers may infer this from the endpoint the
+                client submits requests to. Cannot be updated. In CamelCase. More
+                info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            spec:
+              description: 'spec defines the desired characteristics of a snapshot
+                requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots
+                Required.'
+              properties:
+                source:
+                  description: source specifies where a snapshot will be created from.
+                    This field is immutable after creation. Required.
+                  properties:
+                    persistentVolumeClaimName:
+                      description: persistentVolumeClaimName specifies the name of
+                        the PersistentVolumeClaim object representing the volume from
+                        which a snapshot should be created. This PVC is assumed to
+                        be in the same namespace as the VolumeSnapshot object. This
+                        field should be set if the snapshot does not exists, and needs
+                        to be created. This field is immutable.
+                      type: string
+                    volumeSnapshotContentName:
+                      description: volumeSnapshotContentName specifies the name of
+                        a pre-existing VolumeSnapshotContent object representing an
+                        existing volume snapshot. This field should be set if the
+                        snapshot already exists and only needs a representation in
+                        Kubernetes. This field is immutable.
+                      type: string
+                  type: object
+                volumeSnapshotClassName:
+                  description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass
+                    requested by the VolumeSnapshot. VolumeSnapshotClassName may be
+                    left nil to indicate that the default SnapshotClass should be
+                    used. A given cluster may have multiple default Volume SnapshotClasses:
+                    one default per CSI Driver. If a VolumeSnapshot does not specify
+                    a SnapshotClass, VolumeSnapshotSource will be checked to figure
+                    out what the associated CSI Driver is, and the default VolumeSnapshotClass
+                    associated with that CSI Driver will be used. If more than one
+                    VolumeSnapshotClass exist for a given CSI Driver and more than
+                    one have been marked as default, CreateSnapshot will fail and
+                    generate an event. Empty string is not allowed for this field.'
+                  type: string
+              required: [source]
+              type: object
+            status:
+              description: status represents the current information of a snapshot.
+                Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent
+                objects is successful (by validating that both VolumeSnapshot and
+                VolumeSnapshotContent point at each other) before using this object.
+              properties:
+                boundVolumeSnapshotContentName:
+                  description: 'boundVolumeSnapshotContentName is the name of the
+                    VolumeSnapshotContent object to which this VolumeSnapshot object
+                    intends to bind to. If not specified, it indicates that the VolumeSnapshot
+                    object has not been successfully bound to a VolumeSnapshotContent
+                    object yet. NOTE: To avoid possible security issues, consumers
+                    must verify binding between VolumeSnapshot and VolumeSnapshotContent
+                    objects is successful (by validating that both VolumeSnapshot
+                    and VolumeSnapshotContent point at each other) before using this
+                    object.'
+                  type: string
+                creationTime:
+                  description: creationTime is the timestamp when the point-in-time
+                    snapshot is taken by the underlying storage system. In dynamic
+                    snapshot creation case, this field will be filled in by the snapshot
+                    controller with the "creation_time" value returned from CSI "CreateSnapshot"
+                    gRPC call. For a pre-existing snapshot, this field will be filled
+                    with the "creation_time" value returned from the CSI "ListSnapshots"
+                    gRPC call if the driver supports it. If not specified, it may
+                    indicate that the creation time of the snapshot is unknown.
+                  format: date-time
+                  type: string
+                error:
+                  description: error is the last observed error during snapshot creation,
+                    if any. This field could be helpful to upper level controllers(i.e.,
+                    application controller) to decide whether they should continue
+                    on waiting for the snapshot to be created based on the type of
+                    error reported. The snapshot controller will keep retrying when
+                    an error occurs during the snapshot creation. Upon success, this
+                    error field will be cleared.
+                  properties:
+                    message:
+                      description: 'message is a string detailing the encountered
+                        error during snapshot creation if specified. NOTE: message
+                        may be logged, and it should not contain sensitive information.'
+                      type: string
+                    time:
+                      description: time is the timestamp when the error was encountered.
+                      format: date-time
+                      type: string
+                  type: object
+                readyToUse:
+                  description: readyToUse indicates if the snapshot is ready to be
+                    used to restore a volume. In dynamic snapshot creation case, this
+                    field will be filled in by the snapshot controller with the "ready_to_use"
+                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "ready_to_use" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver
+                    supports it, otherwise, this field will be set to "True". If not
+                    specified, it means the readiness of a snapshot is unknown.
+                  type: boolean
+                restoreSize:
+                  type: string
+                  description: restoreSize represents the minimum size of volume required
+                    to create a volume from this snapshot. In dynamic snapshot creation
+                    case, this field will be filled in by the snapshot controller
+                    with the "size_bytes" value returned from CSI "CreateSnapshot"
+                    gRPC call. For a pre-existing snapshot, this field will be filled
+                    with the "size_bytes" value returned from the CSI "ListSnapshots"
+                    gRPC call if the driver supports it. When restoring a volume from
+                    this snapshot, the size of the volume MUST NOT be smaller than
+                    the restoreSize if it is specified, otherwise the restoration
+                    will fail. If not specified, it indicates that the size is unknown.
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
+              type: object
+          required: [spec]
+          type: object
+      served: false
+      storage: false
+      subresources:
+        status: {}
+status:
+  acceptedNames:
+    kind: ''
+    plural: ''
+  conditions: []
+  storedVersions: []
+{{- end -}}

charts/latest/csi-driver-nfs/values.yaml

@@ -8,6 +8,10 @@ image:
         repository: registry.k8s.io/sig-storage/csi-provisioner
         tag: v3.3.0
         pullPolicy: IfNotPresent
+    csiSnapshotter:
+        repository: registry.k8s.io/sig-storage/csi-snapshotter
+        tag: v6.2.1
+        pullPolicy: IfNotPresent
     livenessProbe:
         repository: registry.k8s.io/sig-storage/livenessprobe
         tag: v2.8.0
@@ -16,6 +20,10 @@ image:
         repository: registry.k8s.io/sig-storage/csi-node-driver-registrar
         tag: v2.6.2
         pullPolicy: IfNotPresent
+    externalSnapshotter:
+        repository: registry.k8s.io/sig-storage/snapshot-controller
+        tag: v6.1.0
+        pullPolicy: IfNotPresent
 
 serviceAccount:
   create: true # When true, service accounts will be created for you. Set to false if you want to use your own.
@@ -110,6 +118,12 @@ node:
         cpu: 10m
         memory: 20Mi
 
+externalSnapshotter:
+  enabled: true
+  name: snapshot-controller
+  controller:
+    replicas: 1
+
 ## Reference to one or more secrets to be used when pulling images
 ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
 ##

deploy/csi-nfs-controller.yaml

@@ -51,6 +51,20 @@ spec:
             requests:
               cpu: 10m
               memory: 20Mi
+        - name: csi-snapshotter
+          image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1
+          args:
+            - "--v=5"
+            - "--csi-address=$(ADDRESS)"
+            - "--leader-election-namespace=kube-system"
+            - "--leader-election"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
         - name: liveness-probe
           image: registry.k8s.io/sig-storage/livenessprobe:v2.8.0
           args:

deploy/csi-snapshot-controller.yaml

@@ -0,0 +1,40 @@
+# This YAML file shows how to deploy the snapshot controller
+
+# The snapshot controller implements the control loop for CSI snapshot functionality.
+# It should be installed as part of the base Kubernetes distribution in an appropriate
+# namespace for components implementing base system functionality. For installing with
+# Vanilla Kubernetes, kube-system makes sense for the namespace.
+
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: snapshot-controller
+  namespace: kube-system
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: snapshot-controller
+  # the snapshot controller won't be marked as ready if the v1 CRDs are unavailable
+  # in #504 the snapshot-controller will exit after around 7.5 seconds if it
+  # can't find the v1 CRDs so this value should be greater than that
+  minReadySeconds: 15
+  strategy:
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: snapshot-controller
+    spec:
+      serviceAccountName: snapshot-controller
+      containers:
+        - name: snapshot-controller
+          image: registry.k8s.io/sig-storage/snapshot-controller:v6.1.0
+          args:
+            - "--v=5"
+            - "--leader-election=true"
+          imagePullPolicy: IfNotPresent

deploy/install-driver.sh

@@ -16,6 +16,13 @@
 
 set -euo pipefail
 
+# min version of this nfs plugin supporting CSI snapshots
+snap_ver='4.3.0'
+
+# external-snapshotter to install for snapshots to work
+snap_controller_repo='github.com/kubernetes-csi/external-snapshotter/deploy/kubernetes/snapshot-controller?ref=v6.2.1'
+snap_crd_repo='github.com/kubernetes-csi/external-snapshotter/client/config/crd?ref=v6.2.1'
+
 ver="master"
 if [[ "$#" -gt 0 ]]; then
   ver="$1"
@@ -33,6 +40,12 @@ if [ $ver != "master" ]; then
   repo="$repo/$ver"
 fi
 
+# check for min supported version for snapshots or master to install external-snapshotter
+if [[ "$ver" == master || "$(printf '%s\n' "$ver" "$snap_ver" | sort -V | head -n1)" = "$snap_ver" ]]; then
+  kubectl kustomize "$snap_crd_repo" | kubectl apply -f -
+  kubectl -n kube-system kustomize "$snap_controller_repo" | kubectl apply -f -
+fi
+
 echo "Installing NFS CSI driver, version: $ver ..."
 kubectl apply -f $repo/rbac-csi-nfs.yaml
 kubectl apply -f $repo/csi-nfs-driverinfo.yaml

deploy/rbac-csi-nfs.yaml

@@ -26,6 +26,15 @@ rules:
   - apiGroups: ["storage.k8s.io"]
     resources: ["storageclasses"]
     verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses", "volumesnapshots"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["get", "update", "patch"]
   - apiGroups: [""]
     resources: ["events"]
     verbs: ["get", "list", "watch", "create", "update", "patch"]

deploy/rbac-snapshot-controller.yaml

@@ -0,0 +1,82 @@
+# RBAC file for the snapshot controller.
+#
+# The snapshot controller implements the control loop for CSI snapshot functionality.
+# It should be installed as part of the base Kubernetes distribution in an appropriate
+# namespace for components implementing base system functionality. For installing with
+# Vanilla Kubernetes, kube-system makes sense for the namespace.
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: snapshot-controller
+  namespace: kube-system
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: snapshot-controller-runner
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots/status"]
+    verbs: ["update", "patch"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: snapshot-controller-role
+subjects:
+  - kind: ServiceAccount
+    name: snapshot-controller
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: snapshot-controller-runner
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: snapshot-controller-leaderelection
+  namespace: kube-system
+rules:
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: snapshot-controller-leaderelection
+  namespace: kube-system
+subjects:
+  - kind: ServiceAccount
+    name: snapshot-controller
+roleRef:
+  kind: Role
+  name: snapshot-controller-leaderelection
+  apiGroup: rbac.authorization.k8s.io

deploy/snapshot.storage.k8s.io_volumesnapshotclasses.yaml

@@ -0,0 +1,144 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.8.0
+    api-approved.kubernetes.io: https://github.com/kubernetes-csi/external-snapshotter/pull/665
+  name: volumesnapshotclasses.snapshot.storage.k8s.io
+spec:
+  group: snapshot.storage.k8s.io
+  names:
+    kind: VolumeSnapshotClass
+    listKind: VolumeSnapshotClassList
+    plural: volumesnapshotclasses
+    shortNames: [vsclass, vsclasses]
+    singular: volumesnapshotclass
+  scope: Cluster
+  versions:
+    - additionalPrinterColumns:
+        - jsonPath: .driver
+          name: Driver
+          type: string
+        - description: Determines whether a VolumeSnapshotContent created through
+            the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot
+            is deleted.
+          jsonPath: .deletionPolicy
+          name: DeletionPolicy
+          type: string
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshotClass specifies parameters that a underlying
+            storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass
+            is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses
+            are non-namespaced
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+                of an object. Servers should convert recognized schemas to the latest
+                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            deletionPolicy:
+              description: deletionPolicy determines whether a VolumeSnapshotContent
+                created through the VolumeSnapshotClass should be deleted when its
+                bound VolumeSnapshot is deleted. Supported values are "Retain" and
+                "Delete". "Retain" means that the VolumeSnapshotContent and its physical
+                snapshot on underlying storage system are kept. "Delete" means that
+                the VolumeSnapshotContent and its physical snapshot on underlying
+                storage system are deleted. Required.
+              enum: [Delete, Retain]
+              type: string
+            driver:
+              description: driver is the name of the storage driver that handles this
+                VolumeSnapshotClass. Required.
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource
+                this object represents. Servers may infer this from the endpoint the
+                client submits requests to. Cannot be updated. In CamelCase. More
+                info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            parameters:
+              additionalProperties:
+                type: string
+              description: parameters is a key-value map with storage driver specific
+                parameters for creating snapshots. These values are opaque to Kubernetes.
+              type: object
+          required: [deletionPolicy, driver]
+          type: object
+      served: true
+      storage: true
+      subresources: {}
+    - additionalPrinterColumns:
+        - jsonPath: .driver
+          name: Driver
+          type: string
+        - description: Determines whether a VolumeSnapshotContent created through
+            the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot
+            is deleted.
+          jsonPath: .deletionPolicy
+          name: DeletionPolicy
+          type: string
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1beta1
+      # This indicates the v1beta1 version of the custom resource is deprecated.
+      # API requests to this version receive a warning in the server response.
+      deprecated: true
+      # This overrides the default warning returned to clients making v1beta1 API requests.
+      deprecationWarning: snapshot.storage.k8s.io/v1beta1 VolumeSnapshotClass is deprecated;
+        use snapshot.storage.k8s.io/v1 VolumeSnapshotClass
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshotClass specifies parameters that a underlying
+            storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass
+            is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses
+            are non-namespaced
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+                of an object. Servers should convert recognized schemas to the latest
+                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            deletionPolicy:
+              description: deletionPolicy determines whether a VolumeSnapshotContent
+                created through the VolumeSnapshotClass should be deleted when its
+                bound VolumeSnapshot is deleted. Supported values are "Retain" and
+                "Delete". "Retain" means that the VolumeSnapshotContent and its physical
+                snapshot on underlying storage system are kept. "Delete" means that
+                the VolumeSnapshotContent and its physical snapshot on underlying
+                storage system are deleted. Required.
+              enum: [Delete, Retain]
+              type: string
+            driver:
+              description: driver is the name of the storage driver that handles this
+                VolumeSnapshotClass. Required.
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource
+                this object represents. Servers may infer this from the endpoint the
+                client submits requests to. Cannot be updated. In CamelCase. More
+                info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            parameters:
+              additionalProperties:
+                type: string
+              description: parameters is a key-value map with storage driver specific
+                parameters for creating snapshots. These values are opaque to Kubernetes.
+              type: object
+          required: [deletionPolicy, driver]
+          type: object
+      served: false
+      storage: false
+      subresources: {}
+status:
+  acceptedNames:
+    kind: ''
+    plural: ''
+  conditions: []
+  storedVersions: []

deploy/snapshot.storage.k8s.io_volumesnapshotcontents.yaml

@@ -0,0 +1,471 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.8.0
+    api-approved.kubernetes.io: https://github.com/kubernetes-csi/external-snapshotter/pull/665
+  name: volumesnapshotcontents.snapshot.storage.k8s.io
+spec:
+  group: snapshot.storage.k8s.io
+  names:
+    kind: VolumeSnapshotContent
+    listKind: VolumeSnapshotContentList
+    plural: volumesnapshotcontents
+    shortNames: [vsc, vscs]
+    singular: volumesnapshotcontent
+  scope: Cluster
+  versions:
+    - additionalPrinterColumns:
+        - description: Indicates if the snapshot is ready to be used to restore a
+            volume.
+          jsonPath: .status.readyToUse
+          name: ReadyToUse
+          type: boolean
+        - description: Represents the complete size of the snapshot in bytes
+          jsonPath: .status.restoreSize
+          name: RestoreSize
+          type: integer
+        - description: Determines whether this VolumeSnapshotContent and its physical
+            snapshot on the underlying storage system should be deleted when its bound
+            VolumeSnapshot is deleted.
+          jsonPath: .spec.deletionPolicy
+          name: DeletionPolicy
+          type: string
+        - description: Name of the CSI driver used to create the physical snapshot
+            on the underlying storage system.
+          jsonPath: .spec.driver
+          name: Driver
+          type: string
+        - description: Name of the VolumeSnapshotClass to which this snapshot belongs.
+          jsonPath: .spec.volumeSnapshotClassName
+          name: VolumeSnapshotClass
+          type: string
+        - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent
+            object is bound.
+          jsonPath: .spec.volumeSnapshotRef.name
+          name: VolumeSnapshot
+          type: string
+        - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent
+            object is bound.
+          jsonPath: .spec.volumeSnapshotRef.namespace
+          name: VolumeSnapshotNamespace
+          type: string
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshotContent represents the actual "on-disk" snapshot
+            object in the underlying storage system
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+                of an object. Servers should convert recognized schemas to the latest
+                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource
+                this object represents. Servers may infer this from the endpoint the
+                client submits requests to. Cannot be updated. In CamelCase. More
+                info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            spec:
+              description: spec defines properties of a VolumeSnapshotContent created
+                by the underlying storage system. Required.
+              properties:
+                deletionPolicy:
+                  description: deletionPolicy determines whether this VolumeSnapshotContent
+                    and its physical snapshot on the underlying storage system should
+                    be deleted when its bound VolumeSnapshot is deleted. Supported
+                    values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent
+                    and its physical snapshot on underlying storage system are kept.
+                    "Delete" means that the VolumeSnapshotContent and its physical
+                    snapshot on underlying storage system are deleted. For dynamically
+                    provisioned snapshots, this field will automatically be filled
+                    in by the CSI snapshotter sidecar with the "DeletionPolicy" field
+                    defined in the corresponding VolumeSnapshotClass. For pre-existing
+                    snapshots, users MUST specify this field when creating the VolumeSnapshotContent
+                    object. Required.
+                  enum: [Delete, Retain]
+                  type: string
+                driver:
+                  description: driver is the name of the CSI driver used to create
+                    the physical snapshot on the underlying storage system. This MUST
+                    be the same as the name returned by the CSI GetPluginName() call
+                    for that driver. Required.
+                  type: string
+                source:
+                  description: source specifies whether the snapshot is (or should
+                    be) dynamically provisioned or already exists, and just requires
+                    a Kubernetes object representation. This field is immutable after
+                    creation. Required.
+                  properties:
+                    snapshotHandle:
+                      description: snapshotHandle specifies the CSI "snapshot_id"
+                        of a pre-existing snapshot on the underlying storage system
+                        for which a Kubernetes object representation was (or should
+                        be) created. This field is immutable.
+                      type: string
+                    volumeHandle:
+                      description: volumeHandle specifies the CSI "volume_id" of the
+                        volume from which a snapshot should be dynamically taken from.
+                        This field is immutable.
+                      type: string
+                  type: object
+                  oneOf:
+                    - required: [snapshotHandle]
+                    - required: [volumeHandle]
+                sourceVolumeMode:
+                  description: SourceVolumeMode is the mode of the volume whose snapshot
+                    is taken. Can be either “Filesystem” or “Block”. If not specified,
+                    it indicates the source volume's mode is unknown. This field is
+                    immutable. This field is an alpha field.
+                  type: string
+                volumeSnapshotClassName:
+                  description: name of the VolumeSnapshotClass from which this snapshot
+                    was (or will be) created. Note that after provisioning, the VolumeSnapshotClass
+                    may be deleted or recreated with different set of values, and
+                    as such, should not be referenced post-snapshot creation.
+                  type: string
+                volumeSnapshotRef:
+                  description: volumeSnapshotRef specifies the VolumeSnapshot object
+                    to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName
+                    field must reference to this VolumeSnapshotContent's name for
+                    the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent
+                    object, name and namespace of the VolumeSnapshot object MUST be
+                    provided for binding to happen. This field is immutable after
+                    creation. Required.
+                  properties:
+                    apiVersion:
+                      description: API version of the referent.
+                      type: string
+                    fieldPath:
+                      description: 'If referring to a piece of an object instead of
+                        an entire object, this string should contain a valid JSON/Go
+                        field access statement, such as desiredState.manifest.containers[2].
+                        For example, if the object reference is to a container within
+                        a pod, this would take on a value like: "spec.containers{name}"
+                        (where "name" refers to the name of the container that triggered
+                        the event) or if no container name is specified "spec.containers[2]"
+                        (container with index 2 in this pod). This syntax is chosen
+                        only to have some well-defined way of referencing a part of
+                        an object. TODO: this design is not final and this field is
+                        subject to change in the future.'
+                      type: string
+                    kind:
+                      description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                      type: string
+                    name:
+                      description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                      type: string
+                    namespace:
+                      description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
+                      type: string
+                    resourceVersion:
+                      description: 'Specific resourceVersion to which this reference
+                        is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
+                      type: string
+                    uid:
+                      description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                      type: string
+                  type: object
+              required: [deletionPolicy, driver, source, volumeSnapshotRef]
+              type: object
+            status:
+              description: status represents the current information of a snapshot.
+              properties:
+                creationTime:
+                  description: creationTime is the timestamp when the point-in-time
+                    snapshot is taken by the underlying storage system. In dynamic
+                    snapshot creation case, this field will be filled in by the CSI
+                    snapshotter sidecar with the "creation_time" value returned from
+                    CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this
+                    field will be filled with the "creation_time" value returned from
+                    the CSI "ListSnapshots" gRPC call if the driver supports it. If
+                    not specified, it indicates the creation time is unknown. The
+                    format of this field is a Unix nanoseconds time encoded as an
+                    int64. On Unix, the command `date +%s%N` returns the current time
+                    in nanoseconds since 1970-01-01 00:00:00 UTC.
+                  format: int64
+                  type: integer
+                error:
+                  description: error is the last observed error during snapshot creation,
+                    if any. Upon success after retry, this error field will be cleared.
+                  properties:
+                    message:
+                      description: 'message is a string detailing the encountered
+                        error during snapshot creation if specified. NOTE: message
+                        may be logged, and it should not contain sensitive information.'
+                      type: string
+                    time:
+                      description: time is the timestamp when the error was encountered.
+                      format: date-time
+                      type: string
+                  type: object
+                readyToUse:
+                  description: readyToUse indicates if a snapshot is ready to be used
+                    to restore a volume. In dynamic snapshot creation case, this field
+                    will be filled in by the CSI snapshotter sidecar with the "ready_to_use"
+                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "ready_to_use" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver
+                    supports it, otherwise, this field will be set to "True". If not
+                    specified, it means the readiness of a snapshot is unknown.
+                  type: boolean
+                restoreSize:
+                  description: restoreSize represents the complete size of the snapshot
+                    in bytes. In dynamic snapshot creation case, this field will be
+                    filled in by the CSI snapshotter sidecar with the "size_bytes"
+                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "size_bytes" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver
+                    supports it. When restoring a volume from this snapshot, the size
+                    of the volume MUST NOT be smaller than the restoreSize if it is
+                    specified, otherwise the restoration will fail. If not specified,
+                    it indicates that the size is unknown.
+                  format: int64
+                  minimum: 0
+                  type: integer
+                snapshotHandle:
+                  description: snapshotHandle is the CSI "snapshot_id" of a snapshot
+                    on the underlying storage system. If not specified, it indicates
+                    that dynamic snapshot creation has either failed or it is still
+                    in progress.
+                  type: string
+              type: object
+          required: [spec]
+          type: object
+      served: true
+      storage: true
+      subresources:
+        status: {}
+    - additionalPrinterColumns:
+        - description: Indicates if the snapshot is ready to be used to restore a
+            volume.
+          jsonPath: .status.readyToUse
+          name: ReadyToUse
+          type: boolean
+        - description: Represents the complete size of the snapshot in bytes
+          jsonPath: .status.restoreSize
+          name: RestoreSize
+          type: integer
+        - description: Determines whether this VolumeSnapshotContent and its physical
+            snapshot on the underlying storage system should be deleted when its bound
+            VolumeSnapshot is deleted.
+          jsonPath: .spec.deletionPolicy
+          name: DeletionPolicy
+          type: string
+        - description: Name of the CSI driver used to create the physical snapshot
+            on the underlying storage system.
+          jsonPath: .spec.driver
+          name: Driver
+          type: string
+        - description: Name of the VolumeSnapshotClass to which this snapshot belongs.
+          jsonPath: .spec.volumeSnapshotClassName
+          name: VolumeSnapshotClass
+          type: string
+        - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent
+            object is bound.
+          jsonPath: .spec.volumeSnapshotRef.name
+          name: VolumeSnapshot
+          type: string
+        - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent
+            object is bound.
+          jsonPath: .spec.volumeSnapshotRef.namespace
+          name: VolumeSnapshotNamespace
+          type: string
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1beta1
+      # This indicates the v1beta1 version of the custom resource is deprecated.
+      # API requests to this version receive a warning in the server response.
+      deprecated: true
+      # This overrides the default warning returned to clients making v1beta1 API requests.
+      deprecationWarning: snapshot.storage.k8s.io/v1beta1 VolumeSnapshotContent is
+        deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotContent
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshotContent represents the actual "on-disk" snapshot
+            object in the underlying storage system
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+                of an object. Servers should convert recognized schemas to the latest
+                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource
+                this object represents. Servers may infer this from the endpoint the
+                client submits requests to. Cannot be updated. In CamelCase. More
+                info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            spec:
+              description: spec defines properties of a VolumeSnapshotContent created
+                by the underlying storage system. Required.
+              properties:
+                deletionPolicy:
+                  description: deletionPolicy determines whether this VolumeSnapshotContent
+                    and its physical snapshot on the underlying storage system should
+                    be deleted when its bound VolumeSnapshot is deleted. Supported
+                    values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent
+                    and its physical snapshot on underlying storage system are kept.
+                    "Delete" means that the VolumeSnapshotContent and its physical
+                    snapshot on underlying storage system are deleted. For dynamically
+                    provisioned snapshots, this field will automatically be filled
+                    in by the CSI snapshotter sidecar with the "DeletionPolicy" field
+                    defined in the corresponding VolumeSnapshotClass. For pre-existing
+                    snapshots, users MUST specify this field when creating the  VolumeSnapshotContent
+                    object. Required.
+                  enum: [Delete, Retain]
+                  type: string
+                driver:
+                  description: driver is the name of the CSI driver used to create
+                    the physical snapshot on the underlying storage system. This MUST
+                    be the same as the name returned by the CSI GetPluginName() call
+                    for that driver. Required.
+                  type: string
+                source:
+                  description: source specifies whether the snapshot is (or should
+                    be) dynamically provisioned or already exists, and just requires
+                    a Kubernetes object representation. This field is immutable after
+                    creation. Required.
+                  properties:
+                    snapshotHandle:
+                      description: snapshotHandle specifies the CSI "snapshot_id"
+                        of a pre-existing snapshot on the underlying storage system
+                        for which a Kubernetes object representation was (or should
+                        be) created. This field is immutable.
+                      type: string
+                    volumeHandle:
+                      description: volumeHandle specifies the CSI "volume_id" of the
+                        volume from which a snapshot should be dynamically taken from.
+                        This field is immutable.
+                      type: string
+                  type: object
+                volumeSnapshotClassName:
+                  description: name of the VolumeSnapshotClass from which this snapshot
+                    was (or will be) created. Note that after provisioning, the VolumeSnapshotClass
+                    may be deleted or recreated with different set of values, and
+                    as such, should not be referenced post-snapshot creation.
+                  type: string
+                volumeSnapshotRef:
+                  description: volumeSnapshotRef specifies the VolumeSnapshot object
+                    to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName
+                    field must reference to this VolumeSnapshotContent's name for
+                    the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent
+                    object, name and namespace of the VolumeSnapshot object MUST be
+                    provided for binding to happen. This field is immutable after
+                    creation. Required.
+                  properties:
+                    apiVersion:
+                      description: API version of the referent.
+                      type: string
+                    fieldPath:
+                      description: 'If referring to a piece of an object instead of
+                        an entire object, this string should contain a valid JSON/Go
+                        field access statement, such as desiredState.manifest.containers[2].
+                        For example, if the object reference is to a container within
+                        a pod, this would take on a value like: "spec.containers{name}"
+                        (where "name" refers to the name of the container that triggered
+                        the event) or if no container name is specified "spec.containers[2]"
+                        (container with index 2 in this pod). This syntax is chosen
+                        only to have some well-defined way of referencing a part of
+                        an object. TODO: this design is not final and this field is
+                        subject to change in the future.'
+                      type: string
+                    kind:
+                      description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                      type: string
+                    name:
+                      description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                      type: string
+                    namespace:
+                      description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
+                      type: string
+                    resourceVersion:
+                      description: 'Specific resourceVersion to which this reference
+                        is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
+                      type: string
+                    uid:
+                      description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                      type: string
+                  type: object
+              required: [deletionPolicy, driver, source, volumeSnapshotRef]
+              type: object
+            status:
+              description: status represents the current information of a snapshot.
+              properties:
+                creationTime:
+                  description: creationTime is the timestamp when the point-in-time
+                    snapshot is taken by the underlying storage system. In dynamic
+                    snapshot creation case, this field will be filled in by the CSI
+                    snapshotter sidecar with the "creation_time" value returned from
+                    CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this
+                    field will be filled with the "creation_time" value returned from
+                    the CSI "ListSnapshots" gRPC call if the driver supports it. If
+                    not specified, it indicates the creation time is unknown. The
+                    format of this field is a Unix nanoseconds time encoded as an
+                    int64. On Unix, the command `date +%s%N` returns the current time
+                    in nanoseconds since 1970-01-01 00:00:00 UTC.
+                  format: int64
+                  type: integer
+                error:
+                  description: error is the last observed error during snapshot creation,
+                    if any. Upon success after retry, this error field will be cleared.
+                  properties:
+                    message:
+                      description: 'message is a string detailing the encountered
+                        error during snapshot creation if specified. NOTE: message
+                        may be logged, and it should not contain sensitive information.'
+                      type: string
+                    time:
+                      description: time is the timestamp when the error was encountered.
+                      format: date-time
+                      type: string
+                  type: object
+                readyToUse:
+                  description: readyToUse indicates if a snapshot is ready to be used
+                    to restore a volume. In dynamic snapshot creation case, this field
+                    will be filled in by the CSI snapshotter sidecar with the "ready_to_use"
+                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "ready_to_use" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver
+                    supports it, otherwise, this field will be set to "True". If not
+                    specified, it means the readiness of a snapshot is unknown.
+                  type: boolean
+                restoreSize:
+                  description: restoreSize represents the complete size of the snapshot
+                    in bytes. In dynamic snapshot creation case, this field will be
+                    filled in by the CSI snapshotter sidecar with the "size_bytes"
+                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "size_bytes" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver
+                    supports it. When restoring a volume from this snapshot, the size
+                    of the volume MUST NOT be smaller than the restoreSize if it is
+                    specified, otherwise the restoration will fail. If not specified,
+                    it indicates that the size is unknown.
+                  format: int64
+                  minimum: 0
+                  type: integer
+                snapshotHandle:
+                  description: snapshotHandle is the CSI "snapshot_id" of a snapshot
+                    on the underlying storage system. If not specified, it indicates
+                    that dynamic snapshot creation has either failed or it is still
+                    in progress.
+                  type: string
+              type: object
+          required: [spec]
+          type: object
+      served: false
+      storage: false
+      subresources:
+        status: {}
+status:
+  acceptedNames:
+    kind: ''
+    plural: ''
+  conditions: []
+  storedVersions: []

deploy/snapshot.storage.k8s.io_volumesnapshots.yaml

@@ -0,0 +1,385 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.8.0
+    api-approved.kubernetes.io: https://github.com/kubernetes-csi/external-snapshotter/pull/665
+  name: volumesnapshots.snapshot.storage.k8s.io
+spec:
+  group: snapshot.storage.k8s.io
+  names:
+    kind: VolumeSnapshot
+    listKind: VolumeSnapshotList
+    plural: volumesnapshots
+    shortNames: [vs]
+    singular: volumesnapshot
+  scope: Namespaced
+  versions:
+    - additionalPrinterColumns:
+        - description: Indicates if the snapshot is ready to be used to restore a
+            volume.
+          jsonPath: .status.readyToUse
+          name: ReadyToUse
+          type: boolean
+        - description: If a new snapshot needs to be created, this contains the name
+            of the source PVC from which this snapshot was (or will be) created.
+          jsonPath: .spec.source.persistentVolumeClaimName
+          name: SourcePVC
+          type: string
+        - description: If a snapshot already exists, this contains the name of the
+            existing VolumeSnapshotContent object representing the existing snapshot.
+          jsonPath: .spec.source.volumeSnapshotContentName
+          name: SourceSnapshotContent
+          type: string
+        - description: Represents the minimum size of volume required to rehydrate
+            from this snapshot.
+          jsonPath: .status.restoreSize
+          name: RestoreSize
+          type: string
+        - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot.
+          jsonPath: .spec.volumeSnapshotClassName
+          name: SnapshotClass
+          type: string
+        - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot
+            object intends to bind to. Please note that verification of binding actually
+            requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure
+            both are pointing at each other. Binding MUST be verified prior to usage
+            of this object.
+          jsonPath: .status.boundVolumeSnapshotContentName
+          name: SnapshotContent
+          type: string
+        - description: Timestamp when the point-in-time snapshot was taken by the
+            underlying storage system.
+          jsonPath: .status.creationTime
+          name: CreationTime
+          type: date
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshot is a user's request for either creating a point-in-time
+            snapshot of a persistent volume, or binding to a pre-existing snapshot.
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+                of an object. Servers should convert recognized schemas to the latest
+                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource
+                this object represents. Servers may infer this from the endpoint the
+                client submits requests to. Cannot be updated. In CamelCase. More
+                info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            spec:
+              description: 'spec defines the desired characteristics of a snapshot
+                requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots
+                Required.'
+              properties:
+                source:
+                  description: source specifies where a snapshot will be created from.
+                    This field is immutable after creation. Required.
+                  properties:
+                    persistentVolumeClaimName:
+                      description: persistentVolumeClaimName specifies the name of
+                        the PersistentVolumeClaim object representing the volume from
+                        which a snapshot should be created. This PVC is assumed to
+                        be in the same namespace as the VolumeSnapshot object. This
+                        field should be set if the snapshot does not exists, and needs
+                        to be created. This field is immutable.
+                      type: string
+                    volumeSnapshotContentName:
+                      description: volumeSnapshotContentName specifies the name of
+                        a pre-existing VolumeSnapshotContent object representing an
+                        existing volume snapshot. This field should be set if the
+                        snapshot already exists and only needs a representation in
+                        Kubernetes. This field is immutable.
+                      type: string
+                  type: object
+                  oneOf:
+                    - required: [persistentVolumeClaimName]
+                    - required: [volumeSnapshotContentName]
+                volumeSnapshotClassName:
+                  description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass
+                    requested by the VolumeSnapshot. VolumeSnapshotClassName may be
+                    left nil to indicate that the default SnapshotClass should be
+                    used. A given cluster may have multiple default Volume SnapshotClasses:
+                    one default per CSI Driver. If a VolumeSnapshot does not specify
+                    a SnapshotClass, VolumeSnapshotSource will be checked to figure
+                    out what the associated CSI Driver is, and the default VolumeSnapshotClass
+                    associated with that CSI Driver will be used. If more than one
+                    VolumeSnapshotClass exist for a given CSI Driver and more than
+                    one have been marked as default, CreateSnapshot will fail and
+                    generate an event. Empty string is not allowed for this field.'
+                  type: string
+              required: [source]
+              type: object
+            status:
+              description: status represents the current information of a snapshot.
+                Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent
+                objects is successful (by validating that both VolumeSnapshot and
+                VolumeSnapshotContent point at each other) before using this object.
+              properties:
+                boundVolumeSnapshotContentName:
+                  description: 'boundVolumeSnapshotContentName is the name of the
+                    VolumeSnapshotContent object to which this VolumeSnapshot object
+                    intends to bind to. If not specified, it indicates that the VolumeSnapshot
+                    object has not been successfully bound to a VolumeSnapshotContent
+                    object yet. NOTE: To avoid possible security issues, consumers
+                    must verify binding between VolumeSnapshot and VolumeSnapshotContent
+                    objects is successful (by validating that both VolumeSnapshot
+                    and VolumeSnapshotContent point at each other) before using this
+                    object.'
+                  type: string
+                creationTime:
+                  description: creationTime is the timestamp when the point-in-time
+                    snapshot is taken by the underlying storage system. In dynamic
+                    snapshot creation case, this field will be filled in by the snapshot
+                    controller with the "creation_time" value returned from CSI "CreateSnapshot"
+                    gRPC call. For a pre-existing snapshot, this field will be filled
+                    with the "creation_time" value returned from the CSI "ListSnapshots"
+                    gRPC call if the driver supports it. If not specified, it may
+                    indicate that the creation time of the snapshot is unknown.
+                  format: date-time
+                  type: string
+                error:
+                  description: error is the last observed error during snapshot creation,
+                    if any. This field could be helpful to upper level controllers(i.e.,
+                    application controller) to decide whether they should continue
+                    on waiting for the snapshot to be created based on the type of
+                    error reported. The snapshot controller will keep retrying when
+                    an error occurs during the snapshot creation. Upon success, this
+                    error field will be cleared.
+                  properties:
+                    message:
+                      description: 'message is a string detailing the encountered
+                        error during snapshot creation if specified. NOTE: message
+                        may be logged, and it should not contain sensitive information.'
+                      type: string
+                    time:
+                      description: time is the timestamp when the error was encountered.
+                      format: date-time
+                      type: string
+                  type: object
+                readyToUse:
+                  description: readyToUse indicates if the snapshot is ready to be
+                    used to restore a volume. In dynamic snapshot creation case, this
+                    field will be filled in by the snapshot controller with the "ready_to_use"
+                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "ready_to_use" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver
+                    supports it, otherwise, this field will be set to "True". If not
+                    specified, it means the readiness of a snapshot is unknown.
+                  type: boolean
+                restoreSize:
+                  type: string
+                  description: restoreSize represents the minimum size of volume required
+                    to create a volume from this snapshot. In dynamic snapshot creation
+                    case, this field will be filled in by the snapshot controller
+                    with the "size_bytes" value returned from CSI "CreateSnapshot"
+                    gRPC call. For a pre-existing snapshot, this field will be filled
+                    with the "size_bytes" value returned from the CSI "ListSnapshots"
+                    gRPC call if the driver supports it. When restoring a volume from
+                    this snapshot, the size of the volume MUST NOT be smaller than
+                    the restoreSize if it is specified, otherwise the restoration
+                    will fail. If not specified, it indicates that the size is unknown.
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
+              type: object
+          required: [spec]
+          type: object
+      served: true
+      storage: true
+      subresources:
+        status: {}
+    - additionalPrinterColumns:
+        - description: Indicates if the snapshot is ready to be used to restore a
+            volume.
+          jsonPath: .status.readyToUse
+          name: ReadyToUse
+          type: boolean
+        - description: If a new snapshot needs to be created, this contains the name
+            of the source PVC from which this snapshot was (or will be) created.
+          jsonPath: .spec.source.persistentVolumeClaimName
+          name: SourcePVC
+          type: string
+        - description: If a snapshot already exists, this contains the name of the
+            existing VolumeSnapshotContent object representing the existing snapshot.
+          jsonPath: .spec.source.volumeSnapshotContentName
+          name: SourceSnapshotContent
+          type: string
+        - description: Represents the minimum size of volume required to rehydrate
+            from this snapshot.
+          jsonPath: .status.restoreSize
+          name: RestoreSize
+          type: string
+        - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot.
+          jsonPath: .spec.volumeSnapshotClassName
+          name: SnapshotClass
+          type: string
+        - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot
+            object intends to bind to. Please note that verification of binding actually
+            requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure
+            both are pointing at each other. Binding MUST be verified prior to usage
+            of this object.
+          jsonPath: .status.boundVolumeSnapshotContentName
+          name: SnapshotContent
+          type: string
+        - description: Timestamp when the point-in-time snapshot was taken by the
+            underlying storage system.
+          jsonPath: .status.creationTime
+          name: CreationTime
+          type: date
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1beta1
+      # This indicates the v1beta1 version of the custom resource is deprecated.
+      # API requests to this version receive a warning in the server response.
+      deprecated: true
+      # This overrides the default warning returned to clients making v1beta1 API requests.
+      deprecationWarning: snapshot.storage.k8s.io/v1beta1 VolumeSnapshot is deprecated;
+        use snapshot.storage.k8s.io/v1 VolumeSnapshot
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshot is a user's request for either creating a point-in-time
+            snapshot of a persistent volume, or binding to a pre-existing snapshot.
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+                of an object. Servers should convert recognized schemas to the latest
+                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource
+                this object represents. Servers may infer this from the endpoint the
+                client submits requests to. Cannot be updated. In CamelCase. More
+                info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            spec:
+              description: 'spec defines the desired characteristics of a snapshot
+                requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots
+                Required.'
+              properties:
+                source:
+                  description: source specifies where a snapshot will be created from.
+                    This field is immutable after creation. Required.
+                  properties:
+                    persistentVolumeClaimName:
+                      description: persistentVolumeClaimName specifies the name of
+                        the PersistentVolumeClaim object representing the volume from
+                        which a snapshot should be created. This PVC is assumed to
+                        be in the same namespace as the VolumeSnapshot object. This
+                        field should be set if the snapshot does not exists, and needs
+                        to be created. This field is immutable.
+                      type: string
+                    volumeSnapshotContentName:
+                      description: volumeSnapshotContentName specifies the name of
+                        a pre-existing VolumeSnapshotContent object representing an
+                        existing volume snapshot. This field should be set if the
+                        snapshot already exists and only needs a representation in
+                        Kubernetes. This field is immutable.
+                      type: string
+                  type: object
+                volumeSnapshotClassName:
+                  description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass
+                    requested by the VolumeSnapshot. VolumeSnapshotClassName may be
+                    left nil to indicate that the default SnapshotClass should be
+                    used. A given cluster may have multiple default Volume SnapshotClasses:
+                    one default per CSI Driver. If a VolumeSnapshot does not specify
+                    a SnapshotClass, VolumeSnapshotSource will be checked to figure
+                    out what the associated CSI Driver is, and the default VolumeSnapshotClass
+                    associated with that CSI Driver will be used. If more than one
+                    VolumeSnapshotClass exist for a given CSI Driver and more than
+                    one have been marked as default, CreateSnapshot will fail and
+                    generate an event. Empty string is not allowed for this field.'
+                  type: string
+              required: [source]
+              type: object
+            status:
+              description: status represents the current information of a snapshot.
+                Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent
+                objects is successful (by validating that both VolumeSnapshot and
+                VolumeSnapshotContent point at each other) before using this object.
+              properties:
+                boundVolumeSnapshotContentName:
+                  description: 'boundVolumeSnapshotContentName is the name of the
+                    VolumeSnapshotContent object to which this VolumeSnapshot object
+                    intends to bind to. If not specified, it indicates that the VolumeSnapshot
+                    object has not been successfully bound to a VolumeSnapshotContent
+                    object yet. NOTE: To avoid possible security issues, consumers
+                    must verify binding between VolumeSnapshot and VolumeSnapshotContent
+                    objects is successful (by validating that both VolumeSnapshot
+                    and VolumeSnapshotContent point at each other) before using this
+                    object.'
+                  type: string
+                creationTime:
+                  description: creationTime is the timestamp when the point-in-time
+                    snapshot is taken by the underlying storage system. In dynamic
+                    snapshot creation case, this field will be filled in by the snapshot
+                    controller with the "creation_time" value returned from CSI "CreateSnapshot"
+                    gRPC call. For a pre-existing snapshot, this field will be filled
+                    with the "creation_time" value returned from the CSI "ListSnapshots"
+                    gRPC call if the driver supports it. If not specified, it may
+                    indicate that the creation time of the snapshot is unknown.
+                  format: date-time
+                  type: string
+                error:
+                  description: error is the last observed error during snapshot creation,
+                    if any. This field could be helpful to upper level controllers(i.e.,
+                    application controller) to decide whether they should continue
+                    on waiting for the snapshot to be created based on the type of
+                    error reported. The snapshot controller will keep retrying when
+                    an error occurs during the snapshot creation. Upon success, this
+                    error field will be cleared.
+                  properties:
+                    message:
+                      description: 'message is a string detailing the encountered
+                        error during snapshot creation if specified. NOTE: message
+                        may be logged, and it should not contain sensitive information.'
+                      type: string
+                    time:
+                      description: time is the timestamp when the error was encountered.
+                      format: date-time
+                      type: string
+                  type: object
+                readyToUse:
+                  description: readyToUse indicates if the snapshot is ready to be
+                    used to restore a volume. In dynamic snapshot creation case, this
+                    field will be filled in by the snapshot controller with the "ready_to_use"
+                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "ready_to_use" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver
+                    supports it, otherwise, this field will be set to "True". If not
+                    specified, it means the readiness of a snapshot is unknown.
+                  type: boolean
+                restoreSize:
+                  type: string
+                  description: restoreSize represents the minimum size of volume required
+                    to create a volume from this snapshot. In dynamic snapshot creation
+                    case, this field will be filled in by the snapshot controller
+                    with the "size_bytes" value returned from CSI "CreateSnapshot"
+                    gRPC call. For a pre-existing snapshot, this field will be filled
+                    with the "size_bytes" value returned from the CSI "ListSnapshots"
+                    gRPC call if the driver supports it. When restoring a volume from
+                    this snapshot, the size of the volume MUST NOT be smaller than
+                    the restoreSize if it is specified, otherwise the restoration
+                    will fail. If not specified, it indicates that the size is unknown.
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
+              type: object
+          required: [spec]
+          type: object
+      served: false
+      storage: false
+      subresources:
+        status: {}
+status:
+  acceptedNames:
+    kind: ''
+    plural: ''
+  conditions: []
+  storedVersions: []

deploy/uninstall-driver.sh

@@ -16,6 +16,10 @@
 
 set -euo pipefail
 
+# external-snapshotter to install for snapshots to work
+snap_controller_repo='github.com/kubernetes-csi/external-snapshotter/deploy/kubernetes/snapshot-controller?ref=v6.2.1'
+snap_crd_repo='github.com/kubernetes-csi/external-snapshotter/client/config/crd?ref=v6.2.1'
+
 ver="master"
 if [[ "$#" -gt 0 ]]; then
   ver="$1"
@@ -34,6 +38,8 @@ if [ $ver != "master" ]; then
 fi
 
 echo "Uninstalling NFS driver, version: $ver ..."
+# this keeps VolumeSnapshot CRDs untouched
+kubectl -n kube-system kustomize "$snap_controller_repo" | kubectl delete --ignore-not-found -f -
 kubectl delete -f $repo/csi-nfs-controller.yaml --ignore-not-found
 kubectl delete -f $repo/csi-nfs-node.yaml --ignore-not-found
 kubectl delete -f $repo/csi-nfs-driverinfo.yaml --ignore-not-found

hack/verify-helm-chart.sh

@@ -62,12 +62,16 @@ pip install yq --ignore-installed PyYAML
 
 # Extract images from csi-nfs-controller.yaml
 expected_csi_provisioner_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[0].image | head -n 1)"
-expected_liveness_probe_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[1].image | head -n 1)"
+expected_csi_snapshotter_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[1].image | head -n 1)"
-expected_nfs_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[2].image | head -n 1)"
+expected_liveness_probe_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[2].image | head -n 1)"
+expected_nfs_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[3].image | head -n 1)"
 
 csi_provisioner_image="$(get_image_from_helm_chart "csiProvisioner")"
 validate_image "${expected_csi_provisioner_image}" "${csi_provisioner_image}"
 
+csi_snapshotter_image="$(get_image_from_helm_chart "csiSnapshotter")"
+validate_image "${expected_csi_snapshotter_image}" "${csi_snapshotter_image}"
+
 liveness_probe_image="$(get_image_from_helm_chart "livenessProbe")"
 validate_image "${expected_liveness_probe_image}" "${liveness_probe_image}"
 

test/external-e2e/run.sh

@@ -32,8 +32,11 @@ setup_e2e_binaries() {
 
     # test on alternative driver name
     sed -i "s/nfs.csi.k8s.io/$DRIVER.csi.k8s.io/g" deploy/example/storageclass-nfs.yaml
+    sed -i "s/nfs.csi.k8s.io/$DRIVER.csi.k8s.io/g" deploy/example/snapshotclass-nfs.yaml
     # install csi driver
-    mkdir -p /tmp/csi && cp deploy/example/storageclass-nfs.yaml /tmp/csi/storageclass.yaml
+    mkdir -p /tmp/csi
+    cp deploy/example/storageclass-nfs.yaml /tmp/csi/storageclass.yaml
+    cp deploy/example/snapshotclass-nfs.yaml /tmp/csi/snapshotclass.yaml
     make e2e-bootstrap
     make install-nfs-server
 }

test/external-e2e/testdriver.yaml

@@ -3,6 +3,8 @@
 
 StorageClass:
   FromFile: /tmp/csi/storageclass.yaml
+SnapshotClass:
+  FromName: true
 DriverInfo:
   Name: test.csi.k8s.io
   SupportedFsType: {"nfs"}
@@ -13,6 +15,7 @@ DriverInfo:
     RWX: true
     fsGroup: true
     pvcDataSource: true
+    snapshotDataSource: true
 InlineVolumes:
 - Attributes:
     server: nfs-server.default.svc.cluster.local