feat: add NodeGetVolumeStats support

go.sum

@@ -1222,11 +1222,13 @@ k8s.io/apiserver v0.20.0/go.mod h1:6gRIWiOkvGvQt12WTYmsiYoUyYW0FXSiMdNl4m+sxY8=
 k8s.io/cli-runtime v0.20.0/go.mod h1:C5tewU1SC1t09D7pmkk83FT4lMAw+bvMDuRxA7f0t2s=
 k8s.io/client-go v0.20.0 h1:Xlax8PKbZsjX4gFvNtt4F5MoJ1V5prDvCuoq9B7iax0=
 k8s.io/client-go v0.20.0/go.mod h1:4KWh/g+Ocd8KkCwKF8vUNnmqgv+EVnQDK4MBF4oB5tY=
+k8s.io/cloud-provider v0.20.0 h1:CVPQ66iyfNgeGomUq2jE/TWrfzE77bdCpemhFS8955U=
 k8s.io/cloud-provider v0.20.0/go.mod h1:Lz/luSVD5BrHDDhtVdjFh0C2qQCRYdf0b9BHQ9L+bXc=
 k8s.io/cluster-bootstrap v0.20.0/go.mod h1:6WZaNIBvcvL7MkPzSRKrZDIr4u+ePW2oIWoRsEFMjmE=
 k8s.io/code-generator v0.20.0/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg=
 k8s.io/component-base v0.20.0 h1:BXGL8iitIQD+0NgW49UsM7MraNUUGDU3FBmrfUAtmVQ=
 k8s.io/component-base v0.20.0/go.mod h1:wKPj+RHnAr8LW2EIBIK7AxOHPde4gme2lzXwVSoRXeA=
+k8s.io/component-helpers v0.20.0-alpha.2.0.20201114090304-7cb42b694587 h1:XV8zcD26n/hrYX129A4vR35MvgGPKKSVAQP38dif/HI=
 k8s.io/component-helpers v0.20.0-alpha.2.0.20201114090304-7cb42b694587/go.mod h1:Fb+7xcdRsPnQNQnXkqMwptC6LsONZr10ris8YPJw1bk=
 k8s.io/controller-manager v0.20.0-alpha.1.0.20201209052538-b2c380a1dc86 h1:22Cxrg/eu1vIlWan5HoJJzZw8/sAZpjlwMrs87uOdEQ=
 k8s.io/controller-manager v0.20.0-alpha.1.0.20201209052538-b2c380a1dc86/go.mod h1:nPjStmyrmF6IE6bhIsSN+5A4nYp1ORzLENSAPp+tSzs=
@@ -1265,6 +1267,7 @@ k8s.io/kubernetes v1.21.0-alpha.0.0.20201210005053-f58c4d8cd725 h1:zDSrjDDKNWvXy
 k8s.io/kubernetes v1.21.0-alpha.0.0.20201210005053-f58c4d8cd725/go.mod h1:/xrHGNfoQphtkhZvyd5bA1lRmz+QkDVmBZu+O8QMoek=
 k8s.io/legacy-cloud-providers v0.20.0/go.mod h1:1jEkaU7h9+b1EYdfWDBvhFAr+QpRfUjQfK+dGhxPGfA=
 k8s.io/metrics v0.20.0/go.mod h1:9yiRhfr8K8sjdj2EthQQE9WvpYDvsXIV3CjN4Ruq4Jw=
+k8s.io/mount-utils v0.21.0-alpha.0 h1:IKn8GtClgNVIb801UbMpDI7F49D6KBXzSf5yTO5VZpo=
 k8s.io/mount-utils v0.21.0-alpha.0/go.mod h1:Jv9NRZ5L2LF87A17GaGlArD+r3JAJdZFvo4XD1cG4Kc=
 k8s.io/repo-infra v0.0.1-alpha.1/go.mod h1:wO1t9WaB99V80ljbeENTnayuEEwNZt7gECYh/CEyOJ8=
 k8s.io/sample-apiserver v0.20.0/go.mod h1:tScvbz/BcUG46IOsu2YLt4EjBP7XeUuMzMbQt2tQYWw=

pkg/nfs/nodeserver.go

@@ -26,6 +26,7 @@ import (
 	"golang.org/x/net/context"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
+	"k8s.io/kubernetes/pkg/volume"
 	"k8s.io/utils/mount"
 )
 
@@ -148,8 +149,69 @@ func (ns *NodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetC
 }
 
 // NodeGetVolumeStats get volume stats
-func (ns *NodeServer) NodeGetVolumeStats(ctx context.Context, in *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) {
+func (ns *NodeServer) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) {
-	return nil, status.Error(codes.Unimplemented, "")
+	if len(req.VolumeId) == 0 {
+		return nil, status.Error(codes.InvalidArgument, "NodeGetVolumeStats volume ID was empty")
+	}
+	if len(req.VolumePath) == 0 {
+		return nil, status.Error(codes.InvalidArgument, "NodeGetVolumeStats volume path was empty")
+	}
+
+	_, err := os.Stat(req.VolumePath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, status.Errorf(codes.NotFound, "path %s does not exist", req.VolumePath)
+		}
+		return nil, status.Errorf(codes.Internal, "failed to stat file %s: %v", req.VolumePath, err)
+	}
+
+	volumeMetrics, err := volume.NewMetricsStatFS(req.VolumePath).GetMetrics()
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "failed to get metrics: %v", err)
+	}
+
+	available, ok := volumeMetrics.Available.AsInt64()
+	if !ok {
+		return nil, status.Errorf(codes.Internal, "failed to transform volume available size(%v)", volumeMetrics.Available)
+	}
+	capacity, ok := volumeMetrics.Capacity.AsInt64()
+	if !ok {
+		return nil, status.Errorf(codes.Internal, "failed to transform volume capacity size(%v)", volumeMetrics.Capacity)
+	}
+	used, ok := volumeMetrics.Used.AsInt64()
+	if !ok {
+		return nil, status.Errorf(codes.Internal, "failed to transform volume used size(%v)", volumeMetrics.Used)
+	}
+
+	inodesFree, ok := volumeMetrics.InodesFree.AsInt64()
+	if !ok {
+		return nil, status.Errorf(codes.Internal, "failed to transform disk inodes free(%v)", volumeMetrics.InodesFree)
+	}
+	inodes, ok := volumeMetrics.Inodes.AsInt64()
+	if !ok {
+		return nil, status.Errorf(codes.Internal, "failed to transform disk inodes(%v)", volumeMetrics.Inodes)
+	}
+	inodesUsed, ok := volumeMetrics.InodesUsed.AsInt64()
+	if !ok {
+		return nil, status.Errorf(codes.Internal, "failed to transform disk inodes used(%v)", volumeMetrics.InodesUsed)
+	}
+
+	return &csi.NodeGetVolumeStatsResponse{
+		Usage: []*csi.VolumeUsage{
+			{
+				Unit:      csi.VolumeUsage_BYTES,
+				Available: available,
+				Total:     capacity,
+				Used:      used,
+			},
+			{
+				Unit:      csi.VolumeUsage_INODES,
+				Available: inodesFree,
+				Total:     inodes,
+				Used:      inodesUsed,
+			},
+		},
+	}, nil
 }
 
 // NodeUnstageVolume unstage volume

vendor/k8s.io/cloud-provider/CONTRIBUTING.md

@@ -0,0 +1,9 @@
+# Contributing guidelines
+
+Welcome to Kubernetes. We are excited about the prospect of you joining our [community](https://github.com/kubernetes/community)! The Kubernetes community abides by the CNCF [code of conduct](code-of-conduct.md).
+
+Do not open pull requests directly against this repository, they will be ignored. Instead, please open pull requests against [kubernetes/kubernetes](https://git.k8s.io/kubernetes/).  Please follow the same [contributing guide](https://git.k8s.io/kubernetes/CONTRIBUTING.md) you would follow for any other pull request made to kubernetes/kubernetes. Changes to this repo should be discussed with [sig cloud-provider](https://github.com/kubernetes/community/tree/master/sig-cloud-provider).
+
+This repository is published from [kubernetes/kubernetes/staging/src/k8s.io/cloud-provider](https://git.k8s.io/kubernetes/staging/src/k8s.io/cloud-provider) by the [kubernetes publishing-bot](https://git.k8s.io/publishing-bot).
+
+Please see [Staging Directory and Publishing](https://git.k8s.io/community/contributors/devel/sig-architecture/staging.md) for more information

vendor/k8s.io/cloud-provider/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

vendor/k8s.io/cloud-provider/OWNERS

@@ -0,0 +1,38 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- mikedanese
+- dims
+- wlan0
+- andrewsykim
+- cheftako
+reviewers:
+- wojtek-t
+- deads2k
+- derekwaynecarr
+- vishh
+- mikedanese
+- liggitt
+- gmarek
+- davidopp
+- pmorie
+- sttts
+- quinton-hoole
+- dchen1107
+- saad-ali
+- luxas
+- justinsb
+- piosz
+- jsafrane
+- dims
+- krousey
+- rootfs
+- freehan
+- jingxu97
+- wlan0
+- cheftako
+- andrewsykim
+- mcrute
+labels:
+- sig/cloud-provider
+- area/cloudprovider

vendor/k8s.io/cloud-provider/README.md

@@ -0,0 +1,32 @@
+# cloud-provider
+
+This repository defines the cloud-provider interface and mechanism to initialize
+a cloud-provider implementation into Kubernetes. Currently multiple processes
+use this code although the intent is that it will eventually only be cloud
+controller manager.
+
+**Note:** go-get or vendor this package as `k8s.io/cloud-provider`.
+
+## Purpose
+
+This library is a shared dependency for processes which need to be able to
+integrate with cloud-provider specific functionality.
+
+## Compatibility
+
+Cloud Providers are expected to keep the HEAD of their implementations in sync
+with the HEAD of this repository.
+
+## Where does it come from?
+
+`cloud-provider` is synced from
+https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/cloud-provider.
+Code changes are made in that location, merged into k8s.io/kubernetes and
+later synced here.
+
+## Things you should NOT do
+
+ 1. Add an cloud provider specific code to this repo.
+ 2. Directly modify anything under vendor/k8s.io/cloud-provider in this repo. Those are driven from `k8s.io/kubernetes/staging/src/k8s.io/cloud-provider`.
+ 3. Make interface changes without first discussing them with
+    sig-cloudprovider.

vendor/k8s.io/cloud-provider/SECURITY_CONTACTS

@@ -0,0 +1,20 @@
+# Defined below are the security contacts for this repo.
+#
+# They are the contact point for the Product Security Committee to reach out
+# to for triaging and handling of incoming issues.
+#
+# The below names agree to abide by the
+# [Embargo Policy](https://git.k8s.io/security/private-distributors-list.md#embargo-policy)
+# and will be removed and replaced if they violate that agreement.
+#
+# DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE
+# INSTRUCTIONS AT https://kubernetes.io/security/
+
+cheftako
+andrewsykim
+dims
+cjcullen
+joelsmith
+liggitt
+philips
+tallclair

vendor/k8s.io/cloud-provider/cloud.go

@@ -0,0 +1,313 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cloudprovider
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/informers"
+	clientset "k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
+)
+
+// ControllerClientBuilder allows you to get clients and configs for controllers
+// Please note a copy also exists in pkg/controller/client_builder.go
+// TODO: Make this depend on the separate controller utilities repo (issues/68947)
+type ControllerClientBuilder interface {
+	Config(name string) (*restclient.Config, error)
+	ConfigOrDie(name string) *restclient.Config
+	Client(name string) (clientset.Interface, error)
+	ClientOrDie(name string) clientset.Interface
+}
+
+// Interface is an abstract, pluggable interface for cloud providers.
+type Interface interface {
+	// Initialize provides the cloud with a kubernetes client builder and may spawn goroutines
+	// to perform housekeeping or run custom controllers specific to the cloud provider.
+	// Any tasks started here should be cleaned up when the stop channel closes.
+	Initialize(clientBuilder ControllerClientBuilder, stop <-chan struct{})
+	// LoadBalancer returns a balancer interface. Also returns true if the interface is supported, false otherwise.
+	LoadBalancer() (LoadBalancer, bool)
+	// Instances returns an instances interface. Also returns true if the interface is supported, false otherwise.
+	Instances() (Instances, bool)
+	// InstancesV2 is an implementation for instances and should only be implemented by external cloud providers.
+	// Implementing InstancesV2 is behaviorally identical to Instances but is optimized to significantly reduce
+	// API calls to the cloud provider when registering and syncing nodes. Implementation of this interface will
+	// disable calls to the Zones interface. Also returns true if the interface is supported, false otherwise.
+	InstancesV2() (InstancesV2, bool)
+	// Zones returns a zones interface. Also returns true if the interface is supported, false otherwise.
+	// DEPRECATED: Zones is deprecated in favor of retrieving zone/region information from InstancesV2.
+	// This interface will not be called if InstancesV2 is enabled.
+	Zones() (Zones, bool)
+	// Clusters returns a clusters interface.  Also returns true if the interface is supported, false otherwise.
+	Clusters() (Clusters, bool)
+	// Routes returns a routes interface along with whether the interface is supported.
+	Routes() (Routes, bool)
+	// ProviderName returns the cloud provider ID.
+	ProviderName() string
+	// HasClusterID returns true if a ClusterID is required and set
+	HasClusterID() bool
+}
+
+type InformerUser interface {
+	// SetInformers sets the informer on the cloud object.
+	SetInformers(informerFactory informers.SharedInformerFactory)
+}
+
+// Clusters is an abstract, pluggable interface for clusters of containers.
+type Clusters interface {
+	// ListClusters lists the names of the available clusters.
+	ListClusters(ctx context.Context) ([]string, error)
+	// Master gets back the address (either DNS name or IP address) of the master node for the cluster.
+	Master(ctx context.Context, clusterName string) (string, error)
+}
+
+// (DEPRECATED) DefaultLoadBalancerName is the default load balancer name that is called from
+// LoadBalancer.GetLoadBalancerName. Use this method to maintain backward compatible names for
+// LoadBalancers that were created prior to Kubernetes v1.12. In the future, each provider should
+// replace this method call in GetLoadBalancerName with a provider-specific implementation that
+// is less cryptic than the Service's UUID.
+func DefaultLoadBalancerName(service *v1.Service) string {
+	//GCE requires that the name of a load balancer starts with a lower case letter.
+	ret := "a" + string(service.UID)
+	ret = strings.Replace(ret, "-", "", -1)
+	//AWS requires that the name of a load balancer is shorter than 32 bytes.
+	if len(ret) > 32 {
+		ret = ret[:32]
+	}
+	return ret
+}
+
+// GetInstanceProviderID builds a ProviderID for a node in a cloud.
+func GetInstanceProviderID(ctx context.Context, cloud Interface, nodeName types.NodeName) (string, error) {
+	instances, ok := cloud.Instances()
+	if !ok {
+		return "", fmt.Errorf("failed to get instances from cloud provider")
+	}
+	instanceID, err := instances.InstanceID(ctx, nodeName)
+	if err != nil {
+		if err == NotImplemented {
+			return "", err
+		}
+
+		return "", fmt.Errorf("failed to get instance ID from cloud provider: %v", err)
+	}
+	return cloud.ProviderName() + "://" + instanceID, nil
+}
+
+// LoadBalancer is an abstract, pluggable interface for load balancers.
+//
+// Cloud provider may chose to implement the logic for
+// constructing/destroying specific kinds of load balancers in a
+// controller separate from the ServiceController.  If this is the case,
+// then {Ensure,Update}LoadBalancer must return the ImplementedElsewhere error.
+// For the given LB service, the GetLoadBalancer must return "exists=True" if
+// there exists a LoadBalancer instance created by ServiceController.
+// In all other cases, GetLoadBalancer must return a NotFound error.
+// EnsureLoadBalancerDeleted must not return ImplementedElsewhere to ensure
+// proper teardown of resources that were allocated by the ServiceController.
+// This can happen if a user changes the type of LB via an update to the resource
+// or when migrating from ServiceController to alternate implementation.
+// The finalizer on the service will be added and removed by ServiceController
+// irrespective of the ImplementedElsewhere error. Additional finalizers for
+// LB services must be managed in the alternate implementation.
+type LoadBalancer interface {
+	// TODO: Break this up into different interfaces (LB, etc) when we have more than one type of service
+	// GetLoadBalancer returns whether the specified load balancer exists, and
+	// if so, what its status is.
+	// Implementations must treat the *v1.Service parameter as read-only and not modify it.
+	// Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager
+	GetLoadBalancer(ctx context.Context, clusterName string, service *v1.Service) (status *v1.LoadBalancerStatus, exists bool, err error)
+	// GetLoadBalancerName returns the name of the load balancer. Implementations must treat the
+	// *v1.Service parameter as read-only and not modify it.
+	GetLoadBalancerName(ctx context.Context, clusterName string, service *v1.Service) string
+	// EnsureLoadBalancer creates a new load balancer 'name', or updates the existing one. Returns the status of the balancer
+	// Implementations must treat the *v1.Service and *v1.Node
+	// parameters as read-only and not modify them.
+	// Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager
+	EnsureLoadBalancer(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node) (*v1.LoadBalancerStatus, error)
+	// UpdateLoadBalancer updates hosts under the specified load balancer.
+	// Implementations must treat the *v1.Service and *v1.Node
+	// parameters as read-only and not modify them.
+	// Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager
+	UpdateLoadBalancer(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node) error
+	// EnsureLoadBalancerDeleted deletes the specified load balancer if it
+	// exists, returning nil if the load balancer specified either didn't exist or
+	// was successfully deleted.
+	// This construction is useful because many cloud providers' load balancers
+	// have multiple underlying components, meaning a Get could say that the LB
+	// doesn't exist even if some part of it is still laying around.
+	// Implementations must treat the *v1.Service parameter as read-only and not modify it.
+	// Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager
+	EnsureLoadBalancerDeleted(ctx context.Context, clusterName string, service *v1.Service) error
+}
+
+// Instances is an abstract, pluggable interface for sets of instances.
+type Instances interface {
+	// NodeAddresses returns the addresses of the specified instance.
+	NodeAddresses(ctx context.Context, name types.NodeName) ([]v1.NodeAddress, error)
+	// NodeAddressesByProviderID returns the addresses of the specified instance.
+	// The instance is specified using the providerID of the node. The
+	// ProviderID is a unique identifier of the node. This will not be called
+	// from the node whose nodeaddresses are being queried. i.e. local metadata
+	// services cannot be used in this method to obtain nodeaddresses
+	NodeAddressesByProviderID(ctx context.Context, providerID string) ([]v1.NodeAddress, error)
+	// InstanceID returns the cloud provider ID of the node with the specified NodeName.
+	// Note that if the instance does not exist, we must return ("", cloudprovider.InstanceNotFound)
+	// cloudprovider.InstanceNotFound should NOT be returned for instances that exist but are stopped/sleeping
+	InstanceID(ctx context.Context, nodeName types.NodeName) (string, error)
+	// InstanceType returns the type of the specified instance.
+	InstanceType(ctx context.Context, name types.NodeName) (string, error)
+	// InstanceTypeByProviderID returns the type of the specified instance.
+	InstanceTypeByProviderID(ctx context.Context, providerID string) (string, error)
+	// AddSSHKeyToAllInstances adds an SSH public key as a legal identity for all instances
+	// expected format for the key is standard ssh-keygen format: <protocol> <blob>
+	AddSSHKeyToAllInstances(ctx context.Context, user string, keyData []byte) error
+	// CurrentNodeName returns the name of the node we are currently running on
+	// On most clouds (e.g. GCE) this is the hostname, so we provide the hostname
+	CurrentNodeName(ctx context.Context, hostname string) (types.NodeName, error)
+	// InstanceExistsByProviderID returns true if the instance for the given provider exists.
+	// If false is returned with no error, the instance will be immediately deleted by the cloud controller manager.
+	// This method should still return true for instances that exist but are stopped/sleeping.
+	InstanceExistsByProviderID(ctx context.Context, providerID string) (bool, error)
+	// InstanceShutdownByProviderID returns true if the instance is shutdown in cloudprovider
+	InstanceShutdownByProviderID(ctx context.Context, providerID string) (bool, error)
+}
+
+// InstancesV2 is an abstract, pluggable interface for cloud provider instances.
+// Unlike the Instances interface, it is designed for external cloud providers and should only be used by them.
+// Implementation of this interface will disable calls to the Zones interface.
+type InstancesV2 interface {
+	// InstanceExists returns true if the instance for the given node exists according to the cloud provider.
+	// Use the node.name or node.spec.providerID field to find the node in the cloud provider.
+	InstanceExists(ctx context.Context, node *v1.Node) (bool, error)
+	// InstanceShutdown returns true if the instance is shutdown according to the cloud provider.
+	// Use the node.name or node.spec.providerID field to find the node in the cloud provider.
+	InstanceShutdown(ctx context.Context, node *v1.Node) (bool, error)
+	// InstanceMetadata returns the instance's metadata. The values returned in InstanceMetadata are
+	// translated into specific fields and labels in the Node object on registration.
+	// Implementations should always check node.spec.providerID first when trying to discover the instance
+	// for a given node. In cases where node.spec.providerID is empty, implementations can use other
+	// properties of the node like its name, labels and annotations.
+	InstanceMetadata(ctx context.Context, node *v1.Node) (*InstanceMetadata, error)
+}
+
+// Route is a representation of an advanced routing rule.
+type Route struct {
+	// Name is the name of the routing rule in the cloud-provider.
+	// It will be ignored in a Create (although nameHint may influence it)
+	Name string
+	// TargetNode is the NodeName of the target instance.
+	TargetNode types.NodeName
+	// DestinationCIDR is the CIDR format IP range that this routing rule
+	// applies to.
+	DestinationCIDR string
+	// Blackhole is set to true if this is a blackhole route
+	// The node controller will delete the route if it is in the managed range.
+	Blackhole bool
+}
+
+// Routes is an abstract, pluggable interface for advanced routing rules.
+type Routes interface {
+	// ListRoutes lists all managed routes that belong to the specified clusterName
+	ListRoutes(ctx context.Context, clusterName string) ([]*Route, error)
+	// CreateRoute creates the described managed route
+	// route.Name will be ignored, although the cloud-provider may use nameHint
+	// to create a more user-meaningful name.
+	CreateRoute(ctx context.Context, clusterName string, nameHint string, route *Route) error
+	// DeleteRoute deletes the specified managed route
+	// Route should be as returned by ListRoutes
+	DeleteRoute(ctx context.Context, clusterName string, route *Route) error
+}
+
+var (
+	DiskNotFound         = errors.New("disk is not found")
+	ImplementedElsewhere = errors.New("implemented by alternate to cloud provider")
+	InstanceNotFound     = errors.New("instance not found")
+	NotImplemented       = errors.New("unimplemented")
+)
+
+// Zone represents the location of a particular machine.
+type Zone struct {
+	FailureDomain string
+	Region        string
+}
+
+// Zones is an abstract, pluggable interface for zone enumeration.
+// DEPRECATED: Zones is deprecated in favor of retrieving zone/region information from InstancesV2.
+// This interface will not be called if InstancesV2 is enabled.
+type Zones interface {
+	// GetZone returns the Zone containing the current failure zone and locality region that the program is running in
+	// In most cases, this method is called from the kubelet querying a local metadata service to acquire its zone.
+	// For the case of external cloud providers, use GetZoneByProviderID or GetZoneByNodeName since GetZone
+	// can no longer be called from the kubelets.
+	GetZone(ctx context.Context) (Zone, error)
+
+	// GetZoneByProviderID returns the Zone containing the current zone and locality region of the node specified by providerID
+	// This method is particularly used in the context of external cloud providers where node initialization must be done
+	// outside the kubelets.
+	GetZoneByProviderID(ctx context.Context, providerID string) (Zone, error)
+
+	// GetZoneByNodeName returns the Zone containing the current zone and locality region of the node specified by node name
+	// This method is particularly used in the context of external cloud providers where node initialization must be done
+	// outside the kubelets.
+	GetZoneByNodeName(ctx context.Context, nodeName types.NodeName) (Zone, error)
+}
+
+// PVLabeler is an abstract, pluggable interface for fetching labels for volumes
+type PVLabeler interface {
+	GetLabelsForVolume(ctx context.Context, pv *v1.PersistentVolume) (map[string]string, error)
+}
+
+// InstanceMetadata contains metadata about a specific instance.
+// Values returned in InstanceMetadata are translated into specific fields and labels for Node.
+type InstanceMetadata struct {
+	// ProviderID is a unique ID used to idenfitify an instance on the cloud provider.
+	// The ProviderID set here will be set on the node's spec.providerID field.
+	// The provider ID format can be set by the cloud provider but providers should
+	// ensure the format does not change in any incompatible way.
+	//
+	// The provider ID format used by existing cloud provider has been:
+	//    <provider-name>://<instance-id>
+	// Existing providers setting this field should preserve the existing format
+	// currently being set in node.spec.providerID.
+	ProviderID string
+	// InstanceType is the instance's type.
+	// The InstanceType set here will be set using the following labels on the node object:
+	//   * node.kubernetes.io/instance-type=<instance-type>
+	//   * beta.kubernetes.io/instance-type=<instance-type> (DEPRECATED)
+	InstanceType string
+	// NodeAddress contains information for the instance's address.
+	// The node addresses returned here will be set on the node's status.addresses field.
+	NodeAddresses []v1.NodeAddress
+
+	// Zone is the zone that the instance is in.
+	// The value set here is applied as the following labels on the node:
+	//   * topology.kubernetes.io/zone=<zone>
+	//   * failure-domain.beta.kubernetes.io/zone=<zone> (DEPRECATED)
+	Zone string
+	// Region is the region that the instance is in.
+	// The value set here is applied as the following labels on the node:
+	//   * topology.kubernetes.io/region=<region>
+	//   * failure-domain.beta.kubernetes.io/region=<region> (DEPRECATED)
+	Region string
+}

vendor/k8s.io/cloud-provider/code-of-conduct.md

@@ -0,0 +1,3 @@
+# Kubernetes Community Code of Conduct
+
+Please refer to our [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md)

vendor/k8s.io/cloud-provider/doc.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package cloudprovider supplies interfaces and implementations for cloud service providers.
+package cloudprovider // import "k8s.io/cloud-provider"

vendor/k8s.io/cloud-provider/go.mod

@@ -0,0 +1,29 @@
+// This is a generated file. Do not edit directly.
+
+module k8s.io/cloud-provider
+
+go 1.15
+
+require (
+	github.com/google/go-cmp v0.5.2
+	github.com/spf13/cobra v1.1.1
+	github.com/spf13/pflag v1.0.5
+	github.com/stretchr/testify v1.6.1
+	k8s.io/api v0.20.0
+	k8s.io/apimachinery v0.20.0
+	k8s.io/apiserver v0.20.0
+	k8s.io/client-go v0.20.0
+	k8s.io/component-base v0.20.0
+	k8s.io/controller-manager v0.20.0
+	k8s.io/klog/v2 v2.4.0
+	k8s.io/utils v0.0.0-20201110183641-67b214c5f920
+)
+
+replace (
+	k8s.io/api => k8s.io/api v0.20.0
+	k8s.io/apimachinery => k8s.io/apimachinery v0.20.0
+	k8s.io/apiserver => k8s.io/apiserver v0.20.0
+	k8s.io/client-go => k8s.io/client-go v0.20.0
+	k8s.io/component-base => k8s.io/component-base v0.20.0
+	k8s.io/controller-manager => k8s.io/controller-manager v0.20.0
+)

vendor/k8s.io/cloud-provider/go.sum

@@ -0,0 +1,694 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
+github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
+github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
+github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw=
+github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg=
+github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
+github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
+github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
+github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
+github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
+github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46 h1:lsxEuwrXEAokXB9qhlbKWPpo3KMLZQ5WB5WLQRW1uq0=
+github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
+github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
+github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
+github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
+github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
+github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y=
+github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
+github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
+github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk=
+github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses=
+github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg=
+github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
+github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY=
+github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
+github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w=
+github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
+github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o=
+github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
+github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc=
+github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
+github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
+github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
+github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 h1:5ZkaAPbicIKTF2I64qf5Fh8Aa83Q/dnOafMYV0OMwjA=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
+github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I=
+github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI=
+github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
+github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
+github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
+github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
+github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
+github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
+github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
+github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
+github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM=
+github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
+github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
+github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
+github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
+github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/moby/term v0.0.0-20200312100748-672ec06f55cd h1:aY7OQNf2XqY/JQ6qREWamhI/81os/agb2BAGpcx5yWI=
+github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
+github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw=
+github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
+github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4=
+github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4=
+github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
+go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
+go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489 h1:1JFLBqwIgdyHN1ZtgjTBwO+blA6gVOmZurpiMEsETKo=
+go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE=
+golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 h1:pE8b58s1HRDMi8RDc79m0HISf9D4TzseP40cEA6IGfs=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201112073958-5cba982894dd h1:5CtCZbICpIOFdgO940moixOPjc0178IU44m4EjOO5IY=
+golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
+golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s=
+golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a h1:pOwg4OoaRYScjmR4LlLgdtnyoHYTSAVhhqe5uPdpII8=
+google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
+gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
+gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+k8s.io/api v0.20.0/go.mod h1:HyLC5l5eoS/ygQYl1BXBgFzWNlkHiAuyNAbevIn+FKg=
+k8s.io/apimachinery v0.20.0/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
+k8s.io/apiserver v0.20.0/go.mod h1:6gRIWiOkvGvQt12WTYmsiYoUyYW0FXSiMdNl4m+sxY8=
+k8s.io/client-go v0.20.0/go.mod h1:4KWh/g+Ocd8KkCwKF8vUNnmqgv+EVnQDK4MBF4oB5tY=
+k8s.io/component-base v0.20.0/go.mod h1:wKPj+RHnAr8LW2EIBIK7AxOHPde4gme2lzXwVSoRXeA=
+k8s.io/controller-manager v0.20.0/go.mod h1:nD4qym/pmCz2v1tpqvlEBVlHW9CAZwedloM8GrJTLpg=
+k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
+k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ=
+k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
+k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c=
+k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM=
+k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw=
+k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14 h1:TihvEz9MPj2u0KWds6E2OBUXfwaL4qRJ33c7HGiJpqk=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
+sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhYV9PmGrh1s8=
+sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
+sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
+sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
+sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

vendor/k8s.io/cloud-provider/plugins.go

@@ -0,0 +1,148 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cloudprovider
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"sync"
+
+	"k8s.io/klog/v2"
+)
+
+// Factory is a function that returns a cloudprovider.Interface.
+// The config parameter provides an io.Reader handler to the factory in
+// order to load specific configurations. If no configuration is provided
+// the parameter is nil.
+type Factory func(config io.Reader) (Interface, error)
+
+// All registered cloud providers.
+var (
+	providersMutex           sync.Mutex
+	providers                = make(map[string]Factory)
+	deprecatedCloudProviders = []struct {
+		name     string
+		external bool
+		detail   string
+	}{
+		{"aws", false, "The AWS provider is deprecated and will be removed in a future release"},
+		{"azure", false, "The Azure provider is deprecated and will be removed in a future release"},
+		{"gce", false, "The GCE provider is deprecated and will be removed in a future release"},
+		{"openstack", true, "https://github.com/kubernetes/cloud-provider-openstack"},
+		{"vsphere", false, "The vSphere provider is deprecated and will be removed in a future release"},
+	}
+)
+
+const externalCloudProvider = "external"
+
+// RegisterCloudProvider registers a cloudprovider.Factory by name.  This
+// is expected to happen during app startup.
+func RegisterCloudProvider(name string, cloud Factory) {
+	providersMutex.Lock()
+	defer providersMutex.Unlock()
+	if _, found := providers[name]; found {
+		klog.Fatalf("Cloud provider %q was registered twice", name)
+	}
+	klog.V(1).Infof("Registered cloud provider %q", name)
+	providers[name] = cloud
+}
+
+// IsCloudProvider returns true if name corresponds to an already registered
+// cloud provider.
+func IsCloudProvider(name string) bool {
+	providersMutex.Lock()
+	defer providersMutex.Unlock()
+	_, found := providers[name]
+	return found
+}
+
+// GetCloudProvider creates an instance of the named cloud provider, or nil if
+// the name is unknown.  The error return is only used if the named provider
+// was known but failed to initialize. The config parameter specifies the
+// io.Reader handler of the configuration file for the cloud provider, or nil
+// for no configuration.
+func GetCloudProvider(name string, config io.Reader) (Interface, error) {
+	providersMutex.Lock()
+	defer providersMutex.Unlock()
+	f, found := providers[name]
+	if !found {
+		return nil, nil
+	}
+	return f(config)
+}
+
+// Detects if the string is an external cloud provider
+func IsExternal(name string) bool {
+	return name == externalCloudProvider
+}
+
+func DeprecationWarningForProvider(providerName string) {
+	for _, provider := range deprecatedCloudProviders {
+		if provider.name != providerName {
+			continue
+		}
+
+		detail := provider.detail
+		if provider.external {
+			detail = fmt.Sprintf("Please use 'external' cloud provider for %s: %s", providerName, provider.detail)
+		}
+
+		klog.Warningf("WARNING: %s built-in cloud provider is now deprecated. %s", providerName, detail)
+		break
+	}
+}
+
+// InitCloudProvider creates an instance of the named cloud provider.
+func InitCloudProvider(name string, configFilePath string) (Interface, error) {
+	var cloud Interface
+	var err error
+
+	if name == "" {
+		return nil, nil
+	}
+
+	if IsExternal(name) {
+		klog.Info("External cloud provider specified")
+		return nil, nil
+	}
+
+	if configFilePath != "" {
+		var config *os.File
+		config, err = os.Open(configFilePath)
+		if err != nil {
+			klog.Fatalf("Couldn't open cloud provider configuration %s: %#v",
+				configFilePath, err)
+		}
+
+		defer config.Close()
+		cloud, err = GetCloudProvider(name, config)
+	} else {
+		// Pass explicit nil so plugins can actually check for nil. See
+		// "Why is my nil error value not equal to nil?" in golang.org/doc/faq.
+		cloud, err = GetCloudProvider(name, nil)
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("could not init cloud provider %q: %v", name, err)
+	}
+	if cloud == nil {
+		return nil, fmt.Errorf("unknown cloud provider %q", name)
+	}
+
+	return cloud, nil
+}

vendor/k8s.io/cloud-provider/ports.go

@@ -0,0 +1,23 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cloudprovider
+
+const (
+	// CloudControllerManagerPort is the default port for the cloud controller manager server.
+	// This value may be overridden by a flag at startup.
+	CloudControllerManagerPort = 10258
+)

vendor/k8s.io/kubernetes/pkg/proxy/util/BUILD

@@ -0,0 +1,58 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "endpoints.go",
+        "network.go",
+        "port.go",
+        "utils.go",
+    ],
+    importpath = "k8s.io/kubernetes/pkg/proxy/util",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/apis/core/v1/helper:go_default_library",
+        "//pkg/util/sysctl:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/rand:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/record:go_default_library",
+        "//vendor/k8s.io/klog/v2:go_default_library",
+        "//vendor/k8s.io/utils/net:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "endpoints_test.go",
+        "port_test.go",
+        "utils_test.go",
+    ],
+    embed = [":go_default_library"],
+    deps = [
+        "//pkg/proxy/util/testing:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//pkg/proxy/util/iptables:all-srcs",
+        "//pkg/proxy/util/testing:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/k8s.io/kubernetes/pkg/proxy/util/endpoints.go

@@ -0,0 +1,74 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+
+	"k8s.io/klog/v2"
+)
+
+// IPPart returns just the IP part of an IP or IP:port or endpoint string. If the IP
+// part is an IPv6 address enclosed in brackets (e.g. "[fd00:1::5]:9999"),
+// then the brackets are stripped as well.
+func IPPart(s string) string {
+	if ip := net.ParseIP(s); ip != nil {
+		// IP address without port
+		return s
+	}
+	// Must be IP:port
+	host, _, err := net.SplitHostPort(s)
+	if err != nil {
+		klog.Errorf("Error parsing '%s': %v", s, err)
+		return ""
+	}
+	// Check if host string is a valid IP address
+	ip := net.ParseIP(host)
+	if ip == nil {
+		klog.Errorf("invalid IP part '%s'", host)
+		return ""
+	}
+	return ip.String()
+}
+
+// PortPart returns just the port part of an endpoint string.
+func PortPart(s string) (int, error) {
+	// Must be IP:port
+	_, port, err := net.SplitHostPort(s)
+	if err != nil {
+		klog.Errorf("Error parsing '%s': %v", s, err)
+		return -1, err
+	}
+	portNumber, err := strconv.Atoi(port)
+	if err != nil {
+		klog.Errorf("Error parsing '%s': %v", port, err)
+		return -1, err
+	}
+	return portNumber, nil
+}
+
+// ToCIDR returns a host address of the form <ip-address>/32 for
+// IPv4 and <ip-address>/128 for IPv6
+func ToCIDR(ip net.IP) string {
+	len := 32
+	if ip.To4() == nil {
+		len = 128
+	}
+	return fmt.Sprintf("%s/%d", ip.String(), len)
+}

vendor/k8s.io/kubernetes/pkg/proxy/util/network.go

@@ -0,0 +1,45 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"net"
+)
+
+// NetworkInterfacer defines an interface for several net library functions. Production
+// code will forward to net library functions, and unit tests will override the methods
+// for testing purposes.
+type NetworkInterfacer interface {
+	Addrs(intf *net.Interface) ([]net.Addr, error)
+	Interfaces() ([]net.Interface, error)
+}
+
+// RealNetwork implements the NetworkInterfacer interface for production code, just
+// wrapping the underlying net library function calls.
+type RealNetwork struct{}
+
+// Addrs wraps net.Interface.Addrs(), it's a part of NetworkInterfacer interface.
+func (RealNetwork) Addrs(intf *net.Interface) ([]net.Addr, error) {
+	return intf.Addrs()
+}
+
+// Interfaces wraps net.Interfaces(), it's a part of NetworkInterfacer interface.
+func (RealNetwork) Interfaces() ([]net.Interface, error) {
+	return net.Interfaces()
+}
+
+var _ NetworkInterfacer = &RealNetwork{}

vendor/k8s.io/kubernetes/pkg/proxy/util/port.go

@@ -0,0 +1,67 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+
+	"k8s.io/klog/v2"
+)
+
+// LocalPort describes a port on specific IP address and protocol
+type LocalPort struct {
+	// Description is the identity message of a given local port.
+	Description string
+	// IP is the IP address part of a given local port.
+	// If this string is empty, the port binds to all local IP addresses.
+	IP string
+	// Port is the port part of a given local port.
+	Port int
+	// Protocol is the protocol part of a given local port.
+	// The value is assumed to be lower-case. For example, "udp" not "UDP", "tcp" not "TCP".
+	Protocol string
+}
+
+func (lp *LocalPort) String() string {
+	ipPort := net.JoinHostPort(lp.IP, strconv.Itoa(lp.Port))
+	return fmt.Sprintf("%q (%s/%s)", lp.Description, ipPort, lp.Protocol)
+}
+
+// Closeable is an interface around closing a port.
+type Closeable interface {
+	Close() error
+}
+
+// PortOpener is an interface around port opening/closing.
+// Abstracted out for testing.
+type PortOpener interface {
+	OpenLocalPort(lp *LocalPort, isIPv6 bool) (Closeable, error)
+}
+
+// RevertPorts is closing ports in replacementPortsMap but not in originalPortsMap. In other words, it only
+// closes the ports opened in this sync.
+func RevertPorts(replacementPortsMap, originalPortsMap map[LocalPort]Closeable) {
+	for k, v := range replacementPortsMap {
+		// Only close newly opened local ports - leave ones that were open before this update
+		if originalPortsMap[k] == nil {
+			klog.V(2).Infof("Closing local port %s", k.String())
+			v.Close()
+		}
+	}
+}

vendor/k8s.io/kubernetes/pkg/proxy/util/utils.go

@@ -0,0 +1,443 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"strconv"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	utilrand "k8s.io/apimachinery/pkg/util/rand"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/client-go/tools/record"
+	helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
+	utilsysctl "k8s.io/kubernetes/pkg/util/sysctl"
+	utilnet "k8s.io/utils/net"
+
+	"k8s.io/klog/v2"
+)
+
+const (
+	// IPv4ZeroCIDR is the CIDR block for the whole IPv4 address space
+	IPv4ZeroCIDR = "0.0.0.0/0"
+
+	// IPv6ZeroCIDR is the CIDR block for the whole IPv6 address space
+	IPv6ZeroCIDR = "::/0"
+)
+
+var (
+	// ErrAddressNotAllowed indicates the address is not allowed
+	ErrAddressNotAllowed = errors.New("address not allowed")
+
+	// ErrNoAddresses indicates there are no addresses for the hostname
+	ErrNoAddresses = errors.New("No addresses for hostname")
+)
+
+// isValidEndpoint checks that the given host / port pair are valid endpoint
+func isValidEndpoint(host string, port int) bool {
+	return host != "" && port > 0
+}
+
+// BuildPortsToEndpointsMap builds a map of portname -> all ip:ports for that
+// portname. Explode Endpoints.Subsets[*] into this structure.
+func BuildPortsToEndpointsMap(endpoints *v1.Endpoints) map[string][]string {
+	portsToEndpoints := map[string][]string{}
+	for i := range endpoints.Subsets {
+		ss := &endpoints.Subsets[i]
+		for i := range ss.Ports {
+			port := &ss.Ports[i]
+			for i := range ss.Addresses {
+				addr := &ss.Addresses[i]
+				if isValidEndpoint(addr.IP, int(port.Port)) {
+					portsToEndpoints[port.Name] = append(portsToEndpoints[port.Name], net.JoinHostPort(addr.IP, strconv.Itoa(int(port.Port))))
+				}
+			}
+		}
+	}
+	return portsToEndpoints
+}
+
+// IsZeroCIDR checks whether the input CIDR string is either
+// the IPv4 or IPv6 zero CIDR
+func IsZeroCIDR(cidr string) bool {
+	if cidr == IPv4ZeroCIDR || cidr == IPv6ZeroCIDR {
+		return true
+	}
+	return false
+}
+
+// IsProxyableIP checks if a given IP address is permitted to be proxied
+func IsProxyableIP(ip string) error {
+	netIP := net.ParseIP(ip)
+	if netIP == nil {
+		return ErrAddressNotAllowed
+	}
+	return isProxyableIP(netIP)
+}
+
+func isProxyableIP(ip net.IP) error {
+	if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsInterfaceLocalMulticast() {
+		return ErrAddressNotAllowed
+	}
+	return nil
+}
+
+// Resolver is an interface for net.Resolver
+type Resolver interface {
+	LookupIPAddr(ctx context.Context, host string) ([]net.IPAddr, error)
+}
+
+// IsProxyableHostname checks if the IP addresses for a given hostname are permitted to be proxied
+func IsProxyableHostname(ctx context.Context, resolv Resolver, hostname string) error {
+	resp, err := resolv.LookupIPAddr(ctx, hostname)
+	if err != nil {
+		return err
+	}
+
+	if len(resp) == 0 {
+		return ErrNoAddresses
+	}
+
+	for _, host := range resp {
+		if err := isProxyableIP(host.IP); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// IsAllowedHost checks if the given IP host address is in a network in the denied list.
+func IsAllowedHost(host net.IP, denied []*net.IPNet) error {
+	for _, ipNet := range denied {
+		if ipNet.Contains(host) {
+			return ErrAddressNotAllowed
+		}
+	}
+	return nil
+}
+
+// GetLocalAddrs returns a list of all network addresses on the local system
+func GetLocalAddrs() ([]net.IP, error) {
+	var localAddrs []net.IP
+
+	addrs, err := net.InterfaceAddrs()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, addr := range addrs {
+		ip, _, err := net.ParseCIDR(addr.String())
+		if err != nil {
+			return nil, err
+		}
+
+		localAddrs = append(localAddrs, ip)
+	}
+
+	return localAddrs, nil
+}
+
+// ShouldSkipService checks if a given service should skip proxying
+func ShouldSkipService(service *v1.Service) bool {
+	// if ClusterIP is "None" or empty, skip proxying
+	if !helper.IsServiceIPSet(service) {
+		klog.V(3).Infof("Skipping service %s in namespace %s due to clusterIP = %q", service.Name, service.Namespace, service.Spec.ClusterIP)
+		return true
+	}
+	// Even if ClusterIP is set, ServiceTypeExternalName services don't get proxied
+	if service.Spec.Type == v1.ServiceTypeExternalName {
+		klog.V(3).Infof("Skipping service %s in namespace %s due to Type=ExternalName", service.Name, service.Namespace)
+		return true
+	}
+	return false
+}
+
+// GetNodeAddresses return all matched node IP addresses based on given cidr slice.
+// Some callers, e.g. IPVS proxier, need concrete IPs, not ranges, which is why this exists.
+// NetworkInterfacer is injected for test purpose.
+// We expect the cidrs passed in is already validated.
+// Given an empty input `[]`, it will return `0.0.0.0/0` and `::/0` directly.
+// If multiple cidrs is given, it will return the minimal IP sets, e.g. given input `[1.2.0.0/16, 0.0.0.0/0]`, it will
+// only return `0.0.0.0/0`.
+// NOTE: GetNodeAddresses only accepts CIDRs, if you want concrete IPs, e.g. 1.2.3.4, then the input should be 1.2.3.4/32.
+func GetNodeAddresses(cidrs []string, nw NetworkInterfacer) (sets.String, error) {
+	uniqueAddressList := sets.NewString()
+	if len(cidrs) == 0 {
+		uniqueAddressList.Insert(IPv4ZeroCIDR)
+		uniqueAddressList.Insert(IPv6ZeroCIDR)
+		return uniqueAddressList, nil
+	}
+	// First round of iteration to pick out `0.0.0.0/0` or `::/0` for the sake of excluding non-zero IPs.
+	for _, cidr := range cidrs {
+		if IsZeroCIDR(cidr) {
+			uniqueAddressList.Insert(cidr)
+		}
+	}
+
+	itfs, err := nw.Interfaces()
+	if err != nil {
+		return nil, fmt.Errorf("error listing all interfaces from host, error: %v", err)
+	}
+
+	// Second round of iteration to parse IPs based on cidr.
+	for _, cidr := range cidrs {
+		if IsZeroCIDR(cidr) {
+			continue
+		}
+
+		_, ipNet, _ := net.ParseCIDR(cidr)
+		for _, itf := range itfs {
+			addrs, err := nw.Addrs(&itf)
+			if err != nil {
+				return nil, fmt.Errorf("error getting address from interface %s, error: %v", itf.Name, err)
+			}
+
+			for _, addr := range addrs {
+				if addr == nil {
+					continue
+				}
+
+				ip, _, err := net.ParseCIDR(addr.String())
+				if err != nil {
+					return nil, fmt.Errorf("error parsing CIDR for interface %s, error: %v", itf.Name, err)
+				}
+
+				if ipNet.Contains(ip) {
+					if utilnet.IsIPv6(ip) && !uniqueAddressList.Has(IPv6ZeroCIDR) {
+						uniqueAddressList.Insert(ip.String())
+					}
+					if !utilnet.IsIPv6(ip) && !uniqueAddressList.Has(IPv4ZeroCIDR) {
+						uniqueAddressList.Insert(ip.String())
+					}
+				}
+			}
+		}
+	}
+
+	if uniqueAddressList.Len() == 0 {
+		return nil, fmt.Errorf("no addresses found for cidrs %v", cidrs)
+	}
+
+	return uniqueAddressList, nil
+}
+
+// LogAndEmitIncorrectIPVersionEvent logs and emits incorrect IP version event.
+func LogAndEmitIncorrectIPVersionEvent(recorder record.EventRecorder, fieldName, fieldValue, svcNamespace, svcName string, svcUID types.UID) {
+	errMsg := fmt.Sprintf("%s in %s has incorrect IP version", fieldValue, fieldName)
+	klog.Errorf("%s (service %s/%s).", errMsg, svcNamespace, svcName)
+	if recorder != nil {
+		recorder.Eventf(
+			&v1.ObjectReference{
+				Kind:      "Service",
+				Name:      svcName,
+				Namespace: svcNamespace,
+				UID:       svcUID,
+			}, v1.EventTypeWarning, "KubeProxyIncorrectIPVersion", errMsg)
+	}
+}
+
+// MapIPsByIPFamily maps a slice of IPs to their respective IP families (v4 or v6)
+func MapIPsByIPFamily(ipStrings []string) map[v1.IPFamily][]string {
+	ipFamilyMap := map[v1.IPFamily][]string{}
+	for _, ip := range ipStrings {
+		// Handle only the valid IPs
+		if ipFamily, err := getIPFamilyFromIP(ip); err == nil {
+			ipFamilyMap[ipFamily] = append(ipFamilyMap[ipFamily], ip)
+		} else {
+			klog.Errorf("Skipping invalid IP: %s", ip)
+		}
+	}
+	return ipFamilyMap
+}
+
+// MapCIDRsByIPFamily maps a slice of IPs to their respective IP families (v4 or v6)
+func MapCIDRsByIPFamily(cidrStrings []string) map[v1.IPFamily][]string {
+	ipFamilyMap := map[v1.IPFamily][]string{}
+	for _, cidr := range cidrStrings {
+		// Handle only the valid CIDRs
+		if ipFamily, err := getIPFamilyFromCIDR(cidr); err == nil {
+			ipFamilyMap[ipFamily] = append(ipFamilyMap[ipFamily], cidr)
+		} else {
+			klog.Errorf("Skipping invalid cidr: %s", cidr)
+		}
+	}
+	return ipFamilyMap
+}
+
+func getIPFamilyFromIP(ipStr string) (v1.IPFamily, error) {
+	netIP := net.ParseIP(ipStr)
+	if netIP == nil {
+		return "", ErrAddressNotAllowed
+	}
+
+	if utilnet.IsIPv6(netIP) {
+		return v1.IPv6Protocol, nil
+	}
+	return v1.IPv4Protocol, nil
+}
+
+func getIPFamilyFromCIDR(cidrStr string) (v1.IPFamily, error) {
+	_, netCIDR, err := net.ParseCIDR(cidrStr)
+	if err != nil {
+		return "", ErrAddressNotAllowed
+	}
+	if utilnet.IsIPv6CIDR(netCIDR) {
+		return v1.IPv6Protocol, nil
+	}
+	return v1.IPv4Protocol, nil
+}
+
+// OtherIPFamily returns the other ip family
+func OtherIPFamily(ipFamily v1.IPFamily) v1.IPFamily {
+	if ipFamily == v1.IPv6Protocol {
+		return v1.IPv4Protocol
+	}
+
+	return v1.IPv6Protocol
+}
+
+// AppendPortIfNeeded appends the given port to IP address unless it is already in
+// "ipv4:port" or "[ipv6]:port" format.
+func AppendPortIfNeeded(addr string, port int32) string {
+	// Return if address is already in "ipv4:port" or "[ipv6]:port" format.
+	if _, _, err := net.SplitHostPort(addr); err == nil {
+		return addr
+	}
+
+	// Simply return for invalid case. This should be caught by validation instead.
+	ip := net.ParseIP(addr)
+	if ip == nil {
+		return addr
+	}
+
+	// Append port to address.
+	if ip.To4() != nil {
+		return fmt.Sprintf("%s:%d", addr, port)
+	}
+	return fmt.Sprintf("[%s]:%d", addr, port)
+}
+
+// ShuffleStrings copies strings from the specified slice into a copy in random
+// order. It returns a new slice.
+func ShuffleStrings(s []string) []string {
+	if s == nil {
+		return nil
+	}
+	shuffled := make([]string, len(s))
+	perm := utilrand.Perm(len(s))
+	for i, j := range perm {
+		shuffled[j] = s[i]
+	}
+	return shuffled
+}
+
+// EnsureSysctl sets a kernel sysctl to a given numeric value.
+func EnsureSysctl(sysctl utilsysctl.Interface, name string, newVal int) error {
+	if oldVal, _ := sysctl.GetSysctl(name); oldVal != newVal {
+		if err := sysctl.SetSysctl(name, newVal); err != nil {
+			return fmt.Errorf("can't set sysctl %s to %d: %v", name, newVal, err)
+		}
+		klog.V(1).Infof("Changed sysctl %q: %d -> %d", name, oldVal, newVal)
+	}
+	return nil
+}
+
+// DialContext is a dial function matching the signature of net.Dialer.DialContext.
+type DialContext = func(context.Context, string, string) (net.Conn, error)
+
+// FilteredDialOptions configures how a DialContext is wrapped by NewFilteredDialContext.
+type FilteredDialOptions struct {
+	// DialHostIPDenylist restricts hosts from being dialed.
+	DialHostCIDRDenylist []*net.IPNet
+	// AllowLocalLoopback controls connections to local loopback hosts (as defined by
+	// IsProxyableIP).
+	AllowLocalLoopback bool
+}
+
+// NewFilteredDialContext returns a DialContext function that filters connections based on a FilteredDialOptions.
+func NewFilteredDialContext(wrapped DialContext, resolv Resolver, opts *FilteredDialOptions) DialContext {
+	if wrapped == nil {
+		wrapped = http.DefaultTransport.(*http.Transport).DialContext
+	}
+	if opts == nil {
+		// Do no filtering
+		return wrapped
+	}
+	if resolv == nil {
+		resolv = net.DefaultResolver
+	}
+	if len(opts.DialHostCIDRDenylist) == 0 && opts.AllowLocalLoopback {
+		// Do no filtering.
+		return wrapped
+	}
+	return func(ctx context.Context, network, address string) (net.Conn, error) {
+		resp, err := resolv.LookupIPAddr(ctx, address)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(resp) == 0 {
+			return nil, ErrNoAddresses
+		}
+
+		for _, host := range resp {
+			if !opts.AllowLocalLoopback {
+				if err := isProxyableIP(host.IP); err != nil {
+					return nil, err
+				}
+			}
+			if opts.DialHostCIDRDenylist != nil {
+				if err := IsAllowedHost(host.IP, opts.DialHostCIDRDenylist); err != nil {
+					return nil, err
+				}
+			}
+		}
+		return wrapped(ctx, network, address)
+	}
+}
+
+// GetClusterIPByFamily returns a service clusterip by family
+func GetClusterIPByFamily(ipFamily v1.IPFamily, service *v1.Service) string {
+	// allowing skew
+	if len(service.Spec.IPFamilies) == 0 {
+		if len(service.Spec.ClusterIP) == 0 || service.Spec.ClusterIP == v1.ClusterIPNone {
+			return ""
+		}
+
+		IsIPv6Family := (ipFamily == v1.IPv6Protocol)
+		if IsIPv6Family == utilnet.IsIPv6String(service.Spec.ClusterIP) {
+			return service.Spec.ClusterIP
+		}
+
+		return ""
+	}
+
+	for idx, family := range service.Spec.IPFamilies {
+		if family == ipFamily {
+			if idx < len(service.Spec.ClusterIPs) {
+				return service.Spec.ClusterIPs[idx]
+			}
+		}
+	}
+
+	return ""
+}

vendor/k8s.io/kubernetes/pkg/util/sysctl/BUILD

@@ -0,0 +1,28 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["sysctl.go"],
+    importpath = "k8s.io/kubernetes/pkg/util/sysctl",
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//pkg/util/sysctl/testing:all-srcs",
+    ],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/pkg/util/sysctl/sysctl.go

@@ -0,0 +1,100 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sysctl
+
+import (
+	"io/ioutil"
+	"path"
+	"strconv"
+	"strings"
+)
+
+const (
+	sysctlBase = "/proc/sys"
+	// VMOvercommitMemory refers to the sysctl variable responsible for defining
+	// the memory over-commit policy used by kernel.
+	VMOvercommitMemory = "vm/overcommit_memory"
+	// VMPanicOnOOM refers to the sysctl variable responsible for defining
+	// the OOM behavior used by kernel.
+	VMPanicOnOOM = "vm/panic_on_oom"
+	// KernelPanic refers to the sysctl variable responsible for defining
+	// the timeout after a panic for the kernel to reboot.
+	KernelPanic = "kernel/panic"
+	// KernelPanicOnOops refers to the sysctl variable responsible for defining
+	// the kernel behavior when an oops or BUG is encountered.
+	KernelPanicOnOops = "kernel/panic_on_oops"
+	// RootMaxKeys refers to the sysctl variable responsible for defining
+	// the maximum number of keys that the root user (UID 0 in the root user namespace) may own.
+	RootMaxKeys = "kernel/keys/root_maxkeys"
+	// RootMaxBytes refers to the sysctl variable responsible for defining
+	// the maximum number of bytes of data that the root user (UID 0 in the root user namespace)
+	// can hold in the payloads of the keys owned by root.
+	RootMaxBytes = "kernel/keys/root_maxbytes"
+
+	// VMOvercommitMemoryAlways represents that kernel performs no memory over-commit handling.
+	VMOvercommitMemoryAlways = 1
+	// VMPanicOnOOMInvokeOOMKiller represents that kernel calls the oom_killer function when OOM occurs.
+	VMPanicOnOOMInvokeOOMKiller = 0
+
+	// KernelPanicOnOopsAlways represents that kernel panics on kernel oops.
+	KernelPanicOnOopsAlways = 1
+	// KernelPanicRebootTimeout is the timeout seconds after a panic for the kernel to reboot.
+	KernelPanicRebootTimeout = 10
+
+	// RootMaxKeysSetting is the maximum number of keys that the root user (UID 0 in the root user namespace) may own.
+	// Needed since docker creates a new key per container.
+	RootMaxKeysSetting = 1000000
+	// RootMaxBytesSetting is the maximum number of bytes of data that the root user (UID 0 in the root user namespace)
+	// can hold in the payloads of the keys owned by root.
+	// Allocate 25 bytes per key * number of MaxKeys.
+	RootMaxBytesSetting = RootMaxKeysSetting * 25
+)
+
+// Interface is an injectable interface for running sysctl commands.
+type Interface interface {
+	// GetSysctl returns the value for the specified sysctl setting
+	GetSysctl(sysctl string) (int, error)
+	// SetSysctl modifies the specified sysctl flag to the new value
+	SetSysctl(sysctl string, newVal int) error
+}
+
+// New returns a new Interface for accessing sysctl
+func New() Interface {
+	return &procSysctl{}
+}
+
+// procSysctl implements Interface by reading and writing files under /proc/sys
+type procSysctl struct {
+}
+
+// GetSysctl returns the value for the specified sysctl setting
+func (*procSysctl) GetSysctl(sysctl string) (int, error) {
+	data, err := ioutil.ReadFile(path.Join(sysctlBase, sysctl))
+	if err != nil {
+		return -1, err
+	}
+	val, err := strconv.Atoi(strings.Trim(string(data), " \n"))
+	if err != nil {
+		return -1, err
+	}
+	return val, nil
+}
+
+// SetSysctl modifies the specified sysctl flag to the new value
+func (*procSysctl) SetSysctl(sysctl string, newVal int) error {
+	return ioutil.WriteFile(path.Join(sysctlBase, sysctl), []byte(strconv.Itoa(newVal)), 0640)
+}

vendor/k8s.io/kubernetes/pkg/volume/BUILD

@@ -0,0 +1,134 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "doc.go",
+        "metrics_cached.go",
+        "metrics_du.go",
+        "metrics_errors.go",
+        "metrics_nil.go",
+        "metrics_statfs.go",
+        "noop_expandable_plugin.go",
+        "plugins.go",
+        "volume.go",
+        "volume_linux.go",
+        "volume_unsupported.go",
+    ],
+    importpath = "k8s.io/kubernetes/pkg/volume",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/proxy/util:go_default_library",
+        "//pkg/volume/util/fs:go_default_library",
+        "//pkg/volume/util/hostutil:go_default_library",
+        "//pkg/volume/util/recyclerclient:go_default_library",
+        "//pkg/volume/util/subpath:go_default_library",
+        "//staging/src/k8s.io/api/authentication/v1:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library",
+        "//staging/src/k8s.io/client-go/informers:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
+        "//staging/src/k8s.io/client-go/listers/storage/v1:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/cache:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/record:go_default_library",
+        "//staging/src/k8s.io/cloud-provider:go_default_library",
+        "//staging/src/k8s.io/mount-utils:go_default_library",
+        "//vendor/k8s.io/klog/v2:go_default_library",
+        "//vendor/k8s.io/utils/exec:go_default_library",
+    ] + select({
+        "@io_bazel_rules_go//go/platform:android": [
+            "//pkg/features:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:linux": [
+            "//pkg/features:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "metrics_du_test.go",
+        "metrics_nil_test.go",
+        "metrics_statfs_test.go",
+        "plugins_test.go",
+        "volume_linux_test.go",
+    ],
+    embed = [":go_default_library"],
+    deps = [
+        "//pkg/volume/testing:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//staging/src/k8s.io/client-go/util/testing:go_default_library",
+    ] + select({
+        "@io_bazel_rules_go//go/platform:android": [
+            "//pkg/features:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+            "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library",
+            "//vendor/golang.org/x/sys/unix:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:linux": [
+            "//pkg/features:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+            "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library",
+            "//vendor/golang.org/x/sys/unix:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//pkg/volume/awsebs:all-srcs",
+        "//pkg/volume/azure_file:all-srcs",
+        "//pkg/volume/azuredd:all-srcs",
+        "//pkg/volume/cephfs:all-srcs",
+        "//pkg/volume/cinder:all-srcs",
+        "//pkg/volume/configmap:all-srcs",
+        "//pkg/volume/csi:all-srcs",
+        "//pkg/volume/csimigration:all-srcs",
+        "//pkg/volume/downwardapi:all-srcs",
+        "//pkg/volume/emptydir:all-srcs",
+        "//pkg/volume/fc:all-srcs",
+        "//pkg/volume/flexvolume:all-srcs",
+        "//pkg/volume/flocker:all-srcs",
+        "//pkg/volume/gcepd:all-srcs",
+        "//pkg/volume/git_repo:all-srcs",
+        "//pkg/volume/glusterfs:all-srcs",
+        "//pkg/volume/hostpath:all-srcs",
+        "//pkg/volume/iscsi:all-srcs",
+        "//pkg/volume/local:all-srcs",
+        "//pkg/volume/nfs:all-srcs",
+        "//pkg/volume/portworx:all-srcs",
+        "//pkg/volume/projected:all-srcs",
+        "//pkg/volume/quobyte:all-srcs",
+        "//pkg/volume/rbd:all-srcs",
+        "//pkg/volume/scaleio:all-srcs",
+        "//pkg/volume/secret:all-srcs",
+        "//pkg/volume/storageos:all-srcs",
+        "//pkg/volume/testing:all-srcs",
+        "//pkg/volume/util:all-srcs",
+        "//pkg/volume/validation:all-srcs",
+        "//pkg/volume/vsphere_volume:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/k8s.io/kubernetes/pkg/volume/OWNERS

@@ -0,0 +1,8 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- sig-storage-approvers
+reviewers:
+- sig-storage-reviewers
+labels:
+- sig/storage

vendor/k8s.io/kubernetes/pkg/volume/doc.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package volume includes internal representations of external volume types
+// as well as utility methods required to mount/unmount volumes to kubelets.
+package volume // import "k8s.io/kubernetes/pkg/volume"

vendor/k8s.io/kubernetes/pkg/volume/metrics_cached.go

@@ -0,0 +1,74 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package volume
+
+import (
+	"sync"
+	"sync/atomic"
+)
+
+var _ MetricsProvider = &cachedMetrics{}
+
+// cachedMetrics represents a MetricsProvider that wraps another provider and
+// caches the result.
+type cachedMetrics struct {
+	wrapped       MetricsProvider
+	resultError   error
+	resultMetrics *Metrics
+	once          cacheOnce
+}
+
+// NewCachedMetrics creates a new cachedMetrics wrapping another
+// MetricsProvider and caching the results.
+func NewCachedMetrics(provider MetricsProvider) MetricsProvider {
+	return &cachedMetrics{wrapped: provider}
+}
+
+// GetMetrics runs the wrapped metrics provider's GetMetrics methd once and
+// caches the result. Will not cache result if there is an error.
+// See MetricsProvider.GetMetrics
+func (md *cachedMetrics) GetMetrics() (*Metrics, error) {
+	md.once.cache(func() error {
+		md.resultMetrics, md.resultError = md.wrapped.GetMetrics()
+		return md.resultError
+	})
+	return md.resultMetrics, md.resultError
+}
+
+// Copied from sync.Once but we don't want to cache the results if there is an
+// error
+type cacheOnce struct {
+	m    sync.Mutex
+	done uint32
+}
+
+// Copied from sync.Once but we don't want to cache the results if there is an
+// error
+func (o *cacheOnce) cache(f func() error) {
+	if atomic.LoadUint32(&o.done) == 1 {
+		return
+	}
+	// Slow-path.
+	o.m.Lock()
+	defer o.m.Unlock()
+	if o.done == 0 {
+		err := f()
+		if err == nil {
+			atomic.StoreUint32(&o.done, 1)
+		}
+	}
+}

vendor/k8s.io/kubernetes/pkg/volume/metrics_du.go

@@ -0,0 +1,99 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package volume
+
+import (
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/pkg/volume/util/fs"
+)
+
+var _ MetricsProvider = &metricsDu{}
+
+// metricsDu represents a MetricsProvider that calculates the used and
+// available Volume space by calling fs.DiskUsage() and gathering
+// filesystem info for the Volume path.
+type metricsDu struct {
+	// the directory path the volume is mounted to.
+	path string
+}
+
+// NewMetricsDu creates a new metricsDu with the Volume path.
+func NewMetricsDu(path string) MetricsProvider {
+	return &metricsDu{path}
+}
+
+// GetMetrics calculates the volume usage and device free space by executing "du"
+// and gathering filesystem info for the Volume path.
+// See MetricsProvider.GetMetrics
+func (md *metricsDu) GetMetrics() (*Metrics, error) {
+	metrics := &Metrics{Time: metav1.Now()}
+	if md.path == "" {
+		return metrics, NewNoPathDefinedError()
+	}
+
+	err := md.runDiskUsage(metrics)
+	if err != nil {
+		return metrics, err
+	}
+
+	err = md.runFind(metrics)
+	if err != nil {
+		return metrics, err
+	}
+
+	err = md.getFsInfo(metrics)
+	if err != nil {
+		return metrics, err
+	}
+
+	return metrics, nil
+}
+
+// runDiskUsage gets disk usage of md.path and writes the results to metrics.Used
+func (md *metricsDu) runDiskUsage(metrics *Metrics) error {
+	used, err := fs.DiskUsage(md.path)
+	if err != nil {
+		return err
+	}
+	metrics.Used = used
+	return nil
+}
+
+// runFind executes the "find" command and writes the results to metrics.InodesUsed
+func (md *metricsDu) runFind(metrics *Metrics) error {
+	inodesUsed, err := fs.Find(md.path)
+	if err != nil {
+		return err
+	}
+	metrics.InodesUsed = resource.NewQuantity(inodesUsed, resource.BinarySI)
+	return nil
+}
+
+// getFsInfo writes metrics.Capacity and metrics.Available from the filesystem
+// info
+func (md *metricsDu) getFsInfo(metrics *Metrics) error {
+	available, capacity, _, inodes, inodesFree, _, err := fs.FsInfo(md.path)
+	if err != nil {
+		return NewFsInfoFailedError(err)
+	}
+	metrics.Available = resource.NewQuantity(available, resource.BinarySI)
+	metrics.Capacity = resource.NewQuantity(capacity, resource.BinarySI)
+	metrics.Inodes = resource.NewQuantity(inodes, resource.BinarySI)
+	metrics.InodesFree = resource.NewQuantity(inodesFree, resource.BinarySI)
+	return nil
+}

vendor/k8s.io/kubernetes/pkg/volume/metrics_errors.go

@@ -0,0 +1,86 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package volume
+
+import (
+	"fmt"
+)
+
+const (
+	// ErrCodeNotSupported code for NotSupported Errors.
+	ErrCodeNotSupported int = iota + 1
+	ErrCodeNoPathDefined
+	ErrCodeFsInfoFailed
+)
+
+// NewNotSupportedError creates a new MetricsError with code NotSupported.
+func NewNotSupportedError() *MetricsError {
+	return &MetricsError{
+		Code: ErrCodeNotSupported,
+		Msg:  "metrics are not supported for MetricsNil Volumes",
+	}
+}
+
+// NewNotSupportedErrorWithDriverName creates a new MetricsError with code NotSupported.
+// driver name is added to the error message.
+func NewNotSupportedErrorWithDriverName(name string) *MetricsError {
+	return &MetricsError{
+		Code: ErrCodeNotSupported,
+		Msg:  fmt.Sprintf("metrics are not supported for %s volumes", name),
+	}
+}
+
+// NewNoPathDefinedError creates a new MetricsError with code NoPathDefined.
+func NewNoPathDefinedError() *MetricsError {
+	return &MetricsError{
+		Code: ErrCodeNoPathDefined,
+		Msg:  "no path defined for disk usage metrics.",
+	}
+}
+
+// NewFsInfoFailedError creates a new MetricsError with code FsInfoFailed.
+func NewFsInfoFailedError(err error) *MetricsError {
+	return &MetricsError{
+		Code: ErrCodeFsInfoFailed,
+		Msg:  fmt.Sprintf("failed to get FsInfo due to error %v", err),
+	}
+}
+
+// MetricsError to distinguish different Metrics Errors.
+type MetricsError struct {
+	Code int
+	Msg  string
+}
+
+func (e *MetricsError) Error() string {
+	return e.Msg
+}
+
+// IsNotSupported returns true if and only if err is "key" not found error.
+func IsNotSupported(err error) bool {
+	return isErrCode(err, ErrCodeNotSupported)
+}
+
+func isErrCode(err error, code int) bool {
+	if err == nil {
+		return false
+	}
+	if e, ok := err.(*MetricsError); ok {
+		return e.Code == code
+	}
+	return false
+}

vendor/k8s.io/kubernetes/pkg/volume/metrics_nil.go

@@ -0,0 +1,30 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package volume
+
+var _ MetricsProvider = &MetricsNil{}
+
+// MetricsNil represents a MetricsProvider that does not support returning
+// Metrics.  It serves as a placeholder for Volumes that do not yet support
+// metrics.
+type MetricsNil struct{}
+
+// GetMetrics returns an empty Metrics and an error.
+// See MetricsProvider.GetMetrics
+func (*MetricsNil) GetMetrics() (*Metrics, error) {
+	return &Metrics{}, NewNotSupportedError()
+}

vendor/k8s.io/kubernetes/pkg/volume/metrics_statfs.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package volume
+
+import (
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/pkg/volume/util/fs"
+)
+
+var _ MetricsProvider = &metricsStatFS{}
+
+// metricsStatFS represents a MetricsProvider that calculates the used and available
+// Volume space by stat'ing and gathering filesystem info for the Volume path.
+type metricsStatFS struct {
+	// the directory path the volume is mounted to.
+	path string
+}
+
+// NewMetricsStatfs creates a new metricsStatFS with the Volume path.
+func NewMetricsStatFS(path string) MetricsProvider {
+	return &metricsStatFS{path}
+}
+
+// See MetricsProvider.GetMetrics
+// GetMetrics calculates the volume usage and device free space by executing "du"
+// and gathering filesystem info for the Volume path.
+func (md *metricsStatFS) GetMetrics() (*Metrics, error) {
+	metrics := &Metrics{Time: metav1.Now()}
+	if md.path == "" {
+		return metrics, NewNoPathDefinedError()
+	}
+
+	err := md.getFsInfo(metrics)
+	if err != nil {
+		return metrics, err
+	}
+
+	return metrics, nil
+}
+
+// getFsInfo writes metrics.Capacity, metrics.Used and metrics.Available from the filesystem info
+func (md *metricsStatFS) getFsInfo(metrics *Metrics) error {
+	available, capacity, usage, inodes, inodesFree, inodesUsed, err := fs.FsInfo(md.path)
+	if err != nil {
+		return NewFsInfoFailedError(err)
+	}
+	metrics.Available = resource.NewQuantity(available, resource.BinarySI)
+	metrics.Capacity = resource.NewQuantity(capacity, resource.BinarySI)
+	metrics.Used = resource.NewQuantity(usage, resource.BinarySI)
+	metrics.Inodes = resource.NewQuantity(inodes, resource.BinarySI)
+	metrics.InodesFree = resource.NewQuantity(inodesFree, resource.BinarySI)
+	metrics.InodesUsed = resource.NewQuantity(inodesUsed, resource.BinarySI)
+	return nil
+}

vendor/k8s.io/kubernetes/pkg/volume/noop_expandable_plugin.go

@@ -0,0 +1,77 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package volume
+
+import (
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+type noopExpandableVolumePluginInstance struct {
+	spec *Spec
+}
+
+var _ ExpandableVolumePlugin = &noopExpandableVolumePluginInstance{}
+
+func (n *noopExpandableVolumePluginInstance) ExpandVolumeDevice(spec *Spec, newSize resource.Quantity, oldSize resource.Quantity) (resource.Quantity, error) {
+	return newSize, nil
+}
+
+func (n *noopExpandableVolumePluginInstance) Init(host VolumeHost) error {
+	return nil
+}
+
+func (n *noopExpandableVolumePluginInstance) GetPluginName() string {
+	return n.spec.KubeletExpandablePluginName()
+}
+
+func (n *noopExpandableVolumePluginInstance) GetVolumeName(spec *Spec) (string, error) {
+	return n.spec.Name(), nil
+}
+
+func (n *noopExpandableVolumePluginInstance) CanSupport(spec *Spec) bool {
+	return true
+}
+
+func (n *noopExpandableVolumePluginInstance) RequiresRemount(spec *Spec) bool {
+	return false
+}
+
+func (n *noopExpandableVolumePluginInstance) NewMounter(spec *Spec, podRef *v1.Pod, opts VolumeOptions) (Mounter, error) {
+	return nil, nil
+}
+
+func (n *noopExpandableVolumePluginInstance) NewUnmounter(name string, podUID types.UID) (Unmounter, error) {
+	return nil, nil
+}
+
+func (n *noopExpandableVolumePluginInstance) ConstructVolumeSpec(volumeName, mountPath string) (*Spec, error) {
+	return n.spec, nil
+}
+
+func (n *noopExpandableVolumePluginInstance) SupportsMountOption() bool {
+	return true
+}
+
+func (n *noopExpandableVolumePluginInstance) SupportsBulkVolumeVerification() bool {
+	return false
+}
+
+func (n *noopExpandableVolumePluginInstance) RequiresFSResize() bool {
+	return true
+}

vendor/k8s.io/kubernetes/pkg/volume/util/fs/BUILD

@@ -0,0 +1,95 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "fs.go",
+        "fs_unsupported.go",
+        "fs_windows.go",
+    ],
+    importpath = "k8s.io/kubernetes/pkg/volume/util/fs",
+    visibility = ["//visibility:public"],
+    deps = select({
+        "@io_bazel_rules_go//go/platform:aix": [
+            "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:android": [
+            "//pkg/volume/util/fsquota:go_default_library",
+            "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+            "//vendor/golang.org/x/sys/unix:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:darwin": [
+            "//pkg/volume/util/fsquota:go_default_library",
+            "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+            "//vendor/golang.org/x/sys/unix:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:dragonfly": [
+            "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:freebsd": [
+            "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:illumos": [
+            "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:ios": [
+            "//pkg/volume/util/fsquota:go_default_library",
+            "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+            "//vendor/golang.org/x/sys/unix:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:js": [
+            "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:linux": [
+            "//pkg/volume/util/fsquota:go_default_library",
+            "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+            "//vendor/golang.org/x/sys/unix:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:nacl": [
+            "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:netbsd": [
+            "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:openbsd": [
+            "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:plan9": [
+            "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:solaris": [
+            "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:windows": [
+            "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+            "//vendor/golang.org/x/sys/windows:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["fs_windows_test.go"],
+    embed = [":go_default_library"],
+    deps = select({
+        "@io_bazel_rules_go//go/platform:windows": [
+            "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
+)

vendor/k8s.io/kubernetes/pkg/volume/util/fs/fs.go

@@ -0,0 +1,117 @@
+// +build linux darwin
+
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fs
+
+import (
+	"bytes"
+	"fmt"
+	"os/exec"
+	"strings"
+
+	"golang.org/x/sys/unix"
+
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/kubernetes/pkg/volume/util/fsquota"
+)
+
+// FsInfo linux returns (available bytes, byte capacity, byte usage, total inodes, inodes free, inode usage, error)
+// for the filesystem that path resides upon.
+func FsInfo(path string) (int64, int64, int64, int64, int64, int64, error) {
+	statfs := &unix.Statfs_t{}
+	err := unix.Statfs(path, statfs)
+	if err != nil {
+		return 0, 0, 0, 0, 0, 0, err
+	}
+
+	// Available is blocks available * fragment size
+	available := int64(statfs.Bavail) * int64(statfs.Bsize)
+
+	// Capacity is total block count * fragment size
+	capacity := int64(statfs.Blocks) * int64(statfs.Bsize)
+
+	// Usage is block being used * fragment size (aka block size).
+	usage := (int64(statfs.Blocks) - int64(statfs.Bfree)) * int64(statfs.Bsize)
+
+	inodes := int64(statfs.Files)
+	inodesFree := int64(statfs.Ffree)
+	inodesUsed := inodes - inodesFree
+
+	return available, capacity, usage, inodes, inodesFree, inodesUsed, nil
+}
+
+// DiskUsage gets disk usage of specified path.
+func DiskUsage(path string) (*resource.Quantity, error) {
+	// First check whether the quota system knows about this directory
+	// A nil quantity with no error means that the path does not support quotas
+	// and we should use other mechanisms.
+	data, err := fsquota.GetConsumption(path)
+	if data != nil {
+		return data, nil
+	} else if err != nil {
+		return nil, fmt.Errorf("unable to retrieve disk consumption via quota for %s: %v", path, err)
+	}
+	// Uses the same niceness level as cadvisor.fs does when running du
+	// Uses -B 1 to always scale to a blocksize of 1 byte
+	out, err := exec.Command("nice", "-n", "19", "du", "-x", "-s", "-B", "1", path).CombinedOutput()
+	if err != nil {
+		return nil, fmt.Errorf("failed command 'du' ($ nice -n 19 du -x -s -B 1) on path %s with error %v", path, err)
+	}
+	used, err := resource.ParseQuantity(strings.Fields(string(out))[0])
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse 'du' output %s due to error %v", out, err)
+	}
+	used.Format = resource.BinarySI
+	return &used, nil
+}
+
+// Find uses the equivalent of the command `find <path> -dev -printf '.' | wc -c` to count files and directories.
+// While this is not an exact measure of inodes used, it is a very good approximation.
+func Find(path string) (int64, error) {
+	if path == "" {
+		return 0, fmt.Errorf("invalid directory")
+	}
+	// First check whether the quota system knows about this directory
+	// A nil quantity with no error means that the path does not support quotas
+	// and we should use other mechanisms.
+	inodes, err := fsquota.GetInodes(path)
+	if inodes != nil {
+		return inodes.Value(), nil
+	} else if err != nil {
+		return 0, fmt.Errorf("unable to retrieve inode consumption via quota for %s: %v", path, err)
+	}
+	var counter byteCounter
+	var stderr bytes.Buffer
+	findCmd := exec.Command("find", path, "-xdev", "-printf", ".")
+	findCmd.Stdout, findCmd.Stderr = &counter, &stderr
+	if err := findCmd.Start(); err != nil {
+		return 0, fmt.Errorf("failed to exec cmd %v - %v; stderr: %v", findCmd.Args, err, stderr.String())
+	}
+	if err := findCmd.Wait(); err != nil {
+		return 0, fmt.Errorf("cmd %v failed. stderr: %s; err: %v", findCmd.Args, stderr.String(), err)
+	}
+	return counter.bytesWritten, nil
+}
+
+// Simple io.Writer implementation that counts how many bytes were written.
+type byteCounter struct{ bytesWritten int64 }
+
+func (b *byteCounter) Write(p []byte) (int, error) {
+	b.bytesWritten += int64(len(p))
+	return len(p), nil
+}

vendor/k8s.io/kubernetes/pkg/volume/util/fs/fs_unsupported.go

@@ -0,0 +1,39 @@
+// +build !linux,!darwin,!windows
+
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fs
+
+import (
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/api/resource"
+)
+
+// FSInfo unsupported returns 0 values for available and capacity and an error.
+func FsInfo(path string) (int64, int64, int64, int64, int64, int64, error) {
+	return 0, 0, 0, 0, 0, 0, fmt.Errorf("FsInfo not supported for this build.")
+}
+
+// DiskUsage gets disk usage of specified path.
+func DiskUsage(path string) (*resource.Quantity, error) {
+	return nil, fmt.Errorf("Du not supported for this build.")
+}
+
+func Find(path string) (int64, error) {
+	return 0, fmt.Errorf("Find not supported for this build.")
+}

vendor/k8s.io/kubernetes/pkg/volume/util/fs/fs_windows.go

@@ -0,0 +1,121 @@
+// +build windows
+
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fs
+
+import (
+	"fmt"
+	"os"
+	"syscall"
+	"unsafe"
+
+	"golang.org/x/sys/windows"
+
+	"k8s.io/apimachinery/pkg/api/resource"
+)
+
+var (
+	modkernel32            = windows.NewLazySystemDLL("kernel32.dll")
+	procGetDiskFreeSpaceEx = modkernel32.NewProc("GetDiskFreeSpaceExW")
+)
+
+// FSInfo returns (available bytes, byte capacity, byte usage, total inodes, inodes free, inode usage, error)
+// for the filesystem that path resides upon.
+func FsInfo(path string) (int64, int64, int64, int64, int64, int64, error) {
+	var freeBytesAvailable, totalNumberOfBytes, totalNumberOfFreeBytes int64
+	var err error
+
+	ret, _, err := syscall.Syscall6(
+		procGetDiskFreeSpaceEx.Addr(),
+		4,
+		uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))),
+		uintptr(unsafe.Pointer(&freeBytesAvailable)),
+		uintptr(unsafe.Pointer(&totalNumberOfBytes)),
+		uintptr(unsafe.Pointer(&totalNumberOfFreeBytes)),
+		0,
+		0,
+	)
+	if ret == 0 {
+		return 0, 0, 0, 0, 0, 0, err
+	}
+
+	return freeBytesAvailable, totalNumberOfBytes, totalNumberOfBytes - freeBytesAvailable, 0, 0, 0, nil
+}
+
+// DiskUsage gets disk usage of specified path.
+func DiskUsage(path string) (*resource.Quantity, error) {
+	info, err := os.Lstat(path)
+	if err != nil {
+		return nil, err
+	}
+
+	usage, err := diskUsage(path, info)
+	if err != nil {
+		return nil, err
+	}
+
+	used, err := resource.ParseQuantity(fmt.Sprintf("%d", usage))
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse fs usage %d due to %v", usage, err)
+	}
+	used.Format = resource.BinarySI
+	return &used, nil
+}
+
+// Always return zero since inodes is not supported on Windows.
+func Find(path string) (int64, error) {
+	return 0, nil
+}
+
+func diskUsage(currPath string, info os.FileInfo) (int64, error) {
+	var size int64
+
+	if info.Mode()&os.ModeSymlink != 0 {
+		return size, nil
+	}
+
+	size += info.Size()
+
+	if !info.IsDir() {
+		return size, nil
+	}
+
+	dir, err := os.Open(currPath)
+	if err != nil {
+		return size, err
+	}
+	defer dir.Close()
+
+	files, err := dir.Readdir(-1)
+	if err != nil {
+		return size, err
+	}
+
+	for _, file := range files {
+		if file.IsDir() {
+			s, err := diskUsage(fmt.Sprintf("%s/%s", currPath, file.Name()), file)
+			if err != nil {
+				return size, err
+			}
+			size += s
+		} else {
+			size += file.Size()
+		}
+	}
+	return size, nil
+}

vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/BUILD

@@ -0,0 +1,78 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "project.go",
+        "quota.go",
+        "quota_linux.go",
+        "quota_unsupported.go",
+    ],
+    importpath = "k8s.io/kubernetes/pkg/volume/util/fsquota",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/features:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//staging/src/k8s.io/mount-utils:go_default_library",
+    ] + select({
+        "@io_bazel_rules_go//go/platform:android": [
+            "//pkg/volume/util/fsquota/common:go_default_library",
+            "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
+            "//vendor/golang.org/x/sys/unix:go_default_library",
+            "//vendor/k8s.io/klog/v2:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:linux": [
+            "//pkg/volume/util/fsquota/common:go_default_library",
+            "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
+            "//vendor/golang.org/x/sys/unix:go_default_library",
+            "//vendor/k8s.io/klog/v2:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["quota_linux_test.go"],
+    embed = [":go_default_library"],
+    deps = select({
+        "@io_bazel_rules_go//go/platform:android": [
+            "//pkg/features:go_default_library",
+            "//pkg/volume/util/fsquota/common:go_default_library",
+            "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+            "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+            "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library",
+            "//staging/src/k8s.io/mount-utils:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:linux": [
+            "//pkg/features:go_default_library",
+            "//pkg/volume/util/fsquota/common:go_default_library",
+            "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+            "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+            "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library",
+            "//staging/src/k8s.io/mount-utils:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//pkg/volume/util/fsquota/common:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/common/BUILD

@@ -0,0 +1,34 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "quota_linux_common.go",
+        "quota_linux_common_impl.go",
+    ],
+    importpath = "k8s.io/kubernetes/pkg/volume/util/fsquota/common",
+    visibility = ["//visibility:public"],
+    deps = select({
+        "@io_bazel_rules_go//go/platform:android": [
+            "//vendor/k8s.io/klog/v2:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:linux": [
+            "//vendor/k8s.io/klog/v2:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/common/quota_linux_common.go

@@ -0,0 +1,100 @@
+// +build linux
+
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package common
+
+import (
+	"regexp"
+)
+
+// QuotaID is generic quota identifier.
+// Data type based on quotactl(2).
+type QuotaID int32
+
+const (
+	// UnknownQuotaID -- cannot determine whether a quota is in force
+	UnknownQuotaID QuotaID = -1
+	// BadQuotaID -- Invalid quota
+	BadQuotaID QuotaID = 0
+)
+
+// QuotaType -- type of quota to be applied
+type QuotaType int
+
+const (
+	// FSQuotaAccounting for quotas for accounting only
+	FSQuotaAccounting QuotaType = 1 << iota
+	// FSQuotaEnforcing for quotas for enforcement
+	FSQuotaEnforcing QuotaType = 1 << iota
+)
+
+// FirstQuota is the quota ID we start with.
+// XXXXXXX Need a better way of doing this...
+var FirstQuota QuotaID = 1048577
+
+// MountsFile is the location of the system mount data
+var MountsFile = "/proc/self/mounts"
+
+// MountParseRegexp parses out /proc/sys/self/mounts
+var MountParseRegexp = regexp.MustCompilePOSIX("^([^ ]*)[ \t]*([^ ]*)[ \t]*([^ ]*)") // Ignore options etc.
+
+// LinuxVolumeQuotaProvider returns an appropriate quota applier
+// object if we can support quotas on this device
+type LinuxVolumeQuotaProvider interface {
+	// GetQuotaApplier retrieves an object that can apply
+	// quotas (or nil if this provider cannot support quotas
+	// on the device)
+	GetQuotaApplier(mountpoint string, backingDev string) LinuxVolumeQuotaApplier
+}
+
+// LinuxVolumeQuotaApplier is a generic interface to any quota
+// mechanism supported by Linux
+type LinuxVolumeQuotaApplier interface {
+	// GetQuotaOnDir gets the quota ID (if any) that applies to
+	// this directory
+	GetQuotaOnDir(path string) (QuotaID, error)
+
+	// SetQuotaOnDir applies the specified quota ID to a directory.
+	// Negative value for bytes means that a non-enforcing quota
+	// should be applied (perhaps by setting a quota too large to
+	// be hit)
+	SetQuotaOnDir(path string, id QuotaID, bytes int64) error
+
+	// QuotaIDIsInUse determines whether the quota ID is in use.
+	// Implementations should not check /etc/project or /etc/projid,
+	// only whether their underlying mechanism already has the ID in
+	// use.
+	// Return value of false with no error means that the ID is not
+	// in use; true means that it is already in use.  An error
+	// return means that any quota ID will fail.
+	QuotaIDIsInUse(id QuotaID) (bool, error)
+
+	// GetConsumption returns the consumption (in bytes) of the
+	// directory, determined by the implementation's quota-based
+	// mechanism.  If it is unable to do so using that mechanism,
+	// it should return an error and allow higher layers to
+	// enumerate the directory.
+	GetConsumption(path string, id QuotaID) (int64, error)
+
+	// GetInodes returns the number of inodes used by the
+	// directory, determined by the implementation's quota-based
+	// mechanism.  If it is unable to do so using that mechanism,
+	// it should return an error and allow higher layers to
+	// enumerate the directory.
+	GetInodes(path string, id QuotaID) (int64, error)
+}

vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/common/quota_linux_common_impl.go

@@ -0,0 +1,286 @@
+// +build linux
+
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package common
+
+import (
+	"bufio"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+
+	"k8s.io/klog/v2"
+)
+
+var quotaCmd string
+var quotaCmdInitialized bool
+var quotaCmdLock sync.RWMutex
+
+// If we later get a filesystem that uses project quota semantics other than
+// XFS, we'll need to change this.
+// Higher levels don't need to know what's inside
+type linuxFilesystemType struct {
+	name             string
+	typeMagic        int64 // Filesystem magic number, per statfs(2)
+	maxQuota         int64
+	allowEmptyOutput bool // Accept empty output from "quota" command
+}
+
+const (
+	bitsPerWord = 32 << (^uint(0) >> 63) // either 32 or 64
+)
+
+var (
+	linuxSupportedFilesystems = []linuxFilesystemType{
+		{
+			name:             "XFS",
+			typeMagic:        0x58465342,
+			maxQuota:         1<<(bitsPerWord-1) - 1,
+			allowEmptyOutput: true, // XFS filesystems report nothing if a quota is not present
+		}, {
+			name:             "ext4fs",
+			typeMagic:        0xef53,
+			maxQuota:         (1<<(bitsPerWord-1) - 1) & (1<<58 - 1),
+			allowEmptyOutput: false, // ext4 filesystems always report something even if a quota is not present
+		},
+	}
+)
+
+// VolumeProvider supplies a quota applier to the generic code.
+type VolumeProvider struct {
+}
+
+var quotaCmds = []string{"/sbin/xfs_quota",
+	"/usr/sbin/xfs_quota",
+	"/bin/xfs_quota"}
+
+var quotaParseRegexp = regexp.MustCompilePOSIX("^[^ \t]*[ \t]*([0-9]+)")
+
+var lsattrCmd = "/usr/bin/lsattr"
+var lsattrParseRegexp = regexp.MustCompilePOSIX("^ *([0-9]+) [^ ]+ (.*)$")
+
+// GetQuotaApplier -- does this backing device support quotas that
+// can be applied to directories?
+func (*VolumeProvider) GetQuotaApplier(mountpoint string, backingDev string) LinuxVolumeQuotaApplier {
+	for _, fsType := range linuxSupportedFilesystems {
+		if isFilesystemOfType(mountpoint, backingDev, fsType.typeMagic) {
+			return linuxVolumeQuotaApplier{mountpoint: mountpoint,
+				maxQuota:         fsType.maxQuota,
+				allowEmptyOutput: fsType.allowEmptyOutput,
+			}
+		}
+	}
+	return nil
+}
+
+type linuxVolumeQuotaApplier struct {
+	mountpoint       string
+	maxQuota         int64
+	allowEmptyOutput bool
+}
+
+func getXFSQuotaCmd() (string, error) {
+	quotaCmdLock.Lock()
+	defer quotaCmdLock.Unlock()
+	if quotaCmdInitialized {
+		return quotaCmd, nil
+	}
+	for _, program := range quotaCmds {
+		fileinfo, err := os.Stat(program)
+		if err == nil && ((fileinfo.Mode().Perm() & (1 << 6)) != 0) {
+			klog.V(3).Infof("Found xfs_quota program %s", program)
+			quotaCmd = program
+			quotaCmdInitialized = true
+			return quotaCmd, nil
+		}
+	}
+	quotaCmdInitialized = true
+	return "", fmt.Errorf("no xfs_quota program found")
+}
+
+func doRunXFSQuotaCommand(mountpoint string, mountsFile, command string) (string, error) {
+	quotaCmd, err := getXFSQuotaCmd()
+	if err != nil {
+		return "", err
+	}
+	// We're using numeric project IDs directly; no need to scan
+	// /etc/projects or /etc/projid
+	klog.V(4).Infof("runXFSQuotaCommand %s -t %s -P/dev/null -D/dev/null -x -f %s -c %s", quotaCmd, mountsFile, mountpoint, command)
+	cmd := exec.Command(quotaCmd, "-t", mountsFile, "-P/dev/null", "-D/dev/null", "-x", "-f", mountpoint, "-c", command)
+
+	data, err := cmd.Output()
+	if err != nil {
+		return "", err
+	}
+	klog.V(4).Infof("runXFSQuotaCommand output %q", string(data))
+	return string(data), nil
+}
+
+// Extract the mountpoint we care about into a temporary mounts file so that xfs_quota does
+// not attempt to scan every mount on the filesystem, which could hang if e. g.
+// a stuck NFS mount is present.
+// See https://bugzilla.redhat.com/show_bug.cgi?id=237120 for an example
+// of the problem that could be caused if this were to happen.
+func runXFSQuotaCommand(mountpoint string, command string) (string, error) {
+	tmpMounts, err := ioutil.TempFile("", "mounts")
+	if err != nil {
+		return "", fmt.Errorf("cannot create temporary mount file: %v", err)
+	}
+	tmpMountsFileName := tmpMounts.Name()
+	defer tmpMounts.Close()
+	defer os.Remove(tmpMountsFileName)
+
+	mounts, err := os.Open(MountsFile)
+	if err != nil {
+		return "", fmt.Errorf("cannot open mounts file %s: %v", MountsFile, err)
+	}
+	defer mounts.Close()
+
+	scanner := bufio.NewScanner(mounts)
+	for scanner.Scan() {
+		match := MountParseRegexp.FindStringSubmatch(scanner.Text())
+		if match != nil {
+			mount := match[2]
+			if mount == mountpoint {
+				if _, err := tmpMounts.WriteString(fmt.Sprintf("%s\n", scanner.Text())); err != nil {
+					return "", fmt.Errorf("cannot write temporary mounts file: %v", err)
+				}
+				if err := tmpMounts.Sync(); err != nil {
+					return "", fmt.Errorf("cannot sync temporary mounts file: %v", err)
+				}
+				return doRunXFSQuotaCommand(mountpoint, tmpMountsFileName, command)
+			}
+		}
+	}
+	return "", fmt.Errorf("cannot run xfs_quota: cannot find mount point %s in %s", mountpoint, MountsFile)
+}
+
+// SupportsQuotas determines whether the filesystem supports quotas.
+func SupportsQuotas(mountpoint string, qType QuotaType) (bool, error) {
+	data, err := runXFSQuotaCommand(mountpoint, "state -p")
+	if err != nil {
+		return false, err
+	}
+	if qType == FSQuotaEnforcing {
+		return strings.Contains(data, "Enforcement: ON"), nil
+	}
+	return strings.Contains(data, "Accounting: ON"), nil
+}
+
+func isFilesystemOfType(mountpoint string, backingDev string, typeMagic int64) bool {
+	var buf syscall.Statfs_t
+	err := syscall.Statfs(mountpoint, &buf)
+	if err != nil {
+		klog.Warningf("Warning: Unable to statfs %s: %v", mountpoint, err)
+		return false
+	}
+	if int64(buf.Type) != typeMagic {
+		return false
+	}
+	if answer, _ := SupportsQuotas(mountpoint, FSQuotaAccounting); answer {
+		return true
+	}
+	return false
+}
+
+// GetQuotaOnDir retrieves the quota ID (if any) associated with the specified directory
+// If we can't make system calls, all we can say is that we don't know whether
+// it has a quota, and higher levels have to make the call.
+func (v linuxVolumeQuotaApplier) GetQuotaOnDir(path string) (QuotaID, error) {
+	cmd := exec.Command(lsattrCmd, "-pd", path)
+	data, err := cmd.Output()
+	if err != nil {
+		return BadQuotaID, fmt.Errorf("cannot run lsattr: %v", err)
+	}
+	match := lsattrParseRegexp.FindStringSubmatch(string(data))
+	if match == nil {
+		return BadQuotaID, fmt.Errorf("unable to parse lsattr -pd %s output %s", path, string(data))
+	}
+	if match[2] != path {
+		return BadQuotaID, fmt.Errorf("mismatch between supplied and returned path (%s != %s)", path, match[2])
+	}
+	projid, err := strconv.ParseInt(match[1], 10, 32)
+	if err != nil {
+		return BadQuotaID, fmt.Errorf("unable to parse project ID from %s (%v)", match[1], err)
+	}
+	return QuotaID(projid), nil
+}
+
+// SetQuotaOnDir applies a quota to the specified directory under the specified mountpoint.
+func (v linuxVolumeQuotaApplier) SetQuotaOnDir(path string, id QuotaID, bytes int64) error {
+	if bytes < 0 || bytes > v.maxQuota {
+		bytes = v.maxQuota
+	}
+	_, err := runXFSQuotaCommand(v.mountpoint, fmt.Sprintf("limit -p bhard=%v bsoft=%v %v", bytes, bytes, id))
+	if err != nil {
+		return err
+	}
+
+	_, err = runXFSQuotaCommand(v.mountpoint, fmt.Sprintf("project -s -p %s %v", path, id))
+	return err
+}
+
+func getQuantity(mountpoint string, id QuotaID, xfsQuotaArg string, multiplier int64, allowEmptyOutput bool) (int64, error) {
+	data, err := runXFSQuotaCommand(mountpoint, fmt.Sprintf("quota -p -N -n -v %s %v", xfsQuotaArg, id))
+	if err != nil {
+		return 0, fmt.Errorf("unable to run xfs_quota: %v", err)
+	}
+	if data == "" && allowEmptyOutput {
+		return 0, nil
+	}
+	match := quotaParseRegexp.FindStringSubmatch(data)
+	if match == nil {
+		return 0, fmt.Errorf("unable to parse quota output '%s'", data)
+	}
+	size, err := strconv.ParseInt(match[1], 10, 64)
+	if err != nil {
+		return 0, fmt.Errorf("unable to parse data size '%s' from '%s': %v", match[1], data, err)
+	}
+	klog.V(4).Infof("getQuantity %s %d %s %d => %d %v", mountpoint, id, xfsQuotaArg, multiplier, size, err)
+	return size * multiplier, nil
+}
+
+// GetConsumption returns the consumption in bytes if available via quotas
+func (v linuxVolumeQuotaApplier) GetConsumption(_ string, id QuotaID) (int64, error) {
+	return getQuantity(v.mountpoint, id, "-b", 1024, v.allowEmptyOutput)
+}
+
+// GetInodes returns the inodes in use if available via quotas
+func (v linuxVolumeQuotaApplier) GetInodes(_ string, id QuotaID) (int64, error) {
+	return getQuantity(v.mountpoint, id, "-i", 1, v.allowEmptyOutput)
+}
+
+// QuotaIDIsInUse checks whether the specified quota ID is in use on the specified
+// filesystem
+func (v linuxVolumeQuotaApplier) QuotaIDIsInUse(id QuotaID) (bool, error) {
+	bytes, err := v.GetConsumption(v.mountpoint, id)
+	if err != nil {
+		return false, err
+	}
+	if bytes > 0 {
+		return true, nil
+	}
+	inodes, err := v.GetInodes(v.mountpoint, id)
+	return inodes > 0, err
+}

vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/project.go

@@ -0,0 +1,357 @@
+// +build linux
+
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fsquota
+
+import (
+	"bufio"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"sync"
+
+	"golang.org/x/sys/unix"
+	"k8s.io/kubernetes/pkg/volume/util/fsquota/common"
+)
+
+var projectsFile = "/etc/projects"
+var projidFile = "/etc/projid"
+
+var projectsParseRegexp = regexp.MustCompilePOSIX("^([[:digit:]]+):(.*)$")
+var projidParseRegexp = regexp.MustCompilePOSIX("^([^#][^:]*):([[:digit:]]+)$")
+
+var quotaIDLock sync.RWMutex
+
+const maxUnusedQuotasToSearch = 128 // Don't go into an infinite loop searching for an unused quota
+
+type projectType struct {
+	isValid bool // False if we need to remove this line
+	id      common.QuotaID
+	data    string // Project name (projid) or directory (projects)
+	line    string
+}
+
+type projectsList struct {
+	projects []projectType
+	projid   []projectType
+}
+
+func projFilesAreOK() error {
+	if sf, err := os.Lstat(projectsFile); err != nil || sf.Mode().IsRegular() {
+		if sf, err := os.Lstat(projidFile); err != nil || sf.Mode().IsRegular() {
+			return nil
+		}
+		return fmt.Errorf("%s exists but is not a plain file, cannot continue", projidFile)
+	}
+	return fmt.Errorf("%s exists but is not a plain file, cannot continue", projectsFile)
+}
+
+func lockFile(file *os.File) error {
+	return unix.Flock(int(file.Fd()), unix.LOCK_EX)
+}
+
+func unlockFile(file *os.File) error {
+	return unix.Flock(int(file.Fd()), unix.LOCK_UN)
+}
+
+// openAndLockProjectFiles opens /etc/projects and /etc/projid locked.
+// Creates them if they don't exist
+func openAndLockProjectFiles() (*os.File, *os.File, error) {
+	// Make sure neither project-related file is a symlink!
+	if err := projFilesAreOK(); err != nil {
+		return nil, nil, fmt.Errorf("system project files failed verification: %v", err)
+	}
+	// We don't actually modify the original files; we create temporaries and
+	// move them over the originals
+	fProjects, err := os.OpenFile(projectsFile, os.O_RDONLY|os.O_CREATE, 0644)
+	if err != nil {
+		err = fmt.Errorf("unable to open %s: %v", projectsFile, err)
+		return nil, nil, err
+	}
+	fProjid, err := os.OpenFile(projidFile, os.O_RDONLY|os.O_CREATE, 0644)
+	if err == nil {
+		// Check once more, to ensure nothing got changed out from under us
+		if err = projFilesAreOK(); err == nil {
+			err = lockFile(fProjects)
+			if err == nil {
+				err = lockFile(fProjid)
+				if err == nil {
+					return fProjects, fProjid, nil
+				}
+				// Nothing useful we can do if we get an error here
+				err = fmt.Errorf("unable to lock %s: %v", projidFile, err)
+				unlockFile(fProjects)
+			} else {
+				err = fmt.Errorf("unable to lock %s: %v", projectsFile, err)
+			}
+		} else {
+			err = fmt.Errorf("system project files failed re-verification: %v", err)
+		}
+		fProjid.Close()
+	} else {
+		err = fmt.Errorf("unable to open %s: %v", projidFile, err)
+	}
+	fProjects.Close()
+	return nil, nil, err
+}
+
+func closeProjectFiles(fProjects *os.File, fProjid *os.File) error {
+	// Nothing useful we can do if either of these fail,
+	// but we have to close (and thereby unlock) the files anyway.
+	var err error
+	var err1 error
+	if fProjid != nil {
+		err = fProjid.Close()
+	}
+	if fProjects != nil {
+		err1 = fProjects.Close()
+	}
+	if err == nil {
+		return err1
+	}
+	return err
+}
+
+func parseProject(l string) projectType {
+	if match := projectsParseRegexp.FindStringSubmatch(l); match != nil {
+		i, err := strconv.Atoi(match[1])
+		if err == nil {
+			return projectType{true, common.QuotaID(i), match[2], l}
+		}
+	}
+	return projectType{true, common.BadQuotaID, "", l}
+}
+
+func parseProjid(l string) projectType {
+	if match := projidParseRegexp.FindStringSubmatch(l); match != nil {
+		i, err := strconv.Atoi(match[2])
+		if err == nil {
+			return projectType{true, common.QuotaID(i), match[1], l}
+		}
+	}
+	return projectType{true, common.BadQuotaID, "", l}
+}
+
+func parseProjFile(f *os.File, parser func(l string) projectType) []projectType {
+	var answer []projectType
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		answer = append(answer, parser(scanner.Text()))
+	}
+	return answer
+}
+
+func readProjectFiles(projects *os.File, projid *os.File) projectsList {
+	return projectsList{parseProjFile(projects, parseProject), parseProjFile(projid, parseProjid)}
+}
+
+func findAvailableQuota(path string, idMap map[common.QuotaID]bool) (common.QuotaID, error) {
+	unusedQuotasSearched := 0
+	for id := common.FirstQuota; true; id++ {
+		if _, ok := idMap[id]; !ok {
+			isInUse, err := getApplier(path).QuotaIDIsInUse(id)
+			if err != nil {
+				return common.BadQuotaID, err
+			} else if !isInUse {
+				return id, nil
+			}
+			unusedQuotasSearched++
+			if unusedQuotasSearched > maxUnusedQuotasToSearch {
+				break
+			}
+		}
+	}
+	return common.BadQuotaID, fmt.Errorf("cannot find available quota ID")
+}
+
+func addDirToProject(path string, id common.QuotaID, list *projectsList) (common.QuotaID, bool, error) {
+	idMap := make(map[common.QuotaID]bool)
+	for _, project := range list.projects {
+		if project.data == path {
+			if id != project.id {
+				return common.BadQuotaID, false, fmt.Errorf("attempt to reassign project ID for %s", path)
+			}
+			// Trying to reassign a directory to the project it's
+			// already in.  Maybe this should be an error, but for
+			// now treat it as an idempotent operation
+			return id, false, nil
+		}
+		idMap[project.id] = true
+	}
+	var needToAddProjid = true
+	for _, projid := range list.projid {
+		idMap[projid.id] = true
+		if projid.id == id && id != common.BadQuotaID {
+			needToAddProjid = false
+		}
+	}
+	var err error
+	if id == common.BadQuotaID {
+		id, err = findAvailableQuota(path, idMap)
+		if err != nil {
+			return common.BadQuotaID, false, err
+		}
+		needToAddProjid = true
+	}
+	if needToAddProjid {
+		name := fmt.Sprintf("volume%v", id)
+		line := fmt.Sprintf("%s:%v", name, id)
+		list.projid = append(list.projid, projectType{true, id, name, line})
+	}
+	line := fmt.Sprintf("%v:%s", id, path)
+	list.projects = append(list.projects, projectType{true, id, path, line})
+	return id, needToAddProjid, nil
+}
+
+func removeDirFromProject(path string, id common.QuotaID, list *projectsList) (bool, error) {
+	if id == common.BadQuotaID {
+		return false, fmt.Errorf("attempt to remove invalid quota ID from %s", path)
+	}
+	foundAt := -1
+	countByID := make(map[common.QuotaID]int)
+	for i, project := range list.projects {
+		if project.data == path {
+			if id != project.id {
+				return false, fmt.Errorf("attempting to remove quota ID %v from path %s, but expecting ID %v", id, path, project.id)
+			} else if foundAt != -1 {
+				return false, fmt.Errorf("found multiple quota IDs for path %s", path)
+			}
+			// Faster and easier than deleting an element
+			list.projects[i].isValid = false
+			foundAt = i
+		}
+		countByID[project.id]++
+	}
+	if foundAt == -1 {
+		return false, fmt.Errorf("cannot find quota associated with path %s", path)
+	}
+	if countByID[id] <= 1 {
+		// Removing the last entry means that we're no longer using
+		// the quota ID, so remove that as well
+		for i, projid := range list.projid {
+			if projid.id == id {
+				list.projid[i].isValid = false
+			}
+		}
+		return true, nil
+	}
+	return false, nil
+}
+
+func writeProjectFile(base *os.File, projects []projectType) (string, error) {
+	oname := base.Name()
+	stat, err := base.Stat()
+	if err != nil {
+		return "", err
+	}
+	mode := stat.Mode() & os.ModePerm
+	f, err := ioutil.TempFile(filepath.Dir(oname), filepath.Base(oname))
+	if err != nil {
+		return "", err
+	}
+	filename := f.Name()
+	if err := os.Chmod(filename, mode); err != nil {
+		return "", err
+	}
+	for _, proj := range projects {
+		if proj.isValid {
+			if _, err := f.WriteString(fmt.Sprintf("%s\n", proj.line)); err != nil {
+				f.Close()
+				os.Remove(filename)
+				return "", err
+			}
+		}
+	}
+	if err := f.Close(); err != nil {
+		os.Remove(filename)
+		return "", err
+	}
+	return filename, nil
+}
+
+func writeProjectFiles(fProjects *os.File, fProjid *os.File, writeProjid bool, list projectsList) error {
+	tmpProjects, err := writeProjectFile(fProjects, list.projects)
+	if err == nil {
+		// Ensure that both files are written before we try to rename either.
+		if writeProjid {
+			tmpProjid, err := writeProjectFile(fProjid, list.projid)
+			if err == nil {
+				err = os.Rename(tmpProjid, fProjid.Name())
+				if err != nil {
+					os.Remove(tmpProjid)
+				}
+			}
+		}
+		if err == nil {
+			err = os.Rename(tmpProjects, fProjects.Name())
+			if err == nil {
+				return nil
+			}
+			// We're in a bit of trouble here; at this
+			// point we've successfully renamed tmpProjid
+			// to the real thing, but renaming tmpProject
+			// to the real file failed.  There's not much we
+			// can do in this position.  Anything we could do
+			// to try to undo it would itself be likely to fail.
+		}
+		os.Remove(tmpProjects)
+	}
+	return fmt.Errorf("unable to write project files: %v", err)
+}
+
+func createProjectID(path string, ID common.QuotaID) (common.QuotaID, error) {
+	quotaIDLock.Lock()
+	defer quotaIDLock.Unlock()
+	fProjects, fProjid, err := openAndLockProjectFiles()
+	if err == nil {
+		defer closeProjectFiles(fProjects, fProjid)
+		list := readProjectFiles(fProjects, fProjid)
+		var writeProjid bool
+		ID, writeProjid, err = addDirToProject(path, ID, &list)
+		if err == nil && ID != common.BadQuotaID {
+			if err = writeProjectFiles(fProjects, fProjid, writeProjid, list); err == nil {
+				return ID, nil
+			}
+		}
+	}
+	return common.BadQuotaID, fmt.Errorf("createProjectID %s %v failed %v", path, ID, err)
+}
+
+func removeProjectID(path string, ID common.QuotaID) error {
+	if ID == common.BadQuotaID {
+		return fmt.Errorf("attempting to remove invalid quota ID %v", ID)
+	}
+	quotaIDLock.Lock()
+	defer quotaIDLock.Unlock()
+	fProjects, fProjid, err := openAndLockProjectFiles()
+	if err == nil {
+		defer closeProjectFiles(fProjects, fProjid)
+		list := readProjectFiles(fProjects, fProjid)
+		var writeProjid bool
+		writeProjid, err = removeDirFromProject(path, ID, &list)
+		if err == nil {
+			if err = writeProjectFiles(fProjects, fProjid, writeProjid, list); err == nil {
+				return nil
+			}
+		}
+	}
+	return fmt.Errorf("removeProjectID %s %v failed %v", path, ID, err)
+}

vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/quota.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fsquota
+
+import (
+	"k8s.io/mount-utils"
+
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/types"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/kubernetes/pkg/features"
+)
+
+// Interface -- quota interface
+type Interface interface {
+	// Does the path provided support quotas, and if so, what types
+	SupportsQuotas(m mount.Interface, path string) (bool, error)
+	// Assign a quota (picked by the quota mechanism) to a path,
+	// and return it.
+	AssignQuota(m mount.Interface, path string, poduid types.UID, bytes *resource.Quantity) error
+
+	// Get the quota-based storage consumption for the path
+	GetConsumption(path string) (*resource.Quantity, error)
+
+	// Get the quota-based inode consumption for the path
+	GetInodes(path string) (*resource.Quantity, error)
+
+	// Remove the quota from a path
+	// Implementations may assume that any data covered by the
+	// quota has already been removed.
+	ClearQuota(m mount.Interface, path string) error
+}
+
+func enabledQuotasForMonitoring() bool {
+	return utilfeature.DefaultFeatureGate.Enabled(features.LocalStorageCapacityIsolationFSQuotaMonitoring)
+}

vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/quota_linux.go

@@ -0,0 +1,452 @@
+// +build linux
+
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fsquota
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"k8s.io/klog/v2"
+	"k8s.io/mount-utils"
+
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	"k8s.io/kubernetes/pkg/volume/util/fsquota/common"
+)
+
+// Pod -> ID
+var podQuotaMap = make(map[types.UID]common.QuotaID)
+
+// Dir -> ID (for convenience)
+var dirQuotaMap = make(map[string]common.QuotaID)
+
+// ID -> pod
+var quotaPodMap = make(map[common.QuotaID]types.UID)
+
+// Directory -> pod
+var dirPodMap = make(map[string]types.UID)
+
+// Backing device -> applier
+// This is *not* cleaned up; its size will be bounded.
+var devApplierMap = make(map[string]common.LinuxVolumeQuotaApplier)
+
+// Directory -> applier
+var dirApplierMap = make(map[string]common.LinuxVolumeQuotaApplier)
+var dirApplierLock sync.RWMutex
+
+// Pod -> refcount
+var podDirCountMap = make(map[types.UID]int)
+
+// ID -> size
+var quotaSizeMap = make(map[common.QuotaID]int64)
+var quotaLock sync.RWMutex
+
+var supportsQuotasMap = make(map[string]bool)
+var supportsQuotasLock sync.RWMutex
+
+// Directory -> backingDev
+var backingDevMap = make(map[string]string)
+var backingDevLock sync.RWMutex
+
+var mountpointMap = make(map[string]string)
+var mountpointLock sync.RWMutex
+
+var providers = []common.LinuxVolumeQuotaProvider{
+	&common.VolumeProvider{},
+}
+
+// Separate the innards for ease of testing
+func detectBackingDevInternal(mountpoint string, mounts string) (string, error) {
+	file, err := os.Open(mounts)
+	if err != nil {
+		return "", err
+	}
+	defer file.Close()
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		match := common.MountParseRegexp.FindStringSubmatch(scanner.Text())
+		if match != nil {
+			device := match[1]
+			mount := match[2]
+			if mount == mountpoint {
+				return device, nil
+			}
+		}
+	}
+	return "", fmt.Errorf("couldn't find backing device for %s", mountpoint)
+}
+
+// detectBackingDev assumes that the mount point provided is valid
+func detectBackingDev(_ mount.Interface, mountpoint string) (string, error) {
+	return detectBackingDevInternal(mountpoint, common.MountsFile)
+}
+
+func clearBackingDev(path string) {
+	backingDevLock.Lock()
+	defer backingDevLock.Unlock()
+	delete(backingDevMap, path)
+}
+
+// Assumes that the path has been fully canonicalized
+// Breaking this up helps with testing
+func detectMountpointInternal(m mount.Interface, path string) (string, error) {
+	for path != "" && path != "/" {
+		// per k8s.io/mount-utils/mount_linux this detects all but
+		// a bind mount from one part of a mount to another.
+		// For our purposes that's fine; we simply want the "true"
+		// mount point
+		//
+		// IsNotMountPoint proved much more troublesome; it actually
+		// scans the mounts, and when a lot of mount/unmount
+		// activity takes place, it is not able to get a consistent
+		// view of /proc/self/mounts, causing it to time out and
+		// report incorrectly.
+		isNotMount, err := m.IsLikelyNotMountPoint(path)
+		if err != nil {
+			return "/", err
+		}
+		if !isNotMount {
+			return path, nil
+		}
+		path = filepath.Dir(path)
+	}
+	return "/", nil
+}
+
+func detectMountpoint(m mount.Interface, path string) (string, error) {
+	xpath, err := filepath.Abs(path)
+	if err != nil {
+		return "/", err
+	}
+	xpath, err = filepath.EvalSymlinks(xpath)
+	if err != nil {
+		return "/", err
+	}
+	if xpath, err = detectMountpointInternal(m, xpath); err == nil {
+		return xpath, nil
+	}
+	return "/", err
+}
+
+func clearMountpoint(path string) {
+	mountpointLock.Lock()
+	defer mountpointLock.Unlock()
+	delete(mountpointMap, path)
+}
+
+// getFSInfo Returns mountpoint and backing device
+// getFSInfo should cache the mountpoint and backing device for the
+// path.
+func getFSInfo(m mount.Interface, path string) (string, string, error) {
+	mountpointLock.Lock()
+	defer mountpointLock.Unlock()
+
+	backingDevLock.Lock()
+	defer backingDevLock.Unlock()
+
+	var err error
+
+	mountpoint, okMountpoint := mountpointMap[path]
+	if !okMountpoint {
+		mountpoint, err = detectMountpoint(m, path)
+		if err != nil {
+			return "", "", fmt.Errorf("cannot determine mountpoint for %s: %v", path, err)
+		}
+	}
+
+	backingDev, okBackingDev := backingDevMap[path]
+	if !okBackingDev {
+		backingDev, err = detectBackingDev(m, mountpoint)
+		if err != nil {
+			return "", "", fmt.Errorf("cannot determine backing device for %s: %v", path, err)
+		}
+	}
+	mountpointMap[path] = mountpoint
+	backingDevMap[path] = backingDev
+	return mountpoint, backingDev, nil
+}
+
+func clearFSInfo(path string) {
+	clearMountpoint(path)
+	clearBackingDev(path)
+}
+
+func getApplier(path string) common.LinuxVolumeQuotaApplier {
+	dirApplierLock.Lock()
+	defer dirApplierLock.Unlock()
+	return dirApplierMap[path]
+}
+
+func setApplier(path string, applier common.LinuxVolumeQuotaApplier) {
+	dirApplierLock.Lock()
+	defer dirApplierLock.Unlock()
+	dirApplierMap[path] = applier
+}
+
+func clearApplier(path string) {
+	dirApplierLock.Lock()
+	defer dirApplierLock.Unlock()
+	delete(dirApplierMap, path)
+}
+
+func setQuotaOnDir(path string, id common.QuotaID, bytes int64) error {
+	return getApplier(path).SetQuotaOnDir(path, id, bytes)
+}
+
+func getQuotaOnDir(m mount.Interface, path string) (common.QuotaID, error) {
+	_, _, err := getFSInfo(m, path)
+	if err != nil {
+		return common.BadQuotaID, err
+	}
+	return getApplier(path).GetQuotaOnDir(path)
+}
+
+func clearQuotaOnDir(m mount.Interface, path string) error {
+	// Since we may be called without path being in the map,
+	// we explicitly have to check in this case.
+	klog.V(4).Infof("clearQuotaOnDir %s", path)
+	supportsQuotas, err := SupportsQuotas(m, path)
+	if err != nil {
+		// Log-and-continue instead of returning an error for now
+		// due to unspecified backwards compatibility concerns (a subject to revise)
+		klog.V(3).Infof("Attempt to check for quota support failed: %v", err)
+	}
+	if !supportsQuotas {
+		return nil
+	}
+	projid, err := getQuotaOnDir(m, path)
+	if err == nil && projid != common.BadQuotaID {
+		// This means that we have a quota on the directory but
+		// we can't clear it.  That's not good.
+		err = setQuotaOnDir(path, projid, 0)
+		if err != nil {
+			klog.V(3).Infof("Attempt to clear quota failed: %v", err)
+		}
+		// Even if clearing the quota failed, we still need to
+		// try to remove the project ID, or that may be left dangling.
+		err1 := removeProjectID(path, projid)
+		if err1 != nil {
+			klog.V(3).Infof("Attempt to remove quota ID from system files failed: %v", err1)
+		}
+		clearFSInfo(path)
+		if err != nil {
+			return err
+		}
+		return err1
+	}
+	// If we couldn't get a quota, that's fine -- there may
+	// never have been one, and we have no way to know otherwise
+	klog.V(3).Infof("clearQuotaOnDir fails %v", err)
+	return nil
+}
+
+// SupportsQuotas -- Does the path support quotas
+// Cache the applier for paths that support quotas.  For paths that don't,
+// don't cache the result because nothing will clean it up.
+// However, do cache the device->applier map; the number of devices
+// is bounded.
+func SupportsQuotas(m mount.Interface, path string) (bool, error) {
+	if !enabledQuotasForMonitoring() {
+		klog.V(3).Info("SupportsQuotas called, but quotas disabled")
+		return false, nil
+	}
+	supportsQuotasLock.Lock()
+	defer supportsQuotasLock.Unlock()
+	if supportsQuotas, ok := supportsQuotasMap[path]; ok {
+		return supportsQuotas, nil
+	}
+	mount, dev, err := getFSInfo(m, path)
+	if err != nil {
+		return false, err
+	}
+	// Do we know about this device?
+	applier, ok := devApplierMap[mount]
+	if !ok {
+		for _, provider := range providers {
+			if applier = provider.GetQuotaApplier(mount, dev); applier != nil {
+				devApplierMap[mount] = applier
+				break
+			}
+		}
+	}
+	if applier != nil {
+		supportsQuotasMap[path] = true
+		setApplier(path, applier)
+		return true, nil
+	}
+	delete(backingDevMap, path)
+	delete(mountpointMap, path)
+	return false, nil
+}
+
+// AssignQuota -- assign a quota to the specified directory.
+// AssignQuota chooses the quota ID based on the pod UID and path.
+// If the pod UID is identical to another one known, it may (but presently
+// doesn't) choose the same quota ID as other volumes in the pod.
+//lint:ignore SA4009 poduid is overwritten by design, see comment below
+func AssignQuota(m mount.Interface, path string, poduid types.UID, bytes *resource.Quantity) error {
+	if bytes == nil {
+		return fmt.Errorf("attempting to assign null quota to %s", path)
+	}
+	ibytes := bytes.Value()
+	if ok, err := SupportsQuotas(m, path); !ok {
+		return fmt.Errorf("quotas not supported on %s: %v", path, err)
+	}
+	quotaLock.Lock()
+	defer quotaLock.Unlock()
+	// Current policy is to set individual quotas on each volumes.
+	// If we decide later that we want to assign one quota for all
+	// volumes in a pod, we can simply remove this line of code.
+	// If and when we decide permanently that we're going to adopt
+	// one quota per volume, we can rip all of the pod code out.
+	poduid = types.UID(uuid.NewUUID())
+	if pod, ok := dirPodMap[path]; ok && pod != poduid {
+		return fmt.Errorf("requesting quota on existing directory %s but different pod %s %s", path, pod, poduid)
+	}
+	oid, ok := podQuotaMap[poduid]
+	if ok {
+		if quotaSizeMap[oid] != ibytes {
+			return fmt.Errorf("requesting quota of different size: old %v new %v", quotaSizeMap[oid], bytes)
+		}
+	} else {
+		oid = common.BadQuotaID
+	}
+	id, err := createProjectID(path, oid)
+	if err == nil {
+		if oid != common.BadQuotaID && oid != id {
+			return fmt.Errorf("attempt to reassign quota %v to %v", oid, id)
+		}
+		// When enforcing quotas are enabled, we'll condition this
+		// on their being disabled also.
+		if ibytes > 0 {
+			ibytes = -1
+		}
+		if err = setQuotaOnDir(path, id, ibytes); err == nil {
+			quotaPodMap[id] = poduid
+			quotaSizeMap[id] = ibytes
+			podQuotaMap[poduid] = id
+			dirQuotaMap[path] = id
+			dirPodMap[path] = poduid
+			podDirCountMap[poduid]++
+			klog.V(4).Infof("Assigning quota ID %d (%d) to %s", id, ibytes, path)
+			return nil
+		}
+		removeProjectID(path, id)
+	}
+	return fmt.Errorf("assign quota FAILED %v", err)
+}
+
+// GetConsumption -- retrieve the consumption (in bytes) of the directory
+func GetConsumption(path string) (*resource.Quantity, error) {
+	// Note that we actually need to hold the lock at least through
+	// running the quota command, so it can't get recycled behind our back
+	quotaLock.Lock()
+	defer quotaLock.Unlock()
+	applier := getApplier(path)
+	// No applier means directory is not under quota management
+	if applier == nil {
+		return nil, nil
+	}
+	ibytes, err := applier.GetConsumption(path, dirQuotaMap[path])
+	if err != nil {
+		return nil, err
+	}
+	return resource.NewQuantity(ibytes, resource.DecimalSI), nil
+}
+
+// GetInodes -- retrieve the number of inodes in use under the directory
+func GetInodes(path string) (*resource.Quantity, error) {
+	// Note that we actually need to hold the lock at least through
+	// running the quota command, so it can't get recycled behind our back
+	quotaLock.Lock()
+	defer quotaLock.Unlock()
+	applier := getApplier(path)
+	// No applier means directory is not under quota management
+	if applier == nil {
+		return nil, nil
+	}
+	inodes, err := applier.GetInodes(path, dirQuotaMap[path])
+	if err != nil {
+		return nil, err
+	}
+	return resource.NewQuantity(inodes, resource.DecimalSI), nil
+}
+
+// ClearQuota -- remove the quota assigned to a directory
+func ClearQuota(m mount.Interface, path string) error {
+	klog.V(3).Infof("ClearQuota %s", path)
+	if !enabledQuotasForMonitoring() {
+		return fmt.Errorf("clearQuota called, but quotas disabled")
+	}
+	quotaLock.Lock()
+	defer quotaLock.Unlock()
+	poduid, ok := dirPodMap[path]
+	if !ok {
+		// Nothing in the map either means that there was no
+		// quota to begin with or that we're clearing a
+		// stale directory, so if we find a quota, just remove it.
+		// The process of clearing the quota requires that an applier
+		// be found, which needs to be cleaned up.
+		defer delete(supportsQuotasMap, path)
+		defer clearApplier(path)
+		return clearQuotaOnDir(m, path)
+	}
+	_, ok = podQuotaMap[poduid]
+	if !ok {
+		return fmt.Errorf("clearQuota: No quota available for %s", path)
+	}
+	projid, err := getQuotaOnDir(m, path)
+	if err != nil {
+		// Log-and-continue instead of returning an error for now
+		// due to unspecified backwards compatibility concerns (a subject to revise)
+		klog.V(3).Infof("Attempt to check quota ID %v on dir %s failed: %v", dirQuotaMap[path], path, err)
+	}
+	if projid != dirQuotaMap[path] {
+		return fmt.Errorf("expected quota ID %v on dir %s does not match actual %v", dirQuotaMap[path], path, projid)
+	}
+	count, ok := podDirCountMap[poduid]
+	if count <= 1 || !ok {
+		err = clearQuotaOnDir(m, path)
+		// This error should be noted; we still need to clean up
+		// and otherwise handle in the same way.
+		if err != nil {
+			klog.V(3).Infof("Unable to clear quota %v %s: %v", dirQuotaMap[path], path, err)
+		}
+		delete(quotaSizeMap, podQuotaMap[poduid])
+		delete(quotaPodMap, podQuotaMap[poduid])
+		delete(podDirCountMap, poduid)
+		delete(podQuotaMap, poduid)
+	} else {
+		err = removeProjectID(path, projid)
+		podDirCountMap[poduid]--
+		klog.V(4).Infof("Not clearing quota for pod %s; still %v dirs outstanding", poduid, podDirCountMap[poduid])
+	}
+	delete(dirPodMap, path)
+	delete(dirQuotaMap, path)
+	delete(supportsQuotasMap, path)
+	clearApplier(path)
+	if err != nil {
+		return fmt.Errorf("unable to clear quota for %s: %v", path, err)
+	}
+	return nil
+}

vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/quota_unsupported.go

@@ -0,0 +1,58 @@
+// +build !linux
+
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fsquota
+
+import (
+	"errors"
+
+	"k8s.io/mount-utils"
+
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+// Dummy quota implementation for systems that do not implement support
+// for volume quotas
+
+var errNotImplemented = errors.New("not implemented")
+
+// SupportsQuotas -- dummy implementation
+func SupportsQuotas(_ mount.Interface, _ string) (bool, error) {
+	return false, errNotImplemented
+}
+
+// AssignQuota -- dummy implementation
+func AssignQuota(_ mount.Interface, _ string, _ types.UID, _ *resource.Quantity) error {
+	return errNotImplemented
+}
+
+// GetConsumption -- dummy implementation
+func GetConsumption(_ string) (*resource.Quantity, error) {
+	return nil, errNotImplemented
+}
+
+// GetInodes -- dummy implementation
+func GetInodes(_ string) (*resource.Quantity, error) {
+	return nil, errNotImplemented
+}
+
+// ClearQuota -- dummy implementation
+func ClearQuota(_ mount.Interface, _ string) error {
+	return errNotImplemented
+}

vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/BUILD

@@ -0,0 +1,65 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "fake_hostutil.go",
+        "hostutil.go",
+        "hostutil_linux.go",
+        "hostutil_unsupported.go",
+        "hostutil_windows.go",
+    ],
+    importpath = "k8s.io/kubernetes/pkg/volume/util/hostutil",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//staging/src/k8s.io/mount-utils:go_default_library",
+    ] + select({
+        "@io_bazel_rules_go//go/platform:android": [
+            "//vendor/golang.org/x/sys/unix:go_default_library",
+            "//vendor/k8s.io/klog/v2:go_default_library",
+            "//vendor/k8s.io/utils/path:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:linux": [
+            "//vendor/golang.org/x/sys/unix:go_default_library",
+            "//vendor/k8s.io/klog/v2:go_default_library",
+            "//vendor/k8s.io/utils/path:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:windows": [
+            "//vendor/k8s.io/klog/v2:go_default_library",
+            "//vendor/k8s.io/utils/path:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "hostutil_linux_test.go",
+        "hostutil_windows_test.go",
+    ],
+    embed = [":go_default_library"],
+    deps = select({
+        "@io_bazel_rules_go//go/platform:android": [
+            "//vendor/k8s.io/utils/exec:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:linux": [
+            "//vendor/k8s.io/utils/exec:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/fake_hostutil.go

@@ -0,0 +1,118 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hostutil
+
+import (
+	"errors"
+	"os"
+	"sync"
+
+	"k8s.io/mount-utils"
+)
+
+// FakeHostUtil is a fake HostUtils implementation for testing
+type FakeHostUtil struct {
+	MountPoints []mount.MountPoint
+	Filesystem  map[string]FileType
+
+	mutex sync.Mutex
+}
+
+// NewFakeHostUtil returns a struct that implements the HostUtils interface
+// for testing
+// TODO: no callers were initializing the struct with any MountPoints. Check
+// if those are still being used by any callers and if MountPoints still need
+// to be a part of the struct.
+func NewFakeHostUtil(fs map[string]FileType) *FakeHostUtil {
+	return &FakeHostUtil{
+		Filesystem: fs,
+	}
+}
+
+// Compile-time check to make sure FakeHostUtil implements interface
+var _ HostUtils = &FakeHostUtil{}
+
+// DeviceOpened checks if block device referenced by pathname is in use by
+// checking if is listed as a device in the in-memory mountpoint table.
+func (hu *FakeHostUtil) DeviceOpened(pathname string) (bool, error) {
+	hu.mutex.Lock()
+	defer hu.mutex.Unlock()
+
+	for _, mp := range hu.MountPoints {
+		if mp.Device == pathname {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// PathIsDevice always returns true
+func (hu *FakeHostUtil) PathIsDevice(pathname string) (bool, error) {
+	return true, nil
+}
+
+// GetDeviceNameFromMount given a mount point, find the volume id
+func (hu *FakeHostUtil) GetDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) {
+	return getDeviceNameFromMount(mounter, mountPath, pluginMountDir)
+}
+
+// MakeRShared checks if path is shared and bind-mounts it as rshared if needed.
+// No-op for testing
+func (hu *FakeHostUtil) MakeRShared(path string) error {
+	return nil
+}
+
+// GetFileType checks for file/directory/socket/block/character devices.
+// Defaults to Directory if otherwise unspecified.
+func (hu *FakeHostUtil) GetFileType(pathname string) (FileType, error) {
+	if t, ok := hu.Filesystem[pathname]; ok {
+		return t, nil
+	}
+	return FileType("Directory"), nil
+}
+
+// PathExists checks if pathname exists.
+func (hu *FakeHostUtil) PathExists(pathname string) (bool, error) {
+	if _, ok := hu.Filesystem[pathname]; ok {
+		return true, nil
+	}
+	return false, nil
+}
+
+// EvalHostSymlinks returns the path name after evaluating symlinks.
+// No-op for testing
+func (hu *FakeHostUtil) EvalHostSymlinks(pathname string) (string, error) {
+	return pathname, nil
+}
+
+// GetOwner returns the integer ID for the user and group of the given path
+// Not implemented for testing
+func (hu *FakeHostUtil) GetOwner(pathname string) (int64, int64, error) {
+	return -1, -1, errors.New("GetOwner not implemented")
+}
+
+// GetSELinuxSupport tests if pathname is on a mount that supports SELinux.
+// Not implemented for testing
+func (hu *FakeHostUtil) GetSELinuxSupport(pathname string) (bool, error) {
+	return false, errors.New("GetSELinuxSupport not implemented")
+}
+
+// GetMode returns permissions of pathname.
+// Not implemented for testing
+func (hu *FakeHostUtil) GetMode(pathname string) (os.FileMode, error) {
+	return 0, errors.New("not implemented")
+}

vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/hostutil.go

@@ -0,0 +1,109 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hostutil
+
+import (
+	"fmt"
+	"os"
+
+	"k8s.io/mount-utils"
+)
+
+// FileType enumerates the known set of possible file types.
+type FileType string
+
+const (
+	// FileTypeBlockDev defines a constant for the block device FileType.
+	FileTypeBlockDev FileType = "BlockDevice"
+	// FileTypeCharDev defines a constant for the character device FileType.
+	FileTypeCharDev FileType = "CharDevice"
+	// FileTypeDirectory defines a constant for the directory FileType.
+	FileTypeDirectory FileType = "Directory"
+	// FileTypeFile defines a constant for the file FileType.
+	FileTypeFile FileType = "File"
+	// FileTypeSocket defines a constant for the socket FileType.
+	FileTypeSocket FileType = "Socket"
+	// FileTypeUnknown defines a constant for an unknown FileType.
+	FileTypeUnknown FileType = ""
+)
+
+// HostUtils defines the set of methods for interacting with paths on a host.
+type HostUtils interface {
+	// DeviceOpened determines if the device (e.g. /dev/sdc) is in use elsewhere
+	// on the system, i.e. still mounted.
+	DeviceOpened(pathname string) (bool, error)
+	// PathIsDevice determines if a path is a device.
+	PathIsDevice(pathname string) (bool, error)
+	// GetDeviceNameFromMount finds the device name by checking the mount path
+	// to get the global mount path within its plugin directory.
+	GetDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error)
+	// MakeRShared checks that given path is on a mount with 'rshared' mount
+	// propagation. If not, it bind-mounts the path as rshared.
+	MakeRShared(path string) error
+	// GetFileType checks for file/directory/socket/block/character devices.
+	GetFileType(pathname string) (FileType, error)
+	// PathExists tests if the given path already exists
+	// Error is returned on any other error than "file not found".
+	PathExists(pathname string) (bool, error)
+	// EvalHostSymlinks returns the path name after evaluating symlinks.
+	EvalHostSymlinks(pathname string) (string, error)
+	// GetOwner returns the integer ID for the user and group of the given path
+	GetOwner(pathname string) (int64, int64, error)
+	// GetSELinuxSupport returns true if given path is on a mount that supports
+	// SELinux.
+	GetSELinuxSupport(pathname string) (bool, error)
+	// GetMode returns permissions of the path.
+	GetMode(pathname string) (os.FileMode, error)
+}
+
+// Compile-time check to ensure all HostUtil implementations satisfy
+// the Interface.
+var _ HostUtils = &HostUtil{}
+
+// getFileType checks for file/directory/socket and block/character devices.
+func getFileType(pathname string) (FileType, error) {
+	var pathType FileType
+	info, err := os.Stat(pathname)
+	if os.IsNotExist(err) {
+		return pathType, fmt.Errorf("path %q does not exist", pathname)
+	}
+	// err in call to os.Stat
+	if err != nil {
+		return pathType, err
+	}
+
+	// checks whether the mode is the target mode.
+	isSpecificMode := func(mode, targetMode os.FileMode) bool {
+		return mode&targetMode == targetMode
+	}
+
+	mode := info.Mode()
+	if mode.IsDir() {
+		return FileTypeDirectory, nil
+	} else if mode.IsRegular() {
+		return FileTypeFile, nil
+	} else if isSpecificMode(mode, os.ModeSocket) {
+		return FileTypeSocket, nil
+	} else if isSpecificMode(mode, os.ModeDevice) {
+		if isSpecificMode(mode, os.ModeCharDevice) {
+			return FileTypeCharDev, nil
+		}
+		return FileTypeBlockDev, nil
+	}
+
+	return pathType, fmt.Errorf("only recognise file, directory, socket, block device and character device")
+}

vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/hostutil_linux.go

@@ -0,0 +1,291 @@
+// +build linux
+
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hostutil
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+	"k8s.io/klog/v2"
+	"k8s.io/mount-utils"
+	utilpath "k8s.io/utils/path"
+)
+
+const (
+	// Location of the mountinfo file
+	procMountInfoPath = "/proc/self/mountinfo"
+)
+
+// HostUtil implements HostUtils for Linux platforms.
+type HostUtil struct {
+}
+
+// NewHostUtil returns a struct that implements the HostUtils interface on
+// linux platforms
+func NewHostUtil() *HostUtil {
+	return &HostUtil{}
+}
+
+// DeviceOpened checks if block device in use by calling Open with O_EXCL flag.
+// If pathname is not a device, log and return false with nil error.
+// If open returns errno EBUSY, return true with nil error.
+// If open returns nil, return false with nil error.
+// Otherwise, return false with error
+func (hu *HostUtil) DeviceOpened(pathname string) (bool, error) {
+	return ExclusiveOpenFailsOnDevice(pathname)
+}
+
+// PathIsDevice uses FileInfo returned from os.Stat to check if path refers
+// to a device.
+func (hu *HostUtil) PathIsDevice(pathname string) (bool, error) {
+	pathType, err := hu.GetFileType(pathname)
+	isDevice := pathType == FileTypeCharDev || pathType == FileTypeBlockDev
+	return isDevice, err
+}
+
+// ExclusiveOpenFailsOnDevice is shared with NsEnterMounter
+func ExclusiveOpenFailsOnDevice(pathname string) (bool, error) {
+	var isDevice bool
+	finfo, err := os.Stat(pathname)
+	if os.IsNotExist(err) {
+		isDevice = false
+	}
+	// err in call to os.Stat
+	if err != nil {
+		return false, fmt.Errorf(
+			"PathIsDevice failed for path %q: %v",
+			pathname,
+			err)
+	}
+	// path refers to a device
+	if finfo.Mode()&os.ModeDevice != 0 {
+		isDevice = true
+	}
+
+	if !isDevice {
+		klog.Errorf("Path %q is not referring to a device.", pathname)
+		return false, nil
+	}
+	fd, errno := unix.Open(pathname, unix.O_RDONLY|unix.O_EXCL|unix.O_CLOEXEC, 0)
+	// If the device is in use, open will return an invalid fd.
+	// When this happens, it is expected that Close will fail and throw an error.
+	defer unix.Close(fd)
+	if errno == nil {
+		// device not in use
+		return false, nil
+	} else if errno == unix.EBUSY {
+		// device is in use
+		return true, nil
+	}
+	// error during call to Open
+	return false, errno
+}
+
+// GetDeviceNameFromMount given a mount point, find the device name from its global mount point
+func (hu *HostUtil) GetDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) {
+	return getDeviceNameFromMount(mounter, mountPath, pluginMountDir)
+}
+
+// getDeviceNameFromMountLinux find the device name from /proc/mounts in which
+// the mount path reference should match the given plugin mount directory. In case no mount path reference
+// matches, returns the volume name taken from its given mountPath
+func getDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) {
+	refs, err := mounter.GetMountRefs(mountPath)
+	if err != nil {
+		klog.V(4).Infof("GetMountRefs failed for mount path %q: %v", mountPath, err)
+		return "", err
+	}
+	if len(refs) == 0 {
+		klog.V(4).Infof("Directory %s is not mounted", mountPath)
+		return "", fmt.Errorf("directory %s is not mounted", mountPath)
+	}
+	for _, ref := range refs {
+		if strings.HasPrefix(ref, pluginMountDir) {
+			volumeID, err := filepath.Rel(pluginMountDir, ref)
+			if err != nil {
+				klog.Errorf("Failed to get volume id from mount %s - %v", mountPath, err)
+				return "", err
+			}
+			return volumeID, nil
+		}
+	}
+
+	return path.Base(mountPath), nil
+}
+
+// MakeRShared checks that given path is on a mount with 'rshared' mount
+// propagation. If not, it bind-mounts the path as rshared.
+func (hu *HostUtil) MakeRShared(path string) error {
+	return DoMakeRShared(path, procMountInfoPath)
+}
+
+// GetFileType checks for file/directory/socket/block/character devices.
+func (hu *HostUtil) GetFileType(pathname string) (FileType, error) {
+	return getFileType(pathname)
+}
+
+// PathExists tests if the given path already exists
+// Error is returned on any other error than "file not found".
+func (hu *HostUtil) PathExists(pathname string) (bool, error) {
+	return utilpath.Exists(utilpath.CheckFollowSymlink, pathname)
+}
+
+// EvalHostSymlinks returns the path name after evaluating symlinks.
+// TODO once the nsenter implementation is removed, this method can be removed
+// from the interface and filepath.EvalSymlinks used directly
+func (hu *HostUtil) EvalHostSymlinks(pathname string) (string, error) {
+	return filepath.EvalSymlinks(pathname)
+}
+
+// FindMountInfo returns the mount info on the given path.
+func (hu *HostUtil) FindMountInfo(path string) (mount.MountInfo, error) {
+	return findMountInfo(path, procMountInfoPath)
+}
+
+// isShared returns true, if given path is on a mount point that has shared
+// mount propagation.
+func isShared(mount string, mountInfoPath string) (bool, error) {
+	info, err := findMountInfo(mount, mountInfoPath)
+	if err != nil {
+		return false, err
+	}
+
+	// parse optional parameters
+	for _, opt := range info.OptionalFields {
+		if strings.HasPrefix(opt, "shared:") {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func findMountInfo(path, mountInfoPath string) (mount.MountInfo, error) {
+	infos, err := mount.ParseMountInfo(mountInfoPath)
+	if err != nil {
+		return mount.MountInfo{}, err
+	}
+
+	// process /proc/xxx/mountinfo in backward order and find the first mount
+	// point that is prefix of 'path' - that's the mount where path resides
+	var info *mount.MountInfo
+	for i := len(infos) - 1; i >= 0; i-- {
+		if mount.PathWithinBase(path, infos[i].MountPoint) {
+			info = &infos[i]
+			break
+		}
+	}
+	if info == nil {
+		return mount.MountInfo{}, fmt.Errorf("cannot find mount point for %q", path)
+	}
+	return *info, nil
+}
+
+// DoMakeRShared is common implementation of MakeRShared on Linux. It checks if
+// path is shared and bind-mounts it as rshared if needed. mountCmd and
+// mountArgs are expected to contain mount-like command, DoMakeRShared will add
+// '--bind <path> <path>' and '--make-rshared <path>' to mountArgs.
+func DoMakeRShared(path string, mountInfoFilename string) error {
+	shared, err := isShared(path, mountInfoFilename)
+	if err != nil {
+		return err
+	}
+	if shared {
+		klog.V(4).Infof("Directory %s is already on a shared mount", path)
+		return nil
+	}
+
+	klog.V(2).Infof("Bind-mounting %q with shared mount propagation", path)
+	// mount --bind /var/lib/kubelet /var/lib/kubelet
+	if err := syscall.Mount(path, path, "" /*fstype*/, syscall.MS_BIND, "" /*data*/); err != nil {
+		return fmt.Errorf("failed to bind-mount %s: %v", path, err)
+	}
+
+	// mount --make-rshared /var/lib/kubelet
+	if err := syscall.Mount(path, path, "" /*fstype*/, syscall.MS_SHARED|syscall.MS_REC, "" /*data*/); err != nil {
+		return fmt.Errorf("failed to make %s rshared: %v", path, err)
+	}
+
+	return nil
+}
+
+// GetSELinux is common implementation of GetSELinuxSupport on Linux.
+func GetSELinux(path string, mountInfoFilename string) (bool, error) {
+	info, err := findMountInfo(path, mountInfoFilename)
+	if err != nil {
+		return false, err
+	}
+
+	// "seclabel" can be both in mount options and super options.
+	for _, opt := range info.SuperOptions {
+		if opt == "seclabel" {
+			return true, nil
+		}
+	}
+	for _, opt := range info.MountOptions {
+		if opt == "seclabel" {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// GetSELinuxSupport returns true if given path is on a mount that supports
+// SELinux.
+func (hu *HostUtil) GetSELinuxSupport(pathname string) (bool, error) {
+	return GetSELinux(pathname, procMountInfoPath)
+}
+
+// GetOwner returns the integer ID for the user and group of the given path
+func (hu *HostUtil) GetOwner(pathname string) (int64, int64, error) {
+	realpath, err := filepath.EvalSymlinks(pathname)
+	if err != nil {
+		return -1, -1, err
+	}
+	return GetOwnerLinux(realpath)
+}
+
+// GetMode returns permissions of the path.
+func (hu *HostUtil) GetMode(pathname string) (os.FileMode, error) {
+	return GetModeLinux(pathname)
+}
+
+// GetOwnerLinux is shared between Linux and NsEnterMounter
+// pathname must already be evaluated for symlinks
+func GetOwnerLinux(pathname string) (int64, int64, error) {
+	info, err := os.Stat(pathname)
+	if err != nil {
+		return -1, -1, err
+	}
+	stat := info.Sys().(*syscall.Stat_t)
+	return int64(stat.Uid), int64(stat.Gid), nil
+}
+
+// GetModeLinux is shared between Linux and NsEnterMounter
+func GetModeLinux(pathname string) (os.FileMode, error) {
+	info, err := os.Stat(pathname)
+	if err != nil {
+		return 0, err
+	}
+	return info.Mode(), nil
+}

vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/hostutil_unsupported.go

@@ -0,0 +1,102 @@
+// +build !linux,!windows
+
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hostutil
+
+import (
+	"errors"
+	"os"
+
+	"k8s.io/mount-utils"
+)
+
+// HostUtil is an HostUtils implementation that allows compilation on
+// unsupported platforms
+type HostUtil struct{}
+
+// NewHostUtil returns a struct that implements the HostUtils interface on
+// unsupported platforms
+func NewHostUtil() *HostUtil {
+	return &HostUtil{}
+}
+
+var errUnsupported = errors.New("volume/util/hostutil on this platform is not supported")
+
+// DeviceOpened always returns an error on unsupported platforms
+func (hu *HostUtil) DeviceOpened(pathname string) (bool, error) {
+	return false, errUnsupported
+}
+
+// PathIsDevice always returns an error on unsupported platforms
+func (hu *HostUtil) PathIsDevice(pathname string) (bool, error) {
+	return true, errUnsupported
+}
+
+// GetDeviceNameFromMount always returns an error on unsupported platforms
+func (hu *HostUtil) GetDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) {
+	return getDeviceNameFromMount(mounter, mountPath, pluginMountDir)
+}
+
+// MakeRShared always returns an error on unsupported platforms
+func (hu *HostUtil) MakeRShared(path string) error {
+	return errUnsupported
+}
+
+// GetFileType always returns an error on unsupported platforms
+func (hu *HostUtil) GetFileType(pathname string) (FileType, error) {
+	return FileType("fake"), errUnsupported
+}
+
+// MakeFile always returns an error on unsupported platforms
+func (hu *HostUtil) MakeFile(pathname string) error {
+	return errUnsupported
+}
+
+// MakeDir always returns an error on unsupported platforms
+func (hu *HostUtil) MakeDir(pathname string) error {
+	return errUnsupported
+}
+
+// PathExists always returns an error on unsupported platforms
+func (hu *HostUtil) PathExists(pathname string) (bool, error) {
+	return true, errUnsupported
+}
+
+// EvalHostSymlinks always returns an error on unsupported platforms
+func (hu *HostUtil) EvalHostSymlinks(pathname string) (string, error) {
+	return "", errUnsupported
+}
+
+// GetOwner always returns an error on unsupported platforms
+func (hu *HostUtil) GetOwner(pathname string) (int64, int64, error) {
+	return -1, -1, errUnsupported
+}
+
+// GetSELinuxSupport always returns an error on unsupported platforms
+func (hu *HostUtil) GetSELinuxSupport(pathname string) (bool, error) {
+	return false, errUnsupported
+}
+
+//GetMode always returns an error on unsupported platforms
+func (hu *HostUtil) GetMode(pathname string) (os.FileMode, error) {
+	return 0, errUnsupported
+}
+
+func getDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) {
+	return "", errUnsupported
+}

vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/hostutil_windows.go

@@ -0,0 +1,124 @@
+// +build windows
+
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hostutil
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"k8s.io/klog/v2"
+	"k8s.io/mount-utils"
+	utilpath "k8s.io/utils/path"
+)
+
+// HostUtil implements HostUtils for Windows platforms.
+type HostUtil struct{}
+
+// NewHostUtil returns a struct that implements HostUtils on Windows platforms
+func NewHostUtil() *HostUtil {
+	return &HostUtil{}
+}
+
+// GetDeviceNameFromMount given a mnt point, find the device
+func (hu *HostUtil) GetDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) {
+	return getDeviceNameFromMount(mounter, mountPath, pluginMountDir)
+}
+
+// getDeviceNameFromMount find the device(drive) name in which
+// the mount path reference should match the given plugin mount directory. In case no mount path reference
+// matches, returns the volume name taken from its given mountPath
+func getDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) {
+	refs, err := mounter.GetMountRefs(mountPath)
+	if err != nil {
+		klog.V(4).Infof("GetMountRefs failed for mount path %q: %v", mountPath, err)
+		return "", err
+	}
+	if len(refs) == 0 {
+		return "", fmt.Errorf("directory %s is not mounted", mountPath)
+	}
+	basemountPath := mount.NormalizeWindowsPath(pluginMountDir)
+	for _, ref := range refs {
+		if strings.Contains(ref, basemountPath) {
+			volumeID, err := filepath.Rel(mount.NormalizeWindowsPath(basemountPath), ref)
+			if err != nil {
+				klog.Errorf("Failed to get volume id from mount %s - %v", mountPath, err)
+				return "", err
+			}
+			return volumeID, nil
+		}
+	}
+
+	return path.Base(mountPath), nil
+}
+
+// DeviceOpened determines if the device is in use elsewhere
+func (hu *HostUtil) DeviceOpened(pathname string) (bool, error) {
+	return false, nil
+}
+
+// PathIsDevice determines if a path is a device.
+func (hu *HostUtil) PathIsDevice(pathname string) (bool, error) {
+	return false, nil
+}
+
+// MakeRShared checks that given path is on a mount with 'rshared' mount
+// propagation. Empty implementation here.
+func (hu *HostUtil) MakeRShared(path string) error {
+	return nil
+}
+
+// GetFileType checks for sockets/block/character devices
+func (hu *(HostUtil)) GetFileType(pathname string) (FileType, error) {
+	return getFileType(pathname)
+}
+
+// PathExists checks whether the path exists
+func (hu *HostUtil) PathExists(pathname string) (bool, error) {
+	return utilpath.Exists(utilpath.CheckFollowSymlink, pathname)
+}
+
+// EvalHostSymlinks returns the path name after evaluating symlinks
+func (hu *HostUtil) EvalHostSymlinks(pathname string) (string, error) {
+	return filepath.EvalSymlinks(pathname)
+}
+
+// GetOwner returns the integer ID for the user and group of the given path
+// Note that on windows, it always returns 0. We actually don't set Group on
+// windows platform, see SetVolumeOwnership implementation.
+func (hu *HostUtil) GetOwner(pathname string) (int64, int64, error) {
+	return -1, -1, nil
+}
+
+// GetSELinuxSupport returns a boolean indicating support for SELinux.
+// Windows does not support SELinux.
+func (hu *HostUtil) GetSELinuxSupport(pathname string) (bool, error) {
+	return false, nil
+}
+
+// GetMode returns permissions of the path.
+func (hu *HostUtil) GetMode(pathname string) (os.FileMode, error) {
+	info, err := os.Stat(pathname)
+	if err != nil {
+		return 0, err
+	}
+	return info.Mode(), nil
+}

vendor/k8s.io/kubernetes/pkg/volume/util/recyclerclient/BUILD

@@ -0,0 +1,44 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["recycler_client.go"],
+    importpath = "k8s.io/kubernetes/pkg/volume/util/recyclerclient",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/klog/v2:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["recycler_client_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//pkg/apis/core:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/k8s.io/kubernetes/pkg/volume/util/recyclerclient/recycler_client.go

@@ -0,0 +1,266 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package recyclerclient
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/watch"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/klog/v2"
+)
+
+// RecycleEventRecorder is a func that defines how to record RecycleEvent.
+type RecycleEventRecorder func(eventtype, message string)
+
+// RecycleVolumeByWatchingPodUntilCompletion is intended for use with volume
+// Recyclers. This function will save the given Pod to the API and watch it
+// until it completes, fails, or the pod's ActiveDeadlineSeconds is exceeded,
+// whichever comes first. An attempt to delete a recycler pod is always
+// attempted before returning.
+//
+// In case there is a pod with the same namespace+name already running, this
+// function deletes it as it is not able to judge if it is an old recycler
+// or user has forged a fake recycler to block Kubernetes from recycling.//
+//
+//  pod - the pod designed by a volume plugin to recycle the volume. pod.Name
+//        will be overwritten with unique name based on PV.Name.
+//	client - kube client for API operations.
+func RecycleVolumeByWatchingPodUntilCompletion(pvName string, pod *v1.Pod, kubeClient clientset.Interface, recorder RecycleEventRecorder) error {
+	return internalRecycleVolumeByWatchingPodUntilCompletion(pvName, pod, newRecyclerClient(kubeClient, recorder))
+}
+
+// same as above func comments, except 'recyclerClient' is a narrower pod API
+// interface to ease testing
+func internalRecycleVolumeByWatchingPodUntilCompletion(pvName string, pod *v1.Pod, recyclerClient recyclerClient) error {
+	klog.V(5).Infof("creating recycler pod for volume %s\n", pod.Name)
+
+	// Generate unique name for the recycler pod - we need to get "already
+	// exists" error when a previous controller has already started recycling
+	// the volume. Here we assume that pv.Name is already unique.
+	pod.Name = "recycler-for-" + pvName
+	pod.GenerateName = ""
+
+	stopChannel := make(chan struct{})
+	defer close(stopChannel)
+	podCh, err := recyclerClient.WatchPod(pod.Name, pod.Namespace, stopChannel)
+	if err != nil {
+		klog.V(4).Infof("cannot start watcher for pod %s/%s: %v", pod.Namespace, pod.Name, err)
+		return err
+	}
+
+	// Start the pod
+	_, err = recyclerClient.CreatePod(pod)
+	if err != nil {
+		if errors.IsAlreadyExists(err) {
+			deleteErr := recyclerClient.DeletePod(pod.Name, pod.Namespace)
+			if deleteErr != nil {
+				return fmt.Errorf("failed to delete old recycler pod %s/%s: %s", pod.Namespace, pod.Name, deleteErr)
+			}
+			// Recycler will try again and the old pod will be hopefully deleted
+			// at that time.
+			return fmt.Errorf("old recycler pod found, will retry later")
+		}
+		return fmt.Errorf("unexpected error creating recycler pod:  %+v", err)
+	}
+	err = waitForPod(pod, recyclerClient, podCh)
+
+	// In all cases delete the recycler pod and log its result.
+	klog.V(2).Infof("deleting recycler pod %s/%s", pod.Namespace, pod.Name)
+	deleteErr := recyclerClient.DeletePod(pod.Name, pod.Namespace)
+	if deleteErr != nil {
+		klog.Errorf("failed to delete recycler pod %s/%s: %v", pod.Namespace, pod.Name, err)
+	}
+
+	// Returning recycler error is preferred, the pod will be deleted again on
+	// the next retry.
+	if err != nil {
+		return fmt.Errorf("failed to recycle volume: %s", err)
+	}
+
+	// Recycle succeeded but we failed to delete the recycler pod. Report it,
+	// the controller will re-try recycling the PV again shortly.
+	if deleteErr != nil {
+		return fmt.Errorf("failed to delete recycler pod: %s", deleteErr)
+	}
+
+	return nil
+}
+
+// waitForPod watches the pod it until it finishes and send all events on the
+// pod to the PV.
+func waitForPod(pod *v1.Pod, recyclerClient recyclerClient, podCh <-chan watch.Event) error {
+	for {
+		event, ok := <-podCh
+		if !ok {
+			return fmt.Errorf("recycler pod %q watch channel had been closed", pod.Name)
+		}
+		switch event.Object.(type) {
+		case *v1.Pod:
+			// POD changed
+			pod := event.Object.(*v1.Pod)
+			klog.V(4).Infof("recycler pod update received: %s %s/%s %s", event.Type, pod.Namespace, pod.Name, pod.Status.Phase)
+			switch event.Type {
+			case watch.Added, watch.Modified:
+				if pod.Status.Phase == v1.PodSucceeded {
+					// Recycle succeeded.
+					return nil
+				}
+				if pod.Status.Phase == v1.PodFailed {
+					if pod.Status.Message != "" {
+						return fmt.Errorf(pod.Status.Message)
+					}
+					return fmt.Errorf("pod failed, pod.Status.Message unknown")
+				}
+
+			case watch.Deleted:
+				return fmt.Errorf("recycler pod was deleted")
+
+			case watch.Error:
+				return fmt.Errorf("recycler pod watcher failed")
+			}
+
+		case *v1.Event:
+			// Event received
+			podEvent := event.Object.(*v1.Event)
+			klog.V(4).Infof("recycler event received: %s %s/%s %s/%s %s", event.Type, podEvent.Namespace, podEvent.Name, podEvent.InvolvedObject.Namespace, podEvent.InvolvedObject.Name, podEvent.Message)
+			if event.Type == watch.Added {
+				recyclerClient.Event(podEvent.Type, podEvent.Message)
+			}
+		}
+	}
+}
+
+// recyclerClient abstracts access to a Pod by providing a narrower interface.
+// This makes it easier to mock a client for testing.
+type recyclerClient interface {
+	CreatePod(pod *v1.Pod) (*v1.Pod, error)
+	GetPod(name, namespace string) (*v1.Pod, error)
+	DeletePod(name, namespace string) error
+	// WatchPod returns a ListWatch for watching a pod.  The stopChannel is used
+	// to close the reflector backing the watch.  The caller is responsible for
+	// derring a close on the channel to stop the reflector.
+	WatchPod(name, namespace string, stopChannel chan struct{}) (<-chan watch.Event, error)
+	// Event sends an event to the volume that is being recycled.
+	Event(eventtype, message string)
+}
+
+func newRecyclerClient(client clientset.Interface, recorder RecycleEventRecorder) recyclerClient {
+	return &realRecyclerClient{
+		client,
+		recorder,
+	}
+}
+
+type realRecyclerClient struct {
+	client   clientset.Interface
+	recorder RecycleEventRecorder
+}
+
+func (c *realRecyclerClient) CreatePod(pod *v1.Pod) (*v1.Pod, error) {
+	return c.client.CoreV1().Pods(pod.Namespace).Create(context.TODO(), pod, metav1.CreateOptions{})
+}
+
+func (c *realRecyclerClient) GetPod(name, namespace string) (*v1.Pod, error) {
+	return c.client.CoreV1().Pods(namespace).Get(context.TODO(), name, metav1.GetOptions{})
+}
+
+func (c *realRecyclerClient) DeletePod(name, namespace string) error {
+	return c.client.CoreV1().Pods(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{})
+}
+
+func (c *realRecyclerClient) Event(eventtype, message string) {
+	c.recorder(eventtype, message)
+}
+
+// WatchPod watches a pod and events related to it. It sends pod updates and events over the returned channel
+// It will continue until stopChannel is closed
+func (c *realRecyclerClient) WatchPod(name, namespace string, stopChannel chan struct{}) (<-chan watch.Event, error) {
+	podSelector, err := fields.ParseSelector("metadata.name=" + name)
+	if err != nil {
+		return nil, err
+	}
+	options := metav1.ListOptions{
+		FieldSelector: podSelector.String(),
+		Watch:         true,
+	}
+
+	podWatch, err := c.client.CoreV1().Pods(namespace).Watch(context.TODO(), options)
+	if err != nil {
+		return nil, err
+	}
+
+	eventSelector, _ := fields.ParseSelector("involvedObject.name=" + name)
+	eventWatch, err := c.client.CoreV1().Events(namespace).Watch(context.TODO(), metav1.ListOptions{
+		FieldSelector: eventSelector.String(),
+		Watch:         true,
+	})
+	if err != nil {
+		podWatch.Stop()
+		return nil, err
+	}
+
+	eventCh := make(chan watch.Event, 30)
+	var wg sync.WaitGroup
+	wg.Add(2)
+
+	go func() {
+		defer close(eventCh)
+		wg.Wait()
+	}()
+
+	go func() {
+		defer eventWatch.Stop()
+		defer wg.Done()
+		for {
+			select {
+			case <-stopChannel:
+				return
+			case eventEvent, ok := <-eventWatch.ResultChan():
+				if !ok {
+					return
+				}
+				eventCh <- eventEvent
+			}
+		}
+	}()
+
+	go func() {
+		defer podWatch.Stop()
+		defer wg.Done()
+		for {
+			select {
+			case <-stopChannel:
+				return
+
+			case podEvent, ok := <-podWatch.ResultChan():
+				if !ok {
+					return
+				}
+				eventCh <- podEvent
+			}
+		}
+	}()
+
+	return eventCh, nil
+}

vendor/k8s.io/kubernetes/pkg/volume/util/subpath/BUILD

@@ -0,0 +1,118 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "subpath.go",
+        "subpath_linux.go",
+        "subpath_unsupported.go",
+        "subpath_windows.go",
+    ],
+    importpath = "k8s.io/kubernetes/pkg/volume/util/subpath",
+    visibility = ["//visibility:public"],
+    deps = select({
+        "@io_bazel_rules_go//go/platform:aix": [
+            "//staging/src/k8s.io/mount-utils:go_default_library",
+            "//vendor/k8s.io/utils/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:android": [
+            "//pkg/volume/util/hostutil:go_default_library",
+            "//staging/src/k8s.io/mount-utils:go_default_library",
+            "//vendor/golang.org/x/sys/unix:go_default_library",
+            "//vendor/k8s.io/klog/v2:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:darwin": [
+            "//staging/src/k8s.io/mount-utils:go_default_library",
+            "//vendor/k8s.io/utils/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:dragonfly": [
+            "//staging/src/k8s.io/mount-utils:go_default_library",
+            "//vendor/k8s.io/utils/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:freebsd": [
+            "//staging/src/k8s.io/mount-utils:go_default_library",
+            "//vendor/k8s.io/utils/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:illumos": [
+            "//staging/src/k8s.io/mount-utils:go_default_library",
+            "//vendor/k8s.io/utils/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:ios": [
+            "//staging/src/k8s.io/mount-utils:go_default_library",
+            "//vendor/k8s.io/utils/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:js": [
+            "//staging/src/k8s.io/mount-utils:go_default_library",
+            "//vendor/k8s.io/utils/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:linux": [
+            "//pkg/volume/util/hostutil:go_default_library",
+            "//staging/src/k8s.io/mount-utils:go_default_library",
+            "//vendor/golang.org/x/sys/unix:go_default_library",
+            "//vendor/k8s.io/klog/v2:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:nacl": [
+            "//staging/src/k8s.io/mount-utils:go_default_library",
+            "//vendor/k8s.io/utils/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:netbsd": [
+            "//staging/src/k8s.io/mount-utils:go_default_library",
+            "//vendor/k8s.io/utils/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:openbsd": [
+            "//staging/src/k8s.io/mount-utils:go_default_library",
+            "//vendor/k8s.io/utils/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:plan9": [
+            "//staging/src/k8s.io/mount-utils:go_default_library",
+            "//vendor/k8s.io/utils/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:solaris": [
+            "//staging/src/k8s.io/mount-utils:go_default_library",
+            "//vendor/k8s.io/utils/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:windows": [
+            "//staging/src/k8s.io/mount-utils:go_default_library",
+            "//vendor/k8s.io/klog/v2:go_default_library",
+            "//vendor/k8s.io/utils/nsenter:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "subpath_linux_test.go",
+        "subpath_windows_test.go",
+    ],
+    embed = [":go_default_library"],
+    deps = select({
+        "@io_bazel_rules_go//go/platform:android": [
+            "//staging/src/k8s.io/mount-utils:go_default_library",
+            "//vendor/k8s.io/klog/v2:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:linux": [
+            "//staging/src/k8s.io/mount-utils:go_default_library",
+            "//vendor/k8s.io/klog/v2:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:windows": [
+            "//vendor/github.com/stretchr/testify/assert:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/k8s.io/kubernetes/pkg/volume/util/subpath/OWNERS

@@ -0,0 +1,13 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+reviewers:
+  - jingxu97
+  - saad-ali
+  - jsafrane
+  - msau42
+  - andyzhangx
+approvers:
+  - jingxu97
+  - saad-ali
+  - jsafrane
+

vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath.go

@@ -0,0 +1,92 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package subpath
+
+import "os"
+
+// Interface defines the set of methods all subpathers must implement
+type Interface interface {
+	// CleanSubPaths removes any bind-mounts created by PrepareSafeSubpath in given
+	// pod volume directory.
+	CleanSubPaths(poodDir string, volumeName string) error
+
+	// PrepareSafeSubpath does everything that's necessary to prepare a subPath
+	// that's 1) inside given volumePath and 2) immutable after this call.
+	//
+	// newHostPath - location of prepared subPath. It should be used instead of
+	// hostName when running the container.
+	// cleanupAction - action to run when the container is running or it failed to start.
+	//
+	// CleanupAction must be called immediately after the container with given
+	// subpath starts. On the other hand, Interface.CleanSubPaths must be called
+	// when the pod finishes.
+	PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error)
+
+	// SafeMakeDir creates subdir within given base. It makes sure that the
+	// created directory does not escape given base directory mis-using
+	// symlinks. Note that the function makes sure that it creates the directory
+	// somewhere under the base, nothing else. E.g. if the directory already
+	// exists, it may exist outside of the base due to symlinks.
+	// This method should be used if the directory to create is inside volume
+	// that's under user control. User must not be able to use symlinks to
+	// escape the volume to create directories somewhere else.
+	SafeMakeDir(subdir string, base string, perm os.FileMode) error
+}
+
+// Subpath defines the attributes of a subpath
+type Subpath struct {
+	// index of the VolumeMount for this container
+	VolumeMountIndex int
+
+	// Full path to the subpath directory on the host
+	Path string
+
+	// name of the volume that is a valid directory name.
+	VolumeName string
+
+	// Full path to the volume path
+	VolumePath string
+
+	// Path to the pod's directory, including pod UID
+	PodDir string
+
+	// Name of the container
+	ContainerName string
+}
+
+// Compile time-check for all implementers of subpath interface
+var _ Interface = &subpath{}
+var _ Interface = &FakeSubpath{}
+
+// FakeSubpath is a subpather implementation for testing
+type FakeSubpath struct{}
+
+// PrepareSafeSubpath is a fake implementation of PrepareSafeSubpath. Always returns
+// newHostPath == subPath.Path
+func (fs *FakeSubpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	return subPath.Path, nil, nil
+}
+
+// CleanSubPaths is a fake implementation of CleanSubPaths. It is a noop
+func (fs *FakeSubpath) CleanSubPaths(podDir string, volumeName string) error {
+	return nil
+}
+
+// SafeMakeDir is a fake implementation of SafeMakeDir. It is a noop
+func (fs *FakeSubpath) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	return nil
+}

vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_linux.go

@@ -0,0 +1,581 @@
+// +build linux
+
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package subpath
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+	"k8s.io/klog/v2"
+	"k8s.io/kubernetes/pkg/volume/util/hostutil"
+	"k8s.io/mount-utils"
+)
+
+const (
+	// place for subpath mounts
+	// TODO: pass in directory using kubelet_getters instead
+	containerSubPathDirectoryName = "volume-subpaths"
+	// syscall.Openat flags used to traverse directories not following symlinks
+	nofollowFlags = unix.O_RDONLY | unix.O_NOFOLLOW
+	// flags for getting file descriptor without following the symlink
+	openFDFlags = unix.O_NOFOLLOW | unix.O_PATH
+)
+
+type subpath struct {
+	mounter mount.Interface
+}
+
+// New returns a subpath.Interface for the current system
+func New(mounter mount.Interface) Interface {
+	return &subpath{
+		mounter: mounter,
+	}
+}
+
+func (sp *subpath) CleanSubPaths(podDir string, volumeName string) error {
+	return doCleanSubPaths(sp.mounter, podDir, volumeName)
+}
+
+func (sp *subpath) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
+	realBase, err := filepath.EvalSymlinks(base)
+	if err != nil {
+		return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
+	}
+
+	realFullPath := filepath.Join(realBase, subdir)
+
+	return doSafeMakeDir(realFullPath, realBase, perm)
+}
+
+func (sp *subpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	newHostPath, err = doBindSubPath(sp.mounter, subPath)
+
+	// There is no action when the container starts. Bind-mount will be cleaned
+	// when container stops by CleanSubPaths.
+	cleanupAction = nil
+	return newHostPath, cleanupAction, err
+}
+
+// This implementation is shared between Linux and NsEnter
+func safeOpenSubPath(mounter mount.Interface, subpath Subpath) (int, error) {
+	if !mount.PathWithinBase(subpath.Path, subpath.VolumePath) {
+		return -1, fmt.Errorf("subpath %q not within volume path %q", subpath.Path, subpath.VolumePath)
+	}
+	fd, err := doSafeOpen(subpath.Path, subpath.VolumePath)
+	if err != nil {
+		return -1, fmt.Errorf("error opening subpath %v: %v", subpath.Path, err)
+	}
+	return fd, nil
+}
+
+// prepareSubpathTarget creates target for bind-mount of subpath. It returns
+// "true" when the target already exists and something is mounted there.
+// Given Subpath must have all paths with already resolved symlinks and with
+// paths relevant to kubelet (when it runs in a container).
+// This function is called also by NsEnterMounter. It works because
+// /var/lib/kubelet is mounted from the host into the container with Kubelet as
+// /var/lib/kubelet too.
+func prepareSubpathTarget(mounter mount.Interface, subpath Subpath) (bool, string, error) {
+	// Early check for already bind-mounted subpath.
+	bindPathTarget := getSubpathBindTarget(subpath)
+	notMount, err := mount.IsNotMountPoint(mounter, bindPathTarget)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return false, "", fmt.Errorf("error checking path %s for mount: %s", bindPathTarget, err)
+		}
+		// Ignore ErrorNotExist: the file/directory will be created below if it does not exist yet.
+		notMount = true
+	}
+	if !notMount {
+		linuxHostUtil := hostutil.NewHostUtil()
+		mntInfo, err := linuxHostUtil.FindMountInfo(bindPathTarget)
+		if err != nil {
+			return false, "", fmt.Errorf("error calling findMountInfo for %s: %s", bindPathTarget, err)
+		}
+		if mntInfo.Root != subpath.Path {
+			// It's already mounted but not what we want, unmount it
+			if err = mounter.Unmount(bindPathTarget); err != nil {
+				return false, "", fmt.Errorf("error ummounting %s: %s", bindPathTarget, err)
+			}
+		} else {
+			// It's already mounted
+			klog.V(5).Infof("Skipping bind-mounting subpath %s: already mounted", bindPathTarget)
+			return true, bindPathTarget, nil
+		}
+	}
+
+	// bindPathTarget is in /var/lib/kubelet and thus reachable without any
+	// translation even to containerized kubelet.
+	bindParent := filepath.Dir(bindPathTarget)
+	err = os.MkdirAll(bindParent, 0750)
+	if err != nil && !os.IsExist(err) {
+		return false, "", fmt.Errorf("error creating directory %s: %s", bindParent, err)
+	}
+
+	t, err := os.Lstat(subpath.Path)
+	if err != nil {
+		return false, "", fmt.Errorf("lstat %s failed: %s", subpath.Path, err)
+	}
+
+	if t.Mode()&os.ModeDir > 0 {
+		if err = os.Mkdir(bindPathTarget, 0750); err != nil && !os.IsExist(err) {
+			return false, "", fmt.Errorf("error creating directory %s: %s", bindPathTarget, err)
+		}
+	} else {
+		// "/bin/touch <bindPathTarget>".
+		// A file is enough for all possible targets (symlink, device, pipe,
+		// socket, ...), bind-mounting them into a file correctly changes type
+		// of the target file.
+		if err = ioutil.WriteFile(bindPathTarget, []byte{}, 0640); err != nil {
+			return false, "", fmt.Errorf("error creating file %s: %s", bindPathTarget, err)
+		}
+	}
+	return false, bindPathTarget, nil
+}
+
+func getSubpathBindTarget(subpath Subpath) string {
+	// containerName is DNS label, i.e. safe as a directory name.
+	return filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName, strconv.Itoa(subpath.VolumeMountIndex))
+}
+
+func doBindSubPath(mounter mount.Interface, subpath Subpath) (hostPath string, err error) {
+	// Linux, kubelet runs on the host:
+	// - safely open the subpath
+	// - bind-mount /proc/<pid of kubelet>/fd/<fd> to subpath target
+	// User can't change /proc/<pid of kubelet>/fd/<fd> to point to a bad place.
+
+	// Evaluate all symlinks here once for all subsequent functions.
+	newVolumePath, err := filepath.EvalSymlinks(subpath.VolumePath)
+	if err != nil {
+		return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err)
+	}
+	newPath, err := filepath.EvalSymlinks(subpath.Path)
+	if err != nil {
+		return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err)
+	}
+	klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, newPath, subpath.VolumePath)
+	subpath.VolumePath = newVolumePath
+	subpath.Path = newPath
+
+	fd, err := safeOpenSubPath(mounter, subpath)
+	if err != nil {
+		return "", err
+	}
+	defer syscall.Close(fd)
+
+	alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath)
+	if err != nil {
+		return "", err
+	}
+	if alreadyMounted {
+		return bindPathTarget, nil
+	}
+
+	success := false
+	defer func() {
+		// Cleanup subpath on error
+		if !success {
+			klog.V(4).Infof("doBindSubPath() failed for %q, cleaning up subpath", bindPathTarget)
+			if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil {
+				klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr)
+			}
+		}
+	}()
+
+	kubeletPid := os.Getpid()
+	mountSource := fmt.Sprintf("/proc/%d/fd/%v", kubeletPid, fd)
+
+	// Do the bind mount
+	options := []string{"bind"}
+	klog.V(5).Infof("bind mounting %q at %q", mountSource, bindPathTarget)
+	if err = mounter.MountSensitiveWithoutSystemd(mountSource, bindPathTarget, "" /*fstype*/, options, nil); err != nil {
+		return "", fmt.Errorf("error mounting %s: %s", subpath.Path, err)
+	}
+	success = true
+
+	klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget)
+	return bindPathTarget, nil
+}
+
+// This implementation is shared between Linux and NsEnter
+func doCleanSubPaths(mounter mount.Interface, podDir string, volumeName string) error {
+	// scan /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/*
+	subPathDir := filepath.Join(podDir, containerSubPathDirectoryName, volumeName)
+	klog.V(4).Infof("Cleaning up subpath mounts for %s", subPathDir)
+
+	containerDirs, err := ioutil.ReadDir(subPathDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("error reading %s: %s", subPathDir, err)
+	}
+
+	for _, containerDir := range containerDirs {
+		if !containerDir.IsDir() {
+			klog.V(4).Infof("Container file is not a directory: %s", containerDir.Name())
+			continue
+		}
+		klog.V(4).Infof("Cleaning up subpath mounts for container %s", containerDir.Name())
+
+		// scan /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/*
+		fullContainerDirPath := filepath.Join(subPathDir, containerDir.Name())
+		err = filepath.Walk(fullContainerDirPath, func(path string, info os.FileInfo, _ error) error {
+			if path == fullContainerDirPath {
+				// Skip top level directory
+				return nil
+			}
+
+			// pass through errors and let doCleanSubPath handle them
+			if err = doCleanSubPath(mounter, fullContainerDirPath, filepath.Base(path)); err != nil {
+				return err
+			}
+
+			// We need to check that info is not nil. This may happen when the incoming err is not nil due to stale mounts or permission errors.
+			if info != nil && info.IsDir() {
+				// skip subdirs of the volume: it only matters the first level to unmount, otherwise it would try to unmount subdir of the volume
+				return filepath.SkipDir
+			}
+
+			return nil
+		})
+		if err != nil {
+			return fmt.Errorf("error processing %s: %s", fullContainerDirPath, err)
+		}
+
+		// Whole container has been processed, remove its directory.
+		if err := os.Remove(fullContainerDirPath); err != nil {
+			return fmt.Errorf("error deleting %s: %s", fullContainerDirPath, err)
+		}
+		klog.V(5).Infof("Removed %s", fullContainerDirPath)
+	}
+	// Whole pod volume subpaths have been cleaned up, remove its subpath directory.
+	if err := os.Remove(subPathDir); err != nil {
+		return fmt.Errorf("error deleting %s: %s", subPathDir, err)
+	}
+	klog.V(5).Infof("Removed %s", subPathDir)
+
+	// Remove entire subpath directory if it's the last one
+	podSubPathDir := filepath.Join(podDir, containerSubPathDirectoryName)
+	if err := os.Remove(podSubPathDir); err != nil && !os.IsExist(err) {
+		return fmt.Errorf("error deleting %s: %s", podSubPathDir, err)
+	}
+	klog.V(5).Infof("Removed %s", podSubPathDir)
+	return nil
+}
+
+// doCleanSubPath tears down the single subpath bind mount
+func doCleanSubPath(mounter mount.Interface, fullContainerDirPath, subPathIndex string) error {
+	// process /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/<subPathName>
+	klog.V(4).Infof("Cleaning up subpath mounts for subpath %v", subPathIndex)
+	fullSubPath := filepath.Join(fullContainerDirPath, subPathIndex)
+
+	if err := mount.CleanupMountPoint(fullSubPath, mounter, true); err != nil {
+		return fmt.Errorf("error cleaning subpath mount %s: %s", fullSubPath, err)
+	}
+
+	klog.V(4).Infof("Successfully cleaned subpath directory %s", fullSubPath)
+	return nil
+}
+
+// cleanSubPath will teardown the subpath bind mount and any remove any directories if empty
+func cleanSubPath(mounter mount.Interface, subpath Subpath) error {
+	containerDir := filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName)
+
+	// Clean subdir bindmount
+	if err := doCleanSubPath(mounter, containerDir, strconv.Itoa(subpath.VolumeMountIndex)); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+
+	// Recusively remove directories if empty
+	if err := removeEmptyDirs(subpath.PodDir, containerDir); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// removeEmptyDirs works backwards from endDir to baseDir and removes each directory
+// if it is empty.  It stops once it encounters a directory that has content
+func removeEmptyDirs(baseDir, endDir string) error {
+	if !mount.PathWithinBase(endDir, baseDir) {
+		return fmt.Errorf("endDir %q is not within baseDir %q", endDir, baseDir)
+	}
+
+	for curDir := endDir; curDir != baseDir; curDir = filepath.Dir(curDir) {
+		s, err := os.Stat(curDir)
+		if err != nil {
+			if os.IsNotExist(err) {
+				klog.V(5).Infof("curDir %q doesn't exist, skipping", curDir)
+				continue
+			}
+			return fmt.Errorf("error stat %q: %v", curDir, err)
+		}
+		if !s.IsDir() {
+			return fmt.Errorf("path %q not a directory", curDir)
+		}
+
+		err = os.Remove(curDir)
+		if os.IsExist(err) {
+			klog.V(5).Infof("Directory %q not empty, not removing", curDir)
+			break
+		} else if err != nil {
+			return fmt.Errorf("error removing directory %q: %v", curDir, err)
+		}
+		klog.V(5).Infof("Removed directory %q", curDir)
+	}
+	return nil
+}
+
+// This implementation is shared between Linux and NsEnterMounter. Both pathname
+// and base must be either already resolved symlinks or thet will be resolved in
+// kubelet's mount namespace (in case it runs containerized).
+func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	klog.V(4).Infof("Creating directory %q within base %q", pathname, base)
+
+	if !mount.PathWithinBase(pathname, base) {
+		return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
+	}
+
+	// Quick check if the directory already exists
+	s, err := os.Stat(pathname)
+	if err == nil {
+		// Path exists
+		if s.IsDir() {
+			// The directory already exists. It can be outside of the parent,
+			// but there is no race-proof check.
+			klog.V(4).Infof("Directory %s already exists", pathname)
+			return nil
+		}
+		return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR}
+	}
+
+	// Find all existing directories
+	existingPath, toCreate, err := findExistingPrefix(base, pathname)
+	if err != nil {
+		return fmt.Errorf("error opening directory %s: %s", pathname, err)
+	}
+	// Ensure the existing directory is inside allowed base
+	fullExistingPath, err := filepath.EvalSymlinks(existingPath)
+	if err != nil {
+		return fmt.Errorf("error opening directory %s: %s", existingPath, err)
+	}
+	if !mount.PathWithinBase(fullExistingPath, base) {
+		return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
+	}
+
+	klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...))
+	parentFD, err := doSafeOpen(fullExistingPath, base)
+	if err != nil {
+		return fmt.Errorf("cannot open directory %s: %s", existingPath, err)
+	}
+	childFD := -1
+	defer func() {
+		if parentFD != -1 {
+			if err = syscall.Close(parentFD); err != nil {
+				klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err)
+			}
+		}
+		if childFD != -1 {
+			if err = syscall.Close(childFD); err != nil {
+				klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", childFD, pathname, err)
+			}
+		}
+	}()
+
+	currentPath := fullExistingPath
+	// create the directories one by one, making sure nobody can change
+	// created directory into symlink.
+	for _, dir := range toCreate {
+		currentPath = filepath.Join(currentPath, dir)
+		klog.V(4).Infof("Creating %s", dir)
+		err = syscall.Mkdirat(parentFD, currentPath, uint32(perm))
+		if err != nil {
+			return fmt.Errorf("cannot create directory %s: %s", currentPath, err)
+		}
+		// Dive into the created directory
+		childFD, err = syscall.Openat(parentFD, dir, nofollowFlags|unix.O_CLOEXEC, 0)
+		if err != nil {
+			return fmt.Errorf("cannot open %s: %s", currentPath, err)
+		}
+		// We can be sure that childFD is safe to use. It could be changed
+		// by user after Mkdirat() and before Openat(), however:
+		// - it could not be changed to symlink - we use nofollowFlags
+		// - it could be changed to a file (or device, pipe, socket, ...)
+		//   but either subsequent Mkdirat() fails or we mount this file
+		//   to user's container. Security is no violated in both cases
+		//   and user either gets error or the file that it can already access.
+
+		if err = syscall.Close(parentFD); err != nil {
+			klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err)
+		}
+		parentFD = childFD
+		childFD = -1
+	}
+
+	// Everything was created. mkdirat(..., perm) above was affected by current
+	// umask and we must apply the right permissions to the last directory
+	// (that's the one that will be available to the container as subpath)
+	// so user can read/write it. This is the behavior of previous code.
+	// TODO: chmod all created directories, not just the last one.
+	// parentFD is the last created directory.
+
+	// Translate perm (os.FileMode) to uint32 that fchmod() expects
+	kernelPerm := uint32(perm & os.ModePerm)
+	if perm&os.ModeSetgid > 0 {
+		kernelPerm |= syscall.S_ISGID
+	}
+	if perm&os.ModeSetuid > 0 {
+		kernelPerm |= syscall.S_ISUID
+	}
+	if perm&os.ModeSticky > 0 {
+		kernelPerm |= syscall.S_ISVTX
+	}
+	if err = syscall.Fchmod(parentFD, kernelPerm); err != nil {
+		return fmt.Errorf("chmod %q failed: %s", currentPath, err)
+	}
+	return nil
+}
+
+// findExistingPrefix finds prefix of pathname that exists. In addition, it
+// returns list of remaining directories that don't exist yet.
+func findExistingPrefix(base, pathname string) (string, []string, error) {
+	rel, err := filepath.Rel(base, pathname)
+	if err != nil {
+		return base, nil, err
+	}
+	dirs := strings.Split(rel, string(filepath.Separator))
+
+	// Do OpenAt in a loop to find the first non-existing dir. Resolve symlinks.
+	// This should be faster than looping through all dirs and calling os.Stat()
+	// on each of them, as the symlinks are resolved only once with OpenAt().
+	currentPath := base
+	fd, err := syscall.Open(currentPath, syscall.O_RDONLY|syscall.O_CLOEXEC, 0)
+	if err != nil {
+		return pathname, nil, fmt.Errorf("error opening %s: %s", currentPath, err)
+	}
+	defer func() {
+		if err = syscall.Close(fd); err != nil {
+			klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err)
+		}
+	}()
+	for i, dir := range dirs {
+		// Using O_PATH here will prevent hangs in case user replaces directory with
+		// fifo
+		childFD, err := syscall.Openat(fd, dir, unix.O_PATH|unix.O_CLOEXEC, 0)
+		if err != nil {
+			if os.IsNotExist(err) {
+				return currentPath, dirs[i:], nil
+			}
+			return base, nil, err
+		}
+		if err = syscall.Close(fd); err != nil {
+			klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err)
+		}
+		fd = childFD
+		currentPath = filepath.Join(currentPath, dir)
+	}
+	return pathname, []string{}, nil
+}
+
+// This implementation is shared between Linux and NsEnterMounter
+// Open path and return its fd.
+// Symlinks are disallowed (pathname must already resolve symlinks),
+// and the path must be within the base directory.
+func doSafeOpen(pathname string, base string) (int, error) {
+	pathname = filepath.Clean(pathname)
+	base = filepath.Clean(base)
+
+	// Calculate segments to follow
+	subpath, err := filepath.Rel(base, pathname)
+	if err != nil {
+		return -1, err
+	}
+	segments := strings.Split(subpath, string(filepath.Separator))
+
+	// Assumption: base is the only directory that we have under control.
+	// Base dir is not allowed to be a symlink.
+	parentFD, err := syscall.Open(base, nofollowFlags|unix.O_CLOEXEC, 0)
+	if err != nil {
+		return -1, fmt.Errorf("cannot open directory %s: %s", base, err)
+	}
+	defer func() {
+		if parentFD != -1 {
+			if err = syscall.Close(parentFD); err != nil {
+				klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", parentFD, pathname, err)
+			}
+		}
+	}()
+
+	childFD := -1
+	defer func() {
+		if childFD != -1 {
+			if err = syscall.Close(childFD); err != nil {
+				klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", childFD, pathname, err)
+			}
+		}
+	}()
+
+	currentPath := base
+
+	// Follow the segments one by one using openat() to make
+	// sure the user cannot change already existing directories into symlinks.
+	for _, seg := range segments {
+		currentPath = filepath.Join(currentPath, seg)
+		if !mount.PathWithinBase(currentPath, base) {
+			return -1, fmt.Errorf("path %s is outside of allowed base %s", currentPath, base)
+		}
+
+		klog.V(5).Infof("Opening path %s", currentPath)
+		childFD, err = syscall.Openat(parentFD, seg, openFDFlags|unix.O_CLOEXEC, 0)
+		if err != nil {
+			return -1, fmt.Errorf("cannot open %s: %s", currentPath, err)
+		}
+
+		var deviceStat unix.Stat_t
+		err := unix.Fstat(childFD, &deviceStat)
+		if err != nil {
+			return -1, fmt.Errorf("error running fstat on %s with %v", currentPath, err)
+		}
+		fileFmt := deviceStat.Mode & syscall.S_IFMT
+		if fileFmt == syscall.S_IFLNK {
+			return -1, fmt.Errorf("unexpected symlink found %s", currentPath)
+		}
+
+		// Close parentFD
+		if err = syscall.Close(parentFD); err != nil {
+			return -1, fmt.Errorf("closing fd for %q failed: %v", filepath.Dir(currentPath), err)
+		}
+		// Set child to new parent
+		parentFD = childFD
+		childFD = -1
+	}
+
+	// We made it to the end, return this fd, don't close it
+	finalFD := parentFD
+	parentFD = -1
+
+	return finalFD, nil
+}

vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_unsupported.go

@@ -0,0 +1,54 @@
+// +build !linux,!windows
+
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package subpath
+
+import (
+	"errors"
+	"os"
+
+	"k8s.io/mount-utils"
+	"k8s.io/utils/nsenter"
+)
+
+type subpath struct{}
+
+var errUnsupported = errors.New("util/subpath on this platform is not supported")
+
+// New returns a subpath.Interface for the current system.
+func New(mount.Interface) Interface {
+	return &subpath{}
+}
+
+// NewNSEnter is to satisfy the compiler for having NewSubpathNSEnter exist for all
+// OS choices. however, NSEnter is only valid on Linux
+func NewNSEnter(mounter mount.Interface, ne *nsenter.Nsenter, rootDir string) Interface {
+	return nil
+}
+
+func (sp *subpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	return subPath.Path, nil, errUnsupported
+}
+
+func (sp *subpath) CleanSubPaths(podDir string, volumeName string) error {
+	return errUnsupported
+}
+
+func (sp *subpath) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	return errUnsupported
+}

vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_windows.go

@@ -0,0 +1,377 @@
+// +build windows
+
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package subpath
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"k8s.io/klog/v2"
+	"k8s.io/mount-utils"
+	"k8s.io/utils/nsenter"
+)
+
+// MaxPathLength is the maximum length of Windows path. Normally, it is 260, but if long path is enable,
+// the max number is 32,767
+const MaxPathLength = 32767
+
+type subpath struct{}
+
+// New returns a subpath.Interface for the current system
+func New(mount.Interface) Interface {
+	return &subpath{}
+}
+
+// NewNSEnter is to satisfy the compiler for having NewSubpathNSEnter exist for all
+// OS choices. however, NSEnter is only valid on Linux
+func NewNSEnter(mounter mount.Interface, ne *nsenter.Nsenter, rootDir string) Interface {
+	return nil
+}
+
+// isDriveLetterPath returns true if the given path is empty or it ends with ":" or ":\" or ":\\"
+func isDriveLetterorEmptyPath(path string) bool {
+	if path == "" || strings.HasSuffix(path, ":\\\\") || strings.HasSuffix(path, ":") || strings.HasSuffix(path, ":\\") {
+		return true
+	}
+	return false
+}
+
+// isVolumePrefix returns true if the given path name starts with "Volume" or volume prefix including
+// "\\.\", "\\?\" for device path or "UNC" or "\\" for UNC path. Otherwise, it returns false.
+func isDeviceOrUncPath(path string) bool {
+	if strings.HasPrefix(path, "Volume") || strings.HasPrefix(path, "\\\\?\\") || strings.HasPrefix(path, "\\\\.\\") || strings.HasPrefix(path, "UNC") {
+		return true
+	}
+	return false
+}
+
+// getUpperPath removes the last level of directory.
+func getUpperPath(path string) string {
+	sep := fmt.Sprintf("%c", filepath.Separator)
+	upperpath := strings.TrimSuffix(path, sep)
+	return filepath.Dir(upperpath)
+}
+
+// Check whether a directory/file is a link type or not
+// LinkType could be SymbolicLink, Junction, or HardLink
+func isLinkPath(path string) (bool, error) {
+	cmd := fmt.Sprintf("(Get-Item -Path %s).LinkType", path)
+	output, err := exec.Command("powershell", "/c", cmd).CombinedOutput()
+	if err != nil {
+		return false, err
+	}
+	if strings.TrimSpace(string(output)) != "" {
+		return true, nil
+	}
+	return false, nil
+}
+
+// evalSymlink returns the path name after the evaluation of any symbolic links.
+// If the path after evaluation is a device path or network connection, the original path is returned
+func evalSymlink(path string) (string, error) {
+	path = mount.NormalizeWindowsPath(path)
+	if isDeviceOrUncPath(path) || isDriveLetterorEmptyPath(path) {
+		klog.V(4).Infof("Path '%s' is not a symlink, return its original form.", path)
+		return path, nil
+	}
+	upperpath := path
+	base := ""
+	for i := 0; i < MaxPathLength; i++ {
+		isLink, err := isLinkPath(upperpath)
+		if err != nil {
+			return "", err
+		}
+		if isLink {
+			break
+		}
+		// continue to check next layer
+		base = filepath.Join(filepath.Base(upperpath), base)
+		upperpath = getUpperPath(upperpath)
+		if isDriveLetterorEmptyPath(upperpath) {
+			klog.V(4).Infof("Path '%s' is not a symlink, return its original form.", path)
+			return path, nil
+		}
+	}
+	// This command will give the target path of a given symlink
+	cmd := fmt.Sprintf("(Get-Item -Path %s).Target", upperpath)
+	output, err := exec.Command("powershell", "/c", cmd).CombinedOutput()
+	if err != nil {
+		return "", err
+	}
+	klog.V(4).Infof("evaluate path %s: symlink from %s to %s", path, upperpath, string(output))
+	linkedPath := strings.TrimSpace(string(output))
+	if linkedPath == "" || isDeviceOrUncPath(linkedPath) {
+		klog.V(4).Infof("Path '%s' has a target %s. Return its original form.", path, linkedPath)
+		return path, nil
+	}
+	// If the target is not an absoluate path, join iit with the current upperpath
+	if !filepath.IsAbs(linkedPath) {
+		linkedPath = filepath.Join(getUpperPath(upperpath), linkedPath)
+	}
+	nextLink, err := evalSymlink(linkedPath)
+	if err != nil {
+		return path, err
+	}
+	return filepath.Join(nextLink, base), nil
+}
+
+// check whether hostPath is within volume path
+// this func will lock all intermediate subpath directories, need to close handle outside of this func after container started
+func lockAndCheckSubPath(volumePath, hostPath string) ([]uintptr, error) {
+	if len(volumePath) == 0 || len(hostPath) == 0 {
+		return []uintptr{}, nil
+	}
+
+	finalSubPath, err := evalSymlink(hostPath)
+	if err != nil {
+		return []uintptr{}, fmt.Errorf("cannot evaluate link %s: %s", hostPath, err)
+	}
+
+	finalVolumePath, err := evalSymlink(volumePath)
+	if err != nil {
+		return []uintptr{}, fmt.Errorf("cannot read link %s: %s", volumePath, err)
+	}
+
+	return lockAndCheckSubPathWithoutSymlink(finalVolumePath, finalSubPath)
+}
+
+// lock all intermediate subPath directories and check they are all within volumePath
+// volumePath & subPath should not contain any symlink, otherwise it will return error
+func lockAndCheckSubPathWithoutSymlink(volumePath, subPath string) ([]uintptr, error) {
+	if len(volumePath) == 0 || len(subPath) == 0 {
+		return []uintptr{}, nil
+	}
+
+	// get relative path to volumePath
+	relSubPath, err := filepath.Rel(volumePath, subPath)
+	if err != nil {
+		return []uintptr{}, fmt.Errorf("Rel(%s, %s) error: %v", volumePath, subPath, err)
+	}
+	if mount.StartsWithBackstep(relSubPath) {
+		return []uintptr{}, fmt.Errorf("SubPath %q not within volume path %q", subPath, volumePath)
+	}
+
+	if relSubPath == "." {
+		// volumePath and subPath are equal
+		return []uintptr{}, nil
+	}
+
+	fileHandles := []uintptr{}
+	var errorResult error
+
+	currentFullPath := volumePath
+	dirs := strings.Split(relSubPath, string(os.PathSeparator))
+	for _, dir := range dirs {
+		// lock intermediate subPath directory first
+		currentFullPath = filepath.Join(currentFullPath, dir)
+		handle, err := lockPath(currentFullPath)
+		if err != nil {
+			errorResult = fmt.Errorf("cannot lock path %s: %s", currentFullPath, err)
+			break
+		}
+		fileHandles = append(fileHandles, handle)
+
+		// make sure intermediate subPath directory does not contain symlink any more
+		stat, err := os.Lstat(currentFullPath)
+		if err != nil {
+			errorResult = fmt.Errorf("Lstat(%q) error: %v", currentFullPath, err)
+			break
+		}
+		if stat.Mode()&os.ModeSymlink != 0 {
+			errorResult = fmt.Errorf("subpath %q is an unexpected symlink after EvalSymlinks", currentFullPath)
+			break
+		}
+
+		if !mount.PathWithinBase(currentFullPath, volumePath) {
+			errorResult = fmt.Errorf("SubPath %q not within volume path %q", currentFullPath, volumePath)
+			break
+		}
+	}
+
+	return fileHandles, errorResult
+}
+
+// unlockPath unlock directories
+func unlockPath(fileHandles []uintptr) {
+	if fileHandles != nil {
+		for _, handle := range fileHandles {
+			syscall.CloseHandle(syscall.Handle(handle))
+		}
+	}
+}
+
+// lockPath locks a directory or symlink, return handle, exec "syscall.CloseHandle(handle)" to unlock the path
+func lockPath(path string) (uintptr, error) {
+	if len(path) == 0 {
+		return uintptr(syscall.InvalidHandle), syscall.ERROR_FILE_NOT_FOUND
+	}
+	pathp, err := syscall.UTF16PtrFromString(path)
+	if err != nil {
+		return uintptr(syscall.InvalidHandle), err
+	}
+	access := uint32(syscall.GENERIC_READ)
+	sharemode := uint32(syscall.FILE_SHARE_READ)
+	createmode := uint32(syscall.OPEN_EXISTING)
+	flags := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS | syscall.FILE_FLAG_OPEN_REPARSE_POINT)
+	fd, err := syscall.CreateFile(pathp, access, sharemode, nil, createmode, flags, 0)
+	return uintptr(fd), err
+}
+
+// Lock all directories in subPath and check they're not symlinks.
+func (sp *subpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	handles, err := lockAndCheckSubPath(subPath.VolumePath, subPath.Path)
+
+	// Unlock the directories when the container starts
+	cleanupAction = func() {
+		unlockPath(handles)
+	}
+	return subPath.Path, cleanupAction, err
+}
+
+// No bind-mounts for subpaths are necessary on Windows
+func (sp *subpath) CleanSubPaths(podDir string, volumeName string) error {
+	return nil
+}
+
+// SafeMakeDir makes sure that the created directory does not escape given base directory mis-using symlinks.
+func (sp *subpath) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
+	realBase, err := evalSymlink(base)
+	if err != nil {
+		return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
+	}
+
+	realFullPath := filepath.Join(realBase, subdir)
+	return doSafeMakeDir(realFullPath, realBase, perm)
+}
+
+func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	klog.V(4).Infof("Creating directory %q within base %q", pathname, base)
+
+	if !mount.PathWithinBase(pathname, base) {
+		return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
+	}
+
+	// Quick check if the directory already exists
+	s, err := os.Stat(pathname)
+	if err == nil {
+		// Path exists
+		if s.IsDir() {
+			// The directory already exists. It can be outside of the parent,
+			// but there is no race-proof check.
+			klog.V(4).Infof("Directory %s already exists", pathname)
+			return nil
+		}
+		return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR}
+	}
+
+	// Find all existing directories
+	existingPath, toCreate, err := findExistingPrefix(base, pathname)
+	if err != nil {
+		return fmt.Errorf("error opening directory %s: %s", pathname, err)
+	}
+	if len(toCreate) == 0 {
+		return nil
+	}
+
+	// Ensure the existing directory is inside allowed base
+	fullExistingPath, err := evalSymlink(existingPath)
+	if err != nil {
+		return fmt.Errorf("error opening existing directory %s: %s", existingPath, err)
+	}
+	fullBasePath, err := evalSymlink(base)
+	if err != nil {
+		return fmt.Errorf("cannot read link %s: %s", base, err)
+	}
+	if !mount.PathWithinBase(fullExistingPath, fullBasePath) {
+		return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
+	}
+
+	// lock all intermediate directories from fullBasePath to fullExistingPath (top to bottom)
+	fileHandles, err := lockAndCheckSubPathWithoutSymlink(fullBasePath, fullExistingPath)
+	defer unlockPath(fileHandles)
+	if err != nil {
+		return err
+	}
+
+	klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...))
+	currentPath := fullExistingPath
+	// create the directories one by one, making sure nobody can change
+	// created directory into symlink by lock that directory immediately
+	for _, dir := range toCreate {
+		currentPath = filepath.Join(currentPath, dir)
+		klog.V(4).Infof("Creating %s", dir)
+		if err := os.Mkdir(currentPath, perm); err != nil {
+			return fmt.Errorf("cannot create directory %s: %s", currentPath, err)
+		}
+		handle, err := lockPath(currentPath)
+		if err != nil {
+			return fmt.Errorf("cannot lock path %s: %s", currentPath, err)
+		}
+		defer syscall.CloseHandle(syscall.Handle(handle))
+		// make sure newly created directory does not contain symlink after lock
+		stat, err := os.Lstat(currentPath)
+		if err != nil {
+			return fmt.Errorf("Lstat(%q) error: %v", currentPath, err)
+		}
+		if stat.Mode()&os.ModeSymlink != 0 {
+			return fmt.Errorf("subpath %q is an unexpected symlink after Mkdir", currentPath)
+		}
+	}
+
+	return nil
+}
+
+// findExistingPrefix finds prefix of pathname that exists. In addition, it
+// returns list of remaining directories that don't exist yet.
+func findExistingPrefix(base, pathname string) (string, []string, error) {
+	rel, err := filepath.Rel(base, pathname)
+	if err != nil {
+		return base, nil, err
+	}
+
+	if mount.StartsWithBackstep(rel) {
+		return base, nil, fmt.Errorf("pathname(%s) is not within base(%s)", pathname, base)
+	}
+
+	if rel == "." {
+		// base and pathname are equal
+		return pathname, []string{}, nil
+	}
+
+	dirs := strings.Split(rel, string(filepath.Separator))
+
+	var parent string
+	currentPath := base
+	for i, dir := range dirs {
+		parent = currentPath
+		currentPath = filepath.Join(parent, dir)
+		if _, err := os.Lstat(currentPath); err != nil {
+			if os.IsNotExist(err) {
+				return parent, dirs[i:], nil
+			}
+			return base, nil, err
+		}
+	}
+
+	return pathname, []string{}, nil
+}

vendor/k8s.io/kubernetes/pkg/volume/volume.go

@@ -0,0 +1,295 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package volume
+
+import (
+	"time"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+// Volume represents a directory used by pods or hosts on a node. All method
+// implementations of methods in the volume interface must be idempotent.
+type Volume interface {
+	// GetPath returns the path to which the volume should be mounted for the
+	// pod.
+	GetPath() string
+
+	// MetricsProvider embeds methods for exposing metrics (e.g.
+	// used, available space).
+	MetricsProvider
+}
+
+// BlockVolume interface provides methods to generate global map path
+// and pod device map path.
+type BlockVolume interface {
+	// GetGlobalMapPath returns a global map path which contains
+	// bind mount associated to a block device.
+	// ex. plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}/{pod uuid}
+	GetGlobalMapPath(spec *Spec) (string, error)
+	// GetPodDeviceMapPath returns a pod device map path
+	// and name of a symbolic link associated to a block device.
+	// ex. pods/{podUid}/{DefaultKubeletVolumeDevicesDirName}/{escapeQualifiedPluginName}/, {volumeName}
+	GetPodDeviceMapPath() (string, string)
+}
+
+// MetricsProvider exposes metrics (e.g. used,available space) related to a
+// Volume.
+type MetricsProvider interface {
+	// GetMetrics returns the Metrics for the Volume. Maybe expensive for
+	// some implementations.
+	GetMetrics() (*Metrics, error)
+}
+
+// Metrics represents the used and available bytes of the Volume.
+type Metrics struct {
+	// The time at which these stats were updated.
+	Time metav1.Time
+
+	// Used represents the total bytes used by the Volume.
+	// Note: For block devices this maybe more than the total size of the files.
+	Used *resource.Quantity
+
+	// Capacity represents the total capacity (bytes) of the volume's
+	// underlying storage. For Volumes that share a filesystem with the host
+	// (e.g. emptydir, hostpath) this is the size of the underlying storage,
+	// and will not equal Used + Available as the fs is shared.
+	Capacity *resource.Quantity
+
+	// Available represents the storage space available (bytes) for the
+	// Volume. For Volumes that share a filesystem with the host (e.g.
+	// emptydir, hostpath), this is the available space on the underlying
+	// storage, and is shared with host processes and other Volumes.
+	Available *resource.Quantity
+
+	// InodesUsed represents the total inodes used by the Volume.
+	InodesUsed *resource.Quantity
+
+	// Inodes represents the total number of inodes available in the volume.
+	// For volumes that share a filesystem with the host (e.g. emptydir, hostpath),
+	// this is the inodes available in the underlying storage,
+	// and will not equal InodesUsed + InodesFree as the fs is shared.
+	Inodes *resource.Quantity
+
+	// InodesFree represent the inodes available for the volume.  For Volumes that share
+	// a filesystem with the host (e.g. emptydir, hostpath), this is the free inodes
+	// on the underlying storage, and is shared with host processes and other volumes
+	InodesFree *resource.Quantity
+}
+
+// Attributes represents the attributes of this mounter.
+type Attributes struct {
+	ReadOnly        bool
+	Managed         bool
+	SupportsSELinux bool
+}
+
+// MounterArgs provides more easily extensible arguments to Mounter
+type MounterArgs struct {
+	// When FsUser is set, the ownership of the volume will be modified to be
+	// owned and writable by FsUser. Otherwise, there is no side effects.
+	// Currently only supported with projected service account tokens.
+	FsUser              *int64
+	FsGroup             *int64
+	FSGroupChangePolicy *v1.PodFSGroupChangePolicy
+	DesiredSize         *resource.Quantity
+}
+
+// Mounter interface provides methods to set up/mount the volume.
+type Mounter interface {
+	// Uses Interface to provide the path for Docker binds.
+	Volume
+
+	// CanMount is called immediately prior to Setup to check if
+	// the required components (binaries, etc.) are available on
+	// the underlying node to complete the subsequent SetUp (mount)
+	// operation. If CanMount returns error, the mount operation is
+	// aborted and an event is generated indicating that the node
+	// does not have the required binaries to complete the mount.
+	// If CanMount succeeds, the mount operation continues
+	// normally. The CanMount check can be enabled or disabled
+	// using the experimental-check-mount-binaries binary flag
+	CanMount() error
+
+	// SetUp prepares and mounts/unpacks the volume to a
+	// self-determined directory path. The mount point and its
+	// content should be owned by `fsUser` or 'fsGroup' so that it can be
+	// accessed by the pod. This may be called more than once, so
+	// implementations must be idempotent.
+	// It could return following types of errors:
+	//   - TransientOperationFailure
+	//   - UncertainProgressError
+	//   - Error of any other type should be considered a final error
+	SetUp(mounterArgs MounterArgs) error
+
+	// SetUpAt prepares and mounts/unpacks the volume to the
+	// specified directory path, which may or may not exist yet.
+	// The mount point and its content should be owned by `fsUser`
+	// 'fsGroup' so that it can be accessed by the pod. This may
+	// be called more than once, so implementations must be
+	// idempotent.
+	SetUpAt(dir string, mounterArgs MounterArgs) error
+	// GetAttributes returns the attributes of the mounter.
+	// This function is called after SetUp()/SetUpAt().
+	GetAttributes() Attributes
+}
+
+// Unmounter interface provides methods to cleanup/unmount the volumes.
+type Unmounter interface {
+	Volume
+	// TearDown unmounts the volume from a self-determined directory and
+	// removes traces of the SetUp procedure.
+	TearDown() error
+	// TearDown unmounts the volume from the specified directory and
+	// removes traces of the SetUp procedure.
+	TearDownAt(dir string) error
+}
+
+// BlockVolumeMapper interface is a mapper interface for block volume.
+type BlockVolumeMapper interface {
+	BlockVolume
+}
+
+// CustomBlockVolumeMapper interface provides custom methods to set up/map the volume.
+type CustomBlockVolumeMapper interface {
+	BlockVolumeMapper
+	// SetUpDevice prepares the volume to the node by the plugin specific way.
+	// For most in-tree plugins, attacher.Attach() and attacher.WaitForAttach()
+	// will do necessary works.
+	// This may be called more than once, so implementations must be idempotent.
+	// SetUpDevice returns stagingPath if device setup was successful
+	SetUpDevice() (stagingPath string, err error)
+
+	// MapPodDevice maps the block device to a path and return the path.
+	// Unique device path across kubelet node reboot is required to avoid
+	// unexpected block volume destruction.
+	// If empty string is returned, the path retuned by attacher.Attach() and
+	// attacher.WaitForAttach() will be used.
+	MapPodDevice() (publishPath string, err error)
+
+	// GetStagingPath returns path that was used for staging the volume
+	// it is mainly used by CSI plugins
+	GetStagingPath() string
+}
+
+// BlockVolumeUnmapper interface is an unmapper interface for block volume.
+type BlockVolumeUnmapper interface {
+	BlockVolume
+}
+
+// CustomBlockVolumeUnmapper interface provides custom methods to cleanup/unmap the volumes.
+type CustomBlockVolumeUnmapper interface {
+	BlockVolumeUnmapper
+	// TearDownDevice removes traces of the SetUpDevice procedure.
+	// If the plugin is non-attachable, this method detaches the volume
+	// from a node.
+	TearDownDevice(mapPath string, devicePath string) error
+
+	// UnmapPodDevice removes traces of the MapPodDevice procedure.
+	UnmapPodDevice() error
+}
+
+// Provisioner is an interface that creates templates for PersistentVolumes
+// and can create the volume as a new resource in the infrastructure provider.
+type Provisioner interface {
+	// Provision creates the resource by allocating the underlying volume in a
+	// storage system. This method should block until completion and returns
+	// PersistentVolume representing the created storage resource.
+	Provision(selectedNode *v1.Node, allowedTopologies []v1.TopologySelectorTerm) (*v1.PersistentVolume, error)
+}
+
+// Deleter removes the resource from the underlying storage provider. Calls
+// to this method should block until the deletion is complete. Any error
+// returned indicates the volume has failed to be reclaimed. A nil return
+// indicates success.
+type Deleter interface {
+	Volume
+	// This method should block until completion.
+	// deletedVolumeInUseError returned from this function will not be reported
+	// as error and it will be sent as "Info" event to the PV being deleted. The
+	// volume controller will retry deleting the volume in the next periodic
+	// sync. This can be used to postpone deletion of a volume that is being
+	// detached from a node. Deletion of such volume would fail anyway and such
+	// error would confuse users.
+	Delete() error
+}
+
+// Attacher can attach a volume to a node.
+type Attacher interface {
+	DeviceMounter
+
+	// Attaches the volume specified by the given spec to the node with the given Name.
+	// On success, returns the device path where the device was attached on the
+	// node.
+	Attach(spec *Spec, nodeName types.NodeName) (string, error)
+
+	// VolumesAreAttached checks whether the list of volumes still attached to the specified
+	// node. It returns a map which maps from the volume spec to the checking result.
+	// If an error is occurred during checking, the error will be returned
+	VolumesAreAttached(specs []*Spec, nodeName types.NodeName) (map[*Spec]bool, error)
+
+	// WaitForAttach blocks until the device is attached to this
+	// node. If it successfully attaches, the path to the device
+	// is returned. Otherwise, if the device does not attach after
+	// the given timeout period, an error will be returned.
+	WaitForAttach(spec *Spec, devicePath string, pod *v1.Pod, timeout time.Duration) (string, error)
+}
+
+// DeviceMounter can mount a block volume to a global path.
+type DeviceMounter interface {
+	// GetDeviceMountPath returns a path where the device should
+	// be mounted after it is attached. This is a global mount
+	// point which should be bind mounted for individual volumes.
+	GetDeviceMountPath(spec *Spec) (string, error)
+
+	// MountDevice mounts the disk to a global path which
+	// individual pods can then bind mount
+	// Note that devicePath can be empty if the volume plugin does not implement any of Attach and WaitForAttach methods.
+	// It could return following types of errors:
+	//   - TransientOperationFailure
+	//   - UncertainProgressError
+	//   - Error of any other type should be considered a final error
+	MountDevice(spec *Spec, devicePath string, deviceMountPath string) error
+}
+
+type BulkVolumeVerifier interface {
+	// BulkVerifyVolumes checks whether the list of volumes still attached to the
+	// the clusters in the node. It returns a map which maps from the volume spec to the checking result.
+	// If an error occurs during check - error should be returned and volume on nodes
+	// should be assumed as still attached.
+	BulkVerifyVolumes(volumesByNode map[types.NodeName][]*Spec) (map[types.NodeName]map[*Spec]bool, error)
+}
+
+// Detacher can detach a volume from a node.
+type Detacher interface {
+	DeviceUnmounter
+	// Detach the given volume from the node with the given Name.
+	// volumeName is name of the volume as returned from plugin's
+	// GetVolumeName().
+	Detach(volumeName string, nodeName types.NodeName) error
+}
+
+// DeviceUnmounter can unmount a block volume from the global path.
+type DeviceUnmounter interface {
+	// UnmountDevice unmounts the global mount of the disk. This
+	// should only be called once all bind mounts have been
+	// unmounted.
+	UnmountDevice(deviceMountPath string) error
+}

vendor/k8s.io/kubernetes/pkg/volume/volume_linux.go

@@ -0,0 +1,241 @@
+// +build linux
+
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package volume
+
+import (
+	"path/filepath"
+	"syscall"
+
+	"os"
+	"time"
+
+	v1 "k8s.io/api/core/v1"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/klog/v2"
+	"k8s.io/kubernetes/pkg/features"
+)
+
+const (
+	rwMask   = os.FileMode(0660)
+	roMask   = os.FileMode(0440)
+	execMask = os.FileMode(0110)
+)
+
+// SetVolumeOwnership modifies the given volume to be owned by
+// fsGroup, and sets SetGid so that newly created files are owned by
+// fsGroup. If fsGroup is nil nothing is done.
+func SetVolumeOwnership(mounter Mounter, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy, completeFunc func(*error)) error {
+	if fsGroup == nil {
+		return nil
+	}
+
+	fsGroupPolicyEnabled := utilfeature.DefaultFeatureGate.Enabled(features.ConfigurableFSGroupPolicy)
+
+	timer := time.AfterFunc(30*time.Second, func() {
+		klog.Warningf("Setting volume ownership for %s and fsGroup set. If the volume has a lot of files then setting volume ownership could be slow, see https://github.com/kubernetes/kubernetes/issues/69699", mounter.GetPath())
+	})
+	defer timer.Stop()
+
+	// This code exists for legacy purposes, so as old behaviour is entirely preserved when feature gate is disabled
+	// TODO: remove this when ConfigurableFSGroupPolicy turns GA.
+	if !fsGroupPolicyEnabled {
+		err := legacyOwnershipChange(mounter, fsGroup)
+		if completeFunc != nil {
+			completeFunc(&err)
+		}
+		return err
+	}
+
+	if skipPermissionChange(mounter, fsGroup, fsGroupChangePolicy) {
+		klog.V(3).Infof("skipping permission and ownership change for volume %s", mounter.GetPath())
+		return nil
+	}
+
+	err := walkDeep(mounter.GetPath(), func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		return changeFilePermission(path, fsGroup, mounter.GetAttributes().ReadOnly, info)
+	})
+	if completeFunc != nil {
+		completeFunc(&err)
+	}
+	return err
+}
+
+func legacyOwnershipChange(mounter Mounter, fsGroup *int64) error {
+	return filepath.Walk(mounter.GetPath(), func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		return changeFilePermission(path, fsGroup, mounter.GetAttributes().ReadOnly, info)
+	})
+}
+
+func changeFilePermission(filename string, fsGroup *int64, readonly bool, info os.FileInfo) error {
+	// chown and chmod pass through to the underlying file for symlinks.
+	// Symlinks have a mode of 777 but this really doesn't mean anything.
+	// The permissions of the underlying file are what matter.
+	// However, if one reads the mode of a symlink then chmods the symlink
+	// with that mode, it changes the mode of the underlying file, overridden
+	// the defaultMode and permissions initialized by the volume plugin, which
+	// is not what we want; thus, we skip chown/chmod for symlinks.
+	if info.Mode()&os.ModeSymlink != 0 {
+		return nil
+	}
+
+	stat, ok := info.Sys().(*syscall.Stat_t)
+	if !ok {
+		return nil
+	}
+
+	if stat == nil {
+		klog.Errorf("Got nil stat_t for path %v while setting ownership of volume", filename)
+		return nil
+	}
+
+	err := os.Chown(filename, int(stat.Uid), int(*fsGroup))
+	if err != nil {
+		klog.Errorf("Chown failed on %v: %v", filename, err)
+	}
+
+	mask := rwMask
+	if readonly {
+		mask = roMask
+	}
+
+	if info.IsDir() {
+		mask |= os.ModeSetgid
+		mask |= execMask
+	}
+
+	err = os.Chmod(filename, info.Mode()|mask)
+	if err != nil {
+		klog.Errorf("Chmod failed on %v: %v", filename, err)
+	}
+
+	return nil
+}
+
+func skipPermissionChange(mounter Mounter, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy) bool {
+	dir := mounter.GetPath()
+
+	if fsGroupChangePolicy == nil || *fsGroupChangePolicy != v1.FSGroupChangeOnRootMismatch {
+		klog.V(4).Infof("perform recursive ownership change for %s", dir)
+		return false
+	}
+	return !requiresPermissionChange(mounter.GetPath(), fsGroup, mounter.GetAttributes().ReadOnly)
+}
+
+func requiresPermissionChange(rootDir string, fsGroup *int64, readonly bool) bool {
+	fsInfo, err := os.Stat(rootDir)
+	if err != nil {
+		klog.Errorf("performing recursive ownership change on %s because reading permissions of root volume failed: %v", rootDir, err)
+		return true
+	}
+	stat, ok := fsInfo.Sys().(*syscall.Stat_t)
+	if !ok || stat == nil {
+		klog.Errorf("performing recursive ownership change on %s because reading permissions of root volume failed", rootDir)
+		return true
+	}
+
+	if int(stat.Gid) != int(*fsGroup) {
+		klog.V(4).Infof("expected group ownership of volume %s did not match with: %d", rootDir, stat.Gid)
+		return true
+	}
+	unixPerms := rwMask
+
+	if readonly {
+		unixPerms = roMask
+	}
+
+	// if rootDir is not a directory then we should apply permission change anyways
+	if !fsInfo.IsDir() {
+		return true
+	}
+	unixPerms |= execMask
+	filePerm := fsInfo.Mode().Perm()
+
+	// We need to check if actual permissions of root directory is a superset of permissions required by unixPerms.
+	// This is done by checking if permission bits expected in unixPerms is set in actual permissions of the directory.
+	// We use bitwise AND operation to check set bits. For example:
+	//     unixPerms: 770, filePerms: 775 : 770&775 = 770 (perms on directory is a superset)
+	//     unixPerms: 770, filePerms: 770 : 770&770 = 770 (perms on directory is a superset)
+	//     unixPerms: 770, filePerms: 750 : 770&750 = 750 (perms on directory is NOT a superset)
+	// We also need to check if setgid bits are set in permissions of the directory.
+	if (unixPerms&filePerm != unixPerms) || (fsInfo.Mode()&os.ModeSetgid == 0) {
+		klog.V(4).Infof("performing recursive ownership change on %s because of mismatching mode", rootDir)
+		return true
+	}
+	return false
+}
+
+// readDirNames reads the directory named by dirname and returns
+// a list of directory entries.
+// We are not using filepath.readDirNames because we do not want to sort files found in a directory before changing
+// permissions for performance reasons.
+func readDirNames(dirname string) ([]string, error) {
+	f, err := os.Open(dirname)
+	if err != nil {
+		return nil, err
+	}
+	names, err := f.Readdirnames(-1)
+	f.Close()
+	if err != nil {
+		return nil, err
+	}
+	return names, nil
+}
+
+// walkDeep can be used to traverse directories and has two minor differences
+// from filepath.Walk:
+//   - List of files/dirs is not sorted for performance reasons
+//   - callback walkFunc is invoked on root directory after visiting children dirs and files
+func walkDeep(root string, walkFunc filepath.WalkFunc) error {
+	info, err := os.Lstat(root)
+	if err != nil {
+		return walkFunc(root, nil, err)
+	}
+	return walk(root, info, walkFunc)
+}
+
+func walk(path string, info os.FileInfo, walkFunc filepath.WalkFunc) error {
+	if !info.IsDir() {
+		return walkFunc(path, info, nil)
+	}
+	names, err := readDirNames(path)
+	if err != nil {
+		return err
+	}
+	for _, name := range names {
+		filename := filepath.Join(path, name)
+		fileInfo, err := os.Lstat(filename)
+		if err != nil {
+			if err := walkFunc(filename, fileInfo, err); err != nil {
+				return err
+			}
+		} else {
+			err = walk(filename, fileInfo, walkFunc)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return walkFunc(path, info, nil)
+}

vendor/k8s.io/kubernetes/pkg/volume/volume_unsupported.go

@@ -0,0 +1,27 @@
+// +build !linux
+
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package volume
+
+import (
+	v1 "k8s.io/api/core/v1"
+)
+
+func SetVolumeOwnership(mounter Mounter, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy, completeFunc func(*error)) error {
+	return nil
+}

vendor/k8s.io/mount-utils/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

vendor/k8s.io/mount-utils/OWNERS

@@ -0,0 +1,14 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+reviewers:
+  - jingxu97
+  - saad-ali
+  - jsafrane
+  - msau42
+  - andyzhangx
+  - gnufied
+approvers:
+  - jingxu97
+  - saad-ali
+  - jsafrane
+

vendor/k8s.io/mount-utils/README.md

@@ -0,0 +1,30 @@
+## Purpose
+
+This repository defines an interface to mounting filesystems to be consumed by 
+various Kubernetes and out-of-tree CSI components. 
+
+Consumers of this repository can make use of functions like 'Mount' to mount 
+source to target as fstype with given options, 'Unmount' to unmount a target.
+Other useful functions include 'List' all mounted file systems and find all
+mount references to a path using 'GetMountRefs'
+
+## Community, discussion, contribution, and support
+
+Learn how to engage with the Kubernetes community on the [community
+page](http://kubernetes.io/community/).
+
+You can reach the maintainers of this repository at:
+
+- Slack: #sig-storage (on https://kubernetes.slack.com -- get an
+  invite at slack.kubernetes.io)
+- Mailing List:
+  https://groups.google.com/forum/#!forum/kubernetes-sig-storage
+
+### Code of Conduct
+
+Participation in the Kubernetes community is governed by the [Kubernetes
+Code of Conduct](code-of-conduct.md).
+
+### Contibution Guidelines
+
+See [CONTRIBUTING.md](CONTRIBUTING.md) for more information.

vendor/k8s.io/mount-utils/SECURITY_CONTACTS

@@ -0,0 +1,18 @@
+# Defined below are the security contacts for this repo.
+#
+# They are the contact point for the Product Security Committee to reach out
+# to for triaging and handling of incoming issues.
+#
+# The below names agree to abide by the
+# [Embargo Policy](https://git.k8s.io/security/private-distributors-list.md#embargo-policy)
+# and will be removed and replaced if they violate that agreement.
+#
+# DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE
+# INSTRUCTIONS AT https://kubernetes.io/security/
+
+saad-ali
+cjcullen
+joelsmith
+liggitt
+philips
+tallclair

vendor/k8s.io/mount-utils/code-of-conduct.md

@@ -0,0 +1,3 @@
+# Kubernetes Community Code of Conduct
+
+Please refer to our [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md)

vendor/k8s.io/mount-utils/doc.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package mount defines an interface to mounting filesystems.
+package mount // import "k8s.io/mount-utils"

vendor/k8s.io/mount-utils/fake_mounter.go

@@ -0,0 +1,220 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mount
+
+import (
+	"os"
+	"path/filepath"
+	"sync"
+
+	"k8s.io/klog/v2"
+)
+
+// FakeMounter implements mount.Interface for tests.
+type FakeMounter struct {
+	MountPoints []MountPoint
+	log         []FakeAction
+	// Error to return for a path when calling IsLikelyNotMountPoint
+	MountCheckErrors map[string]error
+	// Some tests run things in parallel, make sure the mounter does not produce
+	// any golang's DATA RACE warnings.
+	mutex       sync.Mutex
+	UnmountFunc UnmountFunc
+}
+
+// UnmountFunc is a function callback to be executed during the Unmount() call.
+type UnmountFunc func(path string) error
+
+var _ Interface = &FakeMounter{}
+
+const (
+	// FakeActionMount is the string for specifying mount as FakeAction.Action
+	FakeActionMount = "mount"
+	// FakeActionUnmount is the string for specifying unmount as FakeAction.Action
+	FakeActionUnmount = "unmount"
+)
+
+// FakeAction objects are logged every time a fake mount or unmount is called.
+type FakeAction struct {
+	Action string // "mount" or "unmount"
+	Target string // applies to both mount and unmount actions
+	Source string // applies only to "mount" actions
+	FSType string // applies only to "mount" actions
+}
+
+// NewFakeMounter returns a FakeMounter struct that implements Interface and is
+// suitable for testing purposes.
+func NewFakeMounter(mps []MountPoint) *FakeMounter {
+	return &FakeMounter{
+		MountPoints: mps,
+	}
+}
+
+// ResetLog clears all the log entries in FakeMounter
+func (f *FakeMounter) ResetLog() {
+	f.mutex.Lock()
+	defer f.mutex.Unlock()
+
+	f.log = []FakeAction{}
+}
+
+// GetLog returns the slice of FakeActions taken by the mounter
+func (f *FakeMounter) GetLog() []FakeAction {
+	f.mutex.Lock()
+	defer f.mutex.Unlock()
+
+	return f.log
+}
+
+// Mount records the mount event and updates the in-memory mount points for FakeMounter
+func (f *FakeMounter) Mount(source string, target string, fstype string, options []string) error {
+	return f.MountSensitive(source, target, fstype, options, nil /* sensitiveOptions */)
+}
+
+// Mount records the mount event and updates the in-memory mount points for FakeMounter
+// sensitiveOptions to be passed in a separate parameter from the normal
+// mount options and ensures the sensitiveOptions are never logged. This
+// method should be used by callers that pass sensitive material (like
+// passwords) as mount options.
+func (f *FakeMounter) MountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error {
+	f.mutex.Lock()
+	defer f.mutex.Unlock()
+
+	opts := []string{}
+
+	for _, option := range options {
+		// find 'bind' option
+		if option == "bind" {
+			// This is a bind-mount. In order to mimic linux behaviour, we must
+			// use the original device of the bind-mount as the real source.
+			// E.g. when mounted /dev/sda like this:
+			//      $ mount /dev/sda /mnt/test
+			//      $ mount -o bind /mnt/test /mnt/bound
+			// then /proc/mount contains:
+			// /dev/sda /mnt/test
+			// /dev/sda /mnt/bound
+			// (and not /mnt/test /mnt/bound)
+			// I.e. we must use /dev/sda as source instead of /mnt/test in the
+			// bind mount.
+			for _, mnt := range f.MountPoints {
+				if source == mnt.Path {
+					source = mnt.Device
+					break
+				}
+			}
+		}
+		// reuse MountPoint.Opts field to mark mount as readonly
+		opts = append(opts, option)
+	}
+
+	// If target is a symlink, get its absolute path
+	absTarget, err := filepath.EvalSymlinks(target)
+	if err != nil {
+		absTarget = target
+	}
+	f.MountPoints = append(f.MountPoints, MountPoint{Device: source, Path: absTarget, Type: fstype, Opts: append(opts, sensitiveOptions...)})
+	klog.V(5).Infof("Fake mounter: mounted %s to %s", source, absTarget)
+	f.log = append(f.log, FakeAction{Action: FakeActionMount, Target: absTarget, Source: source, FSType: fstype})
+	return nil
+}
+
+func (f *FakeMounter) MountSensitiveWithoutSystemd(source string, target string, fstype string, options []string, sensitiveOptions []string) error {
+	return f.MountSensitive(source, target, fstype, options, nil /* sensitiveOptions */)
+}
+
+// Unmount records the unmount event and updates the in-memory mount points for FakeMounter
+func (f *FakeMounter) Unmount(target string) error {
+	f.mutex.Lock()
+	defer f.mutex.Unlock()
+
+	// If target is a symlink, get its absolute path
+	absTarget, err := filepath.EvalSymlinks(target)
+	if err != nil {
+		absTarget = target
+	}
+
+	newMountpoints := []MountPoint{}
+	for _, mp := range f.MountPoints {
+		if mp.Path == absTarget {
+			if f.UnmountFunc != nil {
+				err := f.UnmountFunc(absTarget)
+				if err != nil {
+					return err
+				}
+			}
+			klog.V(5).Infof("Fake mounter: unmounted %s from %s", mp.Device, absTarget)
+			// Don't copy it to newMountpoints
+			continue
+		}
+		newMountpoints = append(newMountpoints, MountPoint{Device: mp.Device, Path: mp.Path, Type: mp.Type})
+	}
+	f.MountPoints = newMountpoints
+	f.log = append(f.log, FakeAction{Action: FakeActionUnmount, Target: absTarget})
+	delete(f.MountCheckErrors, target)
+	return nil
+}
+
+// List returns all the in-memory mountpoints for FakeMounter
+func (f *FakeMounter) List() ([]MountPoint, error) {
+	f.mutex.Lock()
+	defer f.mutex.Unlock()
+
+	return f.MountPoints, nil
+}
+
+// IsLikelyNotMountPoint determines whether a path is a mountpoint by checking
+// if the absolute path to file is in the in-memory mountpoints
+func (f *FakeMounter) IsLikelyNotMountPoint(file string) (bool, error) {
+	f.mutex.Lock()
+	defer f.mutex.Unlock()
+
+	err := f.MountCheckErrors[file]
+	if err != nil {
+		return false, err
+	}
+
+	_, err = os.Stat(file)
+	if err != nil {
+		return true, err
+	}
+
+	// If file is a symlink, get its absolute path
+	absFile, err := filepath.EvalSymlinks(file)
+	if err != nil {
+		absFile = file
+	}
+
+	for _, mp := range f.MountPoints {
+		if mp.Path == absFile {
+			klog.V(5).Infof("isLikelyNotMountPoint for %s: mounted %s, false", file, mp.Path)
+			return false, nil
+		}
+	}
+	klog.V(5).Infof("isLikelyNotMountPoint for %s: true", file)
+	return true, nil
+}
+
+// GetMountRefs finds all mount references to the path, returns a
+// list of paths.
+func (f *FakeMounter) GetMountRefs(pathname string) ([]string, error) {
+	realpath, err := filepath.EvalSymlinks(pathname)
+	if err != nil {
+		// Ignore error in FakeMounter, because we actually didn't create files.
+		realpath = pathname
+	}
+	return getMountRefsByDev(f, realpath)
+}

vendor/k8s.io/mount-utils/go.mod

@@ -0,0 +1,13 @@
+// This is a generated file. Do not edit directly.
+
+module k8s.io/mount-utils
+
+go 1.15
+
+require (
+	github.com/kr/pretty v0.2.0 // indirect
+	github.com/stretchr/testify v1.6.1
+	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
+	k8s.io/klog/v2 v2.4.0
+	k8s.io/utils v0.0.0-20201110183641-67b214c5f920
+)

vendor/k8s.io/mount-utils/go.sum

@@ -0,0 +1,32 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg=
+github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
+github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY=
+github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+k8s.io/klog/v2 v2.0.0 h1:Foj74zO6RbjjP4hBEKjnYtjjAhGg4jNynUdYF6fJrok=
+k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
+k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ=
+k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
+k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw=
+k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=

vendor/k8s.io/mount-utils/mount.go

@@ -0,0 +1,368 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// TODO(thockin): This whole pkg is pretty linux-centric.  As soon as we have
+// an alternate platform, we will need to abstract further.
+
+package mount
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	utilexec "k8s.io/utils/exec"
+)
+
+const (
+	// Default mount command if mounter path is not specified.
+	defaultMountCommand = "mount"
+	// Log message where sensitive mount options were removed
+	sensitiveOptionsRemoved = "<masked>"
+)
+
+// Interface defines the set of methods to allow for mount operations on a system.
+type Interface interface {
+	// Mount mounts source to target as fstype with given options.
+	// options MUST not contain sensitive material (like passwords).
+	Mount(source string, target string, fstype string, options []string) error
+	// MountSensitive is the same as Mount() but this method allows
+	// sensitiveOptions to be passed in a separate parameter from the normal
+	// mount options and ensures the sensitiveOptions are never logged. This
+	// method should be used by callers that pass sensitive material (like
+	// passwords) as mount options.
+	MountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error
+	// MountSensitiveWithoutSystemd is the same as MountSensitive() but this method disable using systemd mount.
+	MountSensitiveWithoutSystemd(source string, target string, fstype string, options []string, sensitiveOptions []string) error
+	// Unmount unmounts given target.
+	Unmount(target string) error
+	// List returns a list of all mounted filesystems.  This can be large.
+	// On some platforms, reading mounts directly from the OS is not guaranteed
+	// consistent (i.e. it could change between chunked reads). This is guaranteed
+	// to be consistent.
+	List() ([]MountPoint, error)
+	// IsLikelyNotMountPoint uses heuristics to determine if a directory
+	// is not a mountpoint.
+	// It should return ErrNotExist when the directory does not exist.
+	// IsLikelyNotMountPoint does NOT properly detect all mountpoint types
+	// most notably linux bind mounts and symbolic link. For callers that do not
+	// care about such situations, this is a faster alternative to calling List()
+	// and scanning that output.
+	IsLikelyNotMountPoint(file string) (bool, error)
+	// GetMountRefs finds all mount references to pathname, returning a slice of
+	// paths. Pathname can be a mountpoint path or a normal	directory
+	// (for bind mount). On Linux, pathname is excluded from the slice.
+	// For example, if /dev/sdc was mounted at /path/a and /path/b,
+	// GetMountRefs("/path/a") would return ["/path/b"]
+	// GetMountRefs("/path/b") would return ["/path/a"]
+	// On Windows there is no way to query all mount points; as long as pathname is
+	// a valid mount, it will be returned.
+	GetMountRefs(pathname string) ([]string, error)
+}
+
+// Compile-time check to ensure all Mounter implementations satisfy
+// the mount interface.
+var _ Interface = &Mounter{}
+
+// MountPoint represents a single line in /proc/mounts or /etc/fstab.
+type MountPoint struct { // nolint: golint
+	Device string
+	Path   string
+	Type   string
+	Opts   []string // Opts may contain sensitive mount options (like passwords) and MUST be treated as such (e.g. not logged).
+	Freq   int
+	Pass   int
+}
+
+type MountErrorType string // nolint: golint
+
+const (
+	FilesystemMismatch  MountErrorType = "FilesystemMismatch"
+	HasFilesystemErrors MountErrorType = "HasFilesystemErrors"
+	UnformattedReadOnly MountErrorType = "UnformattedReadOnly"
+	FormatFailed        MountErrorType = "FormatFailed"
+	GetDiskFormatFailed MountErrorType = "GetDiskFormatFailed"
+	UnknownMountError   MountErrorType = "UnknownMountError"
+)
+
+type MountError struct { // nolint: golint
+	Type    MountErrorType
+	Message string
+}
+
+func (mountError MountError) String() string {
+	return mountError.Message
+}
+
+func (mountError MountError) Error() string {
+	return mountError.Message
+}
+
+func NewMountError(mountErrorValue MountErrorType, format string, args ...interface{}) error {
+	mountError := MountError{
+		Type:    mountErrorValue,
+		Message: fmt.Sprintf(format, args...),
+	}
+	return mountError
+}
+
+// SafeFormatAndMount probes a device to see if it is formatted.
+// Namely it checks to see if a file system is present. If so it
+// mounts it otherwise the device is formatted first then mounted.
+type SafeFormatAndMount struct {
+	Interface
+	Exec utilexec.Interface
+}
+
+// FormatAndMount formats the given disk, if needed, and mounts it.
+// That is if the disk is not formatted and it is not being mounted as
+// read-only it will format it first then mount it. Otherwise, if the
+// disk is already formatted or it is being mounted as read-only, it
+// will be mounted without formatting.
+// options MUST not contain sensitive material (like passwords).
+func (mounter *SafeFormatAndMount) FormatAndMount(source string, target string, fstype string, options []string) error {
+	return mounter.FormatAndMountSensitive(source, target, fstype, options, nil /* sensitiveOptions */)
+}
+
+// FormatAndMountSensitive is the same as FormatAndMount but this method allows
+// sensitiveOptions to be passed in a separate parameter from the normal mount
+// options and ensures the sensitiveOptions are never logged. This method should
+// be used by callers that pass sensitive material (like passwords) as mount
+// options.
+func (mounter *SafeFormatAndMount) FormatAndMountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error {
+	return mounter.formatAndMountSensitive(source, target, fstype, options, sensitiveOptions)
+}
+
+// getMountRefsByDev finds all references to the device provided
+// by mountPath; returns a list of paths.
+// Note that mountPath should be path after the evaluation of any symblolic links.
+func getMountRefsByDev(mounter Interface, mountPath string) ([]string, error) {
+	mps, err := mounter.List()
+	if err != nil {
+		return nil, err
+	}
+
+	// Finding the device mounted to mountPath.
+	diskDev := ""
+	for i := range mps {
+		if mountPath == mps[i].Path {
+			diskDev = mps[i].Device
+			break
+		}
+	}
+
+	// Find all references to the device.
+	var refs []string
+	for i := range mps {
+		if mps[i].Device == diskDev || mps[i].Device == mountPath {
+			if mps[i].Path != mountPath {
+				refs = append(refs, mps[i].Path)
+			}
+		}
+	}
+	return refs, nil
+}
+
+// GetDeviceNameFromMount given a mnt point, find the device from /proc/mounts
+// returns the device name, reference count, and error code.
+func GetDeviceNameFromMount(mounter Interface, mountPath string) (string, int, error) {
+	mps, err := mounter.List()
+	if err != nil {
+		return "", 0, err
+	}
+
+	// Find the device name.
+	// FIXME if multiple devices mounted on the same mount path, only the first one is returned.
+	device := ""
+	// If mountPath is symlink, need get its target path.
+	slTarget, err := filepath.EvalSymlinks(mountPath)
+	if err != nil {
+		slTarget = mountPath
+	}
+	for i := range mps {
+		if mps[i].Path == slTarget {
+			device = mps[i].Device
+			break
+		}
+	}
+
+	// Find all references to the device.
+	refCount := 0
+	for i := range mps {
+		if mps[i].Device == device {
+			refCount++
+		}
+	}
+	return device, refCount, nil
+}
+
+// IsNotMountPoint determines if a directory is a mountpoint.
+// It should return ErrNotExist when the directory does not exist.
+// IsNotMountPoint is more expensive than IsLikelyNotMountPoint.
+// IsNotMountPoint detects bind mounts in linux.
+// IsNotMountPoint enumerates all the mountpoints using List() and
+// the list of mountpoints may be large, then it uses
+// isMountPointMatch to evaluate whether the directory is a mountpoint.
+func IsNotMountPoint(mounter Interface, file string) (bool, error) {
+	// IsLikelyNotMountPoint provides a quick check
+	// to determine whether file IS A mountpoint.
+	notMnt, notMntErr := mounter.IsLikelyNotMountPoint(file)
+	if notMntErr != nil && os.IsPermission(notMntErr) {
+		// We were not allowed to do the simple stat() check, e.g. on NFS with
+		// root_squash. Fall back to /proc/mounts check below.
+		notMnt = true
+		notMntErr = nil
+	}
+	if notMntErr != nil {
+		return notMnt, notMntErr
+	}
+	// identified as mountpoint, so return this fact.
+	if notMnt == false {
+		return notMnt, nil
+	}
+
+	// Resolve any symlinks in file, kernel would do the same and use the resolved path in /proc/mounts.
+	resolvedFile, err := filepath.EvalSymlinks(file)
+	if err != nil {
+		return true, err
+	}
+
+	// check all mountpoints since IsLikelyNotMountPoint
+	// is not reliable for some mountpoint types.
+	mountPoints, mountPointsErr := mounter.List()
+	if mountPointsErr != nil {
+		return notMnt, mountPointsErr
+	}
+	for _, mp := range mountPoints {
+		if isMountPointMatch(mp, resolvedFile) {
+			notMnt = false
+			break
+		}
+	}
+	return notMnt, nil
+}
+
+// MakeBindOpts detects whether a bind mount is being requested and makes the remount options to
+// use in case of bind mount, due to the fact that bind mount doesn't respect mount options.
+// The list equals:
+//   options - 'bind' + 'remount' (no duplicate)
+func MakeBindOpts(options []string) (bool, []string, []string) {
+	bind, bindOpts, bindRemountOpts, _ := MakeBindOptsSensitive(options, nil /* sensitiveOptions */)
+	return bind, bindOpts, bindRemountOpts
+}
+
+// MakeBindOptsSensitive is the same as MakeBindOpts but this method allows
+// sensitiveOptions to be passed in a separate parameter from the normal mount
+// options and ensures the sensitiveOptions are never logged. This method should
+// be used by callers that pass sensitive material (like passwords) as mount
+// options.
+func MakeBindOptsSensitive(options []string, sensitiveOptions []string) (bool, []string, []string, []string) {
+	// Because we have an FD opened on the subpath bind mount, the "bind" option
+	// needs to be included, otherwise the mount target will error as busy if you
+	// remount as readonly.
+	//
+	// As a consequence, all read only bind mounts will no longer change the underlying
+	// volume mount to be read only.
+	bindRemountOpts := []string{"bind", "remount"}
+	bindRemountSensitiveOpts := []string{}
+	bind := false
+	bindOpts := []string{"bind"}
+
+	// _netdev is a userspace mount option and does not automatically get added when
+	// bind mount is created and hence we must carry it over.
+	if checkForNetDev(options, sensitiveOptions) {
+		bindOpts = append(bindOpts, "_netdev")
+	}
+
+	for _, option := range options {
+		switch option {
+		case "bind":
+			bind = true
+		case "remount":
+		default:
+			bindRemountOpts = append(bindRemountOpts, option)
+		}
+	}
+
+	for _, sensitiveOption := range sensitiveOptions {
+		switch sensitiveOption {
+		case "bind":
+			bind = true
+		case "remount":
+		default:
+			bindRemountSensitiveOpts = append(bindRemountSensitiveOpts, sensitiveOption)
+		}
+	}
+
+	return bind, bindOpts, bindRemountOpts, bindRemountSensitiveOpts
+}
+
+func checkForNetDev(options []string, sensitiveOptions []string) bool {
+	for _, option := range options {
+		if option == "_netdev" {
+			return true
+		}
+	}
+	for _, sensitiveOption := range sensitiveOptions {
+		if sensitiveOption == "_netdev" {
+			return true
+		}
+	}
+	return false
+}
+
+// PathWithinBase checks if give path is within given base directory.
+func PathWithinBase(fullPath, basePath string) bool {
+	rel, err := filepath.Rel(basePath, fullPath)
+	if err != nil {
+		return false
+	}
+	if StartsWithBackstep(rel) {
+		// Needed to escape the base path.
+		return false
+	}
+	return true
+}
+
+// StartsWithBackstep checks if the given path starts with a backstep segment.
+func StartsWithBackstep(rel string) bool {
+	// normalize to / and check for ../
+	return rel == ".." || strings.HasPrefix(filepath.ToSlash(rel), "../")
+}
+
+// sanitizedOptionsForLogging will return a comma separated string containing
+// options and sensitiveOptions. Each entry in sensitiveOptions will be
+// replaced with the string sensitiveOptionsRemoved
+// e.g. o1,o2,<masked>,<masked>
+func sanitizedOptionsForLogging(options []string, sensitiveOptions []string) string {
+	separator := ""
+	if len(options) > 0 && len(sensitiveOptions) > 0 {
+		separator = ","
+	}
+
+	sensitiveOptionsStart := ""
+	sensitiveOptionsEnd := ""
+	if len(sensitiveOptions) > 0 {
+		sensitiveOptionsStart = strings.Repeat(sensitiveOptionsRemoved+",", len(sensitiveOptions)-1)
+		sensitiveOptionsEnd = sensitiveOptionsRemoved
+	}
+
+	return strings.Join(options, ",") +
+		separator +
+		sensitiveOptionsStart +
+		sensitiveOptionsEnd
+}

vendor/k8s.io/mount-utils/mount_helper_common.go

@@ -0,0 +1,103 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mount
+
+import (
+	"fmt"
+	"os"
+
+	"k8s.io/klog/v2"
+)
+
+// CleanupMountPoint unmounts the given path and deletes the remaining directory
+// if successful. If extensiveMountPointCheck is true IsNotMountPoint will be
+// called instead of IsLikelyNotMountPoint. IsNotMountPoint is more expensive
+// but properly handles bind mounts within the same fs.
+func CleanupMountPoint(mountPath string, mounter Interface, extensiveMountPointCheck bool) error {
+	pathExists, pathErr := PathExists(mountPath)
+	if !pathExists {
+		klog.Warningf("Warning: Unmount skipped because path does not exist: %v", mountPath)
+		return nil
+	}
+	corruptedMnt := IsCorruptedMnt(pathErr)
+	if pathErr != nil && !corruptedMnt {
+		return fmt.Errorf("Error checking path: %v", pathErr)
+	}
+	return doCleanupMountPoint(mountPath, mounter, extensiveMountPointCheck, corruptedMnt)
+}
+
+// doCleanupMountPoint unmounts the given path and
+// deletes the remaining directory if successful.
+// if extensiveMountPointCheck is true
+// IsNotMountPoint will be called instead of IsLikelyNotMountPoint.
+// IsNotMountPoint is more expensive but properly handles bind mounts within the same fs.
+// if corruptedMnt is true, it means that the mountPath is a corrupted mountpoint, and the mount point check
+// will be skipped
+func doCleanupMountPoint(mountPath string, mounter Interface, extensiveMountPointCheck bool, corruptedMnt bool) error {
+	var notMnt bool
+	var err error
+	if !corruptedMnt {
+		if extensiveMountPointCheck {
+			notMnt, err = IsNotMountPoint(mounter, mountPath)
+		} else {
+			notMnt, err = mounter.IsLikelyNotMountPoint(mountPath)
+		}
+
+		if err != nil {
+			return err
+		}
+
+		if notMnt {
+			klog.Warningf("Warning: %q is not a mountpoint, deleting", mountPath)
+			return os.Remove(mountPath)
+		}
+	}
+
+	// Unmount the mount path
+	klog.V(4).Infof("%q is a mountpoint, unmounting", mountPath)
+	if err := mounter.Unmount(mountPath); err != nil {
+		return err
+	}
+
+	if extensiveMountPointCheck {
+		notMnt, err = IsNotMountPoint(mounter, mountPath)
+	} else {
+		notMnt, err = mounter.IsLikelyNotMountPoint(mountPath)
+	}
+	if err != nil {
+		return err
+	}
+	if notMnt {
+		klog.V(4).Infof("%q is unmounted, deleting the directory", mountPath)
+		return os.Remove(mountPath)
+	}
+	return fmt.Errorf("Failed to unmount path %v", mountPath)
+}
+
+// PathExists returns true if the specified path exists.
+// TODO: clean this up to use pkg/util/file/FileExists
+func PathExists(path string) (bool, error) {
+	_, err := os.Stat(path)
+	if err == nil {
+		return true, nil
+	} else if os.IsNotExist(err) {
+		return false, nil
+	} else if IsCorruptedMnt(err) {
+		return true, err
+	}
+	return false, err
+}

vendor/k8s.io/mount-utils/mount_helper_unix.go

@@ -0,0 +1,158 @@
+// +build !windows
+
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mount
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"syscall"
+
+	utilio "k8s.io/utils/io"
+)
+
+const (
+	// At least number of fields per line in /proc/<pid>/mountinfo.
+	expectedAtLeastNumFieldsPerMountInfo = 10
+	// How many times to retry for a consistent read of /proc/mounts.
+	maxListTries = 3
+)
+
+// IsCorruptedMnt return true if err is about corrupted mount point
+func IsCorruptedMnt(err error) bool {
+	if err == nil {
+		return false
+	}
+	var underlyingError error
+	switch pe := err.(type) {
+	case nil:
+		return false
+	case *os.PathError:
+		underlyingError = pe.Err
+	case *os.LinkError:
+		underlyingError = pe.Err
+	case *os.SyscallError:
+		underlyingError = pe.Err
+	}
+
+	return underlyingError == syscall.ENOTCONN || underlyingError == syscall.ESTALE || underlyingError == syscall.EIO || underlyingError == syscall.EACCES
+}
+
+// MountInfo represents a single line in /proc/<pid>/mountinfo.
+type MountInfo struct { // nolint: golint
+	// Unique ID for the mount (maybe reused after umount).
+	ID int
+	// The ID of the parent mount (or of self for the root of this mount namespace's mount tree).
+	ParentID int
+	// Major indicates one half of the device ID which identifies the device class
+	// (parsed from `st_dev` for files on this filesystem).
+	Major int
+	// Minor indicates one half of the device ID which identifies a specific
+	// instance of device (parsed from `st_dev` for files on this filesystem).
+	Minor int
+	// The pathname of the directory in the filesystem which forms the root of this mount.
+	Root string
+	// Mount source, filesystem-specific information. e.g. device, tmpfs name.
+	Source string
+	// Mount point, the pathname of the mount point.
+	MountPoint string
+	// Optional fieds, zero or more fields of the form "tag[:value]".
+	OptionalFields []string
+	// The filesystem type in the form "type[.subtype]".
+	FsType string
+	// Per-mount options.
+	MountOptions []string
+	// Per-superblock options.
+	SuperOptions []string
+}
+
+// ParseMountInfo parses /proc/xxx/mountinfo.
+func ParseMountInfo(filename string) ([]MountInfo, error) {
+	content, err := utilio.ConsistentRead(filename, maxListTries)
+	if err != nil {
+		return []MountInfo{}, err
+	}
+	contentStr := string(content)
+	infos := []MountInfo{}
+
+	for _, line := range strings.Split(contentStr, "\n") {
+		if line == "" {
+			// the last split() item is empty string following the last \n
+			continue
+		}
+		// See `man proc` for authoritative description of format of the file.
+		fields := strings.Fields(line)
+		if len(fields) < expectedAtLeastNumFieldsPerMountInfo {
+			return nil, fmt.Errorf("wrong number of fields in (expected at least %d, got %d): %s", expectedAtLeastNumFieldsPerMountInfo, len(fields), line)
+		}
+		id, err := strconv.Atoi(fields[0])
+		if err != nil {
+			return nil, err
+		}
+		parentID, err := strconv.Atoi(fields[1])
+		if err != nil {
+			return nil, err
+		}
+		mm := strings.Split(fields[2], ":")
+		if len(mm) != 2 {
+			return nil, fmt.Errorf("parsing '%s' failed: unexpected minor:major pair %s", line, mm)
+		}
+		major, err := strconv.Atoi(mm[0])
+		if err != nil {
+			return nil, fmt.Errorf("parsing '%s' failed: unable to parse major device id, err:%v", mm[0], err)
+		}
+		minor, err := strconv.Atoi(mm[1])
+		if err != nil {
+			return nil, fmt.Errorf("parsing '%s' failed: unable to parse minor device id, err:%v", mm[1], err)
+		}
+
+		info := MountInfo{
+			ID:           id,
+			ParentID:     parentID,
+			Major:        major,
+			Minor:        minor,
+			Root:         fields[3],
+			MountPoint:   fields[4],
+			MountOptions: strings.Split(fields[5], ","),
+		}
+		// All fields until "-" are "optional fields".
+		i := 6
+		for ; i < len(fields) && fields[i] != "-"; i++ {
+			info.OptionalFields = append(info.OptionalFields, fields[i])
+		}
+		// Parse the rest 3 fields.
+		i++
+		if len(fields)-i < 3 {
+			return nil, fmt.Errorf("expect 3 fields in %s, got %d", line, len(fields)-i)
+		}
+		info.FsType = fields[i]
+		info.Source = fields[i+1]
+		info.SuperOptions = strings.Split(fields[i+2], ",")
+		infos = append(infos, info)
+	}
+	return infos, nil
+}
+
+// isMountPointMatch returns true if the path in mp is the same as dir.
+// Handles case where mountpoint dir has been renamed due to stale NFS mount.
+func isMountPointMatch(mp MountPoint, dir string) bool {
+	deletedDir := fmt.Sprintf("%s\\040(deleted)", dir)
+	return ((mp.Path == dir) || (mp.Path == deletedDir))
+}

vendor/k8s.io/mount-utils/mount_helper_windows.go

@@ -0,0 +1,101 @@
+// +build windows
+
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mount
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"syscall"
+
+	"k8s.io/klog/v2"
+)
+
+// following failure codes are from https://docs.microsoft.com/en-us/windows/desktop/debug/system-error-codes--1300-1699-
+// ERROR_BAD_NETPATH                 = 53
+// ERROR_NETWORK_BUSY                = 54
+// ERROR_UNEXP_NET_ERR               = 59
+// ERROR_NETNAME_DELETED             = 64
+// ERROR_NETWORK_ACCESS_DENIED       = 65
+// ERROR_BAD_DEV_TYPE                = 66
+// ERROR_BAD_NET_NAME                = 67
+// ERROR_SESSION_CREDENTIAL_CONFLICT = 1219
+// ERROR_LOGON_FAILURE               = 1326
+var errorNoList = [...]int{53, 54, 59, 64, 65, 66, 67, 1219, 1326}
+
+// IsCorruptedMnt return true if err is about corrupted mount point
+func IsCorruptedMnt(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	var underlyingError error
+	switch pe := err.(type) {
+	case nil:
+		return false
+	case *os.PathError:
+		underlyingError = pe.Err
+	case *os.LinkError:
+		underlyingError = pe.Err
+	case *os.SyscallError:
+		underlyingError = pe.Err
+	}
+
+	if ee, ok := underlyingError.(syscall.Errno); ok {
+		for _, errno := range errorNoList {
+			if int(ee) == errno {
+				klog.Warningf("IsCorruptedMnt failed with error: %v, error code: %v", err, errno)
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
+// NormalizeWindowsPath makes sure the given path is a valid path on Windows
+// systems by making sure all instances of `/` are replaced with `\\`, and the
+// path beings with `c:`
+func NormalizeWindowsPath(path string) string {
+	normalizedPath := strings.Replace(path, "/", "\\", -1)
+	if strings.HasPrefix(normalizedPath, "\\") {
+		normalizedPath = "c:" + normalizedPath
+	}
+	return normalizedPath
+}
+
+// ValidateDiskNumber : disk number should be a number in [0, 99]
+func ValidateDiskNumber(disk string) error {
+	diskNum, err := strconv.Atoi(disk)
+	if err != nil {
+		return fmt.Errorf("wrong disk number format: %q, err:%v", disk, err)
+	}
+
+	if diskNum < 0 || diskNum > 99 {
+		return fmt.Errorf("disk number out of range: %q", disk)
+	}
+
+	return nil
+}
+
+// isMountPointMatch determines if the mountpoint matches the dir
+func isMountPointMatch(mp MountPoint, dir string) bool {
+	return mp.Path == dir
+}

vendor/k8s.io/mount-utils/mount_linux.go

@@ -0,0 +1,575 @@
+// +build linux
+
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mount
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+
+	"k8s.io/klog/v2"
+	utilexec "k8s.io/utils/exec"
+	utilio "k8s.io/utils/io"
+)
+
+const (
+	// Number of fields per line in /proc/mounts as per the fstab man page.
+	expectedNumFieldsPerLine = 6
+	// Location of the mount file to use
+	procMountsPath = "/proc/mounts"
+	// Location of the mountinfo file
+	procMountInfoPath = "/proc/self/mountinfo"
+	// 'fsck' found errors and corrected them
+	fsckErrorsCorrected = 1
+	// 'fsck' found errors but exited without correcting them
+	fsckErrorsUncorrected = 4
+)
+
+// Mounter provides the default implementation of mount.Interface
+// for the linux platform.  This implementation assumes that the
+// kubelet is running in the host's root mount namespace.
+type Mounter struct {
+	mounterPath string
+	withSystemd bool
+}
+
+// New returns a mount.Interface for the current system.
+// It provides options to override the default mounter behavior.
+// mounterPath allows using an alternative to `/bin/mount` for mounting.
+func New(mounterPath string) Interface {
+	return &Mounter{
+		mounterPath: mounterPath,
+		withSystemd: detectSystemd(),
+	}
+}
+
+// Mount mounts source to target as fstype with given options. 'source' and 'fstype' must
+// be an empty string in case it's not required, e.g. for remount, or for auto filesystem
+// type, where kernel handles fstype for you. The mount 'options' is a list of options,
+// currently come from mount(8), e.g. "ro", "remount", "bind", etc. If no more option is
+// required, call Mount with an empty string list or nil.
+func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error {
+	return mounter.MountSensitive(source, target, fstype, options, nil)
+}
+
+// MountSensitive is the same as Mount() but this method allows
+// sensitiveOptions to be passed in a separate parameter from the normal
+// mount options and ensures the sensitiveOptions are never logged. This
+// method should be used by callers that pass sensitive material (like
+// passwords) as mount options.
+func (mounter *Mounter) MountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error {
+	// Path to mounter binary if containerized mounter is needed. Otherwise, it is set to empty.
+	// All Linux distros are expected to be shipped with a mount utility that a support bind mounts.
+	mounterPath := ""
+	bind, bindOpts, bindRemountOpts, bindRemountOptsSensitive := MakeBindOptsSensitive(options, sensitiveOptions)
+	if bind {
+		err := mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindOpts, bindRemountOptsSensitive, true)
+		if err != nil {
+			return err
+		}
+		return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindRemountOpts, bindRemountOptsSensitive, true)
+	}
+	// The list of filesystems that require containerized mounter on GCI image cluster
+	fsTypesNeedMounter := map[string]struct{}{
+		"nfs":       {},
+		"glusterfs": {},
+		"ceph":      {},
+		"cifs":      {},
+	}
+	if _, ok := fsTypesNeedMounter[fstype]; ok {
+		mounterPath = mounter.mounterPath
+	}
+	return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, options, sensitiveOptions, true)
+}
+
+// MountSensitiveWithoutSystemd is the same as MountSensitive() but disable using systemd mount.
+func (mounter *Mounter) MountSensitiveWithoutSystemd(source string, target string, fstype string, options []string, sensitiveOptions []string) error {
+	mounterPath := ""
+	bind, bindOpts, bindRemountOpts, bindRemountOptsSensitive := MakeBindOptsSensitive(options, sensitiveOptions)
+	if bind {
+		err := mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindOpts, bindRemountOptsSensitive, false)
+		if err != nil {
+			return err
+		}
+		return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindRemountOpts, bindRemountOptsSensitive, false)
+	}
+	// The list of filesystems that require containerized mounter on GCI image cluster
+	fsTypesNeedMounter := map[string]struct{}{
+		"nfs":       {},
+		"glusterfs": {},
+		"ceph":      {},
+		"cifs":      {},
+	}
+	if _, ok := fsTypesNeedMounter[fstype]; ok {
+		mounterPath = mounter.mounterPath
+	}
+	return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, options, sensitiveOptions, false)
+}
+
+// doMount runs the mount command. mounterPath is the path to mounter binary if containerized mounter is used.
+// sensitiveOptions is an extension of options except they will not be logged (because they may contain sensitive material)
+// systemdMountRequired is an extension of option to decide whether uses systemd mount.
+func (mounter *Mounter) doMount(mounterPath string, mountCmd string, source string, target string, fstype string, options []string, sensitiveOptions []string, systemdMountRequired bool) error {
+	mountArgs, mountArgsLogStr := MakeMountArgsSensitive(source, target, fstype, options, sensitiveOptions)
+	if len(mounterPath) > 0 {
+		mountArgs = append([]string{mountCmd}, mountArgs...)
+		mountArgsLogStr = mountCmd + " " + mountArgsLogStr
+		mountCmd = mounterPath
+	}
+
+	if mounter.withSystemd && systemdMountRequired {
+		// Try to run mount via systemd-run --scope. This will escape the
+		// service where kubelet runs and any fuse daemons will be started in a
+		// specific scope. kubelet service than can be restarted without killing
+		// these fuse daemons.
+		//
+		// Complete command line (when mounterPath is not used):
+		// systemd-run --description=... --scope -- mount -t <type> <what> <where>
+		//
+		// Expected flow:
+		// * systemd-run creates a transient scope (=~ cgroup) and executes its
+		//   argument (/bin/mount) there.
+		// * mount does its job, forks a fuse daemon if necessary and finishes.
+		//   (systemd-run --scope finishes at this point, returning mount's exit
+		//   code and stdout/stderr - thats one of --scope benefits).
+		// * systemd keeps the fuse daemon running in the scope (i.e. in its own
+		//   cgroup) until the fuse daemon dies (another --scope benefit).
+		//   Kubelet service can be restarted and the fuse daemon survives.
+		// * When the fuse daemon dies (e.g. during unmount) systemd removes the
+		//   scope automatically.
+		//
+		// systemd-mount is not used because it's too new for older distros
+		// (CentOS 7, Debian Jessie).
+		mountCmd, mountArgs, mountArgsLogStr = AddSystemdScopeSensitive("systemd-run", target, mountCmd, mountArgs, mountArgsLogStr)
+		// } else {
+		// No systemd-run on the host (or we failed to check it), assume kubelet
+		// does not run as a systemd service.
+		// No code here, mountCmd and mountArgs are already populated.
+	}
+
+	// Logging with sensitive mount options removed.
+	klog.V(4).Infof("Mounting cmd (%s) with arguments (%s)", mountCmd, mountArgsLogStr)
+	command := exec.Command(mountCmd, mountArgs...)
+	output, err := command.CombinedOutput()
+	if err != nil {
+		klog.Errorf("Mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s\n", err, mountCmd, mountArgsLogStr, string(output))
+		return fmt.Errorf("mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s",
+			err, mountCmd, mountArgsLogStr, string(output))
+	}
+	return err
+}
+
+// detectSystemd returns true if OS runs with systemd as init. When not sure
+// (permission errors, ...), it returns false.
+// There may be different ways how to detect systemd, this one makes sure that
+// systemd-runs (needed by Mount()) works.
+func detectSystemd() bool {
+	if _, err := exec.LookPath("systemd-run"); err != nil {
+		klog.V(2).Infof("Detected OS without systemd")
+		return false
+	}
+	// Try to run systemd-run --scope /bin/true, that should be enough
+	// to make sure that systemd is really running and not just installed,
+	// which happens when running in a container with a systemd-based image
+	// but with different pid 1.
+	cmd := exec.Command("systemd-run", "--description=Kubernetes systemd probe", "--scope", "true")
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		klog.V(2).Infof("Cannot run systemd-run, assuming non-systemd OS")
+		klog.V(4).Infof("systemd-run output: %s, failed with: %v", string(output), err)
+		return false
+	}
+	klog.V(2).Infof("Detected OS with systemd")
+	return true
+}
+
+// MakeMountArgs makes the arguments to the mount(8) command.
+// options MUST not contain sensitive material (like passwords).
+func MakeMountArgs(source, target, fstype string, options []string) (mountArgs []string) {
+	mountArgs, _ = MakeMountArgsSensitive(source, target, fstype, options, nil /* sensitiveOptions */)
+	return mountArgs
+}
+
+// MakeMountArgsSensitive makes the arguments to the mount(8) command.
+// sensitiveOptions is an extension of options except they will not be logged (because they may contain sensitive material)
+func MakeMountArgsSensitive(source, target, fstype string, options []string, sensitiveOptions []string) (mountArgs []string, mountArgsLogStr string) {
+	// Build mount command as follows:
+	//   mount [-t $fstype] [-o $options] [$source] $target
+	mountArgs = []string{}
+	mountArgsLogStr = ""
+	if len(fstype) > 0 {
+		mountArgs = append(mountArgs, "-t", fstype)
+		mountArgsLogStr += strings.Join(mountArgs, " ")
+	}
+	if len(options) > 0 || len(sensitiveOptions) > 0 {
+		combinedOptions := []string{}
+		combinedOptions = append(combinedOptions, options...)
+		combinedOptions = append(combinedOptions, sensitiveOptions...)
+		mountArgs = append(mountArgs, "-o", strings.Join(combinedOptions, ","))
+		// exclude sensitiveOptions from log string
+		mountArgsLogStr += " -o " + sanitizedOptionsForLogging(options, sensitiveOptions)
+	}
+	if len(source) > 0 {
+		mountArgs = append(mountArgs, source)
+		mountArgsLogStr += " " + source
+	}
+	mountArgs = append(mountArgs, target)
+	mountArgsLogStr += " " + target
+
+	return mountArgs, mountArgsLogStr
+}
+
+// AddSystemdScope adds "system-run --scope" to given command line
+// If args contains sensitive material, use AddSystemdScopeSensitive to construct
+// a safe to log string.
+func AddSystemdScope(systemdRunPath, mountName, command string, args []string) (string, []string) {
+	descriptionArg := fmt.Sprintf("--description=Kubernetes transient mount for %s", mountName)
+	systemdRunArgs := []string{descriptionArg, "--scope", "--", command}
+	return systemdRunPath, append(systemdRunArgs, args...)
+}
+
+// AddSystemdScopeSensitive adds "system-run --scope" to given command line
+// It also accepts takes a sanitized string containing mount arguments, mountArgsLogStr,
+// and returns the string appended to the systemd command for logging.
+func AddSystemdScopeSensitive(systemdRunPath, mountName, command string, args []string, mountArgsLogStr string) (string, []string, string) {
+	descriptionArg := fmt.Sprintf("--description=Kubernetes transient mount for %s", mountName)
+	systemdRunArgs := []string{descriptionArg, "--scope", "--", command}
+	return systemdRunPath, append(systemdRunArgs, args...), strings.Join(systemdRunArgs, " ") + " " + mountArgsLogStr
+}
+
+// Unmount unmounts the target.
+func (mounter *Mounter) Unmount(target string) error {
+	klog.V(4).Infof("Unmounting %s", target)
+	command := exec.Command("umount", target)
+	output, err := command.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("unmount failed: %v\nUnmounting arguments: %s\nOutput: %s", err, target, string(output))
+	}
+	return nil
+}
+
+// List returns a list of all mounted filesystems.
+func (*Mounter) List() ([]MountPoint, error) {
+	return ListProcMounts(procMountsPath)
+}
+
+// IsLikelyNotMountPoint determines if a directory is not a mountpoint.
+// It is fast but not necessarily ALWAYS correct. If the path is in fact
+// a bind mount from one part of a mount to another it will not be detected.
+// It also can not distinguish between mountpoints and symbolic links.
+// mkdir /tmp/a /tmp/b; mount --bind /tmp/a /tmp/b; IsLikelyNotMountPoint("/tmp/b")
+// will return true. When in fact /tmp/b is a mount point. If this situation
+// is of interest to you, don't use this function...
+func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
+	stat, err := os.Stat(file)
+	if err != nil {
+		return true, err
+	}
+	rootStat, err := os.Stat(filepath.Dir(strings.TrimSuffix(file, "/")))
+	if err != nil {
+		return true, err
+	}
+	// If the directory has a different device as parent, then it is a mountpoint.
+	if stat.Sys().(*syscall.Stat_t).Dev != rootStat.Sys().(*syscall.Stat_t).Dev {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+// GetMountRefs finds all mount references to pathname, returns a
+// list of paths. Path could be a mountpoint or a normal
+// directory (for bind mount).
+func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
+	pathExists, pathErr := PathExists(pathname)
+	if !pathExists {
+		return []string{}, nil
+	} else if IsCorruptedMnt(pathErr) {
+		klog.Warningf("GetMountRefs found corrupted mount at %s, treating as unmounted path", pathname)
+		return []string{}, nil
+	} else if pathErr != nil {
+		return nil, fmt.Errorf("error checking path %s: %v", pathname, pathErr)
+	}
+	realpath, err := filepath.EvalSymlinks(pathname)
+	if err != nil {
+		return nil, err
+	}
+	return SearchMountPoints(realpath, procMountInfoPath)
+}
+
+// checkAndRepairFileSystem checks and repairs filesystems using command fsck.
+func (mounter *SafeFormatAndMount) checkAndRepairFilesystem(source string) error {
+	klog.V(4).Infof("Checking for issues with fsck on disk: %s", source)
+	args := []string{"-a", source}
+	out, err := mounter.Exec.Command("fsck", args...).CombinedOutput()
+	if err != nil {
+		ee, isExitError := err.(utilexec.ExitError)
+		switch {
+		case err == utilexec.ErrExecutableNotFound:
+			klog.Warningf("'fsck' not found on system; continuing mount without running 'fsck'.")
+		case isExitError && ee.ExitStatus() == fsckErrorsCorrected:
+			klog.Infof("Device %s has errors which were corrected by fsck.", source)
+		case isExitError && ee.ExitStatus() == fsckErrorsUncorrected:
+			return NewMountError(HasFilesystemErrors, "'fsck' found errors on device %s but could not correct them: %s", source, string(out))
+		case isExitError && ee.ExitStatus() > fsckErrorsUncorrected:
+			klog.Infof("`fsck` error %s", string(out))
+		}
+	}
+	return nil
+}
+
+// formatAndMount uses unix utils to format and mount the given disk
+func (mounter *SafeFormatAndMount) formatAndMountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error {
+	readOnly := false
+	for _, option := range options {
+		if option == "ro" {
+			readOnly = true
+			break
+		}
+	}
+	if !readOnly {
+		// Check sensitiveOptions for ro
+		for _, option := range sensitiveOptions {
+			if option == "ro" {
+				readOnly = true
+				break
+			}
+		}
+	}
+
+	options = append(options, "defaults")
+	mountErrorValue := UnknownMountError
+
+	// Check if the disk is already formatted
+	existingFormat, err := mounter.GetDiskFormat(source)
+	if err != nil {
+		return NewMountError(GetDiskFormatFailed, "failed to get disk format of disk %s: %v", source, err)
+	}
+
+	// Use 'ext4' as the default
+	if len(fstype) == 0 {
+		fstype = "ext4"
+	}
+
+	if existingFormat == "" {
+		// Do not attempt to format the disk if mounting as readonly, return an error to reflect this.
+		if readOnly {
+			return NewMountError(UnformattedReadOnly, "cannot mount unformatted disk %s as we are manipulating it in read-only mode", source)
+		}
+
+		// Disk is unformatted so format it.
+		args := []string{source}
+		if fstype == "ext4" || fstype == "ext3" {
+			args = []string{
+				"-F",  // Force flag
+				"-m0", // Zero blocks reserved for super-user
+				source,
+			}
+		}
+
+		klog.Infof("Disk %q appears to be unformatted, attempting to format as type: %q with options: %v", source, fstype, args)
+		output, err := mounter.Exec.Command("mkfs."+fstype, args...).CombinedOutput()
+		if err != nil {
+			// Do not log sensitiveOptions only options
+			sensitiveOptionsLog := sanitizedOptionsForLogging(options, sensitiveOptions)
+			detailedErr := fmt.Sprintf("format of disk %q failed: type:(%q) target:(%q) options:(%q) errcode:(%v) output:(%v) ", source, fstype, target, sensitiveOptionsLog, err, string(output))
+			klog.Error(detailedErr)
+			return NewMountError(FormatFailed, detailedErr)
+		}
+
+		klog.Infof("Disk successfully formatted (mkfs): %s - %s %s", fstype, source, target)
+	} else {
+		if fstype != existingFormat {
+			// Verify that the disk is formatted with filesystem type we are expecting
+			mountErrorValue = FilesystemMismatch
+			klog.Warningf("Configured to mount disk %s as %s but current format is %s, things might break", source, existingFormat, fstype)
+		}
+
+		if !readOnly {
+			// Run check tools on the disk to fix repairable issues, only do this for formatted volumes requested as rw.
+			err := mounter.checkAndRepairFilesystem(source)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	// Mount the disk
+	klog.V(4).Infof("Attempting to mount disk %s in %s format at %s", source, fstype, target)
+	if err := mounter.MountSensitive(source, target, fstype, options, sensitiveOptions); err != nil {
+		return NewMountError(mountErrorValue, err.Error())
+	}
+
+	return nil
+}
+
+// GetDiskFormat uses 'blkid' to see if the given disk is unformatted
+func (mounter *SafeFormatAndMount) GetDiskFormat(disk string) (string, error) {
+	args := []string{"-p", "-s", "TYPE", "-s", "PTTYPE", "-o", "export", disk}
+	klog.V(4).Infof("Attempting to determine if disk %q is formatted using blkid with args: (%v)", disk, args)
+	dataOut, err := mounter.Exec.Command("blkid", args...).CombinedOutput()
+	output := string(dataOut)
+	klog.V(4).Infof("Output: %q", output)
+
+	if err != nil {
+		if exit, ok := err.(utilexec.ExitError); ok {
+			if exit.ExitStatus() == 2 {
+				// Disk device is unformatted.
+				// For `blkid`, if the specified token (TYPE/PTTYPE, etc) was
+				// not found, or no (specified) devices could be identified, an
+				// exit code of 2 is returned.
+				return "", nil
+			}
+		}
+		klog.Errorf("Could not determine if disk %q is formatted (%v)", disk, err)
+		return "", err
+	}
+
+	var fstype, pttype string
+
+	lines := strings.Split(output, "\n")
+	for _, l := range lines {
+		if len(l) <= 0 {
+			// Ignore empty line.
+			continue
+		}
+		cs := strings.Split(l, "=")
+		if len(cs) != 2 {
+			return "", fmt.Errorf("blkid returns invalid output: %s", output)
+		}
+		// TYPE is filesystem type, and PTTYPE is partition table type, according
+		// to https://www.kernel.org/pub/linux/utils/util-linux/v2.21/libblkid-docs/.
+		if cs[0] == "TYPE" {
+			fstype = cs[1]
+		} else if cs[0] == "PTTYPE" {
+			pttype = cs[1]
+		}
+	}
+
+	if len(pttype) > 0 {
+		klog.V(4).Infof("Disk %s detected partition table type: %s", disk, pttype)
+		// Returns a special non-empty string as filesystem type, then kubelet
+		// will not format it.
+		return "unknown data, probably partitions", nil
+	}
+
+	return fstype, nil
+}
+
+// ListProcMounts is shared with NsEnterMounter
+func ListProcMounts(mountFilePath string) ([]MountPoint, error) {
+	content, err := utilio.ConsistentRead(mountFilePath, maxListTries)
+	if err != nil {
+		return nil, err
+	}
+	return parseProcMounts(content)
+}
+
+func parseProcMounts(content []byte) ([]MountPoint, error) {
+	out := []MountPoint{}
+	lines := strings.Split(string(content), "\n")
+	for _, line := range lines {
+		if line == "" {
+			// the last split() item is empty string following the last \n
+			continue
+		}
+		fields := strings.Fields(line)
+		if len(fields) != expectedNumFieldsPerLine {
+			// Do not log line in case it contains sensitive Mount options
+			return nil, fmt.Errorf("wrong number of fields (expected %d, got %d)", expectedNumFieldsPerLine, len(fields))
+		}
+
+		mp := MountPoint{
+			Device: fields[0],
+			Path:   fields[1],
+			Type:   fields[2],
+			Opts:   strings.Split(fields[3], ","),
+		}
+
+		freq, err := strconv.Atoi(fields[4])
+		if err != nil {
+			return nil, err
+		}
+		mp.Freq = freq
+
+		pass, err := strconv.Atoi(fields[5])
+		if err != nil {
+			return nil, err
+		}
+		mp.Pass = pass
+
+		out = append(out, mp)
+	}
+	return out, nil
+}
+
+// SearchMountPoints finds all mount references to the source, returns a list of
+// mountpoints.
+// The source can be a mount point or a normal directory (bind mount). We
+// didn't support device because there is no use case by now.
+// Some filesystems may share a source name, e.g. tmpfs. And for bind mounting,
+// it's possible to mount a non-root path of a filesystem, so we need to use
+// root path and major:minor to represent mount source uniquely.
+// This implementation is shared between Linux and NsEnterMounter
+func SearchMountPoints(hostSource, mountInfoPath string) ([]string, error) {
+	mis, err := ParseMountInfo(mountInfoPath)
+	if err != nil {
+		return nil, err
+	}
+
+	mountID := 0
+	rootPath := ""
+	major := -1
+	minor := -1
+
+	// Finding the underlying root path and major:minor if possible.
+	// We need search in backward order because it's possible for later mounts
+	// to overlap earlier mounts.
+	for i := len(mis) - 1; i >= 0; i-- {
+		if hostSource == mis[i].MountPoint || PathWithinBase(hostSource, mis[i].MountPoint) {
+			// If it's a mount point or path under a mount point.
+			mountID = mis[i].ID
+			rootPath = filepath.Join(mis[i].Root, strings.TrimPrefix(hostSource, mis[i].MountPoint))
+			major = mis[i].Major
+			minor = mis[i].Minor
+			break
+		}
+	}
+
+	if rootPath == "" || major == -1 || minor == -1 {
+		return nil, fmt.Errorf("failed to get root path and major:minor for %s", hostSource)
+	}
+
+	var refs []string
+	for i := range mis {
+		if mis[i].ID == mountID {
+			// Ignore mount entry for mount source itself.
+			continue
+		}
+		if mis[i].Root == rootPath && mis[i].Major == major && mis[i].Minor == minor {
+			refs = append(refs, mis[i].MountPoint)
+		}
+	}
+
+	return refs, nil
+}

vendor/k8s.io/mount-utils/mount_unsupported.go

@@ -0,0 +1,82 @@
+// +build !linux,!windows
+
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mount
+
+import (
+	"errors"
+)
+
+// Mounter implements mount.Interface for unsupported platforms
+type Mounter struct {
+	mounterPath string
+}
+
+var errUnsupported = errors.New("util/mount on this platform is not supported")
+
+// New returns a mount.Interface for the current system.
+// It provides options to override the default mounter behavior.
+// mounterPath allows using an alternative to `/bin/mount` for mounting.
+func New(mounterPath string) Interface {
+	return &Mounter{
+		mounterPath: mounterPath,
+	}
+}
+
+// Mount always returns an error on unsupported platforms
+func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error {
+	return errUnsupported
+}
+
+// MountSensitive always returns an error on unsupported platforms
+func (mounter *Mounter) MountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error {
+	return errUnsupported
+}
+
+// MountSensitiveWithoutSystemd always returns an error on unsupported platforms
+func (mounter *Mounter) MountSensitiveWithoutSystemd(source string, target string, fstype string, options []string, sensitiveOptions []string) error {
+	return errUnsupported
+}
+
+// Unmount always returns an error on unsupported platforms
+func (mounter *Mounter) Unmount(target string) error {
+	return errUnsupported
+}
+
+// List always returns an error on unsupported platforms
+func (mounter *Mounter) List() ([]MountPoint, error) {
+	return []MountPoint{}, errUnsupported
+}
+
+// IsLikelyNotMountPoint always returns an error on unsupported platforms
+func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
+	return true, errUnsupported
+}
+
+// GetMountRefs always returns an error on unsupported platforms
+func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
+	return nil, errUnsupported
+}
+
+func (mounter *SafeFormatAndMount) formatAndMountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error {
+	return mounter.Interface.Mount(source, target, fstype, options)
+}
+
+func (mounter *SafeFormatAndMount) diskLooksUnformatted(disk string) (bool, error) {
+	return true, errUnsupported
+}

vendor/k8s.io/mount-utils/mount_windows.go

@@ -0,0 +1,330 @@
+// +build windows
+
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mount
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"k8s.io/klog/v2"
+	"k8s.io/utils/keymutex"
+)
+
+const (
+	accessDenied string = "access is denied"
+)
+
+// Mounter provides the default implementation of mount.Interface
+// for the windows platform.  This implementation assumes that the
+// kubelet is running in the host's root mount namespace.
+type Mounter struct {
+	mounterPath string
+}
+
+// New returns a mount.Interface for the current system.
+// It provides options to override the default mounter behavior.
+// mounterPath allows using an alternative to `/bin/mount` for mounting.
+func New(mounterPath string) Interface {
+	return &Mounter{
+		mounterPath: mounterPath,
+	}
+}
+
+// acquire lock for smb mount
+var getSMBMountMutex = keymutex.NewHashed(0)
+
+// Mount : mounts source to target with given options.
+// currently only supports cifs(smb), bind mount(for disk)
+func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error {
+	return mounter.MountSensitive(source, target, fstype, options, nil /* sensitiveOptions */)
+}
+
+// MountSensitiveWithoutSystemd is the same as MountSensitive() but disable using ssytemd mount.
+// Windows not supported systemd mount, this function degrades to MountSensitive().
+func (mounter *Mounter) MountSensitiveWithoutSystemd(source string, target string, fstype string, options []string, sensitiveOptions []string) error {
+	return mounter.MountSensitive(source, target, fstype, options, sensitiveOptions /* sensitiveOptions */)
+}
+
+// MountSensitive is the same as Mount() but this method allows
+// sensitiveOptions to be passed in a separate parameter from the normal
+// mount options and ensures the sensitiveOptions are never logged. This
+// method should be used by callers that pass sensitive material (like
+// passwords) as mount options.
+func (mounter *Mounter) MountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error {
+	target = NormalizeWindowsPath(target)
+	sanitizedOptionsForLogging := sanitizedOptionsForLogging(options, sensitiveOptions)
+
+	if source == "tmpfs" {
+		klog.V(3).Infof("mounting source (%q), target (%q), with options (%q)", source, target, sanitizedOptionsForLogging)
+		return os.MkdirAll(target, 0755)
+	}
+
+	parentDir := filepath.Dir(target)
+	if err := os.MkdirAll(parentDir, 0755); err != nil {
+		return err
+	}
+
+	klog.V(4).Infof("mount options(%q) source:%q, target:%q, fstype:%q, begin to mount",
+		sanitizedOptionsForLogging, source, target, fstype)
+	bindSource := source
+
+	if bind, _, _, _ := MakeBindOptsSensitive(options, sensitiveOptions); bind {
+		bindSource = NormalizeWindowsPath(source)
+	} else {
+		allOptions := []string{}
+		allOptions = append(allOptions, options...)
+		allOptions = append(allOptions, sensitiveOptions...)
+		if len(allOptions) < 2 {
+			return fmt.Errorf("mount options(%q) should have at least 2 options, current number:%d, source:%q, target:%q",
+				sanitizedOptionsForLogging, len(allOptions), source, target)
+		}
+
+		// currently only cifs mount is supported
+		if strings.ToLower(fstype) != "cifs" {
+			return fmt.Errorf("only cifs mount is supported now, fstype: %q, mounting source (%q), target (%q), with options (%q)", fstype, source, target, sanitizedOptionsForLogging)
+		}
+
+		// lock smb mount for the same source
+		getSMBMountMutex.LockKey(source)
+		defer getSMBMountMutex.UnlockKey(source)
+
+		username := allOptions[0]
+		password := allOptions[1]
+		if output, err := newSMBMapping(username, password, source); err != nil {
+			klog.Warningf("SMB Mapping(%s) returned with error(%v), output(%s)", source, err, string(output))
+			if isSMBMappingExist(source) {
+				valid, err := isValidPath(source)
+				if !valid {
+					if err == nil || isAccessDeniedError(err) {
+						klog.V(2).Infof("SMB Mapping(%s) already exists while it's not valid, return error: %v, now begin to remove and remount", source, err)
+						if output, err = removeSMBMapping(source); err != nil {
+							return fmt.Errorf("Remove-SmbGlobalMapping failed: %v, output: %q", err, output)
+						}
+						if output, err := newSMBMapping(username, password, source); err != nil {
+							return fmt.Errorf("New-SmbGlobalMapping(%s) failed: %v, output: %q", source, err, output)
+						}
+					}
+				} else {
+					klog.V(2).Infof("SMB Mapping(%s) already exists and is still valid, skip error(%v)", source, err)
+				}
+			} else {
+				return fmt.Errorf("New-SmbGlobalMapping(%s) failed: %v, output: %q", source, err, output)
+			}
+		}
+	}
+
+	// There is an issue in golang where EvalSymlinks fails on Windows when passed a
+	// UNC share root path without a trailing backslash.
+	// Ex: \\SERVER\share will fail to resolve but \\SERVER\share\ will resolve
+	// containerD on Windows calls EvalSymlinks so we'll add the backslash when making the symlink if it is missing.
+	// https://github.com/golang/go/pull/42096 fixes this issue in golang but a fix will not be available until
+	// golang v1.16
+	mklinkSource := bindSource
+	if !strings.HasSuffix(mklinkSource, "\\") {
+		mklinkSource = mklinkSource + "\\"
+	}
+
+	output, err := exec.Command("cmd", "/c", "mklink", "/D", target, mklinkSource).CombinedOutput()
+	if err != nil {
+		klog.Errorf("mklink failed: %v, source(%q) target(%q) output: %q", err, mklinkSource, target, string(output))
+		return err
+	}
+	klog.V(2).Infof("mklink source(%q) on target(%q) successfully, output: %q", mklinkSource, target, string(output))
+
+	return nil
+}
+
+// do the SMB mount with username, password, remotepath
+// return (output, error)
+func newSMBMapping(username, password, remotepath string) (string, error) {
+	if username == "" || password == "" || remotepath == "" {
+		return "", fmt.Errorf("invalid parameter(username: %s, password: %s, remoteapth: %s)", username, sensitiveOptionsRemoved, remotepath)
+	}
+
+	// use PowerShell Environment Variables to store user input string to prevent command line injection
+	// https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_environment_variables?view=powershell-5.1
+	cmdLine := `$PWord = ConvertTo-SecureString -String $Env:smbpassword -AsPlainText -Force` +
+		`;$Credential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $Env:smbuser, $PWord` +
+		`;New-SmbGlobalMapping -RemotePath $Env:smbremotepath -Credential $Credential`
+	cmd := exec.Command("powershell", "/c", cmdLine)
+	cmd.Env = append(os.Environ(),
+		fmt.Sprintf("smbuser=%s", username),
+		fmt.Sprintf("smbpassword=%s", password),
+		fmt.Sprintf("smbremotepath=%s", remotepath))
+
+	output, err := cmd.CombinedOutput()
+	return string(output), err
+}
+
+// check whether remotepath is already mounted
+func isSMBMappingExist(remotepath string) bool {
+	cmd := exec.Command("powershell", "/c", `Get-SmbGlobalMapping -RemotePath $Env:smbremotepath`)
+	cmd.Env = append(os.Environ(), fmt.Sprintf("smbremotepath=%s", remotepath))
+	_, err := cmd.CombinedOutput()
+	return err == nil
+}
+
+// check whether remotepath is valid
+// return (true, nil) if remotepath is valid
+func isValidPath(remotepath string) (bool, error) {
+	cmd := exec.Command("powershell", "/c", `Test-Path $Env:remoteapth`)
+	cmd.Env = append(os.Environ(), fmt.Sprintf("remoteapth=%s", remotepath))
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return false, fmt.Errorf("returned output: %s, error: %v", string(output), err)
+	}
+
+	return strings.HasPrefix(strings.ToLower(string(output)), "true"), nil
+}
+
+func isAccessDeniedError(err error) bool {
+	return err != nil && strings.Contains(strings.ToLower(err.Error()), accessDenied)
+}
+
+// remove SMB mapping
+func removeSMBMapping(remotepath string) (string, error) {
+	cmd := exec.Command("powershell", "/c", `Remove-SmbGlobalMapping -RemotePath $Env:smbremotepath -Force`)
+	cmd.Env = append(os.Environ(), fmt.Sprintf("smbremotepath=%s", remotepath))
+	output, err := cmd.CombinedOutput()
+	return string(output), err
+}
+
+// Unmount unmounts the target.
+func (mounter *Mounter) Unmount(target string) error {
+	klog.V(4).Infof("azureMount: Unmount target (%q)", target)
+	target = NormalizeWindowsPath(target)
+	if output, err := exec.Command("cmd", "/c", "rmdir", target).CombinedOutput(); err != nil {
+		klog.Errorf("rmdir failed: %v, output: %q", err, string(output))
+		return err
+	}
+	return nil
+}
+
+// List returns a list of all mounted filesystems. todo
+func (mounter *Mounter) List() ([]MountPoint, error) {
+	return []MountPoint{}, nil
+}
+
+// IsLikelyNotMountPoint determines if a directory is not a mountpoint.
+func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
+	stat, err := os.Lstat(file)
+	if err != nil {
+		return true, err
+	}
+
+	if stat.Mode()&os.ModeSymlink != 0 {
+		return false, err
+	}
+	return true, nil
+}
+
+// GetMountRefs : empty implementation here since there is no place to query all mount points on Windows
+func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
+	windowsPath := NormalizeWindowsPath(pathname)
+	pathExists, pathErr := PathExists(windowsPath)
+	if !pathExists {
+		return []string{}, nil
+	} else if IsCorruptedMnt(pathErr) {
+		klog.Warningf("GetMountRefs found corrupted mount at %s, treating as unmounted path", windowsPath)
+		return []string{}, nil
+	} else if pathErr != nil {
+		return nil, fmt.Errorf("error checking path %s: %v", windowsPath, pathErr)
+	}
+	return []string{pathname}, nil
+}
+
+func (mounter *SafeFormatAndMount) formatAndMountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error {
+	// Try to mount the disk
+	klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, source, target)
+
+	if err := ValidateDiskNumber(source); err != nil {
+		klog.Errorf("diskMount: formatAndMount failed, err: %v", err)
+		return err
+	}
+
+	if len(fstype) == 0 {
+		// Use 'NTFS' as the default
+		fstype = "NTFS"
+	}
+
+	// format disk if it is unformatted(raw)
+	cmd := fmt.Sprintf("Get-Disk -Number %s | Where partitionstyle -eq 'raw' | Initialize-Disk -PartitionStyle MBR -PassThru"+
+		" | New-Partition -UseMaximumSize | Format-Volume -FileSystem %s -Confirm:$false", source, fstype)
+	if output, err := mounter.Exec.Command("powershell", "/c", cmd).CombinedOutput(); err != nil {
+		return fmt.Errorf("diskMount: format disk failed, error: %v, output: %q", err, string(output))
+	}
+	klog.V(4).Infof("diskMount: Disk successfully formatted, disk: %q, fstype: %q", source, fstype)
+
+	volumeIds, err := listVolumesOnDisk(source)
+	if err != nil {
+		return err
+	}
+	driverPath := volumeIds[0]
+	target = NormalizeWindowsPath(target)
+	output, err := mounter.Exec.Command("cmd", "/c", "mklink", "/D", target, driverPath).CombinedOutput()
+	if err != nil {
+		klog.Errorf("mklink(%s, %s) failed: %v, output: %q", target, driverPath, err, string(output))
+		return err
+	}
+	klog.V(2).Infof("formatAndMount disk(%s) fstype(%s) on(%s) with output(%s) successfully", driverPath, fstype, target, string(output))
+	return nil
+}
+
+// ListVolumesOnDisk - returns back list of volumes(volumeIDs) in the disk (requested in diskID).
+func listVolumesOnDisk(diskID string) (volumeIDs []string, err error) {
+	cmd := fmt.Sprintf("(Get-Disk -DeviceId %s | Get-Partition | Get-Volume).UniqueId", diskID)
+	output, err := exec.Command("powershell", "/c", cmd).CombinedOutput()
+	klog.V(4).Infof("listVolumesOnDisk id from %s: %s", diskID, string(output))
+	if err != nil {
+		return []string{}, fmt.Errorf("error list volumes on disk. cmd: %s, output: %s, error: %v", cmd, string(output), err)
+	}
+
+	volumeIds := strings.Split(strings.TrimSpace(string(output)), "\r\n")
+	return volumeIds, nil
+}
+
+// getAllParentLinks walks all symbolic links and return all the parent targets recursively
+func getAllParentLinks(path string) ([]string, error) {
+	const maxIter = 255
+	links := []string{}
+	for {
+		links = append(links, path)
+		if len(links) > maxIter {
+			return links, fmt.Errorf("unexpected length of parent links: %v", links)
+		}
+
+		fi, err := os.Lstat(path)
+		if err != nil {
+			return links, fmt.Errorf("Lstat: %v", err)
+		}
+		if fi.Mode()&os.ModeSymlink == 0 {
+			break
+		}
+
+		path, err = os.Readlink(path)
+		if err != nil {
+			return links, fmt.Errorf("Readlink error: %v", err)
+		}
+	}
+
+	return links, nil
+}

vendor/k8s.io/utils/nsenter/OWNERS

@@ -0,0 +1,10 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+reviewers:
+  - jsafrane
+  - msau42
+  - cofyc
+approvers:
+  - jsafrane
+  - msau42
+  - cofyc

vendor/k8s.io/utils/nsenter/README.md

@@ -0,0 +1,4 @@
+# NSEnter
+
+This package provides interfaces for executing and interacting with processes
+running within a namespace.

vendor/k8s.io/utils/nsenter/nsenter.go

@@ -0,0 +1,258 @@
+// +build linux
+
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nsenter
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"k8s.io/klog/v2"
+	"k8s.io/utils/exec"
+)
+
+const (
+	// DefaultHostRootFsPath is path to host's filesystem mounted into container
+	// with kubelet.
+	DefaultHostRootFsPath = "/rootfs"
+	// mountNsPath is the default mount namespace of the host
+	mountNsPath = "/proc/1/ns/mnt"
+	// nsenterPath is the default nsenter command
+	nsenterPath = "nsenter"
+)
+
+// Nsenter is a type alias for backward compatibility
+type Nsenter = NSEnter
+
+// NSEnter is part of experimental support for running the kubelet
+// in a container.
+//
+// NSEnter requires:
+//
+// 1.  Docker >= 1.6 due to the dependency on the slave propagation mode
+//     of the bind-mount of the kubelet root directory in the container.
+//     Docker 1.5 used a private propagation mode for bind-mounts, so mounts
+//     performed in the host's mount namespace do not propagate out to the
+//     bind-mount in this docker version.
+// 2.  The host's root filesystem must be available at /rootfs
+// 3.  The nsenter binary must be on the Kubelet process' PATH in the container's
+//     filesystem.
+// 4.  The Kubelet process must have CAP_SYS_ADMIN (required by nsenter); at
+//     the present, this effectively means that the kubelet is running in a
+//     privileged container.
+// 5.  The volume path used by the Kubelet must be the same inside and outside
+//     the container and be writable by the container (to initialize volume)
+//     contents. TODO: remove this requirement.
+// 6.  The host image must have "mount", "findmnt", "umount", "stat", "touch",
+//     "mkdir", "ls", "sh" and "chmod" binaries in /bin, /usr/sbin, or /usr/bin
+// 7.  The host image should have systemd-run in /bin, /usr/sbin, or /usr/bin if
+//     systemd is installed/enabled in the operating system.
+// For more information about mount propagation modes, see:
+//   https://www.kernel.org/doc/Documentation/filesystems/sharedsubtree.txt
+type NSEnter struct {
+	// a map of commands to their paths on the host filesystem
+	paths map[string]string
+
+	// Path to the host filesystem, typically "/rootfs". Used only for testing.
+	hostRootFsPath string
+
+	// Exec implementation
+	executor exec.Interface
+}
+
+// NewNsenter constructs a new instance of NSEnter
+func NewNsenter(hostRootFsPath string, executor exec.Interface) (*NSEnter, error) {
+	ne := &NSEnter{
+		hostRootFsPath: hostRootFsPath,
+		executor:       executor,
+	}
+	if err := ne.initPaths(); err != nil {
+		return nil, err
+	}
+	return ne, nil
+}
+
+func (ne *NSEnter) initPaths() error {
+	ne.paths = map[string]string{}
+	binaries := []string{
+		"mount",
+		"findmnt",
+		"umount",
+		"systemd-run",
+		"stat",
+		"touch",
+		"mkdir",
+		"sh",
+		"chmod",
+		"realpath",
+	}
+	// search for the required commands in other locations besides /usr/bin
+	for _, binary := range binaries {
+		// check for binary under the following directories
+		for _, path := range []string{"/", "/bin", "/usr/sbin", "/usr/bin"} {
+			binPath := filepath.Join(path, binary)
+			if _, err := os.Stat(filepath.Join(ne.hostRootFsPath, binPath)); err != nil {
+				continue
+			}
+			ne.paths[binary] = binPath
+			break
+		}
+		// systemd-run is optional, bailout if we don't find any of the other binaries
+		if ne.paths[binary] == "" && binary != "systemd-run" {
+			return fmt.Errorf("unable to find %v", binary)
+		}
+	}
+	return nil
+}
+
+// Exec executes nsenter commands in hostProcMountNsPath mount namespace
+func (ne *NSEnter) Exec(cmd string, args []string) exec.Cmd {
+	hostProcMountNsPath := filepath.Join(ne.hostRootFsPath, mountNsPath)
+	fullArgs := append([]string{fmt.Sprintf("--mount=%s", hostProcMountNsPath), "--"},
+		append([]string{ne.AbsHostPath(cmd)}, args...)...)
+	klog.V(5).Infof("Running nsenter command: %v %v", nsenterPath, fullArgs)
+	return ne.executor.Command(nsenterPath, fullArgs...)
+}
+
+// Command returns a command wrapped with nsenter
+func (ne *NSEnter) Command(cmd string, args ...string) exec.Cmd {
+	return ne.Exec(cmd, args)
+}
+
+// CommandContext returns a CommandContext wrapped with nsenter
+func (ne *NSEnter) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd {
+	hostProcMountNsPath := filepath.Join(ne.hostRootFsPath, mountNsPath)
+	fullArgs := append([]string{fmt.Sprintf("--mount=%s", hostProcMountNsPath), "--"},
+		append([]string{ne.AbsHostPath(cmd)}, args...)...)
+	klog.V(5).Infof("Running nsenter command: %v %v", nsenterPath, fullArgs)
+	return ne.executor.CommandContext(ctx, nsenterPath, fullArgs...)
+}
+
+// LookPath returns a LookPath wrapped with nsenter
+func (ne *NSEnter) LookPath(file string) (string, error) {
+	return "", fmt.Errorf("not implemented, error looking up : %s", file)
+}
+
+// AbsHostPath returns the absolute runnable path for a specified command
+func (ne *NSEnter) AbsHostPath(command string) string {
+	path, ok := ne.paths[command]
+	if !ok {
+		return command
+	}
+	return path
+}
+
+// SupportsSystemd checks whether command systemd-run exists
+func (ne *NSEnter) SupportsSystemd() (string, bool) {
+	systemdRunPath, ok := ne.paths["systemd-run"]
+	return systemdRunPath, ok && systemdRunPath != ""
+}
+
+// EvalSymlinks returns the path name on the host after evaluating symlinks on the
+// host.
+// mustExist makes EvalSymlinks to return error when the path does not
+// exist. When it's false, it evaluates symlinks of the existing part and
+// blindly adds the non-existing part:
+// pathname: /mnt/volume/non/existing/directory
+//     /mnt/volume exists
+//                non/existing/directory does not exist
+// -> It resolves symlinks in /mnt/volume to say /mnt/foo and returns
+//    /mnt/foo/non/existing/directory.
+//
+// BEWARE! EvalSymlinks is not able to detect symlink looks with mustExist=false!
+// If /tmp/link is symlink to /tmp/link, EvalSymlinks(/tmp/link/foo) returns /tmp/link/foo.
+func (ne *NSEnter) EvalSymlinks(pathname string, mustExist bool) (string, error) {
+	var args []string
+	if mustExist {
+		// "realpath -e: all components of the path must exist"
+		args = []string{"-e", pathname}
+	} else {
+		// "realpath -m: no path components need exist or be a directory"
+		args = []string{"-m", pathname}
+	}
+	outBytes, err := ne.Exec("realpath", args).CombinedOutput()
+	if err != nil {
+		klog.Infof("failed to resolve symbolic links on %s: %v", pathname, err)
+		return "", err
+	}
+	return strings.TrimSpace(string(outBytes)), nil
+}
+
+// KubeletPath returns the path name that can be accessed by containerized
+// kubelet. It is recommended to resolve symlinks on the host by EvalSymlinks
+// before calling this function
+func (ne *NSEnter) KubeletPath(pathname string) string {
+	return filepath.Join(ne.hostRootFsPath, pathname)
+}
+
+// NewFakeNsenter returns a NSEnter that does not run "nsenter --mount=... --",
+// but runs everything in the same mount namespace as the unit test binary.
+// rootfsPath is supposed to be a symlink, e.g. /tmp/xyz/rootfs -> /.
+// This fake NSEnter is enough for most operations, e.g. to resolve symlinks,
+// but it's not enough to call /bin/mount - unit tests don't run as root.
+func NewFakeNsenter(rootfsPath string) (*NSEnter, error) {
+	executor := &fakeExec{
+		rootfsPath: rootfsPath,
+	}
+	// prepare /rootfs/bin, usr/bin and usr/sbin
+	bin := filepath.Join(rootfsPath, "bin")
+	if err := os.Symlink("/bin", bin); err != nil {
+		return nil, err
+	}
+
+	usr := filepath.Join(rootfsPath, "usr")
+	if err := os.Mkdir(usr, 0755); err != nil {
+		return nil, err
+	}
+	usrbin := filepath.Join(usr, "bin")
+	if err := os.Symlink("/usr/bin", usrbin); err != nil {
+		return nil, err
+	}
+	usrsbin := filepath.Join(usr, "sbin")
+	if err := os.Symlink("/usr/sbin", usrsbin); err != nil {
+		return nil, err
+	}
+
+	return NewNsenter(rootfsPath, executor)
+}
+
+type fakeExec struct {
+	rootfsPath string
+}
+
+func (f fakeExec) Command(cmd string, args ...string) exec.Cmd {
+	// This will intentionaly panic if NSEnter does not provide enough arguments.
+	realCmd := args[2]
+	realArgs := args[3:]
+	return exec.New().Command(realCmd, realArgs...)
+}
+
+func (fakeExec) LookPath(file string) (string, error) {
+	return "", errors.New("not implemented")
+}
+
+func (fakeExec) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd {
+	return nil
+}
+
+var _ exec.Interface = fakeExec{}
+var _ exec.Interface = &NSEnter{}

vendor/k8s.io/utils/nsenter/nsenter_unsupported.go

@@ -0,0 +1,79 @@
+// +build !linux
+
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nsenter
+
+import (
+	"context"
+	"fmt"
+
+	"k8s.io/utils/exec"
+)
+
+const (
+	// DefaultHostRootFsPath is path to host's filesystem mounted into container
+	// with kubelet.
+	DefaultHostRootFsPath = "/rootfs"
+)
+
+// Nsenter is a type alias for backward compatibility
+type Nsenter = NSEnter
+
+// NSEnter is part of experimental support for running the kubelet
+// in a container.
+type NSEnter struct {
+	// a map of commands to their paths on the host filesystem
+	Paths map[string]string
+}
+
+// NewNsenter constructs a new instance of NSEnter
+func NewNsenter(hostRootFsPath string, executor exec.Interface) (*Nsenter, error) {
+	return &Nsenter{}, nil
+}
+
+// Exec executes nsenter commands in hostProcMountNsPath mount namespace
+func (ne *NSEnter) Exec(cmd string, args []string) exec.Cmd {
+	return nil
+}
+
+// AbsHostPath returns the absolute runnable path for a specified command
+func (ne *NSEnter) AbsHostPath(command string) string {
+	return ""
+}
+
+// SupportsSystemd checks whether command systemd-run exists
+func (ne *NSEnter) SupportsSystemd() (string, bool) {
+	return "", false
+}
+
+// Command returns a command wrapped with nenter
+func (ne *NSEnter) Command(cmd string, args ...string) exec.Cmd {
+	return nil
+}
+
+// CommandContext returns a CommandContext wrapped with nsenter
+func (ne *NSEnter) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd {
+	return nil
+}
+
+// LookPath returns a LookPath wrapped with nsenter
+func (ne *NSEnter) LookPath(file string) (string, error) {
+	return "", fmt.Errorf("not implemented, error looking up : %s", file)
+}
+
+var _ exec.Interface = &NSEnter{}

vendor/modules.txt

@@ -648,6 +648,8 @@ k8s.io/client-go/util/homedir
 k8s.io/client-go/util/keyutil
 k8s.io/client-go/util/retry
 k8s.io/client-go/util/workqueue
+# k8s.io/cloud-provider v0.20.0 => k8s.io/cloud-provider v0.20.0
+k8s.io/cloud-provider
 # k8s.io/component-base v0.20.0 => k8s.io/component-base v0.20.0
 k8s.io/component-base/cli/flag
 k8s.io/component-base/config
@@ -695,11 +697,20 @@ k8s.io/kubernetes/pkg/features
 k8s.io/kubernetes/pkg/fieldpath
 k8s.io/kubernetes/pkg/kubelet/apis/config
 k8s.io/kubernetes/pkg/kubelet/types
+k8s.io/kubernetes/pkg/proxy/util
 k8s.io/kubernetes/pkg/security/apparmor
 k8s.io/kubernetes/pkg/util/hash
 k8s.io/kubernetes/pkg/util/labels
 k8s.io/kubernetes/pkg/util/parsers
+k8s.io/kubernetes/pkg/util/sysctl
 k8s.io/kubernetes/pkg/util/taints
+k8s.io/kubernetes/pkg/volume
+k8s.io/kubernetes/pkg/volume/util/fs
+k8s.io/kubernetes/pkg/volume/util/fsquota
+k8s.io/kubernetes/pkg/volume/util/fsquota/common
+k8s.io/kubernetes/pkg/volume/util/hostutil
+k8s.io/kubernetes/pkg/volume/util/recyclerclient
+k8s.io/kubernetes/pkg/volume/util/subpath
 k8s.io/kubernetes/test/e2e/framework
 k8s.io/kubernetes/test/e2e/framework/auth
 k8s.io/kubernetes/test/e2e/framework/config
@@ -718,6 +729,8 @@ k8s.io/kubernetes/test/e2e/perftype
 k8s.io/kubernetes/test/e2e/storage/utils
 k8s.io/kubernetes/test/utils
 k8s.io/kubernetes/test/utils/image
+# k8s.io/mount-utils v0.0.0 => k8s.io/mount-utils v0.21.0-alpha.0
+k8s.io/mount-utils
 # k8s.io/utils v0.0.0-20201110183641-67b214c5f920
 ## explicit
 k8s.io/utils/buffer
@@ -727,6 +740,7 @@ k8s.io/utils/io
 k8s.io/utils/keymutex
 k8s.io/utils/mount
 k8s.io/utils/net
+k8s.io/utils/nsenter
 k8s.io/utils/path
 k8s.io/utils/pointer
 k8s.io/utils/trace