diff --git a/go.sum b/go.sum index 8e287d1f..8f780c3c 100644 --- a/go.sum +++ b/go.sum @@ -1222,11 +1222,13 @@ k8s.io/apiserver v0.20.0/go.mod h1:6gRIWiOkvGvQt12WTYmsiYoUyYW0FXSiMdNl4m+sxY8= k8s.io/cli-runtime v0.20.0/go.mod h1:C5tewU1SC1t09D7pmkk83FT4lMAw+bvMDuRxA7f0t2s= k8s.io/client-go v0.20.0 h1:Xlax8PKbZsjX4gFvNtt4F5MoJ1V5prDvCuoq9B7iax0= k8s.io/client-go v0.20.0/go.mod h1:4KWh/g+Ocd8KkCwKF8vUNnmqgv+EVnQDK4MBF4oB5tY= +k8s.io/cloud-provider v0.20.0 h1:CVPQ66iyfNgeGomUq2jE/TWrfzE77bdCpemhFS8955U= k8s.io/cloud-provider v0.20.0/go.mod h1:Lz/luSVD5BrHDDhtVdjFh0C2qQCRYdf0b9BHQ9L+bXc= k8s.io/cluster-bootstrap v0.20.0/go.mod h1:6WZaNIBvcvL7MkPzSRKrZDIr4u+ePW2oIWoRsEFMjmE= k8s.io/code-generator v0.20.0/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/component-base v0.20.0 h1:BXGL8iitIQD+0NgW49UsM7MraNUUGDU3FBmrfUAtmVQ= k8s.io/component-base v0.20.0/go.mod h1:wKPj+RHnAr8LW2EIBIK7AxOHPde4gme2lzXwVSoRXeA= +k8s.io/component-helpers v0.20.0-alpha.2.0.20201114090304-7cb42b694587 h1:XV8zcD26n/hrYX129A4vR35MvgGPKKSVAQP38dif/HI= k8s.io/component-helpers v0.20.0-alpha.2.0.20201114090304-7cb42b694587/go.mod h1:Fb+7xcdRsPnQNQnXkqMwptC6LsONZr10ris8YPJw1bk= k8s.io/controller-manager v0.20.0-alpha.1.0.20201209052538-b2c380a1dc86 h1:22Cxrg/eu1vIlWan5HoJJzZw8/sAZpjlwMrs87uOdEQ= k8s.io/controller-manager v0.20.0-alpha.1.0.20201209052538-b2c380a1dc86/go.mod h1:nPjStmyrmF6IE6bhIsSN+5A4nYp1ORzLENSAPp+tSzs= @@ -1265,6 +1267,7 @@ k8s.io/kubernetes v1.21.0-alpha.0.0.20201210005053-f58c4d8cd725 h1:zDSrjDDKNWvXy k8s.io/kubernetes v1.21.0-alpha.0.0.20201210005053-f58c4d8cd725/go.mod h1:/xrHGNfoQphtkhZvyd5bA1lRmz+QkDVmBZu+O8QMoek= k8s.io/legacy-cloud-providers v0.20.0/go.mod h1:1jEkaU7h9+b1EYdfWDBvhFAr+QpRfUjQfK+dGhxPGfA= k8s.io/metrics v0.20.0/go.mod h1:9yiRhfr8K8sjdj2EthQQE9WvpYDvsXIV3CjN4Ruq4Jw= +k8s.io/mount-utils v0.21.0-alpha.0 h1:IKn8GtClgNVIb801UbMpDI7F49D6KBXzSf5yTO5VZpo= k8s.io/mount-utils v0.21.0-alpha.0/go.mod h1:Jv9NRZ5L2LF87A17GaGlArD+r3JAJdZFvo4XD1cG4Kc= k8s.io/repo-infra v0.0.1-alpha.1/go.mod h1:wO1t9WaB99V80ljbeENTnayuEEwNZt7gECYh/CEyOJ8= k8s.io/sample-apiserver v0.20.0/go.mod h1:tScvbz/BcUG46IOsu2YLt4EjBP7XeUuMzMbQt2tQYWw= diff --git a/pkg/nfs/nodeserver.go b/pkg/nfs/nodeserver.go index 75b0068d..9e192351 100644 --- a/pkg/nfs/nodeserver.go +++ b/pkg/nfs/nodeserver.go @@ -26,6 +26,7 @@ import ( "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "k8s.io/kubernetes/pkg/volume" "k8s.io/utils/mount" ) @@ -148,8 +149,69 @@ func (ns *NodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetC } // NodeGetVolumeStats get volume stats -func (ns *NodeServer) NodeGetVolumeStats(ctx context.Context, in *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) { - return nil, status.Error(codes.Unimplemented, "") +func (ns *NodeServer) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) { + if len(req.VolumeId) == 0 { + return nil, status.Error(codes.InvalidArgument, "NodeGetVolumeStats volume ID was empty") + } + if len(req.VolumePath) == 0 { + return nil, status.Error(codes.InvalidArgument, "NodeGetVolumeStats volume path was empty") + } + + _, err := os.Stat(req.VolumePath) + if err != nil { + if os.IsNotExist(err) { + return nil, status.Errorf(codes.NotFound, "path %s does not exist", req.VolumePath) + } + return nil, status.Errorf(codes.Internal, "failed to stat file %s: %v", req.VolumePath, err) + } + + volumeMetrics, err := volume.NewMetricsStatFS(req.VolumePath).GetMetrics() + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to get metrics: %v", err) + } + + available, ok := volumeMetrics.Available.AsInt64() + if !ok { + return nil, status.Errorf(codes.Internal, "failed to transform volume available size(%v)", volumeMetrics.Available) + } + capacity, ok := volumeMetrics.Capacity.AsInt64() + if !ok { + return nil, status.Errorf(codes.Internal, "failed to transform volume capacity size(%v)", volumeMetrics.Capacity) + } + used, ok := volumeMetrics.Used.AsInt64() + if !ok { + return nil, status.Errorf(codes.Internal, "failed to transform volume used size(%v)", volumeMetrics.Used) + } + + inodesFree, ok := volumeMetrics.InodesFree.AsInt64() + if !ok { + return nil, status.Errorf(codes.Internal, "failed to transform disk inodes free(%v)", volumeMetrics.InodesFree) + } + inodes, ok := volumeMetrics.Inodes.AsInt64() + if !ok { + return nil, status.Errorf(codes.Internal, "failed to transform disk inodes(%v)", volumeMetrics.Inodes) + } + inodesUsed, ok := volumeMetrics.InodesUsed.AsInt64() + if !ok { + return nil, status.Errorf(codes.Internal, "failed to transform disk inodes used(%v)", volumeMetrics.InodesUsed) + } + + return &csi.NodeGetVolumeStatsResponse{ + Usage: []*csi.VolumeUsage{ + { + Unit: csi.VolumeUsage_BYTES, + Available: available, + Total: capacity, + Used: used, + }, + { + Unit: csi.VolumeUsage_INODES, + Available: inodesFree, + Total: inodes, + Used: inodesUsed, + }, + }, + }, nil } // NodeUnstageVolume unstage volume diff --git a/vendor/k8s.io/cloud-provider/CONTRIBUTING.md b/vendor/k8s.io/cloud-provider/CONTRIBUTING.md new file mode 100644 index 00000000..fe4643fd --- /dev/null +++ b/vendor/k8s.io/cloud-provider/CONTRIBUTING.md @@ -0,0 +1,9 @@ +# Contributing guidelines + +Welcome to Kubernetes. We are excited about the prospect of you joining our [community](https://github.com/kubernetes/community)! The Kubernetes community abides by the CNCF [code of conduct](code-of-conduct.md). + +Do not open pull requests directly against this repository, they will be ignored. Instead, please open pull requests against [kubernetes/kubernetes](https://git.k8s.io/kubernetes/). Please follow the same [contributing guide](https://git.k8s.io/kubernetes/CONTRIBUTING.md) you would follow for any other pull request made to kubernetes/kubernetes. Changes to this repo should be discussed with [sig cloud-provider](https://github.com/kubernetes/community/tree/master/sig-cloud-provider). + +This repository is published from [kubernetes/kubernetes/staging/src/k8s.io/cloud-provider](https://git.k8s.io/kubernetes/staging/src/k8s.io/cloud-provider) by the [kubernetes publishing-bot](https://git.k8s.io/publishing-bot). + +Please see [Staging Directory and Publishing](https://git.k8s.io/community/contributors/devel/sig-architecture/staging.md) for more information diff --git a/vendor/k8s.io/cloud-provider/LICENSE b/vendor/k8s.io/cloud-provider/LICENSE new file mode 100644 index 00000000..8dada3ed --- /dev/null +++ b/vendor/k8s.io/cloud-provider/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/k8s.io/cloud-provider/OWNERS b/vendor/k8s.io/cloud-provider/OWNERS new file mode 100644 index 00000000..b5c5205c --- /dev/null +++ b/vendor/k8s.io/cloud-provider/OWNERS @@ -0,0 +1,38 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: +- mikedanese +- dims +- wlan0 +- andrewsykim +- cheftako +reviewers: +- wojtek-t +- deads2k +- derekwaynecarr +- vishh +- mikedanese +- liggitt +- gmarek +- davidopp +- pmorie +- sttts +- quinton-hoole +- dchen1107 +- saad-ali +- luxas +- justinsb +- piosz +- jsafrane +- dims +- krousey +- rootfs +- freehan +- jingxu97 +- wlan0 +- cheftako +- andrewsykim +- mcrute +labels: +- sig/cloud-provider +- area/cloudprovider diff --git a/vendor/k8s.io/cloud-provider/README.md b/vendor/k8s.io/cloud-provider/README.md new file mode 100644 index 00000000..d53fec5a --- /dev/null +++ b/vendor/k8s.io/cloud-provider/README.md @@ -0,0 +1,32 @@ +# cloud-provider + +This repository defines the cloud-provider interface and mechanism to initialize +a cloud-provider implementation into Kubernetes. Currently multiple processes +use this code although the intent is that it will eventually only be cloud +controller manager. + +**Note:** go-get or vendor this package as `k8s.io/cloud-provider`. + +## Purpose + +This library is a shared dependency for processes which need to be able to +integrate with cloud-provider specific functionality. + +## Compatibility + +Cloud Providers are expected to keep the HEAD of their implementations in sync +with the HEAD of this repository. + +## Where does it come from? + +`cloud-provider` is synced from +https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/cloud-provider. +Code changes are made in that location, merged into k8s.io/kubernetes and +later synced here. + +## Things you should NOT do + + 1. Add an cloud provider specific code to this repo. + 2. Directly modify anything under vendor/k8s.io/cloud-provider in this repo. Those are driven from `k8s.io/kubernetes/staging/src/k8s.io/cloud-provider`. + 3. Make interface changes without first discussing them with + sig-cloudprovider. diff --git a/vendor/k8s.io/cloud-provider/SECURITY_CONTACTS b/vendor/k8s.io/cloud-provider/SECURITY_CONTACTS new file mode 100644 index 00000000..68e73eda --- /dev/null +++ b/vendor/k8s.io/cloud-provider/SECURITY_CONTACTS @@ -0,0 +1,20 @@ +# Defined below are the security contacts for this repo. +# +# They are the contact point for the Product Security Committee to reach out +# to for triaging and handling of incoming issues. +# +# The below names agree to abide by the +# [Embargo Policy](https://git.k8s.io/security/private-distributors-list.md#embargo-policy) +# and will be removed and replaced if they violate that agreement. +# +# DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE +# INSTRUCTIONS AT https://kubernetes.io/security/ + +cheftako +andrewsykim +dims +cjcullen +joelsmith +liggitt +philips +tallclair diff --git a/vendor/k8s.io/cloud-provider/cloud.go b/vendor/k8s.io/cloud-provider/cloud.go new file mode 100644 index 00000000..502443d6 --- /dev/null +++ b/vendor/k8s.io/cloud-provider/cloud.go @@ -0,0 +1,313 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudprovider + +import ( + "context" + "errors" + "fmt" + "strings" + + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/informers" + clientset "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" +) + +// ControllerClientBuilder allows you to get clients and configs for controllers +// Please note a copy also exists in pkg/controller/client_builder.go +// TODO: Make this depend on the separate controller utilities repo (issues/68947) +type ControllerClientBuilder interface { + Config(name string) (*restclient.Config, error) + ConfigOrDie(name string) *restclient.Config + Client(name string) (clientset.Interface, error) + ClientOrDie(name string) clientset.Interface +} + +// Interface is an abstract, pluggable interface for cloud providers. +type Interface interface { + // Initialize provides the cloud with a kubernetes client builder and may spawn goroutines + // to perform housekeeping or run custom controllers specific to the cloud provider. + // Any tasks started here should be cleaned up when the stop channel closes. + Initialize(clientBuilder ControllerClientBuilder, stop <-chan struct{}) + // LoadBalancer returns a balancer interface. Also returns true if the interface is supported, false otherwise. + LoadBalancer() (LoadBalancer, bool) + // Instances returns an instances interface. Also returns true if the interface is supported, false otherwise. + Instances() (Instances, bool) + // InstancesV2 is an implementation for instances and should only be implemented by external cloud providers. + // Implementing InstancesV2 is behaviorally identical to Instances but is optimized to significantly reduce + // API calls to the cloud provider when registering and syncing nodes. Implementation of this interface will + // disable calls to the Zones interface. Also returns true if the interface is supported, false otherwise. + InstancesV2() (InstancesV2, bool) + // Zones returns a zones interface. Also returns true if the interface is supported, false otherwise. + // DEPRECATED: Zones is deprecated in favor of retrieving zone/region information from InstancesV2. + // This interface will not be called if InstancesV2 is enabled. + Zones() (Zones, bool) + // Clusters returns a clusters interface. Also returns true if the interface is supported, false otherwise. + Clusters() (Clusters, bool) + // Routes returns a routes interface along with whether the interface is supported. + Routes() (Routes, bool) + // ProviderName returns the cloud provider ID. + ProviderName() string + // HasClusterID returns true if a ClusterID is required and set + HasClusterID() bool +} + +type InformerUser interface { + // SetInformers sets the informer on the cloud object. + SetInformers(informerFactory informers.SharedInformerFactory) +} + +// Clusters is an abstract, pluggable interface for clusters of containers. +type Clusters interface { + // ListClusters lists the names of the available clusters. + ListClusters(ctx context.Context) ([]string, error) + // Master gets back the address (either DNS name or IP address) of the master node for the cluster. + Master(ctx context.Context, clusterName string) (string, error) +} + +// (DEPRECATED) DefaultLoadBalancerName is the default load balancer name that is called from +// LoadBalancer.GetLoadBalancerName. Use this method to maintain backward compatible names for +// LoadBalancers that were created prior to Kubernetes v1.12. In the future, each provider should +// replace this method call in GetLoadBalancerName with a provider-specific implementation that +// is less cryptic than the Service's UUID. +func DefaultLoadBalancerName(service *v1.Service) string { + //GCE requires that the name of a load balancer starts with a lower case letter. + ret := "a" + string(service.UID) + ret = strings.Replace(ret, "-", "", -1) + //AWS requires that the name of a load balancer is shorter than 32 bytes. + if len(ret) > 32 { + ret = ret[:32] + } + return ret +} + +// GetInstanceProviderID builds a ProviderID for a node in a cloud. +func GetInstanceProviderID(ctx context.Context, cloud Interface, nodeName types.NodeName) (string, error) { + instances, ok := cloud.Instances() + if !ok { + return "", fmt.Errorf("failed to get instances from cloud provider") + } + instanceID, err := instances.InstanceID(ctx, nodeName) + if err != nil { + if err == NotImplemented { + return "", err + } + + return "", fmt.Errorf("failed to get instance ID from cloud provider: %v", err) + } + return cloud.ProviderName() + "://" + instanceID, nil +} + +// LoadBalancer is an abstract, pluggable interface for load balancers. +// +// Cloud provider may chose to implement the logic for +// constructing/destroying specific kinds of load balancers in a +// controller separate from the ServiceController. If this is the case, +// then {Ensure,Update}LoadBalancer must return the ImplementedElsewhere error. +// For the given LB service, the GetLoadBalancer must return "exists=True" if +// there exists a LoadBalancer instance created by ServiceController. +// In all other cases, GetLoadBalancer must return a NotFound error. +// EnsureLoadBalancerDeleted must not return ImplementedElsewhere to ensure +// proper teardown of resources that were allocated by the ServiceController. +// This can happen if a user changes the type of LB via an update to the resource +// or when migrating from ServiceController to alternate implementation. +// The finalizer on the service will be added and removed by ServiceController +// irrespective of the ImplementedElsewhere error. Additional finalizers for +// LB services must be managed in the alternate implementation. +type LoadBalancer interface { + // TODO: Break this up into different interfaces (LB, etc) when we have more than one type of service + // GetLoadBalancer returns whether the specified load balancer exists, and + // if so, what its status is. + // Implementations must treat the *v1.Service parameter as read-only and not modify it. + // Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager + GetLoadBalancer(ctx context.Context, clusterName string, service *v1.Service) (status *v1.LoadBalancerStatus, exists bool, err error) + // GetLoadBalancerName returns the name of the load balancer. Implementations must treat the + // *v1.Service parameter as read-only and not modify it. + GetLoadBalancerName(ctx context.Context, clusterName string, service *v1.Service) string + // EnsureLoadBalancer creates a new load balancer 'name', or updates the existing one. Returns the status of the balancer + // Implementations must treat the *v1.Service and *v1.Node + // parameters as read-only and not modify them. + // Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager + EnsureLoadBalancer(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node) (*v1.LoadBalancerStatus, error) + // UpdateLoadBalancer updates hosts under the specified load balancer. + // Implementations must treat the *v1.Service and *v1.Node + // parameters as read-only and not modify them. + // Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager + UpdateLoadBalancer(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node) error + // EnsureLoadBalancerDeleted deletes the specified load balancer if it + // exists, returning nil if the load balancer specified either didn't exist or + // was successfully deleted. + // This construction is useful because many cloud providers' load balancers + // have multiple underlying components, meaning a Get could say that the LB + // doesn't exist even if some part of it is still laying around. + // Implementations must treat the *v1.Service parameter as read-only and not modify it. + // Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager + EnsureLoadBalancerDeleted(ctx context.Context, clusterName string, service *v1.Service) error +} + +// Instances is an abstract, pluggable interface for sets of instances. +type Instances interface { + // NodeAddresses returns the addresses of the specified instance. + NodeAddresses(ctx context.Context, name types.NodeName) ([]v1.NodeAddress, error) + // NodeAddressesByProviderID returns the addresses of the specified instance. + // The instance is specified using the providerID of the node. The + // ProviderID is a unique identifier of the node. This will not be called + // from the node whose nodeaddresses are being queried. i.e. local metadata + // services cannot be used in this method to obtain nodeaddresses + NodeAddressesByProviderID(ctx context.Context, providerID string) ([]v1.NodeAddress, error) + // InstanceID returns the cloud provider ID of the node with the specified NodeName. + // Note that if the instance does not exist, we must return ("", cloudprovider.InstanceNotFound) + // cloudprovider.InstanceNotFound should NOT be returned for instances that exist but are stopped/sleeping + InstanceID(ctx context.Context, nodeName types.NodeName) (string, error) + // InstanceType returns the type of the specified instance. + InstanceType(ctx context.Context, name types.NodeName) (string, error) + // InstanceTypeByProviderID returns the type of the specified instance. + InstanceTypeByProviderID(ctx context.Context, providerID string) (string, error) + // AddSSHKeyToAllInstances adds an SSH public key as a legal identity for all instances + // expected format for the key is standard ssh-keygen format: + AddSSHKeyToAllInstances(ctx context.Context, user string, keyData []byte) error + // CurrentNodeName returns the name of the node we are currently running on + // On most clouds (e.g. GCE) this is the hostname, so we provide the hostname + CurrentNodeName(ctx context.Context, hostname string) (types.NodeName, error) + // InstanceExistsByProviderID returns true if the instance for the given provider exists. + // If false is returned with no error, the instance will be immediately deleted by the cloud controller manager. + // This method should still return true for instances that exist but are stopped/sleeping. + InstanceExistsByProviderID(ctx context.Context, providerID string) (bool, error) + // InstanceShutdownByProviderID returns true if the instance is shutdown in cloudprovider + InstanceShutdownByProviderID(ctx context.Context, providerID string) (bool, error) +} + +// InstancesV2 is an abstract, pluggable interface for cloud provider instances. +// Unlike the Instances interface, it is designed for external cloud providers and should only be used by them. +// Implementation of this interface will disable calls to the Zones interface. +type InstancesV2 interface { + // InstanceExists returns true if the instance for the given node exists according to the cloud provider. + // Use the node.name or node.spec.providerID field to find the node in the cloud provider. + InstanceExists(ctx context.Context, node *v1.Node) (bool, error) + // InstanceShutdown returns true if the instance is shutdown according to the cloud provider. + // Use the node.name or node.spec.providerID field to find the node in the cloud provider. + InstanceShutdown(ctx context.Context, node *v1.Node) (bool, error) + // InstanceMetadata returns the instance's metadata. The values returned in InstanceMetadata are + // translated into specific fields and labels in the Node object on registration. + // Implementations should always check node.spec.providerID first when trying to discover the instance + // for a given node. In cases where node.spec.providerID is empty, implementations can use other + // properties of the node like its name, labels and annotations. + InstanceMetadata(ctx context.Context, node *v1.Node) (*InstanceMetadata, error) +} + +// Route is a representation of an advanced routing rule. +type Route struct { + // Name is the name of the routing rule in the cloud-provider. + // It will be ignored in a Create (although nameHint may influence it) + Name string + // TargetNode is the NodeName of the target instance. + TargetNode types.NodeName + // DestinationCIDR is the CIDR format IP range that this routing rule + // applies to. + DestinationCIDR string + // Blackhole is set to true if this is a blackhole route + // The node controller will delete the route if it is in the managed range. + Blackhole bool +} + +// Routes is an abstract, pluggable interface for advanced routing rules. +type Routes interface { + // ListRoutes lists all managed routes that belong to the specified clusterName + ListRoutes(ctx context.Context, clusterName string) ([]*Route, error) + // CreateRoute creates the described managed route + // route.Name will be ignored, although the cloud-provider may use nameHint + // to create a more user-meaningful name. + CreateRoute(ctx context.Context, clusterName string, nameHint string, route *Route) error + // DeleteRoute deletes the specified managed route + // Route should be as returned by ListRoutes + DeleteRoute(ctx context.Context, clusterName string, route *Route) error +} + +var ( + DiskNotFound = errors.New("disk is not found") + ImplementedElsewhere = errors.New("implemented by alternate to cloud provider") + InstanceNotFound = errors.New("instance not found") + NotImplemented = errors.New("unimplemented") +) + +// Zone represents the location of a particular machine. +type Zone struct { + FailureDomain string + Region string +} + +// Zones is an abstract, pluggable interface for zone enumeration. +// DEPRECATED: Zones is deprecated in favor of retrieving zone/region information from InstancesV2. +// This interface will not be called if InstancesV2 is enabled. +type Zones interface { + // GetZone returns the Zone containing the current failure zone and locality region that the program is running in + // In most cases, this method is called from the kubelet querying a local metadata service to acquire its zone. + // For the case of external cloud providers, use GetZoneByProviderID or GetZoneByNodeName since GetZone + // can no longer be called from the kubelets. + GetZone(ctx context.Context) (Zone, error) + + // GetZoneByProviderID returns the Zone containing the current zone and locality region of the node specified by providerID + // This method is particularly used in the context of external cloud providers where node initialization must be done + // outside the kubelets. + GetZoneByProviderID(ctx context.Context, providerID string) (Zone, error) + + // GetZoneByNodeName returns the Zone containing the current zone and locality region of the node specified by node name + // This method is particularly used in the context of external cloud providers where node initialization must be done + // outside the kubelets. + GetZoneByNodeName(ctx context.Context, nodeName types.NodeName) (Zone, error) +} + +// PVLabeler is an abstract, pluggable interface for fetching labels for volumes +type PVLabeler interface { + GetLabelsForVolume(ctx context.Context, pv *v1.PersistentVolume) (map[string]string, error) +} + +// InstanceMetadata contains metadata about a specific instance. +// Values returned in InstanceMetadata are translated into specific fields and labels for Node. +type InstanceMetadata struct { + // ProviderID is a unique ID used to idenfitify an instance on the cloud provider. + // The ProviderID set here will be set on the node's spec.providerID field. + // The provider ID format can be set by the cloud provider but providers should + // ensure the format does not change in any incompatible way. + // + // The provider ID format used by existing cloud provider has been: + // :// + // Existing providers setting this field should preserve the existing format + // currently being set in node.spec.providerID. + ProviderID string + // InstanceType is the instance's type. + // The InstanceType set here will be set using the following labels on the node object: + // * node.kubernetes.io/instance-type= + // * beta.kubernetes.io/instance-type= (DEPRECATED) + InstanceType string + // NodeAddress contains information for the instance's address. + // The node addresses returned here will be set on the node's status.addresses field. + NodeAddresses []v1.NodeAddress + + // Zone is the zone that the instance is in. + // The value set here is applied as the following labels on the node: + // * topology.kubernetes.io/zone= + // * failure-domain.beta.kubernetes.io/zone= (DEPRECATED) + Zone string + // Region is the region that the instance is in. + // The value set here is applied as the following labels on the node: + // * topology.kubernetes.io/region= + // * failure-domain.beta.kubernetes.io/region= (DEPRECATED) + Region string +} diff --git a/vendor/k8s.io/cloud-provider/code-of-conduct.md b/vendor/k8s.io/cloud-provider/code-of-conduct.md new file mode 100644 index 00000000..0d15c00c --- /dev/null +++ b/vendor/k8s.io/cloud-provider/code-of-conduct.md @@ -0,0 +1,3 @@ +# Kubernetes Community Code of Conduct + +Please refer to our [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md) diff --git a/vendor/k8s.io/cloud-provider/doc.go b/vendor/k8s.io/cloud-provider/doc.go new file mode 100644 index 00000000..6b401e45 --- /dev/null +++ b/vendor/k8s.io/cloud-provider/doc.go @@ -0,0 +1,18 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package cloudprovider supplies interfaces and implementations for cloud service providers. +package cloudprovider // import "k8s.io/cloud-provider" diff --git a/vendor/k8s.io/cloud-provider/go.mod b/vendor/k8s.io/cloud-provider/go.mod new file mode 100644 index 00000000..ff85ae2b --- /dev/null +++ b/vendor/k8s.io/cloud-provider/go.mod @@ -0,0 +1,29 @@ +// This is a generated file. Do not edit directly. + +module k8s.io/cloud-provider + +go 1.15 + +require ( + github.com/google/go-cmp v0.5.2 + github.com/spf13/cobra v1.1.1 + github.com/spf13/pflag v1.0.5 + github.com/stretchr/testify v1.6.1 + k8s.io/api v0.20.0 + k8s.io/apimachinery v0.20.0 + k8s.io/apiserver v0.20.0 + k8s.io/client-go v0.20.0 + k8s.io/component-base v0.20.0 + k8s.io/controller-manager v0.20.0 + k8s.io/klog/v2 v2.4.0 + k8s.io/utils v0.0.0-20201110183641-67b214c5f920 +) + +replace ( + k8s.io/api => k8s.io/api v0.20.0 + k8s.io/apimachinery => k8s.io/apimachinery v0.20.0 + k8s.io/apiserver => k8s.io/apiserver v0.20.0 + k8s.io/client-go => k8s.io/client-go v0.20.0 + k8s.io/component-base => k8s.io/component-base v0.20.0 + k8s.io/controller-manager => k8s.io/controller-manager v0.20.0 +) diff --git a/vendor/k8s.io/cloud-provider/go.sum b/vendor/k8s.io/cloud-provider/go.sum new file mode 100644 index 00000000..e2791735 --- /dev/null +++ b/vendor/k8s.io/cloud-provider/go.sum @@ -0,0 +1,694 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= +github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46 h1:lsxEuwrXEAokXB9qhlbKWPpo3KMLZQ5WB5WLQRW1uq0= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= +github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= +github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= +github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 h1:5ZkaAPbicIKTF2I64qf5Fh8Aa83Q/dnOafMYV0OMwjA= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= +github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/moby/term v0.0.0-20200312100748-672ec06f55cd h1:aY7OQNf2XqY/JQ6qREWamhI/81os/agb2BAGpcx5yWI= +github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= +github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4= +github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489 h1:1JFLBqwIgdyHN1ZtgjTBwO+blA6gVOmZurpiMEsETKo= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 h1:pE8b58s1HRDMi8RDc79m0HISf9D4TzseP40cEA6IGfs= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd h1:5CtCZbICpIOFdgO940moixOPjc0178IU44m4EjOO5IY= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a h1:pOwg4OoaRYScjmR4LlLgdtnyoHYTSAVhhqe5uPdpII8= +google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.20.0/go.mod h1:HyLC5l5eoS/ygQYl1BXBgFzWNlkHiAuyNAbevIn+FKg= +k8s.io/apimachinery v0.20.0/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apiserver v0.20.0/go.mod h1:6gRIWiOkvGvQt12WTYmsiYoUyYW0FXSiMdNl4m+sxY8= +k8s.io/client-go v0.20.0/go.mod h1:4KWh/g+Ocd8KkCwKF8vUNnmqgv+EVnQDK4MBF4oB5tY= +k8s.io/component-base v0.20.0/go.mod h1:wKPj+RHnAr8LW2EIBIK7AxOHPde4gme2lzXwVSoRXeA= +k8s.io/controller-manager v0.20.0/go.mod h1:nD4qym/pmCz2v1tpqvlEBVlHW9CAZwedloM8GrJTLpg= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= +k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14 h1:TihvEz9MPj2u0KWds6E2OBUXfwaL4qRJ33c7HGiJpqk= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhYV9PmGrh1s8= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/vendor/k8s.io/cloud-provider/plugins.go b/vendor/k8s.io/cloud-provider/plugins.go new file mode 100644 index 00000000..e408e9cb --- /dev/null +++ b/vendor/k8s.io/cloud-provider/plugins.go @@ -0,0 +1,148 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudprovider + +import ( + "fmt" + "io" + "os" + "sync" + + "k8s.io/klog/v2" +) + +// Factory is a function that returns a cloudprovider.Interface. +// The config parameter provides an io.Reader handler to the factory in +// order to load specific configurations. If no configuration is provided +// the parameter is nil. +type Factory func(config io.Reader) (Interface, error) + +// All registered cloud providers. +var ( + providersMutex sync.Mutex + providers = make(map[string]Factory) + deprecatedCloudProviders = []struct { + name string + external bool + detail string + }{ + {"aws", false, "The AWS provider is deprecated and will be removed in a future release"}, + {"azure", false, "The Azure provider is deprecated and will be removed in a future release"}, + {"gce", false, "The GCE provider is deprecated and will be removed in a future release"}, + {"openstack", true, "https://github.com/kubernetes/cloud-provider-openstack"}, + {"vsphere", false, "The vSphere provider is deprecated and will be removed in a future release"}, + } +) + +const externalCloudProvider = "external" + +// RegisterCloudProvider registers a cloudprovider.Factory by name. This +// is expected to happen during app startup. +func RegisterCloudProvider(name string, cloud Factory) { + providersMutex.Lock() + defer providersMutex.Unlock() + if _, found := providers[name]; found { + klog.Fatalf("Cloud provider %q was registered twice", name) + } + klog.V(1).Infof("Registered cloud provider %q", name) + providers[name] = cloud +} + +// IsCloudProvider returns true if name corresponds to an already registered +// cloud provider. +func IsCloudProvider(name string) bool { + providersMutex.Lock() + defer providersMutex.Unlock() + _, found := providers[name] + return found +} + +// GetCloudProvider creates an instance of the named cloud provider, or nil if +// the name is unknown. The error return is only used if the named provider +// was known but failed to initialize. The config parameter specifies the +// io.Reader handler of the configuration file for the cloud provider, or nil +// for no configuration. +func GetCloudProvider(name string, config io.Reader) (Interface, error) { + providersMutex.Lock() + defer providersMutex.Unlock() + f, found := providers[name] + if !found { + return nil, nil + } + return f(config) +} + +// Detects if the string is an external cloud provider +func IsExternal(name string) bool { + return name == externalCloudProvider +} + +func DeprecationWarningForProvider(providerName string) { + for _, provider := range deprecatedCloudProviders { + if provider.name != providerName { + continue + } + + detail := provider.detail + if provider.external { + detail = fmt.Sprintf("Please use 'external' cloud provider for %s: %s", providerName, provider.detail) + } + + klog.Warningf("WARNING: %s built-in cloud provider is now deprecated. %s", providerName, detail) + break + } +} + +// InitCloudProvider creates an instance of the named cloud provider. +func InitCloudProvider(name string, configFilePath string) (Interface, error) { + var cloud Interface + var err error + + if name == "" { + return nil, nil + } + + if IsExternal(name) { + klog.Info("External cloud provider specified") + return nil, nil + } + + if configFilePath != "" { + var config *os.File + config, err = os.Open(configFilePath) + if err != nil { + klog.Fatalf("Couldn't open cloud provider configuration %s: %#v", + configFilePath, err) + } + + defer config.Close() + cloud, err = GetCloudProvider(name, config) + } else { + // Pass explicit nil so plugins can actually check for nil. See + // "Why is my nil error value not equal to nil?" in golang.org/doc/faq. + cloud, err = GetCloudProvider(name, nil) + } + + if err != nil { + return nil, fmt.Errorf("could not init cloud provider %q: %v", name, err) + } + if cloud == nil { + return nil, fmt.Errorf("unknown cloud provider %q", name) + } + + return cloud, nil +} diff --git a/vendor/k8s.io/cloud-provider/ports.go b/vendor/k8s.io/cloud-provider/ports.go new file mode 100644 index 00000000..797de6d9 --- /dev/null +++ b/vendor/k8s.io/cloud-provider/ports.go @@ -0,0 +1,23 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudprovider + +const ( + // CloudControllerManagerPort is the default port for the cloud controller manager server. + // This value may be overridden by a flag at startup. + CloudControllerManagerPort = 10258 +) diff --git a/vendor/k8s.io/kubernetes/pkg/proxy/util/BUILD b/vendor/k8s.io/kubernetes/pkg/proxy/util/BUILD new file mode 100644 index 00000000..4f1ce986 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/proxy/util/BUILD @@ -0,0 +1,58 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "endpoints.go", + "network.go", + "port.go", + "utils.go", + ], + importpath = "k8s.io/kubernetes/pkg/proxy/util", + visibility = ["//visibility:public"], + deps = [ + "//pkg/apis/core/v1/helper:go_default_library", + "//pkg/util/sysctl:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/rand:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//staging/src/k8s.io/client-go/tools/record:go_default_library", + "//vendor/k8s.io/klog/v2:go_default_library", + "//vendor/k8s.io/utils/net:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = [ + "endpoints_test.go", + "port_test.go", + "utils_test.go", + ], + embed = [":go_default_library"], + deps = [ + "//pkg/proxy/util/testing:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [ + ":package-srcs", + "//pkg/proxy/util/iptables:all-srcs", + "//pkg/proxy/util/testing:all-srcs", + ], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/vendor/k8s.io/kubernetes/pkg/proxy/util/endpoints.go b/vendor/k8s.io/kubernetes/pkg/proxy/util/endpoints.go new file mode 100644 index 00000000..0e017c54 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/proxy/util/endpoints.go @@ -0,0 +1,74 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + "net" + "strconv" + + "k8s.io/klog/v2" +) + +// IPPart returns just the IP part of an IP or IP:port or endpoint string. If the IP +// part is an IPv6 address enclosed in brackets (e.g. "[fd00:1::5]:9999"), +// then the brackets are stripped as well. +func IPPart(s string) string { + if ip := net.ParseIP(s); ip != nil { + // IP address without port + return s + } + // Must be IP:port + host, _, err := net.SplitHostPort(s) + if err != nil { + klog.Errorf("Error parsing '%s': %v", s, err) + return "" + } + // Check if host string is a valid IP address + ip := net.ParseIP(host) + if ip == nil { + klog.Errorf("invalid IP part '%s'", host) + return "" + } + return ip.String() +} + +// PortPart returns just the port part of an endpoint string. +func PortPart(s string) (int, error) { + // Must be IP:port + _, port, err := net.SplitHostPort(s) + if err != nil { + klog.Errorf("Error parsing '%s': %v", s, err) + return -1, err + } + portNumber, err := strconv.Atoi(port) + if err != nil { + klog.Errorf("Error parsing '%s': %v", port, err) + return -1, err + } + return portNumber, nil +} + +// ToCIDR returns a host address of the form /32 for +// IPv4 and /128 for IPv6 +func ToCIDR(ip net.IP) string { + len := 32 + if ip.To4() == nil { + len = 128 + } + return fmt.Sprintf("%s/%d", ip.String(), len) +} diff --git a/vendor/k8s.io/kubernetes/pkg/proxy/util/network.go b/vendor/k8s.io/kubernetes/pkg/proxy/util/network.go new file mode 100644 index 00000000..504ad939 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/proxy/util/network.go @@ -0,0 +1,45 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "net" +) + +// NetworkInterfacer defines an interface for several net library functions. Production +// code will forward to net library functions, and unit tests will override the methods +// for testing purposes. +type NetworkInterfacer interface { + Addrs(intf *net.Interface) ([]net.Addr, error) + Interfaces() ([]net.Interface, error) +} + +// RealNetwork implements the NetworkInterfacer interface for production code, just +// wrapping the underlying net library function calls. +type RealNetwork struct{} + +// Addrs wraps net.Interface.Addrs(), it's a part of NetworkInterfacer interface. +func (RealNetwork) Addrs(intf *net.Interface) ([]net.Addr, error) { + return intf.Addrs() +} + +// Interfaces wraps net.Interfaces(), it's a part of NetworkInterfacer interface. +func (RealNetwork) Interfaces() ([]net.Interface, error) { + return net.Interfaces() +} + +var _ NetworkInterfacer = &RealNetwork{} diff --git a/vendor/k8s.io/kubernetes/pkg/proxy/util/port.go b/vendor/k8s.io/kubernetes/pkg/proxy/util/port.go new file mode 100644 index 00000000..e256bd9e --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/proxy/util/port.go @@ -0,0 +1,67 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + "net" + "strconv" + + "k8s.io/klog/v2" +) + +// LocalPort describes a port on specific IP address and protocol +type LocalPort struct { + // Description is the identity message of a given local port. + Description string + // IP is the IP address part of a given local port. + // If this string is empty, the port binds to all local IP addresses. + IP string + // Port is the port part of a given local port. + Port int + // Protocol is the protocol part of a given local port. + // The value is assumed to be lower-case. For example, "udp" not "UDP", "tcp" not "TCP". + Protocol string +} + +func (lp *LocalPort) String() string { + ipPort := net.JoinHostPort(lp.IP, strconv.Itoa(lp.Port)) + return fmt.Sprintf("%q (%s/%s)", lp.Description, ipPort, lp.Protocol) +} + +// Closeable is an interface around closing a port. +type Closeable interface { + Close() error +} + +// PortOpener is an interface around port opening/closing. +// Abstracted out for testing. +type PortOpener interface { + OpenLocalPort(lp *LocalPort, isIPv6 bool) (Closeable, error) +} + +// RevertPorts is closing ports in replacementPortsMap but not in originalPortsMap. In other words, it only +// closes the ports opened in this sync. +func RevertPorts(replacementPortsMap, originalPortsMap map[LocalPort]Closeable) { + for k, v := range replacementPortsMap { + // Only close newly opened local ports - leave ones that were open before this update + if originalPortsMap[k] == nil { + klog.V(2).Infof("Closing local port %s", k.String()) + v.Close() + } + } +} diff --git a/vendor/k8s.io/kubernetes/pkg/proxy/util/utils.go b/vendor/k8s.io/kubernetes/pkg/proxy/util/utils.go new file mode 100644 index 00000000..56752b37 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/proxy/util/utils.go @@ -0,0 +1,443 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "strconv" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + utilrand "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/record" + helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" + utilsysctl "k8s.io/kubernetes/pkg/util/sysctl" + utilnet "k8s.io/utils/net" + + "k8s.io/klog/v2" +) + +const ( + // IPv4ZeroCIDR is the CIDR block for the whole IPv4 address space + IPv4ZeroCIDR = "0.0.0.0/0" + + // IPv6ZeroCIDR is the CIDR block for the whole IPv6 address space + IPv6ZeroCIDR = "::/0" +) + +var ( + // ErrAddressNotAllowed indicates the address is not allowed + ErrAddressNotAllowed = errors.New("address not allowed") + + // ErrNoAddresses indicates there are no addresses for the hostname + ErrNoAddresses = errors.New("No addresses for hostname") +) + +// isValidEndpoint checks that the given host / port pair are valid endpoint +func isValidEndpoint(host string, port int) bool { + return host != "" && port > 0 +} + +// BuildPortsToEndpointsMap builds a map of portname -> all ip:ports for that +// portname. Explode Endpoints.Subsets[*] into this structure. +func BuildPortsToEndpointsMap(endpoints *v1.Endpoints) map[string][]string { + portsToEndpoints := map[string][]string{} + for i := range endpoints.Subsets { + ss := &endpoints.Subsets[i] + for i := range ss.Ports { + port := &ss.Ports[i] + for i := range ss.Addresses { + addr := &ss.Addresses[i] + if isValidEndpoint(addr.IP, int(port.Port)) { + portsToEndpoints[port.Name] = append(portsToEndpoints[port.Name], net.JoinHostPort(addr.IP, strconv.Itoa(int(port.Port)))) + } + } + } + } + return portsToEndpoints +} + +// IsZeroCIDR checks whether the input CIDR string is either +// the IPv4 or IPv6 zero CIDR +func IsZeroCIDR(cidr string) bool { + if cidr == IPv4ZeroCIDR || cidr == IPv6ZeroCIDR { + return true + } + return false +} + +// IsProxyableIP checks if a given IP address is permitted to be proxied +func IsProxyableIP(ip string) error { + netIP := net.ParseIP(ip) + if netIP == nil { + return ErrAddressNotAllowed + } + return isProxyableIP(netIP) +} + +func isProxyableIP(ip net.IP) error { + if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsInterfaceLocalMulticast() { + return ErrAddressNotAllowed + } + return nil +} + +// Resolver is an interface for net.Resolver +type Resolver interface { + LookupIPAddr(ctx context.Context, host string) ([]net.IPAddr, error) +} + +// IsProxyableHostname checks if the IP addresses for a given hostname are permitted to be proxied +func IsProxyableHostname(ctx context.Context, resolv Resolver, hostname string) error { + resp, err := resolv.LookupIPAddr(ctx, hostname) + if err != nil { + return err + } + + if len(resp) == 0 { + return ErrNoAddresses + } + + for _, host := range resp { + if err := isProxyableIP(host.IP); err != nil { + return err + } + } + return nil +} + +// IsAllowedHost checks if the given IP host address is in a network in the denied list. +func IsAllowedHost(host net.IP, denied []*net.IPNet) error { + for _, ipNet := range denied { + if ipNet.Contains(host) { + return ErrAddressNotAllowed + } + } + return nil +} + +// GetLocalAddrs returns a list of all network addresses on the local system +func GetLocalAddrs() ([]net.IP, error) { + var localAddrs []net.IP + + addrs, err := net.InterfaceAddrs() + if err != nil { + return nil, err + } + + for _, addr := range addrs { + ip, _, err := net.ParseCIDR(addr.String()) + if err != nil { + return nil, err + } + + localAddrs = append(localAddrs, ip) + } + + return localAddrs, nil +} + +// ShouldSkipService checks if a given service should skip proxying +func ShouldSkipService(service *v1.Service) bool { + // if ClusterIP is "None" or empty, skip proxying + if !helper.IsServiceIPSet(service) { + klog.V(3).Infof("Skipping service %s in namespace %s due to clusterIP = %q", service.Name, service.Namespace, service.Spec.ClusterIP) + return true + } + // Even if ClusterIP is set, ServiceTypeExternalName services don't get proxied + if service.Spec.Type == v1.ServiceTypeExternalName { + klog.V(3).Infof("Skipping service %s in namespace %s due to Type=ExternalName", service.Name, service.Namespace) + return true + } + return false +} + +// GetNodeAddresses return all matched node IP addresses based on given cidr slice. +// Some callers, e.g. IPVS proxier, need concrete IPs, not ranges, which is why this exists. +// NetworkInterfacer is injected for test purpose. +// We expect the cidrs passed in is already validated. +// Given an empty input `[]`, it will return `0.0.0.0/0` and `::/0` directly. +// If multiple cidrs is given, it will return the minimal IP sets, e.g. given input `[1.2.0.0/16, 0.0.0.0/0]`, it will +// only return `0.0.0.0/0`. +// NOTE: GetNodeAddresses only accepts CIDRs, if you want concrete IPs, e.g. 1.2.3.4, then the input should be 1.2.3.4/32. +func GetNodeAddresses(cidrs []string, nw NetworkInterfacer) (sets.String, error) { + uniqueAddressList := sets.NewString() + if len(cidrs) == 0 { + uniqueAddressList.Insert(IPv4ZeroCIDR) + uniqueAddressList.Insert(IPv6ZeroCIDR) + return uniqueAddressList, nil + } + // First round of iteration to pick out `0.0.0.0/0` or `::/0` for the sake of excluding non-zero IPs. + for _, cidr := range cidrs { + if IsZeroCIDR(cidr) { + uniqueAddressList.Insert(cidr) + } + } + + itfs, err := nw.Interfaces() + if err != nil { + return nil, fmt.Errorf("error listing all interfaces from host, error: %v", err) + } + + // Second round of iteration to parse IPs based on cidr. + for _, cidr := range cidrs { + if IsZeroCIDR(cidr) { + continue + } + + _, ipNet, _ := net.ParseCIDR(cidr) + for _, itf := range itfs { + addrs, err := nw.Addrs(&itf) + if err != nil { + return nil, fmt.Errorf("error getting address from interface %s, error: %v", itf.Name, err) + } + + for _, addr := range addrs { + if addr == nil { + continue + } + + ip, _, err := net.ParseCIDR(addr.String()) + if err != nil { + return nil, fmt.Errorf("error parsing CIDR for interface %s, error: %v", itf.Name, err) + } + + if ipNet.Contains(ip) { + if utilnet.IsIPv6(ip) && !uniqueAddressList.Has(IPv6ZeroCIDR) { + uniqueAddressList.Insert(ip.String()) + } + if !utilnet.IsIPv6(ip) && !uniqueAddressList.Has(IPv4ZeroCIDR) { + uniqueAddressList.Insert(ip.String()) + } + } + } + } + } + + if uniqueAddressList.Len() == 0 { + return nil, fmt.Errorf("no addresses found for cidrs %v", cidrs) + } + + return uniqueAddressList, nil +} + +// LogAndEmitIncorrectIPVersionEvent logs and emits incorrect IP version event. +func LogAndEmitIncorrectIPVersionEvent(recorder record.EventRecorder, fieldName, fieldValue, svcNamespace, svcName string, svcUID types.UID) { + errMsg := fmt.Sprintf("%s in %s has incorrect IP version", fieldValue, fieldName) + klog.Errorf("%s (service %s/%s).", errMsg, svcNamespace, svcName) + if recorder != nil { + recorder.Eventf( + &v1.ObjectReference{ + Kind: "Service", + Name: svcName, + Namespace: svcNamespace, + UID: svcUID, + }, v1.EventTypeWarning, "KubeProxyIncorrectIPVersion", errMsg) + } +} + +// MapIPsByIPFamily maps a slice of IPs to their respective IP families (v4 or v6) +func MapIPsByIPFamily(ipStrings []string) map[v1.IPFamily][]string { + ipFamilyMap := map[v1.IPFamily][]string{} + for _, ip := range ipStrings { + // Handle only the valid IPs + if ipFamily, err := getIPFamilyFromIP(ip); err == nil { + ipFamilyMap[ipFamily] = append(ipFamilyMap[ipFamily], ip) + } else { + klog.Errorf("Skipping invalid IP: %s", ip) + } + } + return ipFamilyMap +} + +// MapCIDRsByIPFamily maps a slice of IPs to their respective IP families (v4 or v6) +func MapCIDRsByIPFamily(cidrStrings []string) map[v1.IPFamily][]string { + ipFamilyMap := map[v1.IPFamily][]string{} + for _, cidr := range cidrStrings { + // Handle only the valid CIDRs + if ipFamily, err := getIPFamilyFromCIDR(cidr); err == nil { + ipFamilyMap[ipFamily] = append(ipFamilyMap[ipFamily], cidr) + } else { + klog.Errorf("Skipping invalid cidr: %s", cidr) + } + } + return ipFamilyMap +} + +func getIPFamilyFromIP(ipStr string) (v1.IPFamily, error) { + netIP := net.ParseIP(ipStr) + if netIP == nil { + return "", ErrAddressNotAllowed + } + + if utilnet.IsIPv6(netIP) { + return v1.IPv6Protocol, nil + } + return v1.IPv4Protocol, nil +} + +func getIPFamilyFromCIDR(cidrStr string) (v1.IPFamily, error) { + _, netCIDR, err := net.ParseCIDR(cidrStr) + if err != nil { + return "", ErrAddressNotAllowed + } + if utilnet.IsIPv6CIDR(netCIDR) { + return v1.IPv6Protocol, nil + } + return v1.IPv4Protocol, nil +} + +// OtherIPFamily returns the other ip family +func OtherIPFamily(ipFamily v1.IPFamily) v1.IPFamily { + if ipFamily == v1.IPv6Protocol { + return v1.IPv4Protocol + } + + return v1.IPv6Protocol +} + +// AppendPortIfNeeded appends the given port to IP address unless it is already in +// "ipv4:port" or "[ipv6]:port" format. +func AppendPortIfNeeded(addr string, port int32) string { + // Return if address is already in "ipv4:port" or "[ipv6]:port" format. + if _, _, err := net.SplitHostPort(addr); err == nil { + return addr + } + + // Simply return for invalid case. This should be caught by validation instead. + ip := net.ParseIP(addr) + if ip == nil { + return addr + } + + // Append port to address. + if ip.To4() != nil { + return fmt.Sprintf("%s:%d", addr, port) + } + return fmt.Sprintf("[%s]:%d", addr, port) +} + +// ShuffleStrings copies strings from the specified slice into a copy in random +// order. It returns a new slice. +func ShuffleStrings(s []string) []string { + if s == nil { + return nil + } + shuffled := make([]string, len(s)) + perm := utilrand.Perm(len(s)) + for i, j := range perm { + shuffled[j] = s[i] + } + return shuffled +} + +// EnsureSysctl sets a kernel sysctl to a given numeric value. +func EnsureSysctl(sysctl utilsysctl.Interface, name string, newVal int) error { + if oldVal, _ := sysctl.GetSysctl(name); oldVal != newVal { + if err := sysctl.SetSysctl(name, newVal); err != nil { + return fmt.Errorf("can't set sysctl %s to %d: %v", name, newVal, err) + } + klog.V(1).Infof("Changed sysctl %q: %d -> %d", name, oldVal, newVal) + } + return nil +} + +// DialContext is a dial function matching the signature of net.Dialer.DialContext. +type DialContext = func(context.Context, string, string) (net.Conn, error) + +// FilteredDialOptions configures how a DialContext is wrapped by NewFilteredDialContext. +type FilteredDialOptions struct { + // DialHostIPDenylist restricts hosts from being dialed. + DialHostCIDRDenylist []*net.IPNet + // AllowLocalLoopback controls connections to local loopback hosts (as defined by + // IsProxyableIP). + AllowLocalLoopback bool +} + +// NewFilteredDialContext returns a DialContext function that filters connections based on a FilteredDialOptions. +func NewFilteredDialContext(wrapped DialContext, resolv Resolver, opts *FilteredDialOptions) DialContext { + if wrapped == nil { + wrapped = http.DefaultTransport.(*http.Transport).DialContext + } + if opts == nil { + // Do no filtering + return wrapped + } + if resolv == nil { + resolv = net.DefaultResolver + } + if len(opts.DialHostCIDRDenylist) == 0 && opts.AllowLocalLoopback { + // Do no filtering. + return wrapped + } + return func(ctx context.Context, network, address string) (net.Conn, error) { + resp, err := resolv.LookupIPAddr(ctx, address) + if err != nil { + return nil, err + } + + if len(resp) == 0 { + return nil, ErrNoAddresses + } + + for _, host := range resp { + if !opts.AllowLocalLoopback { + if err := isProxyableIP(host.IP); err != nil { + return nil, err + } + } + if opts.DialHostCIDRDenylist != nil { + if err := IsAllowedHost(host.IP, opts.DialHostCIDRDenylist); err != nil { + return nil, err + } + } + } + return wrapped(ctx, network, address) + } +} + +// GetClusterIPByFamily returns a service clusterip by family +func GetClusterIPByFamily(ipFamily v1.IPFamily, service *v1.Service) string { + // allowing skew + if len(service.Spec.IPFamilies) == 0 { + if len(service.Spec.ClusterIP) == 0 || service.Spec.ClusterIP == v1.ClusterIPNone { + return "" + } + + IsIPv6Family := (ipFamily == v1.IPv6Protocol) + if IsIPv6Family == utilnet.IsIPv6String(service.Spec.ClusterIP) { + return service.Spec.ClusterIP + } + + return "" + } + + for idx, family := range service.Spec.IPFamilies { + if family == ipFamily { + if idx < len(service.Spec.ClusterIPs) { + return service.Spec.ClusterIPs[idx] + } + } + } + + return "" +} diff --git a/vendor/k8s.io/kubernetes/pkg/util/sysctl/BUILD b/vendor/k8s.io/kubernetes/pkg/util/sysctl/BUILD new file mode 100644 index 00000000..77d69988 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/util/sysctl/BUILD @@ -0,0 +1,28 @@ +package(default_visibility = ["//visibility:public"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", +) + +go_library( + name = "go_default_library", + srcs = ["sysctl.go"], + importpath = "k8s.io/kubernetes/pkg/util/sysctl", +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [ + ":package-srcs", + "//pkg/util/sysctl/testing:all-srcs", + ], + tags = ["automanaged"], +) diff --git a/vendor/k8s.io/kubernetes/pkg/util/sysctl/sysctl.go b/vendor/k8s.io/kubernetes/pkg/util/sysctl/sysctl.go new file mode 100644 index 00000000..311212c1 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/util/sysctl/sysctl.go @@ -0,0 +1,100 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sysctl + +import ( + "io/ioutil" + "path" + "strconv" + "strings" +) + +const ( + sysctlBase = "/proc/sys" + // VMOvercommitMemory refers to the sysctl variable responsible for defining + // the memory over-commit policy used by kernel. + VMOvercommitMemory = "vm/overcommit_memory" + // VMPanicOnOOM refers to the sysctl variable responsible for defining + // the OOM behavior used by kernel. + VMPanicOnOOM = "vm/panic_on_oom" + // KernelPanic refers to the sysctl variable responsible for defining + // the timeout after a panic for the kernel to reboot. + KernelPanic = "kernel/panic" + // KernelPanicOnOops refers to the sysctl variable responsible for defining + // the kernel behavior when an oops or BUG is encountered. + KernelPanicOnOops = "kernel/panic_on_oops" + // RootMaxKeys refers to the sysctl variable responsible for defining + // the maximum number of keys that the root user (UID 0 in the root user namespace) may own. + RootMaxKeys = "kernel/keys/root_maxkeys" + // RootMaxBytes refers to the sysctl variable responsible for defining + // the maximum number of bytes of data that the root user (UID 0 in the root user namespace) + // can hold in the payloads of the keys owned by root. + RootMaxBytes = "kernel/keys/root_maxbytes" + + // VMOvercommitMemoryAlways represents that kernel performs no memory over-commit handling. + VMOvercommitMemoryAlways = 1 + // VMPanicOnOOMInvokeOOMKiller represents that kernel calls the oom_killer function when OOM occurs. + VMPanicOnOOMInvokeOOMKiller = 0 + + // KernelPanicOnOopsAlways represents that kernel panics on kernel oops. + KernelPanicOnOopsAlways = 1 + // KernelPanicRebootTimeout is the timeout seconds after a panic for the kernel to reboot. + KernelPanicRebootTimeout = 10 + + // RootMaxKeysSetting is the maximum number of keys that the root user (UID 0 in the root user namespace) may own. + // Needed since docker creates a new key per container. + RootMaxKeysSetting = 1000000 + // RootMaxBytesSetting is the maximum number of bytes of data that the root user (UID 0 in the root user namespace) + // can hold in the payloads of the keys owned by root. + // Allocate 25 bytes per key * number of MaxKeys. + RootMaxBytesSetting = RootMaxKeysSetting * 25 +) + +// Interface is an injectable interface for running sysctl commands. +type Interface interface { + // GetSysctl returns the value for the specified sysctl setting + GetSysctl(sysctl string) (int, error) + // SetSysctl modifies the specified sysctl flag to the new value + SetSysctl(sysctl string, newVal int) error +} + +// New returns a new Interface for accessing sysctl +func New() Interface { + return &procSysctl{} +} + +// procSysctl implements Interface by reading and writing files under /proc/sys +type procSysctl struct { +} + +// GetSysctl returns the value for the specified sysctl setting +func (*procSysctl) GetSysctl(sysctl string) (int, error) { + data, err := ioutil.ReadFile(path.Join(sysctlBase, sysctl)) + if err != nil { + return -1, err + } + val, err := strconv.Atoi(strings.Trim(string(data), " \n")) + if err != nil { + return -1, err + } + return val, nil +} + +// SetSysctl modifies the specified sysctl flag to the new value +func (*procSysctl) SetSysctl(sysctl string, newVal int) error { + return ioutil.WriteFile(path.Join(sysctlBase, sysctl), []byte(strconv.Itoa(newVal)), 0640) +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/BUILD b/vendor/k8s.io/kubernetes/pkg/volume/BUILD new file mode 100644 index 00000000..d92b63ec --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/BUILD @@ -0,0 +1,134 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "doc.go", + "metrics_cached.go", + "metrics_du.go", + "metrics_errors.go", + "metrics_nil.go", + "metrics_statfs.go", + "noop_expandable_plugin.go", + "plugins.go", + "volume.go", + "volume_linux.go", + "volume_unsupported.go", + ], + importpath = "k8s.io/kubernetes/pkg/volume", + visibility = ["//visibility:public"], + deps = [ + "//pkg/proxy/util:go_default_library", + "//pkg/volume/util/fs:go_default_library", + "//pkg/volume/util/hostutil:go_default_library", + "//pkg/volume/util/recyclerclient:go_default_library", + "//pkg/volume/util/subpath:go_default_library", + "//staging/src/k8s.io/api/authentication/v1:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library", + "//staging/src/k8s.io/client-go/informers:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/listers/storage/v1:go_default_library", + "//staging/src/k8s.io/client-go/tools/cache:go_default_library", + "//staging/src/k8s.io/client-go/tools/record:go_default_library", + "//staging/src/k8s.io/cloud-provider:go_default_library", + "//staging/src/k8s.io/mount-utils:go_default_library", + "//vendor/k8s.io/klog/v2:go_default_library", + "//vendor/k8s.io/utils/exec:go_default_library", + ] + select({ + "@io_bazel_rules_go//go/platform:android": [ + "//pkg/features:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + ], + "@io_bazel_rules_go//go/platform:linux": [ + "//pkg/features:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + ], + "//conditions:default": [], + }), +) + +go_test( + name = "go_default_test", + srcs = [ + "metrics_du_test.go", + "metrics_nil_test.go", + "metrics_statfs_test.go", + "plugins_test.go", + "volume_linux_test.go", + ], + embed = [":go_default_library"], + deps = [ + "//pkg/volume/testing:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/client-go/util/testing:go_default_library", + ] + select({ + "@io_bazel_rules_go//go/platform:android": [ + "//pkg/features:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library", + "//vendor/golang.org/x/sys/unix:go_default_library", + ], + "@io_bazel_rules_go//go/platform:linux": [ + "//pkg/features:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library", + "//vendor/golang.org/x/sys/unix:go_default_library", + ], + "//conditions:default": [], + }), +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [ + ":package-srcs", + "//pkg/volume/awsebs:all-srcs", + "//pkg/volume/azure_file:all-srcs", + "//pkg/volume/azuredd:all-srcs", + "//pkg/volume/cephfs:all-srcs", + "//pkg/volume/cinder:all-srcs", + "//pkg/volume/configmap:all-srcs", + "//pkg/volume/csi:all-srcs", + "//pkg/volume/csimigration:all-srcs", + "//pkg/volume/downwardapi:all-srcs", + "//pkg/volume/emptydir:all-srcs", + "//pkg/volume/fc:all-srcs", + "//pkg/volume/flexvolume:all-srcs", + "//pkg/volume/flocker:all-srcs", + "//pkg/volume/gcepd:all-srcs", + "//pkg/volume/git_repo:all-srcs", + "//pkg/volume/glusterfs:all-srcs", + "//pkg/volume/hostpath:all-srcs", + "//pkg/volume/iscsi:all-srcs", + "//pkg/volume/local:all-srcs", + "//pkg/volume/nfs:all-srcs", + "//pkg/volume/portworx:all-srcs", + "//pkg/volume/projected:all-srcs", + "//pkg/volume/quobyte:all-srcs", + "//pkg/volume/rbd:all-srcs", + "//pkg/volume/scaleio:all-srcs", + "//pkg/volume/secret:all-srcs", + "//pkg/volume/storageos:all-srcs", + "//pkg/volume/testing:all-srcs", + "//pkg/volume/util:all-srcs", + "//pkg/volume/validation:all-srcs", + "//pkg/volume/vsphere_volume:all-srcs", + ], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/vendor/k8s.io/kubernetes/pkg/volume/OWNERS b/vendor/k8s.io/kubernetes/pkg/volume/OWNERS new file mode 100644 index 00000000..6b5f6ba5 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/OWNERS @@ -0,0 +1,8 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: +- sig-storage-approvers +reviewers: +- sig-storage-reviewers +labels: +- sig/storage diff --git a/vendor/k8s.io/kubernetes/pkg/volume/doc.go b/vendor/k8s.io/kubernetes/pkg/volume/doc.go new file mode 100644 index 00000000..c98a5a15 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package volume includes internal representations of external volume types +// as well as utility methods required to mount/unmount volumes to kubelets. +package volume // import "k8s.io/kubernetes/pkg/volume" diff --git a/vendor/k8s.io/kubernetes/pkg/volume/metrics_cached.go b/vendor/k8s.io/kubernetes/pkg/volume/metrics_cached.go new file mode 100644 index 00000000..ac0dc9b7 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/metrics_cached.go @@ -0,0 +1,74 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volume + +import ( + "sync" + "sync/atomic" +) + +var _ MetricsProvider = &cachedMetrics{} + +// cachedMetrics represents a MetricsProvider that wraps another provider and +// caches the result. +type cachedMetrics struct { + wrapped MetricsProvider + resultError error + resultMetrics *Metrics + once cacheOnce +} + +// NewCachedMetrics creates a new cachedMetrics wrapping another +// MetricsProvider and caching the results. +func NewCachedMetrics(provider MetricsProvider) MetricsProvider { + return &cachedMetrics{wrapped: provider} +} + +// GetMetrics runs the wrapped metrics provider's GetMetrics methd once and +// caches the result. Will not cache result if there is an error. +// See MetricsProvider.GetMetrics +func (md *cachedMetrics) GetMetrics() (*Metrics, error) { + md.once.cache(func() error { + md.resultMetrics, md.resultError = md.wrapped.GetMetrics() + return md.resultError + }) + return md.resultMetrics, md.resultError +} + +// Copied from sync.Once but we don't want to cache the results if there is an +// error +type cacheOnce struct { + m sync.Mutex + done uint32 +} + +// Copied from sync.Once but we don't want to cache the results if there is an +// error +func (o *cacheOnce) cache(f func() error) { + if atomic.LoadUint32(&o.done) == 1 { + return + } + // Slow-path. + o.m.Lock() + defer o.m.Unlock() + if o.done == 0 { + err := f() + if err == nil { + atomic.StoreUint32(&o.done, 1) + } + } +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/metrics_du.go b/vendor/k8s.io/kubernetes/pkg/volume/metrics_du.go new file mode 100644 index 00000000..1cae99c1 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/metrics_du.go @@ -0,0 +1,99 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volume + +import ( + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/pkg/volume/util/fs" +) + +var _ MetricsProvider = &metricsDu{} + +// metricsDu represents a MetricsProvider that calculates the used and +// available Volume space by calling fs.DiskUsage() and gathering +// filesystem info for the Volume path. +type metricsDu struct { + // the directory path the volume is mounted to. + path string +} + +// NewMetricsDu creates a new metricsDu with the Volume path. +func NewMetricsDu(path string) MetricsProvider { + return &metricsDu{path} +} + +// GetMetrics calculates the volume usage and device free space by executing "du" +// and gathering filesystem info for the Volume path. +// See MetricsProvider.GetMetrics +func (md *metricsDu) GetMetrics() (*Metrics, error) { + metrics := &Metrics{Time: metav1.Now()} + if md.path == "" { + return metrics, NewNoPathDefinedError() + } + + err := md.runDiskUsage(metrics) + if err != nil { + return metrics, err + } + + err = md.runFind(metrics) + if err != nil { + return metrics, err + } + + err = md.getFsInfo(metrics) + if err != nil { + return metrics, err + } + + return metrics, nil +} + +// runDiskUsage gets disk usage of md.path and writes the results to metrics.Used +func (md *metricsDu) runDiskUsage(metrics *Metrics) error { + used, err := fs.DiskUsage(md.path) + if err != nil { + return err + } + metrics.Used = used + return nil +} + +// runFind executes the "find" command and writes the results to metrics.InodesUsed +func (md *metricsDu) runFind(metrics *Metrics) error { + inodesUsed, err := fs.Find(md.path) + if err != nil { + return err + } + metrics.InodesUsed = resource.NewQuantity(inodesUsed, resource.BinarySI) + return nil +} + +// getFsInfo writes metrics.Capacity and metrics.Available from the filesystem +// info +func (md *metricsDu) getFsInfo(metrics *Metrics) error { + available, capacity, _, inodes, inodesFree, _, err := fs.FsInfo(md.path) + if err != nil { + return NewFsInfoFailedError(err) + } + metrics.Available = resource.NewQuantity(available, resource.BinarySI) + metrics.Capacity = resource.NewQuantity(capacity, resource.BinarySI) + metrics.Inodes = resource.NewQuantity(inodes, resource.BinarySI) + metrics.InodesFree = resource.NewQuantity(inodesFree, resource.BinarySI) + return nil +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/metrics_errors.go b/vendor/k8s.io/kubernetes/pkg/volume/metrics_errors.go new file mode 100644 index 00000000..a6cbdbf7 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/metrics_errors.go @@ -0,0 +1,86 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volume + +import ( + "fmt" +) + +const ( + // ErrCodeNotSupported code for NotSupported Errors. + ErrCodeNotSupported int = iota + 1 + ErrCodeNoPathDefined + ErrCodeFsInfoFailed +) + +// NewNotSupportedError creates a new MetricsError with code NotSupported. +func NewNotSupportedError() *MetricsError { + return &MetricsError{ + Code: ErrCodeNotSupported, + Msg: "metrics are not supported for MetricsNil Volumes", + } +} + +// NewNotSupportedErrorWithDriverName creates a new MetricsError with code NotSupported. +// driver name is added to the error message. +func NewNotSupportedErrorWithDriverName(name string) *MetricsError { + return &MetricsError{ + Code: ErrCodeNotSupported, + Msg: fmt.Sprintf("metrics are not supported for %s volumes", name), + } +} + +// NewNoPathDefinedError creates a new MetricsError with code NoPathDefined. +func NewNoPathDefinedError() *MetricsError { + return &MetricsError{ + Code: ErrCodeNoPathDefined, + Msg: "no path defined for disk usage metrics.", + } +} + +// NewFsInfoFailedError creates a new MetricsError with code FsInfoFailed. +func NewFsInfoFailedError(err error) *MetricsError { + return &MetricsError{ + Code: ErrCodeFsInfoFailed, + Msg: fmt.Sprintf("failed to get FsInfo due to error %v", err), + } +} + +// MetricsError to distinguish different Metrics Errors. +type MetricsError struct { + Code int + Msg string +} + +func (e *MetricsError) Error() string { + return e.Msg +} + +// IsNotSupported returns true if and only if err is "key" not found error. +func IsNotSupported(err error) bool { + return isErrCode(err, ErrCodeNotSupported) +} + +func isErrCode(err error, code int) bool { + if err == nil { + return false + } + if e, ok := err.(*MetricsError); ok { + return e.Code == code + } + return false +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/metrics_nil.go b/vendor/k8s.io/kubernetes/pkg/volume/metrics_nil.go new file mode 100644 index 00000000..5438dc3d --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/metrics_nil.go @@ -0,0 +1,30 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volume + +var _ MetricsProvider = &MetricsNil{} + +// MetricsNil represents a MetricsProvider that does not support returning +// Metrics. It serves as a placeholder for Volumes that do not yet support +// metrics. +type MetricsNil struct{} + +// GetMetrics returns an empty Metrics and an error. +// See MetricsProvider.GetMetrics +func (*MetricsNil) GetMetrics() (*Metrics, error) { + return &Metrics{}, NewNotSupportedError() +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/metrics_statfs.go b/vendor/k8s.io/kubernetes/pkg/volume/metrics_statfs.go new file mode 100644 index 00000000..66f99e30 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/metrics_statfs.go @@ -0,0 +1,69 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volume + +import ( + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/pkg/volume/util/fs" +) + +var _ MetricsProvider = &metricsStatFS{} + +// metricsStatFS represents a MetricsProvider that calculates the used and available +// Volume space by stat'ing and gathering filesystem info for the Volume path. +type metricsStatFS struct { + // the directory path the volume is mounted to. + path string +} + +// NewMetricsStatfs creates a new metricsStatFS with the Volume path. +func NewMetricsStatFS(path string) MetricsProvider { + return &metricsStatFS{path} +} + +// See MetricsProvider.GetMetrics +// GetMetrics calculates the volume usage and device free space by executing "du" +// and gathering filesystem info for the Volume path. +func (md *metricsStatFS) GetMetrics() (*Metrics, error) { + metrics := &Metrics{Time: metav1.Now()} + if md.path == "" { + return metrics, NewNoPathDefinedError() + } + + err := md.getFsInfo(metrics) + if err != nil { + return metrics, err + } + + return metrics, nil +} + +// getFsInfo writes metrics.Capacity, metrics.Used and metrics.Available from the filesystem info +func (md *metricsStatFS) getFsInfo(metrics *Metrics) error { + available, capacity, usage, inodes, inodesFree, inodesUsed, err := fs.FsInfo(md.path) + if err != nil { + return NewFsInfoFailedError(err) + } + metrics.Available = resource.NewQuantity(available, resource.BinarySI) + metrics.Capacity = resource.NewQuantity(capacity, resource.BinarySI) + metrics.Used = resource.NewQuantity(usage, resource.BinarySI) + metrics.Inodes = resource.NewQuantity(inodes, resource.BinarySI) + metrics.InodesFree = resource.NewQuantity(inodesFree, resource.BinarySI) + metrics.InodesUsed = resource.NewQuantity(inodesUsed, resource.BinarySI) + return nil +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/noop_expandable_plugin.go b/vendor/k8s.io/kubernetes/pkg/volume/noop_expandable_plugin.go new file mode 100644 index 00000000..fac27426 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/noop_expandable_plugin.go @@ -0,0 +1,77 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volume + +import ( + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/types" +) + +type noopExpandableVolumePluginInstance struct { + spec *Spec +} + +var _ ExpandableVolumePlugin = &noopExpandableVolumePluginInstance{} + +func (n *noopExpandableVolumePluginInstance) ExpandVolumeDevice(spec *Spec, newSize resource.Quantity, oldSize resource.Quantity) (resource.Quantity, error) { + return newSize, nil +} + +func (n *noopExpandableVolumePluginInstance) Init(host VolumeHost) error { + return nil +} + +func (n *noopExpandableVolumePluginInstance) GetPluginName() string { + return n.spec.KubeletExpandablePluginName() +} + +func (n *noopExpandableVolumePluginInstance) GetVolumeName(spec *Spec) (string, error) { + return n.spec.Name(), nil +} + +func (n *noopExpandableVolumePluginInstance) CanSupport(spec *Spec) bool { + return true +} + +func (n *noopExpandableVolumePluginInstance) RequiresRemount(spec *Spec) bool { + return false +} + +func (n *noopExpandableVolumePluginInstance) NewMounter(spec *Spec, podRef *v1.Pod, opts VolumeOptions) (Mounter, error) { + return nil, nil +} + +func (n *noopExpandableVolumePluginInstance) NewUnmounter(name string, podUID types.UID) (Unmounter, error) { + return nil, nil +} + +func (n *noopExpandableVolumePluginInstance) ConstructVolumeSpec(volumeName, mountPath string) (*Spec, error) { + return n.spec, nil +} + +func (n *noopExpandableVolumePluginInstance) SupportsMountOption() bool { + return true +} + +func (n *noopExpandableVolumePluginInstance) SupportsBulkVolumeVerification() bool { + return false +} + +func (n *noopExpandableVolumePluginInstance) RequiresFSResize() bool { + return true +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/plugins.go b/vendor/k8s.io/kubernetes/pkg/volume/plugins.go new file mode 100644 index 00000000..17ba4811 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/plugins.go @@ -0,0 +1,1110 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volume + +import ( + "fmt" + "net" + "strings" + "sync" + + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" + "k8s.io/mount-utils" + "k8s.io/utils/exec" + + authenticationv1 "k8s.io/api/authentication/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/informers" + clientset "k8s.io/client-go/kubernetes" + storagelistersv1 "k8s.io/client-go/listers/storage/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" + cloudprovider "k8s.io/cloud-provider" + proxyutil "k8s.io/kubernetes/pkg/proxy/util" + "k8s.io/kubernetes/pkg/volume/util/hostutil" + "k8s.io/kubernetes/pkg/volume/util/recyclerclient" + "k8s.io/kubernetes/pkg/volume/util/subpath" +) + +type ProbeOperation uint32 +type ProbeEvent struct { + Plugin VolumePlugin // VolumePlugin that was added/updated/removed. if ProbeEvent.Op is 'ProbeRemove', Plugin should be nil + PluginName string + Op ProbeOperation // The operation to the plugin +} + +// CSIVolumePhaseType stores information about CSI volume path. +type CSIVolumePhaseType string + +const ( + // Common parameter which can be specified in StorageClass to specify the desired FSType + // Provisioners SHOULD implement support for this if they are block device based + // Must be a filesystem type supported by the host operating system. + // Ex. "ext4", "xfs", "ntfs". Default value depends on the provisioner + VolumeParameterFSType = "fstype" + + ProbeAddOrUpdate ProbeOperation = 1 << iota + ProbeRemove + CSIVolumeStaged CSIVolumePhaseType = "staged" + CSIVolumePublished CSIVolumePhaseType = "published" +) + +var ( + deprecatedVolumeProviders = map[string]string{ + "kubernetes.io/cinder": "The Cinder volume provider is deprecated and will be removed in a future release", + "kubernetes.io/scaleio": "The ScaleIO volume provider is deprecated and will be removed in a future release", + } +) + +// VolumeOptions contains option information about a volume. +type VolumeOptions struct { + // The attributes below are required by volume.Provisioner + // TODO: refactor all of this out of volumes when an admin can configure + // many kinds of provisioners. + + // Reclamation policy for a persistent volume + PersistentVolumeReclaimPolicy v1.PersistentVolumeReclaimPolicy + // Mount options for a persistent volume + MountOptions []string + // Suggested PV.Name of the PersistentVolume to provision. + // This is a generated name guaranteed to be unique in Kubernetes cluster. + // If you choose not to use it as volume name, ensure uniqueness by either + // combining it with your value or create unique values of your own. + PVName string + // PVC is reference to the claim that lead to provisioning of a new PV. + // Provisioners *must* create a PV that would be matched by this PVC, + // i.e. with required capacity, accessMode, labels matching PVC.Selector and + // so on. + PVC *v1.PersistentVolumeClaim + // Unique name of Kubernetes cluster. + ClusterName string + // Tags to attach to the real volume in the cloud provider - e.g. AWS EBS + CloudTags *map[string]string + // Volume provisioning parameters from StorageClass + Parameters map[string]string +} + +// NodeResizeOptions contain options to be passed for node expansion. +type NodeResizeOptions struct { + VolumeSpec *Spec + + // DevicePath - location of actual device on the node. In case of CSI + // this just could be volumeID + DevicePath string + + // DeviceMountPath location where device is mounted on the node. If volume type + // is attachable - this would be global mount path otherwise + // it would be location where volume was mounted for the pod + DeviceMountPath string + + // DeviceStagingPath stores location where the volume is staged + DeviceStagePath string + + NewSize resource.Quantity + OldSize resource.Quantity + + // CSIVolumePhase contains volume phase on the node + CSIVolumePhase CSIVolumePhaseType +} + +type DynamicPluginProber interface { + Init() error + + // If an error occurs, events are undefined. + Probe() (events []ProbeEvent, err error) +} + +// VolumePlugin is an interface to volume plugins that can be used on a +// kubernetes node (e.g. by kubelet) to instantiate and manage volumes. +type VolumePlugin interface { + // Init initializes the plugin. This will be called exactly once + // before any New* calls are made - implementations of plugins may + // depend on this. + Init(host VolumeHost) error + + // Name returns the plugin's name. Plugins must use namespaced names + // such as "example.com/volume" and contain exactly one '/' character. + // The "kubernetes.io" namespace is reserved for plugins which are + // bundled with kubernetes. + GetPluginName() string + + // GetVolumeName returns the name/ID to uniquely identifying the actual + // backing device, directory, path, etc. referenced by the specified volume + // spec. + // For Attachable volumes, this value must be able to be passed back to + // volume Detach methods to identify the device to act on. + // If the plugin does not support the given spec, this returns an error. + GetVolumeName(spec *Spec) (string, error) + + // CanSupport tests whether the plugin supports a given volume + // specification from the API. The spec pointer should be considered + // const. + CanSupport(spec *Spec) bool + + // RequiresRemount returns true if this plugin requires mount calls to be + // reexecuted. Atomically updating volumes, like Downward API, depend on + // this to update the contents of the volume. + RequiresRemount(spec *Spec) bool + + // NewMounter creates a new volume.Mounter from an API specification. + // Ownership of the spec pointer in *not* transferred. + // - spec: The v1.Volume spec + // - pod: The enclosing pod + NewMounter(spec *Spec, podRef *v1.Pod, opts VolumeOptions) (Mounter, error) + + // NewUnmounter creates a new volume.Unmounter from recoverable state. + // - name: The volume name, as per the v1.Volume spec. + // - podUID: The UID of the enclosing pod + NewUnmounter(name string, podUID types.UID) (Unmounter, error) + + // ConstructVolumeSpec constructs a volume spec based on the given volume name + // and volumePath. The spec may have incomplete information due to limited + // information from input. This function is used by volume manager to reconstruct + // volume spec by reading the volume directories from disk + ConstructVolumeSpec(volumeName, volumePath string) (*Spec, error) + + // SupportsMountOption returns true if volume plugins supports Mount options + // Specifying mount options in a volume plugin that doesn't support + // user specified mount options will result in error creating persistent volumes + SupportsMountOption() bool + + // SupportsBulkVolumeVerification checks if volume plugin type is capable + // of enabling bulk polling of all nodes. This can speed up verification of + // attached volumes by quite a bit, but underlying pluging must support it. + SupportsBulkVolumeVerification() bool +} + +// PersistentVolumePlugin is an extended interface of VolumePlugin and is used +// by volumes that want to provide long term persistence of data +type PersistentVolumePlugin interface { + VolumePlugin + // GetAccessModes describes the ways a given volume can be accessed/mounted. + GetAccessModes() []v1.PersistentVolumeAccessMode +} + +// RecyclableVolumePlugin is an extended interface of VolumePlugin and is used +// by persistent volumes that want to be recycled before being made available +// again to new claims +type RecyclableVolumePlugin interface { + VolumePlugin + + // Recycle knows how to reclaim this + // resource after the volume's release from a PersistentVolumeClaim. + // Recycle will use the provided recorder to write any events that might be + // interesting to user. It's expected that caller will pass these events to + // the PV being recycled. + Recycle(pvName string, spec *Spec, eventRecorder recyclerclient.RecycleEventRecorder) error +} + +// DeletableVolumePlugin is an extended interface of VolumePlugin and is used +// by persistent volumes that want to be deleted from the cluster after their +// release from a PersistentVolumeClaim. +type DeletableVolumePlugin interface { + VolumePlugin + // NewDeleter creates a new volume.Deleter which knows how to delete this + // resource in accordance with the underlying storage provider after the + // volume's release from a claim + NewDeleter(spec *Spec) (Deleter, error) +} + +// ProvisionableVolumePlugin is an extended interface of VolumePlugin and is +// used to create volumes for the cluster. +type ProvisionableVolumePlugin interface { + VolumePlugin + // NewProvisioner creates a new volume.Provisioner which knows how to + // create PersistentVolumes in accordance with the plugin's underlying + // storage provider + NewProvisioner(options VolumeOptions) (Provisioner, error) +} + +// AttachableVolumePlugin is an extended interface of VolumePlugin and is used for volumes that require attachment +// to a node before mounting. +type AttachableVolumePlugin interface { + DeviceMountableVolumePlugin + NewAttacher() (Attacher, error) + NewDetacher() (Detacher, error) + // CanAttach tests if provided volume spec is attachable + CanAttach(spec *Spec) (bool, error) +} + +// DeviceMountableVolumePlugin is an extended interface of VolumePlugin and is used +// for volumes that requires mount device to a node before binding to volume to pod. +type DeviceMountableVolumePlugin interface { + VolumePlugin + NewDeviceMounter() (DeviceMounter, error) + NewDeviceUnmounter() (DeviceUnmounter, error) + GetDeviceMountRefs(deviceMountPath string) ([]string, error) + // CanDeviceMount determines if device in volume.Spec is mountable + CanDeviceMount(spec *Spec) (bool, error) +} + +// ExpandableVolumePlugin is an extended interface of VolumePlugin and is used for volumes that can be +// expanded via control-plane ExpandVolumeDevice call. +type ExpandableVolumePlugin interface { + VolumePlugin + ExpandVolumeDevice(spec *Spec, newSize resource.Quantity, oldSize resource.Quantity) (resource.Quantity, error) + RequiresFSResize() bool +} + +// NodeExpandableVolumePlugin is an expanded interface of VolumePlugin and is used for volumes that +// require expansion on the node via NodeExpand call. +type NodeExpandableVolumePlugin interface { + VolumePlugin + RequiresFSResize() bool + // NodeExpand expands volume on given deviceMountPath and returns true if resize is successful. + NodeExpand(resizeOptions NodeResizeOptions) (bool, error) +} + +// VolumePluginWithAttachLimits is an extended interface of VolumePlugin that restricts number of +// volumes that can be attached to a node. +type VolumePluginWithAttachLimits interface { + VolumePlugin + // Return maximum number of volumes that can be attached to a node for this plugin. + // The key must be same as string returned by VolumeLimitKey function. The returned + // map may look like: + // - { "storage-limits-aws-ebs": 39 } + // - { "storage-limits-gce-pd": 10 } + // A volume plugin may return error from this function - if it can not be used on a given node or not + // applicable in given environment (where environment could be cloudprovider or any other dependency) + // For example - calling this function for EBS volume plugin on a GCE node should + // result in error. + // The returned values are stored in node allocatable property and will be used + // by scheduler to determine how many pods with volumes can be scheduled on given node. + GetVolumeLimits() (map[string]int64, error) + // Return volume limit key string to be used in node capacity constraints + // The key must start with prefix storage-limits-. For example: + // - storage-limits-aws-ebs + // - storage-limits-csi-cinder + // The key should respect character limit of ResourceName type + // This function may be called by kubelet or scheduler to identify node allocatable property + // which stores volumes limits. + VolumeLimitKey(spec *Spec) string +} + +// BlockVolumePlugin is an extend interface of VolumePlugin and is used for block volumes support. +type BlockVolumePlugin interface { + VolumePlugin + // NewBlockVolumeMapper creates a new volume.BlockVolumeMapper from an API specification. + // Ownership of the spec pointer in *not* transferred. + // - spec: The v1.Volume spec + // - pod: The enclosing pod + NewBlockVolumeMapper(spec *Spec, podRef *v1.Pod, opts VolumeOptions) (BlockVolumeMapper, error) + // NewBlockVolumeUnmapper creates a new volume.BlockVolumeUnmapper from recoverable state. + // - name: The volume name, as per the v1.Volume spec. + // - podUID: The UID of the enclosing pod + NewBlockVolumeUnmapper(name string, podUID types.UID) (BlockVolumeUnmapper, error) + // ConstructBlockVolumeSpec constructs a volume spec based on the given + // podUID, volume name and a pod device map path. + // The spec may have incomplete information due to limited information + // from input. This function is used by volume manager to reconstruct + // volume spec by reading the volume directories from disk. + ConstructBlockVolumeSpec(podUID types.UID, volumeName, volumePath string) (*Spec, error) +} + +// TODO(#14217) +// As part of the Volume Host refactor we are starting to create Volume Hosts +// for specific hosts. New methods for each specific host can be added here. +// Currently consumers will do type assertions to get the specific type of Volume +// Host; however, the end result should be that specific Volume Hosts are passed +// to the specific functions they are needed in (instead of using a catch-all +// VolumeHost interface) + +// KubeletVolumeHost is a Kubelet specific interface that plugins can use to access the kubelet. +type KubeletVolumeHost interface { + // SetKubeletError lets plugins set an error on the Kubelet runtime status + // that will cause the Kubelet to post NotReady status with the error message provided + SetKubeletError(err error) + + // GetInformerFactory returns the informer factory for CSIDriverLister + GetInformerFactory() informers.SharedInformerFactory + // CSIDriverLister returns the informer lister for the CSIDriver API Object + CSIDriverLister() storagelistersv1.CSIDriverLister + // CSIDriverSynced returns the informer synced for the CSIDriver API Object + CSIDriversSynced() cache.InformerSynced + // WaitForCacheSync is a helper function that waits for cache sync for CSIDriverLister + WaitForCacheSync() error + // Returns hostutil.HostUtils + GetHostUtil() hostutil.HostUtils +} + +// AttachDetachVolumeHost is a AttachDetach Controller specific interface that plugins can use +// to access methods on the Attach Detach Controller. +type AttachDetachVolumeHost interface { + // CSINodeLister returns the informer lister for the CSINode API Object + CSINodeLister() storagelistersv1.CSINodeLister + + // CSIDriverLister returns the informer lister for the CSIDriver API Object + CSIDriverLister() storagelistersv1.CSIDriverLister + + // VolumeAttachmentLister returns the informer lister for the VolumeAttachment API Object + VolumeAttachmentLister() storagelistersv1.VolumeAttachmentLister + // IsAttachDetachController is an interface marker to strictly tie AttachDetachVolumeHost + // to the attachDetachController + IsAttachDetachController() bool +} + +// VolumeHost is an interface that plugins can use to access the kubelet. +type VolumeHost interface { + // GetPluginDir returns the absolute path to a directory under which + // a given plugin may store data. This directory might not actually + // exist on disk yet. For plugin data that is per-pod, see + // GetPodPluginDir(). + GetPluginDir(pluginName string) string + + // GetVolumeDevicePluginDir returns the absolute path to a directory + // under which a given plugin may store data. + // ex. plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}/ + GetVolumeDevicePluginDir(pluginName string) string + + // GetPodsDir returns the absolute path to a directory where all the pods + // information is stored + GetPodsDir() string + + // GetPodVolumeDir returns the absolute path a directory which + // represents the named volume under the named plugin for the given + // pod. If the specified pod does not exist, the result of this call + // might not exist. + GetPodVolumeDir(podUID types.UID, pluginName string, volumeName string) string + + // GetPodPluginDir returns the absolute path to a directory under which + // a given plugin may store data for a given pod. If the specified pod + // does not exist, the result of this call might not exist. This + // directory might not actually exist on disk yet. + GetPodPluginDir(podUID types.UID, pluginName string) string + + // GetPodVolumeDeviceDir returns the absolute path a directory which + // represents the named plugin for the given pod. + // If the specified pod does not exist, the result of this call + // might not exist. + // ex. pods/{podUid}/{DefaultKubeletVolumeDevicesDirName}/{escapeQualifiedPluginName}/ + GetPodVolumeDeviceDir(podUID types.UID, pluginName string) string + + // GetKubeClient returns a client interface + GetKubeClient() clientset.Interface + + // NewWrapperMounter finds an appropriate plugin with which to handle + // the provided spec. This is used to implement volume plugins which + // "wrap" other plugins. For example, the "secret" volume is + // implemented in terms of the "emptyDir" volume. + NewWrapperMounter(volName string, spec Spec, pod *v1.Pod, opts VolumeOptions) (Mounter, error) + + // NewWrapperUnmounter finds an appropriate plugin with which to handle + // the provided spec. See comments on NewWrapperMounter for more + // context. + NewWrapperUnmounter(volName string, spec Spec, podUID types.UID) (Unmounter, error) + + // Get cloud provider from kubelet. + GetCloudProvider() cloudprovider.Interface + + // Get mounter interface. + GetMounter(pluginName string) mount.Interface + + // Returns the hostname of the host kubelet is running on + GetHostName() string + + // Returns host IP or nil in the case of error. + GetHostIP() (net.IP, error) + + // Returns node allocatable. + GetNodeAllocatable() (v1.ResourceList, error) + + // Returns a function that returns a secret. + GetSecretFunc() func(namespace, name string) (*v1.Secret, error) + + // Returns a function that returns a configmap. + GetConfigMapFunc() func(namespace, name string) (*v1.ConfigMap, error) + + GetServiceAccountTokenFunc() func(namespace, name string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error) + + DeleteServiceAccountTokenFunc() func(podUID types.UID) + + // Returns an interface that should be used to execute any utilities in volume plugins + GetExec(pluginName string) exec.Interface + + // Returns the labels on the node + GetNodeLabels() (map[string]string, error) + + // Returns the name of the node + GetNodeName() types.NodeName + + // Returns the event recorder of kubelet. + GetEventRecorder() record.EventRecorder + + // Returns an interface that should be used to execute subpath operations + GetSubpather() subpath.Interface + + // Returns options to pass for proxyutil filtered dialers. + GetFilteredDialOptions() *proxyutil.FilteredDialOptions +} + +// VolumePluginMgr tracks registered plugins. +type VolumePluginMgr struct { + mutex sync.Mutex + plugins map[string]VolumePlugin + prober DynamicPluginProber + probedPlugins map[string]VolumePlugin + loggedDeprecationWarnings sets.String + Host VolumeHost +} + +// Spec is an internal representation of a volume. All API volume types translate to Spec. +type Spec struct { + Volume *v1.Volume + PersistentVolume *v1.PersistentVolume + ReadOnly bool + InlineVolumeSpecForCSIMigration bool +} + +// Name returns the name of either Volume or PersistentVolume, one of which must not be nil. +func (spec *Spec) Name() string { + switch { + case spec.Volume != nil: + return spec.Volume.Name + case spec.PersistentVolume != nil: + return spec.PersistentVolume.Name + default: + return "" + } +} + +// IsKubeletExpandable returns true for volume types that can be expanded only by the node +// and not the controller. Currently Flex volume is the only one in this category since +// it is typically not installed on the controller +func (spec *Spec) IsKubeletExpandable() bool { + switch { + case spec.Volume != nil: + return spec.Volume.FlexVolume != nil + case spec.PersistentVolume != nil: + return spec.PersistentVolume.Spec.FlexVolume != nil + default: + return false + + } +} + +// KubeletExpandablePluginName creates and returns a name for the plugin +// this is used in context on the controller where the plugin lookup fails +// as volume expansion on controller isn't supported, but a plugin name is +// required +func (spec *Spec) KubeletExpandablePluginName() string { + switch { + case spec.Volume != nil && spec.Volume.FlexVolume != nil: + return spec.Volume.FlexVolume.Driver + case spec.PersistentVolume != nil && spec.PersistentVolume.Spec.FlexVolume != nil: + return spec.PersistentVolume.Spec.FlexVolume.Driver + default: + return "" + } +} + +// VolumeConfig is how volume plugins receive configuration. An instance +// specific to the plugin will be passed to the plugin's +// ProbeVolumePlugins(config) func. Reasonable defaults will be provided by +// the binary hosting the plugins while allowing override of those default +// values. Those config values are then set to an instance of VolumeConfig +// and passed to the plugin. +// +// Values in VolumeConfig are intended to be relevant to several plugins, but +// not necessarily all plugins. The preference is to leverage strong typing +// in this struct. All config items must have a descriptive but non-specific +// name (i.e, RecyclerMinimumTimeout is OK but RecyclerMinimumTimeoutForNFS is +// !OK). An instance of config will be given directly to the plugin, so +// config names specific to plugins are unneeded and wrongly expose plugins in +// this VolumeConfig struct. +// +// OtherAttributes is a map of string values intended for one-off +// configuration of a plugin or config that is only relevant to a single +// plugin. All values are passed by string and require interpretation by the +// plugin. Passing config as strings is the least desirable option but can be +// used for truly one-off configuration. The binary should still use strong +// typing for this value when binding CLI values before they are passed as +// strings in OtherAttributes. +type VolumeConfig struct { + // RecyclerPodTemplate is pod template that understands how to scrub clean + // a persistent volume after its release. The template is used by plugins + // which override specific properties of the pod in accordance with that + // plugin. See NewPersistentVolumeRecyclerPodTemplate for the properties + // that are expected to be overridden. + RecyclerPodTemplate *v1.Pod + + // RecyclerMinimumTimeout is the minimum amount of time in seconds for the + // recycler pod's ActiveDeadlineSeconds attribute. Added to the minimum + // timeout is the increment per Gi of capacity. + RecyclerMinimumTimeout int + + // RecyclerTimeoutIncrement is the number of seconds added to the recycler + // pod's ActiveDeadlineSeconds for each Gi of capacity in the persistent + // volume. Example: 5Gi volume x 30s increment = 150s + 30s minimum = 180s + // ActiveDeadlineSeconds for recycler pod + RecyclerTimeoutIncrement int + + // PVName is name of the PersistentVolume instance that is being recycled. + // It is used to generate unique recycler pod name. + PVName string + + // OtherAttributes stores config as strings. These strings are opaque to + // the system and only understood by the binary hosting the plugin and the + // plugin itself. + OtherAttributes map[string]string + + // ProvisioningEnabled configures whether provisioning of this plugin is + // enabled or not. Currently used only in host_path plugin. + ProvisioningEnabled bool +} + +// NewSpecFromVolume creates an Spec from an v1.Volume +func NewSpecFromVolume(vs *v1.Volume) *Spec { + return &Spec{ + Volume: vs, + } +} + +// NewSpecFromPersistentVolume creates an Spec from an v1.PersistentVolume +func NewSpecFromPersistentVolume(pv *v1.PersistentVolume, readOnly bool) *Spec { + return &Spec{ + PersistentVolume: pv, + ReadOnly: readOnly, + } +} + +// InitPlugins initializes each plugin. All plugins must have unique names. +// This must be called exactly once before any New* methods are called on any +// plugins. +func (pm *VolumePluginMgr) InitPlugins(plugins []VolumePlugin, prober DynamicPluginProber, host VolumeHost) error { + pm.mutex.Lock() + defer pm.mutex.Unlock() + + pm.Host = host + pm.loggedDeprecationWarnings = sets.NewString() + + if prober == nil { + // Use a dummy prober to prevent nil deference. + pm.prober = &dummyPluginProber{} + } else { + pm.prober = prober + } + if err := pm.prober.Init(); err != nil { + // Prober init failure should not affect the initialization of other plugins. + klog.Errorf("Error initializing dynamic plugin prober: %s", err) + pm.prober = &dummyPluginProber{} + } + + if pm.plugins == nil { + pm.plugins = map[string]VolumePlugin{} + } + if pm.probedPlugins == nil { + pm.probedPlugins = map[string]VolumePlugin{} + } + + allErrs := []error{} + for _, plugin := range plugins { + name := plugin.GetPluginName() + if errs := validation.IsQualifiedName(name); len(errs) != 0 { + allErrs = append(allErrs, fmt.Errorf("volume plugin has invalid name: %q: %s", name, strings.Join(errs, ";"))) + continue + } + + if _, found := pm.plugins[name]; found { + allErrs = append(allErrs, fmt.Errorf("volume plugin %q was registered more than once", name)) + continue + } + err := plugin.Init(host) + if err != nil { + klog.Errorf("Failed to load volume plugin %s, error: %s", name, err.Error()) + allErrs = append(allErrs, err) + continue + } + pm.plugins[name] = plugin + klog.V(1).Infof("Loaded volume plugin %q", name) + } + return utilerrors.NewAggregate(allErrs) +} + +func (pm *VolumePluginMgr) initProbedPlugin(probedPlugin VolumePlugin) error { + name := probedPlugin.GetPluginName() + if errs := validation.IsQualifiedName(name); len(errs) != 0 { + return fmt.Errorf("volume plugin has invalid name: %q: %s", name, strings.Join(errs, ";")) + } + + err := probedPlugin.Init(pm.Host) + if err != nil { + return fmt.Errorf("Failed to load volume plugin %s, error: %s", name, err.Error()) + } + + klog.V(1).Infof("Loaded volume plugin %q", name) + return nil +} + +// FindPluginBySpec looks for a plugin that can support a given volume +// specification. If no plugins can support or more than one plugin can +// support it, return error. +func (pm *VolumePluginMgr) FindPluginBySpec(spec *Spec) (VolumePlugin, error) { + pm.mutex.Lock() + defer pm.mutex.Unlock() + + if spec == nil { + return nil, fmt.Errorf("Could not find plugin because volume spec is nil") + } + + matches := []VolumePlugin{} + for _, v := range pm.plugins { + if v.CanSupport(spec) { + matches = append(matches, v) + } + } + + pm.refreshProbedPlugins() + for _, plugin := range pm.probedPlugins { + if plugin.CanSupport(spec) { + matches = append(matches, plugin) + } + } + + if len(matches) == 0 { + return nil, fmt.Errorf("no volume plugin matched") + } + if len(matches) > 1 { + matchedPluginNames := []string{} + for _, plugin := range matches { + matchedPluginNames = append(matchedPluginNames, plugin.GetPluginName()) + } + return nil, fmt.Errorf("multiple volume plugins matched: %s", strings.Join(matchedPluginNames, ",")) + } + + // Issue warning if the matched provider is deprecated + pm.logDeprecation(matches[0].GetPluginName()) + return matches[0], nil +} + +// FindPluginByName fetches a plugin by name or by legacy name. If no plugin +// is found, returns error. +func (pm *VolumePluginMgr) FindPluginByName(name string) (VolumePlugin, error) { + pm.mutex.Lock() + defer pm.mutex.Unlock() + + // Once we can get rid of legacy names we can reduce this to a map lookup. + matches := []VolumePlugin{} + if v, found := pm.plugins[name]; found { + matches = append(matches, v) + } + + pm.refreshProbedPlugins() + if plugin, found := pm.probedPlugins[name]; found { + matches = append(matches, plugin) + } + + if len(matches) == 0 { + return nil, fmt.Errorf("no volume plugin matched name: %s", name) + } + if len(matches) > 1 { + matchedPluginNames := []string{} + for _, plugin := range matches { + matchedPluginNames = append(matchedPluginNames, plugin.GetPluginName()) + } + return nil, fmt.Errorf("multiple volume plugins matched: %s", strings.Join(matchedPluginNames, ",")) + } + + // Issue warning if the matched provider is deprecated + pm.logDeprecation(matches[0].GetPluginName()) + return matches[0], nil +} + +// logDeprecation logs warning when a deprecated plugin is used. +func (pm *VolumePluginMgr) logDeprecation(plugin string) { + if detail, ok := deprecatedVolumeProviders[plugin]; ok && !pm.loggedDeprecationWarnings.Has(plugin) { + klog.Warningf("WARNING: %s built-in volume provider is now deprecated. %s", plugin, detail) + // Make sure the message is logged only once. It has Warning severity + // and we don't want to spam the log too much. + pm.loggedDeprecationWarnings.Insert(plugin) + } +} + +// Check if probedPlugin cache update is required. +// If it is, initialize all probed plugins and replace the cache with them. +func (pm *VolumePluginMgr) refreshProbedPlugins() { + events, err := pm.prober.Probe() + if err != nil { + klog.Errorf("Error dynamically probing plugins: %s", err) + return // Use cached plugins upon failure. + } + + for _, event := range events { + if event.Op == ProbeAddOrUpdate { + if err := pm.initProbedPlugin(event.Plugin); err != nil { + klog.Errorf("Error initializing dynamically probed plugin %s; error: %s", + event.Plugin.GetPluginName(), err) + continue + } + pm.probedPlugins[event.Plugin.GetPluginName()] = event.Plugin + } else if event.Op == ProbeRemove { + // Plugin is not available on ProbeRemove event, only PluginName + delete(pm.probedPlugins, event.PluginName) + } else { + klog.Errorf("Unknown Operation on PluginName: %s.", + event.Plugin.GetPluginName()) + } + } +} + +// ListVolumePluginWithLimits returns plugins that have volume limits on nodes +func (pm *VolumePluginMgr) ListVolumePluginWithLimits() []VolumePluginWithAttachLimits { + matchedPlugins := []VolumePluginWithAttachLimits{} + for _, v := range pm.plugins { + if plugin, ok := v.(VolumePluginWithAttachLimits); ok { + matchedPlugins = append(matchedPlugins, plugin) + } + } + return matchedPlugins +} + +// FindPersistentPluginBySpec looks for a persistent volume plugin that can +// support a given volume specification. If no plugin is found, return an +// error +func (pm *VolumePluginMgr) FindPersistentPluginBySpec(spec *Spec) (PersistentVolumePlugin, error) { + volumePlugin, err := pm.FindPluginBySpec(spec) + if err != nil { + return nil, fmt.Errorf("Could not find volume plugin for spec: %#v", spec) + } + if persistentVolumePlugin, ok := volumePlugin.(PersistentVolumePlugin); ok { + return persistentVolumePlugin, nil + } + return nil, fmt.Errorf("no persistent volume plugin matched") +} + +// FindVolumePluginWithLimitsBySpec returns volume plugin that has a limit on how many +// of them can be attached to a node +func (pm *VolumePluginMgr) FindVolumePluginWithLimitsBySpec(spec *Spec) (VolumePluginWithAttachLimits, error) { + volumePlugin, err := pm.FindPluginBySpec(spec) + if err != nil { + return nil, fmt.Errorf("Could not find volume plugin for spec : %#v", spec) + } + + if limitedPlugin, ok := volumePlugin.(VolumePluginWithAttachLimits); ok { + return limitedPlugin, nil + } + return nil, fmt.Errorf("no plugin with limits found") +} + +// FindPersistentPluginByName fetches a persistent volume plugin by name. If +// no plugin is found, returns error. +func (pm *VolumePluginMgr) FindPersistentPluginByName(name string) (PersistentVolumePlugin, error) { + volumePlugin, err := pm.FindPluginByName(name) + if err != nil { + return nil, err + } + if persistentVolumePlugin, ok := volumePlugin.(PersistentVolumePlugin); ok { + return persistentVolumePlugin, nil + } + return nil, fmt.Errorf("no persistent volume plugin matched") +} + +// FindRecyclablePluginByName fetches a persistent volume plugin by name. If +// no plugin is found, returns error. +func (pm *VolumePluginMgr) FindRecyclablePluginBySpec(spec *Spec) (RecyclableVolumePlugin, error) { + volumePlugin, err := pm.FindPluginBySpec(spec) + if err != nil { + return nil, err + } + if recyclableVolumePlugin, ok := volumePlugin.(RecyclableVolumePlugin); ok { + return recyclableVolumePlugin, nil + } + return nil, fmt.Errorf("no recyclable volume plugin matched") +} + +// FindProvisionablePluginByName fetches a persistent volume plugin by name. If +// no plugin is found, returns error. +func (pm *VolumePluginMgr) FindProvisionablePluginByName(name string) (ProvisionableVolumePlugin, error) { + volumePlugin, err := pm.FindPluginByName(name) + if err != nil { + return nil, err + } + if provisionableVolumePlugin, ok := volumePlugin.(ProvisionableVolumePlugin); ok { + return provisionableVolumePlugin, nil + } + return nil, fmt.Errorf("no provisionable volume plugin matched") +} + +// FindDeletablePluginBySpec fetches a persistent volume plugin by spec. If +// no plugin is found, returns error. +func (pm *VolumePluginMgr) FindDeletablePluginBySpec(spec *Spec) (DeletableVolumePlugin, error) { + volumePlugin, err := pm.FindPluginBySpec(spec) + if err != nil { + return nil, err + } + if deletableVolumePlugin, ok := volumePlugin.(DeletableVolumePlugin); ok { + return deletableVolumePlugin, nil + } + return nil, fmt.Errorf("no deletable volume plugin matched") +} + +// FindDeletablePluginByName fetches a persistent volume plugin by name. If +// no plugin is found, returns error. +func (pm *VolumePluginMgr) FindDeletablePluginByName(name string) (DeletableVolumePlugin, error) { + volumePlugin, err := pm.FindPluginByName(name) + if err != nil { + return nil, err + } + if deletableVolumePlugin, ok := volumePlugin.(DeletableVolumePlugin); ok { + return deletableVolumePlugin, nil + } + return nil, fmt.Errorf("no deletable volume plugin matched") +} + +// FindCreatablePluginBySpec fetches a persistent volume plugin by name. If +// no plugin is found, returns error. +func (pm *VolumePluginMgr) FindCreatablePluginBySpec(spec *Spec) (ProvisionableVolumePlugin, error) { + volumePlugin, err := pm.FindPluginBySpec(spec) + if err != nil { + return nil, err + } + if provisionableVolumePlugin, ok := volumePlugin.(ProvisionableVolumePlugin); ok { + return provisionableVolumePlugin, nil + } + return nil, fmt.Errorf("no creatable volume plugin matched") +} + +// FindAttachablePluginBySpec fetches a persistent volume plugin by spec. +// Unlike the other "FindPlugin" methods, this does not return error if no +// plugin is found. All volumes require a mounter and unmounter, but not +// every volume will have an attacher/detacher. +func (pm *VolumePluginMgr) FindAttachablePluginBySpec(spec *Spec) (AttachableVolumePlugin, error) { + volumePlugin, err := pm.FindPluginBySpec(spec) + if err != nil { + return nil, err + } + if attachableVolumePlugin, ok := volumePlugin.(AttachableVolumePlugin); ok { + if canAttach, err := attachableVolumePlugin.CanAttach(spec); err != nil { + return nil, err + } else if canAttach { + return attachableVolumePlugin, nil + } + } + return nil, nil +} + +// FindAttachablePluginByName fetches an attachable volume plugin by name. +// Unlike the other "FindPlugin" methods, this does not return error if no +// plugin is found. All volumes require a mounter and unmounter, but not +// every volume will have an attacher/detacher. +func (pm *VolumePluginMgr) FindAttachablePluginByName(name string) (AttachableVolumePlugin, error) { + volumePlugin, err := pm.FindPluginByName(name) + if err != nil { + return nil, err + } + if attachablePlugin, ok := volumePlugin.(AttachableVolumePlugin); ok { + return attachablePlugin, nil + } + return nil, nil +} + +// FindDeviceMountablePluginBySpec fetches a persistent volume plugin by spec. +func (pm *VolumePluginMgr) FindDeviceMountablePluginBySpec(spec *Spec) (DeviceMountableVolumePlugin, error) { + volumePlugin, err := pm.FindPluginBySpec(spec) + if err != nil { + return nil, err + } + if deviceMountableVolumePlugin, ok := volumePlugin.(DeviceMountableVolumePlugin); ok { + if canMount, err := deviceMountableVolumePlugin.CanDeviceMount(spec); err != nil { + return nil, err + } else if canMount { + return deviceMountableVolumePlugin, nil + } + } + return nil, nil +} + +// FindDeviceMountablePluginByName fetches a devicemountable volume plugin by name. +func (pm *VolumePluginMgr) FindDeviceMountablePluginByName(name string) (DeviceMountableVolumePlugin, error) { + volumePlugin, err := pm.FindPluginByName(name) + if err != nil { + return nil, err + } + if deviceMountableVolumePlugin, ok := volumePlugin.(DeviceMountableVolumePlugin); ok { + return deviceMountableVolumePlugin, nil + } + return nil, nil +} + +// FindExpandablePluginBySpec fetches a persistent volume plugin by spec. +func (pm *VolumePluginMgr) FindExpandablePluginBySpec(spec *Spec) (ExpandableVolumePlugin, error) { + volumePlugin, err := pm.FindPluginBySpec(spec) + if err != nil { + if spec.IsKubeletExpandable() { + // for kubelet expandable volumes, return a noop plugin that + // returns success for expand on the controller + klog.V(4).Infof("FindExpandablePluginBySpec(%s) -> returning noopExpandableVolumePluginInstance", spec.Name()) + return &noopExpandableVolumePluginInstance{spec}, nil + } + klog.V(4).Infof("FindExpandablePluginBySpec(%s) -> err:%v", spec.Name(), err) + return nil, err + } + + if expandableVolumePlugin, ok := volumePlugin.(ExpandableVolumePlugin); ok { + return expandableVolumePlugin, nil + } + return nil, nil +} + +// FindExpandablePluginBySpec fetches a persistent volume plugin by name. +func (pm *VolumePluginMgr) FindExpandablePluginByName(name string) (ExpandableVolumePlugin, error) { + volumePlugin, err := pm.FindPluginByName(name) + if err != nil { + return nil, err + } + + if expandableVolumePlugin, ok := volumePlugin.(ExpandableVolumePlugin); ok { + return expandableVolumePlugin, nil + } + return nil, nil +} + +// FindMapperPluginBySpec fetches a block volume plugin by spec. +func (pm *VolumePluginMgr) FindMapperPluginBySpec(spec *Spec) (BlockVolumePlugin, error) { + volumePlugin, err := pm.FindPluginBySpec(spec) + if err != nil { + return nil, err + } + + if blockVolumePlugin, ok := volumePlugin.(BlockVolumePlugin); ok { + return blockVolumePlugin, nil + } + return nil, nil +} + +// FindMapperPluginByName fetches a block volume plugin by name. +func (pm *VolumePluginMgr) FindMapperPluginByName(name string) (BlockVolumePlugin, error) { + volumePlugin, err := pm.FindPluginByName(name) + if err != nil { + return nil, err + } + + if blockVolumePlugin, ok := volumePlugin.(BlockVolumePlugin); ok { + return blockVolumePlugin, nil + } + return nil, nil +} + +// FindNodeExpandablePluginBySpec fetches a persistent volume plugin by spec +func (pm *VolumePluginMgr) FindNodeExpandablePluginBySpec(spec *Spec) (NodeExpandableVolumePlugin, error) { + volumePlugin, err := pm.FindPluginBySpec(spec) + if err != nil { + return nil, err + } + if fsResizablePlugin, ok := volumePlugin.(NodeExpandableVolumePlugin); ok { + return fsResizablePlugin, nil + } + return nil, nil +} + +// FindNodeExpandablePluginByName fetches a persistent volume plugin by name +func (pm *VolumePluginMgr) FindNodeExpandablePluginByName(name string) (NodeExpandableVolumePlugin, error) { + volumePlugin, err := pm.FindPluginByName(name) + if err != nil { + return nil, err + } + + if fsResizablePlugin, ok := volumePlugin.(NodeExpandableVolumePlugin); ok { + return fsResizablePlugin, nil + } + + return nil, nil +} + +func (pm *VolumePluginMgr) Run(stopCh <-chan struct{}) { + kletHost, ok := pm.Host.(KubeletVolumeHost) + if ok { + // start informer for CSIDriver + informerFactory := kletHost.GetInformerFactory() + informerFactory.Start(stopCh) + informerFactory.WaitForCacheSync(stopCh) + } +} + +// NewPersistentVolumeRecyclerPodTemplate creates a template for a recycler +// pod. By default, a recycler pod simply runs "rm -rf" on a volume and tests +// for emptiness. Most attributes of the template will be correct for most +// plugin implementations. The following attributes can be overridden per +// plugin via configuration: +// +// 1. pod.Spec.Volumes[0].VolumeSource must be overridden. Recycler +// implementations without a valid VolumeSource will fail. +// 2. pod.GenerateName helps distinguish recycler pods by name. Recommended. +// Default is "pv-recycler-". +// 3. pod.Spec.ActiveDeadlineSeconds gives the recycler pod a maximum timeout +// before failing. Recommended. Default is 60 seconds. +// +// See HostPath and NFS for working recycler examples +func NewPersistentVolumeRecyclerPodTemplate() *v1.Pod { + timeout := int64(60) + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "pv-recycler-", + Namespace: metav1.NamespaceDefault, + }, + Spec: v1.PodSpec{ + ActiveDeadlineSeconds: &timeout, + RestartPolicy: v1.RestartPolicyNever, + Volumes: []v1.Volume{ + { + Name: "vol", + // IMPORTANT! All plugins using this template MUST + // override pod.Spec.Volumes[0].VolumeSource Recycler + // implementations without a valid VolumeSource will fail. + VolumeSource: v1.VolumeSource{}, + }, + }, + Containers: []v1.Container{ + { + Name: "pv-recycler", + Image: "busybox:1.27", + Command: []string{"/bin/sh"}, + Args: []string{"-c", "test -e /scrub && rm -rf /scrub/..?* /scrub/.[!.]* /scrub/* && test -z \"$(ls -A /scrub)\" || exit 1"}, + VolumeMounts: []v1.VolumeMount{ + { + Name: "vol", + MountPath: "/scrub", + }, + }, + }, + }, + }, + } + return pod +} + +// Check validity of recycle pod template +// List of checks: +// - at least one volume is defined in the recycle pod template +// If successful, returns nil +// if unsuccessful, returns an error. +func ValidateRecyclerPodTemplate(pod *v1.Pod) error { + if len(pod.Spec.Volumes) < 1 { + return fmt.Errorf("does not contain any volume(s)") + } + return nil +} + +type dummyPluginProber struct{} + +func (*dummyPluginProber) Init() error { return nil } +func (*dummyPluginProber) Probe() ([]ProbeEvent, error) { return nil, nil } diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/fs/BUILD b/vendor/k8s.io/kubernetes/pkg/volume/util/fs/BUILD new file mode 100644 index 00000000..b51ee597 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/fs/BUILD @@ -0,0 +1,95 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "fs.go", + "fs_unsupported.go", + "fs_windows.go", + ], + importpath = "k8s.io/kubernetes/pkg/volume/util/fs", + visibility = ["//visibility:public"], + deps = select({ + "@io_bazel_rules_go//go/platform:aix": [ + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + ], + "@io_bazel_rules_go//go/platform:android": [ + "//pkg/volume/util/fsquota:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//vendor/golang.org/x/sys/unix:go_default_library", + ], + "@io_bazel_rules_go//go/platform:darwin": [ + "//pkg/volume/util/fsquota:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//vendor/golang.org/x/sys/unix:go_default_library", + ], + "@io_bazel_rules_go//go/platform:dragonfly": [ + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + ], + "@io_bazel_rules_go//go/platform:freebsd": [ + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + ], + "@io_bazel_rules_go//go/platform:illumos": [ + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + ], + "@io_bazel_rules_go//go/platform:ios": [ + "//pkg/volume/util/fsquota:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//vendor/golang.org/x/sys/unix:go_default_library", + ], + "@io_bazel_rules_go//go/platform:js": [ + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + ], + "@io_bazel_rules_go//go/platform:linux": [ + "//pkg/volume/util/fsquota:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//vendor/golang.org/x/sys/unix:go_default_library", + ], + "@io_bazel_rules_go//go/platform:nacl": [ + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + ], + "@io_bazel_rules_go//go/platform:netbsd": [ + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + ], + "@io_bazel_rules_go//go/platform:openbsd": [ + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + ], + "@io_bazel_rules_go//go/platform:plan9": [ + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + ], + "@io_bazel_rules_go//go/platform:solaris": [ + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + ], + "@io_bazel_rules_go//go/platform:windows": [ + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//vendor/golang.org/x/sys/windows:go_default_library", + ], + "//conditions:default": [], + }), +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) + +go_test( + name = "go_default_test", + srcs = ["fs_windows_test.go"], + embed = [":go_default_library"], + deps = select({ + "@io_bazel_rules_go//go/platform:windows": [ + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + ], + "//conditions:default": [], + }), +) diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/fs/fs.go b/vendor/k8s.io/kubernetes/pkg/volume/util/fs/fs.go new file mode 100644 index 00000000..0050c5fe --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/fs/fs.go @@ -0,0 +1,117 @@ +// +build linux darwin + +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fs + +import ( + "bytes" + "fmt" + "os/exec" + "strings" + + "golang.org/x/sys/unix" + + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/kubernetes/pkg/volume/util/fsquota" +) + +// FsInfo linux returns (available bytes, byte capacity, byte usage, total inodes, inodes free, inode usage, error) +// for the filesystem that path resides upon. +func FsInfo(path string) (int64, int64, int64, int64, int64, int64, error) { + statfs := &unix.Statfs_t{} + err := unix.Statfs(path, statfs) + if err != nil { + return 0, 0, 0, 0, 0, 0, err + } + + // Available is blocks available * fragment size + available := int64(statfs.Bavail) * int64(statfs.Bsize) + + // Capacity is total block count * fragment size + capacity := int64(statfs.Blocks) * int64(statfs.Bsize) + + // Usage is block being used * fragment size (aka block size). + usage := (int64(statfs.Blocks) - int64(statfs.Bfree)) * int64(statfs.Bsize) + + inodes := int64(statfs.Files) + inodesFree := int64(statfs.Ffree) + inodesUsed := inodes - inodesFree + + return available, capacity, usage, inodes, inodesFree, inodesUsed, nil +} + +// DiskUsage gets disk usage of specified path. +func DiskUsage(path string) (*resource.Quantity, error) { + // First check whether the quota system knows about this directory + // A nil quantity with no error means that the path does not support quotas + // and we should use other mechanisms. + data, err := fsquota.GetConsumption(path) + if data != nil { + return data, nil + } else if err != nil { + return nil, fmt.Errorf("unable to retrieve disk consumption via quota for %s: %v", path, err) + } + // Uses the same niceness level as cadvisor.fs does when running du + // Uses -B 1 to always scale to a blocksize of 1 byte + out, err := exec.Command("nice", "-n", "19", "du", "-x", "-s", "-B", "1", path).CombinedOutput() + if err != nil { + return nil, fmt.Errorf("failed command 'du' ($ nice -n 19 du -x -s -B 1) on path %s with error %v", path, err) + } + used, err := resource.ParseQuantity(strings.Fields(string(out))[0]) + if err != nil { + return nil, fmt.Errorf("failed to parse 'du' output %s due to error %v", out, err) + } + used.Format = resource.BinarySI + return &used, nil +} + +// Find uses the equivalent of the command `find -dev -printf '.' | wc -c` to count files and directories. +// While this is not an exact measure of inodes used, it is a very good approximation. +func Find(path string) (int64, error) { + if path == "" { + return 0, fmt.Errorf("invalid directory") + } + // First check whether the quota system knows about this directory + // A nil quantity with no error means that the path does not support quotas + // and we should use other mechanisms. + inodes, err := fsquota.GetInodes(path) + if inodes != nil { + return inodes.Value(), nil + } else if err != nil { + return 0, fmt.Errorf("unable to retrieve inode consumption via quota for %s: %v", path, err) + } + var counter byteCounter + var stderr bytes.Buffer + findCmd := exec.Command("find", path, "-xdev", "-printf", ".") + findCmd.Stdout, findCmd.Stderr = &counter, &stderr + if err := findCmd.Start(); err != nil { + return 0, fmt.Errorf("failed to exec cmd %v - %v; stderr: %v", findCmd.Args, err, stderr.String()) + } + if err := findCmd.Wait(); err != nil { + return 0, fmt.Errorf("cmd %v failed. stderr: %s; err: %v", findCmd.Args, stderr.String(), err) + } + return counter.bytesWritten, nil +} + +// Simple io.Writer implementation that counts how many bytes were written. +type byteCounter struct{ bytesWritten int64 } + +func (b *byteCounter) Write(p []byte) (int, error) { + b.bytesWritten += int64(len(p)) + return len(p), nil +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/fs/fs_unsupported.go b/vendor/k8s.io/kubernetes/pkg/volume/util/fs/fs_unsupported.go new file mode 100644 index 00000000..340b4fdc --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/fs/fs_unsupported.go @@ -0,0 +1,39 @@ +// +build !linux,!darwin,!windows + +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fs + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/api/resource" +) + +// FSInfo unsupported returns 0 values for available and capacity and an error. +func FsInfo(path string) (int64, int64, int64, int64, int64, int64, error) { + return 0, 0, 0, 0, 0, 0, fmt.Errorf("FsInfo not supported for this build.") +} + +// DiskUsage gets disk usage of specified path. +func DiskUsage(path string) (*resource.Quantity, error) { + return nil, fmt.Errorf("Du not supported for this build.") +} + +func Find(path string) (int64, error) { + return 0, fmt.Errorf("Find not supported for this build.") +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/fs/fs_windows.go b/vendor/k8s.io/kubernetes/pkg/volume/util/fs/fs_windows.go new file mode 100644 index 00000000..07c4e6bd --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/fs/fs_windows.go @@ -0,0 +1,121 @@ +// +build windows + +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fs + +import ( + "fmt" + "os" + "syscall" + "unsafe" + + "golang.org/x/sys/windows" + + "k8s.io/apimachinery/pkg/api/resource" +) + +var ( + modkernel32 = windows.NewLazySystemDLL("kernel32.dll") + procGetDiskFreeSpaceEx = modkernel32.NewProc("GetDiskFreeSpaceExW") +) + +// FSInfo returns (available bytes, byte capacity, byte usage, total inodes, inodes free, inode usage, error) +// for the filesystem that path resides upon. +func FsInfo(path string) (int64, int64, int64, int64, int64, int64, error) { + var freeBytesAvailable, totalNumberOfBytes, totalNumberOfFreeBytes int64 + var err error + + ret, _, err := syscall.Syscall6( + procGetDiskFreeSpaceEx.Addr(), + 4, + uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))), + uintptr(unsafe.Pointer(&freeBytesAvailable)), + uintptr(unsafe.Pointer(&totalNumberOfBytes)), + uintptr(unsafe.Pointer(&totalNumberOfFreeBytes)), + 0, + 0, + ) + if ret == 0 { + return 0, 0, 0, 0, 0, 0, err + } + + return freeBytesAvailable, totalNumberOfBytes, totalNumberOfBytes - freeBytesAvailable, 0, 0, 0, nil +} + +// DiskUsage gets disk usage of specified path. +func DiskUsage(path string) (*resource.Quantity, error) { + info, err := os.Lstat(path) + if err != nil { + return nil, err + } + + usage, err := diskUsage(path, info) + if err != nil { + return nil, err + } + + used, err := resource.ParseQuantity(fmt.Sprintf("%d", usage)) + if err != nil { + return nil, fmt.Errorf("failed to parse fs usage %d due to %v", usage, err) + } + used.Format = resource.BinarySI + return &used, nil +} + +// Always return zero since inodes is not supported on Windows. +func Find(path string) (int64, error) { + return 0, nil +} + +func diskUsage(currPath string, info os.FileInfo) (int64, error) { + var size int64 + + if info.Mode()&os.ModeSymlink != 0 { + return size, nil + } + + size += info.Size() + + if !info.IsDir() { + return size, nil + } + + dir, err := os.Open(currPath) + if err != nil { + return size, err + } + defer dir.Close() + + files, err := dir.Readdir(-1) + if err != nil { + return size, err + } + + for _, file := range files { + if file.IsDir() { + s, err := diskUsage(fmt.Sprintf("%s/%s", currPath, file.Name()), file) + if err != nil { + return size, err + } + size += s + } else { + size += file.Size() + } + } + return size, nil +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/BUILD b/vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/BUILD new file mode 100644 index 00000000..ef48ca78 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/BUILD @@ -0,0 +1,78 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "project.go", + "quota.go", + "quota_linux.go", + "quota_unsupported.go", + ], + importpath = "k8s.io/kubernetes/pkg/volume/util/fsquota", + visibility = ["//visibility:public"], + deps = [ + "//pkg/features:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/mount-utils:go_default_library", + ] + select({ + "@io_bazel_rules_go//go/platform:android": [ + "//pkg/volume/util/fsquota/common:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", + "//vendor/golang.org/x/sys/unix:go_default_library", + "//vendor/k8s.io/klog/v2:go_default_library", + ], + "@io_bazel_rules_go//go/platform:linux": [ + "//pkg/volume/util/fsquota/common:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", + "//vendor/golang.org/x/sys/unix:go_default_library", + "//vendor/k8s.io/klog/v2:go_default_library", + ], + "//conditions:default": [], + }), +) + +go_test( + name = "go_default_test", + srcs = ["quota_linux_test.go"], + embed = [":go_default_library"], + deps = select({ + "@io_bazel_rules_go//go/platform:android": [ + "//pkg/features:go_default_library", + "//pkg/volume/util/fsquota/common:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library", + "//staging/src/k8s.io/mount-utils:go_default_library", + ], + "@io_bazel_rules_go//go/platform:linux": [ + "//pkg/features:go_default_library", + "//pkg/volume/util/fsquota/common:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library", + "//staging/src/k8s.io/mount-utils:go_default_library", + ], + "//conditions:default": [], + }), +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [ + ":package-srcs", + "//pkg/volume/util/fsquota/common:all-srcs", + ], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/common/BUILD b/vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/common/BUILD new file mode 100644 index 00000000..443359ab --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/common/BUILD @@ -0,0 +1,34 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "quota_linux_common.go", + "quota_linux_common_impl.go", + ], + importpath = "k8s.io/kubernetes/pkg/volume/util/fsquota/common", + visibility = ["//visibility:public"], + deps = select({ + "@io_bazel_rules_go//go/platform:android": [ + "//vendor/k8s.io/klog/v2:go_default_library", + ], + "@io_bazel_rules_go//go/platform:linux": [ + "//vendor/k8s.io/klog/v2:go_default_library", + ], + "//conditions:default": [], + }), +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/common/quota_linux_common.go b/vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/common/quota_linux_common.go new file mode 100644 index 00000000..c7232e8d --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/common/quota_linux_common.go @@ -0,0 +1,100 @@ +// +build linux + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import ( + "regexp" +) + +// QuotaID is generic quota identifier. +// Data type based on quotactl(2). +type QuotaID int32 + +const ( + // UnknownQuotaID -- cannot determine whether a quota is in force + UnknownQuotaID QuotaID = -1 + // BadQuotaID -- Invalid quota + BadQuotaID QuotaID = 0 +) + +// QuotaType -- type of quota to be applied +type QuotaType int + +const ( + // FSQuotaAccounting for quotas for accounting only + FSQuotaAccounting QuotaType = 1 << iota + // FSQuotaEnforcing for quotas for enforcement + FSQuotaEnforcing QuotaType = 1 << iota +) + +// FirstQuota is the quota ID we start with. +// XXXXXXX Need a better way of doing this... +var FirstQuota QuotaID = 1048577 + +// MountsFile is the location of the system mount data +var MountsFile = "/proc/self/mounts" + +// MountParseRegexp parses out /proc/sys/self/mounts +var MountParseRegexp = regexp.MustCompilePOSIX("^([^ ]*)[ \t]*([^ ]*)[ \t]*([^ ]*)") // Ignore options etc. + +// LinuxVolumeQuotaProvider returns an appropriate quota applier +// object if we can support quotas on this device +type LinuxVolumeQuotaProvider interface { + // GetQuotaApplier retrieves an object that can apply + // quotas (or nil if this provider cannot support quotas + // on the device) + GetQuotaApplier(mountpoint string, backingDev string) LinuxVolumeQuotaApplier +} + +// LinuxVolumeQuotaApplier is a generic interface to any quota +// mechanism supported by Linux +type LinuxVolumeQuotaApplier interface { + // GetQuotaOnDir gets the quota ID (if any) that applies to + // this directory + GetQuotaOnDir(path string) (QuotaID, error) + + // SetQuotaOnDir applies the specified quota ID to a directory. + // Negative value for bytes means that a non-enforcing quota + // should be applied (perhaps by setting a quota too large to + // be hit) + SetQuotaOnDir(path string, id QuotaID, bytes int64) error + + // QuotaIDIsInUse determines whether the quota ID is in use. + // Implementations should not check /etc/project or /etc/projid, + // only whether their underlying mechanism already has the ID in + // use. + // Return value of false with no error means that the ID is not + // in use; true means that it is already in use. An error + // return means that any quota ID will fail. + QuotaIDIsInUse(id QuotaID) (bool, error) + + // GetConsumption returns the consumption (in bytes) of the + // directory, determined by the implementation's quota-based + // mechanism. If it is unable to do so using that mechanism, + // it should return an error and allow higher layers to + // enumerate the directory. + GetConsumption(path string, id QuotaID) (int64, error) + + // GetInodes returns the number of inodes used by the + // directory, determined by the implementation's quota-based + // mechanism. If it is unable to do so using that mechanism, + // it should return an error and allow higher layers to + // enumerate the directory. + GetInodes(path string, id QuotaID) (int64, error) +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/common/quota_linux_common_impl.go b/vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/common/quota_linux_common_impl.go new file mode 100644 index 00000000..ad9a53ad --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/common/quota_linux_common_impl.go @@ -0,0 +1,286 @@ +// +build linux + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import ( + "bufio" + "fmt" + "io/ioutil" + "os" + "os/exec" + "regexp" + "strconv" + "strings" + "sync" + "syscall" + + "k8s.io/klog/v2" +) + +var quotaCmd string +var quotaCmdInitialized bool +var quotaCmdLock sync.RWMutex + +// If we later get a filesystem that uses project quota semantics other than +// XFS, we'll need to change this. +// Higher levels don't need to know what's inside +type linuxFilesystemType struct { + name string + typeMagic int64 // Filesystem magic number, per statfs(2) + maxQuota int64 + allowEmptyOutput bool // Accept empty output from "quota" command +} + +const ( + bitsPerWord = 32 << (^uint(0) >> 63) // either 32 or 64 +) + +var ( + linuxSupportedFilesystems = []linuxFilesystemType{ + { + name: "XFS", + typeMagic: 0x58465342, + maxQuota: 1<<(bitsPerWord-1) - 1, + allowEmptyOutput: true, // XFS filesystems report nothing if a quota is not present + }, { + name: "ext4fs", + typeMagic: 0xef53, + maxQuota: (1<<(bitsPerWord-1) - 1) & (1<<58 - 1), + allowEmptyOutput: false, // ext4 filesystems always report something even if a quota is not present + }, + } +) + +// VolumeProvider supplies a quota applier to the generic code. +type VolumeProvider struct { +} + +var quotaCmds = []string{"/sbin/xfs_quota", + "/usr/sbin/xfs_quota", + "/bin/xfs_quota"} + +var quotaParseRegexp = regexp.MustCompilePOSIX("^[^ \t]*[ \t]*([0-9]+)") + +var lsattrCmd = "/usr/bin/lsattr" +var lsattrParseRegexp = regexp.MustCompilePOSIX("^ *([0-9]+) [^ ]+ (.*)$") + +// GetQuotaApplier -- does this backing device support quotas that +// can be applied to directories? +func (*VolumeProvider) GetQuotaApplier(mountpoint string, backingDev string) LinuxVolumeQuotaApplier { + for _, fsType := range linuxSupportedFilesystems { + if isFilesystemOfType(mountpoint, backingDev, fsType.typeMagic) { + return linuxVolumeQuotaApplier{mountpoint: mountpoint, + maxQuota: fsType.maxQuota, + allowEmptyOutput: fsType.allowEmptyOutput, + } + } + } + return nil +} + +type linuxVolumeQuotaApplier struct { + mountpoint string + maxQuota int64 + allowEmptyOutput bool +} + +func getXFSQuotaCmd() (string, error) { + quotaCmdLock.Lock() + defer quotaCmdLock.Unlock() + if quotaCmdInitialized { + return quotaCmd, nil + } + for _, program := range quotaCmds { + fileinfo, err := os.Stat(program) + if err == nil && ((fileinfo.Mode().Perm() & (1 << 6)) != 0) { + klog.V(3).Infof("Found xfs_quota program %s", program) + quotaCmd = program + quotaCmdInitialized = true + return quotaCmd, nil + } + } + quotaCmdInitialized = true + return "", fmt.Errorf("no xfs_quota program found") +} + +func doRunXFSQuotaCommand(mountpoint string, mountsFile, command string) (string, error) { + quotaCmd, err := getXFSQuotaCmd() + if err != nil { + return "", err + } + // We're using numeric project IDs directly; no need to scan + // /etc/projects or /etc/projid + klog.V(4).Infof("runXFSQuotaCommand %s -t %s -P/dev/null -D/dev/null -x -f %s -c %s", quotaCmd, mountsFile, mountpoint, command) + cmd := exec.Command(quotaCmd, "-t", mountsFile, "-P/dev/null", "-D/dev/null", "-x", "-f", mountpoint, "-c", command) + + data, err := cmd.Output() + if err != nil { + return "", err + } + klog.V(4).Infof("runXFSQuotaCommand output %q", string(data)) + return string(data), nil +} + +// Extract the mountpoint we care about into a temporary mounts file so that xfs_quota does +// not attempt to scan every mount on the filesystem, which could hang if e. g. +// a stuck NFS mount is present. +// See https://bugzilla.redhat.com/show_bug.cgi?id=237120 for an example +// of the problem that could be caused if this were to happen. +func runXFSQuotaCommand(mountpoint string, command string) (string, error) { + tmpMounts, err := ioutil.TempFile("", "mounts") + if err != nil { + return "", fmt.Errorf("cannot create temporary mount file: %v", err) + } + tmpMountsFileName := tmpMounts.Name() + defer tmpMounts.Close() + defer os.Remove(tmpMountsFileName) + + mounts, err := os.Open(MountsFile) + if err != nil { + return "", fmt.Errorf("cannot open mounts file %s: %v", MountsFile, err) + } + defer mounts.Close() + + scanner := bufio.NewScanner(mounts) + for scanner.Scan() { + match := MountParseRegexp.FindStringSubmatch(scanner.Text()) + if match != nil { + mount := match[2] + if mount == mountpoint { + if _, err := tmpMounts.WriteString(fmt.Sprintf("%s\n", scanner.Text())); err != nil { + return "", fmt.Errorf("cannot write temporary mounts file: %v", err) + } + if err := tmpMounts.Sync(); err != nil { + return "", fmt.Errorf("cannot sync temporary mounts file: %v", err) + } + return doRunXFSQuotaCommand(mountpoint, tmpMountsFileName, command) + } + } + } + return "", fmt.Errorf("cannot run xfs_quota: cannot find mount point %s in %s", mountpoint, MountsFile) +} + +// SupportsQuotas determines whether the filesystem supports quotas. +func SupportsQuotas(mountpoint string, qType QuotaType) (bool, error) { + data, err := runXFSQuotaCommand(mountpoint, "state -p") + if err != nil { + return false, err + } + if qType == FSQuotaEnforcing { + return strings.Contains(data, "Enforcement: ON"), nil + } + return strings.Contains(data, "Accounting: ON"), nil +} + +func isFilesystemOfType(mountpoint string, backingDev string, typeMagic int64) bool { + var buf syscall.Statfs_t + err := syscall.Statfs(mountpoint, &buf) + if err != nil { + klog.Warningf("Warning: Unable to statfs %s: %v", mountpoint, err) + return false + } + if int64(buf.Type) != typeMagic { + return false + } + if answer, _ := SupportsQuotas(mountpoint, FSQuotaAccounting); answer { + return true + } + return false +} + +// GetQuotaOnDir retrieves the quota ID (if any) associated with the specified directory +// If we can't make system calls, all we can say is that we don't know whether +// it has a quota, and higher levels have to make the call. +func (v linuxVolumeQuotaApplier) GetQuotaOnDir(path string) (QuotaID, error) { + cmd := exec.Command(lsattrCmd, "-pd", path) + data, err := cmd.Output() + if err != nil { + return BadQuotaID, fmt.Errorf("cannot run lsattr: %v", err) + } + match := lsattrParseRegexp.FindStringSubmatch(string(data)) + if match == nil { + return BadQuotaID, fmt.Errorf("unable to parse lsattr -pd %s output %s", path, string(data)) + } + if match[2] != path { + return BadQuotaID, fmt.Errorf("mismatch between supplied and returned path (%s != %s)", path, match[2]) + } + projid, err := strconv.ParseInt(match[1], 10, 32) + if err != nil { + return BadQuotaID, fmt.Errorf("unable to parse project ID from %s (%v)", match[1], err) + } + return QuotaID(projid), nil +} + +// SetQuotaOnDir applies a quota to the specified directory under the specified mountpoint. +func (v linuxVolumeQuotaApplier) SetQuotaOnDir(path string, id QuotaID, bytes int64) error { + if bytes < 0 || bytes > v.maxQuota { + bytes = v.maxQuota + } + _, err := runXFSQuotaCommand(v.mountpoint, fmt.Sprintf("limit -p bhard=%v bsoft=%v %v", bytes, bytes, id)) + if err != nil { + return err + } + + _, err = runXFSQuotaCommand(v.mountpoint, fmt.Sprintf("project -s -p %s %v", path, id)) + return err +} + +func getQuantity(mountpoint string, id QuotaID, xfsQuotaArg string, multiplier int64, allowEmptyOutput bool) (int64, error) { + data, err := runXFSQuotaCommand(mountpoint, fmt.Sprintf("quota -p -N -n -v %s %v", xfsQuotaArg, id)) + if err != nil { + return 0, fmt.Errorf("unable to run xfs_quota: %v", err) + } + if data == "" && allowEmptyOutput { + return 0, nil + } + match := quotaParseRegexp.FindStringSubmatch(data) + if match == nil { + return 0, fmt.Errorf("unable to parse quota output '%s'", data) + } + size, err := strconv.ParseInt(match[1], 10, 64) + if err != nil { + return 0, fmt.Errorf("unable to parse data size '%s' from '%s': %v", match[1], data, err) + } + klog.V(4).Infof("getQuantity %s %d %s %d => %d %v", mountpoint, id, xfsQuotaArg, multiplier, size, err) + return size * multiplier, nil +} + +// GetConsumption returns the consumption in bytes if available via quotas +func (v linuxVolumeQuotaApplier) GetConsumption(_ string, id QuotaID) (int64, error) { + return getQuantity(v.mountpoint, id, "-b", 1024, v.allowEmptyOutput) +} + +// GetInodes returns the inodes in use if available via quotas +func (v linuxVolumeQuotaApplier) GetInodes(_ string, id QuotaID) (int64, error) { + return getQuantity(v.mountpoint, id, "-i", 1, v.allowEmptyOutput) +} + +// QuotaIDIsInUse checks whether the specified quota ID is in use on the specified +// filesystem +func (v linuxVolumeQuotaApplier) QuotaIDIsInUse(id QuotaID) (bool, error) { + bytes, err := v.GetConsumption(v.mountpoint, id) + if err != nil { + return false, err + } + if bytes > 0 { + return true, nil + } + inodes, err := v.GetInodes(v.mountpoint, id) + return inodes > 0, err +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/project.go b/vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/project.go new file mode 100644 index 00000000..e6c7d82e --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/project.go @@ -0,0 +1,357 @@ +// +build linux + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fsquota + +import ( + "bufio" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "strconv" + "sync" + + "golang.org/x/sys/unix" + "k8s.io/kubernetes/pkg/volume/util/fsquota/common" +) + +var projectsFile = "/etc/projects" +var projidFile = "/etc/projid" + +var projectsParseRegexp = regexp.MustCompilePOSIX("^([[:digit:]]+):(.*)$") +var projidParseRegexp = regexp.MustCompilePOSIX("^([^#][^:]*):([[:digit:]]+)$") + +var quotaIDLock sync.RWMutex + +const maxUnusedQuotasToSearch = 128 // Don't go into an infinite loop searching for an unused quota + +type projectType struct { + isValid bool // False if we need to remove this line + id common.QuotaID + data string // Project name (projid) or directory (projects) + line string +} + +type projectsList struct { + projects []projectType + projid []projectType +} + +func projFilesAreOK() error { + if sf, err := os.Lstat(projectsFile); err != nil || sf.Mode().IsRegular() { + if sf, err := os.Lstat(projidFile); err != nil || sf.Mode().IsRegular() { + return nil + } + return fmt.Errorf("%s exists but is not a plain file, cannot continue", projidFile) + } + return fmt.Errorf("%s exists but is not a plain file, cannot continue", projectsFile) +} + +func lockFile(file *os.File) error { + return unix.Flock(int(file.Fd()), unix.LOCK_EX) +} + +func unlockFile(file *os.File) error { + return unix.Flock(int(file.Fd()), unix.LOCK_UN) +} + +// openAndLockProjectFiles opens /etc/projects and /etc/projid locked. +// Creates them if they don't exist +func openAndLockProjectFiles() (*os.File, *os.File, error) { + // Make sure neither project-related file is a symlink! + if err := projFilesAreOK(); err != nil { + return nil, nil, fmt.Errorf("system project files failed verification: %v", err) + } + // We don't actually modify the original files; we create temporaries and + // move them over the originals + fProjects, err := os.OpenFile(projectsFile, os.O_RDONLY|os.O_CREATE, 0644) + if err != nil { + err = fmt.Errorf("unable to open %s: %v", projectsFile, err) + return nil, nil, err + } + fProjid, err := os.OpenFile(projidFile, os.O_RDONLY|os.O_CREATE, 0644) + if err == nil { + // Check once more, to ensure nothing got changed out from under us + if err = projFilesAreOK(); err == nil { + err = lockFile(fProjects) + if err == nil { + err = lockFile(fProjid) + if err == nil { + return fProjects, fProjid, nil + } + // Nothing useful we can do if we get an error here + err = fmt.Errorf("unable to lock %s: %v", projidFile, err) + unlockFile(fProjects) + } else { + err = fmt.Errorf("unable to lock %s: %v", projectsFile, err) + } + } else { + err = fmt.Errorf("system project files failed re-verification: %v", err) + } + fProjid.Close() + } else { + err = fmt.Errorf("unable to open %s: %v", projidFile, err) + } + fProjects.Close() + return nil, nil, err +} + +func closeProjectFiles(fProjects *os.File, fProjid *os.File) error { + // Nothing useful we can do if either of these fail, + // but we have to close (and thereby unlock) the files anyway. + var err error + var err1 error + if fProjid != nil { + err = fProjid.Close() + } + if fProjects != nil { + err1 = fProjects.Close() + } + if err == nil { + return err1 + } + return err +} + +func parseProject(l string) projectType { + if match := projectsParseRegexp.FindStringSubmatch(l); match != nil { + i, err := strconv.Atoi(match[1]) + if err == nil { + return projectType{true, common.QuotaID(i), match[2], l} + } + } + return projectType{true, common.BadQuotaID, "", l} +} + +func parseProjid(l string) projectType { + if match := projidParseRegexp.FindStringSubmatch(l); match != nil { + i, err := strconv.Atoi(match[2]) + if err == nil { + return projectType{true, common.QuotaID(i), match[1], l} + } + } + return projectType{true, common.BadQuotaID, "", l} +} + +func parseProjFile(f *os.File, parser func(l string) projectType) []projectType { + var answer []projectType + scanner := bufio.NewScanner(f) + for scanner.Scan() { + answer = append(answer, parser(scanner.Text())) + } + return answer +} + +func readProjectFiles(projects *os.File, projid *os.File) projectsList { + return projectsList{parseProjFile(projects, parseProject), parseProjFile(projid, parseProjid)} +} + +func findAvailableQuota(path string, idMap map[common.QuotaID]bool) (common.QuotaID, error) { + unusedQuotasSearched := 0 + for id := common.FirstQuota; true; id++ { + if _, ok := idMap[id]; !ok { + isInUse, err := getApplier(path).QuotaIDIsInUse(id) + if err != nil { + return common.BadQuotaID, err + } else if !isInUse { + return id, nil + } + unusedQuotasSearched++ + if unusedQuotasSearched > maxUnusedQuotasToSearch { + break + } + } + } + return common.BadQuotaID, fmt.Errorf("cannot find available quota ID") +} + +func addDirToProject(path string, id common.QuotaID, list *projectsList) (common.QuotaID, bool, error) { + idMap := make(map[common.QuotaID]bool) + for _, project := range list.projects { + if project.data == path { + if id != project.id { + return common.BadQuotaID, false, fmt.Errorf("attempt to reassign project ID for %s", path) + } + // Trying to reassign a directory to the project it's + // already in. Maybe this should be an error, but for + // now treat it as an idempotent operation + return id, false, nil + } + idMap[project.id] = true + } + var needToAddProjid = true + for _, projid := range list.projid { + idMap[projid.id] = true + if projid.id == id && id != common.BadQuotaID { + needToAddProjid = false + } + } + var err error + if id == common.BadQuotaID { + id, err = findAvailableQuota(path, idMap) + if err != nil { + return common.BadQuotaID, false, err + } + needToAddProjid = true + } + if needToAddProjid { + name := fmt.Sprintf("volume%v", id) + line := fmt.Sprintf("%s:%v", name, id) + list.projid = append(list.projid, projectType{true, id, name, line}) + } + line := fmt.Sprintf("%v:%s", id, path) + list.projects = append(list.projects, projectType{true, id, path, line}) + return id, needToAddProjid, nil +} + +func removeDirFromProject(path string, id common.QuotaID, list *projectsList) (bool, error) { + if id == common.BadQuotaID { + return false, fmt.Errorf("attempt to remove invalid quota ID from %s", path) + } + foundAt := -1 + countByID := make(map[common.QuotaID]int) + for i, project := range list.projects { + if project.data == path { + if id != project.id { + return false, fmt.Errorf("attempting to remove quota ID %v from path %s, but expecting ID %v", id, path, project.id) + } else if foundAt != -1 { + return false, fmt.Errorf("found multiple quota IDs for path %s", path) + } + // Faster and easier than deleting an element + list.projects[i].isValid = false + foundAt = i + } + countByID[project.id]++ + } + if foundAt == -1 { + return false, fmt.Errorf("cannot find quota associated with path %s", path) + } + if countByID[id] <= 1 { + // Removing the last entry means that we're no longer using + // the quota ID, so remove that as well + for i, projid := range list.projid { + if projid.id == id { + list.projid[i].isValid = false + } + } + return true, nil + } + return false, nil +} + +func writeProjectFile(base *os.File, projects []projectType) (string, error) { + oname := base.Name() + stat, err := base.Stat() + if err != nil { + return "", err + } + mode := stat.Mode() & os.ModePerm + f, err := ioutil.TempFile(filepath.Dir(oname), filepath.Base(oname)) + if err != nil { + return "", err + } + filename := f.Name() + if err := os.Chmod(filename, mode); err != nil { + return "", err + } + for _, proj := range projects { + if proj.isValid { + if _, err := f.WriteString(fmt.Sprintf("%s\n", proj.line)); err != nil { + f.Close() + os.Remove(filename) + return "", err + } + } + } + if err := f.Close(); err != nil { + os.Remove(filename) + return "", err + } + return filename, nil +} + +func writeProjectFiles(fProjects *os.File, fProjid *os.File, writeProjid bool, list projectsList) error { + tmpProjects, err := writeProjectFile(fProjects, list.projects) + if err == nil { + // Ensure that both files are written before we try to rename either. + if writeProjid { + tmpProjid, err := writeProjectFile(fProjid, list.projid) + if err == nil { + err = os.Rename(tmpProjid, fProjid.Name()) + if err != nil { + os.Remove(tmpProjid) + } + } + } + if err == nil { + err = os.Rename(tmpProjects, fProjects.Name()) + if err == nil { + return nil + } + // We're in a bit of trouble here; at this + // point we've successfully renamed tmpProjid + // to the real thing, but renaming tmpProject + // to the real file failed. There's not much we + // can do in this position. Anything we could do + // to try to undo it would itself be likely to fail. + } + os.Remove(tmpProjects) + } + return fmt.Errorf("unable to write project files: %v", err) +} + +func createProjectID(path string, ID common.QuotaID) (common.QuotaID, error) { + quotaIDLock.Lock() + defer quotaIDLock.Unlock() + fProjects, fProjid, err := openAndLockProjectFiles() + if err == nil { + defer closeProjectFiles(fProjects, fProjid) + list := readProjectFiles(fProjects, fProjid) + var writeProjid bool + ID, writeProjid, err = addDirToProject(path, ID, &list) + if err == nil && ID != common.BadQuotaID { + if err = writeProjectFiles(fProjects, fProjid, writeProjid, list); err == nil { + return ID, nil + } + } + } + return common.BadQuotaID, fmt.Errorf("createProjectID %s %v failed %v", path, ID, err) +} + +func removeProjectID(path string, ID common.QuotaID) error { + if ID == common.BadQuotaID { + return fmt.Errorf("attempting to remove invalid quota ID %v", ID) + } + quotaIDLock.Lock() + defer quotaIDLock.Unlock() + fProjects, fProjid, err := openAndLockProjectFiles() + if err == nil { + defer closeProjectFiles(fProjects, fProjid) + list := readProjectFiles(fProjects, fProjid) + var writeProjid bool + writeProjid, err = removeDirFromProject(path, ID, &list) + if err == nil { + if err = writeProjectFiles(fProjects, fProjid, writeProjid, list); err == nil { + return nil + } + } + } + return fmt.Errorf("removeProjectID %s %v failed %v", path, ID, err) +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/quota.go b/vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/quota.go new file mode 100644 index 00000000..fbd29fba --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/quota.go @@ -0,0 +1,50 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fsquota + +import ( + "k8s.io/mount-utils" + + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/types" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/kubernetes/pkg/features" +) + +// Interface -- quota interface +type Interface interface { + // Does the path provided support quotas, and if so, what types + SupportsQuotas(m mount.Interface, path string) (bool, error) + // Assign a quota (picked by the quota mechanism) to a path, + // and return it. + AssignQuota(m mount.Interface, path string, poduid types.UID, bytes *resource.Quantity) error + + // Get the quota-based storage consumption for the path + GetConsumption(path string) (*resource.Quantity, error) + + // Get the quota-based inode consumption for the path + GetInodes(path string) (*resource.Quantity, error) + + // Remove the quota from a path + // Implementations may assume that any data covered by the + // quota has already been removed. + ClearQuota(m mount.Interface, path string) error +} + +func enabledQuotasForMonitoring() bool { + return utilfeature.DefaultFeatureGate.Enabled(features.LocalStorageCapacityIsolationFSQuotaMonitoring) +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/quota_linux.go b/vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/quota_linux.go new file mode 100644 index 00000000..d1ad9f77 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/quota_linux.go @@ -0,0 +1,452 @@ +// +build linux + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fsquota + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "sync" + + "k8s.io/klog/v2" + "k8s.io/mount-utils" + + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/kubernetes/pkg/volume/util/fsquota/common" +) + +// Pod -> ID +var podQuotaMap = make(map[types.UID]common.QuotaID) + +// Dir -> ID (for convenience) +var dirQuotaMap = make(map[string]common.QuotaID) + +// ID -> pod +var quotaPodMap = make(map[common.QuotaID]types.UID) + +// Directory -> pod +var dirPodMap = make(map[string]types.UID) + +// Backing device -> applier +// This is *not* cleaned up; its size will be bounded. +var devApplierMap = make(map[string]common.LinuxVolumeQuotaApplier) + +// Directory -> applier +var dirApplierMap = make(map[string]common.LinuxVolumeQuotaApplier) +var dirApplierLock sync.RWMutex + +// Pod -> refcount +var podDirCountMap = make(map[types.UID]int) + +// ID -> size +var quotaSizeMap = make(map[common.QuotaID]int64) +var quotaLock sync.RWMutex + +var supportsQuotasMap = make(map[string]bool) +var supportsQuotasLock sync.RWMutex + +// Directory -> backingDev +var backingDevMap = make(map[string]string) +var backingDevLock sync.RWMutex + +var mountpointMap = make(map[string]string) +var mountpointLock sync.RWMutex + +var providers = []common.LinuxVolumeQuotaProvider{ + &common.VolumeProvider{}, +} + +// Separate the innards for ease of testing +func detectBackingDevInternal(mountpoint string, mounts string) (string, error) { + file, err := os.Open(mounts) + if err != nil { + return "", err + } + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + match := common.MountParseRegexp.FindStringSubmatch(scanner.Text()) + if match != nil { + device := match[1] + mount := match[2] + if mount == mountpoint { + return device, nil + } + } + } + return "", fmt.Errorf("couldn't find backing device for %s", mountpoint) +} + +// detectBackingDev assumes that the mount point provided is valid +func detectBackingDev(_ mount.Interface, mountpoint string) (string, error) { + return detectBackingDevInternal(mountpoint, common.MountsFile) +} + +func clearBackingDev(path string) { + backingDevLock.Lock() + defer backingDevLock.Unlock() + delete(backingDevMap, path) +} + +// Assumes that the path has been fully canonicalized +// Breaking this up helps with testing +func detectMountpointInternal(m mount.Interface, path string) (string, error) { + for path != "" && path != "/" { + // per k8s.io/mount-utils/mount_linux this detects all but + // a bind mount from one part of a mount to another. + // For our purposes that's fine; we simply want the "true" + // mount point + // + // IsNotMountPoint proved much more troublesome; it actually + // scans the mounts, and when a lot of mount/unmount + // activity takes place, it is not able to get a consistent + // view of /proc/self/mounts, causing it to time out and + // report incorrectly. + isNotMount, err := m.IsLikelyNotMountPoint(path) + if err != nil { + return "/", err + } + if !isNotMount { + return path, nil + } + path = filepath.Dir(path) + } + return "/", nil +} + +func detectMountpoint(m mount.Interface, path string) (string, error) { + xpath, err := filepath.Abs(path) + if err != nil { + return "/", err + } + xpath, err = filepath.EvalSymlinks(xpath) + if err != nil { + return "/", err + } + if xpath, err = detectMountpointInternal(m, xpath); err == nil { + return xpath, nil + } + return "/", err +} + +func clearMountpoint(path string) { + mountpointLock.Lock() + defer mountpointLock.Unlock() + delete(mountpointMap, path) +} + +// getFSInfo Returns mountpoint and backing device +// getFSInfo should cache the mountpoint and backing device for the +// path. +func getFSInfo(m mount.Interface, path string) (string, string, error) { + mountpointLock.Lock() + defer mountpointLock.Unlock() + + backingDevLock.Lock() + defer backingDevLock.Unlock() + + var err error + + mountpoint, okMountpoint := mountpointMap[path] + if !okMountpoint { + mountpoint, err = detectMountpoint(m, path) + if err != nil { + return "", "", fmt.Errorf("cannot determine mountpoint for %s: %v", path, err) + } + } + + backingDev, okBackingDev := backingDevMap[path] + if !okBackingDev { + backingDev, err = detectBackingDev(m, mountpoint) + if err != nil { + return "", "", fmt.Errorf("cannot determine backing device for %s: %v", path, err) + } + } + mountpointMap[path] = mountpoint + backingDevMap[path] = backingDev + return mountpoint, backingDev, nil +} + +func clearFSInfo(path string) { + clearMountpoint(path) + clearBackingDev(path) +} + +func getApplier(path string) common.LinuxVolumeQuotaApplier { + dirApplierLock.Lock() + defer dirApplierLock.Unlock() + return dirApplierMap[path] +} + +func setApplier(path string, applier common.LinuxVolumeQuotaApplier) { + dirApplierLock.Lock() + defer dirApplierLock.Unlock() + dirApplierMap[path] = applier +} + +func clearApplier(path string) { + dirApplierLock.Lock() + defer dirApplierLock.Unlock() + delete(dirApplierMap, path) +} + +func setQuotaOnDir(path string, id common.QuotaID, bytes int64) error { + return getApplier(path).SetQuotaOnDir(path, id, bytes) +} + +func getQuotaOnDir(m mount.Interface, path string) (common.QuotaID, error) { + _, _, err := getFSInfo(m, path) + if err != nil { + return common.BadQuotaID, err + } + return getApplier(path).GetQuotaOnDir(path) +} + +func clearQuotaOnDir(m mount.Interface, path string) error { + // Since we may be called without path being in the map, + // we explicitly have to check in this case. + klog.V(4).Infof("clearQuotaOnDir %s", path) + supportsQuotas, err := SupportsQuotas(m, path) + if err != nil { + // Log-and-continue instead of returning an error for now + // due to unspecified backwards compatibility concerns (a subject to revise) + klog.V(3).Infof("Attempt to check for quota support failed: %v", err) + } + if !supportsQuotas { + return nil + } + projid, err := getQuotaOnDir(m, path) + if err == nil && projid != common.BadQuotaID { + // This means that we have a quota on the directory but + // we can't clear it. That's not good. + err = setQuotaOnDir(path, projid, 0) + if err != nil { + klog.V(3).Infof("Attempt to clear quota failed: %v", err) + } + // Even if clearing the quota failed, we still need to + // try to remove the project ID, or that may be left dangling. + err1 := removeProjectID(path, projid) + if err1 != nil { + klog.V(3).Infof("Attempt to remove quota ID from system files failed: %v", err1) + } + clearFSInfo(path) + if err != nil { + return err + } + return err1 + } + // If we couldn't get a quota, that's fine -- there may + // never have been one, and we have no way to know otherwise + klog.V(3).Infof("clearQuotaOnDir fails %v", err) + return nil +} + +// SupportsQuotas -- Does the path support quotas +// Cache the applier for paths that support quotas. For paths that don't, +// don't cache the result because nothing will clean it up. +// However, do cache the device->applier map; the number of devices +// is bounded. +func SupportsQuotas(m mount.Interface, path string) (bool, error) { + if !enabledQuotasForMonitoring() { + klog.V(3).Info("SupportsQuotas called, but quotas disabled") + return false, nil + } + supportsQuotasLock.Lock() + defer supportsQuotasLock.Unlock() + if supportsQuotas, ok := supportsQuotasMap[path]; ok { + return supportsQuotas, nil + } + mount, dev, err := getFSInfo(m, path) + if err != nil { + return false, err + } + // Do we know about this device? + applier, ok := devApplierMap[mount] + if !ok { + for _, provider := range providers { + if applier = provider.GetQuotaApplier(mount, dev); applier != nil { + devApplierMap[mount] = applier + break + } + } + } + if applier != nil { + supportsQuotasMap[path] = true + setApplier(path, applier) + return true, nil + } + delete(backingDevMap, path) + delete(mountpointMap, path) + return false, nil +} + +// AssignQuota -- assign a quota to the specified directory. +// AssignQuota chooses the quota ID based on the pod UID and path. +// If the pod UID is identical to another one known, it may (but presently +// doesn't) choose the same quota ID as other volumes in the pod. +//lint:ignore SA4009 poduid is overwritten by design, see comment below +func AssignQuota(m mount.Interface, path string, poduid types.UID, bytes *resource.Quantity) error { + if bytes == nil { + return fmt.Errorf("attempting to assign null quota to %s", path) + } + ibytes := bytes.Value() + if ok, err := SupportsQuotas(m, path); !ok { + return fmt.Errorf("quotas not supported on %s: %v", path, err) + } + quotaLock.Lock() + defer quotaLock.Unlock() + // Current policy is to set individual quotas on each volumes. + // If we decide later that we want to assign one quota for all + // volumes in a pod, we can simply remove this line of code. + // If and when we decide permanently that we're going to adopt + // one quota per volume, we can rip all of the pod code out. + poduid = types.UID(uuid.NewUUID()) + if pod, ok := dirPodMap[path]; ok && pod != poduid { + return fmt.Errorf("requesting quota on existing directory %s but different pod %s %s", path, pod, poduid) + } + oid, ok := podQuotaMap[poduid] + if ok { + if quotaSizeMap[oid] != ibytes { + return fmt.Errorf("requesting quota of different size: old %v new %v", quotaSizeMap[oid], bytes) + } + } else { + oid = common.BadQuotaID + } + id, err := createProjectID(path, oid) + if err == nil { + if oid != common.BadQuotaID && oid != id { + return fmt.Errorf("attempt to reassign quota %v to %v", oid, id) + } + // When enforcing quotas are enabled, we'll condition this + // on their being disabled also. + if ibytes > 0 { + ibytes = -1 + } + if err = setQuotaOnDir(path, id, ibytes); err == nil { + quotaPodMap[id] = poduid + quotaSizeMap[id] = ibytes + podQuotaMap[poduid] = id + dirQuotaMap[path] = id + dirPodMap[path] = poduid + podDirCountMap[poduid]++ + klog.V(4).Infof("Assigning quota ID %d (%d) to %s", id, ibytes, path) + return nil + } + removeProjectID(path, id) + } + return fmt.Errorf("assign quota FAILED %v", err) +} + +// GetConsumption -- retrieve the consumption (in bytes) of the directory +func GetConsumption(path string) (*resource.Quantity, error) { + // Note that we actually need to hold the lock at least through + // running the quota command, so it can't get recycled behind our back + quotaLock.Lock() + defer quotaLock.Unlock() + applier := getApplier(path) + // No applier means directory is not under quota management + if applier == nil { + return nil, nil + } + ibytes, err := applier.GetConsumption(path, dirQuotaMap[path]) + if err != nil { + return nil, err + } + return resource.NewQuantity(ibytes, resource.DecimalSI), nil +} + +// GetInodes -- retrieve the number of inodes in use under the directory +func GetInodes(path string) (*resource.Quantity, error) { + // Note that we actually need to hold the lock at least through + // running the quota command, so it can't get recycled behind our back + quotaLock.Lock() + defer quotaLock.Unlock() + applier := getApplier(path) + // No applier means directory is not under quota management + if applier == nil { + return nil, nil + } + inodes, err := applier.GetInodes(path, dirQuotaMap[path]) + if err != nil { + return nil, err + } + return resource.NewQuantity(inodes, resource.DecimalSI), nil +} + +// ClearQuota -- remove the quota assigned to a directory +func ClearQuota(m mount.Interface, path string) error { + klog.V(3).Infof("ClearQuota %s", path) + if !enabledQuotasForMonitoring() { + return fmt.Errorf("clearQuota called, but quotas disabled") + } + quotaLock.Lock() + defer quotaLock.Unlock() + poduid, ok := dirPodMap[path] + if !ok { + // Nothing in the map either means that there was no + // quota to begin with or that we're clearing a + // stale directory, so if we find a quota, just remove it. + // The process of clearing the quota requires that an applier + // be found, which needs to be cleaned up. + defer delete(supportsQuotasMap, path) + defer clearApplier(path) + return clearQuotaOnDir(m, path) + } + _, ok = podQuotaMap[poduid] + if !ok { + return fmt.Errorf("clearQuota: No quota available for %s", path) + } + projid, err := getQuotaOnDir(m, path) + if err != nil { + // Log-and-continue instead of returning an error for now + // due to unspecified backwards compatibility concerns (a subject to revise) + klog.V(3).Infof("Attempt to check quota ID %v on dir %s failed: %v", dirQuotaMap[path], path, err) + } + if projid != dirQuotaMap[path] { + return fmt.Errorf("expected quota ID %v on dir %s does not match actual %v", dirQuotaMap[path], path, projid) + } + count, ok := podDirCountMap[poduid] + if count <= 1 || !ok { + err = clearQuotaOnDir(m, path) + // This error should be noted; we still need to clean up + // and otherwise handle in the same way. + if err != nil { + klog.V(3).Infof("Unable to clear quota %v %s: %v", dirQuotaMap[path], path, err) + } + delete(quotaSizeMap, podQuotaMap[poduid]) + delete(quotaPodMap, podQuotaMap[poduid]) + delete(podDirCountMap, poduid) + delete(podQuotaMap, poduid) + } else { + err = removeProjectID(path, projid) + podDirCountMap[poduid]-- + klog.V(4).Infof("Not clearing quota for pod %s; still %v dirs outstanding", poduid, podDirCountMap[poduid]) + } + delete(dirPodMap, path) + delete(dirQuotaMap, path) + delete(supportsQuotasMap, path) + clearApplier(path) + if err != nil { + return fmt.Errorf("unable to clear quota for %s: %v", path, err) + } + return nil +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/quota_unsupported.go b/vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/quota_unsupported.go new file mode 100644 index 00000000..b8c4ead4 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/quota_unsupported.go @@ -0,0 +1,58 @@ +// +build !linux + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fsquota + +import ( + "errors" + + "k8s.io/mount-utils" + + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/types" +) + +// Dummy quota implementation for systems that do not implement support +// for volume quotas + +var errNotImplemented = errors.New("not implemented") + +// SupportsQuotas -- dummy implementation +func SupportsQuotas(_ mount.Interface, _ string) (bool, error) { + return false, errNotImplemented +} + +// AssignQuota -- dummy implementation +func AssignQuota(_ mount.Interface, _ string, _ types.UID, _ *resource.Quantity) error { + return errNotImplemented +} + +// GetConsumption -- dummy implementation +func GetConsumption(_ string) (*resource.Quantity, error) { + return nil, errNotImplemented +} + +// GetInodes -- dummy implementation +func GetInodes(_ string) (*resource.Quantity, error) { + return nil, errNotImplemented +} + +// ClearQuota -- dummy implementation +func ClearQuota(_ mount.Interface, _ string) error { + return errNotImplemented +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/BUILD b/vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/BUILD new file mode 100644 index 00000000..57b546e7 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/BUILD @@ -0,0 +1,65 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "fake_hostutil.go", + "hostutil.go", + "hostutil_linux.go", + "hostutil_unsupported.go", + "hostutil_windows.go", + ], + importpath = "k8s.io/kubernetes/pkg/volume/util/hostutil", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/mount-utils:go_default_library", + ] + select({ + "@io_bazel_rules_go//go/platform:android": [ + "//vendor/golang.org/x/sys/unix:go_default_library", + "//vendor/k8s.io/klog/v2:go_default_library", + "//vendor/k8s.io/utils/path:go_default_library", + ], + "@io_bazel_rules_go//go/platform:linux": [ + "//vendor/golang.org/x/sys/unix:go_default_library", + "//vendor/k8s.io/klog/v2:go_default_library", + "//vendor/k8s.io/utils/path:go_default_library", + ], + "@io_bazel_rules_go//go/platform:windows": [ + "//vendor/k8s.io/klog/v2:go_default_library", + "//vendor/k8s.io/utils/path:go_default_library", + ], + "//conditions:default": [], + }), +) + +go_test( + name = "go_default_test", + srcs = [ + "hostutil_linux_test.go", + "hostutil_windows_test.go", + ], + embed = [":go_default_library"], + deps = select({ + "@io_bazel_rules_go//go/platform:android": [ + "//vendor/k8s.io/utils/exec:go_default_library", + ], + "@io_bazel_rules_go//go/platform:linux": [ + "//vendor/k8s.io/utils/exec:go_default_library", + ], + "//conditions:default": [], + }), +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/fake_hostutil.go b/vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/fake_hostutil.go new file mode 100644 index 00000000..cc5fe628 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/fake_hostutil.go @@ -0,0 +1,118 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package hostutil + +import ( + "errors" + "os" + "sync" + + "k8s.io/mount-utils" +) + +// FakeHostUtil is a fake HostUtils implementation for testing +type FakeHostUtil struct { + MountPoints []mount.MountPoint + Filesystem map[string]FileType + + mutex sync.Mutex +} + +// NewFakeHostUtil returns a struct that implements the HostUtils interface +// for testing +// TODO: no callers were initializing the struct with any MountPoints. Check +// if those are still being used by any callers and if MountPoints still need +// to be a part of the struct. +func NewFakeHostUtil(fs map[string]FileType) *FakeHostUtil { + return &FakeHostUtil{ + Filesystem: fs, + } +} + +// Compile-time check to make sure FakeHostUtil implements interface +var _ HostUtils = &FakeHostUtil{} + +// DeviceOpened checks if block device referenced by pathname is in use by +// checking if is listed as a device in the in-memory mountpoint table. +func (hu *FakeHostUtil) DeviceOpened(pathname string) (bool, error) { + hu.mutex.Lock() + defer hu.mutex.Unlock() + + for _, mp := range hu.MountPoints { + if mp.Device == pathname { + return true, nil + } + } + return false, nil +} + +// PathIsDevice always returns true +func (hu *FakeHostUtil) PathIsDevice(pathname string) (bool, error) { + return true, nil +} + +// GetDeviceNameFromMount given a mount point, find the volume id +func (hu *FakeHostUtil) GetDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) { + return getDeviceNameFromMount(mounter, mountPath, pluginMountDir) +} + +// MakeRShared checks if path is shared and bind-mounts it as rshared if needed. +// No-op for testing +func (hu *FakeHostUtil) MakeRShared(path string) error { + return nil +} + +// GetFileType checks for file/directory/socket/block/character devices. +// Defaults to Directory if otherwise unspecified. +func (hu *FakeHostUtil) GetFileType(pathname string) (FileType, error) { + if t, ok := hu.Filesystem[pathname]; ok { + return t, nil + } + return FileType("Directory"), nil +} + +// PathExists checks if pathname exists. +func (hu *FakeHostUtil) PathExists(pathname string) (bool, error) { + if _, ok := hu.Filesystem[pathname]; ok { + return true, nil + } + return false, nil +} + +// EvalHostSymlinks returns the path name after evaluating symlinks. +// No-op for testing +func (hu *FakeHostUtil) EvalHostSymlinks(pathname string) (string, error) { + return pathname, nil +} + +// GetOwner returns the integer ID for the user and group of the given path +// Not implemented for testing +func (hu *FakeHostUtil) GetOwner(pathname string) (int64, int64, error) { + return -1, -1, errors.New("GetOwner not implemented") +} + +// GetSELinuxSupport tests if pathname is on a mount that supports SELinux. +// Not implemented for testing +func (hu *FakeHostUtil) GetSELinuxSupport(pathname string) (bool, error) { + return false, errors.New("GetSELinuxSupport not implemented") +} + +// GetMode returns permissions of pathname. +// Not implemented for testing +func (hu *FakeHostUtil) GetMode(pathname string) (os.FileMode, error) { + return 0, errors.New("not implemented") +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/hostutil.go b/vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/hostutil.go new file mode 100644 index 00000000..561278b7 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/hostutil.go @@ -0,0 +1,109 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package hostutil + +import ( + "fmt" + "os" + + "k8s.io/mount-utils" +) + +// FileType enumerates the known set of possible file types. +type FileType string + +const ( + // FileTypeBlockDev defines a constant for the block device FileType. + FileTypeBlockDev FileType = "BlockDevice" + // FileTypeCharDev defines a constant for the character device FileType. + FileTypeCharDev FileType = "CharDevice" + // FileTypeDirectory defines a constant for the directory FileType. + FileTypeDirectory FileType = "Directory" + // FileTypeFile defines a constant for the file FileType. + FileTypeFile FileType = "File" + // FileTypeSocket defines a constant for the socket FileType. + FileTypeSocket FileType = "Socket" + // FileTypeUnknown defines a constant for an unknown FileType. + FileTypeUnknown FileType = "" +) + +// HostUtils defines the set of methods for interacting with paths on a host. +type HostUtils interface { + // DeviceOpened determines if the device (e.g. /dev/sdc) is in use elsewhere + // on the system, i.e. still mounted. + DeviceOpened(pathname string) (bool, error) + // PathIsDevice determines if a path is a device. + PathIsDevice(pathname string) (bool, error) + // GetDeviceNameFromMount finds the device name by checking the mount path + // to get the global mount path within its plugin directory. + GetDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) + // MakeRShared checks that given path is on a mount with 'rshared' mount + // propagation. If not, it bind-mounts the path as rshared. + MakeRShared(path string) error + // GetFileType checks for file/directory/socket/block/character devices. + GetFileType(pathname string) (FileType, error) + // PathExists tests if the given path already exists + // Error is returned on any other error than "file not found". + PathExists(pathname string) (bool, error) + // EvalHostSymlinks returns the path name after evaluating symlinks. + EvalHostSymlinks(pathname string) (string, error) + // GetOwner returns the integer ID for the user and group of the given path + GetOwner(pathname string) (int64, int64, error) + // GetSELinuxSupport returns true if given path is on a mount that supports + // SELinux. + GetSELinuxSupport(pathname string) (bool, error) + // GetMode returns permissions of the path. + GetMode(pathname string) (os.FileMode, error) +} + +// Compile-time check to ensure all HostUtil implementations satisfy +// the Interface. +var _ HostUtils = &HostUtil{} + +// getFileType checks for file/directory/socket and block/character devices. +func getFileType(pathname string) (FileType, error) { + var pathType FileType + info, err := os.Stat(pathname) + if os.IsNotExist(err) { + return pathType, fmt.Errorf("path %q does not exist", pathname) + } + // err in call to os.Stat + if err != nil { + return pathType, err + } + + // checks whether the mode is the target mode. + isSpecificMode := func(mode, targetMode os.FileMode) bool { + return mode&targetMode == targetMode + } + + mode := info.Mode() + if mode.IsDir() { + return FileTypeDirectory, nil + } else if mode.IsRegular() { + return FileTypeFile, nil + } else if isSpecificMode(mode, os.ModeSocket) { + return FileTypeSocket, nil + } else if isSpecificMode(mode, os.ModeDevice) { + if isSpecificMode(mode, os.ModeCharDevice) { + return FileTypeCharDev, nil + } + return FileTypeBlockDev, nil + } + + return pathType, fmt.Errorf("only recognise file, directory, socket, block device and character device") +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/hostutil_linux.go b/vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/hostutil_linux.go new file mode 100644 index 00000000..160d22c2 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/hostutil_linux.go @@ -0,0 +1,291 @@ +// +build linux + +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package hostutil + +import ( + "fmt" + "os" + "path" + "path/filepath" + "strings" + "syscall" + + "golang.org/x/sys/unix" + "k8s.io/klog/v2" + "k8s.io/mount-utils" + utilpath "k8s.io/utils/path" +) + +const ( + // Location of the mountinfo file + procMountInfoPath = "/proc/self/mountinfo" +) + +// HostUtil implements HostUtils for Linux platforms. +type HostUtil struct { +} + +// NewHostUtil returns a struct that implements the HostUtils interface on +// linux platforms +func NewHostUtil() *HostUtil { + return &HostUtil{} +} + +// DeviceOpened checks if block device in use by calling Open with O_EXCL flag. +// If pathname is not a device, log and return false with nil error. +// If open returns errno EBUSY, return true with nil error. +// If open returns nil, return false with nil error. +// Otherwise, return false with error +func (hu *HostUtil) DeviceOpened(pathname string) (bool, error) { + return ExclusiveOpenFailsOnDevice(pathname) +} + +// PathIsDevice uses FileInfo returned from os.Stat to check if path refers +// to a device. +func (hu *HostUtil) PathIsDevice(pathname string) (bool, error) { + pathType, err := hu.GetFileType(pathname) + isDevice := pathType == FileTypeCharDev || pathType == FileTypeBlockDev + return isDevice, err +} + +// ExclusiveOpenFailsOnDevice is shared with NsEnterMounter +func ExclusiveOpenFailsOnDevice(pathname string) (bool, error) { + var isDevice bool + finfo, err := os.Stat(pathname) + if os.IsNotExist(err) { + isDevice = false + } + // err in call to os.Stat + if err != nil { + return false, fmt.Errorf( + "PathIsDevice failed for path %q: %v", + pathname, + err) + } + // path refers to a device + if finfo.Mode()&os.ModeDevice != 0 { + isDevice = true + } + + if !isDevice { + klog.Errorf("Path %q is not referring to a device.", pathname) + return false, nil + } + fd, errno := unix.Open(pathname, unix.O_RDONLY|unix.O_EXCL|unix.O_CLOEXEC, 0) + // If the device is in use, open will return an invalid fd. + // When this happens, it is expected that Close will fail and throw an error. + defer unix.Close(fd) + if errno == nil { + // device not in use + return false, nil + } else if errno == unix.EBUSY { + // device is in use + return true, nil + } + // error during call to Open + return false, errno +} + +// GetDeviceNameFromMount given a mount point, find the device name from its global mount point +func (hu *HostUtil) GetDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) { + return getDeviceNameFromMount(mounter, mountPath, pluginMountDir) +} + +// getDeviceNameFromMountLinux find the device name from /proc/mounts in which +// the mount path reference should match the given plugin mount directory. In case no mount path reference +// matches, returns the volume name taken from its given mountPath +func getDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) { + refs, err := mounter.GetMountRefs(mountPath) + if err != nil { + klog.V(4).Infof("GetMountRefs failed for mount path %q: %v", mountPath, err) + return "", err + } + if len(refs) == 0 { + klog.V(4).Infof("Directory %s is not mounted", mountPath) + return "", fmt.Errorf("directory %s is not mounted", mountPath) + } + for _, ref := range refs { + if strings.HasPrefix(ref, pluginMountDir) { + volumeID, err := filepath.Rel(pluginMountDir, ref) + if err != nil { + klog.Errorf("Failed to get volume id from mount %s - %v", mountPath, err) + return "", err + } + return volumeID, nil + } + } + + return path.Base(mountPath), nil +} + +// MakeRShared checks that given path is on a mount with 'rshared' mount +// propagation. If not, it bind-mounts the path as rshared. +func (hu *HostUtil) MakeRShared(path string) error { + return DoMakeRShared(path, procMountInfoPath) +} + +// GetFileType checks for file/directory/socket/block/character devices. +func (hu *HostUtil) GetFileType(pathname string) (FileType, error) { + return getFileType(pathname) +} + +// PathExists tests if the given path already exists +// Error is returned on any other error than "file not found". +func (hu *HostUtil) PathExists(pathname string) (bool, error) { + return utilpath.Exists(utilpath.CheckFollowSymlink, pathname) +} + +// EvalHostSymlinks returns the path name after evaluating symlinks. +// TODO once the nsenter implementation is removed, this method can be removed +// from the interface and filepath.EvalSymlinks used directly +func (hu *HostUtil) EvalHostSymlinks(pathname string) (string, error) { + return filepath.EvalSymlinks(pathname) +} + +// FindMountInfo returns the mount info on the given path. +func (hu *HostUtil) FindMountInfo(path string) (mount.MountInfo, error) { + return findMountInfo(path, procMountInfoPath) +} + +// isShared returns true, if given path is on a mount point that has shared +// mount propagation. +func isShared(mount string, mountInfoPath string) (bool, error) { + info, err := findMountInfo(mount, mountInfoPath) + if err != nil { + return false, err + } + + // parse optional parameters + for _, opt := range info.OptionalFields { + if strings.HasPrefix(opt, "shared:") { + return true, nil + } + } + return false, nil +} + +func findMountInfo(path, mountInfoPath string) (mount.MountInfo, error) { + infos, err := mount.ParseMountInfo(mountInfoPath) + if err != nil { + return mount.MountInfo{}, err + } + + // process /proc/xxx/mountinfo in backward order and find the first mount + // point that is prefix of 'path' - that's the mount where path resides + var info *mount.MountInfo + for i := len(infos) - 1; i >= 0; i-- { + if mount.PathWithinBase(path, infos[i].MountPoint) { + info = &infos[i] + break + } + } + if info == nil { + return mount.MountInfo{}, fmt.Errorf("cannot find mount point for %q", path) + } + return *info, nil +} + +// DoMakeRShared is common implementation of MakeRShared on Linux. It checks if +// path is shared and bind-mounts it as rshared if needed. mountCmd and +// mountArgs are expected to contain mount-like command, DoMakeRShared will add +// '--bind ' and '--make-rshared ' to mountArgs. +func DoMakeRShared(path string, mountInfoFilename string) error { + shared, err := isShared(path, mountInfoFilename) + if err != nil { + return err + } + if shared { + klog.V(4).Infof("Directory %s is already on a shared mount", path) + return nil + } + + klog.V(2).Infof("Bind-mounting %q with shared mount propagation", path) + // mount --bind /var/lib/kubelet /var/lib/kubelet + if err := syscall.Mount(path, path, "" /*fstype*/, syscall.MS_BIND, "" /*data*/); err != nil { + return fmt.Errorf("failed to bind-mount %s: %v", path, err) + } + + // mount --make-rshared /var/lib/kubelet + if err := syscall.Mount(path, path, "" /*fstype*/, syscall.MS_SHARED|syscall.MS_REC, "" /*data*/); err != nil { + return fmt.Errorf("failed to make %s rshared: %v", path, err) + } + + return nil +} + +// GetSELinux is common implementation of GetSELinuxSupport on Linux. +func GetSELinux(path string, mountInfoFilename string) (bool, error) { + info, err := findMountInfo(path, mountInfoFilename) + if err != nil { + return false, err + } + + // "seclabel" can be both in mount options and super options. + for _, opt := range info.SuperOptions { + if opt == "seclabel" { + return true, nil + } + } + for _, opt := range info.MountOptions { + if opt == "seclabel" { + return true, nil + } + } + return false, nil +} + +// GetSELinuxSupport returns true if given path is on a mount that supports +// SELinux. +func (hu *HostUtil) GetSELinuxSupport(pathname string) (bool, error) { + return GetSELinux(pathname, procMountInfoPath) +} + +// GetOwner returns the integer ID for the user and group of the given path +func (hu *HostUtil) GetOwner(pathname string) (int64, int64, error) { + realpath, err := filepath.EvalSymlinks(pathname) + if err != nil { + return -1, -1, err + } + return GetOwnerLinux(realpath) +} + +// GetMode returns permissions of the path. +func (hu *HostUtil) GetMode(pathname string) (os.FileMode, error) { + return GetModeLinux(pathname) +} + +// GetOwnerLinux is shared between Linux and NsEnterMounter +// pathname must already be evaluated for symlinks +func GetOwnerLinux(pathname string) (int64, int64, error) { + info, err := os.Stat(pathname) + if err != nil { + return -1, -1, err + } + stat := info.Sys().(*syscall.Stat_t) + return int64(stat.Uid), int64(stat.Gid), nil +} + +// GetModeLinux is shared between Linux and NsEnterMounter +func GetModeLinux(pathname string) (os.FileMode, error) { + info, err := os.Stat(pathname) + if err != nil { + return 0, err + } + return info.Mode(), nil +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/hostutil_unsupported.go b/vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/hostutil_unsupported.go new file mode 100644 index 00000000..132ea3d0 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/hostutil_unsupported.go @@ -0,0 +1,102 @@ +// +build !linux,!windows + +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package hostutil + +import ( + "errors" + "os" + + "k8s.io/mount-utils" +) + +// HostUtil is an HostUtils implementation that allows compilation on +// unsupported platforms +type HostUtil struct{} + +// NewHostUtil returns a struct that implements the HostUtils interface on +// unsupported platforms +func NewHostUtil() *HostUtil { + return &HostUtil{} +} + +var errUnsupported = errors.New("volume/util/hostutil on this platform is not supported") + +// DeviceOpened always returns an error on unsupported platforms +func (hu *HostUtil) DeviceOpened(pathname string) (bool, error) { + return false, errUnsupported +} + +// PathIsDevice always returns an error on unsupported platforms +func (hu *HostUtil) PathIsDevice(pathname string) (bool, error) { + return true, errUnsupported +} + +// GetDeviceNameFromMount always returns an error on unsupported platforms +func (hu *HostUtil) GetDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) { + return getDeviceNameFromMount(mounter, mountPath, pluginMountDir) +} + +// MakeRShared always returns an error on unsupported platforms +func (hu *HostUtil) MakeRShared(path string) error { + return errUnsupported +} + +// GetFileType always returns an error on unsupported platforms +func (hu *HostUtil) GetFileType(pathname string) (FileType, error) { + return FileType("fake"), errUnsupported +} + +// MakeFile always returns an error on unsupported platforms +func (hu *HostUtil) MakeFile(pathname string) error { + return errUnsupported +} + +// MakeDir always returns an error on unsupported platforms +func (hu *HostUtil) MakeDir(pathname string) error { + return errUnsupported +} + +// PathExists always returns an error on unsupported platforms +func (hu *HostUtil) PathExists(pathname string) (bool, error) { + return true, errUnsupported +} + +// EvalHostSymlinks always returns an error on unsupported platforms +func (hu *HostUtil) EvalHostSymlinks(pathname string) (string, error) { + return "", errUnsupported +} + +// GetOwner always returns an error on unsupported platforms +func (hu *HostUtil) GetOwner(pathname string) (int64, int64, error) { + return -1, -1, errUnsupported +} + +// GetSELinuxSupport always returns an error on unsupported platforms +func (hu *HostUtil) GetSELinuxSupport(pathname string) (bool, error) { + return false, errUnsupported +} + +//GetMode always returns an error on unsupported platforms +func (hu *HostUtil) GetMode(pathname string) (os.FileMode, error) { + return 0, errUnsupported +} + +func getDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) { + return "", errUnsupported +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/hostutil_windows.go b/vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/hostutil_windows.go new file mode 100644 index 00000000..bd44dc54 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/hostutil_windows.go @@ -0,0 +1,124 @@ +// +build windows + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package hostutil + +import ( + "fmt" + "os" + "path" + "path/filepath" + "strings" + + "k8s.io/klog/v2" + "k8s.io/mount-utils" + utilpath "k8s.io/utils/path" +) + +// HostUtil implements HostUtils for Windows platforms. +type HostUtil struct{} + +// NewHostUtil returns a struct that implements HostUtils on Windows platforms +func NewHostUtil() *HostUtil { + return &HostUtil{} +} + +// GetDeviceNameFromMount given a mnt point, find the device +func (hu *HostUtil) GetDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) { + return getDeviceNameFromMount(mounter, mountPath, pluginMountDir) +} + +// getDeviceNameFromMount find the device(drive) name in which +// the mount path reference should match the given plugin mount directory. In case no mount path reference +// matches, returns the volume name taken from its given mountPath +func getDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) { + refs, err := mounter.GetMountRefs(mountPath) + if err != nil { + klog.V(4).Infof("GetMountRefs failed for mount path %q: %v", mountPath, err) + return "", err + } + if len(refs) == 0 { + return "", fmt.Errorf("directory %s is not mounted", mountPath) + } + basemountPath := mount.NormalizeWindowsPath(pluginMountDir) + for _, ref := range refs { + if strings.Contains(ref, basemountPath) { + volumeID, err := filepath.Rel(mount.NormalizeWindowsPath(basemountPath), ref) + if err != nil { + klog.Errorf("Failed to get volume id from mount %s - %v", mountPath, err) + return "", err + } + return volumeID, nil + } + } + + return path.Base(mountPath), nil +} + +// DeviceOpened determines if the device is in use elsewhere +func (hu *HostUtil) DeviceOpened(pathname string) (bool, error) { + return false, nil +} + +// PathIsDevice determines if a path is a device. +func (hu *HostUtil) PathIsDevice(pathname string) (bool, error) { + return false, nil +} + +// MakeRShared checks that given path is on a mount with 'rshared' mount +// propagation. Empty implementation here. +func (hu *HostUtil) MakeRShared(path string) error { + return nil +} + +// GetFileType checks for sockets/block/character devices +func (hu *(HostUtil)) GetFileType(pathname string) (FileType, error) { + return getFileType(pathname) +} + +// PathExists checks whether the path exists +func (hu *HostUtil) PathExists(pathname string) (bool, error) { + return utilpath.Exists(utilpath.CheckFollowSymlink, pathname) +} + +// EvalHostSymlinks returns the path name after evaluating symlinks +func (hu *HostUtil) EvalHostSymlinks(pathname string) (string, error) { + return filepath.EvalSymlinks(pathname) +} + +// GetOwner returns the integer ID for the user and group of the given path +// Note that on windows, it always returns 0. We actually don't set Group on +// windows platform, see SetVolumeOwnership implementation. +func (hu *HostUtil) GetOwner(pathname string) (int64, int64, error) { + return -1, -1, nil +} + +// GetSELinuxSupport returns a boolean indicating support for SELinux. +// Windows does not support SELinux. +func (hu *HostUtil) GetSELinuxSupport(pathname string) (bool, error) { + return false, nil +} + +// GetMode returns permissions of the path. +func (hu *HostUtil) GetMode(pathname string) (os.FileMode, error) { + info, err := os.Stat(pathname) + if err != nil { + return 0, err + } + return info.Mode(), nil +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/recyclerclient/BUILD b/vendor/k8s.io/kubernetes/pkg/volume/util/recyclerclient/BUILD new file mode 100644 index 00000000..54fde842 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/recyclerclient/BUILD @@ -0,0 +1,44 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["recycler_client.go"], + importpath = "k8s.io/kubernetes/pkg/volume/util/recyclerclient", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//vendor/k8s.io/klog/v2:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["recycler_client_test.go"], + embed = [":go_default_library"], + deps = [ + "//pkg/apis/core:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/recyclerclient/recycler_client.go b/vendor/k8s.io/kubernetes/pkg/volume/util/recyclerclient/recycler_client.go new file mode 100644 index 00000000..2afbc9a5 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/recyclerclient/recycler_client.go @@ -0,0 +1,266 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recyclerclient + +import ( + "context" + "fmt" + "sync" + + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/watch" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" +) + +// RecycleEventRecorder is a func that defines how to record RecycleEvent. +type RecycleEventRecorder func(eventtype, message string) + +// RecycleVolumeByWatchingPodUntilCompletion is intended for use with volume +// Recyclers. This function will save the given Pod to the API and watch it +// until it completes, fails, or the pod's ActiveDeadlineSeconds is exceeded, +// whichever comes first. An attempt to delete a recycler pod is always +// attempted before returning. +// +// In case there is a pod with the same namespace+name already running, this +// function deletes it as it is not able to judge if it is an old recycler +// or user has forged a fake recycler to block Kubernetes from recycling.// +// +// pod - the pod designed by a volume plugin to recycle the volume. pod.Name +// will be overwritten with unique name based on PV.Name. +// client - kube client for API operations. +func RecycleVolumeByWatchingPodUntilCompletion(pvName string, pod *v1.Pod, kubeClient clientset.Interface, recorder RecycleEventRecorder) error { + return internalRecycleVolumeByWatchingPodUntilCompletion(pvName, pod, newRecyclerClient(kubeClient, recorder)) +} + +// same as above func comments, except 'recyclerClient' is a narrower pod API +// interface to ease testing +func internalRecycleVolumeByWatchingPodUntilCompletion(pvName string, pod *v1.Pod, recyclerClient recyclerClient) error { + klog.V(5).Infof("creating recycler pod for volume %s\n", pod.Name) + + // Generate unique name for the recycler pod - we need to get "already + // exists" error when a previous controller has already started recycling + // the volume. Here we assume that pv.Name is already unique. + pod.Name = "recycler-for-" + pvName + pod.GenerateName = "" + + stopChannel := make(chan struct{}) + defer close(stopChannel) + podCh, err := recyclerClient.WatchPod(pod.Name, pod.Namespace, stopChannel) + if err != nil { + klog.V(4).Infof("cannot start watcher for pod %s/%s: %v", pod.Namespace, pod.Name, err) + return err + } + + // Start the pod + _, err = recyclerClient.CreatePod(pod) + if err != nil { + if errors.IsAlreadyExists(err) { + deleteErr := recyclerClient.DeletePod(pod.Name, pod.Namespace) + if deleteErr != nil { + return fmt.Errorf("failed to delete old recycler pod %s/%s: %s", pod.Namespace, pod.Name, deleteErr) + } + // Recycler will try again and the old pod will be hopefully deleted + // at that time. + return fmt.Errorf("old recycler pod found, will retry later") + } + return fmt.Errorf("unexpected error creating recycler pod: %+v", err) + } + err = waitForPod(pod, recyclerClient, podCh) + + // In all cases delete the recycler pod and log its result. + klog.V(2).Infof("deleting recycler pod %s/%s", pod.Namespace, pod.Name) + deleteErr := recyclerClient.DeletePod(pod.Name, pod.Namespace) + if deleteErr != nil { + klog.Errorf("failed to delete recycler pod %s/%s: %v", pod.Namespace, pod.Name, err) + } + + // Returning recycler error is preferred, the pod will be deleted again on + // the next retry. + if err != nil { + return fmt.Errorf("failed to recycle volume: %s", err) + } + + // Recycle succeeded but we failed to delete the recycler pod. Report it, + // the controller will re-try recycling the PV again shortly. + if deleteErr != nil { + return fmt.Errorf("failed to delete recycler pod: %s", deleteErr) + } + + return nil +} + +// waitForPod watches the pod it until it finishes and send all events on the +// pod to the PV. +func waitForPod(pod *v1.Pod, recyclerClient recyclerClient, podCh <-chan watch.Event) error { + for { + event, ok := <-podCh + if !ok { + return fmt.Errorf("recycler pod %q watch channel had been closed", pod.Name) + } + switch event.Object.(type) { + case *v1.Pod: + // POD changed + pod := event.Object.(*v1.Pod) + klog.V(4).Infof("recycler pod update received: %s %s/%s %s", event.Type, pod.Namespace, pod.Name, pod.Status.Phase) + switch event.Type { + case watch.Added, watch.Modified: + if pod.Status.Phase == v1.PodSucceeded { + // Recycle succeeded. + return nil + } + if pod.Status.Phase == v1.PodFailed { + if pod.Status.Message != "" { + return fmt.Errorf(pod.Status.Message) + } + return fmt.Errorf("pod failed, pod.Status.Message unknown") + } + + case watch.Deleted: + return fmt.Errorf("recycler pod was deleted") + + case watch.Error: + return fmt.Errorf("recycler pod watcher failed") + } + + case *v1.Event: + // Event received + podEvent := event.Object.(*v1.Event) + klog.V(4).Infof("recycler event received: %s %s/%s %s/%s %s", event.Type, podEvent.Namespace, podEvent.Name, podEvent.InvolvedObject.Namespace, podEvent.InvolvedObject.Name, podEvent.Message) + if event.Type == watch.Added { + recyclerClient.Event(podEvent.Type, podEvent.Message) + } + } + } +} + +// recyclerClient abstracts access to a Pod by providing a narrower interface. +// This makes it easier to mock a client for testing. +type recyclerClient interface { + CreatePod(pod *v1.Pod) (*v1.Pod, error) + GetPod(name, namespace string) (*v1.Pod, error) + DeletePod(name, namespace string) error + // WatchPod returns a ListWatch for watching a pod. The stopChannel is used + // to close the reflector backing the watch. The caller is responsible for + // derring a close on the channel to stop the reflector. + WatchPod(name, namespace string, stopChannel chan struct{}) (<-chan watch.Event, error) + // Event sends an event to the volume that is being recycled. + Event(eventtype, message string) +} + +func newRecyclerClient(client clientset.Interface, recorder RecycleEventRecorder) recyclerClient { + return &realRecyclerClient{ + client, + recorder, + } +} + +type realRecyclerClient struct { + client clientset.Interface + recorder RecycleEventRecorder +} + +func (c *realRecyclerClient) CreatePod(pod *v1.Pod) (*v1.Pod, error) { + return c.client.CoreV1().Pods(pod.Namespace).Create(context.TODO(), pod, metav1.CreateOptions{}) +} + +func (c *realRecyclerClient) GetPod(name, namespace string) (*v1.Pod, error) { + return c.client.CoreV1().Pods(namespace).Get(context.TODO(), name, metav1.GetOptions{}) +} + +func (c *realRecyclerClient) DeletePod(name, namespace string) error { + return c.client.CoreV1().Pods(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) +} + +func (c *realRecyclerClient) Event(eventtype, message string) { + c.recorder(eventtype, message) +} + +// WatchPod watches a pod and events related to it. It sends pod updates and events over the returned channel +// It will continue until stopChannel is closed +func (c *realRecyclerClient) WatchPod(name, namespace string, stopChannel chan struct{}) (<-chan watch.Event, error) { + podSelector, err := fields.ParseSelector("metadata.name=" + name) + if err != nil { + return nil, err + } + options := metav1.ListOptions{ + FieldSelector: podSelector.String(), + Watch: true, + } + + podWatch, err := c.client.CoreV1().Pods(namespace).Watch(context.TODO(), options) + if err != nil { + return nil, err + } + + eventSelector, _ := fields.ParseSelector("involvedObject.name=" + name) + eventWatch, err := c.client.CoreV1().Events(namespace).Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: eventSelector.String(), + Watch: true, + }) + if err != nil { + podWatch.Stop() + return nil, err + } + + eventCh := make(chan watch.Event, 30) + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer close(eventCh) + wg.Wait() + }() + + go func() { + defer eventWatch.Stop() + defer wg.Done() + for { + select { + case <-stopChannel: + return + case eventEvent, ok := <-eventWatch.ResultChan(): + if !ok { + return + } + eventCh <- eventEvent + } + } + }() + + go func() { + defer podWatch.Stop() + defer wg.Done() + for { + select { + case <-stopChannel: + return + + case podEvent, ok := <-podWatch.ResultChan(): + if !ok { + return + } + eventCh <- podEvent + } + } + }() + + return eventCh, nil +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/subpath/BUILD b/vendor/k8s.io/kubernetes/pkg/volume/util/subpath/BUILD new file mode 100644 index 00000000..37908f61 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/subpath/BUILD @@ -0,0 +1,118 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "subpath.go", + "subpath_linux.go", + "subpath_unsupported.go", + "subpath_windows.go", + ], + importpath = "k8s.io/kubernetes/pkg/volume/util/subpath", + visibility = ["//visibility:public"], + deps = select({ + "@io_bazel_rules_go//go/platform:aix": [ + "//staging/src/k8s.io/mount-utils:go_default_library", + "//vendor/k8s.io/utils/nsenter:go_default_library", + ], + "@io_bazel_rules_go//go/platform:android": [ + "//pkg/volume/util/hostutil:go_default_library", + "//staging/src/k8s.io/mount-utils:go_default_library", + "//vendor/golang.org/x/sys/unix:go_default_library", + "//vendor/k8s.io/klog/v2:go_default_library", + ], + "@io_bazel_rules_go//go/platform:darwin": [ + "//staging/src/k8s.io/mount-utils:go_default_library", + "//vendor/k8s.io/utils/nsenter:go_default_library", + ], + "@io_bazel_rules_go//go/platform:dragonfly": [ + "//staging/src/k8s.io/mount-utils:go_default_library", + "//vendor/k8s.io/utils/nsenter:go_default_library", + ], + "@io_bazel_rules_go//go/platform:freebsd": [ + "//staging/src/k8s.io/mount-utils:go_default_library", + "//vendor/k8s.io/utils/nsenter:go_default_library", + ], + "@io_bazel_rules_go//go/platform:illumos": [ + "//staging/src/k8s.io/mount-utils:go_default_library", + "//vendor/k8s.io/utils/nsenter:go_default_library", + ], + "@io_bazel_rules_go//go/platform:ios": [ + "//staging/src/k8s.io/mount-utils:go_default_library", + "//vendor/k8s.io/utils/nsenter:go_default_library", + ], + "@io_bazel_rules_go//go/platform:js": [ + "//staging/src/k8s.io/mount-utils:go_default_library", + "//vendor/k8s.io/utils/nsenter:go_default_library", + ], + "@io_bazel_rules_go//go/platform:linux": [ + "//pkg/volume/util/hostutil:go_default_library", + "//staging/src/k8s.io/mount-utils:go_default_library", + "//vendor/golang.org/x/sys/unix:go_default_library", + "//vendor/k8s.io/klog/v2:go_default_library", + ], + "@io_bazel_rules_go//go/platform:nacl": [ + "//staging/src/k8s.io/mount-utils:go_default_library", + "//vendor/k8s.io/utils/nsenter:go_default_library", + ], + "@io_bazel_rules_go//go/platform:netbsd": [ + "//staging/src/k8s.io/mount-utils:go_default_library", + "//vendor/k8s.io/utils/nsenter:go_default_library", + ], + "@io_bazel_rules_go//go/platform:openbsd": [ + "//staging/src/k8s.io/mount-utils:go_default_library", + "//vendor/k8s.io/utils/nsenter:go_default_library", + ], + "@io_bazel_rules_go//go/platform:plan9": [ + "//staging/src/k8s.io/mount-utils:go_default_library", + "//vendor/k8s.io/utils/nsenter:go_default_library", + ], + "@io_bazel_rules_go//go/platform:solaris": [ + "//staging/src/k8s.io/mount-utils:go_default_library", + "//vendor/k8s.io/utils/nsenter:go_default_library", + ], + "@io_bazel_rules_go//go/platform:windows": [ + "//staging/src/k8s.io/mount-utils:go_default_library", + "//vendor/k8s.io/klog/v2:go_default_library", + "//vendor/k8s.io/utils/nsenter:go_default_library", + ], + "//conditions:default": [], + }), +) + +go_test( + name = "go_default_test", + srcs = [ + "subpath_linux_test.go", + "subpath_windows_test.go", + ], + embed = [":go_default_library"], + deps = select({ + "@io_bazel_rules_go//go/platform:android": [ + "//staging/src/k8s.io/mount-utils:go_default_library", + "//vendor/k8s.io/klog/v2:go_default_library", + ], + "@io_bazel_rules_go//go/platform:linux": [ + "//staging/src/k8s.io/mount-utils:go_default_library", + "//vendor/k8s.io/klog/v2:go_default_library", + ], + "@io_bazel_rules_go//go/platform:windows": [ + "//vendor/github.com/stretchr/testify/assert:go_default_library", + ], + "//conditions:default": [], + }), +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/subpath/OWNERS b/vendor/k8s.io/kubernetes/pkg/volume/util/subpath/OWNERS new file mode 100644 index 00000000..c8821ce7 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/subpath/OWNERS @@ -0,0 +1,13 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +reviewers: + - jingxu97 + - saad-ali + - jsafrane + - msau42 + - andyzhangx +approvers: + - jingxu97 + - saad-ali + - jsafrane + diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath.go b/vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath.go new file mode 100644 index 00000000..c843f6cf --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath.go @@ -0,0 +1,92 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package subpath + +import "os" + +// Interface defines the set of methods all subpathers must implement +type Interface interface { + // CleanSubPaths removes any bind-mounts created by PrepareSafeSubpath in given + // pod volume directory. + CleanSubPaths(poodDir string, volumeName string) error + + // PrepareSafeSubpath does everything that's necessary to prepare a subPath + // that's 1) inside given volumePath and 2) immutable after this call. + // + // newHostPath - location of prepared subPath. It should be used instead of + // hostName when running the container. + // cleanupAction - action to run when the container is running or it failed to start. + // + // CleanupAction must be called immediately after the container with given + // subpath starts. On the other hand, Interface.CleanSubPaths must be called + // when the pod finishes. + PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) + + // SafeMakeDir creates subdir within given base. It makes sure that the + // created directory does not escape given base directory mis-using + // symlinks. Note that the function makes sure that it creates the directory + // somewhere under the base, nothing else. E.g. if the directory already + // exists, it may exist outside of the base due to symlinks. + // This method should be used if the directory to create is inside volume + // that's under user control. User must not be able to use symlinks to + // escape the volume to create directories somewhere else. + SafeMakeDir(subdir string, base string, perm os.FileMode) error +} + +// Subpath defines the attributes of a subpath +type Subpath struct { + // index of the VolumeMount for this container + VolumeMountIndex int + + // Full path to the subpath directory on the host + Path string + + // name of the volume that is a valid directory name. + VolumeName string + + // Full path to the volume path + VolumePath string + + // Path to the pod's directory, including pod UID + PodDir string + + // Name of the container + ContainerName string +} + +// Compile time-check for all implementers of subpath interface +var _ Interface = &subpath{} +var _ Interface = &FakeSubpath{} + +// FakeSubpath is a subpather implementation for testing +type FakeSubpath struct{} + +// PrepareSafeSubpath is a fake implementation of PrepareSafeSubpath. Always returns +// newHostPath == subPath.Path +func (fs *FakeSubpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { + return subPath.Path, nil, nil +} + +// CleanSubPaths is a fake implementation of CleanSubPaths. It is a noop +func (fs *FakeSubpath) CleanSubPaths(podDir string, volumeName string) error { + return nil +} + +// SafeMakeDir is a fake implementation of SafeMakeDir. It is a noop +func (fs *FakeSubpath) SafeMakeDir(pathname string, base string, perm os.FileMode) error { + return nil +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_linux.go b/vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_linux.go new file mode 100644 index 00000000..c04f0a78 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_linux.go @@ -0,0 +1,581 @@ +// +build linux + +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package subpath + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + "syscall" + + "golang.org/x/sys/unix" + "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/volume/util/hostutil" + "k8s.io/mount-utils" +) + +const ( + // place for subpath mounts + // TODO: pass in directory using kubelet_getters instead + containerSubPathDirectoryName = "volume-subpaths" + // syscall.Openat flags used to traverse directories not following symlinks + nofollowFlags = unix.O_RDONLY | unix.O_NOFOLLOW + // flags for getting file descriptor without following the symlink + openFDFlags = unix.O_NOFOLLOW | unix.O_PATH +) + +type subpath struct { + mounter mount.Interface +} + +// New returns a subpath.Interface for the current system +func New(mounter mount.Interface) Interface { + return &subpath{ + mounter: mounter, + } +} + +func (sp *subpath) CleanSubPaths(podDir string, volumeName string) error { + return doCleanSubPaths(sp.mounter, podDir, volumeName) +} + +func (sp *subpath) SafeMakeDir(subdir string, base string, perm os.FileMode) error { + realBase, err := filepath.EvalSymlinks(base) + if err != nil { + return fmt.Errorf("error resolving symlinks in %s: %s", base, err) + } + + realFullPath := filepath.Join(realBase, subdir) + + return doSafeMakeDir(realFullPath, realBase, perm) +} + +func (sp *subpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { + newHostPath, err = doBindSubPath(sp.mounter, subPath) + + // There is no action when the container starts. Bind-mount will be cleaned + // when container stops by CleanSubPaths. + cleanupAction = nil + return newHostPath, cleanupAction, err +} + +// This implementation is shared between Linux and NsEnter +func safeOpenSubPath(mounter mount.Interface, subpath Subpath) (int, error) { + if !mount.PathWithinBase(subpath.Path, subpath.VolumePath) { + return -1, fmt.Errorf("subpath %q not within volume path %q", subpath.Path, subpath.VolumePath) + } + fd, err := doSafeOpen(subpath.Path, subpath.VolumePath) + if err != nil { + return -1, fmt.Errorf("error opening subpath %v: %v", subpath.Path, err) + } + return fd, nil +} + +// prepareSubpathTarget creates target for bind-mount of subpath. It returns +// "true" when the target already exists and something is mounted there. +// Given Subpath must have all paths with already resolved symlinks and with +// paths relevant to kubelet (when it runs in a container). +// This function is called also by NsEnterMounter. It works because +// /var/lib/kubelet is mounted from the host into the container with Kubelet as +// /var/lib/kubelet too. +func prepareSubpathTarget(mounter mount.Interface, subpath Subpath) (bool, string, error) { + // Early check for already bind-mounted subpath. + bindPathTarget := getSubpathBindTarget(subpath) + notMount, err := mount.IsNotMountPoint(mounter, bindPathTarget) + if err != nil { + if !os.IsNotExist(err) { + return false, "", fmt.Errorf("error checking path %s for mount: %s", bindPathTarget, err) + } + // Ignore ErrorNotExist: the file/directory will be created below if it does not exist yet. + notMount = true + } + if !notMount { + linuxHostUtil := hostutil.NewHostUtil() + mntInfo, err := linuxHostUtil.FindMountInfo(bindPathTarget) + if err != nil { + return false, "", fmt.Errorf("error calling findMountInfo for %s: %s", bindPathTarget, err) + } + if mntInfo.Root != subpath.Path { + // It's already mounted but not what we want, unmount it + if err = mounter.Unmount(bindPathTarget); err != nil { + return false, "", fmt.Errorf("error ummounting %s: %s", bindPathTarget, err) + } + } else { + // It's already mounted + klog.V(5).Infof("Skipping bind-mounting subpath %s: already mounted", bindPathTarget) + return true, bindPathTarget, nil + } + } + + // bindPathTarget is in /var/lib/kubelet and thus reachable without any + // translation even to containerized kubelet. + bindParent := filepath.Dir(bindPathTarget) + err = os.MkdirAll(bindParent, 0750) + if err != nil && !os.IsExist(err) { + return false, "", fmt.Errorf("error creating directory %s: %s", bindParent, err) + } + + t, err := os.Lstat(subpath.Path) + if err != nil { + return false, "", fmt.Errorf("lstat %s failed: %s", subpath.Path, err) + } + + if t.Mode()&os.ModeDir > 0 { + if err = os.Mkdir(bindPathTarget, 0750); err != nil && !os.IsExist(err) { + return false, "", fmt.Errorf("error creating directory %s: %s", bindPathTarget, err) + } + } else { + // "/bin/touch ". + // A file is enough for all possible targets (symlink, device, pipe, + // socket, ...), bind-mounting them into a file correctly changes type + // of the target file. + if err = ioutil.WriteFile(bindPathTarget, []byte{}, 0640); err != nil { + return false, "", fmt.Errorf("error creating file %s: %s", bindPathTarget, err) + } + } + return false, bindPathTarget, nil +} + +func getSubpathBindTarget(subpath Subpath) string { + // containerName is DNS label, i.e. safe as a directory name. + return filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName, strconv.Itoa(subpath.VolumeMountIndex)) +} + +func doBindSubPath(mounter mount.Interface, subpath Subpath) (hostPath string, err error) { + // Linux, kubelet runs on the host: + // - safely open the subpath + // - bind-mount /proc//fd/ to subpath target + // User can't change /proc//fd/ to point to a bad place. + + // Evaluate all symlinks here once for all subsequent functions. + newVolumePath, err := filepath.EvalSymlinks(subpath.VolumePath) + if err != nil { + return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err) + } + newPath, err := filepath.EvalSymlinks(subpath.Path) + if err != nil { + return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err) + } + klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, newPath, subpath.VolumePath) + subpath.VolumePath = newVolumePath + subpath.Path = newPath + + fd, err := safeOpenSubPath(mounter, subpath) + if err != nil { + return "", err + } + defer syscall.Close(fd) + + alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath) + if err != nil { + return "", err + } + if alreadyMounted { + return bindPathTarget, nil + } + + success := false + defer func() { + // Cleanup subpath on error + if !success { + klog.V(4).Infof("doBindSubPath() failed for %q, cleaning up subpath", bindPathTarget) + if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil { + klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr) + } + } + }() + + kubeletPid := os.Getpid() + mountSource := fmt.Sprintf("/proc/%d/fd/%v", kubeletPid, fd) + + // Do the bind mount + options := []string{"bind"} + klog.V(5).Infof("bind mounting %q at %q", mountSource, bindPathTarget) + if err = mounter.MountSensitiveWithoutSystemd(mountSource, bindPathTarget, "" /*fstype*/, options, nil); err != nil { + return "", fmt.Errorf("error mounting %s: %s", subpath.Path, err) + } + success = true + + klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget) + return bindPathTarget, nil +} + +// This implementation is shared between Linux and NsEnter +func doCleanSubPaths(mounter mount.Interface, podDir string, volumeName string) error { + // scan /var/lib/kubelet/pods//volume-subpaths//* + subPathDir := filepath.Join(podDir, containerSubPathDirectoryName, volumeName) + klog.V(4).Infof("Cleaning up subpath mounts for %s", subPathDir) + + containerDirs, err := ioutil.ReadDir(subPathDir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("error reading %s: %s", subPathDir, err) + } + + for _, containerDir := range containerDirs { + if !containerDir.IsDir() { + klog.V(4).Infof("Container file is not a directory: %s", containerDir.Name()) + continue + } + klog.V(4).Infof("Cleaning up subpath mounts for container %s", containerDir.Name()) + + // scan /var/lib/kubelet/pods//volume-subpaths///* + fullContainerDirPath := filepath.Join(subPathDir, containerDir.Name()) + err = filepath.Walk(fullContainerDirPath, func(path string, info os.FileInfo, _ error) error { + if path == fullContainerDirPath { + // Skip top level directory + return nil + } + + // pass through errors and let doCleanSubPath handle them + if err = doCleanSubPath(mounter, fullContainerDirPath, filepath.Base(path)); err != nil { + return err + } + + // We need to check that info is not nil. This may happen when the incoming err is not nil due to stale mounts or permission errors. + if info != nil && info.IsDir() { + // skip subdirs of the volume: it only matters the first level to unmount, otherwise it would try to unmount subdir of the volume + return filepath.SkipDir + } + + return nil + }) + if err != nil { + return fmt.Errorf("error processing %s: %s", fullContainerDirPath, err) + } + + // Whole container has been processed, remove its directory. + if err := os.Remove(fullContainerDirPath); err != nil { + return fmt.Errorf("error deleting %s: %s", fullContainerDirPath, err) + } + klog.V(5).Infof("Removed %s", fullContainerDirPath) + } + // Whole pod volume subpaths have been cleaned up, remove its subpath directory. + if err := os.Remove(subPathDir); err != nil { + return fmt.Errorf("error deleting %s: %s", subPathDir, err) + } + klog.V(5).Infof("Removed %s", subPathDir) + + // Remove entire subpath directory if it's the last one + podSubPathDir := filepath.Join(podDir, containerSubPathDirectoryName) + if err := os.Remove(podSubPathDir); err != nil && !os.IsExist(err) { + return fmt.Errorf("error deleting %s: %s", podSubPathDir, err) + } + klog.V(5).Infof("Removed %s", podSubPathDir) + return nil +} + +// doCleanSubPath tears down the single subpath bind mount +func doCleanSubPath(mounter mount.Interface, fullContainerDirPath, subPathIndex string) error { + // process /var/lib/kubelet/pods//volume-subpaths/// + klog.V(4).Infof("Cleaning up subpath mounts for subpath %v", subPathIndex) + fullSubPath := filepath.Join(fullContainerDirPath, subPathIndex) + + if err := mount.CleanupMountPoint(fullSubPath, mounter, true); err != nil { + return fmt.Errorf("error cleaning subpath mount %s: %s", fullSubPath, err) + } + + klog.V(4).Infof("Successfully cleaned subpath directory %s", fullSubPath) + return nil +} + +// cleanSubPath will teardown the subpath bind mount and any remove any directories if empty +func cleanSubPath(mounter mount.Interface, subpath Subpath) error { + containerDir := filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName) + + // Clean subdir bindmount + if err := doCleanSubPath(mounter, containerDir, strconv.Itoa(subpath.VolumeMountIndex)); err != nil && !os.IsNotExist(err) { + return err + } + + // Recusively remove directories if empty + if err := removeEmptyDirs(subpath.PodDir, containerDir); err != nil { + return err + } + + return nil +} + +// removeEmptyDirs works backwards from endDir to baseDir and removes each directory +// if it is empty. It stops once it encounters a directory that has content +func removeEmptyDirs(baseDir, endDir string) error { + if !mount.PathWithinBase(endDir, baseDir) { + return fmt.Errorf("endDir %q is not within baseDir %q", endDir, baseDir) + } + + for curDir := endDir; curDir != baseDir; curDir = filepath.Dir(curDir) { + s, err := os.Stat(curDir) + if err != nil { + if os.IsNotExist(err) { + klog.V(5).Infof("curDir %q doesn't exist, skipping", curDir) + continue + } + return fmt.Errorf("error stat %q: %v", curDir, err) + } + if !s.IsDir() { + return fmt.Errorf("path %q not a directory", curDir) + } + + err = os.Remove(curDir) + if os.IsExist(err) { + klog.V(5).Infof("Directory %q not empty, not removing", curDir) + break + } else if err != nil { + return fmt.Errorf("error removing directory %q: %v", curDir, err) + } + klog.V(5).Infof("Removed directory %q", curDir) + } + return nil +} + +// This implementation is shared between Linux and NsEnterMounter. Both pathname +// and base must be either already resolved symlinks or thet will be resolved in +// kubelet's mount namespace (in case it runs containerized). +func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { + klog.V(4).Infof("Creating directory %q within base %q", pathname, base) + + if !mount.PathWithinBase(pathname, base) { + return fmt.Errorf("path %s is outside of allowed base %s", pathname, base) + } + + // Quick check if the directory already exists + s, err := os.Stat(pathname) + if err == nil { + // Path exists + if s.IsDir() { + // The directory already exists. It can be outside of the parent, + // but there is no race-proof check. + klog.V(4).Infof("Directory %s already exists", pathname) + return nil + } + return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR} + } + + // Find all existing directories + existingPath, toCreate, err := findExistingPrefix(base, pathname) + if err != nil { + return fmt.Errorf("error opening directory %s: %s", pathname, err) + } + // Ensure the existing directory is inside allowed base + fullExistingPath, err := filepath.EvalSymlinks(existingPath) + if err != nil { + return fmt.Errorf("error opening directory %s: %s", existingPath, err) + } + if !mount.PathWithinBase(fullExistingPath, base) { + return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err) + } + + klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...)) + parentFD, err := doSafeOpen(fullExistingPath, base) + if err != nil { + return fmt.Errorf("cannot open directory %s: %s", existingPath, err) + } + childFD := -1 + defer func() { + if parentFD != -1 { + if err = syscall.Close(parentFD); err != nil { + klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err) + } + } + if childFD != -1 { + if err = syscall.Close(childFD); err != nil { + klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", childFD, pathname, err) + } + } + }() + + currentPath := fullExistingPath + // create the directories one by one, making sure nobody can change + // created directory into symlink. + for _, dir := range toCreate { + currentPath = filepath.Join(currentPath, dir) + klog.V(4).Infof("Creating %s", dir) + err = syscall.Mkdirat(parentFD, currentPath, uint32(perm)) + if err != nil { + return fmt.Errorf("cannot create directory %s: %s", currentPath, err) + } + // Dive into the created directory + childFD, err = syscall.Openat(parentFD, dir, nofollowFlags|unix.O_CLOEXEC, 0) + if err != nil { + return fmt.Errorf("cannot open %s: %s", currentPath, err) + } + // We can be sure that childFD is safe to use. It could be changed + // by user after Mkdirat() and before Openat(), however: + // - it could not be changed to symlink - we use nofollowFlags + // - it could be changed to a file (or device, pipe, socket, ...) + // but either subsequent Mkdirat() fails or we mount this file + // to user's container. Security is no violated in both cases + // and user either gets error or the file that it can already access. + + if err = syscall.Close(parentFD); err != nil { + klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err) + } + parentFD = childFD + childFD = -1 + } + + // Everything was created. mkdirat(..., perm) above was affected by current + // umask and we must apply the right permissions to the last directory + // (that's the one that will be available to the container as subpath) + // so user can read/write it. This is the behavior of previous code. + // TODO: chmod all created directories, not just the last one. + // parentFD is the last created directory. + + // Translate perm (os.FileMode) to uint32 that fchmod() expects + kernelPerm := uint32(perm & os.ModePerm) + if perm&os.ModeSetgid > 0 { + kernelPerm |= syscall.S_ISGID + } + if perm&os.ModeSetuid > 0 { + kernelPerm |= syscall.S_ISUID + } + if perm&os.ModeSticky > 0 { + kernelPerm |= syscall.S_ISVTX + } + if err = syscall.Fchmod(parentFD, kernelPerm); err != nil { + return fmt.Errorf("chmod %q failed: %s", currentPath, err) + } + return nil +} + +// findExistingPrefix finds prefix of pathname that exists. In addition, it +// returns list of remaining directories that don't exist yet. +func findExistingPrefix(base, pathname string) (string, []string, error) { + rel, err := filepath.Rel(base, pathname) + if err != nil { + return base, nil, err + } + dirs := strings.Split(rel, string(filepath.Separator)) + + // Do OpenAt in a loop to find the first non-existing dir. Resolve symlinks. + // This should be faster than looping through all dirs and calling os.Stat() + // on each of them, as the symlinks are resolved only once with OpenAt(). + currentPath := base + fd, err := syscall.Open(currentPath, syscall.O_RDONLY|syscall.O_CLOEXEC, 0) + if err != nil { + return pathname, nil, fmt.Errorf("error opening %s: %s", currentPath, err) + } + defer func() { + if err = syscall.Close(fd); err != nil { + klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err) + } + }() + for i, dir := range dirs { + // Using O_PATH here will prevent hangs in case user replaces directory with + // fifo + childFD, err := syscall.Openat(fd, dir, unix.O_PATH|unix.O_CLOEXEC, 0) + if err != nil { + if os.IsNotExist(err) { + return currentPath, dirs[i:], nil + } + return base, nil, err + } + if err = syscall.Close(fd); err != nil { + klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err) + } + fd = childFD + currentPath = filepath.Join(currentPath, dir) + } + return pathname, []string{}, nil +} + +// This implementation is shared between Linux and NsEnterMounter +// Open path and return its fd. +// Symlinks are disallowed (pathname must already resolve symlinks), +// and the path must be within the base directory. +func doSafeOpen(pathname string, base string) (int, error) { + pathname = filepath.Clean(pathname) + base = filepath.Clean(base) + + // Calculate segments to follow + subpath, err := filepath.Rel(base, pathname) + if err != nil { + return -1, err + } + segments := strings.Split(subpath, string(filepath.Separator)) + + // Assumption: base is the only directory that we have under control. + // Base dir is not allowed to be a symlink. + parentFD, err := syscall.Open(base, nofollowFlags|unix.O_CLOEXEC, 0) + if err != nil { + return -1, fmt.Errorf("cannot open directory %s: %s", base, err) + } + defer func() { + if parentFD != -1 { + if err = syscall.Close(parentFD); err != nil { + klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", parentFD, pathname, err) + } + } + }() + + childFD := -1 + defer func() { + if childFD != -1 { + if err = syscall.Close(childFD); err != nil { + klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", childFD, pathname, err) + } + } + }() + + currentPath := base + + // Follow the segments one by one using openat() to make + // sure the user cannot change already existing directories into symlinks. + for _, seg := range segments { + currentPath = filepath.Join(currentPath, seg) + if !mount.PathWithinBase(currentPath, base) { + return -1, fmt.Errorf("path %s is outside of allowed base %s", currentPath, base) + } + + klog.V(5).Infof("Opening path %s", currentPath) + childFD, err = syscall.Openat(parentFD, seg, openFDFlags|unix.O_CLOEXEC, 0) + if err != nil { + return -1, fmt.Errorf("cannot open %s: %s", currentPath, err) + } + + var deviceStat unix.Stat_t + err := unix.Fstat(childFD, &deviceStat) + if err != nil { + return -1, fmt.Errorf("error running fstat on %s with %v", currentPath, err) + } + fileFmt := deviceStat.Mode & syscall.S_IFMT + if fileFmt == syscall.S_IFLNK { + return -1, fmt.Errorf("unexpected symlink found %s", currentPath) + } + + // Close parentFD + if err = syscall.Close(parentFD); err != nil { + return -1, fmt.Errorf("closing fd for %q failed: %v", filepath.Dir(currentPath), err) + } + // Set child to new parent + parentFD = childFD + childFD = -1 + } + + // We made it to the end, return this fd, don't close it + finalFD := parentFD + parentFD = -1 + + return finalFD, nil +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_unsupported.go b/vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_unsupported.go new file mode 100644 index 00000000..c9ff2bc0 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_unsupported.go @@ -0,0 +1,54 @@ +// +build !linux,!windows + +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package subpath + +import ( + "errors" + "os" + + "k8s.io/mount-utils" + "k8s.io/utils/nsenter" +) + +type subpath struct{} + +var errUnsupported = errors.New("util/subpath on this platform is not supported") + +// New returns a subpath.Interface for the current system. +func New(mount.Interface) Interface { + return &subpath{} +} + +// NewNSEnter is to satisfy the compiler for having NewSubpathNSEnter exist for all +// OS choices. however, NSEnter is only valid on Linux +func NewNSEnter(mounter mount.Interface, ne *nsenter.Nsenter, rootDir string) Interface { + return nil +} + +func (sp *subpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { + return subPath.Path, nil, errUnsupported +} + +func (sp *subpath) CleanSubPaths(podDir string, volumeName string) error { + return errUnsupported +} + +func (sp *subpath) SafeMakeDir(pathname string, base string, perm os.FileMode) error { + return errUnsupported +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_windows.go b/vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_windows.go new file mode 100644 index 00000000..0e9367ea --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_windows.go @@ -0,0 +1,377 @@ +// +build windows + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package subpath + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + + "k8s.io/klog/v2" + "k8s.io/mount-utils" + "k8s.io/utils/nsenter" +) + +// MaxPathLength is the maximum length of Windows path. Normally, it is 260, but if long path is enable, +// the max number is 32,767 +const MaxPathLength = 32767 + +type subpath struct{} + +// New returns a subpath.Interface for the current system +func New(mount.Interface) Interface { + return &subpath{} +} + +// NewNSEnter is to satisfy the compiler for having NewSubpathNSEnter exist for all +// OS choices. however, NSEnter is only valid on Linux +func NewNSEnter(mounter mount.Interface, ne *nsenter.Nsenter, rootDir string) Interface { + return nil +} + +// isDriveLetterPath returns true if the given path is empty or it ends with ":" or ":\" or ":\\" +func isDriveLetterorEmptyPath(path string) bool { + if path == "" || strings.HasSuffix(path, ":\\\\") || strings.HasSuffix(path, ":") || strings.HasSuffix(path, ":\\") { + return true + } + return false +} + +// isVolumePrefix returns true if the given path name starts with "Volume" or volume prefix including +// "\\.\", "\\?\" for device path or "UNC" or "\\" for UNC path. Otherwise, it returns false. +func isDeviceOrUncPath(path string) bool { + if strings.HasPrefix(path, "Volume") || strings.HasPrefix(path, "\\\\?\\") || strings.HasPrefix(path, "\\\\.\\") || strings.HasPrefix(path, "UNC") { + return true + } + return false +} + +// getUpperPath removes the last level of directory. +func getUpperPath(path string) string { + sep := fmt.Sprintf("%c", filepath.Separator) + upperpath := strings.TrimSuffix(path, sep) + return filepath.Dir(upperpath) +} + +// Check whether a directory/file is a link type or not +// LinkType could be SymbolicLink, Junction, or HardLink +func isLinkPath(path string) (bool, error) { + cmd := fmt.Sprintf("(Get-Item -Path %s).LinkType", path) + output, err := exec.Command("powershell", "/c", cmd).CombinedOutput() + if err != nil { + return false, err + } + if strings.TrimSpace(string(output)) != "" { + return true, nil + } + return false, nil +} + +// evalSymlink returns the path name after the evaluation of any symbolic links. +// If the path after evaluation is a device path or network connection, the original path is returned +func evalSymlink(path string) (string, error) { + path = mount.NormalizeWindowsPath(path) + if isDeviceOrUncPath(path) || isDriveLetterorEmptyPath(path) { + klog.V(4).Infof("Path '%s' is not a symlink, return its original form.", path) + return path, nil + } + upperpath := path + base := "" + for i := 0; i < MaxPathLength; i++ { + isLink, err := isLinkPath(upperpath) + if err != nil { + return "", err + } + if isLink { + break + } + // continue to check next layer + base = filepath.Join(filepath.Base(upperpath), base) + upperpath = getUpperPath(upperpath) + if isDriveLetterorEmptyPath(upperpath) { + klog.V(4).Infof("Path '%s' is not a symlink, return its original form.", path) + return path, nil + } + } + // This command will give the target path of a given symlink + cmd := fmt.Sprintf("(Get-Item -Path %s).Target", upperpath) + output, err := exec.Command("powershell", "/c", cmd).CombinedOutput() + if err != nil { + return "", err + } + klog.V(4).Infof("evaluate path %s: symlink from %s to %s", path, upperpath, string(output)) + linkedPath := strings.TrimSpace(string(output)) + if linkedPath == "" || isDeviceOrUncPath(linkedPath) { + klog.V(4).Infof("Path '%s' has a target %s. Return its original form.", path, linkedPath) + return path, nil + } + // If the target is not an absoluate path, join iit with the current upperpath + if !filepath.IsAbs(linkedPath) { + linkedPath = filepath.Join(getUpperPath(upperpath), linkedPath) + } + nextLink, err := evalSymlink(linkedPath) + if err != nil { + return path, err + } + return filepath.Join(nextLink, base), nil +} + +// check whether hostPath is within volume path +// this func will lock all intermediate subpath directories, need to close handle outside of this func after container started +func lockAndCheckSubPath(volumePath, hostPath string) ([]uintptr, error) { + if len(volumePath) == 0 || len(hostPath) == 0 { + return []uintptr{}, nil + } + + finalSubPath, err := evalSymlink(hostPath) + if err != nil { + return []uintptr{}, fmt.Errorf("cannot evaluate link %s: %s", hostPath, err) + } + + finalVolumePath, err := evalSymlink(volumePath) + if err != nil { + return []uintptr{}, fmt.Errorf("cannot read link %s: %s", volumePath, err) + } + + return lockAndCheckSubPathWithoutSymlink(finalVolumePath, finalSubPath) +} + +// lock all intermediate subPath directories and check they are all within volumePath +// volumePath & subPath should not contain any symlink, otherwise it will return error +func lockAndCheckSubPathWithoutSymlink(volumePath, subPath string) ([]uintptr, error) { + if len(volumePath) == 0 || len(subPath) == 0 { + return []uintptr{}, nil + } + + // get relative path to volumePath + relSubPath, err := filepath.Rel(volumePath, subPath) + if err != nil { + return []uintptr{}, fmt.Errorf("Rel(%s, %s) error: %v", volumePath, subPath, err) + } + if mount.StartsWithBackstep(relSubPath) { + return []uintptr{}, fmt.Errorf("SubPath %q not within volume path %q", subPath, volumePath) + } + + if relSubPath == "." { + // volumePath and subPath are equal + return []uintptr{}, nil + } + + fileHandles := []uintptr{} + var errorResult error + + currentFullPath := volumePath + dirs := strings.Split(relSubPath, string(os.PathSeparator)) + for _, dir := range dirs { + // lock intermediate subPath directory first + currentFullPath = filepath.Join(currentFullPath, dir) + handle, err := lockPath(currentFullPath) + if err != nil { + errorResult = fmt.Errorf("cannot lock path %s: %s", currentFullPath, err) + break + } + fileHandles = append(fileHandles, handle) + + // make sure intermediate subPath directory does not contain symlink any more + stat, err := os.Lstat(currentFullPath) + if err != nil { + errorResult = fmt.Errorf("Lstat(%q) error: %v", currentFullPath, err) + break + } + if stat.Mode()&os.ModeSymlink != 0 { + errorResult = fmt.Errorf("subpath %q is an unexpected symlink after EvalSymlinks", currentFullPath) + break + } + + if !mount.PathWithinBase(currentFullPath, volumePath) { + errorResult = fmt.Errorf("SubPath %q not within volume path %q", currentFullPath, volumePath) + break + } + } + + return fileHandles, errorResult +} + +// unlockPath unlock directories +func unlockPath(fileHandles []uintptr) { + if fileHandles != nil { + for _, handle := range fileHandles { + syscall.CloseHandle(syscall.Handle(handle)) + } + } +} + +// lockPath locks a directory or symlink, return handle, exec "syscall.CloseHandle(handle)" to unlock the path +func lockPath(path string) (uintptr, error) { + if len(path) == 0 { + return uintptr(syscall.InvalidHandle), syscall.ERROR_FILE_NOT_FOUND + } + pathp, err := syscall.UTF16PtrFromString(path) + if err != nil { + return uintptr(syscall.InvalidHandle), err + } + access := uint32(syscall.GENERIC_READ) + sharemode := uint32(syscall.FILE_SHARE_READ) + createmode := uint32(syscall.OPEN_EXISTING) + flags := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS | syscall.FILE_FLAG_OPEN_REPARSE_POINT) + fd, err := syscall.CreateFile(pathp, access, sharemode, nil, createmode, flags, 0) + return uintptr(fd), err +} + +// Lock all directories in subPath and check they're not symlinks. +func (sp *subpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { + handles, err := lockAndCheckSubPath(subPath.VolumePath, subPath.Path) + + // Unlock the directories when the container starts + cleanupAction = func() { + unlockPath(handles) + } + return subPath.Path, cleanupAction, err +} + +// No bind-mounts for subpaths are necessary on Windows +func (sp *subpath) CleanSubPaths(podDir string, volumeName string) error { + return nil +} + +// SafeMakeDir makes sure that the created directory does not escape given base directory mis-using symlinks. +func (sp *subpath) SafeMakeDir(subdir string, base string, perm os.FileMode) error { + realBase, err := evalSymlink(base) + if err != nil { + return fmt.Errorf("error resolving symlinks in %s: %s", base, err) + } + + realFullPath := filepath.Join(realBase, subdir) + return doSafeMakeDir(realFullPath, realBase, perm) +} + +func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { + klog.V(4).Infof("Creating directory %q within base %q", pathname, base) + + if !mount.PathWithinBase(pathname, base) { + return fmt.Errorf("path %s is outside of allowed base %s", pathname, base) + } + + // Quick check if the directory already exists + s, err := os.Stat(pathname) + if err == nil { + // Path exists + if s.IsDir() { + // The directory already exists. It can be outside of the parent, + // but there is no race-proof check. + klog.V(4).Infof("Directory %s already exists", pathname) + return nil + } + return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR} + } + + // Find all existing directories + existingPath, toCreate, err := findExistingPrefix(base, pathname) + if err != nil { + return fmt.Errorf("error opening directory %s: %s", pathname, err) + } + if len(toCreate) == 0 { + return nil + } + + // Ensure the existing directory is inside allowed base + fullExistingPath, err := evalSymlink(existingPath) + if err != nil { + return fmt.Errorf("error opening existing directory %s: %s", existingPath, err) + } + fullBasePath, err := evalSymlink(base) + if err != nil { + return fmt.Errorf("cannot read link %s: %s", base, err) + } + if !mount.PathWithinBase(fullExistingPath, fullBasePath) { + return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err) + } + + // lock all intermediate directories from fullBasePath to fullExistingPath (top to bottom) + fileHandles, err := lockAndCheckSubPathWithoutSymlink(fullBasePath, fullExistingPath) + defer unlockPath(fileHandles) + if err != nil { + return err + } + + klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...)) + currentPath := fullExistingPath + // create the directories one by one, making sure nobody can change + // created directory into symlink by lock that directory immediately + for _, dir := range toCreate { + currentPath = filepath.Join(currentPath, dir) + klog.V(4).Infof("Creating %s", dir) + if err := os.Mkdir(currentPath, perm); err != nil { + return fmt.Errorf("cannot create directory %s: %s", currentPath, err) + } + handle, err := lockPath(currentPath) + if err != nil { + return fmt.Errorf("cannot lock path %s: %s", currentPath, err) + } + defer syscall.CloseHandle(syscall.Handle(handle)) + // make sure newly created directory does not contain symlink after lock + stat, err := os.Lstat(currentPath) + if err != nil { + return fmt.Errorf("Lstat(%q) error: %v", currentPath, err) + } + if stat.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("subpath %q is an unexpected symlink after Mkdir", currentPath) + } + } + + return nil +} + +// findExistingPrefix finds prefix of pathname that exists. In addition, it +// returns list of remaining directories that don't exist yet. +func findExistingPrefix(base, pathname string) (string, []string, error) { + rel, err := filepath.Rel(base, pathname) + if err != nil { + return base, nil, err + } + + if mount.StartsWithBackstep(rel) { + return base, nil, fmt.Errorf("pathname(%s) is not within base(%s)", pathname, base) + } + + if rel == "." { + // base and pathname are equal + return pathname, []string{}, nil + } + + dirs := strings.Split(rel, string(filepath.Separator)) + + var parent string + currentPath := base + for i, dir := range dirs { + parent = currentPath + currentPath = filepath.Join(parent, dir) + if _, err := os.Lstat(currentPath); err != nil { + if os.IsNotExist(err) { + return parent, dirs[i:], nil + } + return base, nil, err + } + } + + return pathname, []string{}, nil +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/volume.go b/vendor/k8s.io/kubernetes/pkg/volume/volume.go new file mode 100644 index 00000000..85b8c4aa --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/volume.go @@ -0,0 +1,295 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volume + +import ( + "time" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +// Volume represents a directory used by pods or hosts on a node. All method +// implementations of methods in the volume interface must be idempotent. +type Volume interface { + // GetPath returns the path to which the volume should be mounted for the + // pod. + GetPath() string + + // MetricsProvider embeds methods for exposing metrics (e.g. + // used, available space). + MetricsProvider +} + +// BlockVolume interface provides methods to generate global map path +// and pod device map path. +type BlockVolume interface { + // GetGlobalMapPath returns a global map path which contains + // bind mount associated to a block device. + // ex. plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}/{pod uuid} + GetGlobalMapPath(spec *Spec) (string, error) + // GetPodDeviceMapPath returns a pod device map path + // and name of a symbolic link associated to a block device. + // ex. pods/{podUid}/{DefaultKubeletVolumeDevicesDirName}/{escapeQualifiedPluginName}/, {volumeName} + GetPodDeviceMapPath() (string, string) +} + +// MetricsProvider exposes metrics (e.g. used,available space) related to a +// Volume. +type MetricsProvider interface { + // GetMetrics returns the Metrics for the Volume. Maybe expensive for + // some implementations. + GetMetrics() (*Metrics, error) +} + +// Metrics represents the used and available bytes of the Volume. +type Metrics struct { + // The time at which these stats were updated. + Time metav1.Time + + // Used represents the total bytes used by the Volume. + // Note: For block devices this maybe more than the total size of the files. + Used *resource.Quantity + + // Capacity represents the total capacity (bytes) of the volume's + // underlying storage. For Volumes that share a filesystem with the host + // (e.g. emptydir, hostpath) this is the size of the underlying storage, + // and will not equal Used + Available as the fs is shared. + Capacity *resource.Quantity + + // Available represents the storage space available (bytes) for the + // Volume. For Volumes that share a filesystem with the host (e.g. + // emptydir, hostpath), this is the available space on the underlying + // storage, and is shared with host processes and other Volumes. + Available *resource.Quantity + + // InodesUsed represents the total inodes used by the Volume. + InodesUsed *resource.Quantity + + // Inodes represents the total number of inodes available in the volume. + // For volumes that share a filesystem with the host (e.g. emptydir, hostpath), + // this is the inodes available in the underlying storage, + // and will not equal InodesUsed + InodesFree as the fs is shared. + Inodes *resource.Quantity + + // InodesFree represent the inodes available for the volume. For Volumes that share + // a filesystem with the host (e.g. emptydir, hostpath), this is the free inodes + // on the underlying storage, and is shared with host processes and other volumes + InodesFree *resource.Quantity +} + +// Attributes represents the attributes of this mounter. +type Attributes struct { + ReadOnly bool + Managed bool + SupportsSELinux bool +} + +// MounterArgs provides more easily extensible arguments to Mounter +type MounterArgs struct { + // When FsUser is set, the ownership of the volume will be modified to be + // owned and writable by FsUser. Otherwise, there is no side effects. + // Currently only supported with projected service account tokens. + FsUser *int64 + FsGroup *int64 + FSGroupChangePolicy *v1.PodFSGroupChangePolicy + DesiredSize *resource.Quantity +} + +// Mounter interface provides methods to set up/mount the volume. +type Mounter interface { + // Uses Interface to provide the path for Docker binds. + Volume + + // CanMount is called immediately prior to Setup to check if + // the required components (binaries, etc.) are available on + // the underlying node to complete the subsequent SetUp (mount) + // operation. If CanMount returns error, the mount operation is + // aborted and an event is generated indicating that the node + // does not have the required binaries to complete the mount. + // If CanMount succeeds, the mount operation continues + // normally. The CanMount check can be enabled or disabled + // using the experimental-check-mount-binaries binary flag + CanMount() error + + // SetUp prepares and mounts/unpacks the volume to a + // self-determined directory path. The mount point and its + // content should be owned by `fsUser` or 'fsGroup' so that it can be + // accessed by the pod. This may be called more than once, so + // implementations must be idempotent. + // It could return following types of errors: + // - TransientOperationFailure + // - UncertainProgressError + // - Error of any other type should be considered a final error + SetUp(mounterArgs MounterArgs) error + + // SetUpAt prepares and mounts/unpacks the volume to the + // specified directory path, which may or may not exist yet. + // The mount point and its content should be owned by `fsUser` + // 'fsGroup' so that it can be accessed by the pod. This may + // be called more than once, so implementations must be + // idempotent. + SetUpAt(dir string, mounterArgs MounterArgs) error + // GetAttributes returns the attributes of the mounter. + // This function is called after SetUp()/SetUpAt(). + GetAttributes() Attributes +} + +// Unmounter interface provides methods to cleanup/unmount the volumes. +type Unmounter interface { + Volume + // TearDown unmounts the volume from a self-determined directory and + // removes traces of the SetUp procedure. + TearDown() error + // TearDown unmounts the volume from the specified directory and + // removes traces of the SetUp procedure. + TearDownAt(dir string) error +} + +// BlockVolumeMapper interface is a mapper interface for block volume. +type BlockVolumeMapper interface { + BlockVolume +} + +// CustomBlockVolumeMapper interface provides custom methods to set up/map the volume. +type CustomBlockVolumeMapper interface { + BlockVolumeMapper + // SetUpDevice prepares the volume to the node by the plugin specific way. + // For most in-tree plugins, attacher.Attach() and attacher.WaitForAttach() + // will do necessary works. + // This may be called more than once, so implementations must be idempotent. + // SetUpDevice returns stagingPath if device setup was successful + SetUpDevice() (stagingPath string, err error) + + // MapPodDevice maps the block device to a path and return the path. + // Unique device path across kubelet node reboot is required to avoid + // unexpected block volume destruction. + // If empty string is returned, the path retuned by attacher.Attach() and + // attacher.WaitForAttach() will be used. + MapPodDevice() (publishPath string, err error) + + // GetStagingPath returns path that was used for staging the volume + // it is mainly used by CSI plugins + GetStagingPath() string +} + +// BlockVolumeUnmapper interface is an unmapper interface for block volume. +type BlockVolumeUnmapper interface { + BlockVolume +} + +// CustomBlockVolumeUnmapper interface provides custom methods to cleanup/unmap the volumes. +type CustomBlockVolumeUnmapper interface { + BlockVolumeUnmapper + // TearDownDevice removes traces of the SetUpDevice procedure. + // If the plugin is non-attachable, this method detaches the volume + // from a node. + TearDownDevice(mapPath string, devicePath string) error + + // UnmapPodDevice removes traces of the MapPodDevice procedure. + UnmapPodDevice() error +} + +// Provisioner is an interface that creates templates for PersistentVolumes +// and can create the volume as a new resource in the infrastructure provider. +type Provisioner interface { + // Provision creates the resource by allocating the underlying volume in a + // storage system. This method should block until completion and returns + // PersistentVolume representing the created storage resource. + Provision(selectedNode *v1.Node, allowedTopologies []v1.TopologySelectorTerm) (*v1.PersistentVolume, error) +} + +// Deleter removes the resource from the underlying storage provider. Calls +// to this method should block until the deletion is complete. Any error +// returned indicates the volume has failed to be reclaimed. A nil return +// indicates success. +type Deleter interface { + Volume + // This method should block until completion. + // deletedVolumeInUseError returned from this function will not be reported + // as error and it will be sent as "Info" event to the PV being deleted. The + // volume controller will retry deleting the volume in the next periodic + // sync. This can be used to postpone deletion of a volume that is being + // detached from a node. Deletion of such volume would fail anyway and such + // error would confuse users. + Delete() error +} + +// Attacher can attach a volume to a node. +type Attacher interface { + DeviceMounter + + // Attaches the volume specified by the given spec to the node with the given Name. + // On success, returns the device path where the device was attached on the + // node. + Attach(spec *Spec, nodeName types.NodeName) (string, error) + + // VolumesAreAttached checks whether the list of volumes still attached to the specified + // node. It returns a map which maps from the volume spec to the checking result. + // If an error is occurred during checking, the error will be returned + VolumesAreAttached(specs []*Spec, nodeName types.NodeName) (map[*Spec]bool, error) + + // WaitForAttach blocks until the device is attached to this + // node. If it successfully attaches, the path to the device + // is returned. Otherwise, if the device does not attach after + // the given timeout period, an error will be returned. + WaitForAttach(spec *Spec, devicePath string, pod *v1.Pod, timeout time.Duration) (string, error) +} + +// DeviceMounter can mount a block volume to a global path. +type DeviceMounter interface { + // GetDeviceMountPath returns a path where the device should + // be mounted after it is attached. This is a global mount + // point which should be bind mounted for individual volumes. + GetDeviceMountPath(spec *Spec) (string, error) + + // MountDevice mounts the disk to a global path which + // individual pods can then bind mount + // Note that devicePath can be empty if the volume plugin does not implement any of Attach and WaitForAttach methods. + // It could return following types of errors: + // - TransientOperationFailure + // - UncertainProgressError + // - Error of any other type should be considered a final error + MountDevice(spec *Spec, devicePath string, deviceMountPath string) error +} + +type BulkVolumeVerifier interface { + // BulkVerifyVolumes checks whether the list of volumes still attached to the + // the clusters in the node. It returns a map which maps from the volume spec to the checking result. + // If an error occurs during check - error should be returned and volume on nodes + // should be assumed as still attached. + BulkVerifyVolumes(volumesByNode map[types.NodeName][]*Spec) (map[types.NodeName]map[*Spec]bool, error) +} + +// Detacher can detach a volume from a node. +type Detacher interface { + DeviceUnmounter + // Detach the given volume from the node with the given Name. + // volumeName is name of the volume as returned from plugin's + // GetVolumeName(). + Detach(volumeName string, nodeName types.NodeName) error +} + +// DeviceUnmounter can unmount a block volume from the global path. +type DeviceUnmounter interface { + // UnmountDevice unmounts the global mount of the disk. This + // should only be called once all bind mounts have been + // unmounted. + UnmountDevice(deviceMountPath string) error +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/volume_linux.go b/vendor/k8s.io/kubernetes/pkg/volume/volume_linux.go new file mode 100644 index 00000000..17a1d3ef --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/volume_linux.go @@ -0,0 +1,241 @@ +// +build linux + +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volume + +import ( + "path/filepath" + "syscall" + + "os" + "time" + + v1 "k8s.io/api/core/v1" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/features" +) + +const ( + rwMask = os.FileMode(0660) + roMask = os.FileMode(0440) + execMask = os.FileMode(0110) +) + +// SetVolumeOwnership modifies the given volume to be owned by +// fsGroup, and sets SetGid so that newly created files are owned by +// fsGroup. If fsGroup is nil nothing is done. +func SetVolumeOwnership(mounter Mounter, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy, completeFunc func(*error)) error { + if fsGroup == nil { + return nil + } + + fsGroupPolicyEnabled := utilfeature.DefaultFeatureGate.Enabled(features.ConfigurableFSGroupPolicy) + + timer := time.AfterFunc(30*time.Second, func() { + klog.Warningf("Setting volume ownership for %s and fsGroup set. If the volume has a lot of files then setting volume ownership could be slow, see https://github.com/kubernetes/kubernetes/issues/69699", mounter.GetPath()) + }) + defer timer.Stop() + + // This code exists for legacy purposes, so as old behaviour is entirely preserved when feature gate is disabled + // TODO: remove this when ConfigurableFSGroupPolicy turns GA. + if !fsGroupPolicyEnabled { + err := legacyOwnershipChange(mounter, fsGroup) + if completeFunc != nil { + completeFunc(&err) + } + return err + } + + if skipPermissionChange(mounter, fsGroup, fsGroupChangePolicy) { + klog.V(3).Infof("skipping permission and ownership change for volume %s", mounter.GetPath()) + return nil + } + + err := walkDeep(mounter.GetPath(), func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + return changeFilePermission(path, fsGroup, mounter.GetAttributes().ReadOnly, info) + }) + if completeFunc != nil { + completeFunc(&err) + } + return err +} + +func legacyOwnershipChange(mounter Mounter, fsGroup *int64) error { + return filepath.Walk(mounter.GetPath(), func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + return changeFilePermission(path, fsGroup, mounter.GetAttributes().ReadOnly, info) + }) +} + +func changeFilePermission(filename string, fsGroup *int64, readonly bool, info os.FileInfo) error { + // chown and chmod pass through to the underlying file for symlinks. + // Symlinks have a mode of 777 but this really doesn't mean anything. + // The permissions of the underlying file are what matter. + // However, if one reads the mode of a symlink then chmods the symlink + // with that mode, it changes the mode of the underlying file, overridden + // the defaultMode and permissions initialized by the volume plugin, which + // is not what we want; thus, we skip chown/chmod for symlinks. + if info.Mode()&os.ModeSymlink != 0 { + return nil + } + + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return nil + } + + if stat == nil { + klog.Errorf("Got nil stat_t for path %v while setting ownership of volume", filename) + return nil + } + + err := os.Chown(filename, int(stat.Uid), int(*fsGroup)) + if err != nil { + klog.Errorf("Chown failed on %v: %v", filename, err) + } + + mask := rwMask + if readonly { + mask = roMask + } + + if info.IsDir() { + mask |= os.ModeSetgid + mask |= execMask + } + + err = os.Chmod(filename, info.Mode()|mask) + if err != nil { + klog.Errorf("Chmod failed on %v: %v", filename, err) + } + + return nil +} + +func skipPermissionChange(mounter Mounter, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy) bool { + dir := mounter.GetPath() + + if fsGroupChangePolicy == nil || *fsGroupChangePolicy != v1.FSGroupChangeOnRootMismatch { + klog.V(4).Infof("perform recursive ownership change for %s", dir) + return false + } + return !requiresPermissionChange(mounter.GetPath(), fsGroup, mounter.GetAttributes().ReadOnly) +} + +func requiresPermissionChange(rootDir string, fsGroup *int64, readonly bool) bool { + fsInfo, err := os.Stat(rootDir) + if err != nil { + klog.Errorf("performing recursive ownership change on %s because reading permissions of root volume failed: %v", rootDir, err) + return true + } + stat, ok := fsInfo.Sys().(*syscall.Stat_t) + if !ok || stat == nil { + klog.Errorf("performing recursive ownership change on %s because reading permissions of root volume failed", rootDir) + return true + } + + if int(stat.Gid) != int(*fsGroup) { + klog.V(4).Infof("expected group ownership of volume %s did not match with: %d", rootDir, stat.Gid) + return true + } + unixPerms := rwMask + + if readonly { + unixPerms = roMask + } + + // if rootDir is not a directory then we should apply permission change anyways + if !fsInfo.IsDir() { + return true + } + unixPerms |= execMask + filePerm := fsInfo.Mode().Perm() + + // We need to check if actual permissions of root directory is a superset of permissions required by unixPerms. + // This is done by checking if permission bits expected in unixPerms is set in actual permissions of the directory. + // We use bitwise AND operation to check set bits. For example: + // unixPerms: 770, filePerms: 775 : 770&775 = 770 (perms on directory is a superset) + // unixPerms: 770, filePerms: 770 : 770&770 = 770 (perms on directory is a superset) + // unixPerms: 770, filePerms: 750 : 770&750 = 750 (perms on directory is NOT a superset) + // We also need to check if setgid bits are set in permissions of the directory. + if (unixPerms&filePerm != unixPerms) || (fsInfo.Mode()&os.ModeSetgid == 0) { + klog.V(4).Infof("performing recursive ownership change on %s because of mismatching mode", rootDir) + return true + } + return false +} + +// readDirNames reads the directory named by dirname and returns +// a list of directory entries. +// We are not using filepath.readDirNames because we do not want to sort files found in a directory before changing +// permissions for performance reasons. +func readDirNames(dirname string) ([]string, error) { + f, err := os.Open(dirname) + if err != nil { + return nil, err + } + names, err := f.Readdirnames(-1) + f.Close() + if err != nil { + return nil, err + } + return names, nil +} + +// walkDeep can be used to traverse directories and has two minor differences +// from filepath.Walk: +// - List of files/dirs is not sorted for performance reasons +// - callback walkFunc is invoked on root directory after visiting children dirs and files +func walkDeep(root string, walkFunc filepath.WalkFunc) error { + info, err := os.Lstat(root) + if err != nil { + return walkFunc(root, nil, err) + } + return walk(root, info, walkFunc) +} + +func walk(path string, info os.FileInfo, walkFunc filepath.WalkFunc) error { + if !info.IsDir() { + return walkFunc(path, info, nil) + } + names, err := readDirNames(path) + if err != nil { + return err + } + for _, name := range names { + filename := filepath.Join(path, name) + fileInfo, err := os.Lstat(filename) + if err != nil { + if err := walkFunc(filename, fileInfo, err); err != nil { + return err + } + } else { + err = walk(filename, fileInfo, walkFunc) + if err != nil { + return err + } + } + } + return walkFunc(path, info, nil) +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/volume_unsupported.go b/vendor/k8s.io/kubernetes/pkg/volume/volume_unsupported.go new file mode 100644 index 00000000..9cb094fe --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/volume_unsupported.go @@ -0,0 +1,27 @@ +// +build !linux + +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volume + +import ( + v1 "k8s.io/api/core/v1" +) + +func SetVolumeOwnership(mounter Mounter, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy, completeFunc func(*error)) error { + return nil +} diff --git a/vendor/k8s.io/mount-utils/LICENSE b/vendor/k8s.io/mount-utils/LICENSE new file mode 100644 index 00000000..8dada3ed --- /dev/null +++ b/vendor/k8s.io/mount-utils/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/k8s.io/mount-utils/OWNERS b/vendor/k8s.io/mount-utils/OWNERS new file mode 100644 index 00000000..5fa5cc24 --- /dev/null +++ b/vendor/k8s.io/mount-utils/OWNERS @@ -0,0 +1,14 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +reviewers: + - jingxu97 + - saad-ali + - jsafrane + - msau42 + - andyzhangx + - gnufied +approvers: + - jingxu97 + - saad-ali + - jsafrane + diff --git a/vendor/k8s.io/mount-utils/README.md b/vendor/k8s.io/mount-utils/README.md new file mode 100644 index 00000000..5510e99b --- /dev/null +++ b/vendor/k8s.io/mount-utils/README.md @@ -0,0 +1,30 @@ +## Purpose + +This repository defines an interface to mounting filesystems to be consumed by +various Kubernetes and out-of-tree CSI components. + +Consumers of this repository can make use of functions like 'Mount' to mount +source to target as fstype with given options, 'Unmount' to unmount a target. +Other useful functions include 'List' all mounted file systems and find all +mount references to a path using 'GetMountRefs' + +## Community, discussion, contribution, and support + +Learn how to engage with the Kubernetes community on the [community +page](http://kubernetes.io/community/). + +You can reach the maintainers of this repository at: + +- Slack: #sig-storage (on https://kubernetes.slack.com -- get an + invite at slack.kubernetes.io) +- Mailing List: + https://groups.google.com/forum/#!forum/kubernetes-sig-storage + +### Code of Conduct + +Participation in the Kubernetes community is governed by the [Kubernetes +Code of Conduct](code-of-conduct.md). + +### Contibution Guidelines + +See [CONTRIBUTING.md](CONTRIBUTING.md) for more information. diff --git a/vendor/k8s.io/mount-utils/SECURITY_CONTACTS b/vendor/k8s.io/mount-utils/SECURITY_CONTACTS new file mode 100644 index 00000000..14fe23e1 --- /dev/null +++ b/vendor/k8s.io/mount-utils/SECURITY_CONTACTS @@ -0,0 +1,18 @@ +# Defined below are the security contacts for this repo. +# +# They are the contact point for the Product Security Committee to reach out +# to for triaging and handling of incoming issues. +# +# The below names agree to abide by the +# [Embargo Policy](https://git.k8s.io/security/private-distributors-list.md#embargo-policy) +# and will be removed and replaced if they violate that agreement. +# +# DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE +# INSTRUCTIONS AT https://kubernetes.io/security/ + +saad-ali +cjcullen +joelsmith +liggitt +philips +tallclair diff --git a/vendor/k8s.io/mount-utils/code-of-conduct.md b/vendor/k8s.io/mount-utils/code-of-conduct.md new file mode 100644 index 00000000..0d15c00c --- /dev/null +++ b/vendor/k8s.io/mount-utils/code-of-conduct.md @@ -0,0 +1,3 @@ +# Kubernetes Community Code of Conduct + +Please refer to our [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md) diff --git a/vendor/k8s.io/mount-utils/doc.go b/vendor/k8s.io/mount-utils/doc.go new file mode 100644 index 00000000..b7cac03a --- /dev/null +++ b/vendor/k8s.io/mount-utils/doc.go @@ -0,0 +1,18 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package mount defines an interface to mounting filesystems. +package mount // import "k8s.io/mount-utils" diff --git a/vendor/k8s.io/mount-utils/fake_mounter.go b/vendor/k8s.io/mount-utils/fake_mounter.go new file mode 100644 index 00000000..393ed043 --- /dev/null +++ b/vendor/k8s.io/mount-utils/fake_mounter.go @@ -0,0 +1,220 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "os" + "path/filepath" + "sync" + + "k8s.io/klog/v2" +) + +// FakeMounter implements mount.Interface for tests. +type FakeMounter struct { + MountPoints []MountPoint + log []FakeAction + // Error to return for a path when calling IsLikelyNotMountPoint + MountCheckErrors map[string]error + // Some tests run things in parallel, make sure the mounter does not produce + // any golang's DATA RACE warnings. + mutex sync.Mutex + UnmountFunc UnmountFunc +} + +// UnmountFunc is a function callback to be executed during the Unmount() call. +type UnmountFunc func(path string) error + +var _ Interface = &FakeMounter{} + +const ( + // FakeActionMount is the string for specifying mount as FakeAction.Action + FakeActionMount = "mount" + // FakeActionUnmount is the string for specifying unmount as FakeAction.Action + FakeActionUnmount = "unmount" +) + +// FakeAction objects are logged every time a fake mount or unmount is called. +type FakeAction struct { + Action string // "mount" or "unmount" + Target string // applies to both mount and unmount actions + Source string // applies only to "mount" actions + FSType string // applies only to "mount" actions +} + +// NewFakeMounter returns a FakeMounter struct that implements Interface and is +// suitable for testing purposes. +func NewFakeMounter(mps []MountPoint) *FakeMounter { + return &FakeMounter{ + MountPoints: mps, + } +} + +// ResetLog clears all the log entries in FakeMounter +func (f *FakeMounter) ResetLog() { + f.mutex.Lock() + defer f.mutex.Unlock() + + f.log = []FakeAction{} +} + +// GetLog returns the slice of FakeActions taken by the mounter +func (f *FakeMounter) GetLog() []FakeAction { + f.mutex.Lock() + defer f.mutex.Unlock() + + return f.log +} + +// Mount records the mount event and updates the in-memory mount points for FakeMounter +func (f *FakeMounter) Mount(source string, target string, fstype string, options []string) error { + return f.MountSensitive(source, target, fstype, options, nil /* sensitiveOptions */) +} + +// Mount records the mount event and updates the in-memory mount points for FakeMounter +// sensitiveOptions to be passed in a separate parameter from the normal +// mount options and ensures the sensitiveOptions are never logged. This +// method should be used by callers that pass sensitive material (like +// passwords) as mount options. +func (f *FakeMounter) MountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error { + f.mutex.Lock() + defer f.mutex.Unlock() + + opts := []string{} + + for _, option := range options { + // find 'bind' option + if option == "bind" { + // This is a bind-mount. In order to mimic linux behaviour, we must + // use the original device of the bind-mount as the real source. + // E.g. when mounted /dev/sda like this: + // $ mount /dev/sda /mnt/test + // $ mount -o bind /mnt/test /mnt/bound + // then /proc/mount contains: + // /dev/sda /mnt/test + // /dev/sda /mnt/bound + // (and not /mnt/test /mnt/bound) + // I.e. we must use /dev/sda as source instead of /mnt/test in the + // bind mount. + for _, mnt := range f.MountPoints { + if source == mnt.Path { + source = mnt.Device + break + } + } + } + // reuse MountPoint.Opts field to mark mount as readonly + opts = append(opts, option) + } + + // If target is a symlink, get its absolute path + absTarget, err := filepath.EvalSymlinks(target) + if err != nil { + absTarget = target + } + f.MountPoints = append(f.MountPoints, MountPoint{Device: source, Path: absTarget, Type: fstype, Opts: append(opts, sensitiveOptions...)}) + klog.V(5).Infof("Fake mounter: mounted %s to %s", source, absTarget) + f.log = append(f.log, FakeAction{Action: FakeActionMount, Target: absTarget, Source: source, FSType: fstype}) + return nil +} + +func (f *FakeMounter) MountSensitiveWithoutSystemd(source string, target string, fstype string, options []string, sensitiveOptions []string) error { + return f.MountSensitive(source, target, fstype, options, nil /* sensitiveOptions */) +} + +// Unmount records the unmount event and updates the in-memory mount points for FakeMounter +func (f *FakeMounter) Unmount(target string) error { + f.mutex.Lock() + defer f.mutex.Unlock() + + // If target is a symlink, get its absolute path + absTarget, err := filepath.EvalSymlinks(target) + if err != nil { + absTarget = target + } + + newMountpoints := []MountPoint{} + for _, mp := range f.MountPoints { + if mp.Path == absTarget { + if f.UnmountFunc != nil { + err := f.UnmountFunc(absTarget) + if err != nil { + return err + } + } + klog.V(5).Infof("Fake mounter: unmounted %s from %s", mp.Device, absTarget) + // Don't copy it to newMountpoints + continue + } + newMountpoints = append(newMountpoints, MountPoint{Device: mp.Device, Path: mp.Path, Type: mp.Type}) + } + f.MountPoints = newMountpoints + f.log = append(f.log, FakeAction{Action: FakeActionUnmount, Target: absTarget}) + delete(f.MountCheckErrors, target) + return nil +} + +// List returns all the in-memory mountpoints for FakeMounter +func (f *FakeMounter) List() ([]MountPoint, error) { + f.mutex.Lock() + defer f.mutex.Unlock() + + return f.MountPoints, nil +} + +// IsLikelyNotMountPoint determines whether a path is a mountpoint by checking +// if the absolute path to file is in the in-memory mountpoints +func (f *FakeMounter) IsLikelyNotMountPoint(file string) (bool, error) { + f.mutex.Lock() + defer f.mutex.Unlock() + + err := f.MountCheckErrors[file] + if err != nil { + return false, err + } + + _, err = os.Stat(file) + if err != nil { + return true, err + } + + // If file is a symlink, get its absolute path + absFile, err := filepath.EvalSymlinks(file) + if err != nil { + absFile = file + } + + for _, mp := range f.MountPoints { + if mp.Path == absFile { + klog.V(5).Infof("isLikelyNotMountPoint for %s: mounted %s, false", file, mp.Path) + return false, nil + } + } + klog.V(5).Infof("isLikelyNotMountPoint for %s: true", file) + return true, nil +} + +// GetMountRefs finds all mount references to the path, returns a +// list of paths. +func (f *FakeMounter) GetMountRefs(pathname string) ([]string, error) { + realpath, err := filepath.EvalSymlinks(pathname) + if err != nil { + // Ignore error in FakeMounter, because we actually didn't create files. + realpath = pathname + } + return getMountRefsByDev(f, realpath) +} diff --git a/vendor/k8s.io/mount-utils/go.mod b/vendor/k8s.io/mount-utils/go.mod new file mode 100644 index 00000000..396e921f --- /dev/null +++ b/vendor/k8s.io/mount-utils/go.mod @@ -0,0 +1,13 @@ +// This is a generated file. Do not edit directly. + +module k8s.io/mount-utils + +go 1.15 + +require ( + github.com/kr/pretty v0.2.0 // indirect + github.com/stretchr/testify v1.6.1 + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect + k8s.io/klog/v2 v2.4.0 + k8s.io/utils v0.0.0-20201110183641-67b214c5f920 +) diff --git a/vendor/k8s.io/mount-utils/go.sum b/vendor/k8s.io/mount-utils/go.sum new file mode 100644 index 00000000..6cc32d74 --- /dev/null +++ b/vendor/k8s.io/mount-utils/go.sum @@ -0,0 +1,32 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/klog/v2 v2.0.0 h1:Foj74zO6RbjjP4hBEKjnYtjjAhGg4jNynUdYF6fJrok= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= +k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= diff --git a/vendor/k8s.io/mount-utils/mount.go b/vendor/k8s.io/mount-utils/mount.go new file mode 100644 index 00000000..c78cf13d --- /dev/null +++ b/vendor/k8s.io/mount-utils/mount.go @@ -0,0 +1,368 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// TODO(thockin): This whole pkg is pretty linux-centric. As soon as we have +// an alternate platform, we will need to abstract further. + +package mount + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + utilexec "k8s.io/utils/exec" +) + +const ( + // Default mount command if mounter path is not specified. + defaultMountCommand = "mount" + // Log message where sensitive mount options were removed + sensitiveOptionsRemoved = "" +) + +// Interface defines the set of methods to allow for mount operations on a system. +type Interface interface { + // Mount mounts source to target as fstype with given options. + // options MUST not contain sensitive material (like passwords). + Mount(source string, target string, fstype string, options []string) error + // MountSensitive is the same as Mount() but this method allows + // sensitiveOptions to be passed in a separate parameter from the normal + // mount options and ensures the sensitiveOptions are never logged. This + // method should be used by callers that pass sensitive material (like + // passwords) as mount options. + MountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error + // MountSensitiveWithoutSystemd is the same as MountSensitive() but this method disable using systemd mount. + MountSensitiveWithoutSystemd(source string, target string, fstype string, options []string, sensitiveOptions []string) error + // Unmount unmounts given target. + Unmount(target string) error + // List returns a list of all mounted filesystems. This can be large. + // On some platforms, reading mounts directly from the OS is not guaranteed + // consistent (i.e. it could change between chunked reads). This is guaranteed + // to be consistent. + List() ([]MountPoint, error) + // IsLikelyNotMountPoint uses heuristics to determine if a directory + // is not a mountpoint. + // It should return ErrNotExist when the directory does not exist. + // IsLikelyNotMountPoint does NOT properly detect all mountpoint types + // most notably linux bind mounts and symbolic link. For callers that do not + // care about such situations, this is a faster alternative to calling List() + // and scanning that output. + IsLikelyNotMountPoint(file string) (bool, error) + // GetMountRefs finds all mount references to pathname, returning a slice of + // paths. Pathname can be a mountpoint path or a normal directory + // (for bind mount). On Linux, pathname is excluded from the slice. + // For example, if /dev/sdc was mounted at /path/a and /path/b, + // GetMountRefs("/path/a") would return ["/path/b"] + // GetMountRefs("/path/b") would return ["/path/a"] + // On Windows there is no way to query all mount points; as long as pathname is + // a valid mount, it will be returned. + GetMountRefs(pathname string) ([]string, error) +} + +// Compile-time check to ensure all Mounter implementations satisfy +// the mount interface. +var _ Interface = &Mounter{} + +// MountPoint represents a single line in /proc/mounts or /etc/fstab. +type MountPoint struct { // nolint: golint + Device string + Path string + Type string + Opts []string // Opts may contain sensitive mount options (like passwords) and MUST be treated as such (e.g. not logged). + Freq int + Pass int +} + +type MountErrorType string // nolint: golint + +const ( + FilesystemMismatch MountErrorType = "FilesystemMismatch" + HasFilesystemErrors MountErrorType = "HasFilesystemErrors" + UnformattedReadOnly MountErrorType = "UnformattedReadOnly" + FormatFailed MountErrorType = "FormatFailed" + GetDiskFormatFailed MountErrorType = "GetDiskFormatFailed" + UnknownMountError MountErrorType = "UnknownMountError" +) + +type MountError struct { // nolint: golint + Type MountErrorType + Message string +} + +func (mountError MountError) String() string { + return mountError.Message +} + +func (mountError MountError) Error() string { + return mountError.Message +} + +func NewMountError(mountErrorValue MountErrorType, format string, args ...interface{}) error { + mountError := MountError{ + Type: mountErrorValue, + Message: fmt.Sprintf(format, args...), + } + return mountError +} + +// SafeFormatAndMount probes a device to see if it is formatted. +// Namely it checks to see if a file system is present. If so it +// mounts it otherwise the device is formatted first then mounted. +type SafeFormatAndMount struct { + Interface + Exec utilexec.Interface +} + +// FormatAndMount formats the given disk, if needed, and mounts it. +// That is if the disk is not formatted and it is not being mounted as +// read-only it will format it first then mount it. Otherwise, if the +// disk is already formatted or it is being mounted as read-only, it +// will be mounted without formatting. +// options MUST not contain sensitive material (like passwords). +func (mounter *SafeFormatAndMount) FormatAndMount(source string, target string, fstype string, options []string) error { + return mounter.FormatAndMountSensitive(source, target, fstype, options, nil /* sensitiveOptions */) +} + +// FormatAndMountSensitive is the same as FormatAndMount but this method allows +// sensitiveOptions to be passed in a separate parameter from the normal mount +// options and ensures the sensitiveOptions are never logged. This method should +// be used by callers that pass sensitive material (like passwords) as mount +// options. +func (mounter *SafeFormatAndMount) FormatAndMountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error { + return mounter.formatAndMountSensitive(source, target, fstype, options, sensitiveOptions) +} + +// getMountRefsByDev finds all references to the device provided +// by mountPath; returns a list of paths. +// Note that mountPath should be path after the evaluation of any symblolic links. +func getMountRefsByDev(mounter Interface, mountPath string) ([]string, error) { + mps, err := mounter.List() + if err != nil { + return nil, err + } + + // Finding the device mounted to mountPath. + diskDev := "" + for i := range mps { + if mountPath == mps[i].Path { + diskDev = mps[i].Device + break + } + } + + // Find all references to the device. + var refs []string + for i := range mps { + if mps[i].Device == diskDev || mps[i].Device == mountPath { + if mps[i].Path != mountPath { + refs = append(refs, mps[i].Path) + } + } + } + return refs, nil +} + +// GetDeviceNameFromMount given a mnt point, find the device from /proc/mounts +// returns the device name, reference count, and error code. +func GetDeviceNameFromMount(mounter Interface, mountPath string) (string, int, error) { + mps, err := mounter.List() + if err != nil { + return "", 0, err + } + + // Find the device name. + // FIXME if multiple devices mounted on the same mount path, only the first one is returned. + device := "" + // If mountPath is symlink, need get its target path. + slTarget, err := filepath.EvalSymlinks(mountPath) + if err != nil { + slTarget = mountPath + } + for i := range mps { + if mps[i].Path == slTarget { + device = mps[i].Device + break + } + } + + // Find all references to the device. + refCount := 0 + for i := range mps { + if mps[i].Device == device { + refCount++ + } + } + return device, refCount, nil +} + +// IsNotMountPoint determines if a directory is a mountpoint. +// It should return ErrNotExist when the directory does not exist. +// IsNotMountPoint is more expensive than IsLikelyNotMountPoint. +// IsNotMountPoint detects bind mounts in linux. +// IsNotMountPoint enumerates all the mountpoints using List() and +// the list of mountpoints may be large, then it uses +// isMountPointMatch to evaluate whether the directory is a mountpoint. +func IsNotMountPoint(mounter Interface, file string) (bool, error) { + // IsLikelyNotMountPoint provides a quick check + // to determine whether file IS A mountpoint. + notMnt, notMntErr := mounter.IsLikelyNotMountPoint(file) + if notMntErr != nil && os.IsPermission(notMntErr) { + // We were not allowed to do the simple stat() check, e.g. on NFS with + // root_squash. Fall back to /proc/mounts check below. + notMnt = true + notMntErr = nil + } + if notMntErr != nil { + return notMnt, notMntErr + } + // identified as mountpoint, so return this fact. + if notMnt == false { + return notMnt, nil + } + + // Resolve any symlinks in file, kernel would do the same and use the resolved path in /proc/mounts. + resolvedFile, err := filepath.EvalSymlinks(file) + if err != nil { + return true, err + } + + // check all mountpoints since IsLikelyNotMountPoint + // is not reliable for some mountpoint types. + mountPoints, mountPointsErr := mounter.List() + if mountPointsErr != nil { + return notMnt, mountPointsErr + } + for _, mp := range mountPoints { + if isMountPointMatch(mp, resolvedFile) { + notMnt = false + break + } + } + return notMnt, nil +} + +// MakeBindOpts detects whether a bind mount is being requested and makes the remount options to +// use in case of bind mount, due to the fact that bind mount doesn't respect mount options. +// The list equals: +// options - 'bind' + 'remount' (no duplicate) +func MakeBindOpts(options []string) (bool, []string, []string) { + bind, bindOpts, bindRemountOpts, _ := MakeBindOptsSensitive(options, nil /* sensitiveOptions */) + return bind, bindOpts, bindRemountOpts +} + +// MakeBindOptsSensitive is the same as MakeBindOpts but this method allows +// sensitiveOptions to be passed in a separate parameter from the normal mount +// options and ensures the sensitiveOptions are never logged. This method should +// be used by callers that pass sensitive material (like passwords) as mount +// options. +func MakeBindOptsSensitive(options []string, sensitiveOptions []string) (bool, []string, []string, []string) { + // Because we have an FD opened on the subpath bind mount, the "bind" option + // needs to be included, otherwise the mount target will error as busy if you + // remount as readonly. + // + // As a consequence, all read only bind mounts will no longer change the underlying + // volume mount to be read only. + bindRemountOpts := []string{"bind", "remount"} + bindRemountSensitiveOpts := []string{} + bind := false + bindOpts := []string{"bind"} + + // _netdev is a userspace mount option and does not automatically get added when + // bind mount is created and hence we must carry it over. + if checkForNetDev(options, sensitiveOptions) { + bindOpts = append(bindOpts, "_netdev") + } + + for _, option := range options { + switch option { + case "bind": + bind = true + case "remount": + default: + bindRemountOpts = append(bindRemountOpts, option) + } + } + + for _, sensitiveOption := range sensitiveOptions { + switch sensitiveOption { + case "bind": + bind = true + case "remount": + default: + bindRemountSensitiveOpts = append(bindRemountSensitiveOpts, sensitiveOption) + } + } + + return bind, bindOpts, bindRemountOpts, bindRemountSensitiveOpts +} + +func checkForNetDev(options []string, sensitiveOptions []string) bool { + for _, option := range options { + if option == "_netdev" { + return true + } + } + for _, sensitiveOption := range sensitiveOptions { + if sensitiveOption == "_netdev" { + return true + } + } + return false +} + +// PathWithinBase checks if give path is within given base directory. +func PathWithinBase(fullPath, basePath string) bool { + rel, err := filepath.Rel(basePath, fullPath) + if err != nil { + return false + } + if StartsWithBackstep(rel) { + // Needed to escape the base path. + return false + } + return true +} + +// StartsWithBackstep checks if the given path starts with a backstep segment. +func StartsWithBackstep(rel string) bool { + // normalize to / and check for ../ + return rel == ".." || strings.HasPrefix(filepath.ToSlash(rel), "../") +} + +// sanitizedOptionsForLogging will return a comma separated string containing +// options and sensitiveOptions. Each entry in sensitiveOptions will be +// replaced with the string sensitiveOptionsRemoved +// e.g. o1,o2,, +func sanitizedOptionsForLogging(options []string, sensitiveOptions []string) string { + separator := "" + if len(options) > 0 && len(sensitiveOptions) > 0 { + separator = "," + } + + sensitiveOptionsStart := "" + sensitiveOptionsEnd := "" + if len(sensitiveOptions) > 0 { + sensitiveOptionsStart = strings.Repeat(sensitiveOptionsRemoved+",", len(sensitiveOptions)-1) + sensitiveOptionsEnd = sensitiveOptionsRemoved + } + + return strings.Join(options, ",") + + separator + + sensitiveOptionsStart + + sensitiveOptionsEnd +} diff --git a/vendor/k8s.io/mount-utils/mount_helper_common.go b/vendor/k8s.io/mount-utils/mount_helper_common.go new file mode 100644 index 00000000..1d40549b --- /dev/null +++ b/vendor/k8s.io/mount-utils/mount_helper_common.go @@ -0,0 +1,103 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "fmt" + "os" + + "k8s.io/klog/v2" +) + +// CleanupMountPoint unmounts the given path and deletes the remaining directory +// if successful. If extensiveMountPointCheck is true IsNotMountPoint will be +// called instead of IsLikelyNotMountPoint. IsNotMountPoint is more expensive +// but properly handles bind mounts within the same fs. +func CleanupMountPoint(mountPath string, mounter Interface, extensiveMountPointCheck bool) error { + pathExists, pathErr := PathExists(mountPath) + if !pathExists { + klog.Warningf("Warning: Unmount skipped because path does not exist: %v", mountPath) + return nil + } + corruptedMnt := IsCorruptedMnt(pathErr) + if pathErr != nil && !corruptedMnt { + return fmt.Errorf("Error checking path: %v", pathErr) + } + return doCleanupMountPoint(mountPath, mounter, extensiveMountPointCheck, corruptedMnt) +} + +// doCleanupMountPoint unmounts the given path and +// deletes the remaining directory if successful. +// if extensiveMountPointCheck is true +// IsNotMountPoint will be called instead of IsLikelyNotMountPoint. +// IsNotMountPoint is more expensive but properly handles bind mounts within the same fs. +// if corruptedMnt is true, it means that the mountPath is a corrupted mountpoint, and the mount point check +// will be skipped +func doCleanupMountPoint(mountPath string, mounter Interface, extensiveMountPointCheck bool, corruptedMnt bool) error { + var notMnt bool + var err error + if !corruptedMnt { + if extensiveMountPointCheck { + notMnt, err = IsNotMountPoint(mounter, mountPath) + } else { + notMnt, err = mounter.IsLikelyNotMountPoint(mountPath) + } + + if err != nil { + return err + } + + if notMnt { + klog.Warningf("Warning: %q is not a mountpoint, deleting", mountPath) + return os.Remove(mountPath) + } + } + + // Unmount the mount path + klog.V(4).Infof("%q is a mountpoint, unmounting", mountPath) + if err := mounter.Unmount(mountPath); err != nil { + return err + } + + if extensiveMountPointCheck { + notMnt, err = IsNotMountPoint(mounter, mountPath) + } else { + notMnt, err = mounter.IsLikelyNotMountPoint(mountPath) + } + if err != nil { + return err + } + if notMnt { + klog.V(4).Infof("%q is unmounted, deleting the directory", mountPath) + return os.Remove(mountPath) + } + return fmt.Errorf("Failed to unmount path %v", mountPath) +} + +// PathExists returns true if the specified path exists. +// TODO: clean this up to use pkg/util/file/FileExists +func PathExists(path string) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return true, nil + } else if os.IsNotExist(err) { + return false, nil + } else if IsCorruptedMnt(err) { + return true, err + } + return false, err +} diff --git a/vendor/k8s.io/mount-utils/mount_helper_unix.go b/vendor/k8s.io/mount-utils/mount_helper_unix.go new file mode 100644 index 00000000..11b70ebc --- /dev/null +++ b/vendor/k8s.io/mount-utils/mount_helper_unix.go @@ -0,0 +1,158 @@ +// +build !windows + +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "fmt" + "os" + "strconv" + "strings" + "syscall" + + utilio "k8s.io/utils/io" +) + +const ( + // At least number of fields per line in /proc//mountinfo. + expectedAtLeastNumFieldsPerMountInfo = 10 + // How many times to retry for a consistent read of /proc/mounts. + maxListTries = 3 +) + +// IsCorruptedMnt return true if err is about corrupted mount point +func IsCorruptedMnt(err error) bool { + if err == nil { + return false + } + var underlyingError error + switch pe := err.(type) { + case nil: + return false + case *os.PathError: + underlyingError = pe.Err + case *os.LinkError: + underlyingError = pe.Err + case *os.SyscallError: + underlyingError = pe.Err + } + + return underlyingError == syscall.ENOTCONN || underlyingError == syscall.ESTALE || underlyingError == syscall.EIO || underlyingError == syscall.EACCES +} + +// MountInfo represents a single line in /proc//mountinfo. +type MountInfo struct { // nolint: golint + // Unique ID for the mount (maybe reused after umount). + ID int + // The ID of the parent mount (or of self for the root of this mount namespace's mount tree). + ParentID int + // Major indicates one half of the device ID which identifies the device class + // (parsed from `st_dev` for files on this filesystem). + Major int + // Minor indicates one half of the device ID which identifies a specific + // instance of device (parsed from `st_dev` for files on this filesystem). + Minor int + // The pathname of the directory in the filesystem which forms the root of this mount. + Root string + // Mount source, filesystem-specific information. e.g. device, tmpfs name. + Source string + // Mount point, the pathname of the mount point. + MountPoint string + // Optional fieds, zero or more fields of the form "tag[:value]". + OptionalFields []string + // The filesystem type in the form "type[.subtype]". + FsType string + // Per-mount options. + MountOptions []string + // Per-superblock options. + SuperOptions []string +} + +// ParseMountInfo parses /proc/xxx/mountinfo. +func ParseMountInfo(filename string) ([]MountInfo, error) { + content, err := utilio.ConsistentRead(filename, maxListTries) + if err != nil { + return []MountInfo{}, err + } + contentStr := string(content) + infos := []MountInfo{} + + for _, line := range strings.Split(contentStr, "\n") { + if line == "" { + // the last split() item is empty string following the last \n + continue + } + // See `man proc` for authoritative description of format of the file. + fields := strings.Fields(line) + if len(fields) < expectedAtLeastNumFieldsPerMountInfo { + return nil, fmt.Errorf("wrong number of fields in (expected at least %d, got %d): %s", expectedAtLeastNumFieldsPerMountInfo, len(fields), line) + } + id, err := strconv.Atoi(fields[0]) + if err != nil { + return nil, err + } + parentID, err := strconv.Atoi(fields[1]) + if err != nil { + return nil, err + } + mm := strings.Split(fields[2], ":") + if len(mm) != 2 { + return nil, fmt.Errorf("parsing '%s' failed: unexpected minor:major pair %s", line, mm) + } + major, err := strconv.Atoi(mm[0]) + if err != nil { + return nil, fmt.Errorf("parsing '%s' failed: unable to parse major device id, err:%v", mm[0], err) + } + minor, err := strconv.Atoi(mm[1]) + if err != nil { + return nil, fmt.Errorf("parsing '%s' failed: unable to parse minor device id, err:%v", mm[1], err) + } + + info := MountInfo{ + ID: id, + ParentID: parentID, + Major: major, + Minor: minor, + Root: fields[3], + MountPoint: fields[4], + MountOptions: strings.Split(fields[5], ","), + } + // All fields until "-" are "optional fields". + i := 6 + for ; i < len(fields) && fields[i] != "-"; i++ { + info.OptionalFields = append(info.OptionalFields, fields[i]) + } + // Parse the rest 3 fields. + i++ + if len(fields)-i < 3 { + return nil, fmt.Errorf("expect 3 fields in %s, got %d", line, len(fields)-i) + } + info.FsType = fields[i] + info.Source = fields[i+1] + info.SuperOptions = strings.Split(fields[i+2], ",") + infos = append(infos, info) + } + return infos, nil +} + +// isMountPointMatch returns true if the path in mp is the same as dir. +// Handles case where mountpoint dir has been renamed due to stale NFS mount. +func isMountPointMatch(mp MountPoint, dir string) bool { + deletedDir := fmt.Sprintf("%s\\040(deleted)", dir) + return ((mp.Path == dir) || (mp.Path == deletedDir)) +} diff --git a/vendor/k8s.io/mount-utils/mount_helper_windows.go b/vendor/k8s.io/mount-utils/mount_helper_windows.go new file mode 100644 index 00000000..b308ce76 --- /dev/null +++ b/vendor/k8s.io/mount-utils/mount_helper_windows.go @@ -0,0 +1,101 @@ +// +build windows + +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "fmt" + "os" + "strconv" + "strings" + "syscall" + + "k8s.io/klog/v2" +) + +// following failure codes are from https://docs.microsoft.com/en-us/windows/desktop/debug/system-error-codes--1300-1699- +// ERROR_BAD_NETPATH = 53 +// ERROR_NETWORK_BUSY = 54 +// ERROR_UNEXP_NET_ERR = 59 +// ERROR_NETNAME_DELETED = 64 +// ERROR_NETWORK_ACCESS_DENIED = 65 +// ERROR_BAD_DEV_TYPE = 66 +// ERROR_BAD_NET_NAME = 67 +// ERROR_SESSION_CREDENTIAL_CONFLICT = 1219 +// ERROR_LOGON_FAILURE = 1326 +var errorNoList = [...]int{53, 54, 59, 64, 65, 66, 67, 1219, 1326} + +// IsCorruptedMnt return true if err is about corrupted mount point +func IsCorruptedMnt(err error) bool { + if err == nil { + return false + } + + var underlyingError error + switch pe := err.(type) { + case nil: + return false + case *os.PathError: + underlyingError = pe.Err + case *os.LinkError: + underlyingError = pe.Err + case *os.SyscallError: + underlyingError = pe.Err + } + + if ee, ok := underlyingError.(syscall.Errno); ok { + for _, errno := range errorNoList { + if int(ee) == errno { + klog.Warningf("IsCorruptedMnt failed with error: %v, error code: %v", err, errno) + return true + } + } + } + + return false +} + +// NormalizeWindowsPath makes sure the given path is a valid path on Windows +// systems by making sure all instances of `/` are replaced with `\\`, and the +// path beings with `c:` +func NormalizeWindowsPath(path string) string { + normalizedPath := strings.Replace(path, "/", "\\", -1) + if strings.HasPrefix(normalizedPath, "\\") { + normalizedPath = "c:" + normalizedPath + } + return normalizedPath +} + +// ValidateDiskNumber : disk number should be a number in [0, 99] +func ValidateDiskNumber(disk string) error { + diskNum, err := strconv.Atoi(disk) + if err != nil { + return fmt.Errorf("wrong disk number format: %q, err:%v", disk, err) + } + + if diskNum < 0 || diskNum > 99 { + return fmt.Errorf("disk number out of range: %q", disk) + } + + return nil +} + +// isMountPointMatch determines if the mountpoint matches the dir +func isMountPointMatch(mp MountPoint, dir string) bool { + return mp.Path == dir +} diff --git a/vendor/k8s.io/mount-utils/mount_linux.go b/vendor/k8s.io/mount-utils/mount_linux.go new file mode 100644 index 00000000..20993cf0 --- /dev/null +++ b/vendor/k8s.io/mount-utils/mount_linux.go @@ -0,0 +1,575 @@ +// +build linux + +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + + "k8s.io/klog/v2" + utilexec "k8s.io/utils/exec" + utilio "k8s.io/utils/io" +) + +const ( + // Number of fields per line in /proc/mounts as per the fstab man page. + expectedNumFieldsPerLine = 6 + // Location of the mount file to use + procMountsPath = "/proc/mounts" + // Location of the mountinfo file + procMountInfoPath = "/proc/self/mountinfo" + // 'fsck' found errors and corrected them + fsckErrorsCorrected = 1 + // 'fsck' found errors but exited without correcting them + fsckErrorsUncorrected = 4 +) + +// Mounter provides the default implementation of mount.Interface +// for the linux platform. This implementation assumes that the +// kubelet is running in the host's root mount namespace. +type Mounter struct { + mounterPath string + withSystemd bool +} + +// New returns a mount.Interface for the current system. +// It provides options to override the default mounter behavior. +// mounterPath allows using an alternative to `/bin/mount` for mounting. +func New(mounterPath string) Interface { + return &Mounter{ + mounterPath: mounterPath, + withSystemd: detectSystemd(), + } +} + +// Mount mounts source to target as fstype with given options. 'source' and 'fstype' must +// be an empty string in case it's not required, e.g. for remount, or for auto filesystem +// type, where kernel handles fstype for you. The mount 'options' is a list of options, +// currently come from mount(8), e.g. "ro", "remount", "bind", etc. If no more option is +// required, call Mount with an empty string list or nil. +func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { + return mounter.MountSensitive(source, target, fstype, options, nil) +} + +// MountSensitive is the same as Mount() but this method allows +// sensitiveOptions to be passed in a separate parameter from the normal +// mount options and ensures the sensitiveOptions are never logged. This +// method should be used by callers that pass sensitive material (like +// passwords) as mount options. +func (mounter *Mounter) MountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error { + // Path to mounter binary if containerized mounter is needed. Otherwise, it is set to empty. + // All Linux distros are expected to be shipped with a mount utility that a support bind mounts. + mounterPath := "" + bind, bindOpts, bindRemountOpts, bindRemountOptsSensitive := MakeBindOptsSensitive(options, sensitiveOptions) + if bind { + err := mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindOpts, bindRemountOptsSensitive, true) + if err != nil { + return err + } + return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindRemountOpts, bindRemountOptsSensitive, true) + } + // The list of filesystems that require containerized mounter on GCI image cluster + fsTypesNeedMounter := map[string]struct{}{ + "nfs": {}, + "glusterfs": {}, + "ceph": {}, + "cifs": {}, + } + if _, ok := fsTypesNeedMounter[fstype]; ok { + mounterPath = mounter.mounterPath + } + return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, options, sensitiveOptions, true) +} + +// MountSensitiveWithoutSystemd is the same as MountSensitive() but disable using systemd mount. +func (mounter *Mounter) MountSensitiveWithoutSystemd(source string, target string, fstype string, options []string, sensitiveOptions []string) error { + mounterPath := "" + bind, bindOpts, bindRemountOpts, bindRemountOptsSensitive := MakeBindOptsSensitive(options, sensitiveOptions) + if bind { + err := mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindOpts, bindRemountOptsSensitive, false) + if err != nil { + return err + } + return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindRemountOpts, bindRemountOptsSensitive, false) + } + // The list of filesystems that require containerized mounter on GCI image cluster + fsTypesNeedMounter := map[string]struct{}{ + "nfs": {}, + "glusterfs": {}, + "ceph": {}, + "cifs": {}, + } + if _, ok := fsTypesNeedMounter[fstype]; ok { + mounterPath = mounter.mounterPath + } + return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, options, sensitiveOptions, false) +} + +// doMount runs the mount command. mounterPath is the path to mounter binary if containerized mounter is used. +// sensitiveOptions is an extension of options except they will not be logged (because they may contain sensitive material) +// systemdMountRequired is an extension of option to decide whether uses systemd mount. +func (mounter *Mounter) doMount(mounterPath string, mountCmd string, source string, target string, fstype string, options []string, sensitiveOptions []string, systemdMountRequired bool) error { + mountArgs, mountArgsLogStr := MakeMountArgsSensitive(source, target, fstype, options, sensitiveOptions) + if len(mounterPath) > 0 { + mountArgs = append([]string{mountCmd}, mountArgs...) + mountArgsLogStr = mountCmd + " " + mountArgsLogStr + mountCmd = mounterPath + } + + if mounter.withSystemd && systemdMountRequired { + // Try to run mount via systemd-run --scope. This will escape the + // service where kubelet runs and any fuse daemons will be started in a + // specific scope. kubelet service than can be restarted without killing + // these fuse daemons. + // + // Complete command line (when mounterPath is not used): + // systemd-run --description=... --scope -- mount -t + // + // Expected flow: + // * systemd-run creates a transient scope (=~ cgroup) and executes its + // argument (/bin/mount) there. + // * mount does its job, forks a fuse daemon if necessary and finishes. + // (systemd-run --scope finishes at this point, returning mount's exit + // code and stdout/stderr - thats one of --scope benefits). + // * systemd keeps the fuse daemon running in the scope (i.e. in its own + // cgroup) until the fuse daemon dies (another --scope benefit). + // Kubelet service can be restarted and the fuse daemon survives. + // * When the fuse daemon dies (e.g. during unmount) systemd removes the + // scope automatically. + // + // systemd-mount is not used because it's too new for older distros + // (CentOS 7, Debian Jessie). + mountCmd, mountArgs, mountArgsLogStr = AddSystemdScopeSensitive("systemd-run", target, mountCmd, mountArgs, mountArgsLogStr) + // } else { + // No systemd-run on the host (or we failed to check it), assume kubelet + // does not run as a systemd service. + // No code here, mountCmd and mountArgs are already populated. + } + + // Logging with sensitive mount options removed. + klog.V(4).Infof("Mounting cmd (%s) with arguments (%s)", mountCmd, mountArgsLogStr) + command := exec.Command(mountCmd, mountArgs...) + output, err := command.CombinedOutput() + if err != nil { + klog.Errorf("Mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s\n", err, mountCmd, mountArgsLogStr, string(output)) + return fmt.Errorf("mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s", + err, mountCmd, mountArgsLogStr, string(output)) + } + return err +} + +// detectSystemd returns true if OS runs with systemd as init. When not sure +// (permission errors, ...), it returns false. +// There may be different ways how to detect systemd, this one makes sure that +// systemd-runs (needed by Mount()) works. +func detectSystemd() bool { + if _, err := exec.LookPath("systemd-run"); err != nil { + klog.V(2).Infof("Detected OS without systemd") + return false + } + // Try to run systemd-run --scope /bin/true, that should be enough + // to make sure that systemd is really running and not just installed, + // which happens when running in a container with a systemd-based image + // but with different pid 1. + cmd := exec.Command("systemd-run", "--description=Kubernetes systemd probe", "--scope", "true") + output, err := cmd.CombinedOutput() + if err != nil { + klog.V(2).Infof("Cannot run systemd-run, assuming non-systemd OS") + klog.V(4).Infof("systemd-run output: %s, failed with: %v", string(output), err) + return false + } + klog.V(2).Infof("Detected OS with systemd") + return true +} + +// MakeMountArgs makes the arguments to the mount(8) command. +// options MUST not contain sensitive material (like passwords). +func MakeMountArgs(source, target, fstype string, options []string) (mountArgs []string) { + mountArgs, _ = MakeMountArgsSensitive(source, target, fstype, options, nil /* sensitiveOptions */) + return mountArgs +} + +// MakeMountArgsSensitive makes the arguments to the mount(8) command. +// sensitiveOptions is an extension of options except they will not be logged (because they may contain sensitive material) +func MakeMountArgsSensitive(source, target, fstype string, options []string, sensitiveOptions []string) (mountArgs []string, mountArgsLogStr string) { + // Build mount command as follows: + // mount [-t $fstype] [-o $options] [$source] $target + mountArgs = []string{} + mountArgsLogStr = "" + if len(fstype) > 0 { + mountArgs = append(mountArgs, "-t", fstype) + mountArgsLogStr += strings.Join(mountArgs, " ") + } + if len(options) > 0 || len(sensitiveOptions) > 0 { + combinedOptions := []string{} + combinedOptions = append(combinedOptions, options...) + combinedOptions = append(combinedOptions, sensitiveOptions...) + mountArgs = append(mountArgs, "-o", strings.Join(combinedOptions, ",")) + // exclude sensitiveOptions from log string + mountArgsLogStr += " -o " + sanitizedOptionsForLogging(options, sensitiveOptions) + } + if len(source) > 0 { + mountArgs = append(mountArgs, source) + mountArgsLogStr += " " + source + } + mountArgs = append(mountArgs, target) + mountArgsLogStr += " " + target + + return mountArgs, mountArgsLogStr +} + +// AddSystemdScope adds "system-run --scope" to given command line +// If args contains sensitive material, use AddSystemdScopeSensitive to construct +// a safe to log string. +func AddSystemdScope(systemdRunPath, mountName, command string, args []string) (string, []string) { + descriptionArg := fmt.Sprintf("--description=Kubernetes transient mount for %s", mountName) + systemdRunArgs := []string{descriptionArg, "--scope", "--", command} + return systemdRunPath, append(systemdRunArgs, args...) +} + +// AddSystemdScopeSensitive adds "system-run --scope" to given command line +// It also accepts takes a sanitized string containing mount arguments, mountArgsLogStr, +// and returns the string appended to the systemd command for logging. +func AddSystemdScopeSensitive(systemdRunPath, mountName, command string, args []string, mountArgsLogStr string) (string, []string, string) { + descriptionArg := fmt.Sprintf("--description=Kubernetes transient mount for %s", mountName) + systemdRunArgs := []string{descriptionArg, "--scope", "--", command} + return systemdRunPath, append(systemdRunArgs, args...), strings.Join(systemdRunArgs, " ") + " " + mountArgsLogStr +} + +// Unmount unmounts the target. +func (mounter *Mounter) Unmount(target string) error { + klog.V(4).Infof("Unmounting %s", target) + command := exec.Command("umount", target) + output, err := command.CombinedOutput() + if err != nil { + return fmt.Errorf("unmount failed: %v\nUnmounting arguments: %s\nOutput: %s", err, target, string(output)) + } + return nil +} + +// List returns a list of all mounted filesystems. +func (*Mounter) List() ([]MountPoint, error) { + return ListProcMounts(procMountsPath) +} + +// IsLikelyNotMountPoint determines if a directory is not a mountpoint. +// It is fast but not necessarily ALWAYS correct. If the path is in fact +// a bind mount from one part of a mount to another it will not be detected. +// It also can not distinguish between mountpoints and symbolic links. +// mkdir /tmp/a /tmp/b; mount --bind /tmp/a /tmp/b; IsLikelyNotMountPoint("/tmp/b") +// will return true. When in fact /tmp/b is a mount point. If this situation +// is of interest to you, don't use this function... +func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { + stat, err := os.Stat(file) + if err != nil { + return true, err + } + rootStat, err := os.Stat(filepath.Dir(strings.TrimSuffix(file, "/"))) + if err != nil { + return true, err + } + // If the directory has a different device as parent, then it is a mountpoint. + if stat.Sys().(*syscall.Stat_t).Dev != rootStat.Sys().(*syscall.Stat_t).Dev { + return false, nil + } + + return true, nil +} + +// GetMountRefs finds all mount references to pathname, returns a +// list of paths. Path could be a mountpoint or a normal +// directory (for bind mount). +func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { + pathExists, pathErr := PathExists(pathname) + if !pathExists { + return []string{}, nil + } else if IsCorruptedMnt(pathErr) { + klog.Warningf("GetMountRefs found corrupted mount at %s, treating as unmounted path", pathname) + return []string{}, nil + } else if pathErr != nil { + return nil, fmt.Errorf("error checking path %s: %v", pathname, pathErr) + } + realpath, err := filepath.EvalSymlinks(pathname) + if err != nil { + return nil, err + } + return SearchMountPoints(realpath, procMountInfoPath) +} + +// checkAndRepairFileSystem checks and repairs filesystems using command fsck. +func (mounter *SafeFormatAndMount) checkAndRepairFilesystem(source string) error { + klog.V(4).Infof("Checking for issues with fsck on disk: %s", source) + args := []string{"-a", source} + out, err := mounter.Exec.Command("fsck", args...).CombinedOutput() + if err != nil { + ee, isExitError := err.(utilexec.ExitError) + switch { + case err == utilexec.ErrExecutableNotFound: + klog.Warningf("'fsck' not found on system; continuing mount without running 'fsck'.") + case isExitError && ee.ExitStatus() == fsckErrorsCorrected: + klog.Infof("Device %s has errors which were corrected by fsck.", source) + case isExitError && ee.ExitStatus() == fsckErrorsUncorrected: + return NewMountError(HasFilesystemErrors, "'fsck' found errors on device %s but could not correct them: %s", source, string(out)) + case isExitError && ee.ExitStatus() > fsckErrorsUncorrected: + klog.Infof("`fsck` error %s", string(out)) + } + } + return nil +} + +// formatAndMount uses unix utils to format and mount the given disk +func (mounter *SafeFormatAndMount) formatAndMountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error { + readOnly := false + for _, option := range options { + if option == "ro" { + readOnly = true + break + } + } + if !readOnly { + // Check sensitiveOptions for ro + for _, option := range sensitiveOptions { + if option == "ro" { + readOnly = true + break + } + } + } + + options = append(options, "defaults") + mountErrorValue := UnknownMountError + + // Check if the disk is already formatted + existingFormat, err := mounter.GetDiskFormat(source) + if err != nil { + return NewMountError(GetDiskFormatFailed, "failed to get disk format of disk %s: %v", source, err) + } + + // Use 'ext4' as the default + if len(fstype) == 0 { + fstype = "ext4" + } + + if existingFormat == "" { + // Do not attempt to format the disk if mounting as readonly, return an error to reflect this. + if readOnly { + return NewMountError(UnformattedReadOnly, "cannot mount unformatted disk %s as we are manipulating it in read-only mode", source) + } + + // Disk is unformatted so format it. + args := []string{source} + if fstype == "ext4" || fstype == "ext3" { + args = []string{ + "-F", // Force flag + "-m0", // Zero blocks reserved for super-user + source, + } + } + + klog.Infof("Disk %q appears to be unformatted, attempting to format as type: %q with options: %v", source, fstype, args) + output, err := mounter.Exec.Command("mkfs."+fstype, args...).CombinedOutput() + if err != nil { + // Do not log sensitiveOptions only options + sensitiveOptionsLog := sanitizedOptionsForLogging(options, sensitiveOptions) + detailedErr := fmt.Sprintf("format of disk %q failed: type:(%q) target:(%q) options:(%q) errcode:(%v) output:(%v) ", source, fstype, target, sensitiveOptionsLog, err, string(output)) + klog.Error(detailedErr) + return NewMountError(FormatFailed, detailedErr) + } + + klog.Infof("Disk successfully formatted (mkfs): %s - %s %s", fstype, source, target) + } else { + if fstype != existingFormat { + // Verify that the disk is formatted with filesystem type we are expecting + mountErrorValue = FilesystemMismatch + klog.Warningf("Configured to mount disk %s as %s but current format is %s, things might break", source, existingFormat, fstype) + } + + if !readOnly { + // Run check tools on the disk to fix repairable issues, only do this for formatted volumes requested as rw. + err := mounter.checkAndRepairFilesystem(source) + if err != nil { + return err + } + } + } + + // Mount the disk + klog.V(4).Infof("Attempting to mount disk %s in %s format at %s", source, fstype, target) + if err := mounter.MountSensitive(source, target, fstype, options, sensitiveOptions); err != nil { + return NewMountError(mountErrorValue, err.Error()) + } + + return nil +} + +// GetDiskFormat uses 'blkid' to see if the given disk is unformatted +func (mounter *SafeFormatAndMount) GetDiskFormat(disk string) (string, error) { + args := []string{"-p", "-s", "TYPE", "-s", "PTTYPE", "-o", "export", disk} + klog.V(4).Infof("Attempting to determine if disk %q is formatted using blkid with args: (%v)", disk, args) + dataOut, err := mounter.Exec.Command("blkid", args...).CombinedOutput() + output := string(dataOut) + klog.V(4).Infof("Output: %q", output) + + if err != nil { + if exit, ok := err.(utilexec.ExitError); ok { + if exit.ExitStatus() == 2 { + // Disk device is unformatted. + // For `blkid`, if the specified token (TYPE/PTTYPE, etc) was + // not found, or no (specified) devices could be identified, an + // exit code of 2 is returned. + return "", nil + } + } + klog.Errorf("Could not determine if disk %q is formatted (%v)", disk, err) + return "", err + } + + var fstype, pttype string + + lines := strings.Split(output, "\n") + for _, l := range lines { + if len(l) <= 0 { + // Ignore empty line. + continue + } + cs := strings.Split(l, "=") + if len(cs) != 2 { + return "", fmt.Errorf("blkid returns invalid output: %s", output) + } + // TYPE is filesystem type, and PTTYPE is partition table type, according + // to https://www.kernel.org/pub/linux/utils/util-linux/v2.21/libblkid-docs/. + if cs[0] == "TYPE" { + fstype = cs[1] + } else if cs[0] == "PTTYPE" { + pttype = cs[1] + } + } + + if len(pttype) > 0 { + klog.V(4).Infof("Disk %s detected partition table type: %s", disk, pttype) + // Returns a special non-empty string as filesystem type, then kubelet + // will not format it. + return "unknown data, probably partitions", nil + } + + return fstype, nil +} + +// ListProcMounts is shared with NsEnterMounter +func ListProcMounts(mountFilePath string) ([]MountPoint, error) { + content, err := utilio.ConsistentRead(mountFilePath, maxListTries) + if err != nil { + return nil, err + } + return parseProcMounts(content) +} + +func parseProcMounts(content []byte) ([]MountPoint, error) { + out := []MountPoint{} + lines := strings.Split(string(content), "\n") + for _, line := range lines { + if line == "" { + // the last split() item is empty string following the last \n + continue + } + fields := strings.Fields(line) + if len(fields) != expectedNumFieldsPerLine { + // Do not log line in case it contains sensitive Mount options + return nil, fmt.Errorf("wrong number of fields (expected %d, got %d)", expectedNumFieldsPerLine, len(fields)) + } + + mp := MountPoint{ + Device: fields[0], + Path: fields[1], + Type: fields[2], + Opts: strings.Split(fields[3], ","), + } + + freq, err := strconv.Atoi(fields[4]) + if err != nil { + return nil, err + } + mp.Freq = freq + + pass, err := strconv.Atoi(fields[5]) + if err != nil { + return nil, err + } + mp.Pass = pass + + out = append(out, mp) + } + return out, nil +} + +// SearchMountPoints finds all mount references to the source, returns a list of +// mountpoints. +// The source can be a mount point or a normal directory (bind mount). We +// didn't support device because there is no use case by now. +// Some filesystems may share a source name, e.g. tmpfs. And for bind mounting, +// it's possible to mount a non-root path of a filesystem, so we need to use +// root path and major:minor to represent mount source uniquely. +// This implementation is shared between Linux and NsEnterMounter +func SearchMountPoints(hostSource, mountInfoPath string) ([]string, error) { + mis, err := ParseMountInfo(mountInfoPath) + if err != nil { + return nil, err + } + + mountID := 0 + rootPath := "" + major := -1 + minor := -1 + + // Finding the underlying root path and major:minor if possible. + // We need search in backward order because it's possible for later mounts + // to overlap earlier mounts. + for i := len(mis) - 1; i >= 0; i-- { + if hostSource == mis[i].MountPoint || PathWithinBase(hostSource, mis[i].MountPoint) { + // If it's a mount point or path under a mount point. + mountID = mis[i].ID + rootPath = filepath.Join(mis[i].Root, strings.TrimPrefix(hostSource, mis[i].MountPoint)) + major = mis[i].Major + minor = mis[i].Minor + break + } + } + + if rootPath == "" || major == -1 || minor == -1 { + return nil, fmt.Errorf("failed to get root path and major:minor for %s", hostSource) + } + + var refs []string + for i := range mis { + if mis[i].ID == mountID { + // Ignore mount entry for mount source itself. + continue + } + if mis[i].Root == rootPath && mis[i].Major == major && mis[i].Minor == minor { + refs = append(refs, mis[i].MountPoint) + } + } + + return refs, nil +} diff --git a/vendor/k8s.io/mount-utils/mount_unsupported.go b/vendor/k8s.io/mount-utils/mount_unsupported.go new file mode 100644 index 00000000..0e8e683a --- /dev/null +++ b/vendor/k8s.io/mount-utils/mount_unsupported.go @@ -0,0 +1,82 @@ +// +build !linux,!windows + +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "errors" +) + +// Mounter implements mount.Interface for unsupported platforms +type Mounter struct { + mounterPath string +} + +var errUnsupported = errors.New("util/mount on this platform is not supported") + +// New returns a mount.Interface for the current system. +// It provides options to override the default mounter behavior. +// mounterPath allows using an alternative to `/bin/mount` for mounting. +func New(mounterPath string) Interface { + return &Mounter{ + mounterPath: mounterPath, + } +} + +// Mount always returns an error on unsupported platforms +func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { + return errUnsupported +} + +// MountSensitive always returns an error on unsupported platforms +func (mounter *Mounter) MountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error { + return errUnsupported +} + +// MountSensitiveWithoutSystemd always returns an error on unsupported platforms +func (mounter *Mounter) MountSensitiveWithoutSystemd(source string, target string, fstype string, options []string, sensitiveOptions []string) error { + return errUnsupported +} + +// Unmount always returns an error on unsupported platforms +func (mounter *Mounter) Unmount(target string) error { + return errUnsupported +} + +// List always returns an error on unsupported platforms +func (mounter *Mounter) List() ([]MountPoint, error) { + return []MountPoint{}, errUnsupported +} + +// IsLikelyNotMountPoint always returns an error on unsupported platforms +func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { + return true, errUnsupported +} + +// GetMountRefs always returns an error on unsupported platforms +func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { + return nil, errUnsupported +} + +func (mounter *SafeFormatAndMount) formatAndMountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error { + return mounter.Interface.Mount(source, target, fstype, options) +} + +func (mounter *SafeFormatAndMount) diskLooksUnformatted(disk string) (bool, error) { + return true, errUnsupported +} diff --git a/vendor/k8s.io/mount-utils/mount_windows.go b/vendor/k8s.io/mount-utils/mount_windows.go new file mode 100644 index 00000000..358bcf52 --- /dev/null +++ b/vendor/k8s.io/mount-utils/mount_windows.go @@ -0,0 +1,330 @@ +// +build windows + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "k8s.io/klog/v2" + "k8s.io/utils/keymutex" +) + +const ( + accessDenied string = "access is denied" +) + +// Mounter provides the default implementation of mount.Interface +// for the windows platform. This implementation assumes that the +// kubelet is running in the host's root mount namespace. +type Mounter struct { + mounterPath string +} + +// New returns a mount.Interface for the current system. +// It provides options to override the default mounter behavior. +// mounterPath allows using an alternative to `/bin/mount` for mounting. +func New(mounterPath string) Interface { + return &Mounter{ + mounterPath: mounterPath, + } +} + +// acquire lock for smb mount +var getSMBMountMutex = keymutex.NewHashed(0) + +// Mount : mounts source to target with given options. +// currently only supports cifs(smb), bind mount(for disk) +func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { + return mounter.MountSensitive(source, target, fstype, options, nil /* sensitiveOptions */) +} + +// MountSensitiveWithoutSystemd is the same as MountSensitive() but disable using ssytemd mount. +// Windows not supported systemd mount, this function degrades to MountSensitive(). +func (mounter *Mounter) MountSensitiveWithoutSystemd(source string, target string, fstype string, options []string, sensitiveOptions []string) error { + return mounter.MountSensitive(source, target, fstype, options, sensitiveOptions /* sensitiveOptions */) +} + +// MountSensitive is the same as Mount() but this method allows +// sensitiveOptions to be passed in a separate parameter from the normal +// mount options and ensures the sensitiveOptions are never logged. This +// method should be used by callers that pass sensitive material (like +// passwords) as mount options. +func (mounter *Mounter) MountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error { + target = NormalizeWindowsPath(target) + sanitizedOptionsForLogging := sanitizedOptionsForLogging(options, sensitiveOptions) + + if source == "tmpfs" { + klog.V(3).Infof("mounting source (%q), target (%q), with options (%q)", source, target, sanitizedOptionsForLogging) + return os.MkdirAll(target, 0755) + } + + parentDir := filepath.Dir(target) + if err := os.MkdirAll(parentDir, 0755); err != nil { + return err + } + + klog.V(4).Infof("mount options(%q) source:%q, target:%q, fstype:%q, begin to mount", + sanitizedOptionsForLogging, source, target, fstype) + bindSource := source + + if bind, _, _, _ := MakeBindOptsSensitive(options, sensitiveOptions); bind { + bindSource = NormalizeWindowsPath(source) + } else { + allOptions := []string{} + allOptions = append(allOptions, options...) + allOptions = append(allOptions, sensitiveOptions...) + if len(allOptions) < 2 { + return fmt.Errorf("mount options(%q) should have at least 2 options, current number:%d, source:%q, target:%q", + sanitizedOptionsForLogging, len(allOptions), source, target) + } + + // currently only cifs mount is supported + if strings.ToLower(fstype) != "cifs" { + return fmt.Errorf("only cifs mount is supported now, fstype: %q, mounting source (%q), target (%q), with options (%q)", fstype, source, target, sanitizedOptionsForLogging) + } + + // lock smb mount for the same source + getSMBMountMutex.LockKey(source) + defer getSMBMountMutex.UnlockKey(source) + + username := allOptions[0] + password := allOptions[1] + if output, err := newSMBMapping(username, password, source); err != nil { + klog.Warningf("SMB Mapping(%s) returned with error(%v), output(%s)", source, err, string(output)) + if isSMBMappingExist(source) { + valid, err := isValidPath(source) + if !valid { + if err == nil || isAccessDeniedError(err) { + klog.V(2).Infof("SMB Mapping(%s) already exists while it's not valid, return error: %v, now begin to remove and remount", source, err) + if output, err = removeSMBMapping(source); err != nil { + return fmt.Errorf("Remove-SmbGlobalMapping failed: %v, output: %q", err, output) + } + if output, err := newSMBMapping(username, password, source); err != nil { + return fmt.Errorf("New-SmbGlobalMapping(%s) failed: %v, output: %q", source, err, output) + } + } + } else { + klog.V(2).Infof("SMB Mapping(%s) already exists and is still valid, skip error(%v)", source, err) + } + } else { + return fmt.Errorf("New-SmbGlobalMapping(%s) failed: %v, output: %q", source, err, output) + } + } + } + + // There is an issue in golang where EvalSymlinks fails on Windows when passed a + // UNC share root path without a trailing backslash. + // Ex: \\SERVER\share will fail to resolve but \\SERVER\share\ will resolve + // containerD on Windows calls EvalSymlinks so we'll add the backslash when making the symlink if it is missing. + // https://github.com/golang/go/pull/42096 fixes this issue in golang but a fix will not be available until + // golang v1.16 + mklinkSource := bindSource + if !strings.HasSuffix(mklinkSource, "\\") { + mklinkSource = mklinkSource + "\\" + } + + output, err := exec.Command("cmd", "/c", "mklink", "/D", target, mklinkSource).CombinedOutput() + if err != nil { + klog.Errorf("mklink failed: %v, source(%q) target(%q) output: %q", err, mklinkSource, target, string(output)) + return err + } + klog.V(2).Infof("mklink source(%q) on target(%q) successfully, output: %q", mklinkSource, target, string(output)) + + return nil +} + +// do the SMB mount with username, password, remotepath +// return (output, error) +func newSMBMapping(username, password, remotepath string) (string, error) { + if username == "" || password == "" || remotepath == "" { + return "", fmt.Errorf("invalid parameter(username: %s, password: %s, remoteapth: %s)", username, sensitiveOptionsRemoved, remotepath) + } + + // use PowerShell Environment Variables to store user input string to prevent command line injection + // https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_environment_variables?view=powershell-5.1 + cmdLine := `$PWord = ConvertTo-SecureString -String $Env:smbpassword -AsPlainText -Force` + + `;$Credential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $Env:smbuser, $PWord` + + `;New-SmbGlobalMapping -RemotePath $Env:smbremotepath -Credential $Credential` + cmd := exec.Command("powershell", "/c", cmdLine) + cmd.Env = append(os.Environ(), + fmt.Sprintf("smbuser=%s", username), + fmt.Sprintf("smbpassword=%s", password), + fmt.Sprintf("smbremotepath=%s", remotepath)) + + output, err := cmd.CombinedOutput() + return string(output), err +} + +// check whether remotepath is already mounted +func isSMBMappingExist(remotepath string) bool { + cmd := exec.Command("powershell", "/c", `Get-SmbGlobalMapping -RemotePath $Env:smbremotepath`) + cmd.Env = append(os.Environ(), fmt.Sprintf("smbremotepath=%s", remotepath)) + _, err := cmd.CombinedOutput() + return err == nil +} + +// check whether remotepath is valid +// return (true, nil) if remotepath is valid +func isValidPath(remotepath string) (bool, error) { + cmd := exec.Command("powershell", "/c", `Test-Path $Env:remoteapth`) + cmd.Env = append(os.Environ(), fmt.Sprintf("remoteapth=%s", remotepath)) + output, err := cmd.CombinedOutput() + if err != nil { + return false, fmt.Errorf("returned output: %s, error: %v", string(output), err) + } + + return strings.HasPrefix(strings.ToLower(string(output)), "true"), nil +} + +func isAccessDeniedError(err error) bool { + return err != nil && strings.Contains(strings.ToLower(err.Error()), accessDenied) +} + +// remove SMB mapping +func removeSMBMapping(remotepath string) (string, error) { + cmd := exec.Command("powershell", "/c", `Remove-SmbGlobalMapping -RemotePath $Env:smbremotepath -Force`) + cmd.Env = append(os.Environ(), fmt.Sprintf("smbremotepath=%s", remotepath)) + output, err := cmd.CombinedOutput() + return string(output), err +} + +// Unmount unmounts the target. +func (mounter *Mounter) Unmount(target string) error { + klog.V(4).Infof("azureMount: Unmount target (%q)", target) + target = NormalizeWindowsPath(target) + if output, err := exec.Command("cmd", "/c", "rmdir", target).CombinedOutput(); err != nil { + klog.Errorf("rmdir failed: %v, output: %q", err, string(output)) + return err + } + return nil +} + +// List returns a list of all mounted filesystems. todo +func (mounter *Mounter) List() ([]MountPoint, error) { + return []MountPoint{}, nil +} + +// IsLikelyNotMountPoint determines if a directory is not a mountpoint. +func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { + stat, err := os.Lstat(file) + if err != nil { + return true, err + } + + if stat.Mode()&os.ModeSymlink != 0 { + return false, err + } + return true, nil +} + +// GetMountRefs : empty implementation here since there is no place to query all mount points on Windows +func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { + windowsPath := NormalizeWindowsPath(pathname) + pathExists, pathErr := PathExists(windowsPath) + if !pathExists { + return []string{}, nil + } else if IsCorruptedMnt(pathErr) { + klog.Warningf("GetMountRefs found corrupted mount at %s, treating as unmounted path", windowsPath) + return []string{}, nil + } else if pathErr != nil { + return nil, fmt.Errorf("error checking path %s: %v", windowsPath, pathErr) + } + return []string{pathname}, nil +} + +func (mounter *SafeFormatAndMount) formatAndMountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error { + // Try to mount the disk + klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, source, target) + + if err := ValidateDiskNumber(source); err != nil { + klog.Errorf("diskMount: formatAndMount failed, err: %v", err) + return err + } + + if len(fstype) == 0 { + // Use 'NTFS' as the default + fstype = "NTFS" + } + + // format disk if it is unformatted(raw) + cmd := fmt.Sprintf("Get-Disk -Number %s | Where partitionstyle -eq 'raw' | Initialize-Disk -PartitionStyle MBR -PassThru"+ + " | New-Partition -UseMaximumSize | Format-Volume -FileSystem %s -Confirm:$false", source, fstype) + if output, err := mounter.Exec.Command("powershell", "/c", cmd).CombinedOutput(); err != nil { + return fmt.Errorf("diskMount: format disk failed, error: %v, output: %q", err, string(output)) + } + klog.V(4).Infof("diskMount: Disk successfully formatted, disk: %q, fstype: %q", source, fstype) + + volumeIds, err := listVolumesOnDisk(source) + if err != nil { + return err + } + driverPath := volumeIds[0] + target = NormalizeWindowsPath(target) + output, err := mounter.Exec.Command("cmd", "/c", "mklink", "/D", target, driverPath).CombinedOutput() + if err != nil { + klog.Errorf("mklink(%s, %s) failed: %v, output: %q", target, driverPath, err, string(output)) + return err + } + klog.V(2).Infof("formatAndMount disk(%s) fstype(%s) on(%s) with output(%s) successfully", driverPath, fstype, target, string(output)) + return nil +} + +// ListVolumesOnDisk - returns back list of volumes(volumeIDs) in the disk (requested in diskID). +func listVolumesOnDisk(diskID string) (volumeIDs []string, err error) { + cmd := fmt.Sprintf("(Get-Disk -DeviceId %s | Get-Partition | Get-Volume).UniqueId", diskID) + output, err := exec.Command("powershell", "/c", cmd).CombinedOutput() + klog.V(4).Infof("listVolumesOnDisk id from %s: %s", diskID, string(output)) + if err != nil { + return []string{}, fmt.Errorf("error list volumes on disk. cmd: %s, output: %s, error: %v", cmd, string(output), err) + } + + volumeIds := strings.Split(strings.TrimSpace(string(output)), "\r\n") + return volumeIds, nil +} + +// getAllParentLinks walks all symbolic links and return all the parent targets recursively +func getAllParentLinks(path string) ([]string, error) { + const maxIter = 255 + links := []string{} + for { + links = append(links, path) + if len(links) > maxIter { + return links, fmt.Errorf("unexpected length of parent links: %v", links) + } + + fi, err := os.Lstat(path) + if err != nil { + return links, fmt.Errorf("Lstat: %v", err) + } + if fi.Mode()&os.ModeSymlink == 0 { + break + } + + path, err = os.Readlink(path) + if err != nil { + return links, fmt.Errorf("Readlink error: %v", err) + } + } + + return links, nil +} diff --git a/vendor/k8s.io/utils/nsenter/OWNERS b/vendor/k8s.io/utils/nsenter/OWNERS new file mode 100644 index 00000000..46895cbd --- /dev/null +++ b/vendor/k8s.io/utils/nsenter/OWNERS @@ -0,0 +1,10 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +reviewers: + - jsafrane + - msau42 + - cofyc +approvers: + - jsafrane + - msau42 + - cofyc diff --git a/vendor/k8s.io/utils/nsenter/README.md b/vendor/k8s.io/utils/nsenter/README.md new file mode 100644 index 00000000..aaacf8e3 --- /dev/null +++ b/vendor/k8s.io/utils/nsenter/README.md @@ -0,0 +1,4 @@ +# NSEnter + +This package provides interfaces for executing and interacting with processes +running within a namespace. diff --git a/vendor/k8s.io/utils/nsenter/nsenter.go b/vendor/k8s.io/utils/nsenter/nsenter.go new file mode 100644 index 00000000..54588390 --- /dev/null +++ b/vendor/k8s.io/utils/nsenter/nsenter.go @@ -0,0 +1,258 @@ +// +build linux + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nsenter + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "k8s.io/klog/v2" + "k8s.io/utils/exec" +) + +const ( + // DefaultHostRootFsPath is path to host's filesystem mounted into container + // with kubelet. + DefaultHostRootFsPath = "/rootfs" + // mountNsPath is the default mount namespace of the host + mountNsPath = "/proc/1/ns/mnt" + // nsenterPath is the default nsenter command + nsenterPath = "nsenter" +) + +// Nsenter is a type alias for backward compatibility +type Nsenter = NSEnter + +// NSEnter is part of experimental support for running the kubelet +// in a container. +// +// NSEnter requires: +// +// 1. Docker >= 1.6 due to the dependency on the slave propagation mode +// of the bind-mount of the kubelet root directory in the container. +// Docker 1.5 used a private propagation mode for bind-mounts, so mounts +// performed in the host's mount namespace do not propagate out to the +// bind-mount in this docker version. +// 2. The host's root filesystem must be available at /rootfs +// 3. The nsenter binary must be on the Kubelet process' PATH in the container's +// filesystem. +// 4. The Kubelet process must have CAP_SYS_ADMIN (required by nsenter); at +// the present, this effectively means that the kubelet is running in a +// privileged container. +// 5. The volume path used by the Kubelet must be the same inside and outside +// the container and be writable by the container (to initialize volume) +// contents. TODO: remove this requirement. +// 6. The host image must have "mount", "findmnt", "umount", "stat", "touch", +// "mkdir", "ls", "sh" and "chmod" binaries in /bin, /usr/sbin, or /usr/bin +// 7. The host image should have systemd-run in /bin, /usr/sbin, or /usr/bin if +// systemd is installed/enabled in the operating system. +// For more information about mount propagation modes, see: +// https://www.kernel.org/doc/Documentation/filesystems/sharedsubtree.txt +type NSEnter struct { + // a map of commands to their paths on the host filesystem + paths map[string]string + + // Path to the host filesystem, typically "/rootfs". Used only for testing. + hostRootFsPath string + + // Exec implementation + executor exec.Interface +} + +// NewNsenter constructs a new instance of NSEnter +func NewNsenter(hostRootFsPath string, executor exec.Interface) (*NSEnter, error) { + ne := &NSEnter{ + hostRootFsPath: hostRootFsPath, + executor: executor, + } + if err := ne.initPaths(); err != nil { + return nil, err + } + return ne, nil +} + +func (ne *NSEnter) initPaths() error { + ne.paths = map[string]string{} + binaries := []string{ + "mount", + "findmnt", + "umount", + "systemd-run", + "stat", + "touch", + "mkdir", + "sh", + "chmod", + "realpath", + } + // search for the required commands in other locations besides /usr/bin + for _, binary := range binaries { + // check for binary under the following directories + for _, path := range []string{"/", "/bin", "/usr/sbin", "/usr/bin"} { + binPath := filepath.Join(path, binary) + if _, err := os.Stat(filepath.Join(ne.hostRootFsPath, binPath)); err != nil { + continue + } + ne.paths[binary] = binPath + break + } + // systemd-run is optional, bailout if we don't find any of the other binaries + if ne.paths[binary] == "" && binary != "systemd-run" { + return fmt.Errorf("unable to find %v", binary) + } + } + return nil +} + +// Exec executes nsenter commands in hostProcMountNsPath mount namespace +func (ne *NSEnter) Exec(cmd string, args []string) exec.Cmd { + hostProcMountNsPath := filepath.Join(ne.hostRootFsPath, mountNsPath) + fullArgs := append([]string{fmt.Sprintf("--mount=%s", hostProcMountNsPath), "--"}, + append([]string{ne.AbsHostPath(cmd)}, args...)...) + klog.V(5).Infof("Running nsenter command: %v %v", nsenterPath, fullArgs) + return ne.executor.Command(nsenterPath, fullArgs...) +} + +// Command returns a command wrapped with nsenter +func (ne *NSEnter) Command(cmd string, args ...string) exec.Cmd { + return ne.Exec(cmd, args) +} + +// CommandContext returns a CommandContext wrapped with nsenter +func (ne *NSEnter) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd { + hostProcMountNsPath := filepath.Join(ne.hostRootFsPath, mountNsPath) + fullArgs := append([]string{fmt.Sprintf("--mount=%s", hostProcMountNsPath), "--"}, + append([]string{ne.AbsHostPath(cmd)}, args...)...) + klog.V(5).Infof("Running nsenter command: %v %v", nsenterPath, fullArgs) + return ne.executor.CommandContext(ctx, nsenterPath, fullArgs...) +} + +// LookPath returns a LookPath wrapped with nsenter +func (ne *NSEnter) LookPath(file string) (string, error) { + return "", fmt.Errorf("not implemented, error looking up : %s", file) +} + +// AbsHostPath returns the absolute runnable path for a specified command +func (ne *NSEnter) AbsHostPath(command string) string { + path, ok := ne.paths[command] + if !ok { + return command + } + return path +} + +// SupportsSystemd checks whether command systemd-run exists +func (ne *NSEnter) SupportsSystemd() (string, bool) { + systemdRunPath, ok := ne.paths["systemd-run"] + return systemdRunPath, ok && systemdRunPath != "" +} + +// EvalSymlinks returns the path name on the host after evaluating symlinks on the +// host. +// mustExist makes EvalSymlinks to return error when the path does not +// exist. When it's false, it evaluates symlinks of the existing part and +// blindly adds the non-existing part: +// pathname: /mnt/volume/non/existing/directory +// /mnt/volume exists +// non/existing/directory does not exist +// -> It resolves symlinks in /mnt/volume to say /mnt/foo and returns +// /mnt/foo/non/existing/directory. +// +// BEWARE! EvalSymlinks is not able to detect symlink looks with mustExist=false! +// If /tmp/link is symlink to /tmp/link, EvalSymlinks(/tmp/link/foo) returns /tmp/link/foo. +func (ne *NSEnter) EvalSymlinks(pathname string, mustExist bool) (string, error) { + var args []string + if mustExist { + // "realpath -e: all components of the path must exist" + args = []string{"-e", pathname} + } else { + // "realpath -m: no path components need exist or be a directory" + args = []string{"-m", pathname} + } + outBytes, err := ne.Exec("realpath", args).CombinedOutput() + if err != nil { + klog.Infof("failed to resolve symbolic links on %s: %v", pathname, err) + return "", err + } + return strings.TrimSpace(string(outBytes)), nil +} + +// KubeletPath returns the path name that can be accessed by containerized +// kubelet. It is recommended to resolve symlinks on the host by EvalSymlinks +// before calling this function +func (ne *NSEnter) KubeletPath(pathname string) string { + return filepath.Join(ne.hostRootFsPath, pathname) +} + +// NewFakeNsenter returns a NSEnter that does not run "nsenter --mount=... --", +// but runs everything in the same mount namespace as the unit test binary. +// rootfsPath is supposed to be a symlink, e.g. /tmp/xyz/rootfs -> /. +// This fake NSEnter is enough for most operations, e.g. to resolve symlinks, +// but it's not enough to call /bin/mount - unit tests don't run as root. +func NewFakeNsenter(rootfsPath string) (*NSEnter, error) { + executor := &fakeExec{ + rootfsPath: rootfsPath, + } + // prepare /rootfs/bin, usr/bin and usr/sbin + bin := filepath.Join(rootfsPath, "bin") + if err := os.Symlink("/bin", bin); err != nil { + return nil, err + } + + usr := filepath.Join(rootfsPath, "usr") + if err := os.Mkdir(usr, 0755); err != nil { + return nil, err + } + usrbin := filepath.Join(usr, "bin") + if err := os.Symlink("/usr/bin", usrbin); err != nil { + return nil, err + } + usrsbin := filepath.Join(usr, "sbin") + if err := os.Symlink("/usr/sbin", usrsbin); err != nil { + return nil, err + } + + return NewNsenter(rootfsPath, executor) +} + +type fakeExec struct { + rootfsPath string +} + +func (f fakeExec) Command(cmd string, args ...string) exec.Cmd { + // This will intentionaly panic if NSEnter does not provide enough arguments. + realCmd := args[2] + realArgs := args[3:] + return exec.New().Command(realCmd, realArgs...) +} + +func (fakeExec) LookPath(file string) (string, error) { + return "", errors.New("not implemented") +} + +func (fakeExec) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd { + return nil +} + +var _ exec.Interface = fakeExec{} +var _ exec.Interface = &NSEnter{} diff --git a/vendor/k8s.io/utils/nsenter/nsenter_unsupported.go b/vendor/k8s.io/utils/nsenter/nsenter_unsupported.go new file mode 100644 index 00000000..e38c7e81 --- /dev/null +++ b/vendor/k8s.io/utils/nsenter/nsenter_unsupported.go @@ -0,0 +1,79 @@ +// +build !linux + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nsenter + +import ( + "context" + "fmt" + + "k8s.io/utils/exec" +) + +const ( + // DefaultHostRootFsPath is path to host's filesystem mounted into container + // with kubelet. + DefaultHostRootFsPath = "/rootfs" +) + +// Nsenter is a type alias for backward compatibility +type Nsenter = NSEnter + +// NSEnter is part of experimental support for running the kubelet +// in a container. +type NSEnter struct { + // a map of commands to their paths on the host filesystem + Paths map[string]string +} + +// NewNsenter constructs a new instance of NSEnter +func NewNsenter(hostRootFsPath string, executor exec.Interface) (*Nsenter, error) { + return &Nsenter{}, nil +} + +// Exec executes nsenter commands in hostProcMountNsPath mount namespace +func (ne *NSEnter) Exec(cmd string, args []string) exec.Cmd { + return nil +} + +// AbsHostPath returns the absolute runnable path for a specified command +func (ne *NSEnter) AbsHostPath(command string) string { + return "" +} + +// SupportsSystemd checks whether command systemd-run exists +func (ne *NSEnter) SupportsSystemd() (string, bool) { + return "", false +} + +// Command returns a command wrapped with nenter +func (ne *NSEnter) Command(cmd string, args ...string) exec.Cmd { + return nil +} + +// CommandContext returns a CommandContext wrapped with nsenter +func (ne *NSEnter) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd { + return nil +} + +// LookPath returns a LookPath wrapped with nsenter +func (ne *NSEnter) LookPath(file string) (string, error) { + return "", fmt.Errorf("not implemented, error looking up : %s", file) +} + +var _ exec.Interface = &NSEnter{} diff --git a/vendor/modules.txt b/vendor/modules.txt index 333c3ad9..81ba03d8 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -648,6 +648,8 @@ k8s.io/client-go/util/homedir k8s.io/client-go/util/keyutil k8s.io/client-go/util/retry k8s.io/client-go/util/workqueue +# k8s.io/cloud-provider v0.20.0 => k8s.io/cloud-provider v0.20.0 +k8s.io/cloud-provider # k8s.io/component-base v0.20.0 => k8s.io/component-base v0.20.0 k8s.io/component-base/cli/flag k8s.io/component-base/config @@ -695,11 +697,20 @@ k8s.io/kubernetes/pkg/features k8s.io/kubernetes/pkg/fieldpath k8s.io/kubernetes/pkg/kubelet/apis/config k8s.io/kubernetes/pkg/kubelet/types +k8s.io/kubernetes/pkg/proxy/util k8s.io/kubernetes/pkg/security/apparmor k8s.io/kubernetes/pkg/util/hash k8s.io/kubernetes/pkg/util/labels k8s.io/kubernetes/pkg/util/parsers +k8s.io/kubernetes/pkg/util/sysctl k8s.io/kubernetes/pkg/util/taints +k8s.io/kubernetes/pkg/volume +k8s.io/kubernetes/pkg/volume/util/fs +k8s.io/kubernetes/pkg/volume/util/fsquota +k8s.io/kubernetes/pkg/volume/util/fsquota/common +k8s.io/kubernetes/pkg/volume/util/hostutil +k8s.io/kubernetes/pkg/volume/util/recyclerclient +k8s.io/kubernetes/pkg/volume/util/subpath k8s.io/kubernetes/test/e2e/framework k8s.io/kubernetes/test/e2e/framework/auth k8s.io/kubernetes/test/e2e/framework/config @@ -718,6 +729,8 @@ k8s.io/kubernetes/test/e2e/perftype k8s.io/kubernetes/test/e2e/storage/utils k8s.io/kubernetes/test/utils k8s.io/kubernetes/test/utils/image +# k8s.io/mount-utils v0.0.0 => k8s.io/mount-utils v0.21.0-alpha.0 +k8s.io/mount-utils # k8s.io/utils v0.0.0-20201110183641-67b214c5f920 ## explicit k8s.io/utils/buffer @@ -727,6 +740,7 @@ k8s.io/utils/io k8s.io/utils/keymutex k8s.io/utils/mount k8s.io/utils/net +k8s.io/utils/nsenter k8s.io/utils/path k8s.io/utils/pointer k8s.io/utils/trace