From cc564f929a4e32a829d35272533242b94797aa22 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 6 Mar 2019 17:23:07 +0100 Subject: [PATCH 01/71] verify-subtree.sh: relax check and ignore old content If for whatever reasons a repo already had a `release-tools` directory before doing a clean import of it with `git subtree`, the check used to fail because it found those old commits. This can be fixed by telling `git log` to stop when the directory disappears from the repo. There has to be a commit with removes the old content, because otherwise `git subtree add` doesn't work. Fixes: https://github.com/kubernetes-csi/external-resizer/issues/21 --- verify-subtree.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/verify-subtree.sh b/verify-subtree.sh index ce8375fc..f04a9fa2 100755 --- a/verify-subtree.sh +++ b/verify-subtree.sh @@ -30,7 +30,7 @@ if [ ! "$DIR" ]; then exit 1 fi -REV=$(git log -n1 --format=format:%H --no-merges -- "$DIR") +REV=$(git log -n1 --remove-empty --format=format:%H --no-merges -- "$DIR") if [ "$REV" ]; then echo "Directory '$DIR' contains non-upstream changes:" echo From 94fc1e31d8acc93663992d117ecf9fc3ca9fceb0 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 15 Mar 2019 11:08:53 +0100 Subject: [PATCH 02/71] build.make: avoid unit-testing E2E test suite In repos that have a test/e2e, that test suite should be run separately because it depends on a running cluster. --- build.make | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.make b/build.make index 8ca0b2c2..6ea32795 100644 --- a/build.make +++ b/build.make @@ -98,7 +98,7 @@ test: test: test-go test-go: @ echo; echo "### $@:" - go test `go list ./... | grep -v 'vendor' $(TEST_GO_FILTER_CMD)` $(TESTARGS) + go test `go list ./... | grep -v -e 'vendor' -e '/test/e2e$$' $(TEST_GO_FILTER_CMD)` $(TESTARGS) .PHONY: test-vet test: test-vet From fb13c5198f899657baac7bb44156abc6d0f20724 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 15 Mar 2019 11:55:06 +0100 Subject: [PATCH 03/71] verify-shellcheck.sh: import from Kubernetes This is an unmodified copy of kubernetes/hack/verify-shellcheck.sh revision d5a3db003916b1d33b503ccd2e4897e094d8af77. --- util.sh | 839 +++++++++++++++++++++++++++++++++++++++++++ verify-shellcheck.sh | 187 ++++++++++ 2 files changed, 1026 insertions(+) create mode 100755 util.sh create mode 100755 verify-shellcheck.sh diff --git a/util.sh b/util.sh new file mode 100755 index 00000000..31ce9fc4 --- /dev/null +++ b/util.sh @@ -0,0 +1,839 @@ +#!/usr/bin/env bash + +# Copyright 2014 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function kube::util::sourced_variable { + # Call this function to tell shellcheck that a variable is supposed to + # be used from other calling context. This helps quiet an "unused + # variable" warning from shellcheck and also document your code. + true +} + +kube::util::sortable_date() { + date "+%Y%m%d-%H%M%S" +} + +# arguments: target, item1, item2, item3, ... +# returns 0 if target is in the given items, 1 otherwise. +kube::util::array_contains() { + local search="$1" + local element + shift + for element; do + if [[ "${element}" == "${search}" ]]; then + return 0 + fi + done + return 1 +} + +kube::util::wait_for_url() { + local url=$1 + local prefix=${2:-} + local wait=${3:-1} + local times=${4:-30} + local maxtime=${5:-1} + + command -v curl >/dev/null || { + kube::log::usage "curl must be installed" + exit 1 + } + + local i + for i in $(seq 1 "${times}"); do + local out + if out=$(curl --max-time "${maxtime}" -gkfs "${url}" 2>/dev/null); then + kube::log::status "On try ${i}, ${prefix}: ${out}" + return 0 + fi + sleep "${wait}" + done + kube::log::error "Timed out waiting for ${prefix} to answer at ${url}; tried ${times} waiting ${wait} between each" + return 1 +} + +# Example: kube::util::trap_add 'echo "in trap DEBUG"' DEBUG +# See: http://stackoverflow.com/questions/3338030/multiple-bash-traps-for-the-same-signal +kube::util::trap_add() { + local trap_add_cmd + trap_add_cmd=$1 + shift + + for trap_add_name in "$@"; do + local existing_cmd + local new_cmd + + # Grab the currently defined trap commands for this trap + existing_cmd=$(trap -p "${trap_add_name}" | awk -F"'" '{print $2}') + + if [[ -z "${existing_cmd}" ]]; then + new_cmd="${trap_add_cmd}" + else + new_cmd="${trap_add_cmd};${existing_cmd}" + fi + + # Assign the test. Disable the shellcheck warning telling that trap + # commands should be single quoted to avoid evaluating them at this + # point instead evaluating them at run time. The logic of adding new + # commands to a single trap requires them to be evaluated right away. + # shellcheck disable=SC2064 + trap "${new_cmd}" "${trap_add_name}" + done +} + +# Opposite of kube::util::ensure-temp-dir() +kube::util::cleanup-temp-dir() { + rm -rf "${KUBE_TEMP}" +} + +# Create a temp dir that'll be deleted at the end of this bash session. +# +# Vars set: +# KUBE_TEMP +kube::util::ensure-temp-dir() { + if [[ -z ${KUBE_TEMP-} ]]; then + KUBE_TEMP=$(mktemp -d 2>/dev/null || mktemp -d -t kubernetes.XXXXXX) + kube::util::trap_add kube::util::cleanup-temp-dir EXIT + fi +} + +kube::util::host_os() { + local host_os + case "$(uname -s)" in + Darwin) + host_os=darwin + ;; + Linux) + host_os=linux + ;; + *) + kube::log::error "Unsupported host OS. Must be Linux or Mac OS X." + exit 1 + ;; + esac + echo "${host_os}" +} + +kube::util::host_arch() { + local host_arch + case "$(uname -m)" in + x86_64*) + host_arch=amd64 + ;; + i?86_64*) + host_arch=amd64 + ;; + amd64*) + host_arch=amd64 + ;; + aarch64*) + host_arch=arm64 + ;; + arm64*) + host_arch=arm64 + ;; + arm*) + host_arch=arm + ;; + i?86*) + host_arch=x86 + ;; + s390x*) + host_arch=s390x + ;; + ppc64le*) + host_arch=ppc64le + ;; + *) + kube::log::error "Unsupported host arch. Must be x86_64, 386, arm, arm64, s390x or ppc64le." + exit 1 + ;; + esac + echo "${host_arch}" +} + +# This figures out the host platform without relying on golang. We need this as +# we don't want a golang install to be a prerequisite to building yet we need +# this info to figure out where the final binaries are placed. +kube::util::host_platform() { + echo "$(kube::util::host_os)/$(kube::util::host_arch)" +} + +kube::util::find-binary-for-platform() { + local -r lookfor="$1" + local -r platform="$2" + local locations=( + "${KUBE_ROOT}/_output/bin/${lookfor}" + "${KUBE_ROOT}/_output/dockerized/bin/${platform}/${lookfor}" + "${KUBE_ROOT}/_output/local/bin/${platform}/${lookfor}" + "${KUBE_ROOT}/platforms/${platform}/${lookfor}" + ) + # Also search for binary in bazel build tree. + # The bazel go rules place some binaries in subtrees like + # "bazel-bin/source/path/linux_amd64_pure_stripped/binaryname", so make sure + # the platform name is matched in the path. + while IFS=$'\n' read -r location; do + locations+=("$location"); + done < <(find "${KUBE_ROOT}/bazel-bin/" -type f -executable \ + \( -path "*/${platform/\//_}*/${lookfor}" -o -path "*/${lookfor}" \) 2>/dev/null || true) + + # List most recently-updated location. + local -r bin=$( (ls -t "${locations[@]}" 2>/dev/null || true) | head -1 ) + echo -n "${bin}" +} + +kube::util::find-binary() { + kube::util::find-binary-for-platform "$1" "$(kube::util::host_platform)" +} + +# Run all known doc generators (today gendocs and genman for kubectl) +# $1 is the directory to put those generated documents +kube::util::gen-docs() { + local dest="$1" + + # Find binary + gendocs=$(kube::util::find-binary "gendocs") + genkubedocs=$(kube::util::find-binary "genkubedocs") + genman=$(kube::util::find-binary "genman") + genyaml=$(kube::util::find-binary "genyaml") + genfeddocs=$(kube::util::find-binary "genfeddocs") + + # TODO: If ${genfeddocs} is not used from anywhere (it isn't used at + # least from k/k tree), remove it completely. + kube::util::sourced_variable "${genfeddocs}" + + mkdir -p "${dest}/docs/user-guide/kubectl/" + "${gendocs}" "${dest}/docs/user-guide/kubectl/" + mkdir -p "${dest}/docs/admin/" + "${genkubedocs}" "${dest}/docs/admin/" "kube-apiserver" + "${genkubedocs}" "${dest}/docs/admin/" "kube-controller-manager" + "${genkubedocs}" "${dest}/docs/admin/" "cloud-controller-manager" + "${genkubedocs}" "${dest}/docs/admin/" "kube-proxy" + "${genkubedocs}" "${dest}/docs/admin/" "kube-scheduler" + "${genkubedocs}" "${dest}/docs/admin/" "kubelet" + "${genkubedocs}" "${dest}/docs/admin/" "kubeadm" + + mkdir -p "${dest}/docs/man/man1/" + "${genman}" "${dest}/docs/man/man1/" "kube-apiserver" + "${genman}" "${dest}/docs/man/man1/" "kube-controller-manager" + "${genman}" "${dest}/docs/man/man1/" "cloud-controller-manager" + "${genman}" "${dest}/docs/man/man1/" "kube-proxy" + "${genman}" "${dest}/docs/man/man1/" "kube-scheduler" + "${genman}" "${dest}/docs/man/man1/" "kubelet" + "${genman}" "${dest}/docs/man/man1/" "kubectl" + "${genman}" "${dest}/docs/man/man1/" "kubeadm" + + mkdir -p "${dest}/docs/yaml/kubectl/" + "${genyaml}" "${dest}/docs/yaml/kubectl/" + + # create the list of generated files + pushd "${dest}" > /dev/null || return 1 + touch docs/.generated_docs + find . -type f | cut -sd / -f 2- | LC_ALL=C sort > docs/.generated_docs + popd > /dev/null || return 1 +} + +# Removes previously generated docs-- we don't want to check them in. $KUBE_ROOT +# must be set. +kube::util::remove-gen-docs() { + if [ -e "${KUBE_ROOT}/docs/.generated_docs" ]; then + # remove all of the old docs; we don't want to check them in. + while read -r file; do + rm "${KUBE_ROOT}/${file}" 2>/dev/null || true + done <"${KUBE_ROOT}/docs/.generated_docs" + # The docs/.generated_docs file lists itself, so we don't need to explicitly + # delete it. + fi +} + +# Takes a group/version and returns the path to its location on disk, sans +# "pkg". E.g.: +# * default behavior: extensions/v1beta1 -> apis/extensions/v1beta1 +# * default behavior for only a group: experimental -> apis/experimental +# * Special handling for empty group: v1 -> api/v1, unversioned -> api/unversioned +# * Special handling for groups suffixed with ".k8s.io": foo.k8s.io/v1 -> apis/foo/v1 +# * Very special handling for when both group and version are "": / -> api +kube::util::group-version-to-pkg-path() { + local group_version="$1" + + while IFS=$'\n' read -r api; do + if [[ "${api}" = "${group_version/.*k8s.io/}" ]]; then + echo "vendor/k8s.io/api/${group_version/.*k8s.io/}" + return + fi + done < <(cd "${KUBE_ROOT}/staging/src/k8s.io/api" && find . -name types.go -exec dirname {} \; | sed "s|\./||g" | sort) + + # "v1" is the API GroupVersion + if [[ "${group_version}" == "v1" ]]; then + echo "vendor/k8s.io/api/core/v1" + return + fi + + # Special cases first. + # TODO(lavalamp): Simplify this by moving pkg/api/v1 and splitting pkg/api, + # moving the results to pkg/apis/api. + case "${group_version}" in + # both group and version are "", this occurs when we generate deep copies for internal objects of the legacy v1 API. + __internal) + echo "pkg/apis/core" + ;; + meta/v1) + echo "vendor/k8s.io/apimachinery/pkg/apis/meta/v1" + ;; + meta/v1beta1) + echo "vendor/k8s.io/apimachinery/pkg/apis/meta/v1beta1" + ;; + *.k8s.io) + echo "pkg/apis/${group_version%.*k8s.io}" + ;; + *.k8s.io/*) + echo "pkg/apis/${group_version/.*k8s.io/}" + ;; + *) + echo "pkg/apis/${group_version%__internal}" + ;; + esac +} + +# Takes a group/version and returns the swagger-spec file name. +# default behavior: extensions/v1beta1 -> extensions_v1beta1 +# special case for v1: v1 -> v1 +kube::util::gv-to-swagger-name() { + local group_version="$1" + case "${group_version}" in + v1) + echo "v1" + ;; + *) + echo "${group_version%/*}_${group_version#*/}" + ;; + esac +} + +# Returns the name of the upstream remote repository name for the local git +# repo, e.g. "upstream" or "origin". +kube::util::git_upstream_remote_name() { + git remote -v | grep fetch |\ + grep -E 'github.com[/:]kubernetes/kubernetes|k8s.io/kubernetes' |\ + head -n 1 | awk '{print $1}' +} + +# Ensures the current directory is a git tree for doing things like restoring or +# validating godeps +kube::util::create-fake-git-tree() { + local -r target_dir=${1:-$(pwd)} + + pushd "${target_dir}" >/dev/null || return 1 + git init >/dev/null + git config --local user.email "nobody@k8s.io" + git config --local user.name "$0" + git add . >/dev/null + git commit -q -m "Snapshot" >/dev/null + if (( ${KUBE_VERBOSE:-5} >= 6 )); then + kube::log::status "${target_dir} is now a git tree." + fi + popd >/dev/null || return 1 +} + +# Checks whether godep restore was run in the current GOPATH, i.e. that all referenced repos exist +# and are checked out to the referenced rev. +kube::util::godep_restored() { + local -r godeps_json=${1:-Godeps/Godeps.json} + local -r gopath=${2:-${GOPATH%:*}} + + kube::util::require-jq + + local root + local old_rev="" + while read -r path rev; do + rev="${rev//[\'\"]}" # remove quotes which are around revs sometimes + + if [[ "${rev}" == "${old_rev}" ]] && [[ "${path}" == "${root}"* ]]; then + # avoid checking the same git/hg root again + continue + fi + + root="${path}" + while [ "${root}" != "." ] && [ ! -d "${gopath}/src/${root}/.git" ] && [ ! -d "${gopath}/src/${root}/.hg" ]; do + root=$(dirname "${root}") + done + if [ "${root}" == "." ]; then + echo "No checkout of ${path} found in GOPATH \"${gopath}\"." 1>&2 + return 1 + fi + local head + if [ -d "${gopath}/src/${root}/.git" ]; then + head="$(cd "${gopath}/src/${root}" && git rev-parse HEAD)" + else + head="$(cd "${gopath}/src/${root}" && hg parent --template '{node}')" + fi + if [ "${head}" != "${rev}" ]; then + echo "Unexpected HEAD '${head}' at ${gopath}/src/${root}, expected '${rev}'." 1>&2 + return 1 + fi + old_rev="${rev}" + done < <(jq '.Deps|.[]|.ImportPath + " " + .Rev' -r < "${godeps_json}") + return 0 +} + +# Exits script if working directory is dirty. If it's run interactively in the terminal +# the user can commit changes in a second terminal. This script will wait. +kube::util::ensure_clean_working_dir() { + while ! git diff HEAD --exit-code &>/dev/null; do + echo -e "\nUnexpected dirty working directory:\n" + if tty -s; then + git status -s + else + git diff -a # be more verbose in log files without tty + exit 1 + fi | sed 's/^/ /' + echo -e "\nCommit your changes in another terminal and then continue here by pressing enter." + read -r + done 1>&2 +} + +# Ensure that the given godep version is installed and in the path. Almost +# nobody should use any version but the default. +# +# Sets: +# KUBE_GODEP: The path to the godep binary +# +kube::util::ensure_godep_version() { + local godep_target_version=${1:-"v80-k8s-r1"} # this version is known to work + + # If KUBE_GODEP is already set, and it's the right version, then use it. + if [[ -n "${KUBE_GODEP:-}" && "$(${KUBE_GODEP:?} version 2>/dev/null)" == *"godep ${godep_target_version}"* ]]; then + kube::log::status "Using ${KUBE_GODEP}" + return + fi + + # Otherwise, install forked godep + kube::log::status "Installing godep version ${godep_target_version}" + GOBIN="${KUBE_OUTPUT_BINPATH}" go install k8s.io/kubernetes/third_party/forked/godep + export KUBE_GODEP="${KUBE_OUTPUT_BINPATH}/godep" + kube::log::status "Installed ${KUBE_GODEP}" + + # Verify that the installed godep from fork is what we expect + if [[ "$(${KUBE_GODEP:?} version 2>/dev/null)" != *"godep ${godep_target_version}"* ]]; then + kube::log::error "Expected godep ${godep_target_version} from ${KUBE_GODEP}, got $(${KUBE_GODEP:?} version)" + return 1 + fi +} + +# Ensure that none of the staging repos is checked out in the GOPATH because this +# easily confused godep. +kube::util::ensure_no_staging_repos_in_gopath() { + kube::util::ensure_single_dir_gopath + local error=0 + for repo_file in "${KUBE_ROOT}"/staging/src/k8s.io/*; do + if [[ ! -d "${repo_file}" ]]; then + # not a directory or there were no files + continue; + fi + repo="$(basename "${repo_file}")" + if [ -e "${GOPATH}/src/k8s.io/${repo}" ]; then + echo "k8s.io/${repo} exists in GOPATH. Remove before running godep-save.sh." 1>&2 + error=1 + fi + done + if [ "${error}" = "1" ]; then + exit 1 + fi +} + +# Checks that the GOPATH is simple, i.e. consists only of one directory, not multiple. +kube::util::ensure_single_dir_gopath() { + if [[ "${GOPATH}" == *:* ]]; then + echo "GOPATH must consist of a single directory." 1>&2 + exit 1 + fi +} + +# Find the base commit using: +# $PULL_BASE_SHA if set (from Prow) +# current ref from the remote upstream branch +kube::util::base_ref() { + local -r git_branch=$1 + + if [[ -n ${PULL_BASE_SHA:-} ]]; then + echo "${PULL_BASE_SHA}" + return + fi + + full_branch="$(kube::util::git_upstream_remote_name)/${git_branch}" + + # make sure the branch is valid, otherwise the check will pass erroneously. + if ! git describe "${full_branch}" >/dev/null; then + # abort! + exit 1 + fi + + echo "${full_branch}" +} + +# Checks whether there are any files matching pattern $2 changed between the +# current branch and upstream branch named by $1. +# Returns 1 (false) if there are no changes +# 0 (true) if there are changes detected. +kube::util::has_changes() { + local -r git_branch=$1 + local -r pattern=$2 + local -r not_pattern=${3:-totallyimpossiblepattern} + + local base_ref + base_ref=$(kube::util::base_ref "${git_branch}") + echo "Checking for '${pattern}' changes against '${base_ref}'" + + # notice this uses ... to find the first shared ancestor + if git diff --name-only "${base_ref}...HEAD" | grep -v -E "${not_pattern}" | grep "${pattern}" > /dev/null; then + return 0 + fi + # also check for pending changes + if git status --porcelain | grep -v -E "${not_pattern}" | grep "${pattern}" > /dev/null; then + echo "Detected '${pattern}' uncommitted changes." + return 0 + fi + echo "No '${pattern}' changes detected." + return 1 +} + +kube::util::download_file() { + local -r url=$1 + local -r destination_file=$2 + + rm "${destination_file}" 2&> /dev/null || true + + for i in $(seq 5) + do + if ! curl -fsSL --retry 3 --keepalive-time 2 "${url}" -o "${destination_file}"; then + echo "Downloading ${url} failed. $((5-i)) retries left." + sleep 1 + else + echo "Downloading ${url} succeed" + return 0 + fi + done + return 1 +} + +# Test whether openssl is installed. +# Sets: +# OPENSSL_BIN: The path to the openssl binary to use +function kube::util::test_openssl_installed { + if ! openssl version >& /dev/null; then + echo "Failed to run openssl. Please ensure openssl is installed" + exit 1 + fi + + OPENSSL_BIN=$(command -v openssl) +} + +# creates a client CA, args are sudo, dest-dir, ca-id, purpose +# purpose is dropped in after "key encipherment", you usually want +# '"client auth"' +# '"server auth"' +# '"client auth","server auth"' +function kube::util::create_signing_certkey { + local sudo=$1 + local dest_dir=$2 + local id=$3 + local purpose=$4 + # Create client ca + ${sudo} /usr/bin/env bash -e < "${dest_dir}/${id}-ca-config.json" +EOF +} + +# signs a client certificate: args are sudo, dest-dir, CA, filename (roughly), username, groups... +function kube::util::create_client_certkey { + local sudo=$1 + local dest_dir=$2 + local ca=$3 + local id=$4 + local cn=${5:-$4} + local groups="" + local SEP="" + shift 5 + while [ -n "${1:-}" ]; do + groups+="${SEP}{\"O\":\"$1\"}" + SEP="," + shift 1 + done + ${sudo} /usr/bin/env bash -e < /dev/null +apiVersion: v1 +kind: Config +clusters: + - cluster: + certificate-authority: ${ca_file} + server: https://${api_host}:${api_port}/ + name: local-up-cluster +users: + - user: + token: ${token} + client-certificate: ${dest_dir}/client-${client_id}.crt + client-key: ${dest_dir}/client-${client_id}.key + name: local-up-cluster +contexts: + - context: + cluster: local-up-cluster + user: local-up-cluster + name: local-up-cluster +current-context: local-up-cluster +EOF + + # flatten the kubeconfig files to make them self contained + username=$(whoami) + ${sudo} /usr/bin/env bash -e < "/tmp/${client_id}.kubeconfig" + mv -f "/tmp/${client_id}.kubeconfig" "${dest_dir}/${client_id}.kubeconfig" + chown ${username} "${dest_dir}/${client_id}.kubeconfig" +EOF +} + +# Determines if docker can be run, failures may simply require that the user be added to the docker group. +function kube::util::ensure_docker_daemon_connectivity { + IFS=" " read -ra DOCKER <<< "${DOCKER_OPTS}" + # Expand ${DOCKER[@]} only if it's not unset. This is to work around + # Bash 3 issue with unbound variable. + DOCKER=(docker ${DOCKER[@]:+"${DOCKER[@]}"}) + if ! "${DOCKER[@]}" info > /dev/null 2>&1 ; then + cat <<'EOF' >&2 +Can't connect to 'docker' daemon. please fix and retry. + +Possible causes: + - Docker Daemon not started + - Linux: confirm via your init system + - macOS w/ docker-machine: run `docker-machine ls` and `docker-machine start ` + - macOS w/ Docker for Mac: Check the menu bar and start the Docker application + - DOCKER_HOST hasn't been set or is set incorrectly + - Linux: domain socket is used, DOCKER_* should be unset. In Bash run `unset ${!DOCKER_*}` + - macOS w/ docker-machine: run `eval "$(docker-machine env )"` + - macOS w/ Docker for Mac: domain socket is used, DOCKER_* should be unset. In Bash run `unset ${!DOCKER_*}` + - Other things to check: + - Linux: User isn't in 'docker' group. Add and relogin. + - Something like 'sudo usermod -a -G docker ${USER}' + - RHEL7 bug and workaround: https://bugzilla.redhat.com/show_bug.cgi?id=1119282#c8 +EOF + return 1 + fi +} + +# Wait for background jobs to finish. Return with +# an error status if any of the jobs failed. +kube::util::wait-for-jobs() { + local fail=0 + local job + for job in $(jobs -p); do + wait "${job}" || fail=$((fail + 1)) + done + return ${fail} +} + +# kube::util::join +# Concatenates the list elements with the delimiter passed as first parameter +# +# Ex: kube::util::join , a b c +# -> a,b,c +function kube::util::join { + local IFS="$1" + shift + echo "$*" +} + +# Downloads cfssl/cfssljson into $1 directory if they do not already exist in PATH +# +# Assumed vars: +# $1 (cfssl directory) (optional) +# +# Sets: +# CFSSL_BIN: The path of the installed cfssl binary +# CFSSLJSON_BIN: The path of the installed cfssljson binary +# +function kube::util::ensure-cfssl { + if command -v cfssl &>/dev/null && command -v cfssljson &>/dev/null; then + CFSSL_BIN=$(command -v cfssl) + CFSSLJSON_BIN=$(command -v cfssljson) + return 0 + fi + + host_arch=$(kube::util::host_arch) + + if [[ "${host_arch}" != "amd64" ]]; then + echo "Cannot download cfssl on non-amd64 hosts and cfssl does not appear to be installed." + echo "Please install cfssl and cfssljson and verify they are in \$PATH." + echo "Hint: export PATH=\$PATH:\$GOPATH/bin; go get -u github.com/cloudflare/cfssl/cmd/..." + exit 1 + fi + + # Create a temp dir for cfssl if no directory was given + local cfssldir=${1:-} + if [[ -z "${cfssldir}" ]]; then + kube::util::ensure-temp-dir + cfssldir="${KUBE_TEMP}/cfssl" + fi + + mkdir -p "${cfssldir}" + pushd "${cfssldir}" > /dev/null || return 1 + + echo "Unable to successfully run 'cfssl' from ${PATH}; downloading instead..." + kernel=$(uname -s) + case "${kernel}" in + Linux) + curl --retry 10 -L -o cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 + curl --retry 10 -L -o cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 + ;; + Darwin) + curl --retry 10 -L -o cfssl https://pkg.cfssl.org/R1.2/cfssl_darwin-amd64 + curl --retry 10 -L -o cfssljson https://pkg.cfssl.org/R1.2/cfssljson_darwin-amd64 + ;; + *) + echo "Unknown, unsupported platform: ${kernel}." >&2 + echo "Supported platforms: Linux, Darwin." >&2 + exit 2 + esac + + chmod +x cfssl || true + chmod +x cfssljson || true + + CFSSL_BIN="${cfssldir}/cfssl" + CFSSLJSON_BIN="${cfssldir}/cfssljson" + if [[ ! -x ${CFSSL_BIN} || ! -x ${CFSSLJSON_BIN} ]]; then + echo "Failed to download 'cfssl'. Please install cfssl and cfssljson and verify they are in \$PATH." + echo "Hint: export PATH=\$PATH:\$GOPATH/bin; go get -u github.com/cloudflare/cfssl/cmd/..." + exit 1 + fi + popd > /dev/null || return 1 +} + +# kube::util::ensure_dockerized +# Confirms that the script is being run inside a kube-build image +# +function kube::util::ensure_dockerized { + if [[ -f /kube-build-image ]]; then + return 0 + else + echo "ERROR: This script is designed to be run inside a kube-build container" + exit 1 + fi +} + +# kube::util::ensure-gnu-sed +# Determines which sed binary is gnu-sed on linux/darwin +# +# Sets: +# SED: The name of the gnu-sed binary +# +function kube::util::ensure-gnu-sed { + if LANG=C sed --help 2>&1 | grep -q GNU; then + SED="sed" + elif command -v gsed &>/dev/null; then + SED="gsed" + else + kube::log::error "Failed to find GNU sed as sed or gsed. If you are on Mac: brew install gnu-sed." >&2 + return 1 + fi + kube::util::sourced_variable "${SED}" +} + +# kube::util::check-file-in-alphabetical-order +# Check that the file is in alphabetical order +# +function kube::util::check-file-in-alphabetical-order { + local failure_file="$1" + if ! diff -u "${failure_file}" <(LC_ALL=C sort "${failure_file}"); then + { + echo + echo "${failure_file} is not in alphabetical order. Please sort it:" + echo + echo " LC_ALL=C sort -o ${failure_file} ${failure_file}" + echo + } >&2 + false + fi +} + +# kube::util::require-jq +# Checks whether jq is installed. +function kube::util::require-jq { + if ! command -v jq &>/dev/null; then + echo "jq not found. Please install." 1>&2 + return 1 + fi +} + +# Some useful colors. +if [[ -z "${color_start-}" ]]; then + declare -r color_start="\033[" + declare -r color_red="${color_start}0;31m" + declare -r color_yellow="${color_start}0;33m" + declare -r color_green="${color_start}0;32m" + declare -r color_blue="${color_start}1;34m" + declare -r color_cyan="${color_start}1;36m" + declare -r color_norm="${color_start}0m" + + kube::util::sourced_variable "${color_start}" + kube::util::sourced_variable "${color_red}" + kube::util::sourced_variable "${color_yellow}" + kube::util::sourced_variable "${color_green}" + kube::util::sourced_variable "${color_blue}" + kube::util::sourced_variable "${color_cyan}" + kube::util::sourced_variable "${color_norm}" +fi + +# ex: ts=2 sw=2 et filetype=sh diff --git a/verify-shellcheck.sh b/verify-shellcheck.sh new file mode 100755 index 00000000..1b882fa8 --- /dev/null +++ b/verify-shellcheck.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash + +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +source "${KUBE_ROOT}/hack/lib/init.sh" +source "${KUBE_ROOT}/hack/lib/util.sh" + +# required version for this script, if not installed on the host we will +# use the official docker image instead. keep this in sync with SHELLCHECK_IMAGE +SHELLCHECK_VERSION="0.6.0" +# upstream shellcheck latest stable image as of January 10th, 2019 +SHELLCHECK_IMAGE="koalaman/shellcheck-alpine:v0.6.0@sha256:7d4d712a2686da99d37580b4e2f45eb658b74e4b01caf67c1099adc294b96b52" + +# fixed name for the shellcheck docker container so we can reliably clean it up +SHELLCHECK_CONTAINER="k8s-shellcheck" + +# disabled lints +disabled=( + # this lint disallows non-constant source, which we use extensively without + # any known bugs + 1090 + # this lint prefers command -v to which, they are not the same + 2230 +) +# comma separate for passing to shellcheck +join_by() { + local IFS="$1"; + shift; + echo "$*"; +} +SHELLCHECK_DISABLED="$(join_by , "${disabled[@]}")" +readonly SHELLCHECK_DISABLED + +# creates the shellcheck container for later use +create_container () { + # TODO(bentheelder): this is a performance hack, we create the container with + # a sleep MAX_INT32 so that it is effectively paused. + # We then repeatedly exec to it to run each shellcheck, and later rm it when + # we're done. + # This is incredibly much faster than creating a container for each shellcheck + # call ... + docker run --name "${SHELLCHECK_CONTAINER}" -d --rm -v "${KUBE_ROOT}:${KUBE_ROOT}" -w "${KUBE_ROOT}" --entrypoint="sleep" "${SHELLCHECK_IMAGE}" 2147483647 +} +# removes the shellcheck container +remove_container () { + docker rm -f "${SHELLCHECK_CONTAINER}" &> /dev/null || true +} + +# ensure we're linting the k8s source tree +cd "${KUBE_ROOT}" + +# find all shell scripts excluding ./_*, ./.git/*, ./vendor*, +# and anything git-ignored +all_shell_scripts=() +while IFS=$'\n' read -r script; + do git check-ignore -q "$script" || all_shell_scripts+=("$script"); +done < <(find . -name "*.sh" \ + -not \( \ + -path ./_\* -o \ + -path ./.git\* -o \ + -path ./vendor\* \ + \)) + +# make sure known failures are sorted +failure_file="${KUBE_ROOT}/hack/.shellcheck_failures" +kube::util::check-file-in-alphabetical-order "${failure_file}" + +# load known failure files +failing_files=() +while IFS=$'\n' read -r script; + do failing_files+=("$script"); +done < <(cat "${failure_file}") + +# detect if the host machine has the required shellcheck version installed +# if so, we will use that instead. +HAVE_SHELLCHECK=false +if which shellcheck &>/dev/null; then + detected_version="$(shellcheck --version | grep 'version: .*')" + if [[ "${detected_version}" = "version: ${SHELLCHECK_VERSION}" ]]; then + HAVE_SHELLCHECK=true + fi +fi + +# tell the user which we've selected and possibly set up the container +if ${HAVE_SHELLCHECK}; then + echo "Using host shellcheck ${SHELLCHECK_VERSION} binary." +else + echo "Using shellcheck ${SHELLCHECK_VERSION} docker image." + # remove any previous container, ensure we will attempt to cleanup on exit, + # and create the container + remove_container + kube::util::trap_add 'remove_container' EXIT + if ! output="$(create_container 2>&1)"; then + { + echo "Failed to create shellcheck container with output: " + echo "" + echo "${output}" + } >&2 + exit 1 + fi +fi + +# lint each script, tracking failures +errors=() +not_failing=() +for f in "${all_shell_scripts[@]}"; do + set +o errexit + if ${HAVE_SHELLCHECK}; then + failedLint=$(shellcheck --exclude="${SHELLCHECK_DISABLED}" "${f}") + else + failedLint=$(docker exec -t ${SHELLCHECK_CONTAINER} \ + shellcheck --exclude="${SHELLCHECK_DISABLED}" "${f}") + fi + set -o errexit + kube::util::array_contains "${f}" "${failing_files[@]}" && in_failing=$? || in_failing=$? + if [[ -n "${failedLint}" ]] && [[ "${in_failing}" -ne "0" ]]; then + errors+=( "${failedLint}" ) + fi + if [[ -z "${failedLint}" ]] && [[ "${in_failing}" -eq "0" ]]; then + not_failing+=( "${f}" ) + fi +done + +# Check to be sure all the packages that should pass lint are. +if [ ${#errors[@]} -eq 0 ]; then + echo 'Congratulations! All shell files are passing lint (excluding those in hack/.shellcheck_failures).' +else + { + echo "Errors from shellcheck:" + for err in "${errors[@]}"; do + echo "$err" + done + echo + echo 'Please review the above warnings. You can test via "./hack/verify-shellcheck"' + echo 'If the above warnings do not make sense, you can exempt this package from shellcheck' + echo 'checking by adding it to hack/.shellcheck_failures (if your reviewer is okay with it).' + echo + } >&2 + false +fi + +if [[ ${#not_failing[@]} -gt 0 ]]; then + { + echo "Some packages in hack/.shellcheck_failures are passing shellcheck. Please remove them." + echo + for f in "${not_failing[@]}"; do + echo " $f" + done + echo + } >&2 + false +fi + +# Check that all failing_packages actually still exist +gone=() +for f in "${failing_files[@]}"; do + kube::util::array_contains "$f" "${all_shell_scripts[@]}" || gone+=( "$f" ) +done + +if [[ ${#gone[@]} -gt 0 ]]; then + { + echo "Some files in hack/.shellcheck_failures do not exist anymore. Please remove them." + echo + for f in "${gone[@]}"; do + echo " $f" + done + echo + } >&2 + false +fi From e6db50df7eefbc7a0f116e5dff79b01d9cc22741 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 27 Mar 2019 17:39:43 +0100 Subject: [PATCH 04/71] check vendor directory This runs "dep check" to verify that the vendor directory is up-to-date and meets expectations (= done with dep >= 0.5.0). --- build.make | 10 ++++++++++ travis.yml | 5 +++++ 2 files changed, 15 insertions(+) diff --git a/build.make b/build.make index 8ca0b2c2..b4084061 100644 --- a/build.make +++ b/build.make @@ -117,6 +117,16 @@ test-fmt: false; \ fi +# This test only runs when dep >= 0.5 is installed, which is the case for the CI setup. +.PHONY: test-vendor +test: test-vendor +test-vendor: + @ echo; echo "### $@:" + @ case "$$(dep version 2>/dev/null | grep 'version *:')" in \ + *v0.[56789]*) dep check && echo "vendor up-to-date" || false;; \ + *) echo "skipping check, dep >= 0.5 required";; \ + esac + .PHONY: test-subtree test: test-subtree test-subtree: diff --git a/travis.yml b/travis.yml index b5a360af..1c05dfd9 100644 --- a/travis.yml +++ b/travis.yml @@ -5,6 +5,11 @@ services: matrix: include: - go: 1.11.1 +before_script: +- mkdir -p bin +- wget https://github.com/golang/dep/releases/download/v0.5.1/dep-linux-amd64 -O bin/dep +- chmod u+x bin/dep +- export PATH=$PWD/bin:$PATH script: - make -k all test after_success: From 104a1ac9699f22e815c54afc2620aef85bf490e2 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 15 Mar 2019 11:08:53 +0100 Subject: [PATCH 05/71] build.make: avoid unit-testing E2E test suite In repos that have a test/e2e, that test suite should be run separately because it depends on a running cluster. --- build.make | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.make b/build.make index 8ca0b2c2..6ea32795 100644 --- a/build.make +++ b/build.make @@ -98,7 +98,7 @@ test: test: test-go test-go: @ echo; echo "### $@:" - go test `go list ./... | grep -v 'vendor' $(TEST_GO_FILTER_CMD)` $(TESTARGS) + go test `go list ./... | grep -v -e 'vendor' -e '/test/e2e$$' $(TEST_GO_FILTER_CMD)` $(TESTARGS) .PHONY: test-vet test: test-vet From b2d25d4f4d7886b1d9ad40943dd06748e0f06874 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 15 Mar 2019 16:42:10 +0100 Subject: [PATCH 06/71] verify-shellcheck.sh: make it usable in csi-release-tools These are the modifications that were necessary to call this outside of Kubernetes. The support for excluding files from checking gets removed to simplify the script. It shouldn't be needed, because linting can be enabled after fixing whatever scripts might fail the check. --- util.sh | 691 ------------------------------------------- verify-shellcheck.sh | 71 +---- 2 files changed, 15 insertions(+), 747 deletions(-) diff --git a/util.sh b/util.sh index 31ce9fc4..abeb1b2e 100755 --- a/util.sh +++ b/util.sh @@ -39,31 +39,6 @@ kube::util::array_contains() { return 1 } -kube::util::wait_for_url() { - local url=$1 - local prefix=${2:-} - local wait=${3:-1} - local times=${4:-30} - local maxtime=${5:-1} - - command -v curl >/dev/null || { - kube::log::usage "curl must be installed" - exit 1 - } - - local i - for i in $(seq 1 "${times}"); do - local out - if out=$(curl --max-time "${maxtime}" -gkfs "${url}" 2>/dev/null); then - kube::log::status "On try ${i}, ${prefix}: ${out}" - return 0 - fi - sleep "${wait}" - done - kube::log::error "Timed out waiting for ${prefix} to answer at ${url}; tried ${times} waiting ${wait} between each" - return 1 -} - # Example: kube::util::trap_add 'echo "in trap DEBUG"' DEBUG # See: http://stackoverflow.com/questions/3338030/multiple-bash-traps-for-the-same-signal kube::util::trap_add() { @@ -93,422 +68,6 @@ kube::util::trap_add() { done } -# Opposite of kube::util::ensure-temp-dir() -kube::util::cleanup-temp-dir() { - rm -rf "${KUBE_TEMP}" -} - -# Create a temp dir that'll be deleted at the end of this bash session. -# -# Vars set: -# KUBE_TEMP -kube::util::ensure-temp-dir() { - if [[ -z ${KUBE_TEMP-} ]]; then - KUBE_TEMP=$(mktemp -d 2>/dev/null || mktemp -d -t kubernetes.XXXXXX) - kube::util::trap_add kube::util::cleanup-temp-dir EXIT - fi -} - -kube::util::host_os() { - local host_os - case "$(uname -s)" in - Darwin) - host_os=darwin - ;; - Linux) - host_os=linux - ;; - *) - kube::log::error "Unsupported host OS. Must be Linux or Mac OS X." - exit 1 - ;; - esac - echo "${host_os}" -} - -kube::util::host_arch() { - local host_arch - case "$(uname -m)" in - x86_64*) - host_arch=amd64 - ;; - i?86_64*) - host_arch=amd64 - ;; - amd64*) - host_arch=amd64 - ;; - aarch64*) - host_arch=arm64 - ;; - arm64*) - host_arch=arm64 - ;; - arm*) - host_arch=arm - ;; - i?86*) - host_arch=x86 - ;; - s390x*) - host_arch=s390x - ;; - ppc64le*) - host_arch=ppc64le - ;; - *) - kube::log::error "Unsupported host arch. Must be x86_64, 386, arm, arm64, s390x or ppc64le." - exit 1 - ;; - esac - echo "${host_arch}" -} - -# This figures out the host platform without relying on golang. We need this as -# we don't want a golang install to be a prerequisite to building yet we need -# this info to figure out where the final binaries are placed. -kube::util::host_platform() { - echo "$(kube::util::host_os)/$(kube::util::host_arch)" -} - -kube::util::find-binary-for-platform() { - local -r lookfor="$1" - local -r platform="$2" - local locations=( - "${KUBE_ROOT}/_output/bin/${lookfor}" - "${KUBE_ROOT}/_output/dockerized/bin/${platform}/${lookfor}" - "${KUBE_ROOT}/_output/local/bin/${platform}/${lookfor}" - "${KUBE_ROOT}/platforms/${platform}/${lookfor}" - ) - # Also search for binary in bazel build tree. - # The bazel go rules place some binaries in subtrees like - # "bazel-bin/source/path/linux_amd64_pure_stripped/binaryname", so make sure - # the platform name is matched in the path. - while IFS=$'\n' read -r location; do - locations+=("$location"); - done < <(find "${KUBE_ROOT}/bazel-bin/" -type f -executable \ - \( -path "*/${platform/\//_}*/${lookfor}" -o -path "*/${lookfor}" \) 2>/dev/null || true) - - # List most recently-updated location. - local -r bin=$( (ls -t "${locations[@]}" 2>/dev/null || true) | head -1 ) - echo -n "${bin}" -} - -kube::util::find-binary() { - kube::util::find-binary-for-platform "$1" "$(kube::util::host_platform)" -} - -# Run all known doc generators (today gendocs and genman for kubectl) -# $1 is the directory to put those generated documents -kube::util::gen-docs() { - local dest="$1" - - # Find binary - gendocs=$(kube::util::find-binary "gendocs") - genkubedocs=$(kube::util::find-binary "genkubedocs") - genman=$(kube::util::find-binary "genman") - genyaml=$(kube::util::find-binary "genyaml") - genfeddocs=$(kube::util::find-binary "genfeddocs") - - # TODO: If ${genfeddocs} is not used from anywhere (it isn't used at - # least from k/k tree), remove it completely. - kube::util::sourced_variable "${genfeddocs}" - - mkdir -p "${dest}/docs/user-guide/kubectl/" - "${gendocs}" "${dest}/docs/user-guide/kubectl/" - mkdir -p "${dest}/docs/admin/" - "${genkubedocs}" "${dest}/docs/admin/" "kube-apiserver" - "${genkubedocs}" "${dest}/docs/admin/" "kube-controller-manager" - "${genkubedocs}" "${dest}/docs/admin/" "cloud-controller-manager" - "${genkubedocs}" "${dest}/docs/admin/" "kube-proxy" - "${genkubedocs}" "${dest}/docs/admin/" "kube-scheduler" - "${genkubedocs}" "${dest}/docs/admin/" "kubelet" - "${genkubedocs}" "${dest}/docs/admin/" "kubeadm" - - mkdir -p "${dest}/docs/man/man1/" - "${genman}" "${dest}/docs/man/man1/" "kube-apiserver" - "${genman}" "${dest}/docs/man/man1/" "kube-controller-manager" - "${genman}" "${dest}/docs/man/man1/" "cloud-controller-manager" - "${genman}" "${dest}/docs/man/man1/" "kube-proxy" - "${genman}" "${dest}/docs/man/man1/" "kube-scheduler" - "${genman}" "${dest}/docs/man/man1/" "kubelet" - "${genman}" "${dest}/docs/man/man1/" "kubectl" - "${genman}" "${dest}/docs/man/man1/" "kubeadm" - - mkdir -p "${dest}/docs/yaml/kubectl/" - "${genyaml}" "${dest}/docs/yaml/kubectl/" - - # create the list of generated files - pushd "${dest}" > /dev/null || return 1 - touch docs/.generated_docs - find . -type f | cut -sd / -f 2- | LC_ALL=C sort > docs/.generated_docs - popd > /dev/null || return 1 -} - -# Removes previously generated docs-- we don't want to check them in. $KUBE_ROOT -# must be set. -kube::util::remove-gen-docs() { - if [ -e "${KUBE_ROOT}/docs/.generated_docs" ]; then - # remove all of the old docs; we don't want to check them in. - while read -r file; do - rm "${KUBE_ROOT}/${file}" 2>/dev/null || true - done <"${KUBE_ROOT}/docs/.generated_docs" - # The docs/.generated_docs file lists itself, so we don't need to explicitly - # delete it. - fi -} - -# Takes a group/version and returns the path to its location on disk, sans -# "pkg". E.g.: -# * default behavior: extensions/v1beta1 -> apis/extensions/v1beta1 -# * default behavior for only a group: experimental -> apis/experimental -# * Special handling for empty group: v1 -> api/v1, unversioned -> api/unversioned -# * Special handling for groups suffixed with ".k8s.io": foo.k8s.io/v1 -> apis/foo/v1 -# * Very special handling for when both group and version are "": / -> api -kube::util::group-version-to-pkg-path() { - local group_version="$1" - - while IFS=$'\n' read -r api; do - if [[ "${api}" = "${group_version/.*k8s.io/}" ]]; then - echo "vendor/k8s.io/api/${group_version/.*k8s.io/}" - return - fi - done < <(cd "${KUBE_ROOT}/staging/src/k8s.io/api" && find . -name types.go -exec dirname {} \; | sed "s|\./||g" | sort) - - # "v1" is the API GroupVersion - if [[ "${group_version}" == "v1" ]]; then - echo "vendor/k8s.io/api/core/v1" - return - fi - - # Special cases first. - # TODO(lavalamp): Simplify this by moving pkg/api/v1 and splitting pkg/api, - # moving the results to pkg/apis/api. - case "${group_version}" in - # both group and version are "", this occurs when we generate deep copies for internal objects of the legacy v1 API. - __internal) - echo "pkg/apis/core" - ;; - meta/v1) - echo "vendor/k8s.io/apimachinery/pkg/apis/meta/v1" - ;; - meta/v1beta1) - echo "vendor/k8s.io/apimachinery/pkg/apis/meta/v1beta1" - ;; - *.k8s.io) - echo "pkg/apis/${group_version%.*k8s.io}" - ;; - *.k8s.io/*) - echo "pkg/apis/${group_version/.*k8s.io/}" - ;; - *) - echo "pkg/apis/${group_version%__internal}" - ;; - esac -} - -# Takes a group/version and returns the swagger-spec file name. -# default behavior: extensions/v1beta1 -> extensions_v1beta1 -# special case for v1: v1 -> v1 -kube::util::gv-to-swagger-name() { - local group_version="$1" - case "${group_version}" in - v1) - echo "v1" - ;; - *) - echo "${group_version%/*}_${group_version#*/}" - ;; - esac -} - -# Returns the name of the upstream remote repository name for the local git -# repo, e.g. "upstream" or "origin". -kube::util::git_upstream_remote_name() { - git remote -v | grep fetch |\ - grep -E 'github.com[/:]kubernetes/kubernetes|k8s.io/kubernetes' |\ - head -n 1 | awk '{print $1}' -} - -# Ensures the current directory is a git tree for doing things like restoring or -# validating godeps -kube::util::create-fake-git-tree() { - local -r target_dir=${1:-$(pwd)} - - pushd "${target_dir}" >/dev/null || return 1 - git init >/dev/null - git config --local user.email "nobody@k8s.io" - git config --local user.name "$0" - git add . >/dev/null - git commit -q -m "Snapshot" >/dev/null - if (( ${KUBE_VERBOSE:-5} >= 6 )); then - kube::log::status "${target_dir} is now a git tree." - fi - popd >/dev/null || return 1 -} - -# Checks whether godep restore was run in the current GOPATH, i.e. that all referenced repos exist -# and are checked out to the referenced rev. -kube::util::godep_restored() { - local -r godeps_json=${1:-Godeps/Godeps.json} - local -r gopath=${2:-${GOPATH%:*}} - - kube::util::require-jq - - local root - local old_rev="" - while read -r path rev; do - rev="${rev//[\'\"]}" # remove quotes which are around revs sometimes - - if [[ "${rev}" == "${old_rev}" ]] && [[ "${path}" == "${root}"* ]]; then - # avoid checking the same git/hg root again - continue - fi - - root="${path}" - while [ "${root}" != "." ] && [ ! -d "${gopath}/src/${root}/.git" ] && [ ! -d "${gopath}/src/${root}/.hg" ]; do - root=$(dirname "${root}") - done - if [ "${root}" == "." ]; then - echo "No checkout of ${path} found in GOPATH \"${gopath}\"." 1>&2 - return 1 - fi - local head - if [ -d "${gopath}/src/${root}/.git" ]; then - head="$(cd "${gopath}/src/${root}" && git rev-parse HEAD)" - else - head="$(cd "${gopath}/src/${root}" && hg parent --template '{node}')" - fi - if [ "${head}" != "${rev}" ]; then - echo "Unexpected HEAD '${head}' at ${gopath}/src/${root}, expected '${rev}'." 1>&2 - return 1 - fi - old_rev="${rev}" - done < <(jq '.Deps|.[]|.ImportPath + " " + .Rev' -r < "${godeps_json}") - return 0 -} - -# Exits script if working directory is dirty. If it's run interactively in the terminal -# the user can commit changes in a second terminal. This script will wait. -kube::util::ensure_clean_working_dir() { - while ! git diff HEAD --exit-code &>/dev/null; do - echo -e "\nUnexpected dirty working directory:\n" - if tty -s; then - git status -s - else - git diff -a # be more verbose in log files without tty - exit 1 - fi | sed 's/^/ /' - echo -e "\nCommit your changes in another terminal and then continue here by pressing enter." - read -r - done 1>&2 -} - -# Ensure that the given godep version is installed and in the path. Almost -# nobody should use any version but the default. -# -# Sets: -# KUBE_GODEP: The path to the godep binary -# -kube::util::ensure_godep_version() { - local godep_target_version=${1:-"v80-k8s-r1"} # this version is known to work - - # If KUBE_GODEP is already set, and it's the right version, then use it. - if [[ -n "${KUBE_GODEP:-}" && "$(${KUBE_GODEP:?} version 2>/dev/null)" == *"godep ${godep_target_version}"* ]]; then - kube::log::status "Using ${KUBE_GODEP}" - return - fi - - # Otherwise, install forked godep - kube::log::status "Installing godep version ${godep_target_version}" - GOBIN="${KUBE_OUTPUT_BINPATH}" go install k8s.io/kubernetes/third_party/forked/godep - export KUBE_GODEP="${KUBE_OUTPUT_BINPATH}/godep" - kube::log::status "Installed ${KUBE_GODEP}" - - # Verify that the installed godep from fork is what we expect - if [[ "$(${KUBE_GODEP:?} version 2>/dev/null)" != *"godep ${godep_target_version}"* ]]; then - kube::log::error "Expected godep ${godep_target_version} from ${KUBE_GODEP}, got $(${KUBE_GODEP:?} version)" - return 1 - fi -} - -# Ensure that none of the staging repos is checked out in the GOPATH because this -# easily confused godep. -kube::util::ensure_no_staging_repos_in_gopath() { - kube::util::ensure_single_dir_gopath - local error=0 - for repo_file in "${KUBE_ROOT}"/staging/src/k8s.io/*; do - if [[ ! -d "${repo_file}" ]]; then - # not a directory or there were no files - continue; - fi - repo="$(basename "${repo_file}")" - if [ -e "${GOPATH}/src/k8s.io/${repo}" ]; then - echo "k8s.io/${repo} exists in GOPATH. Remove before running godep-save.sh." 1>&2 - error=1 - fi - done - if [ "${error}" = "1" ]; then - exit 1 - fi -} - -# Checks that the GOPATH is simple, i.e. consists only of one directory, not multiple. -kube::util::ensure_single_dir_gopath() { - if [[ "${GOPATH}" == *:* ]]; then - echo "GOPATH must consist of a single directory." 1>&2 - exit 1 - fi -} - -# Find the base commit using: -# $PULL_BASE_SHA if set (from Prow) -# current ref from the remote upstream branch -kube::util::base_ref() { - local -r git_branch=$1 - - if [[ -n ${PULL_BASE_SHA:-} ]]; then - echo "${PULL_BASE_SHA}" - return - fi - - full_branch="$(kube::util::git_upstream_remote_name)/${git_branch}" - - # make sure the branch is valid, otherwise the check will pass erroneously. - if ! git describe "${full_branch}" >/dev/null; then - # abort! - exit 1 - fi - - echo "${full_branch}" -} - -# Checks whether there are any files matching pattern $2 changed between the -# current branch and upstream branch named by $1. -# Returns 1 (false) if there are no changes -# 0 (true) if there are changes detected. -kube::util::has_changes() { - local -r git_branch=$1 - local -r pattern=$2 - local -r not_pattern=${3:-totallyimpossiblepattern} - - local base_ref - base_ref=$(kube::util::base_ref "${git_branch}") - echo "Checking for '${pattern}' changes against '${base_ref}'" - - # notice this uses ... to find the first shared ancestor - if git diff --name-only "${base_ref}...HEAD" | grep -v -E "${not_pattern}" | grep "${pattern}" > /dev/null; then - return 0 - fi - # also check for pending changes - if git status --porcelain | grep -v -E "${not_pattern}" | grep "${pattern}" > /dev/null; then - echo "Detected '${pattern}' uncommitted changes." - return 0 - fi - echo "No '${pattern}' changes detected." - return 1 -} - kube::util::download_file() { local -r url=$1 local -r destination_file=$2 @@ -528,152 +87,6 @@ kube::util::download_file() { return 1 } -# Test whether openssl is installed. -# Sets: -# OPENSSL_BIN: The path to the openssl binary to use -function kube::util::test_openssl_installed { - if ! openssl version >& /dev/null; then - echo "Failed to run openssl. Please ensure openssl is installed" - exit 1 - fi - - OPENSSL_BIN=$(command -v openssl) -} - -# creates a client CA, args are sudo, dest-dir, ca-id, purpose -# purpose is dropped in after "key encipherment", you usually want -# '"client auth"' -# '"server auth"' -# '"client auth","server auth"' -function kube::util::create_signing_certkey { - local sudo=$1 - local dest_dir=$2 - local id=$3 - local purpose=$4 - # Create client ca - ${sudo} /usr/bin/env bash -e < "${dest_dir}/${id}-ca-config.json" -EOF -} - -# signs a client certificate: args are sudo, dest-dir, CA, filename (roughly), username, groups... -function kube::util::create_client_certkey { - local sudo=$1 - local dest_dir=$2 - local ca=$3 - local id=$4 - local cn=${5:-$4} - local groups="" - local SEP="" - shift 5 - while [ -n "${1:-}" ]; do - groups+="${SEP}{\"O\":\"$1\"}" - SEP="," - shift 1 - done - ${sudo} /usr/bin/env bash -e < /dev/null -apiVersion: v1 -kind: Config -clusters: - - cluster: - certificate-authority: ${ca_file} - server: https://${api_host}:${api_port}/ - name: local-up-cluster -users: - - user: - token: ${token} - client-certificate: ${dest_dir}/client-${client_id}.crt - client-key: ${dest_dir}/client-${client_id}.key - name: local-up-cluster -contexts: - - context: - cluster: local-up-cluster - user: local-up-cluster - name: local-up-cluster -current-context: local-up-cluster -EOF - - # flatten the kubeconfig files to make them self contained - username=$(whoami) - ${sudo} /usr/bin/env bash -e < "/tmp/${client_id}.kubeconfig" - mv -f "/tmp/${client_id}.kubeconfig" "${dest_dir}/${client_id}.kubeconfig" - chown ${username} "${dest_dir}/${client_id}.kubeconfig" -EOF -} - -# Determines if docker can be run, failures may simply require that the user be added to the docker group. -function kube::util::ensure_docker_daemon_connectivity { - IFS=" " read -ra DOCKER <<< "${DOCKER_OPTS}" - # Expand ${DOCKER[@]} only if it's not unset. This is to work around - # Bash 3 issue with unbound variable. - DOCKER=(docker ${DOCKER[@]:+"${DOCKER[@]}"}) - if ! "${DOCKER[@]}" info > /dev/null 2>&1 ; then - cat <<'EOF' >&2 -Can't connect to 'docker' daemon. please fix and retry. - -Possible causes: - - Docker Daemon not started - - Linux: confirm via your init system - - macOS w/ docker-machine: run `docker-machine ls` and `docker-machine start ` - - macOS w/ Docker for Mac: Check the menu bar and start the Docker application - - DOCKER_HOST hasn't been set or is set incorrectly - - Linux: domain socket is used, DOCKER_* should be unset. In Bash run `unset ${!DOCKER_*}` - - macOS w/ docker-machine: run `eval "$(docker-machine env )"` - - macOS w/ Docker for Mac: domain socket is used, DOCKER_* should be unset. In Bash run `unset ${!DOCKER_*}` - - Other things to check: - - Linux: User isn't in 'docker' group. Add and relogin. - - Something like 'sudo usermod -a -G docker ${USER}' - - RHEL7 bug and workaround: https://bugzilla.redhat.com/show_bug.cgi?id=1119282#c8 -EOF - return 1 - fi -} - # Wait for background jobs to finish. Return with # an error status if any of the jobs failed. kube::util::wait-for-jobs() { @@ -696,101 +109,6 @@ function kube::util::join { echo "$*" } -# Downloads cfssl/cfssljson into $1 directory if they do not already exist in PATH -# -# Assumed vars: -# $1 (cfssl directory) (optional) -# -# Sets: -# CFSSL_BIN: The path of the installed cfssl binary -# CFSSLJSON_BIN: The path of the installed cfssljson binary -# -function kube::util::ensure-cfssl { - if command -v cfssl &>/dev/null && command -v cfssljson &>/dev/null; then - CFSSL_BIN=$(command -v cfssl) - CFSSLJSON_BIN=$(command -v cfssljson) - return 0 - fi - - host_arch=$(kube::util::host_arch) - - if [[ "${host_arch}" != "amd64" ]]; then - echo "Cannot download cfssl on non-amd64 hosts and cfssl does not appear to be installed." - echo "Please install cfssl and cfssljson and verify they are in \$PATH." - echo "Hint: export PATH=\$PATH:\$GOPATH/bin; go get -u github.com/cloudflare/cfssl/cmd/..." - exit 1 - fi - - # Create a temp dir for cfssl if no directory was given - local cfssldir=${1:-} - if [[ -z "${cfssldir}" ]]; then - kube::util::ensure-temp-dir - cfssldir="${KUBE_TEMP}/cfssl" - fi - - mkdir -p "${cfssldir}" - pushd "${cfssldir}" > /dev/null || return 1 - - echo "Unable to successfully run 'cfssl' from ${PATH}; downloading instead..." - kernel=$(uname -s) - case "${kernel}" in - Linux) - curl --retry 10 -L -o cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 - curl --retry 10 -L -o cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 - ;; - Darwin) - curl --retry 10 -L -o cfssl https://pkg.cfssl.org/R1.2/cfssl_darwin-amd64 - curl --retry 10 -L -o cfssljson https://pkg.cfssl.org/R1.2/cfssljson_darwin-amd64 - ;; - *) - echo "Unknown, unsupported platform: ${kernel}." >&2 - echo "Supported platforms: Linux, Darwin." >&2 - exit 2 - esac - - chmod +x cfssl || true - chmod +x cfssljson || true - - CFSSL_BIN="${cfssldir}/cfssl" - CFSSLJSON_BIN="${cfssldir}/cfssljson" - if [[ ! -x ${CFSSL_BIN} || ! -x ${CFSSLJSON_BIN} ]]; then - echo "Failed to download 'cfssl'. Please install cfssl and cfssljson and verify they are in \$PATH." - echo "Hint: export PATH=\$PATH:\$GOPATH/bin; go get -u github.com/cloudflare/cfssl/cmd/..." - exit 1 - fi - popd > /dev/null || return 1 -} - -# kube::util::ensure_dockerized -# Confirms that the script is being run inside a kube-build image -# -function kube::util::ensure_dockerized { - if [[ -f /kube-build-image ]]; then - return 0 - else - echo "ERROR: This script is designed to be run inside a kube-build container" - exit 1 - fi -} - -# kube::util::ensure-gnu-sed -# Determines which sed binary is gnu-sed on linux/darwin -# -# Sets: -# SED: The name of the gnu-sed binary -# -function kube::util::ensure-gnu-sed { - if LANG=C sed --help 2>&1 | grep -q GNU; then - SED="sed" - elif command -v gsed &>/dev/null; then - SED="gsed" - else - kube::log::error "Failed to find GNU sed as sed or gsed. If you are on Mac: brew install gnu-sed." >&2 - return 1 - fi - kube::util::sourced_variable "${SED}" -} - # kube::util::check-file-in-alphabetical-order # Check that the file is in alphabetical order # @@ -808,15 +126,6 @@ function kube::util::check-file-in-alphabetical-order { fi } -# kube::util::require-jq -# Checks whether jq is installed. -function kube::util::require-jq { - if ! command -v jq &>/dev/null; then - echo "jq not found. Please install." 1>&2 - return 1 - fi -} - # Some useful colors. if [[ -z "${color_start-}" ]]; then declare -r color_start="\033[" diff --git a/verify-shellcheck.sh b/verify-shellcheck.sh index 1b882fa8..fd28021a 100755 --- a/verify-shellcheck.sh +++ b/verify-shellcheck.sh @@ -18,9 +18,12 @@ set -o errexit set -o nounset set -o pipefail -KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. -source "${KUBE_ROOT}/hack/lib/init.sh" -source "${KUBE_ROOT}/hack/lib/util.sh" +# The csi-release-tools directory. +TOOLS="$(dirname "${BASH_SOURCE[0]}")" +. "${TOOLS}/util.sh" + +# Directory to check. Default is the parent of the tools themselves. +ROOT="${1:-${TOOLS}/..}" # required version for this script, if not installed on the host we will # use the official docker image instead. keep this in sync with SHELLCHECK_IMAGE @@ -56,15 +59,15 @@ create_container () { # we're done. # This is incredibly much faster than creating a container for each shellcheck # call ... - docker run --name "${SHELLCHECK_CONTAINER}" -d --rm -v "${KUBE_ROOT}:${KUBE_ROOT}" -w "${KUBE_ROOT}" --entrypoint="sleep" "${SHELLCHECK_IMAGE}" 2147483647 + docker run --name "${SHELLCHECK_CONTAINER}" -d --rm -v "${ROOT}:${ROOT}" -w "${ROOT}" --entrypoint="sleep" "${SHELLCHECK_IMAGE}" 2147483647 } # removes the shellcheck container remove_container () { docker rm -f "${SHELLCHECK_CONTAINER}" &> /dev/null || true } -# ensure we're linting the k8s source tree -cd "${KUBE_ROOT}" +# ensure we're linting the source tree +cd "${ROOT}" # find all shell scripts excluding ./_*, ./.git/*, ./vendor*, # and anything git-ignored @@ -78,16 +81,6 @@ done < <(find . -name "*.sh" \ -path ./vendor\* \ \)) -# make sure known failures are sorted -failure_file="${KUBE_ROOT}/hack/.shellcheck_failures" -kube::util::check-file-in-alphabetical-order "${failure_file}" - -# load known failure files -failing_files=() -while IFS=$'\n' read -r script; - do failing_files+=("$script"); -done < <(cat "${failure_file}") - # detect if the host machine has the required shellcheck version installed # if so, we will use that instead. HAVE_SHELLCHECK=false @@ -119,7 +112,6 @@ fi # lint each script, tracking failures errors=() -not_failing=() for f in "${all_shell_scripts[@]}"; do set +o errexit if ${HAVE_SHELLCHECK}; then @@ -129,18 +121,14 @@ for f in "${all_shell_scripts[@]}"; do shellcheck --exclude="${SHELLCHECK_DISABLED}" "${f}") fi set -o errexit - kube::util::array_contains "${f}" "${failing_files[@]}" && in_failing=$? || in_failing=$? - if [[ -n "${failedLint}" ]] && [[ "${in_failing}" -ne "0" ]]; then - errors+=( "${failedLint}" ) - fi - if [[ -z "${failedLint}" ]] && [[ "${in_failing}" -eq "0" ]]; then - not_failing+=( "${f}" ) + if [[ -n "${failedLint}" ]]; then + errors+=( "${failedLint}" ) fi done # Check to be sure all the packages that should pass lint are. if [ ${#errors[@]} -eq 0 ]; then - echo 'Congratulations! All shell files are passing lint (excluding those in hack/.shellcheck_failures).' + echo 'Congratulations! All shell files are passing lint.' else { echo "Errors from shellcheck:" @@ -149,38 +137,9 @@ else done echo echo 'Please review the above warnings. You can test via "./hack/verify-shellcheck"' - echo 'If the above warnings do not make sense, you can exempt this package from shellcheck' - echo 'checking by adding it to hack/.shellcheck_failures (if your reviewer is okay with it).' - echo - } >&2 - false -fi - -if [[ ${#not_failing[@]} -gt 0 ]]; then - { - echo "Some packages in hack/.shellcheck_failures are passing shellcheck. Please remove them." - echo - for f in "${not_failing[@]}"; do - echo " $f" - done - echo - } >&2 - false -fi - -# Check that all failing_packages actually still exist -gone=() -for f in "${failing_files[@]}"; do - kube::util::array_contains "$f" "${all_shell_scripts[@]}" || gone+=( "$f" ) -done - -if [[ ${#gone[@]} -gt 0 ]]; then - { - echo "Some files in hack/.shellcheck_failures do not exist anymore. Please remove them." - echo - for f in "${gone[@]}"; do - echo " $f" - done + echo 'If the above warnings do not make sense, you can exempt them from shellcheck' + echo 'checking by adding the "shellcheck disable" directive' + echo '(https://github.com/koalaman/shellcheck/wiki/Directive#disable).' echo } >&2 false From 6c7ba1be0fbb012b6b3d6d204fcc363164bce12d Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Tue, 26 Mar 2019 13:09:00 +0100 Subject: [PATCH 07/71] build.make: integrate shellcheck into "make test" By default this only tests the scripts inside the "release-tools" directory, which is useful when making experimental changes to them in a component that uses csi-release-tools. But a component can also enable checking for other directories. --- README.md | 12 ++++++++++++ build.make | 15 +++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/README.md b/README.md index 56d2248c..cb028154 100644 --- a/README.md +++ b/README.md @@ -49,3 +49,15 @@ Cheat sheet: - `git subtree add --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master` - add release tools to a repo which does not have them yet (only once) - `git subtree pull --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master` - update local copy to latest upstream (whenever upstream changes) - edit, `git commit`, `git subtree push --prefix=release-tools git@github.com:/csi-release-tools.git ` - push to a new branch before submitting a PR + +verify-shellcheck.sh +-------------------- + +The [verify-shellcheck.sh](./verify-shellcheck.sh) script in this repo +is a stripped down copy of the [corresponding +script](https://github.com/kubernetes/kubernetes/blob/release-1.14/hack/verify-shellcheck.sh) +in the Kubernetes repository. It can be used to check for certain +errors shell scripts, like missing quotation marks. The default +`test-shellcheck` target in [build.make](./build.make) only checks the +scripts in this directory. Components can add more directories to +`TEST_SHELLCHECK_DIRS` to check also other scripts. diff --git a/build.make b/build.make index 6ea32795..3cac9d30 100644 --- a/build.make +++ b/build.make @@ -122,3 +122,18 @@ test: test-subtree test-subtree: @ echo; echo "### $@:" ./release-tools/verify-subtree.sh release-tools + +# Components can extend the set of directories which must pass shellcheck. +# The default is to check only the release-tools directory itself. +TEST_SHELLCHECK_DIRS=release-tools +.PHONY: test-shellcheck +test: test-shellcheck +test-shellcheck: + @ echo; echo "### $@:" + @ ret=0; \ + for dir in $(abspath $(TEST_SHELLCHECK_DIRS)); do \ + echo; \ + echo "$$dir:"; \ + ./release-tools/verify-shellcheck.sh "$$dir" || ret=1; \ + done; \ + return $$ret From 55212ff2b4f6a83c22cf2344b101d31cd11ec271 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 15 Mar 2019 16:45:16 +0100 Subject: [PATCH 08/71] initial Prow test job This enables testing of other repos and of this repo itself inside Prow. Currently supported is unit testing ("make test") and E2E testing (either via a local test suite or the Kubernetes E2E test suite applied to the hostpath driver example deployment). The script passes shellcheck and uses Prow to verify that for future PRs. --- .prow.sh | 7 + README.md | 45 +++ filter-junit.go | 133 +++++++ prow.sh | 916 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 1101 insertions(+) create mode 100755 .prow.sh create mode 100644 filter-junit.go create mode 100755 prow.sh diff --git a/.prow.sh b/.prow.sh new file mode 100755 index 00000000..b18c5358 --- /dev/null +++ b/.prow.sh @@ -0,0 +1,7 @@ +#! /bin/bash -e +# +# This is for testing csi-release-tools itself in Prow. All other +# repos use prow.sh for that, but as csi-release-tools isn't a normal +# repo with some Go code in it, it has a custom Prow test script. + +./verify-shellcheck.sh "$(pwd)" diff --git a/README.md b/README.md index cb028154..bc061aee 100644 --- a/README.md +++ b/README.md @@ -61,3 +61,48 @@ errors shell scripts, like missing quotation marks. The default `test-shellcheck` target in [build.make](./build.make) only checks the scripts in this directory. Components can add more directories to `TEST_SHELLCHECK_DIRS` to check also other scripts. + +End-to-end testing +------------------ + +A repo that wants to opt into testing via Prow must set up a top-level +`.prow.sh`. Typically that will source `prow.sh` and then transfer +control to it: + +``` bash +#! /bin/bash -e + +. release-tools/prow.sh +main +``` + +All Kubernetes-CSI repos are expected to switch to Prow. For details +on what is enabled in Prow, see +https://github.com/kubernetes/test-infra/tree/master/config/jobs/kubernetes-csi + +Test results for periodic jobs are visible in +https://testgrid.k8s.io/sig-storage-csi + +It is possible to reproduce the Prow testing locally on a suitable machine: +- Linux host +- Docker installed +- code to be tested checkout out in `$GOPATH/src/` +- `cd $GOPATH/src/ && ./.prow.sh` + +Beware that the script intentionally doesn't clean up after itself and +modifies the content of `$GOPATH`, in particular the `kubernetes` and +`kind` repositories there. Better run it in an empty, disposable +`$GOPATH`. + +When it terminates, the following command can be used to get access to +the Kubernetes cluster that was brought up for testing (assuming that +this step succeeded): + + export KUBECONFIG="$(kind get kubeconfig-path --name="csi-prow")" + +It is possible to control the execution via environment variables. See +`prow.sh` for details. Particularly useful is testing against different +Kubernetes releases: + + CSI_PROW_KUBERNETES_VERSION=1.13.3 ./.prow.sh + CSI_PROW_KUBERNETES_VERSION=latest ./.prow.sh diff --git a/filter-junit.go b/filter-junit.go new file mode 100644 index 00000000..2f51be00 --- /dev/null +++ b/filter-junit.go @@ -0,0 +1,133 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* + * This command filters a JUnit file such that only tests with a name + * matching a regular expression are passed through. By concatenating + * multiple input files it is possible to merge them into a single file. + */ +package main + +import ( + "encoding/xml" + "flag" + "io/ioutil" + "os" + "regexp" +) + +var ( + output = flag.String("o", "-", "junit file to write, - for stdout") + tests = flag.String("t", "", "regular expression matching the test names that are to be included in the output") +) + +/* + * TestSuite represents a JUnit file. Due to how encoding/xml works, we have + * represent all fields that we want to be passed through. It's therefore + * not a complete solution, but good enough for Ginkgo + Spyglass. + */ +type TestSuite struct { + XMLName string `xml:"testsuite"` + TestCases []TestCase `xml:"testcase"` +} + +type TestCase struct { + Name string `xml:"name,attr"` + Time string `xml:"time,attr"` + SystemOut string `xml:"system-out,omitempty"` + Failure string `xml:"failure,omitempty"` + Skipped SkipReason `xml:"skipped,omitempty"` +} + +// SkipReason deals with the special : +// if present, we must re-encode it, even if empty. +type SkipReason string + +func (s *SkipReason) UnmarshalText(text []byte) error { + *s = SkipReason(text) + if *s == "" { + *s = " " + } + return nil +} + +func (s SkipReason) MarshalText() ([]byte, error) { + if s == " " { + return []byte{}, nil + } + return []byte(s), nil +} + +func main() { + var junit TestSuite + var data []byte + + flag.Parse() + + re := regexp.MustCompile(*tests) + + // Read all input files. + for _, input := range flag.Args() { + if input == "-" { + if _, err := os.Stdin.Read(data); err != nil { + panic(err) + } + } else { + var err error + data, err = ioutil.ReadFile(input) + if err != nil { + panic(err) + } + } + if err := xml.Unmarshal(data, &junit); err != nil { + panic(err) + } + } + + // Keep only matching testcases. Testcases skipped in all test runs are only stored once. + filtered := map[string]TestCase{} + for _, testcase := range junit.TestCases { + if !re.MatchString(testcase.Name) { + continue + } + entry, ok := filtered[testcase.Name] + if !ok || // not present yet + entry.Skipped != "" && testcase.Skipped == "" { // replaced skipped test with real test run + filtered[testcase.Name] = testcase + } + } + junit.TestCases = nil + for _, testcase := range filtered { + junit.TestCases = append(junit.TestCases, testcase) + } + + // Re-encode. + data, err := xml.MarshalIndent(junit, "", " ") + if err != nil { + panic(err) + } + + // Write to output. + if *output == "-" { + if _, err := os.Stdout.Write(data); err != nil { + panic(err) + } + } else { + if err := ioutil.WriteFile(*output, data, 0644); err != nil { + panic(err) + } + } +} diff --git a/prow.sh b/prow.sh new file mode 100755 index 00000000..17e8ae4a --- /dev/null +++ b/prow.sh @@ -0,0 +1,916 @@ +#! /bin/bash +# +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This script runs inside a Prow job. It runs unit tests ("make test") +# and E2E testing. This E2E testing covers different scenarios (see +# https://github.com/kubernetes/enhancements/pull/807): +# - running the stable hostpath example against a Kubernetes release +# - running the canary hostpath example against a Kubernetes release +# - building the component in the current repo and running the +# stable hostpath example with that one component replaced against +# a Kubernetes release +# +# The intended usage of this script is that individual repos import +# csi-release-tools, then link their top-level prow.sh to this or +# include it in that file. When including it, several of the variables +# can be overridden in the top-level prow.sh to customize the script +# for the repo. +# +# The expected environment is: +# - $GOPATH/src/ for the repository that is to be tested, +# with PR branch merged (when testing a PR) +# - running on linux-amd64 +# - bazel installed (when testing against Kubernetes master), must be recent +# enough for Kubernetes master +# - kind (https://github.com/kubernetes-sigs/kind) installed +# - optional: Go already installed + +# Sets the default value for a variable if not set already and logs the value. +# Any variable set this way is usually something that a repo's .prow.sh +# or the job can set. +configvar () { + # Ignore: Word is of the form "A"B"C" (B indicated). Did you mean "ABC" or "A\"B\"C"? + # shellcheck disable=SC2140 + eval : \$\{"$1":="\$2"\} + eval echo "\$3:" "$1=\${$1}" +} + +# Go versions can be specified seperately for different tasks +# If the pre-installed Go is missing or a different +# version, the required version here will get installed +# from https://golang.org/dl/. +configvar CSI_PROW_GO_VERSION_BUILD 1.11.4 "Go version for building the component" # depends on component's source code +configvar CSI_PROW_GO_VERSION_K8S 1.12.1 "Go version for building Kubernetes for the test cluster" # depends on Kubernetes version +configvar CSI_PROW_GO_VERSION_E2E 1.12.1 "Go version for building the Kubernetes E2E test suite" # depends on CSI_PROW_E2E settings below +configvar CSI_PROW_GO_VERSION_SANITY "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building the csi-sanity test suite" # depends on CSI_PROW_SANITY settings below +configvar CSI_PROW_GO_VERSION_KIND "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building 'kind'" # depends on CSI_PROW_KIND_VERSION below +configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building ginkgo" # depends on CSI_PROW_GINKGO_VERSION below + +# kind version to use. If the pre-installed version is different, +# the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases/download/ +# (if available), otherwise it is built from source. +configvar CSI_PROW_KIND_VERSION 0.2.1 "kind" + +# ginkgo test runner version to use. If the pre-installed version is +# different, the desired version is built from source. +configvar CSI_PROW_GINKGO_VERSION v1.7.0 "Ginkgo" + +# Ginkgo runs the E2E test in parallel. The default is based on the number +# of CPUs, but typically this can be set to something higher in the job. +configvar CSI_PROW_GINKO_PARALLEL "-p" "Ginko parallelism parameter(s)" + +# Enables building the code in the repository. On by default, can be +# disabled in jobs which only use pre-built components. +configvar CSI_PROW_BUILD_JOB true "building code in repo enabled" + +# Kubernetes version to test against. This must be a version number +# (like 1.13.3) for which there is a pre-built kind image (see +# https://hub.docker.com/r/kindest/node/tags), "latest" (builds +# Kubernetes from the master branch) or "release-x.yy" (builds +# Kubernetes from a release branch). +# +# This can also be a version that was not released yet at the time +# that the settings below were chose. The script will then +# use the same settings as for "latest" Kubernetes. This works +# as long as there are no breaking changes in Kubernetes, like +# deprecating or changing the implementation of an alpha feature. +configvar CSI_PROW_KUBERNETES_VERSION 1.13.3 "Kubernetes" + +# CSI_PROW_KUBERNETES_VERSION reduced to first two version numbers and +# with underscore (1_13 instead of 1.13.3) and in uppercase (LATEST +# instead of latest). +# +# This is used to derive the right defaults for the variables below +# when a Prow job just defines the Kubernetes version. +csi_prow_kubernetes_version_suffix="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | tr . _ | tr '[:lower:]' '[:upper:]' | sed -e 's/^RELEASE-//' -e 's/\([0-9]*\)_\([0-9]*\).*/\1_\2/')" + +# Work directory. It has to allow running executables, therefore /tmp +# is avoided. Cleaning up after the script is intentionally left to +# the caller. +configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory" + +# The hostpath deployment script is searched for in several places. +# +# - The "deploy" directory in the current repository: this is useful +# for the situation that a component becomes incompatible with the +# shared deployment, because then it can (temporarily!) provide its +# own example until the shared one can be updated; it's also how +# csi-driver-host-path itself provides the example. +# +# - CSI_PROW_HOSTPATH_VERSION of the CSI_PROW_HOSTPATH_REPO is checked +# out: this allows other repos to reference a version of the example +# that is known to be compatible. +# +# - The csi-driver-host-path/deploy directory has multiple sub-directories, +# each with different deployments (stable set of images for Kubernetes 1.13, +# stable set of images for Kubernetes 1.14, canary for latest Kubernetes, etc.). +# This is necessary because there may be incompatible changes in the +# "API" of a component (for example, its command line options or RBAC rules) +# or in its support for different Kubernetes versions (CSIDriverInfo as +# CRD in Kubernetes 1.13 vs builtin API in Kubernetes 1.14). +# +# When testing an update for a component in a PR job, the +# CSI_PROW_DEPLOYMENT variable can be set in the +# .prow.sh of each component when there are breaking changes +# that require using a non-default deployment. The default +# is a deployment named "kubernetes-x.yy" (if available), +# otherwise "kubernetes-latest". +# "none" disables the deployment of the hostpath driver. +# +# When no deploy script is found (nothing in `deploy` directory, +# CSI_PROW_HOSTPATH_REPO=none), nothing gets deployed. +# +# TODO: merge https://github.com/kubernetes-csi/csi-driver-host-path/pull/29, switch to revision from master +configvar CSI_PROW_HOSTPATH_VERSION deployment "hostpath driver" +configvar CSI_PROW_HOSTPATH_REPO https://github.com/pohly/csi-driver-host-path "hostpath repo" +# Ignore 'See if you can use ${variable//search/replace} instead.' +# shellcheck disable=SC2001 +configvar CSI_PROW_DEPLOYMENT "" "deployment" + +# If CSI_PROW_HOSTPATH_CANARY is set (typically to "canary", but also +# "1.0-canary"), then all image versions are replaced with that +# version tag. +configvar CSI_PROW_HOSTPATH_CANARY "" "hostpath image" + +# The E2E testing can come from an arbitrary repo. The expectation is that +# the repo supports "go test ./test/e2e -args --storage.testdriver" (https://github.com/kubernetes/kubernetes/pull/72836) +# after setting KUBECONFIG. As a special case, if the repository is Kubernetes, +# then `make WHAT=test/e2e/e2e.test` is called first to ensure that +# all generated files are present. +# +# CSI_PROW_E2E_REPO=none disables E2E testing. +configvar CSI_PROW_E2E_VERSION v1.14.0 "E2E version" +configvar CSI_PROW_E2E_REPO https://github.com/kubernetes/kubernetes "E2E repo" +configvar CSI_PROW_E2E_IMPORT_PATH k8s.io/kubernetes "E2E package" + +# csi-sanity testing from the csi-test repo can be run against the installed +# CSI driver. For this to work, deploying the driver must expose the Unix domain +# csi.sock as a TCP service for use by the csi-sanity command, which runs outside +# of the cluster. The alternative would have been to (cross-)compile csi-sanity +# and install it inside the cluster, which is not necessarily easier. +configvar CSI_PROW_SANITY_REPO https://github.com/kubernetes-csi/csi-test "csi-test repo" +configvar CSI_PROW_SANITY_VERSION 5421d9f3c37be3b95b241b44a094a3db11bee789 "csi-test version" # latest master +configvar CSI_PROW_SANITY_IMPORT_PATH github.com/kubernetes-csi/csi-test "csi-test package" +configvar CSI_PROW_SANITY_SERVICE "hostpath-service" "Kubernetes TCP service name that exposes csi.sock" +configvar CSI_PROW_SANITY_POD "csi-hostpathplugin-0" "Kubernetes pod with CSI driver" +configvar CSI_PROW_SANITY_CONTAINER "hostpath" "Kubernetes container with CSI driver" + +# Each job can run one or more of the following tests, identified by +# a single word: +# - unit testing +# - parallel excluding alpha features +# - serial excluding alpha features +# - parallel, only alpha feature +# - serial, only alpha features +# - sanity +# +# Sanity testing with csi-sanity only covers the CSI driver itself and thus +# is off by default. A CSI driver can change that default in its .prow.sh +# by setting CSI_PROW_TESTS_SANITY. +configvar CSI_PROW_TESTS "unit parallel serial parallel-alpha serial-alpha ${CSI_PROW_TESTS_SANITY}" "tests to run" +test_enabled () { + echo "${CSI_PROW_TESTS}" | grep -q -w -e "$1" +} + +# Serial vs. parallel is always determined by these regular expressions. +# Individual regular expressions are seperated by spaces for readability +# and expected to not contain spaces. Use dots instead. The complete +# regex for Ginkgo will be created by joining the individual terms. +configvar CSI_PROW_E2E_SERIAL '\[Serial\] \[Disruptive\]' "tags for serial E2E tests" +regex_join () { + echo "$@" | sed -e 's/ */|/g' -e 's/^|*//' -e 's/|*$//' -e 's/^$/this-matches-nothing/g' +} + +# Prints the value of a variable + version suffix, falling back to variable + "LATEST". +get_versioned_variable () { + local var="$1" + local version="$2" + local value + + eval value="\${${var}_${version}}" + if ! [ "$value" ]; then + eval value="\${${var}_LATEST}" + fi + echo "$value" +} + +# Which tests are alpha depends on the Kubernetes version. The same +# E2E test suite is used for all Kubernetes versions, including older +# Kubernetes. +# +# Feature tags in the test are set based on what is an alpha +# feature in the Kubernetes version that contains the E2E test suite's +# source code. +# +# So when testing against an older Kubernetes release, some tests +# might be enabled by default which don't pass for that older +# Kubernetes version. In that case, the +# CSI_PROW_E2E_ALPHA_ variable cannot just +# be based on the `Feature` tag, it also must contain the names +# of tests that used to be alpha but no longer have that tag. +# +# is just major+minor version separated by +# underscore. "latest" matches master. It is also used for +# unknown versions, so when master gets released there is +# no need to add a new versioned variable. That only needs to be +# done when updating something in the configuration that leads +# to "latest" no longer being suitable. +configvar CSI_PROW_E2E_ALPHA_1_13 '\[Feature: \[Testpattern:.Dynamic.PV..block.volmode.\] should.create.and.delete.block.persistent.volumes' "alpha tests for Kubernetes 1.13" # Raw block was an alpha feature in 1.13. +configvar CSI_PROW_E2E_ALPHA_LATEST '\[Feature:' "alpha tests for Kubernetes master" +configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi_prow_kubernetes_version_suffix}")" "alpha tests" + +# After the parallel E2E test without alpha features, a test cluster +# with alpha features is brought up and tests that were previously +# disabled are run. The alpha gates in each release have to be listed +# explicitly. If none are set (= variable empty), alpha testing +# is skipped. +# +# Testing against "latest" Kubernetes is problematic because some alpha +# feature which used to work might stop working or change their behavior +# such that the current tests no longer pass. If that happens, +# kubernetes-csi components must be updated, either by disabling +# the failing test for "latest" or by updating the test and not running +# it anymore for older releases. +configvar CSI_PROW_E2E_ALPHA_GATES_1_13 'VolumeSnapshotDataSource=true,BlockVolume=true,CSIBlockVolume=true' "alpha feature gates for Kubernetes 1.13" +configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'VolumeSnapshotDataSource=true' "alpha feature gates for latest Kubernetes" +configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" + +# Some tests are known to be unusable in a KinD cluster. For example, +# stopping kubelet with "ssh systemctl stop kubelet" simply +# doesn't work. Such tests should be written in a way that they verify +# whether they can run with the current cluster provider, but until +# they are, we filter them out by name. Like the other test selection +# variables, this is again a space separated list of regular expressions. +configvar CSI_PROW_E2E_SKIP 'while.kubelet.is.down.*Disruptive' "tests that need to be skipped" + +# This is the directory for additional result files. Usually set by Prow, but +# if not (for example, when invoking manually) it defaults to the work directory. +configvar ARTIFACTS "${CSI_PROW_WORK}/artifacts" "artifacts" +mkdir -p "${ARTIFACTS}" + +RELEASE_TOOLS_ROOT="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" +REPO_DIR="$(pwd)" + +run () { + echo "$(date) $(go version | sed -e 's/.*version \(go[^ ]*\).*/\1/') $(if [ "$(pwd)" != "${REPO_DIR}" ]; then pwd; fi)\$" "$@" >&2 + "$@" +} + +info () { + echo >&2 INFO: "$@" +} + +warn () { + echo >&2 WARNING: "$@" +} + +die () { + echo >&2 ERROR: "$@" + exit 1 +} + +# For additional tools. +CSI_PROW_BIN="${CSI_PROW_WORK}/bin" +mkdir -p "${CSI_PROW_BIN}" +PATH="${CSI_PROW_BIN}:$PATH" + +# Ensure that PATH has the desired version of the Go tools, then run command given as argument. +run_with_go () { + local version + version="$1" + shift + + if go version 2>/dev/null | grep -q "go$version"; then + run "$@" + else + if ! [ -d "${CSI_PROW_WORK}/go-$version" ]; then + run curl --fail --location "https://dl.google.com/go/go$version.linux-amd64.tar.gz" | tar -C "${CSI_PROW_WORK}" -zxf - || die "installation of Go $version failed" + mv "${CSI_PROW_WORK}/go" "${CSI_PROW_WORK}/go-$version" + fi + PATH="${CSI_PROW_WORK}/go-$version/bin:$PATH" run "$@" + fi +} + +# Ensure that we have the desired version of kind. +install_kind () { + if kind --version 2>/dev/null | grep -q " ${CSI_PROW_KIND_VERSION}$"; then + return + fi + if run curl --fail --location -o "${CSI_PROW_WORK}/bin/kind" "https://github.com/kubernetes-sigs/kind/releases/download/${CSI_PROW_KIND_VERSION}/kind-linux-amd64"; then + chmod u+x "${CSI_PROW_WORK}/bin/kind" + else + git_checkout https://github.com/kubernetes-sigs/kind "$GOPATH/src/sigs.k8s.io/kind" "${CSI_PROW_KIND_VERSION}" --depth=1 && + run_with_go "${CSI_PROW_GO_VERSION_KIND}" go build -o "${CSI_PROW_WORK}/bin/kind" sigs.k8s.io/kind + fi +} + +# Ensure that we have the desired version of the ginkgo test runner. +install_ginkgo () { + # CSI_PROW_GINKGO_VERSION contains the tag with v prefix, the command line output does not. + if [ "v$(ginkgo version 2>/dev/null | sed -e 's/.* //')" = "${CSI_PROW_GINKGO_VERSION}" ]; then + return + fi + git_checkout https://github.com/onsi/ginkgo "$GOPATH/src/github.com/onsi/ginkgo" "${CSI_PROW_GINKGO_VERSION}" --depth=1 && + # We have to get dependencies and hence can't call just "go build". + run_with_go "${CSI_PROW_GO_VERSION_GINKGO}" go get github.com/onsi/ginkgo/ginkgo || die "building ginkgo failed" && + mv "$GOPATH/bin/ginkgo" "${CSI_PROW_BIN}" +} + +# This checks out a repo ("https://github.com/kubernetes/kubernetes") +# in a certain location ("$GOPATH/src/k8s.io/kubernetes") at +# a certain revision (a hex commit hash, v1.13.1, master). It's okay +# for that directory to exist already. +git_checkout () { + local repo path revision + repo="$1" + shift + path="$1" + shift + revision="$1" + shift + + mkdir -p "$path" + if ! [ -d "$path/.git" ]; then + run git init "$path" + fi + if (cd "$path" && run git fetch "$@" "$repo" "$revision"); then + (cd "$path" && run git checkout FETCH_HEAD) || die "checking out $repo $revision failed" + else + # Might have been because fetching by revision is not + # supported by GitHub (https://github.com/isaacs/github/issues/436). + # Fall back to fetching everything. + (cd "$path" && run git fetch "$repo" '+refs/heads/*:refs/remotes/csiprow/heads/*' '+refs/tags/*:refs/tags/*') || die "fetching $repo failed" + (cd "$path" && run git checkout "$revision") || die "checking out $repo $revision failed" + fi +} + +list_gates () ( + set -f; IFS=',' + # Ignore: Double quote to prevent globbing and word splitting. + # shellcheck disable=SC2086 + set -- $1 + while [ "$1" ]; do + # Ignore: See if you can use ${variable//search/replace} instead. + # shellcheck disable=SC2001 + echo "$1" | sed -e 's/ *\([^ =]*\) *= *\([^ ]*\) */ \1: \2/' + shift + done +) + +csi_prow_kind_have_kubernetes=false +# Brings up a Kubernetes cluster and sets KUBECONFIG. +# Accepts additional feature gates in the form gate1=true|false,gate2=... +start_cluster () { + local image gates + gates="$1" + + if kind get clusters | grep -q csi-prow; then + run kind delete cluster --name=csi-prow || die "kind delete failed" + fi + + # Build from source? + if [[ "${CSI_PROW_KUBERNETES_VERSION}" =~ ^release-|^latest$ ]]; then + if ! ${csi_prow_kind_have_kubernetes}; then + local version="${CSI_PROW_KUBERNETES_VERSION}" + if [ "$version" = "latest" ]; then + version=master + fi + git_checkout https://github.com/kubernetes/kubernetes "$GOPATH/src/k8s.io/kubernetes" "$version" --depth=1 || die "checking out Kubernetes $version failed" + + # "kind build" and/or the Kubernetes build rules need at least one tag, which we don't have + # when doing a shallow fetch. Therefore we fake one. For some reason, v1.0.0 didn't work + # while v1.14.0-fake.1 did. + (cd "$GOPATH/src/k8s.io/kubernetes" && run git tag -f v1.14.0-fake.1) || die "git tag failed" + + run_with_go "${CSI_PROW_GO_VERSION_K8S}" kind build node-image --type bazel --image csiprow/node:latest --kube-root "$GOPATH/src/k8s.io/kubernetes" || die "'kind build node-image' failed" + csi_prow_kind_have_kubernetes=true + fi + image="csiprow/node:latest" + else + image="kindest/node:v${CSI_PROW_KUBERNETES_VERSION}" + fi + cat >"${CSI_PROW_WORK}/kind-config.yaml" </dev/null; wait) + info "For container output see job artifacts." + die "deploying the hostpath driver with ${deploy_hostpath} failed" + fi +} + +# collect logs and cluster status (like the version of all components, Kubernetes version, test version) +collect_cluster_info () { + cat <>"${ARTIFACTS}/$pod/$container.log" & + echo "$!" + done + done +} + +# Makes the E2E test suite binary available as "${CSI_PROW_WORK}/e2e.test". +install_e2e () { + if [ -e "${CSI_PROW_WORK}/e2e.test" ]; then + return + fi + + git_checkout "${CSI_PROW_E2E_REPO}" "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}" --depth=1 && + if [ "${CSI_PROW_E2E_IMPORT_PATH}" = "k8s.io/kubernetes" ]; then + run_with_go "${CSI_PROW_GO_VERSION_E2E}" make WHAT=test/e2e/e2e.test "-C${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" && + ln -s "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}/_output/bin/e2e.test" "${CSI_PROW_WORK}" + else + run_with_go "${CSI_PROW_GO_VERSION_E2E}" go test -c -o "${CSI_PROW_WORK}/e2e.test" "${CSI_PROW_E2E_IMPORT_PATH}/test/e2e" + fi +} + +# Makes the csi-sanity test suite binary available as +# "${CSI_PROW_WORK}/csi-sanity". +install_sanity () ( + if [ -e "${CSI_PROW_WORK}/csi-sanity" ]; then + return + fi + + git_checkout "${CSI_PROW_SANITY_REPO}" "${GOPATH}/src/${CSI_PROW_SANITY_IMPORT_PATH}" "${CSI_PROW_SANITY_VERSION}" --depth=1 || die "checking out csi-sanity failed" + run_with_go "${CSI_PROW_GO_VERSION_SANITY}" go test -c -o "${CSI_PROW_WORK}/csi-sanity" "${CSI_PROW_SANITY_IMPORT_PATH}/cmd/csi-sanity" || die "building csi-sanity failed" +) + +# Whether the hostpath driver supports raw block devices depends on which version +# we are testing. It would be much nicer if we could determine that by querying the +# installed driver. +hostpath_supports_block () { + if [ -e "cmd/hostpathplugin" ] && ${CSI_PROW_BUILD_JOB}; then + # The assumption is that if we build the hostpath driver, then it is + # a current version with support. + echo true + return + fi + + case "${CSI_PROW_DEPLOYMENT}" in kubernetes-1.13) echo false;; # wasn't supported and probably won't be backported + *) echo true;; # probably all other deployments have a recent driver + esac +} + +# Captures pod output while running some other command. +run_with_loggers () ( + loggers=$(start_loggers -f) + trap 'kill $loggers' EXIT + + run "$@" +) + +# Invokes the filter-junit.go tool. +run_filter_junit () { + run_with_go "${CSI_PROW_GO_VERSION_BUILD}" go run "${RELEASE_TOOLS_ROOT}/filter-junit.go" "$@" +} + +# Runs the E2E test suite in a sub-shell. +run_e2e () ( + name="$1" + shift + + install_e2e || die "building e2e.test failed" + install_ginkgo || die "installing ginkgo failed" + + # TODO (?): multi-node cluster (depends on https://github.com/kubernetes-csi/csi-driver-host-path/pull/14). + # When running on a multi-node cluster, we need to figure out where the + # hostpath driver was deployed and set ClientNodeName accordingly. + + # The content of this file depends on both what the E2E suite expects and + # what the installed hostpath driver supports. Generating it here seems + # prone to breakage, but it is uncertain where a better place might be. + cat >"${CSI_PROW_WORK}/hostpath-test-driver.yaml" </dev/null >/dev/null; then + run_filter_junit -t="External Storage" -o "${ARTIFACTS}/junit_${name}.xml" "${ARTIFACTS}"/junit_[0-9]*.xml && rm -f "${ARTIFACTS}"/junit_[0-9]*.xml + fi + } + trap move_junit EXIT + + cd "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" && + run_with_loggers ginkgo -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/hostpath-test-driver.yaml" +) + +# Run csi-sanity against installed CSI driver. +run_sanity () ( + install_sanity || die "installing csi-sanity failed" + + cat >"${CSI_PROW_WORK}/mkdir_in_pod.sh" <"${CSI_PROW_WORK}/rmdir_in_pod.sh" </\>/g' -e 's/\x1B...//g' +} + +# The "make test" output starts each test with "### :" +# and then ends when the next test starts or with "make: *** +# [] Error 1" when there was a failure. Here we read each +# line of that output, split it up into individual tests and generate +# a make-test.xml file in JUnit format. +make_test_to_junit () { + local ret out testname testoutput + ret=0 + # Plain make-test.xml was not delivered as text/xml by the web + # server and ignored by spyglass. It seems that the name has to + # match junit*.xml. + out="${ARTIFACTS}/junit_make_test.xml" + testname= + echo "" >>"$out" + + while IFS= read -r line; do + echo "$line" # pass through + if echo "$line" | grep -q "^### [^ ]*:$"; then + if [ "$testname" ]; then + # previous test succesful + echo " " >>"$out" + echo " " >>"$out" + fi + # Ignore: See if you can use ${variable//search/replace} instead. + # shellcheck disable=SC2001 + # + # start new test + testname="$(echo "$line" | sed -e 's/^### \([^ ]*\):$/\1/')" + testoutput= + echo " " >>"$out" + echo " " >>"$out" + elif echo "$line" | grep -q '^make: .*Error [0-9]*$'; then + if [ "$testname" ]; then + # Ignore: Consider using { cmd1; cmd2; } >> file instead of individual redirects. + # shellcheck disable=SC2129 + # + # end test with failure + echo " " >>"$out" + # Include the same text as in also in , + # because then it is easier to view in spyglass (shown directly + # instead of having to click through to stdout). + echo " " >>"$out" + echo -n "$testoutput" | ascii_to_xml >>"$out" + echo " " >>"$out" + echo " " >>"$out" + fi + # remember failure for exit code + ret=1 + # not currently inside a test + testname= + else + if [ "$testname" ]; then + # Test output. + echo "$line" | ascii_to_xml >>"$out" + testoutput="$testoutput$line +" + fi + fi + done + # if still in a test, close it now + if [ "$testname" ]; then + echo " " >>"$out" + echo " " >>"$out" + fi + echo "" >>"$out" + + # this makes the error more visible in spyglass + if [ "$ret" -ne 0 ]; then + echo "ERROR: 'make test' failed" + return 1 + fi +} + +main () { + local images ret + ret=0 + + images= + if ${CSI_PROW_BUILD_JOB}; then + # A successful build is required for testing. + run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make all || die "'make all' failed" + # We don't want test failures to prevent E2E testing below, because the failure + # might have been minor or unavoidable, for example when experimenting with + # changes in "release-tools" in a PR (that fails the "is release-tools unmodified" + # test). + if test_enabled "unit"; then + if ! run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make -k test 2>&1 | make_test_to_junit; then + warn "'make test' failed, proceeding anyway" + ret=1 + fi + fi + # Required for E2E testing. + run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make container || die "'make container' failed" + fi + + install_kind || die "installing kind failed" + start_cluster || die "starting the cluster failed" + + if ${CSI_PROW_BUILD_JOB}; then + cmds="$(grep '^\s*CMDS\s*=' Makefile | sed -e 's/\s*CMDS\s*=//')" + # Get the image that was just built (if any) from the + # top-level Makefile CMDS variable and set the + # deploy-hostpath.sh env variables for it. We also need to + # side-load those images into the cluster. + for i in $cmds; do + e=$(echo "$i" | tr '[:lower:]' '[:upper:]' | tr - _) + images="$images ${e}_REGISTRY=none ${e}_TAG=csiprow" + + # We must avoid the tag "latest" because that implies + # always pulling the image + # (https://github.com/kubernetes-sigs/kind/issues/328). + docker tag "$i:latest" "$i:csiprow" || die "tagging the locally built container image for $i failed" + kind load docker-image --name csi-prow "$i:csiprow" || die "could not load the $i:latest image into the kind cluster" + done + + if [ -e deploy/kubernetes/rbac.yaml ]; then + # This is one of those components which has its own RBAC rules (like external-provisioner). + # We are testing a locally built image and also want to test with the the current, + # potentially modified RBAC rules. + if [ "$(echo "$cmds" | wc -w)" != 1 ]; then + die "ambiguous deploy/kubernetes/rbac.yaml: need exactly one command, got: $cmds" + fi + e=$(echo "$cmds" | tr '[:lower:]' '[:upper:]' | tr - _) + images="$images ${e}_RBAC=$(pwd)/deploy/kubernetes/rbac.yaml" + fi + fi + + # Installing the driver might be disabled, in which case we bail out early. + if ! install_hostpath "$images"; then + info "hostpath driver installation disabled, skipping E2E testing" + return "$ret" + fi + + collect_cluster_info + + if test_enabled "sanity"; then + if ! run_sanity; then + ret=1 + fi + fi + + if test_enabled "parallel"; then + # Ignore: Double quote to prevent globbing and word splitting. + # shellcheck disable=SC2086 + if ! run_e2e parallel ${CSI_PROW_GINKO_PARALLEL} \ + -focus="External.Storage" \ + -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}" "${CSI_PROW_E2E_ALPHA}" "${CSI_PROW_E2E_SKIP}")"; then + warn "E2E parallel failed" + ret=1 + fi + fi + + if test_enabled "serial"; then + if ! run_e2e serial \ + -focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_SERIAL}"))" \ + -skip="$(regex_join "${CSI_PROW_E2E_ALPHA}" "${CSI_PROW_E2E_SKIP}")"; then + warn "E2E serial failed" + ret=1 + fi + fi + + if (test_enabled "parallel-alpha" || test_enabled "serial-alpha") && [ "${CSI_PROW_E2E_ALPHA_GATES}" ]; then + # Need to (re)create the cluster. + start_cluster "${CSI_PROW_E2E_ALPHA_GATES}" || die "starting alpha cluster failed" + if ${CSI_PROW_BUILD_JOB}; then + # Ignore: Double quote to prevent globbing and word splitting. + # Ignore: To read lines rather than words, pipe/redirect to a 'while read' loop. + # shellcheck disable=SC2086 disable=SC2013 + for i in $(grep '^\s*CMDS\s*=' Makefile | sed -e 's/\s*CMDS\s*=//'); do + kind load docker-image --name csi-prow $i:csiprow || die "could not load the $i:latest image into the kind cluster" + done + fi + install_hostpath "$images" || die "hostpath driver installation failed unexpectedly on alpha cluster" + + if test_enabled "parallel-alpha"; then + # Ignore: Double quote to prevent globbing and word splitting. + # shellcheck disable=SC2086 + if ! run_e2e parallel-alpha ${CSI_PROW_GINKO_PARALLEL} \ + -focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_ALPHA}"))" \ + -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}" "${CSI_PROW_E2E_SKIP}")"; then + warn "E2E parallel alpha failed" + ret=1 + fi + fi + + if test_enabled "serial-alpha"; then + if ! run_e2e serial-alpha \ + -focus="External.Storage.*(($(regex_join "${CSI_PROW_E2E_SERIAL}")).*($(regex_join "${CSI_PROW_E2E_ALPHA}"))|($(regex_join "${CSI_PROW_E2E_ALPHA}")).*($(regex_join "${CSI_PROW_E2E_SERIAL}")))" \ + -skip="$(regex_join "${CSI_PROW_E2E_SKIP}")"; then + warn "E2E serial alpha failed" + ret=1 + fi + fi + fi + + # Merge all junit files into one. This gets rid of duplicated "skipped" tests. + if ls "${ARTIFACTS}"/junit_*.xml 2>/dev/null >&2; then + run_filter_junit -o "${CSI_PROW_WORK}/junit_final.xml" "${ARTIFACTS}"/junit_*.xml && rm "${ARTIFACTS}"/junit_*.xml && mv "${CSI_PROW_WORK}/junit_final.xml" "${ARTIFACTS}" + fi + + return "$ret" +} From 0a0fd49b8b146c21e9c90ffbfd696b83b6ed86b7 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Tue, 2 Apr 2019 20:53:08 +0200 Subject: [PATCH 09/71] prow.sh: comment clarification --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 17e8ae4a..faa5c361 100755 --- a/prow.sh +++ b/prow.sh @@ -15,7 +15,7 @@ # limitations under the License. -# This script runs inside a Prow job. It runs unit tests ("make test") +# This script runs inside a Prow job. It can run unit tests ("make test") # and E2E testing. This E2E testing covers different scenarios (see # https://github.com/kubernetes/enhancements/pull/807): # - running the stable hostpath example against a Kubernetes release From 429581c52d1cdbd1d1c3f9c25411fe6aa8c66610 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Tue, 2 Apr 2019 20:53:12 +0200 Subject: [PATCH 10/71] prow.sh: pull Go version from travis.yml The travis.yml is now the only place where the Go version for the component itself needs to be configured. --- prow.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/prow.sh b/prow.sh index faa5c361..bab7eb04 100755 --- a/prow.sh +++ b/prow.sh @@ -39,6 +39,9 @@ # - kind (https://github.com/kubernetes-sigs/kind) installed # - optional: Go already installed +RELEASE_TOOLS_ROOT="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" +REPO_DIR="$(pwd)" + # Sets the default value for a variable if not set already and logs the value. # Any variable set this way is usually something that a repo's .prow.sh # or the job can set. @@ -53,7 +56,10 @@ configvar () { # If the pre-installed Go is missing or a different # version, the required version here will get installed # from https://golang.org/dl/. -configvar CSI_PROW_GO_VERSION_BUILD 1.11.4 "Go version for building the component" # depends on component's source code +go_from_travis_yml () { + grep "^ *- go:" "${RELEASE_TOOLS_ROOT}/travis.yml" | sed -e 's/.*go: *//' +} +configvar CSI_PROW_GO_VERSION_BUILD "$(go_from_travis_yml)" "Go version for building the component" # depends on component's source code configvar CSI_PROW_GO_VERSION_K8S 1.12.1 "Go version for building Kubernetes for the test cluster" # depends on Kubernetes version configvar CSI_PROW_GO_VERSION_E2E 1.12.1 "Go version for building the Kubernetes E2E test suite" # depends on CSI_PROW_E2E settings below configvar CSI_PROW_GO_VERSION_SANITY "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building the csi-sanity test suite" # depends on CSI_PROW_SANITY settings below @@ -262,9 +268,6 @@ configvar CSI_PROW_E2E_SKIP 'while.kubelet.is.down.*Disruptive' "tests that need configvar ARTIFACTS "${CSI_PROW_WORK}/artifacts" "artifacts" mkdir -p "${ARTIFACTS}" -RELEASE_TOOLS_ROOT="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" -REPO_DIR="$(pwd)" - run () { echo "$(date) $(go version | sed -e 's/.*version \(go[^ ]*\).*/\1/') $(if [ "$(pwd)" != "${REPO_DIR}" ]; then pwd; fi)\$" "$@" >&2 "$@" From 29545bb012b6cbeab74ffbced61951e9b36137d2 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 3 Apr 2019 12:38:33 +0200 Subject: [PATCH 11/71] prow.sh: take Go version from Kubernetes source Using the same (recent) Go version for all Kubernetes versions can break for older versions when there are incompatible changes in Go. To avoid that, we use exactly the minimum version of Go required for each Kubernetes version. This is based on the assumption that this combination was tested successfully. When building the E2E suite from Kubernetes (the default) we do the same, but still allow building it from elsewhere. We allow the Go version to be empty when it doesn't matter. --- prow.sh | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/prow.sh b/prow.sh index bab7eb04..6fd55cc2 100755 --- a/prow.sh +++ b/prow.sh @@ -60,8 +60,7 @@ go_from_travis_yml () { grep "^ *- go:" "${RELEASE_TOOLS_ROOT}/travis.yml" | sed -e 's/.*go: *//' } configvar CSI_PROW_GO_VERSION_BUILD "$(go_from_travis_yml)" "Go version for building the component" # depends on component's source code -configvar CSI_PROW_GO_VERSION_K8S 1.12.1 "Go version for building Kubernetes for the test cluster" # depends on Kubernetes version -configvar CSI_PROW_GO_VERSION_E2E 1.12.1 "Go version for building the Kubernetes E2E test suite" # depends on CSI_PROW_E2E settings below +configvar CSI_PROW_GO_VERSION_E2E "" "override Go version for building the Kubernetes E2E test suite" # normally doesn't need to be set, see install_e2e configvar CSI_PROW_GO_VERSION_SANITY "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building the csi-sanity test suite" # depends on CSI_PROW_SANITY settings below configvar CSI_PROW_GO_VERSION_KIND "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building 'kind'" # depends on CSI_PROW_KIND_VERSION below configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building ginkgo" # depends on CSI_PROW_GINKGO_VERSION below @@ -292,12 +291,14 @@ mkdir -p "${CSI_PROW_BIN}" PATH="${CSI_PROW_BIN}:$PATH" # Ensure that PATH has the desired version of the Go tools, then run command given as argument. +# Empty parameter uses the already installed Go. In Prow, that version is kept up-to-date by +# bumping the container image regularly. run_with_go () { local version version="$1" shift - if go version 2>/dev/null | grep -q "go$version"; then + if ! [ "$version" ] || go version 2>/dev/null | grep -q "go$version"; then run "$@" else if ! [ -d "${CSI_PROW_WORK}/go-$version" ]; then @@ -374,6 +375,20 @@ list_gates () ( done ) +go_version_for_kubernetes () ( + local path="$1" + local version="$2" + local go_version + + # We use the minimal Go version specified for each K8S release (= minimum_go_version in hack/lib/golang.sh). + # More recent versions might also work, but we don't want to count on that. + go_version="$(grep minimum_go_version= "$path/hack/lib/golang.sh" | sed -e 's/.*=go//')" + if ! [ "$go_version" ]; then + die "Unable to determine Go version for Kubernetes $version from hack/lib/golang.sh." + fi + echo "$go_version" +) + csi_prow_kind_have_kubernetes=false # Brings up a Kubernetes cluster and sets KUBECONFIG. # Accepts additional feature gates in the form gate1=true|false,gate2=... @@ -399,7 +414,8 @@ start_cluster () { # while v1.14.0-fake.1 did. (cd "$GOPATH/src/k8s.io/kubernetes" && run git tag -f v1.14.0-fake.1) || die "git tag failed" - run_with_go "${CSI_PROW_GO_VERSION_K8S}" kind build node-image --type bazel --image csiprow/node:latest --kube-root "$GOPATH/src/k8s.io/kubernetes" || die "'kind build node-image' failed" + go_version="$(go_version_for_kubernetes "$GOPATH/src/k8s.io/kubernetes" "$version")" || die "cannot proceed without knowing Go version for Kubernetes" + run_with_go "$go_version" kind build node-image --type bazel --image csiprow/node:latest --kube-root "$GOPATH/src/k8s.io/kubernetes" || die "'kind build node-image' failed" csi_prow_kind_have_kubernetes=true fi image="csiprow/node:latest" @@ -563,7 +579,8 @@ install_e2e () { git_checkout "${CSI_PROW_E2E_REPO}" "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}" --depth=1 && if [ "${CSI_PROW_E2E_IMPORT_PATH}" = "k8s.io/kubernetes" ]; then - run_with_go "${CSI_PROW_GO_VERSION_E2E}" make WHAT=test/e2e/e2e.test "-C${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" && + go_version="${CSI_PROW_GO_VERSION_E2E:-$(go_version_for_kubernetes "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}")}" && + run_with_go "$go_version" make WHAT=test/e2e/e2e.test "-C${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" && ln -s "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}/_output/bin/e2e.test" "${CSI_PROW_WORK}" else run_with_go "${CSI_PROW_GO_VERSION_E2E}" go test -c -o "${CSI_PROW_WORK}/e2e.test" "${CSI_PROW_E2E_IMPORT_PATH}/test/e2e" From 741319bd308c10b6c7b16c3cce71ef941cb7086b Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 3 Apr 2019 12:42:33 +0200 Subject: [PATCH 12/71] prow.sh: improve building Kubernetes from source While switching back and forth between release-1.13 and release-1.14 locally, there was the problem that the local kind image kept using the wrong kubelet binary despite rebuilding from source. The problem went away after cleaning the Bazel cache. Exact root cause unknown, but perhaps using unique tags and properly cleaning the repo helps. If not, then the unique tags at least will show what the problem is. --- prow.sh | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/prow.sh b/prow.sh index 6fd55cc2..a7ea66f2 100755 --- a/prow.sh +++ b/prow.sh @@ -360,6 +360,9 @@ git_checkout () { (cd "$path" && run git fetch "$repo" '+refs/heads/*:refs/remotes/csiprow/heads/*' '+refs/tags/*:refs/tags/*') || die "fetching $repo failed" (cd "$path" && run git checkout "$revision") || die "checking out $repo $revision failed" fi + # This is useful for local testing or when switching between different revisions in the same + # repo. + (cd "$path" && run git clean -fdx) || die "failed to clean $path" } list_gates () ( @@ -410,10 +413,20 @@ start_cluster () { git_checkout https://github.com/kubernetes/kubernetes "$GOPATH/src/k8s.io/kubernetes" "$version" --depth=1 || die "checking out Kubernetes $version failed" # "kind build" and/or the Kubernetes build rules need at least one tag, which we don't have - # when doing a shallow fetch. Therefore we fake one. For some reason, v1.0.0 didn't work - # while v1.14.0-fake.1 did. - (cd "$GOPATH/src/k8s.io/kubernetes" && run git tag -f v1.14.0-fake.1) || die "git tag failed" - + # when doing a shallow fetch. Therefore we fake one: + # release-1.12 -> v1.12.0-release..csiprow + # latest or -> v1.14.0-.csiprow + case "${CSI_PROW_KUBERNETES_VERSION}" in + release-*) + # Ignore: See if you can use ${variable//search/replace} instead. + # shellcheck disable=SC2001 + tag="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/release-\(.*\)/v\1.0-release./')";; + *) + # We have to make something up. v1.0.0 did not work for some reasons. + tag="v1.14.0-";; + esac + tag="$tag$(cd "$GOPATH/src/k8s.io/kubernetes" && git rev-list --abbrev-commit HEAD).csiprow" + (cd "$GOPATH/src/k8s.io/kubernetes" && run git tag -f "$tag") || die "git tag failed" go_version="$(go_version_for_kubernetes "$GOPATH/src/k8s.io/kubernetes" "$version")" || die "cannot proceed without knowing Go version for Kubernetes" run_with_go "$go_version" kind build node-image --type bazel --image csiprow/node:latest --kube-root "$GOPATH/src/k8s.io/kubernetes" || die "'kind build node-image' failed" csi_prow_kind_have_kubernetes=true From 6602d38bfbce6a526f9225c39aa6b86adbd86978 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 3 Apr 2019 13:54:01 +0200 Subject: [PATCH 13/71] prow.sh: different E2E suite depending on Kubernetes version Instead of always using the latest E2E tests for all Kubernetes versions, the plan now is to use the tests that match the Kubernetes version. However, for 1.13 we have to make an exception because the suite for that version did not support the necessary --storage.testdriver parameter. --- prow.sh | 72 ++++++++++++++++++++++++++++----------------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/prow.sh b/prow.sh index a7ea66f2..c54a81b4 100755 --- a/prow.sh +++ b/prow.sh @@ -52,6 +52,19 @@ configvar () { eval echo "\$3:" "$1=\${$1}" } +# Prints the value of a variable + version suffix, falling back to variable + "LATEST". +get_versioned_variable () { + local var="$1" + local version="$2" + local value + + eval value="\${${var}_${version}}" + if ! [ "$value" ]; then + eval value="\${${var}_LATEST}" + fi + echo "$value" +} + # Go versions can be specified seperately for different tasks # If the pre-installed Go is missing or a different # version, the required version here will get installed @@ -158,9 +171,15 @@ configvar CSI_PROW_HOSTPATH_CANARY "" "hostpath image" # all generated files are present. # # CSI_PROW_E2E_REPO=none disables E2E testing. -configvar CSI_PROW_E2E_VERSION v1.14.0 "E2E version" -configvar CSI_PROW_E2E_REPO https://github.com/kubernetes/kubernetes "E2E repo" -configvar CSI_PROW_E2E_IMPORT_PATH k8s.io/kubernetes "E2E package" +configvar CSI_PROW_E2E_VERSION_1_13 v1.14.0 "E2E version for Kubernetes 1.13.x" # we can't use the one from 1.13.x because it didn't have --storage.testdriver +configvar CSI_PROW_E2E_VERSION_1_14 v1.14.0 "E2E version for Kubernetes 1.14.x" +# TODO: add new CSI_PROW_E2E_VERSION entry for future Kubernetes releases +configvar CSI_PROW_E2E_VERSION_LATEST master "E2E version for Kubernetes master" # testing against Kubernetes master is already tracking a moving target, so we might as well use a moving E2E version +configvar CSI_PROW_E2E_REPO_LATEST https://github.com/kubernetes/kubernetes "E2E repo for Kubernetes >= 1.13.x" # currently the same for all versions +configvar CSI_PROW_E2E_IMPORT_PATH_LATEST k8s.io/kubernetes "E2E package for Kubernetes >= 1.13.x" # currently the same for all versions +configvar CSI_PROW_E2E_VERSION "$(get_versioned_variable CSI_PROW_E2E_VERSION "${csi_prow_kubernetes_version_suffix}")" "E2E version" +configvar CSI_PROW_E2E_REPO "$(get_versioned_variable CSI_PROW_E2E_REPO "${csi_prow_kubernetes_version_suffix}")" "E2E repo" +configvar CSI_PROW_E2E_IMPORT_PATH "$(get_versioned_variable CSI_PROW_E2E_IMPORT_PATH "${csi_prow_kubernetes_version_suffix}")" "E2E package" # csi-sanity testing from the csi-test repo can be run against the installed # CSI driver. For this to work, deploying the driver must expose the Unix domain @@ -200,42 +219,22 @@ regex_join () { echo "$@" | sed -e 's/ */|/g' -e 's/^|*//' -e 's/|*$//' -e 's/^$/this-matches-nothing/g' } -# Prints the value of a variable + version suffix, falling back to variable + "LATEST". -get_versioned_variable () { - local var="$1" - local version="$2" - local value - - eval value="\${${var}_${version}}" - if ! [ "$value" ]; then - eval value="\${${var}_LATEST}" - fi - echo "$value" -} - -# Which tests are alpha depends on the Kubernetes version. The same -# E2E test suite is used for all Kubernetes versions, including older -# Kubernetes. +# Which tests are alpha depends on the Kubernetes version. We could +# use the same E2E test for all Kubernetes version. This would have +# the advantage that new tests can be applied to older versions +# without having to backport tests. # -# Feature tags in the test are set based on what is an alpha -# feature in the Kubernetes version that contains the E2E test suite's -# source code. +# But the feature tag gets removed from E2E tests when the corresponding +# feature becomes beta, so we would have to track which tests were +# alpha in previous Kubernetes releases. This was considered too +# error prone. Therefore we use E2E tests that match the Kubernetes +# version that is getting tested. # -# So when testing against an older Kubernetes release, some tests -# might be enabled by default which don't pass for that older -# Kubernetes version. In that case, the -# CSI_PROW_E2E_ALPHA_ variable cannot just -# be based on the `Feature` tag, it also must contain the names -# of tests that used to be alpha but no longer have that tag. -# -# is just major+minor version separated by -# underscore. "latest" matches master. It is also used for -# unknown versions, so when master gets released there is -# no need to add a new versioned variable. That only needs to be -# done when updating something in the configuration that leads -# to "latest" no longer being suitable. +# However, for 1.13.x testing we have to use the E2E tests from 1.14 +# because 1.13 didn't have --storage.testdriver yet, so for that (and only +# that version) we have to define alpha tests differently. configvar CSI_PROW_E2E_ALPHA_1_13 '\[Feature: \[Testpattern:.Dynamic.PV..block.volmode.\] should.create.and.delete.block.persistent.volumes' "alpha tests for Kubernetes 1.13" # Raw block was an alpha feature in 1.13. -configvar CSI_PROW_E2E_ALPHA_LATEST '\[Feature:' "alpha tests for Kubernetes master" +configvar CSI_PROW_E2E_ALPHA_LATEST '\[Feature:' "alpha tests for Kubernetes >= 1.14" # there's no need to update this, adding a new case for CSI_PROW_E2E for a new Kubernetes is enough configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi_prow_kubernetes_version_suffix}")" "alpha tests" # After the parallel E2E test without alpha features, a test cluster @@ -251,6 +250,7 @@ configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi # the failing test for "latest" or by updating the test and not running # it anymore for older releases. configvar CSI_PROW_E2E_ALPHA_GATES_1_13 'VolumeSnapshotDataSource=true,BlockVolume=true,CSIBlockVolume=true' "alpha feature gates for Kubernetes 1.13" +# TODO: add new CSI_PROW_ALPHA_GATES entry for future Kubernetes releases configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'VolumeSnapshotDataSource=true' "alpha feature gates for latest Kubernetes" configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" From d87eccb46dca47496f7f03bce1909e9ba548f00e Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 3 Apr 2019 14:59:33 +0200 Subject: [PATCH 14/71] prow.sh: switch back to upstream csi-driver-host-path The temporary fork was merged, we can use the upstream repo again. --- prow.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/prow.sh b/prow.sh index c54a81b4..490fe99f 100755 --- a/prow.sh +++ b/prow.sh @@ -151,12 +151,8 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip # # When no deploy script is found (nothing in `deploy` directory, # CSI_PROW_HOSTPATH_REPO=none), nothing gets deployed. -# -# TODO: merge https://github.com/kubernetes-csi/csi-driver-host-path/pull/29, switch to revision from master -configvar CSI_PROW_HOSTPATH_VERSION deployment "hostpath driver" -configvar CSI_PROW_HOSTPATH_REPO https://github.com/pohly/csi-driver-host-path "hostpath repo" -# Ignore 'See if you can use ${variable//search/replace} instead.' -# shellcheck disable=SC2001 +configvar CSI_PROW_HOSTPATH_VERSION 486074dc3beef59955faf7bb5210418d9844e0a7 "hostpath driver" # pre-1.1.0 +configvar CSI_PROW_HOSTPATH_REPO https://github.com/kubernetes-csi/csi-driver-host-path "hostpath repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" # If CSI_PROW_HOSTPATH_CANARY is set (typically to "canary", but also From f5014439fc2b977a357ff3b49fd6aab052302f7c Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 5 Apr 2019 08:44:53 +0200 Subject: [PATCH 15/71] prow.sh: AllAlpha=true for unknown Kubernetes versions This ensures that also new, currently unknown alpha gates are enabled when testing against a future Kubernetes versions. For all currently known Kubernetes versions we just use the minimal set of alpha gates, which ensures that we don't miss any of them in our documentation. --- prow.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 490fe99f..5a636bc3 100755 --- a/prow.sh +++ b/prow.sh @@ -246,8 +246,9 @@ configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi # the failing test for "latest" or by updating the test and not running # it anymore for older releases. configvar CSI_PROW_E2E_ALPHA_GATES_1_13 'VolumeSnapshotDataSource=true,BlockVolume=true,CSIBlockVolume=true' "alpha feature gates for Kubernetes 1.13" +configvar CSI_PROW_E2E_ALPHA_GATES_1_14 'VolumeSnapshotDataSource=true' "alpha feature gates for Kubernetes 1.14" # TODO: add new CSI_PROW_ALPHA_GATES entry for future Kubernetes releases -configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'VolumeSnapshotDataSource=true' "alpha feature gates for latest Kubernetes" +configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'AllAlpha=true' "alpha feature gates for latest Kubernetes" configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" # Some tests are known to be unusable in a KinD cluster. For example, From 31dfaf31dc6fb132255f9696a4562ab3e8a741ae Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 8 Apr 2019 08:44:10 +0200 Subject: [PATCH 16/71] prow.sh: fix running of just "alpha" tests "grep -w" treated "serial-alpha" as two words and therefore CSI_PROW_TESTS sometimes ran too many tests. --- prow.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 5a636bc3..f39b3394 100755 --- a/prow.sh +++ b/prow.sh @@ -203,7 +203,16 @@ configvar CSI_PROW_SANITY_CONTAINER "hostpath" "Kubernetes container with CSI dr # by setting CSI_PROW_TESTS_SANITY. configvar CSI_PROW_TESTS "unit parallel serial parallel-alpha serial-alpha ${CSI_PROW_TESTS_SANITY}" "tests to run" test_enabled () { - echo "${CSI_PROW_TESTS}" | grep -q -w -e "$1" + local test="$1" + # We want word-splitting here, so ignore: Double quote to prevent globbing and word splitting. + # shellcheck disable=SC2086 + set ${CSI_PROW_TESTS} + for t in "$@"; do + if [ "$t" = "$test" ]; then + return + fi + done + return 1 } # Serial vs. parallel is always determined by these regular expressions. From f3d1d2df5c85dede9cf25201c69634551a538b92 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 8 Apr 2019 08:47:15 +0200 Subject: [PATCH 17/71] prow.sh: fix hostpath driver version check The previous logic failed for canary jobs, those also deploy a recent driver. Instead of guessing what driver gets installed based on job parameters, check what really runs in the cluster and base the decision on that. We only need to maintain this blacklist for 1.0.x until we replace it with 1.1.0, then this entire hostpath_supports_block can be removed. --- prow.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/prow.sh b/prow.sh index f39b3394..cd041319 100755 --- a/prow.sh +++ b/prow.sh @@ -619,18 +619,18 @@ install_sanity () ( # Whether the hostpath driver supports raw block devices depends on which version # we are testing. It would be much nicer if we could determine that by querying the -# installed driver. +# installed driver's capabilities instead of having to do a version check. hostpath_supports_block () { - if [ -e "cmd/hostpathplugin" ] && ${CSI_PROW_BUILD_JOB}; then - # The assumption is that if we build the hostpath driver, then it is - # a current version with support. - echo true - return - fi - - case "${CSI_PROW_DEPLOYMENT}" in kubernetes-1.13) echo false;; # wasn't supported and probably won't be backported - *) echo true;; # probably all other deployments have a recent driver - esac + local result + result="$(docker exec csi-prow-control-plane docker image ls --format='{{.Repository}} {{.Tag}} {{.ID}}' | grep hostpath | while read -r repo tag id; do + if [ "$tag" == "v1.0.1" ]; then + # Old version because the revision label is missing: didn't have support yet. + echo "false" + return + fi + done)" + # If not set, then it must be a newer driver with support. + echo "${result:-true}" } # Captures pod output while running some other command. From aa45a1cd9b9fafde60f7d8a8e314287e3b8a7794 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 8 Apr 2019 08:51:11 +0200 Subject: [PATCH 18/71] prow.sh: more efficient execution of individual tests When running only some tests, sometimes extra, unnecessarily work was done, like bringing up the cluster without alpha gates. --- prow.sh | 218 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 118 insertions(+), 100 deletions(-) diff --git a/prow.sh b/prow.sh index cd041319..72be16f5 100755 --- a/prow.sh +++ b/prow.sh @@ -202,18 +202,30 @@ configvar CSI_PROW_SANITY_CONTAINER "hostpath" "Kubernetes container with CSI dr # is off by default. A CSI driver can change that default in its .prow.sh # by setting CSI_PROW_TESTS_SANITY. configvar CSI_PROW_TESTS "unit parallel serial parallel-alpha serial-alpha ${CSI_PROW_TESTS_SANITY}" "tests to run" -test_enabled () { - local test="$1" - # We want word-splitting here, so ignore: Double quote to prevent globbing and word splitting. - # shellcheck disable=SC2086 - set ${CSI_PROW_TESTS} - for t in "$@"; do - if [ "$t" = "$test" ]; then - return - fi +tests_enabled () { + local t1 t2 + # We want word-splitting here, so ignore: Quote to prevent word splitting, or split robustly with mapfile or read -a. + # shellcheck disable=SC2206 + local tests=(${CSI_PROW_TESTS}) + for t1 in "$@"; do + for t2 in "${tests[@]}"; do + if [ "$t1" = "$t2" ]; then + return + fi + done done return 1 } +tests_need_kind () { + tests_enabled "sanity" "parallel" "serial" "serial-alpha" "parallel-alpha" +} +tests_need_non_alpha_cluster () { + tests_enabled "sanity" "parallel" "serial" +} +tests_need_alpha_cluster () { + tests_enabled "parallel-alpha" "serial-alpha" +} + # Serial vs. parallel is always determined by these regular expressions. # Individual regular expressions are seperated by spaces for readability @@ -521,6 +533,15 @@ install_hostpath () { return 1 fi + if ${CSI_PROW_BUILD_JOB}; then + # Ignore: Double quote to prevent globbing and word splitting. + # Ignore: To read lines rather than words, pipe/redirect to a 'while read' loop. + # shellcheck disable=SC2086 disable=SC2013 + for i in $(grep '^\s*CMDS\s*=' Makefile | sed -e 's/\s*CMDS\s*=//'); do + kind load docker-image --name csi-prow $i:csiprow || die "could not load the $i:latest image into the kind cluster" + done + fi + if deploy_hostpath="$(find_deployment "$(pwd)/deploy")"; then : elif [ "${CSI_PROW_HOSTPATH_REPO}" = "none" ]; then @@ -836,7 +857,7 @@ main () { # might have been minor or unavoidable, for example when experimenting with # changes in "release-tools" in a PR (that fails the "is release-tools unmodified" # test). - if test_enabled "unit"; then + if tests_enabled "unit"; then if ! run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make -k test 2>&1 | make_test_to_junit; then warn "'make test' failed, proceeding anyway" ret=1 @@ -846,102 +867,99 @@ main () { run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make container || die "'make container' failed" fi - install_kind || die "installing kind failed" - start_cluster || die "starting the cluster failed" + if tests_need_kind; then + install_kind || die "installing kind failed" - if ${CSI_PROW_BUILD_JOB}; then - cmds="$(grep '^\s*CMDS\s*=' Makefile | sed -e 's/\s*CMDS\s*=//')" - # Get the image that was just built (if any) from the - # top-level Makefile CMDS variable and set the - # deploy-hostpath.sh env variables for it. We also need to - # side-load those images into the cluster. - for i in $cmds; do - e=$(echo "$i" | tr '[:lower:]' '[:upper:]' | tr - _) - images="$images ${e}_REGISTRY=none ${e}_TAG=csiprow" - - # We must avoid the tag "latest" because that implies - # always pulling the image - # (https://github.com/kubernetes-sigs/kind/issues/328). - docker tag "$i:latest" "$i:csiprow" || die "tagging the locally built container image for $i failed" - kind load docker-image --name csi-prow "$i:csiprow" || die "could not load the $i:latest image into the kind cluster" - done - - if [ -e deploy/kubernetes/rbac.yaml ]; then - # This is one of those components which has its own RBAC rules (like external-provisioner). - # We are testing a locally built image and also want to test with the the current, - # potentially modified RBAC rules. - if [ "$(echo "$cmds" | wc -w)" != 1 ]; then - die "ambiguous deploy/kubernetes/rbac.yaml: need exactly one command, got: $cmds" - fi - e=$(echo "$cmds" | tr '[:lower:]' '[:upper:]' | tr - _) - images="$images ${e}_RBAC=$(pwd)/deploy/kubernetes/rbac.yaml" - fi - fi - - # Installing the driver might be disabled, in which case we bail out early. - if ! install_hostpath "$images"; then - info "hostpath driver installation disabled, skipping E2E testing" - return "$ret" - fi - - collect_cluster_info - - if test_enabled "sanity"; then - if ! run_sanity; then - ret=1 - fi - fi - - if test_enabled "parallel"; then - # Ignore: Double quote to prevent globbing and word splitting. - # shellcheck disable=SC2086 - if ! run_e2e parallel ${CSI_PROW_GINKO_PARALLEL} \ - -focus="External.Storage" \ - -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}" "${CSI_PROW_E2E_ALPHA}" "${CSI_PROW_E2E_SKIP}")"; then - warn "E2E parallel failed" - ret=1 - fi - fi - - if test_enabled "serial"; then - if ! run_e2e serial \ - -focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_SERIAL}"))" \ - -skip="$(regex_join "${CSI_PROW_E2E_ALPHA}" "${CSI_PROW_E2E_SKIP}")"; then - warn "E2E serial failed" - ret=1 - fi - fi - - if (test_enabled "parallel-alpha" || test_enabled "serial-alpha") && [ "${CSI_PROW_E2E_ALPHA_GATES}" ]; then - # Need to (re)create the cluster. - start_cluster "${CSI_PROW_E2E_ALPHA_GATES}" || die "starting alpha cluster failed" if ${CSI_PROW_BUILD_JOB}; then - # Ignore: Double quote to prevent globbing and word splitting. - # Ignore: To read lines rather than words, pipe/redirect to a 'while read' loop. - # shellcheck disable=SC2086 disable=SC2013 - for i in $(grep '^\s*CMDS\s*=' Makefile | sed -e 's/\s*CMDS\s*=//'); do - kind load docker-image --name csi-prow $i:csiprow || die "could not load the $i:latest image into the kind cluster" - done - fi - install_hostpath "$images" || die "hostpath driver installation failed unexpectedly on alpha cluster" + cmds="$(grep '^\s*CMDS\s*=' Makefile | sed -e 's/\s*CMDS\s*=//')" + # Get the image that was just built (if any) from the + # top-level Makefile CMDS variable and set the + # deploy-hostpath.sh env variables for it. We also need to + # side-load those images into the cluster. + for i in $cmds; do + e=$(echo "$i" | tr '[:lower:]' '[:upper:]' | tr - _) + images="$images ${e}_REGISTRY=none ${e}_TAG=csiprow" - if test_enabled "parallel-alpha"; then - # Ignore: Double quote to prevent globbing and word splitting. - # shellcheck disable=SC2086 - if ! run_e2e parallel-alpha ${CSI_PROW_GINKO_PARALLEL} \ - -focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_ALPHA}"))" \ - -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}" "${CSI_PROW_E2E_SKIP}")"; then - warn "E2E parallel alpha failed" - ret=1 + # We must avoid the tag "latest" because that implies + # always pulling the image + # (https://github.com/kubernetes-sigs/kind/issues/328). + docker tag "$i:latest" "$i:csiprow" || die "tagging the locally built container image for $i failed" + done + + if [ -e deploy/kubernetes/rbac.yaml ]; then + # This is one of those components which has its own RBAC rules (like external-provisioner). + # We are testing a locally built image and also want to test with the the current, + # potentially modified RBAC rules. + if [ "$(echo "$cmds" | wc -w)" != 1 ]; then + die "ambiguous deploy/kubernetes/rbac.yaml: need exactly one command, got: $cmds" + fi + e=$(echo "$cmds" | tr '[:lower:]' '[:upper:]' | tr - _) + images="$images ${e}_RBAC=$(pwd)/deploy/kubernetes/rbac.yaml" fi fi - if test_enabled "serial-alpha"; then - if ! run_e2e serial-alpha \ - -focus="External.Storage.*(($(regex_join "${CSI_PROW_E2E_SERIAL}")).*($(regex_join "${CSI_PROW_E2E_ALPHA}"))|($(regex_join "${CSI_PROW_E2E_ALPHA}")).*($(regex_join "${CSI_PROW_E2E_SERIAL}")))" \ - -skip="$(regex_join "${CSI_PROW_E2E_SKIP}")"; then - warn "E2E serial alpha failed" - ret=1 + if tests_need_non_alpha_cluster; then + start_cluster || die "starting the non-alpha cluster failed" + + # Installing the driver might be disabled. + if install_hostpath "$images"; then + collect_cluster_info + + if tests_enabled "sanity"; then + if ! run_sanity; then + ret=1 + fi + fi + + if tests_enabled "parallel"; then + # Ignore: Double quote to prevent globbing and word splitting. + # shellcheck disable=SC2086 + if ! run_e2e parallel ${CSI_PROW_GINKO_PARALLEL} \ + -focus="External.Storage" \ + -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}" "${CSI_PROW_E2E_ALPHA}" "${CSI_PROW_E2E_SKIP}")"; then + warn "E2E parallel failed" + ret=1 + fi + fi + + if tests_enabled "serial"; then + if ! run_e2e serial \ + -focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_SERIAL}"))" \ + -skip="$(regex_join "${CSI_PROW_E2E_ALPHA}" "${CSI_PROW_E2E_SKIP}")"; then + warn "E2E serial failed" + ret=1 + fi + fi + fi + fi + + if tests_need_alpha_cluster && [ "${CSI_PROW_E2E_ALPHA_GATES}" ]; then + # Need to (re)create the cluster. + start_cluster "${CSI_PROW_E2E_ALPHA_GATES}" || die "starting alpha cluster failed" + + # Installing the driver might be disabled. + if install_hostpath "$images"; then + collect_cluster_info + + if tests_enabled "parallel-alpha"; then + # Ignore: Double quote to prevent globbing and word splitting. + # shellcheck disable=SC2086 + if ! run_e2e parallel-alpha ${CSI_PROW_GINKO_PARALLEL} \ + -focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_ALPHA}"))" \ + -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}" "${CSI_PROW_E2E_SKIP}")"; then + warn "E2E parallel alpha failed" + ret=1 + fi + fi + + if tests_enabled "serial-alpha"; then + if ! run_e2e serial-alpha \ + -focus="External.Storage.*(($(regex_join "${CSI_PROW_E2E_SERIAL}")).*($(regex_join "${CSI_PROW_E2E_ALPHA}"))|($(regex_join "${CSI_PROW_E2E_ALPHA}")).*($(regex_join "${CSI_PROW_E2E_SERIAL}")))" \ + -skip="$(regex_join "${CSI_PROW_E2E_SKIP}")"; then + warn "E2E serial alpha failed" + ret=1 + fi + fi fi fi fi From 9b0d9cd74370bff4dfaf86c4ab75973ec9aa56a2 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 8 Apr 2019 11:37:01 +0200 Subject: [PATCH 19/71] build.make: skip shellcheck if Docker is not available Not all environments have Docker. The simplifying assumption here is that if the Docker command is available, it's also usable. --- build.make | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build.make b/build.make index 078100c9..e5769c58 100644 --- a/build.make +++ b/build.make @@ -141,6 +141,10 @@ test: test-shellcheck test-shellcheck: @ echo; echo "### $@:" @ ret=0; \ + if ! command -v docker; then \ + echo "skipped, no Docker"; \ + return 0; \ + fi; \ for dir in $(abspath $(TEST_SHELLCHECK_DIRS)); do \ echo; \ echo "$$dir:"; \ From 546d5504a1902276561ee1bfc3811afb61346855 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 8 Apr 2019 19:42:38 +0200 Subject: [PATCH 20/71] prow.sh: debug failing KinD cluster creation When KinD fails in a Prow job, we need additional information to understand why it failed. --- prow.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 72be16f5..c2075685 100755 --- a/prow.sh +++ b/prow.sh @@ -488,7 +488,17 @@ $(list_gates "$gates") featureGates: $(list_gates "$gates") EOF - run kind create cluster --name csi-prow --config "${CSI_PROW_WORK}/kind-config.yaml" --wait 5m --image "$image" || die "'kind create cluster' failed" + info "kind-config.yaml:" + cat "${CSI_PROW_WORK}/kind-config.yaml" + if ! run kind create cluster --name csi-prow --config "${CSI_PROW_WORK}/kind-config.yaml" --wait 5m --image "$image"; then + warn "Cluster creation failed. Will try again with higher verbosity." + info "Available Docker images:" + docker image ls + if ! run kind --loglevel debug create cluster --retain --name csi-prow --config "${CSI_PROW_WORK}/kind-config.yaml" --wait 5m --image "$image"; then + run kind export logs --name csi-prow "$ARTIFACTS/kind-cluster" + die "Cluster creation failed again, giving up. See the 'kind-cluster' artifact directory for additional logs." + fi + fi KUBECONFIG="$(kind get kubeconfig-path --name=csi-prow)" export KUBECONFIG } From cda2fc5874b42ed68b68cbb60e92fff6ab7ba4c0 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 8 Apr 2019 19:45:44 +0200 Subject: [PATCH 21/71] prow.sh: avoid AllAlpha=true It turned out to not work. Instead of reverting the commit which introduced this, let's better document this explicitly. --- prow.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prow.sh b/prow.sh index c2075685..0a2b37d7 100755 --- a/prow.sh +++ b/prow.sh @@ -267,9 +267,9 @@ configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi # the failing test for "latest" or by updating the test and not running # it anymore for older releases. configvar CSI_PROW_E2E_ALPHA_GATES_1_13 'VolumeSnapshotDataSource=true,BlockVolume=true,CSIBlockVolume=true' "alpha feature gates for Kubernetes 1.13" -configvar CSI_PROW_E2E_ALPHA_GATES_1_14 'VolumeSnapshotDataSource=true' "alpha feature gates for Kubernetes 1.14" +configvar CSI_PROW_E2E_ALPHA_GATES_1_14 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for Kubernetes 1.14" # TODO: add new CSI_PROW_ALPHA_GATES entry for future Kubernetes releases -configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'AllAlpha=true' "alpha feature gates for latest Kubernetes" +configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'AllAlpha=true,ExpandCSIVolumes=true' "alpha feature gates for latest Kubernetes" configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" # Some tests are known to be unusable in a KinD cluster. For example, From 7aaac225719beb59cf984b885bd8241c699723e7 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 10 Apr 2019 08:44:54 +0200 Subject: [PATCH 22/71] prow.sh: remove AllAlpha=all, part II This was already meant to be done earlier in cda2fc5874b42ed68b. While at it, extend the permanent TODO with guidance on future feature gates. --- prow.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/prow.sh b/prow.sh index 0a2b37d7..57415d43 100755 --- a/prow.sh +++ b/prow.sh @@ -268,8 +268,9 @@ configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi # it anymore for older releases. configvar CSI_PROW_E2E_ALPHA_GATES_1_13 'VolumeSnapshotDataSource=true,BlockVolume=true,CSIBlockVolume=true' "alpha feature gates for Kubernetes 1.13" configvar CSI_PROW_E2E_ALPHA_GATES_1_14 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for Kubernetes 1.14" -# TODO: add new CSI_PROW_ALPHA_GATES entry for future Kubernetes releases -configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'AllAlpha=true,ExpandCSIVolumes=true' "alpha feature gates for latest Kubernetes" +# TODO: add new CSI_PROW_ALPHA_GATES_xxx entry for future Kubernetes releases and +# add new gates to CSI_PROW_E2E_ALPHA_GATES_LATEST. +configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for latest Kubernetes" configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" # Some tests are known to be unusable in a KinD cluster. For example, From ff9bce4a71feb31e69d3e5a8ed9874da542066f1 Mon Sep 17 00:00:00 2001 From: Pengzhi Sun Date: Thu, 11 Apr 2019 17:42:44 +0800 Subject: [PATCH 23/71] Replace 'return' to 'exit' to fix shellcheck error --- build.make | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.make b/build.make index e5769c58..e3d4795d 100644 --- a/build.make +++ b/build.make @@ -143,11 +143,11 @@ test-shellcheck: @ ret=0; \ if ! command -v docker; then \ echo "skipped, no Docker"; \ - return 0; \ + exit 0; \ fi; \ for dir in $(abspath $(TEST_SHELLCHECK_DIRS)); do \ echo; \ echo "$$dir:"; \ ./release-tools/verify-shellcheck.sh "$$dir" || ret=1; \ done; \ - return $$ret + exit $$ret From 0b10f6a4b026a52bdda5d8d63b80a45b79b84ff3 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 11 Apr 2019 13:37:43 +0200 Subject: [PATCH 24/71] prow.sh: update csi-driver-host-path This pulls in https://github.com/kubernetes-csi/csi-driver-host-path/pull/37, which turned out to be necessary for pre-submit testing of the livenessprobe. --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 57415d43..ab3a1a57 100755 --- a/prow.sh +++ b/prow.sh @@ -151,7 +151,7 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip # # When no deploy script is found (nothing in `deploy` directory, # CSI_PROW_HOSTPATH_REPO=none), nothing gets deployed. -configvar CSI_PROW_HOSTPATH_VERSION 486074dc3beef59955faf7bb5210418d9844e0a7 "hostpath driver" # pre-1.1.0 +configvar CSI_PROW_HOSTPATH_VERSION fc52d13ba07922c80555a24616a5b16480350c3f "hostpath driver" # pre-1.1.0 configvar CSI_PROW_HOSTPATH_REPO https://github.com/kubernetes-csi/csi-driver-host-path "hostpath repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" From 0fafc663dc8c0d43cbc411b69ad68a74ecacbc8b Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 11 Apr 2019 22:32:01 +0200 Subject: [PATCH 25/71] prow.sh: skip sanity testing if component doesn't support it Whether a component supports sanity testing depends on the component. For example, csi-driver-host-path enables it because it makes sense there (and only there). Letting the prow.sh script decide whether it actually runs simplifies the job definitions in test-infra. --- prow.sh | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/prow.sh b/prow.sh index ab3a1a57..204cc294 100755 --- a/prow.sh +++ b/prow.sh @@ -198,10 +198,13 @@ configvar CSI_PROW_SANITY_CONTAINER "hostpath" "Kubernetes container with CSI dr # - serial, only alpha features # - sanity # -# Sanity testing with csi-sanity only covers the CSI driver itself and thus -# is off by default. A CSI driver can change that default in its .prow.sh -# by setting CSI_PROW_TESTS_SANITY. -configvar CSI_PROW_TESTS "unit parallel serial parallel-alpha serial-alpha ${CSI_PROW_TESTS_SANITY}" "tests to run" +# Unknown or unsupported entries are ignored. +# +# Sanity testing with csi-sanity only covers the CSI driver itself and +# thus only makes sense in repos which provide their own CSI +# driver. Repos can enable sanity testing by setting +# CSI_PROW_TESTS_SANITY=sanity. +configvar CSI_PROW_TESTS "unit parallel serial parallel-alpha serial-alpha sanity" "tests to run" tests_enabled () { local t1 t2 # We want word-splitting here, so ignore: Quote to prevent word splitting, or split robustly with mapfile or read -a. @@ -216,11 +219,16 @@ tests_enabled () { done return 1 } +sanity_enabled () { + [ "${CSI_PROW_TESTS_SANITY}" = "sanity" ] && tests_enabled "sanity" +} tests_need_kind () { - tests_enabled "sanity" "parallel" "serial" "serial-alpha" "parallel-alpha" + tests_enabled "parallel" "serial" "serial-alpha" "parallel-alpha" || + sanity_enabled } tests_need_non_alpha_cluster () { - tests_enabled "sanity" "parallel" "serial" + tests_enabled "parallel" "serial" || + sanity_enabled } tests_need_alpha_cluster () { tests_enabled "parallel-alpha" "serial-alpha" @@ -916,7 +924,7 @@ main () { if install_hostpath "$images"; then collect_cluster_info - if tests_enabled "sanity"; then + if sanity_enabled; then if ! run_sanity; then ret=1 fi From e157b6b514ace191f46d4c879491cb7d11684659 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 15 Apr 2019 09:45:23 +0200 Subject: [PATCH 26/71] update to Go 1.12.4 Kubernetes also requires 1.12. We pick the latest stable release for CI builds. --- travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis.yml b/travis.yml index 1c05dfd9..5b042522 100644 --- a/travis.yml +++ b/travis.yml @@ -4,7 +4,7 @@ services: - docker matrix: include: - - go: 1.11.1 + - go: 1.12.4 before_script: - mkdir -p bin - wget https://github.com/golang/dep/releases/download/v0.5.1/dep-linux-amd64 -O bin/dep From 066143d146253748b9391445b25804e85b5cc208 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 10 May 2019 17:37:49 +0200 Subject: [PATCH 27/71] build.make: allow repos to use 'go mod' for vendoring How a repo does vendoring is detected based on the presence of Gopkg.toml. The vendor check with `dep` was all done locally, but the corresponding check for `go mod` requires network access. The check therefore gets skipped when running in the Prow CI in situations where we are sure that it isn't needed (for example, in a periodic job). --- build.make | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/build.make b/build.make index e3d4795d..142c8578 100644 --- a/build.make +++ b/build.make @@ -118,14 +118,39 @@ test-fmt: fi # This test only runs when dep >= 0.5 is installed, which is the case for the CI setup. +# When using 'go mod', we allow the test to be skipped in the Prow CI under some special +# circumstances, because it depends on accessing all remote repos and thus +# running it all the time would defeat the purpose of vendoring: +# - not handling a PR or +# - the fabricated merge commit leaves go.mod, go.sum and vendor dir unchanged +# - release-tools also didn't change (changing rules or Go version might lead to +# a different result and thus must be tested) .PHONY: test-vendor test: test-vendor test-vendor: @ echo; echo "### $@:" - @ case "$$(dep version 2>/dev/null | grep 'version *:')" in \ - *v0.[56789]*) dep check && echo "vendor up-to-date" || false;; \ - *) echo "skipping check, dep >= 0.5 required";; \ - esac + @ if [ -f Gopkg.toml ]; then \ + echo "Repo uses 'dep' for vendoring."; \ + case "$$(dep version 2>/dev/null | grep 'version *:')" in \ + *v0.[56789]*) dep check && echo "vendor up-to-date" || false;; \ + *) echo "skipping check, dep >= 0.5 required";; \ + esac; \ + else \ + echo "Repo uses 'go mod' for vendoring."; \ + if [ "$${JOB_NAME}" ] && \ + ( [ "$${JOB_TYPE}" != "presubmit" ] || \ + [ $$(git diff "${PULL_BASE_SHA}..HEAD" -- go.mod go.sum vendor release-tools | wc -l) -eq 0 ] ); then \ + echo "Skipping vendor check because the Prow pre-submit job does not change vendoring."; \ + elif ! GO111MODULE=on go mod vendor; then \ + echo "ERROR: vendor check failed."; \ + false; \ + elif [ $$(git status --porcelain -- vendor | wc -l) -gt 0 ]; then \ + echo "ERROR: vendor directory *not* up-to-date, it did get modified by 'GO111MODULE=on go mod vendor':"; \ + git status -- vendor; \ + git diff -- vendor; \ + false; \ + fi; \ + fi; .PHONY: test-subtree test: test-subtree From b2f4e051d4ffeadffc0980db745a33bacf84f36c Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 8 Jul 2019 20:36:55 +0200 Subject: [PATCH 28/71] prow.sh: flexible test driver config By moving the code into a separate function, other CSI drivers have a chance to overwrite it. For the hostpath driver itself we need the ability to set the driver name depending on which revision is getting installed. --- prow.sh | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/prow.sh b/prow.sh index 204cc294..dd335540 100755 --- a/prow.sh +++ b/prow.sh @@ -154,6 +154,7 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip configvar CSI_PROW_HOSTPATH_VERSION fc52d13ba07922c80555a24616a5b16480350c3f "hostpath driver" # pre-1.1.0 configvar CSI_PROW_HOSTPATH_REPO https://github.com/kubernetes-csi/csi-driver-host-path "hostpath repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" +configvar CSI_PROW_HOSTPATH_DRIVER_NAME "csi-hostpath" "the driver (aka provisioner) name of the chosen hostpath driver" # If CSI_PROW_HOSTPATH_CANARY is set (typically to "canary", but also # "1.0-canary"), then all image versions are replaced with that @@ -673,6 +674,29 @@ hostpath_supports_block () { echo "${result:-true}" } +# The default implementation of this function generates a external +# driver test configuration for the hostpath driver. +# +# The content depends on both what the E2E suite expects and what the +# installed hostpath driver supports. Generating it here seems prone +# to breakage, but it is uncertain where a better place might be. +generate_test_driver () { + cat <"${CSI_PROW_WORK}/hostpath-test-driver.yaml" <"${CSI_PROW_WORK}/test-driver.yaml" || die "generating test-driver.yaml failed" # Rename, merge and filter JUnit files. Necessary in case that we run the E2E suite again # and to avoid the large number of "skipped" tests that we get from using @@ -727,7 +735,7 @@ EOF trap move_junit EXIT cd "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" && - run_with_loggers ginkgo -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/hostpath-test-driver.yaml" + run_with_loggers ginkgo -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml" ) # Run csi-sanity against installed CSI driver. From a6f21d4054a1b70f9efd957a6eebe8f85892b666 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Thu, 11 Jul 2019 14:33:47 -0700 Subject: [PATCH 29/71] Add variables for 1.15 --- prow.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/prow.sh b/prow.sh index dd335540..fc562b05 100755 --- a/prow.sh +++ b/prow.sh @@ -170,6 +170,7 @@ configvar CSI_PROW_HOSTPATH_CANARY "" "hostpath image" # CSI_PROW_E2E_REPO=none disables E2E testing. configvar CSI_PROW_E2E_VERSION_1_13 v1.14.0 "E2E version for Kubernetes 1.13.x" # we can't use the one from 1.13.x because it didn't have --storage.testdriver configvar CSI_PROW_E2E_VERSION_1_14 v1.14.0 "E2E version for Kubernetes 1.14.x" +configvar CSI_PROW_E2E_VERSION_1_15 v1.15.0 "E2E version for Kubernetes 1.15.x" # TODO: add new CSI_PROW_E2E_VERSION entry for future Kubernetes releases configvar CSI_PROW_E2E_VERSION_LATEST master "E2E version for Kubernetes master" # testing against Kubernetes master is already tracking a moving target, so we might as well use a moving E2E version configvar CSI_PROW_E2E_REPO_LATEST https://github.com/kubernetes/kubernetes "E2E repo for Kubernetes >= 1.13.x" # currently the same for all versions @@ -277,6 +278,7 @@ configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi # it anymore for older releases. configvar CSI_PROW_E2E_ALPHA_GATES_1_13 'VolumeSnapshotDataSource=true,BlockVolume=true,CSIBlockVolume=true' "alpha feature gates for Kubernetes 1.13" configvar CSI_PROW_E2E_ALPHA_GATES_1_14 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for Kubernetes 1.14" +configvar CSI_PROW_E2E_ALPHA_GATES_1_15 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for Kubernetes 1.15" # TODO: add new CSI_PROW_ALPHA_GATES_xxx entry for future Kubernetes releases and # add new gates to CSI_PROW_E2E_ALPHA_GATES_LATEST. configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for latest Kubernetes" From ecc79187c50d2c9aa2f27851b2dc5a59190032e2 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Thu, 11 Jul 2019 14:42:59 -0700 Subject: [PATCH 30/71] Update kind to v0.4.0. This requires overriding Kubernetes versions with specific patch versions that kind 0.4.0 supports. Also, feature gate setting is only supported on 1.15+ due to kind.sigs.k8s.io/v1alpha3 and kubeadm.k8s.io/v1beta2 dependencies. --- prow.sh | 105 +++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 74 insertions(+), 31 deletions(-) diff --git a/prow.sh b/prow.sh index fc562b05..be49ff57 100755 --- a/prow.sh +++ b/prow.sh @@ -52,6 +52,26 @@ configvar () { eval echo "\$3:" "$1=\${$1}" } +# Takes the minor version of $CSI_PROW_KUBERNETES_VERSION and overrides it to +# $1 if they are equal minor versions. Ignores versions that begin with +# "release-". +override_k8s_version () { + local current_minor_version + local override_minor_version + + # Ignore: See if you can use ${variable//search/replace} instead. + # shellcheck disable=SC2001 + current_minor_version="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1\.\2/')" + + # Ignore: See if you can use ${variable//search/replace} instead. + # shellcheck disable=SC2001 + override_minor_version="$(echo "${1}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1\.\2/')" + if [ "${current_minor_version}" == "${override_minor_version}" ]; then + CSI_PROW_KUBERNETES_VERSION="$1" + echo "Overriding CSI_PROW_KUBERNETES_VERSION with $1: $CSI_PROW_KUBERNETES_VERSION" + fi +} + # Prints the value of a variable + version suffix, falling back to variable + "LATEST". get_versioned_variable () { local var="$1" @@ -81,7 +101,7 @@ configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version # kind version to use. If the pre-installed version is different, # the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases/download/ # (if available), otherwise it is built from source. -configvar CSI_PROW_KIND_VERSION 0.2.1 "kind" +configvar CSI_PROW_KIND_VERSION v0.4.0 "kind" # ginkgo test runner version to use. If the pre-installed version is # different, the desired version is built from source. @@ -108,6 +128,18 @@ configvar CSI_PROW_BUILD_JOB true "building code in repo enabled" # deprecating or changing the implementation of an alpha feature. configvar CSI_PROW_KUBERNETES_VERSION 1.13.3 "Kubernetes" +# This is a hack to workaround the issue that each version +# of kind currently only supports specific patch versions of +# Kubernetes. We need to override CSI_PROW_KUBERNETES_VERSION +# passed in by our CI/pull jobs to the versions that +# kind v0.4.0 supports. +# +# If the version is prefixed with "release-", then nothing +# is overridden. +override_k8s_version "1.13.7" +override_k8s_version "1.14.3" +override_k8s_version "1.15.0" + # CSI_PROW_KUBERNETES_VERSION reduced to first two version numbers and # with underscore (1_13 instead of 1.13.3) and in uppercase (LATEST # instead of latest). @@ -466,40 +498,51 @@ start_cluster () { image="kindest/node:v${CSI_PROW_KUBERNETES_VERSION}" fi cat >"${CSI_PROW_WORK}/kind-config.yaml" <>"${CSI_PROW_WORK}/kind-config.yaml" < Date: Thu, 11 Jul 2019 14:57:27 -0700 Subject: [PATCH 31/71] Update hostpath version for sidecar testing to v1.2.0-rc2 --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index be49ff57..d998ebba 100755 --- a/prow.sh +++ b/prow.sh @@ -183,7 +183,7 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip # # When no deploy script is found (nothing in `deploy` directory, # CSI_PROW_HOSTPATH_REPO=none), nothing gets deployed. -configvar CSI_PROW_HOSTPATH_VERSION fc52d13ba07922c80555a24616a5b16480350c3f "hostpath driver" # pre-1.1.0 +configvar CSI_PROW_HOSTPATH_VERSION "v1.2.0-rc2" "hostpath driver" configvar CSI_PROW_HOSTPATH_REPO https://github.com/kubernetes-csi/csi-driver-host-path "hostpath repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" configvar CSI_PROW_HOSTPATH_DRIVER_NAME "csi-hostpath" "the driver (aka provisioner) name of the chosen hostpath driver" From 4e31f078a2fb7a891a7cbcdf2299b94891d030ce Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Thu, 11 Jul 2019 17:13:06 -0700 Subject: [PATCH 32/71] Change default hostpath driver name to hostpath.csi.k8s.io --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index d998ebba..f1f0a1b2 100755 --- a/prow.sh +++ b/prow.sh @@ -186,7 +186,7 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip configvar CSI_PROW_HOSTPATH_VERSION "v1.2.0-rc2" "hostpath driver" configvar CSI_PROW_HOSTPATH_REPO https://github.com/kubernetes-csi/csi-driver-host-path "hostpath repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" -configvar CSI_PROW_HOSTPATH_DRIVER_NAME "csi-hostpath" "the driver (aka provisioner) name of the chosen hostpath driver" +configvar CSI_PROW_HOSTPATH_DRIVER_NAME "hostpath.csi.k8s.io" "the hostpath driver name" # If CSI_PROW_HOSTPATH_CANARY is set (typically to "canary", but also # "1.0-canary"), then all image versions are replaced with that From 0c0dc300cafca09a27dd7206547a8191baae477e Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Mon, 15 Jul 2019 12:42:20 -0700 Subject: [PATCH 33/71] prow.sh: tag master images with a large version number --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index f1f0a1b2..dcfcec61 100755 --- a/prow.sh +++ b/prow.sh @@ -485,7 +485,7 @@ start_cluster () { tag="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/release-\(.*\)/v\1.0-release./')";; *) # We have to make something up. v1.0.0 did not work for some reasons. - tag="v1.14.0-";; + tag="v999.999.999-";; esac tag="$tag$(cd "$GOPATH/src/k8s.io/kubernetes" && git rev-list --abbrev-commit HEAD).csiprow" (cd "$GOPATH/src/k8s.io/kubernetes" && run git tag -f "$tag") || die "git tag failed" From f46191d9a8924babaa9c74c7620c5f4caa5cae24 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Mon, 5 Aug 2019 09:45:55 -0700 Subject: [PATCH 34/71] Kubernetes master changed the way that releases are tagged, which needed changes to kind. There are 3 changes made to prow.sh: 1. Use a master commit of kind that includes the fix for Kubernetes master. 2. Use git clone instead of git checkout (shallow) to source Kubernetes. This lets kind correctly figure out the Kubernetes release tag. 3. Build kind with make install. The kind fix was not working correctly when built with go build. --- prow.sh | 50 ++++++++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/prow.sh b/prow.sh index dcfcec61..fd32f79e 100755 --- a/prow.sh +++ b/prow.sh @@ -101,7 +101,7 @@ configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version # kind version to use. If the pre-installed version is different, # the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases/download/ # (if available), otherwise it is built from source. -configvar CSI_PROW_KIND_VERSION v0.4.0 "kind" +configvar CSI_PROW_KIND_VERSION 2555d8e09d5a77ee718414cec9f6083dfa028dc5 "kind" # ginkgo test runner version to use. If the pre-installed version is # different, the desired version is built from source. @@ -322,7 +322,7 @@ configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_ # whether they can run with the current cluster provider, but until # they are, we filter them out by name. Like the other test selection # variables, this is again a space separated list of regular expressions. -configvar CSI_PROW_E2E_SKIP 'while.kubelet.is.down.*Disruptive' "tests that need to be skipped" +configvar CSI_PROW_E2E_SKIP 'Disruptive' "tests that need to be skipped" # This is the directory for additional result files. Usually set by Prow, but # if not (for example, when invoking manually) it defaults to the work directory. @@ -379,8 +379,8 @@ install_kind () { if run curl --fail --location -o "${CSI_PROW_WORK}/bin/kind" "https://github.com/kubernetes-sigs/kind/releases/download/${CSI_PROW_KIND_VERSION}/kind-linux-amd64"; then chmod u+x "${CSI_PROW_WORK}/bin/kind" else - git_checkout https://github.com/kubernetes-sigs/kind "$GOPATH/src/sigs.k8s.io/kind" "${CSI_PROW_KIND_VERSION}" --depth=1 && - run_with_go "${CSI_PROW_GO_VERSION_KIND}" go build -o "${CSI_PROW_WORK}/bin/kind" sigs.k8s.io/kind + git_checkout https://github.com/kubernetes-sigs/kind "${GOPATH}/src/sigs.k8s.io/kind" "${CSI_PROW_KIND_VERSION}" --depth=1 && + (cd "${GOPATH}/src/sigs.k8s.io/kind" && make install INSTALL_DIR="${CSI_PROW_WORK}/bin") fi } @@ -427,6 +427,27 @@ git_checkout () { (cd "$path" && run git clean -fdx) || die "failed to clean $path" } +# This clones a repo ("https://github.com/kubernetes/kubernetes") +# in a certain location ("$GOPATH/src/k8s.io/kubernetes") at +# a the head of a specific branch (i.e., release-1.13, master). +# The directory cannot exist. +git_clone_branch () { + local repo path branch parent + repo="$1" + shift + path="$1" + shift + branch="$1" + shift + + parent="$(dirname "$path")" + mkdir -p "$parent" + (cd "$parent" && run git clone --single-branch --branch "$branch" "$repo" "$path") || die "cloning $repo" failed + # This is useful for local testing or when switching between different revisions in the same + # repo. + (cd "$path" && run git clean -fdx) || die "failed to clean $path" +} + list_gates () ( set -f; IFS=',' # Ignore: Double quote to prevent globbing and word splitting. @@ -472,25 +493,10 @@ start_cluster () { if [ "$version" = "latest" ]; then version=master fi - git_checkout https://github.com/kubernetes/kubernetes "$GOPATH/src/k8s.io/kubernetes" "$version" --depth=1 || die "checking out Kubernetes $version failed" + git_clone_branch https://github.com/kubernetes/kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version" || die "checking out Kubernetes $version failed" - # "kind build" and/or the Kubernetes build rules need at least one tag, which we don't have - # when doing a shallow fetch. Therefore we fake one: - # release-1.12 -> v1.12.0-release..csiprow - # latest or -> v1.14.0-.csiprow - case "${CSI_PROW_KUBERNETES_VERSION}" in - release-*) - # Ignore: See if you can use ${variable//search/replace} instead. - # shellcheck disable=SC2001 - tag="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/release-\(.*\)/v\1.0-release./')";; - *) - # We have to make something up. v1.0.0 did not work for some reasons. - tag="v999.999.999-";; - esac - tag="$tag$(cd "$GOPATH/src/k8s.io/kubernetes" && git rev-list --abbrev-commit HEAD).csiprow" - (cd "$GOPATH/src/k8s.io/kubernetes" && run git tag -f "$tag") || die "git tag failed" - go_version="$(go_version_for_kubernetes "$GOPATH/src/k8s.io/kubernetes" "$version")" || die "cannot proceed without knowing Go version for Kubernetes" - run_with_go "$go_version" kind build node-image --type bazel --image csiprow/node:latest --kube-root "$GOPATH/src/k8s.io/kubernetes" || die "'kind build node-image' failed" + go_version="$(go_version_for_kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version")" || die "cannot proceed without knowing Go version for Kubernetes" + run_with_go "$go_version" kind build node-image --type bazel --image csiprow/node:latest --kube-root "${CSI_PROW_WORK}/src/kubernetes" || die "'kind build node-image' failed" csi_prow_kind_have_kubernetes=true fi image="csiprow/node:latest" From dc0a5d8380459b2ddb8ac11307dd91749fd43b99 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Wed, 21 Aug 2019 15:19:39 -0700 Subject: [PATCH 35/71] Update kind to v0.5.0 --- prow.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/prow.sh b/prow.sh index fd32f79e..c334a5dd 100755 --- a/prow.sh +++ b/prow.sh @@ -101,7 +101,7 @@ configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version # kind version to use. If the pre-installed version is different, # the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases/download/ # (if available), otherwise it is built from source. -configvar CSI_PROW_KIND_VERSION 2555d8e09d5a77ee718414cec9f6083dfa028dc5 "kind" +configvar CSI_PROW_KIND_VERSION "v0.5.0" "kind" # ginkgo test runner version to use. If the pre-installed version is # different, the desired version is built from source. @@ -126,19 +126,19 @@ configvar CSI_PROW_BUILD_JOB true "building code in repo enabled" # use the same settings as for "latest" Kubernetes. This works # as long as there are no breaking changes in Kubernetes, like # deprecating or changing the implementation of an alpha feature. -configvar CSI_PROW_KUBERNETES_VERSION 1.13.3 "Kubernetes" +configvar CSI_PROW_KUBERNETES_VERSION 1.15.3 "Kubernetes" # This is a hack to workaround the issue that each version # of kind currently only supports specific patch versions of # Kubernetes. We need to override CSI_PROW_KUBERNETES_VERSION # passed in by our CI/pull jobs to the versions that -# kind v0.4.0 supports. +# kind v0.5.0 supports. # # If the version is prefixed with "release-", then nothing # is overridden. -override_k8s_version "1.13.7" -override_k8s_version "1.14.3" -override_k8s_version "1.15.0" +override_k8s_version "1.13.10" +override_k8s_version "1.14.6" +override_k8s_version "1.15.3" # CSI_PROW_KUBERNETES_VERSION reduced to first two version numbers and # with underscore (1_13 instead of 1.13.3) and in uppercase (LATEST From 9fba09b41405b93a9bad8cd0d39544e21cdc99e3 Mon Sep 17 00:00:00 2001 From: Deep Debroy Date: Wed, 28 Aug 2019 06:24:26 -0700 Subject: [PATCH 36/71] Add rule for building Windows binaries Signed-off-by: Deep Debroy --- build.make | 1 + 1 file changed, 1 insertion(+) diff --git a/build.make b/build.make index 142c8578..f8b737f9 100644 --- a/build.make +++ b/build.make @@ -63,6 +63,7 @@ endif build-%: mkdir -p bin CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$* + CGO_ENABLED=0 GOOS=windows go build -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$* container-%: build-% docker build -t $*:latest -f $(shell if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi) --label revision=$(REV) . From 35ceaedcbddf39c13651ae07ca8fd73083d4f3e4 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 29 Aug 2019 15:00:21 +0200 Subject: [PATCH 37/71] prow.sh: install dep if needed "make test-vendor" depends on dep if the current project uses dep. Without it, the vendor directory checking was skipped under Prow. --- prow.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/prow.sh b/prow.sh index c334a5dd..33d93e49 100755 --- a/prow.sh +++ b/prow.sh @@ -223,6 +223,10 @@ configvar CSI_PROW_SANITY_SERVICE "hostpath-service" "Kubernetes TCP service nam configvar CSI_PROW_SANITY_POD "csi-hostpathplugin-0" "Kubernetes pod with CSI driver" configvar CSI_PROW_SANITY_CONTAINER "hostpath" "Kubernetes container with CSI driver" +# The version of dep to use for 'make test-vendor'. Ignored if the project doesn't +# use dep. Only binary releases of dep are supported (https://github.com/golang/dep/releases). +configvar CSI_PROW_DEP_VERSION v0.5.1 "golang dep version to be used for vendor checking" + # Each job can run one or more of the following tests, identified by # a single word: # - unit testing @@ -396,6 +400,15 @@ install_ginkgo () { mv "$GOPATH/bin/ginkgo" "${CSI_PROW_BIN}" } +# Ensure that we have the desired version of dep. +install_dep () { + if dep version 2>/dev/null | grep -q "version:.*${CSI_PROW_DEP_VERSION}$"; then + return + fi + run curl --fail --location -o "${CSI_PROW_WORK}/bin/dep" "https://github.com/golang/dep/releases/download/v0.5.4/dep-linux-amd64" && + chmod u+x "${CSI_PROW_WORK}/bin/dep" +} + # This checks out a repo ("https://github.com/kubernetes/kubernetes") # in a certain location ("$GOPATH/src/k8s.io/kubernetes") at # a certain revision (a hex commit hash, v1.13.1, master). It's okay @@ -936,6 +949,10 @@ main () { # changes in "release-tools" in a PR (that fails the "is release-tools unmodified" # test). if tests_enabled "unit"; then + if [ -f Gopkg.toml ] && ! install_dep; then + warn "installing 'dep' failed, cannot test vendoring" + ret=1 + fi if ! run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make -k test 2>&1 | make_test_to_junit; then warn "'make test' failed, proceeding anyway" ret=1 From a8ea8bcc5c6b440cb85cb5341f751b63e063fe62 Mon Sep 17 00:00:00 2001 From: Mucahit Kurt Date: Sat, 7 Sep 2019 02:31:15 +0300 Subject: [PATCH 38/71] create 2-node kind cluster since topology support is added to hostpath driver Signed-off-by: Mucahit Kurt --- prow.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/prow.sh b/prow.sh index 33d93e49..5e7277a3 100755 --- a/prow.sh +++ b/prow.sh @@ -521,6 +521,7 @@ kind: Cluster apiVersion: kind.sigs.k8s.io/v1alpha3 nodes: - role: control-plane +- role: worker EOF # kubeadm has API dependencies between apiVersion and Kubernetes version From ea2f1b5277eef96e0855d4d817ceae832079f59b Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Fri, 6 Sep 2019 19:15:35 -0700 Subject: [PATCH 39/71] build windows binaries with .exe suffix --- build.make | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.make b/build.make index f8b737f9..36e19be4 100644 --- a/build.make +++ b/build.make @@ -63,7 +63,7 @@ endif build-%: mkdir -p bin CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$* - CGO_ENABLED=0 GOOS=windows go build -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$* + CGO_ENABLED=0 GOOS=windows go build -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*.exe ./cmd/$* container-%: build-% docker build -t $*:latest -f $(shell if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi) --label revision=$(REV) . From 6208f6ab47bc2d2973db3c276fd270fe13ac60b5 Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Mon, 16 Sep 2019 17:59:04 -0400 Subject: [PATCH 40/71] Enable hostpath expansion --- prow.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 5e7277a3..d93b6fa7 100755 --- a/prow.sh +++ b/prow.sh @@ -183,7 +183,7 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip # # When no deploy script is found (nothing in `deploy` directory, # CSI_PROW_HOSTPATH_REPO=none), nothing gets deployed. -configvar CSI_PROW_HOSTPATH_VERSION "v1.2.0-rc2" "hostpath driver" +configvar CSI_PROW_HOSTPATH_VERSION "v1.2.0-rc8" "hostpath driver" configvar CSI_PROW_HOSTPATH_REPO https://github.com/kubernetes-csi/csi-driver-host-path "hostpath repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" configvar CSI_PROW_HOSTPATH_DRIVER_NAME "hostpath.csi.k8s.io" "the hostpath driver name" @@ -759,6 +759,8 @@ DriverInfo: persistence: true dataSource: true multipods: true + nodeExpansion: true + controllerExpansion: true EOF } From 194289aa8eece02c19f537b26a361932cc0ff73d Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 4 Oct 2019 14:08:31 +0200 Subject: [PATCH 41/71] update Go mod support It turned out that changes like https://github.com/kubernetes-csi/csi-lib-utils/pull/33 should better have been committed after `go mod tidy` because that adds some indirect dependencies in that example. The revised `test-vendor` checks for that and (just in case that this ever becomes desired) allows projects to not have a vendor directory when using `go mod`. How to use `go mod` properly gets documented in the README.md, because there are such pitfalls. --- README.md | 34 +++++++++++++++++++++++++++++++++ build.make | 55 ++++++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 79 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index bc061aee..d5fdb3a0 100644 --- a/README.md +++ b/README.md @@ -106,3 +106,37 @@ Kubernetes releases: CSI_PROW_KUBERNETES_VERSION=1.13.3 ./.prow.sh CSI_PROW_KUBERNETES_VERSION=latest ./.prow.sh + +Dependencies and vendoring +-------------------------- + +Most projects will (eventually) use `go mod` to manage +dependencies. `dep` is also still supported by `csi-release-tools`, +but not documented here because it's not recommended anymore. + +The usual instructions for using [go +modules](https://github.com/golang/go/wiki/Modules) apply. Here's a cheat sheet +for some of the relevant commands: +- list available updates: `GO111MODULE=on go list -u -m all` +- update or add a single dependency: `GO111MODULE=on go get ` +- update all dependencies to their next minor or patch release: + `GO111MODULE=on go get ./...` (add `-u=patch` to limit to patch + releases) +- lock onto a specific version: `GO111MODULE=on go get @` +- clean up `go.mod`: `GO111MODULE=on go mod tidy` +- update vendor directory: `GO111MODULE=on go mod vendor` + +`GO111MODULE=on` can be left out when using Go >= 1.13 or when the +source is checked out outside of `$GOPATH`. + +`go mod tidy` must be used to ensure that the listed dependencies are +really still needed. Changing import statements or a tentative `go +get` can result in stale dependencies. + +The `test-vendor` verifies that it was used when run locally or in a +pre-merge CI job. If a `vendor` directory is present, it will also +verify that it's content is up-to-date. + +The `vendor` directory is optional. It is still present in projects +because it avoids downloading sources during CI builds. If this is no +longer deemed necessary, then a project can also remove the directory. diff --git a/build.make b/build.make index 142c8578..c8384c16 100644 --- a/build.make +++ b/build.make @@ -125,6 +125,26 @@ test-fmt: # - the fabricated merge commit leaves go.mod, go.sum and vendor dir unchanged # - release-tools also didn't change (changing rules or Go version might lead to # a different result and thus must be tested) +# - import statements not changed (because if they change, go.mod might have to be updated) +# +# "git diff" is intelligent enough to annotate changes inside the "import" block in +# the start of the diff hunk: +# +# diff --git a/rpc/common.go b/rpc/common.go +# index bb4a5c4..5fa4271 100644 +# --- a/rpc/common.go +# +++ b/rpc/common.go +# @@ -21,7 +21,6 @@ import ( +# "fmt" +# "time" +# +# - "google.golang.org/grpc" +# "google.golang.org/grpc/codes" +# "google.golang.org/grpc/status" +# +# We rely on that to find such changes. +# +# Vendoring is optional when using go.mod. .PHONY: test-vendor test: test-vendor test-vendor: @@ -135,22 +155,37 @@ test-vendor: *v0.[56789]*) dep check && echo "vendor up-to-date" || false;; \ *) echo "skipping check, dep >= 0.5 required";; \ esac; \ - else \ - echo "Repo uses 'go mod' for vendoring."; \ + elif [ -f go.mod ]; then \ + echo "Repo uses 'go mod'."; \ if [ "$${JOB_NAME}" ] && \ ( [ "$${JOB_TYPE}" != "presubmit" ] || \ - [ $$(git diff "${PULL_BASE_SHA}..HEAD" -- go.mod go.sum vendor release-tools | wc -l) -eq 0 ] ); then \ - echo "Skipping vendor check because the Prow pre-submit job does not change vendoring."; \ - elif ! GO111MODULE=on go mod vendor; then \ + [ $$( (git diff "${PULL_BASE_SHA}..HEAD" -- go.mod go.sum vendor release-tools; \ + git diff "${PULL_BASE_SHA}..HEAD" | grep -e '^@@.*@@ import (' -e '^[+-]import') | \ + wc -l) -eq 0 ] ); then \ + echo "Skipping vendor check because the Prow pre-submit job does not affect dependencies."; \ + elif ! GO111MODULE=on go mod tidy; then \ echo "ERROR: vendor check failed."; \ false; \ - elif [ $$(git status --porcelain -- vendor | wc -l) -gt 0 ]; then \ - echo "ERROR: vendor directory *not* up-to-date, it did get modified by 'GO111MODULE=on go mod vendor':"; \ - git status -- vendor; \ - git diff -- vendor; \ + elif [ $$(git status --porcelain -- go.mod go.sum | wc -l) -gt 0 ]; then \ + echo "ERROR: go module files *not* up-to-date, they did get modified by 'GO111MODULE=on go mod tidy':"; \ + git diff -- go.mod go.sum; \ false; \ + elif [ -d vendor ]; then \ + if ! GO111MODULE=on go mod vendor; then \ + echo "ERROR: vendor check failed."; \ + false; \ + elif [ $$(git status --porcelain -- vendor | wc -l) -gt 0 ]; then \ + echo "ERROR: vendor directory *not* up-to-date, it did get modified by 'GO111MODULE=on go mod vendor':"; \ + git status -- vendor; \ + git diff -- vendor; \ + false; \ + else \ + echo "Go dependencies and vendor directory up-to-date."; \ + fi; \ + else \ + echo "Go dependencies up-to-date."; \ fi; \ - fi; + fi .PHONY: test-subtree test: test-subtree From c1078a6585735bfcad4d273dee57860544d6fd2a Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 7 Oct 2019 12:21:09 +0200 Subject: [PATCH 42/71] go-get-kubernetes.sh: automate Kubernetes dependency handling This script handles the necessary "replace" statements and determines which packages need to be updated in lockstep. --- README.md | 20 +++++++++ go-get-kubernetes.sh | 104 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100755 go-get-kubernetes.sh diff --git a/README.md b/README.md index d5fdb3a0..cc40f161 100644 --- a/README.md +++ b/README.md @@ -140,3 +140,23 @@ verify that it's content is up-to-date. The `vendor` directory is optional. It is still present in projects because it avoids downloading sources during CI builds. If this is no longer deemed necessary, then a project can also remove the directory. + +When using packages that are part of the Kubernetes source code, the +commands above are not enough because the [lack of semantic +versioning](https://github.com/kubernetes/kubernetes/issues/72638) +prevents `go mod` from finding newer releases. Importing directly from +`kubernetes/kubernetes` also needs `replace` statements to override +the fake `v0.0.0` versions +(https://github.com/kubernetes/kubernetes/issues/79384). The +`go-get-kubernetes.sh` script can be used to update all packages in +lockstep to a different Kubernetes version. It takes a single version +number like "1.16.0". + +Conversion of a repository that uses `dep` to `go mod` can be done with: + + GO111MODULE=on go mod init + release-tools/go-get-kubernetes.sh + GO111MODULE=on go mod tidy + GO111MODULE=on go mod vendor + git rm -f Gopkg.toml Gopkg.lock + git add go.mod go.sum vendor diff --git a/go-get-kubernetes.sh b/go-get-kubernetes.sh new file mode 100755 index 00000000..8c4e3024 --- /dev/null +++ b/go-get-kubernetes.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script can be used while converting a repo from "dep" to "go mod" +# by calling it after "go mod init" or to update the Kubernetes packages +# in a repo that has already been converted. Only packages that are +# part of kubernetes/kubernetes and thus part of a Kubernetes release +# are modified. Other k8.io packages (like k8s.io/klog, k8s.io/utils) +# need to be updated separately. + +set -o pipefail + +cmd=$0 + +function help () { + echo "$cmd - update all components from kubernetes/kubernetes to that version" +} + +if [ $# -ne 1 ]; then + help + exit 1 +fi +case "$1" in -h|--help|help) help; exit 0;; esac + +die () { + echo >&2 "$@" + exit 1 +} + +k8s="$1" + +# If the repo imports k8s.io/kubernetes (directly or indirectly), then +# "go mod" will try to find "v0.0.0" versions because +# k8s.io/kubernetes has those in it's go.mod file +# (https://github.com/kubernetes/kubernetes/blob/2bd9643cee5b3b3a5ecbd3af49d09018f0773c77/go.mod#L146-L157). +# (https://github.com/kubernetes/kubernetes/issues/79384). +# +# We need to replicate the replace statements to override those fake +# versions also in our go.mod file (idea and some code from +# https://github.com/kubernetes/kubernetes/issues/79384#issuecomment-521493597). +mods=$( (set -x; curl --silent --show-error --fail "https://raw.githubusercontent.com/kubernetes/kubernetes/v${k8s}/go.mod") | + sed -n 's|.*k8s.io/\(.*\) => ./staging/src/k8s.io/.*|k8s.io/\1|p' + ) || die "failed to determine Kubernetes staging modules" +for mod in $mods; do + # The presence of a potentially incomplete go.mod file affects this command, + # so move elsewhere. + modinfo=$(set -x; cd /; env GO111MODULE=on go mod download -json "$mod@kubernetes-${k8s}") || + die "failed to determine version of $mod: $modinfo" + v=$(echo "$modinfo" | sed -n 's|.*"Version": "\(.*\)".*|\1|p') + (set -x; env GO111MODULE=on go mod edit "-replace=$mod=$mod@$v") || die "'go mod edit' failed" +done + +packages= + +# Beware that we have to work with packages, not modules (i.e. no -m +# flag), because some modules trigger a "no Go code except tests" +# error. Getting their packages works. +if ! packages=$( (set -x; env GO111MODULE=on go list all) | grep ^k8s.io/ | sed -e 's; *;;'); then + cat >&2 <&2 <" go.mod; then + deps="$deps $(echo "$package" | sed -e "s;\$;@kubernetes-$k8s;" -e 's;^k8s.io/kubernetes\(/.*\)@kubernetes-;k8s.io/kubernetes\1@v;')" + fi +done + +# shellcheck disable=SC2086 +(set -x; env GO111MODULE=on go get $deps 2>&1) || die "go get failed" +echo "SUCCESS" From 2d6b3ce85f79a020c4929f8951840dcd1785434b Mon Sep 17 00:00:00 2001 From: Deep Debroy Date: Mon, 7 Oct 2019 16:38:04 -0700 Subject: [PATCH 43/71] Build Windows only for amd64 Signed-off-by: Deep Debroy --- build.make | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build.make b/build.make index 36e19be4..8d715e49 100644 --- a/build.make +++ b/build.make @@ -57,13 +57,17 @@ else TESTARGS = endif +ARCH := $(if $(GOARCH),$(GOARCH),$(shell go env GOARCH)) + # Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables # to something like "| grep -v 'github.com/kubernetes-csi/project/pkg/foobar'". See usage below. build-%: mkdir -p bin CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$* - CGO_ENABLED=0 GOOS=windows go build -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*.exe ./cmd/$* + if [ "$$ARCH" = "amd64" ]; then \ + CGO_ENABLED=0 GOOS=windows go build -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*.exe ./cmd/$* ; \ + fi container-%: build-% docker build -t $*:latest -f $(shell if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi) --label revision=$(REV) . From f1697d2cac2e184d60b96d45720ed7a37a4ef9ed Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Tue, 8 Oct 2019 14:44:56 -0700 Subject: [PATCH 44/71] Do full git clones in travis. Shallow clones are causing test-subtree errors when the depth is exactly 50. --- travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/travis.yml b/travis.yml index 5b042522..a713bed3 100644 --- a/travis.yml +++ b/travis.yml @@ -2,6 +2,8 @@ language: go sudo: required services: - docker +git: + depth: false matrix: include: - go: 1.12.4 From e0fde8c4f5684f5c259c8c7a1d116def587cabf0 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Wed, 30 Oct 2019 18:08:41 -0700 Subject: [PATCH 45/71] Add new variables for 1.16 and remove 1.13 --- prow.sh | 38 ++++++++++---------------------------- 1 file changed, 10 insertions(+), 28 deletions(-) diff --git a/prow.sh b/prow.sh index d93b6fa7..e3308e77 100755 --- a/prow.sh +++ b/prow.sh @@ -101,7 +101,8 @@ configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version # kind version to use. If the pre-installed version is different, # the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases/download/ # (if available), otherwise it is built from source. -configvar CSI_PROW_KIND_VERSION "v0.5.0" "kind" +# TODO: https://github.com/kubernetes-csi/csi-release-tools/issues/39 +configvar CSI_PROW_KIND_VERSION "86bc23d84ac12dcb56a0528890736e2c347c2dc3" "kind" # ginkgo test runner version to use. If the pre-installed version is # different, the desired version is built from source. @@ -136,7 +137,6 @@ configvar CSI_PROW_KUBERNETES_VERSION 1.15.3 "Kubernetes" # # If the version is prefixed with "release-", then nothing # is overridden. -override_k8s_version "1.13.10" override_k8s_version "1.14.6" override_k8s_version "1.15.3" @@ -183,7 +183,7 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip # # When no deploy script is found (nothing in `deploy` directory, # CSI_PROW_HOSTPATH_REPO=none), nothing gets deployed. -configvar CSI_PROW_HOSTPATH_VERSION "v1.2.0-rc8" "hostpath driver" +configvar CSI_PROW_HOSTPATH_VERSION "v1.2.0" "hostpath driver" configvar CSI_PROW_HOSTPATH_REPO https://github.com/kubernetes-csi/csi-driver-host-path "hostpath repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" configvar CSI_PROW_HOSTPATH_DRIVER_NAME "hostpath.csi.k8s.io" "the hostpath driver name" @@ -200,9 +200,10 @@ configvar CSI_PROW_HOSTPATH_CANARY "" "hostpath image" # all generated files are present. # # CSI_PROW_E2E_REPO=none disables E2E testing. -configvar CSI_PROW_E2E_VERSION_1_13 v1.14.0 "E2E version for Kubernetes 1.13.x" # we can't use the one from 1.13.x because it didn't have --storage.testdriver +# TOOO: remove versioned variables and make e2e version match k8s version configvar CSI_PROW_E2E_VERSION_1_14 v1.14.0 "E2E version for Kubernetes 1.14.x" configvar CSI_PROW_E2E_VERSION_1_15 v1.15.0 "E2E version for Kubernetes 1.15.x" +configvar CSI_PROW_E2E_VERSION_1_16 v1.16.0 "E2E version for Kubernetes 1.16.x" # TODO: add new CSI_PROW_E2E_VERSION entry for future Kubernetes releases configvar CSI_PROW_E2E_VERSION_LATEST master "E2E version for Kubernetes master" # testing against Kubernetes master is already tracking a moving target, so we might as well use a moving E2E version configvar CSI_PROW_E2E_REPO_LATEST https://github.com/kubernetes/kubernetes "E2E repo for Kubernetes >= 1.13.x" # currently the same for all versions @@ -292,11 +293,6 @@ regex_join () { # alpha in previous Kubernetes releases. This was considered too # error prone. Therefore we use E2E tests that match the Kubernetes # version that is getting tested. -# -# However, for 1.13.x testing we have to use the E2E tests from 1.14 -# because 1.13 didn't have --storage.testdriver yet, so for that (and only -# that version) we have to define alpha tests differently. -configvar CSI_PROW_E2E_ALPHA_1_13 '\[Feature: \[Testpattern:.Dynamic.PV..block.volmode.\] should.create.and.delete.block.persistent.volumes' "alpha tests for Kubernetes 1.13" # Raw block was an alpha feature in 1.13. configvar CSI_PROW_E2E_ALPHA_LATEST '\[Feature:' "alpha tests for Kubernetes >= 1.14" # there's no need to update this, adding a new case for CSI_PROW_E2E for a new Kubernetes is enough configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi_prow_kubernetes_version_suffix}")" "alpha tests" @@ -312,12 +308,12 @@ configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi # kubernetes-csi components must be updated, either by disabling # the failing test for "latest" or by updating the test and not running # it anymore for older releases. -configvar CSI_PROW_E2E_ALPHA_GATES_1_13 'VolumeSnapshotDataSource=true,BlockVolume=true,CSIBlockVolume=true' "alpha feature gates for Kubernetes 1.13" configvar CSI_PROW_E2E_ALPHA_GATES_1_14 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for Kubernetes 1.14" configvar CSI_PROW_E2E_ALPHA_GATES_1_15 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for Kubernetes 1.15" +configvar CSI_PROW_E2E_ALPHA_GATES_1_16 'VolumeSnapshotDataSource=true' "alpha feature gates for Kubernetes 1.16" # TODO: add new CSI_PROW_ALPHA_GATES_xxx entry for future Kubernetes releases and # add new gates to CSI_PROW_E2E_ALPHA_GATES_LATEST. -configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for latest Kubernetes" +configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'VolumeSnapshotDataSource=true' "alpha feature gates for latest Kubernetes" configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" # Some tests are known to be unusable in a KinD cluster. For example, @@ -723,22 +719,6 @@ install_sanity () ( run_with_go "${CSI_PROW_GO_VERSION_SANITY}" go test -c -o "${CSI_PROW_WORK}/csi-sanity" "${CSI_PROW_SANITY_IMPORT_PATH}/cmd/csi-sanity" || die "building csi-sanity failed" ) -# Whether the hostpath driver supports raw block devices depends on which version -# we are testing. It would be much nicer if we could determine that by querying the -# installed driver's capabilities instead of having to do a version check. -hostpath_supports_block () { - local result - result="$(docker exec csi-prow-control-plane docker image ls --format='{{.Repository}} {{.Tag}} {{.ID}}' | grep hostpath | while read -r repo tag id; do - if [ "$tag" == "v1.0.1" ]; then - # Old version because the revision label is missing: didn't have support yet. - echo "false" - return - fi - done)" - # If not set, then it must be a newer driver with support. - echo "${result:-true}" -} - # The default implementation of this function generates a external # driver test configuration for the hostpath driver. # @@ -755,12 +735,14 @@ SnapshotClass: DriverInfo: Name: ${CSI_PROW_HOSTPATH_DRIVER_NAME} Capabilities: - block: $(hostpath_supports_block) + block: true persistence: true dataSource: true multipods: true nodeExpansion: true controllerExpansion: true + snapshotDataSource: true + singleNodeVolume: true EOF } From 5e773d2db08fa624d3b9352dd577f885efb7a2b2 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 31 Oct 2019 08:43:31 +0100 Subject: [PATCH 46/71] update CI to use Go 1.13.3 This is the latest release. Updating is useful to ensure that we have all of the latest fixes and enhancements. --- travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis.yml b/travis.yml index a713bed3..494c1144 100644 --- a/travis.yml +++ b/travis.yml @@ -6,7 +6,7 @@ git: depth: false matrix: include: - - go: 1.12.4 + - go: 1.13.3 before_script: - mkdir -p bin - wget https://github.com/golang/dep/releases/download/v0.5.1/dep-linux-amd64 -O bin/dep From c8a1c4af933311a7e63765cd2b64ca45a0fb7dba Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 31 Oct 2019 11:49:23 +0100 Subject: [PATCH 47/71] better handling of Go version Some operations are sensitive to the version of Go that is used. In the past, formatting of source differed depending on the version. Right now it is the content of the vendor directory which changes when switch back and forth between 1.12 and 1.13. We don't want to impose a certain workflow on developers, like forcing all invocations of Go to run inside a container. If developers want that, they can set up their development environment accordingly. But we should warn about this aspect to raise awareness. "make" invocations which involve Go now compare against the projects Go version (specified in travis.yml) once at the beginning. This is only a warning because we don't know which future version will be compatible with the project. Vendor directory handling gets updated, too: verification is now a separate script (became too complex for make) and there is a corresponding "update-vendor.sh". In contrast to verification, updating vendor is not integrated into make and thus itself invokes the go version check. --- build.make | 50 ++++++++---------------------------- update-vendor.sh | 23 +++++++++++++++++ verify-go-version.sh | 51 +++++++++++++++++++++++++++++++++++++ verify-vendor.sh | 60 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 145 insertions(+), 39 deletions(-) create mode 100755 update-vendor.sh create mode 100755 verify-go-version.sh create mode 100755 verify-vendor.sh diff --git a/build.make b/build.make index cbf6d455..6b89f5b2 100644 --- a/build.make +++ b/build.make @@ -62,7 +62,7 @@ ARCH := $(if $(GOARCH),$(GOARCH),$(shell go env GOARCH)) # Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables # to something like "| grep -v 'github.com/kubernetes-csi/project/pkg/foobar'". See usage below. -build-%: +build-%: check-go-version-go mkdir -p bin CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$* if [ "$$ARCH" = "amd64" ]; then \ @@ -97,7 +97,7 @@ push: $(CMDS:%=push-%) clean: -rm -rf bin -test: +test: check-go-version-go .PHONY: test-go test: test-go @@ -154,43 +154,7 @@ test-fmt: test: test-vendor test-vendor: @ echo; echo "### $@:" - @ if [ -f Gopkg.toml ]; then \ - echo "Repo uses 'dep' for vendoring."; \ - case "$$(dep version 2>/dev/null | grep 'version *:')" in \ - *v0.[56789]*) dep check && echo "vendor up-to-date" || false;; \ - *) echo "skipping check, dep >= 0.5 required";; \ - esac; \ - elif [ -f go.mod ]; then \ - echo "Repo uses 'go mod'."; \ - if [ "$${JOB_NAME}" ] && \ - ( [ "$${JOB_TYPE}" != "presubmit" ] || \ - [ $$( (git diff "${PULL_BASE_SHA}..HEAD" -- go.mod go.sum vendor release-tools; \ - git diff "${PULL_BASE_SHA}..HEAD" | grep -e '^@@.*@@ import (' -e '^[+-]import') | \ - wc -l) -eq 0 ] ); then \ - echo "Skipping vendor check because the Prow pre-submit job does not affect dependencies."; \ - elif ! GO111MODULE=on go mod tidy; then \ - echo "ERROR: vendor check failed."; \ - false; \ - elif [ $$(git status --porcelain -- go.mod go.sum | wc -l) -gt 0 ]; then \ - echo "ERROR: go module files *not* up-to-date, they did get modified by 'GO111MODULE=on go mod tidy':"; \ - git diff -- go.mod go.sum; \ - false; \ - elif [ -d vendor ]; then \ - if ! GO111MODULE=on go mod vendor; then \ - echo "ERROR: vendor check failed."; \ - false; \ - elif [ $$(git status --porcelain -- vendor | wc -l) -gt 0 ]; then \ - echo "ERROR: vendor directory *not* up-to-date, it did get modified by 'GO111MODULE=on go mod vendor':"; \ - git status -- vendor; \ - git diff -- vendor; \ - false; \ - else \ - echo "Go dependencies and vendor directory up-to-date."; \ - fi; \ - else \ - echo "Go dependencies up-to-date."; \ - fi; \ - fi + @ ./release-tools/verify-vendor.sh .PHONY: test-subtree test: test-subtree @@ -216,3 +180,11 @@ test-shellcheck: ./release-tools/verify-shellcheck.sh "$$dir" || ret=1; \ done; \ exit $$ret + +# Targets in the makefile can depend on check-go-version- +# to trigger a warning if the x.y version of that binary does not match +# what the project uses. Make ensures that this is only checked once per +# invocation. +.PHONY: check-go-version-% +check-go-version-%: + ./release-tools/verify-go-version.sh "$*" diff --git a/update-vendor.sh b/update-vendor.sh new file mode 100755 index 00000000..6f4c27ae --- /dev/null +++ b/update-vendor.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ -f Gopkg.toml ]; then + echo "Repo uses 'dep' for vendoring." + (set -x; dep ensure) +elif [ -f go.mod ]; then + release-tools/verify-go-version.sh "go" + (set -x; env GO111MODULE=on go mod tidy && env GO111MODULE=on go mod vendor) +fi diff --git a/verify-go-version.sh b/verify-go-version.sh new file mode 100755 index 00000000..f242e769 --- /dev/null +++ b/verify-go-version.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +GO="$1" + +if [ ! "$GO" ]; then + echo >&2 "usage: $0 " + exit 1 +fi + +die () { + echo "ERROR: $*" + exit 1 +} + +version=$("$GO" version) || die "determining version of $GO failed" +# shellcheck disable=SC2001 +majorminor=$(echo "$version" | sed -e 's/.*go\([0-9]*\)\.\([0-9]*\).*/\1.\2/') +# shellcheck disable=SC2001 +expected=$(grep "^ *- go:" "release-tools/travis.yml" | sed -e 's/.*go: *\([0-9]*\)\.\([0-9]*\).*/\1.\2/') + +if [ "$majorminor" != "$expected" ]; then + cat >&2 </dev/null | grep 'version *:')" in + *v0.[56789]*) + if dep check; then + echo "vendor up-to-date" + else + exit 1 + fi + ;; + *) echo "skipping check, dep >= 0.5 required";; + esac +elif [ -f go.mod ]; then + echo "Repo uses 'go mod'." + # shellcheck disable=SC2235 + if [ "${JOB_NAME}" ] && + ( [ "${JOB_TYPE}" != "presubmit" ] || + [ "$( (git diff "${PULL_BASE_SHA}..HEAD" -- go.mod go.sum vendor release-tools; + git diff "${PULL_BASE_SHA}..HEAD" | grep -e '^@@.*@@ import (' -e '^[+-]import') | + wc -l)" -eq 0 ] ); then + echo "Skipping vendor check because the Prow pre-submit job does not affect dependencies." + elif ! (set -x; env GO111MODULE=on go mod tidy); then + echo "ERROR: vendor check failed." + exit 1 + elif [ "$(git status --porcelain -- go.mod go.sum | wc -l)" -gt 0 ]; then + echo "ERROR: go module files *not* up-to-date, they did get modified by 'GO111MODULE=on go mod tidy':"; + git diff -- go.mod go.sum + exit 1 + elif [ -d vendor ]; then + if ! (set -x; env GO111MODULE=on go mod vendor); then + echo "ERROR: vendor check failed." + exit 1 + elif [ "$(git status --porcelain -- vendor | wc -l)" -gt 0 ]; then + echo "ERROR: vendor directory *not* up-to-date, it did get modified by 'GO111MODULE=on go mod vendor':" + git status -- vendor + git diff -- vendor + exit 1 + else + echo "Go dependencies and vendor directory up-to-date." + fi + else + echo "Go dependencies up-to-date." + fi +fi From 23df4aef51ee82896bc8326a693c3fdbd0baa07f Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 4 Nov 2019 11:16:38 +0100 Subject: [PATCH 48/71] prow.sh: use vendor directory if available This avoids dependencies on the Go module cache or the upstream code hosting. --- build.make | 11 +++++++---- prow.sh | 12 +++++++++--- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/build.make b/build.make index 6b89f5b2..1b6f35fe 100644 --- a/build.make +++ b/build.make @@ -22,6 +22,9 @@ # including build.make. REGISTRY_NAME=quay.io/k8scsi +# Can be set to -mod=vendor to ensure that the "vendor" directory is used. +GOFLAGS_VENDOR= + # Revision that gets built into each binary via the main.version # string. Uses the `git describe` output based on the most recent # version tag with a short revision suffix or, if nothing has been @@ -64,9 +67,9 @@ ARCH := $(if $(GOARCH),$(GOARCH),$(shell go env GOARCH)) build-%: check-go-version-go mkdir -p bin - CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$* + CGO_ENABLED=0 GOOS=linux go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$* if [ "$$ARCH" = "amd64" ]; then \ - CGO_ENABLED=0 GOOS=windows go build -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*.exe ./cmd/$* ; \ + CGO_ENABLED=0 GOOS=windows go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*.exe ./cmd/$* ; \ fi container-%: build-% @@ -103,13 +106,13 @@ test: check-go-version-go test: test-go test-go: @ echo; echo "### $@:" - go test `go list ./... | grep -v -e 'vendor' -e '/test/e2e$$' $(TEST_GO_FILTER_CMD)` $(TESTARGS) + go test $(GOFLAGS_VENDOR) `go list $(GOFLAGS_VENDOR) ./... | grep -v -e 'vendor' -e '/test/e2e$$' $(TEST_GO_FILTER_CMD)` $(TESTARGS) .PHONY: test-vet test: test-vet test-vet: @ echo; echo "### $@:" - go vet `go list ./... | grep -v vendor $(TEST_VET_FILTER_CMD)` + go test $(GOFLAGS_VENDOR) `go list $(GOFLAGS_VENDOR) ./... | grep -v vendor $(TEST_VET_FILTER_CMD)` .PHONY: test-fmt test: test-fmt diff --git a/prow.sh b/prow.sh index e3308e77..fe68bb7d 100755 --- a/prow.sh +++ b/prow.sh @@ -85,6 +85,12 @@ get_versioned_variable () { echo "$value" } +# If we have a vendor directory, then use it. We must be careful to only +# use this for "make" invocations inside the project's repo itself because +# setting it globally can break other go usages (like "go get " +# which is disabled with GOFLAGS=-mod=vendor). +configvar GOFLAGS_VENDOR "$( [ -d vendor ] && echo '-mod=vendor' )" "Go flags for using the vendor directory" + # Go versions can be specified seperately for different tasks # If the pre-installed Go is missing or a different # version, the required version here will get installed @@ -928,7 +934,7 @@ main () { images= if ${CSI_PROW_BUILD_JOB}; then # A successful build is required for testing. - run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make all || die "'make all' failed" + run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make all "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" || die "'make all' failed" # We don't want test failures to prevent E2E testing below, because the failure # might have been minor or unavoidable, for example when experimenting with # changes in "release-tools" in a PR (that fails the "is release-tools unmodified" @@ -938,13 +944,13 @@ main () { warn "installing 'dep' failed, cannot test vendoring" ret=1 fi - if ! run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make -k test 2>&1 | make_test_to_junit; then + if ! run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make -k test "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" 2>&1 | make_test_to_junit; then warn "'make test' failed, proceeding anyway" ret=1 fi fi # Required for E2E testing. - run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make container || die "'make container' failed" + run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make container "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" || die "'make container' failed" fi if tests_need_kind; then From 806784565969caa302d40f98f6fb6dd3dafb39ba Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 4 Nov 2019 15:40:31 +0100 Subject: [PATCH 49/71] travis.yml: also use vendor directory --- travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/travis.yml b/travis.yml index 494c1144..bfd7647b 100644 --- a/travis.yml +++ b/travis.yml @@ -13,9 +13,9 @@ before_script: - chmod u+x bin/dep - export PATH=$PWD/bin:$PATH script: -- make -k all test +- make -k all test GOFLAGS_VENDOR=$( [ -d vendor ] && echo '-mod=vendor' ) after_success: - if [ "${TRAVIS_PULL_REQUEST}" == "false" ]; then docker login -u "${DOCKER_USERNAME}" -p "${DOCKER_PASSWORD}" quay.io; - make push; + make push GOFLAGS_VENDOR=$( [ -d vendor ] && echo '-mod=vendor' ); fi From f41c1351a38ad936cbdec298fff3f49b8e2e8613 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Tue, 5 Nov 2019 16:40:29 +0100 Subject: [PATCH 50/71] prow.sh: also log output of system containers Depending on the error, those logs are needed to debug failures. --- prow.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/prow.sh b/prow.sh index e3308e77..0777ae37 100755 --- a/prow.sh +++ b/prow.sh @@ -677,16 +677,16 @@ EOF } -# Gets logs of all containers in the default namespace. When passed -f, kubectl will +# Gets logs of all containers in all namespaces. When passed -f, kubectl will # keep running and capture new output. Prints the pid of all background processes. # The caller must kill (when using -f) and/or wait for them. # # May be called multiple times and thus appends. start_loggers () { - kubectl get pods -o go-template --template='{{range .items}}{{.metadata.name}} {{range .spec.containers}}{{.name}} {{end}}{{"\n"}}{{end}}' | while read -r pod containers; do + kubectl get pods --all-namespaces -o go-template --template='{{range .items}}{{.metadata.namespace}} {{.metadata.name}} {{range .spec.containers}}{{.name}} {{end}}{{"\n"}}{{end}}' | while read -r namespace pod containers; do for container in $containers; do - mkdir -p "${ARTIFACTS}/$pod" - kubectl logs "$@" "$pod" "$container" >>"${ARTIFACTS}/$pod/$container.log" & + mkdir -p "${ARTIFACTS}/$namespace/$pod" + kubectl logs -n "$namespace" "$@" "$pod" "$container" >>"${ARTIFACTS}/$namespace/$pod/$container.log" & echo "$!" done done From 83a4ef15dfda6403862437f8acf3606f255e888a Mon Sep 17 00:00:00 2001 From: Peeyush Gupta Date: Tue, 19 Nov 2019 08:58:58 -0500 Subject: [PATCH 51/71] Adding build for ppc64le --- build.make | 1 + 1 file changed, 1 insertion(+) diff --git a/build.make b/build.make index 1b6f35fe..7075a37e 100644 --- a/build.make +++ b/build.make @@ -70,6 +70,7 @@ build-%: check-go-version-go CGO_ENABLED=0 GOOS=linux go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$* if [ "$$ARCH" = "amd64" ]; then \ CGO_ENABLED=0 GOOS=windows go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*.exe ./cmd/$* ; \ + CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*-ppc64le ./cmd/$* fi container-%: build-% From 1eaaaa1cb49713e8e4af87d009c24d310cf66260 Mon Sep 17 00:00:00 2001 From: Mucahit Kurt Date: Sat, 16 Nov 2019 06:23:08 +0300 Subject: [PATCH 52/71] Delete kind cluster after tests run. Inside a real Prow job it is better to clean up at runtime instead of leaving that to the Prow job cleanup code because the later sometimes times out. Signed-off-by: Mucahit Kurt --- prow.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/prow.sh b/prow.sh index 3b9621a2..8b431c56 100755 --- a/prow.sh +++ b/prow.sh @@ -580,6 +580,19 @@ EOF export KUBECONFIG } +# Deletes kind cluster inside a prow job +delete_cluster_inside_prow_job() { + # Inside a real Prow job it is better to clean up at runtime + # instead of leaving that to the Prow job cleanup code + # because the later sometimes times out (https://github.com/kubernetes-csi/csi-release-tools/issues/24#issuecomment-554765872). + if [ "$JOB_NAME" ]; then + if kind get clusters | grep -q csi-prow; then + run kind delete cluster --name=csi-prow || die "kind delete failed" + fi + unset KUBECONFIG + fi +} + # Looks for the deployment as specified by CSI_PROW_DEPLOYMENT and CSI_PROW_KUBERNETES_VERSION # in the given directory. find_deployment () { @@ -1017,6 +1030,7 @@ main () { fi fi fi + delete_cluster_inside_prow_job fi if tests_need_alpha_cluster && [ "${CSI_PROW_E2E_ALPHA_GATES}" ]; then @@ -1047,6 +1061,7 @@ main () { fi fi fi + delete_cluster_inside_prow_job fi fi From 003c14b2d4ae3b1463db5e5b3ff91f39b03f5ba8 Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Mon, 11 Nov 2019 23:49:42 -0800 Subject: [PATCH 53/71] Add snapshotter CRDs after cluster setup Signed-off-by: Grant Griffiths --- prow.sh | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/prow.sh b/prow.sh index 3b9621a2..bc9c9f03 100755 --- a/prow.sh +++ b/prow.sh @@ -322,6 +322,9 @@ configvar CSI_PROW_E2E_ALPHA_GATES_1_16 'VolumeSnapshotDataSource=true' "alpha f configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'VolumeSnapshotDataSource=true' "alpha feature gates for latest Kubernetes" configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" +# Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment +configvar CSI_SNAPSHOTTER_VERSION 'v2.0.0-rc4' "external-snapshotter version tag" + # Some tests are known to be unusable in a KinD cluster. For example, # stopping kubelet with "ssh systemctl stop kubelet" simply # doesn't work. Such tests should be written in a way that they verify @@ -657,6 +660,59 @@ install_hostpath () { fi } +# Installs all nessesary snapshotter CRDs +install_snapshot_crds() { + # Wait until volumesnapshot CRDs are in place. + CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/config/crd" + kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshotclasses.yaml" --validate=false + kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshots.yaml" --validate=false + kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshotcontents.yaml" --validate=false + cnt=0 + until kubectl get volumesnapshotclasses.snapshot.storage.k8s.io \ + && kubectl get volumesnapshots.snapshot.storage.k8s.io \ + && kubectl get volumesnapshotcontents.snapshot.storage.k8s.io; do + if [ $cnt -gt 30 ]; then + echo >&2 "ERROR: snapshot CRDs not ready after over 1 min" + exit 1 + fi + echo "$(date +%H:%M:%S)" "waiting for snapshot CRDs, attempt #$cnt" + cnt=$((cnt + 1)) + sleep 2 + done +} + +# Install snapshot controller and associated RBAC, retrying until the pod is running. +install_snapshot_controller() { + kubectl apply -f "https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml" + cnt=0 + until kubectl get clusterrolebinding snapshot-controller-role; do + if [ $cnt -gt 30 ]; then + echo "Cluster role bindings:" + kubectl describe clusterrolebinding + echo >&2 "ERROR: snapshot controller RBAC not ready after over 5 min" + exit 1 + fi + echo "$(date +%H:%M:%S)" "waiting for snapshot RBAC setup complete, attempt #$cnt" + cnt=$((cnt + 1)) + sleep 10 + done + + + kubectl apply -f "https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml" + cnt=0 + until kubectl get statefulset snapshot-controller | grep snapshot-controller | grep "1/1"; do + if [ $cnt -gt 30 ]; then + echo "Running statefulsets:" + kubectl describe statefulsets + echo >&2 "ERROR: snapshot controller not ready after over 5 min" + exit 1 + fi + echo "$(date +%H:%M:%S)" "waiting for snapshot controller deployment to complete, attempt #$cnt" + cnt=$((cnt + 1)) + sleep 10 + done +} + # collect logs and cluster status (like the version of all components, Kubernetes version, test version) collect_cluster_info () { cat < Date: Sat, 30 Nov 2019 00:29:00 +0530 Subject: [PATCH 54/71] Use kind v0.6.0 kind v0.6.0 appends the kubeconfig with the default config at ~/.kube/config. --- prow.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/prow.sh b/prow.sh index d719e980..b8823062 100755 --- a/prow.sh +++ b/prow.sh @@ -107,8 +107,7 @@ configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version # kind version to use. If the pre-installed version is different, # the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases/download/ # (if available), otherwise it is built from source. -# TODO: https://github.com/kubernetes-csi/csi-release-tools/issues/39 -configvar CSI_PROW_KIND_VERSION "86bc23d84ac12dcb56a0528890736e2c347c2dc3" "kind" +configvar CSI_PROW_KIND_VERSION "v0.6.0" "kind" # ginkgo test runner version to use. If the pre-installed version is # different, the desired version is built from source. @@ -579,8 +578,7 @@ EOF die "Cluster creation failed again, giving up. See the 'kind-cluster' artifact directory for additional logs." fi fi - KUBECONFIG="$(kind get kubeconfig-path --name=csi-prow)" - export KUBECONFIG + export KUBECONFIG="${HOME}/.kube/config" } # Deletes kind cluster inside a prow job From 9a7a685ee169d669a0182532315aeecd8a8715dc Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Mon, 2 Dec 2019 17:45:57 -0800 Subject: [PATCH 55/71] Create a kind cluster with two worker nodes so that the topology feature can be tested. Test cases that test accessing volumes from multiple nodes need to be skipped --- prow.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/prow.sh b/prow.sh index b8823062..815bc9d8 100755 --- a/prow.sh +++ b/prow.sh @@ -330,7 +330,11 @@ configvar CSI_SNAPSHOTTER_VERSION 'v2.0.0-rc4' "external-snapshotter version tag # whether they can run with the current cluster provider, but until # they are, we filter them out by name. Like the other test selection # variables, this is again a space separated list of regular expressions. -configvar CSI_PROW_E2E_SKIP 'Disruptive' "tests that need to be skipped" +# +# "different node" test skips can be removed once +# https://github.com/kubernetes/kubernetes/pull/82678 has been backported +# to all the K8s versions we test against +configvar CSI_PROW_E2E_SKIP 'Disruptive|different\s+node' "tests that need to be skipped" # This is the directory for additional result files. Usually set by Prow, but # if not (for example, when invoking manually) it defaults to the work directory. @@ -526,6 +530,7 @@ apiVersion: kind.sigs.k8s.io/v1alpha3 nodes: - role: control-plane - role: worker +- role: worker EOF # kubeadm has API dependencies between apiVersion and Kubernetes version @@ -840,10 +845,6 @@ run_e2e () ( install_e2e || die "building e2e.test failed" install_ginkgo || die "installing ginkgo failed" - # TODO (?): multi-node cluster (depends on https://github.com/kubernetes-csi/csi-driver-host-path/pull/14). - # When running on a multi-node cluster, we need to figure out where the - # hostpath driver was deployed and set ClientNodeName accordingly. - generate_test_driver >"${CSI_PROW_WORK}/test-driver.yaml" || die "generating test-driver.yaml failed" # Rename, merge and filter JUnit files. Necessary in case that we run the E2E suite again From 53888ae7d59b862bfded46b883570c98f5fc5fff Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Tue, 3 Dec 2019 18:18:38 -0800 Subject: [PATCH 56/71] Improve README by adding an explicit Kubernetes dependency section --- README.md | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index cc40f161..08c82884 100644 --- a/README.md +++ b/README.md @@ -141,17 +141,6 @@ The `vendor` directory is optional. It is still present in projects because it avoids downloading sources during CI builds. If this is no longer deemed necessary, then a project can also remove the directory. -When using packages that are part of the Kubernetes source code, the -commands above are not enough because the [lack of semantic -versioning](https://github.com/kubernetes/kubernetes/issues/72638) -prevents `go mod` from finding newer releases. Importing directly from -`kubernetes/kubernetes` also needs `replace` statements to override -the fake `v0.0.0` versions -(https://github.com/kubernetes/kubernetes/issues/79384). The -`go-get-kubernetes.sh` script can be used to update all packages in -lockstep to a different Kubernetes version. It takes a single version -number like "1.16.0". - Conversion of a repository that uses `dep` to `go mod` can be done with: GO111MODULE=on go mod init @@ -160,3 +149,18 @@ Conversion of a repository that uses `dep` to `go mod` can be done with: GO111MODULE=on go mod vendor git rm -f Gopkg.toml Gopkg.lock git add go.mod go.sum vendor + +### Updating Kubernetes dependencies + +When using packages that are part of the Kubernetes source code, the +commands above are not enough because the [lack of semantic +versioning](https://github.com/kubernetes/kubernetes/issues/72638) +prevents `go mod` from finding newer releases. Importing directly from +`kubernetes/kubernetes` also needs `replace` statements to override +the fake `v0.0.0` versions +(https://github.com/kubernetes/kubernetes/issues/79384). The +`go-get-kubernetes.sh` script can be used to update all packages in +lockstep to a different Kubernetes version. Example usage: +``` +$ ./release-tools/go-get-kubernetes.sh 1.16.4 +``` From 4ad69492c97834b52d74d0b67b6f47b0590e4be1 Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Tue, 3 Dec 2019 23:48:29 -0800 Subject: [PATCH 57/71] Improve snapshot pod running checks and improve version_gt Signed-off-by: Grant Griffiths --- prow.sh | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/prow.sh b/prow.sh index d719e980..27c70512 100755 --- a/prow.sh +++ b/prow.sh @@ -713,10 +713,11 @@ install_snapshot_controller() { kubectl apply -f "https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml" cnt=0 - until kubectl get statefulset snapshot-controller | grep snapshot-controller | grep "1/1"; do + expected_running_pods=$(curl https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${CSI_SNAPSHOTTER_VERSION}"/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml | grep replicas | cut -d ':' -f 2-) + while [ "$(kubectl get pods -l app=snapshot-controller | grep 'Running' -c)" -lt "$expected_running_pods" ]; do if [ $cnt -gt 30 ]; then - echo "Running statefulsets:" - kubectl describe statefulsets + echo "snapshot-controller pod status:" + kubectl describe pods -l app=snapshot-controller echo >&2 "ERROR: snapshot controller not ready after over 5 min" exit 1 fi @@ -996,8 +997,30 @@ make_test_to_junit () { fi } +# version_gt returns true if arg1 is greater than arg2. +# +# This function expects versions to be one of the following formats: +# X.Y.Z, release-X.Y.Z, vX.Y.Z +# +# where X,Y, and Z are any number. +# +# Partial versions (1.2, release-1.2) work as well. +# The follow substrings are stripped before version comparison: +# - "v" +# - "release-" +# +# Usage: +# version_gt release-1.3 v1.2.0 (returns true) +# version_gt v1.1.1 v1.2.0 (returns false) +# version_gt 1.1.1 v1.2.0 (returns false) +# version_gt 1.3.1 v1.2.0 (returns true) +# version_gt 1.1.1 release-1.2.0 (returns false) +# version_gt 1.2.0 1.2.2 (returns false) function version_gt() { - test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"; + versions=$(for ver in "$@"; do ver=${ver#release-}; echo "${ver#v}"; done) + greaterVersion=${1#"release-"}; + greaterVersion=${greaterVersion#"v"}; + test "$(printf '%s' "$versions" | sort -V | head -n 1)" != "$greaterVersion" } main () { From a4e629966848d0097461f49f8aab0dfeb68d7a68 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Wed, 4 Dec 2019 14:12:50 -0800 Subject: [PATCH 58/71] fix syntax for ppc64le build --- build.make | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.make b/build.make index 7075a37e..a9b9d25d 100644 --- a/build.make +++ b/build.make @@ -70,7 +70,7 @@ build-%: check-go-version-go CGO_ENABLED=0 GOOS=linux go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$* if [ "$$ARCH" = "amd64" ]; then \ CGO_ENABLED=0 GOOS=windows go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*.exe ./cmd/$* ; \ - CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*-ppc64le ./cmd/$* + CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*-ppc64le ./cmd/$* ; \ fi container-%: build-% From b98b2aed087f4e768db673a9d43dff4c7abd1482 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Mon, 16 Dec 2019 19:13:38 -0800 Subject: [PATCH 59/71] Enable snapshot tests in 1.17 to be run in non-alpha jobs. This requires adding one more parallel e2e test run with a special focus flag because snapshot tests are still guarded with a "[Feature:VolumeSnapshotDataSource]" tag. The setting that skips all tests with "[Feature:.*]" has to be removed because it overrides the focus. We don't have serial snapshot tests yet. This needs to be modified again if we add any in the future. --- prow.sh | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/prow.sh b/prow.sh index 9bfdc2e2..c160f886 100755 --- a/prow.sh +++ b/prow.sh @@ -132,7 +132,7 @@ configvar CSI_PROW_BUILD_JOB true "building code in repo enabled" # use the same settings as for "latest" Kubernetes. This works # as long as there are no breaking changes in Kubernetes, like # deprecating or changing the implementation of an alpha feature. -configvar CSI_PROW_KUBERNETES_VERSION 1.15.3 "Kubernetes" +configvar CSI_PROW_KUBERNETES_VERSION 1.17.0 "Kubernetes" # This is a hack to workaround the issue that each version # of kind currently only supports specific patch versions of @@ -142,7 +142,6 @@ configvar CSI_PROW_KUBERNETES_VERSION 1.15.3 "Kubernetes" # # If the version is prefixed with "release-", then nothing # is overridden. -override_k8s_version "1.14.6" override_k8s_version "1.15.3" # CSI_PROW_KUBERNETES_VERSION reduced to first two version numbers and @@ -206,9 +205,9 @@ configvar CSI_PROW_HOSTPATH_CANARY "" "hostpath image" # # CSI_PROW_E2E_REPO=none disables E2E testing. # TOOO: remove versioned variables and make e2e version match k8s version -configvar CSI_PROW_E2E_VERSION_1_14 v1.14.0 "E2E version for Kubernetes 1.14.x" configvar CSI_PROW_E2E_VERSION_1_15 v1.15.0 "E2E version for Kubernetes 1.15.x" configvar CSI_PROW_E2E_VERSION_1_16 v1.16.0 "E2E version for Kubernetes 1.16.x" +configvar CSI_PROW_E2E_VERSION_1_17 v1.17.0 "E2E version for Kubernetes 1.17.x" # TODO: add new CSI_PROW_E2E_VERSION entry for future Kubernetes releases configvar CSI_PROW_E2E_VERSION_LATEST master "E2E version for Kubernetes master" # testing against Kubernetes master is already tracking a moving target, so we might as well use a moving E2E version configvar CSI_PROW_E2E_REPO_LATEST https://github.com/kubernetes/kubernetes "E2E repo for Kubernetes >= 1.13.x" # currently the same for all versions @@ -278,6 +277,14 @@ tests_need_alpha_cluster () { tests_enabled "parallel-alpha" "serial-alpha" } +# Regex for non-alpha, feature-tagged tests that should be run. +# +# Starting with 1.17, snapshots is beta, but the E2E tests still have the +# [Feature:] tag. They need to be explicitly enabled. +configvar CSI_PROW_E2E_FOCUS_1_15 '^' "non-alpha, feature-tagged tests for Kubernetes = 1.15" # no tests to run, match nothing +configvar CSI_PROW_E2E_FOCUS_1_16 '^' "non-alpha, feature-tagged tests for Kubernetes = 1.16" # no tests to run, match nothing +configvar CSI_PROW_E2E_FOCUS_LATEST '\[Feature:VolumeSnapshotDataSource\]' "non-alpha, feature-tagged tests for Kubernetes >= 1.17" +configvar CSI_PROW_E2E_FOCUS "$(get_versioned_variable CSI_PROW_E2E_FOCUS "${csi_prow_kubernetes_version_suffix}")" "non-alpha, feature-tagged tests" # Serial vs. parallel is always determined by these regular expressions. # Individual regular expressions are seperated by spaces for readability @@ -313,12 +320,11 @@ configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi # kubernetes-csi components must be updated, either by disabling # the failing test for "latest" or by updating the test and not running # it anymore for older releases. -configvar CSI_PROW_E2E_ALPHA_GATES_1_14 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for Kubernetes 1.14" configvar CSI_PROW_E2E_ALPHA_GATES_1_15 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for Kubernetes 1.15" configvar CSI_PROW_E2E_ALPHA_GATES_1_16 'VolumeSnapshotDataSource=true' "alpha feature gates for Kubernetes 1.16" # TODO: add new CSI_PROW_ALPHA_GATES_xxx entry for future Kubernetes releases and # add new gates to CSI_PROW_E2E_ALPHA_GATES_LATEST. -configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'VolumeSnapshotDataSource=true' "alpha feature gates for latest Kubernetes" +configvar CSI_PROW_E2E_ALPHA_GATES_LATEST '' "alpha feature gates for latest Kubernetes" configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" # Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment @@ -1111,6 +1117,16 @@ main () { warn "E2E parallel failed" ret=1 fi + + # Run tests that are feature tagged, but non-alpha + # Ignore: Double quote to prevent globbing and word splitting. + # shellcheck disable=SC2086 + if ! run_e2e parallel ${CSI_PROW_GINKO_PARALLEL} \ + -focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_FOCUS}"))" \ + -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}")"; then + warn "E2E parallel features failed" + ret=1 + fi fi if tests_enabled "serial"; then From fc80975954a9720a1611fa9a5195cbfc48b64167 Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Fri, 20 Dec 2019 16:30:25 -0800 Subject: [PATCH 60/71] Fix version_gt to work with kubernetes prefix Signed-off-by: Grant Griffiths --- prow.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index c160f886..832da1ee 100755 --- a/prow.sh +++ b/prow.sh @@ -1013,6 +1013,7 @@ make_test_to_junit () { # The follow substrings are stripped before version comparison: # - "v" # - "release-" +# - "kubernetes-" # # Usage: # version_gt release-1.3 v1.2.0 (returns true) @@ -1022,8 +1023,9 @@ make_test_to_junit () { # version_gt 1.1.1 release-1.2.0 (returns false) # version_gt 1.2.0 1.2.2 (returns false) function version_gt() { - versions=$(for ver in "$@"; do ver=${ver#release-}; echo "${ver#v}"; done) + versions=$(for ver in "$@"; do ver=${ver#release-}; ver=${ver#kubernetes-}; echo "${ver#v}"; done) greaterVersion=${1#"release-"}; + greaterVersion=${greaterVersion#"kubernetes-"}; greaterVersion=${greaterVersion#"v"}; test "$(printf '%s' "$versions" | sort -V | head -n 1)" != "$greaterVersion" } From af9549b5a11c4849ff7b4a1a42a7b8aea59137c3 Mon Sep 17 00:00:00 2001 From: saad-ali Date: Thu, 2 Jan 2020 14:29:40 -0800 Subject: [PATCH 61/71] Update prow hostpath driver version to 1.3.0-rc2 --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 832da1ee..ae51947d 100755 --- a/prow.sh +++ b/prow.sh @@ -187,7 +187,7 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip # # When no deploy script is found (nothing in `deploy` directory, # CSI_PROW_HOSTPATH_REPO=none), nothing gets deployed. -configvar CSI_PROW_HOSTPATH_VERSION "v1.2.0" "hostpath driver" +configvar CSI_PROW_HOSTPATH_VERSION "v1.3.0-rc2" "hostpath driver" configvar CSI_PROW_HOSTPATH_REPO https://github.com/kubernetes-csi/csi-driver-host-path "hostpath repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" configvar CSI_PROW_HOSTPATH_DRIVER_NAME "hostpath.csi.k8s.io" "the hostpath driver name" From 8b0316c7e4d94a3206cdab0ea2e8d7f7b3be637b Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Thu, 2 Jan 2020 14:33:46 -0800 Subject: [PATCH 62/71] Fix overriding of junit results by using unique names for each e2e run --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index ae51947d..e21a7ba9 100755 --- a/prow.sh +++ b/prow.sh @@ -1123,7 +1123,7 @@ main () { # Run tests that are feature tagged, but non-alpha # Ignore: Double quote to prevent globbing and word splitting. # shellcheck disable=SC2086 - if ! run_e2e parallel ${CSI_PROW_GINKO_PARALLEL} \ + if ! run_e2e parallel-features ${CSI_PROW_GINKO_PARALLEL} \ -focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_FOCUS}"))" \ -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}")"; then warn "E2E parallel features failed" From 8191eab6ffe694d11bf1e0d254330287c961dc77 Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Tue, 7 Jan 2020 18:09:31 -0800 Subject: [PATCH 63/71] Update snapshotter to version v2.0.0 Signed-off-by: Grant Griffiths --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index e21a7ba9..eb451ef1 100755 --- a/prow.sh +++ b/prow.sh @@ -328,7 +328,7 @@ configvar CSI_PROW_E2E_ALPHA_GATES_LATEST '' "alpha feature gates for latest Kub configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" # Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment -configvar CSI_SNAPSHOTTER_VERSION 'v2.0.0-rc4' "external-snapshotter version tag" +configvar CSI_SNAPSHOTTER_VERSION 'v2.0.0' "external-snapshotter version tag" # Some tests are known to be unusable in a KinD cluster. For example, # stopping kubelet with "ssh systemctl stop kubelet" simply From 6582f2ff3bd584e662035b7da2a90efc52184b1b Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Thu, 9 Jan 2020 17:25:38 -0800 Subject: [PATCH 64/71] Update hostpath driver version to get fix for connection-timeout --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index e21a7ba9..27309a79 100755 --- a/prow.sh +++ b/prow.sh @@ -187,7 +187,7 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip # # When no deploy script is found (nothing in `deploy` directory, # CSI_PROW_HOSTPATH_REPO=none), nothing gets deployed. -configvar CSI_PROW_HOSTPATH_VERSION "v1.3.0-rc2" "hostpath driver" +configvar CSI_PROW_HOSTPATH_VERSION "v1.3.0-rc3" "hostpath driver" configvar CSI_PROW_HOSTPATH_REPO https://github.com/kubernetes-csi/csi-driver-host-path "hostpath repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" configvar CSI_PROW_HOSTPATH_DRIVER_NAME "hostpath.csi.k8s.io" "the hostpath driver name" From ac8a0212b93cad817689029be503839ad1959e21 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Wed, 11 Dec 2019 17:56:08 -0800 Subject: [PATCH 65/71] Document the process for releasing a new sidecar --- SIDECAR_RELEASE_PROCESS.md | 90 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 SIDECAR_RELEASE_PROCESS.md diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md new file mode 100644 index 00000000..91a1e1b8 --- /dev/null +++ b/SIDECAR_RELEASE_PROCESS.md @@ -0,0 +1,90 @@ +# Sidecar Release Process + +This page describes the process for releasing a kubernetes-csi sidecar. + +## Prerequisites + +The release manager must: + +* Be a member of the kubernetes-csi organization. Open an + [issue](https://github.com/kubernetes/org/issues/new?assignees=&labels=area%2Fgithub-membership&template=membership.md&title=REQUEST%3A+New+membership+for+%3Cyour-GH-handle%3E) in + kubernetes/org to request membership +* Be a top level approver for the repository. To become a top level approver, + the candidate must demonstrate ownership and deep knowledge of the repository + through active maintainence, responding to and fixing issues, reviewing PRs, + test triage. +* Be part of the maintainers or admin group for the repository. admin is a + superset of maintainers, only maintainers level is required for cutting a + release. Membership can be requested by submitting a PR to kubernetes/org. + [Example](https://github.com/kubernetes/org/pull/1467) + +## Updating CI Jobs +Whenever a new Kubernetes minor version is released, our kubernetes-csi CI jobs +must be updated. + +[Our CI jobs](https://k8s-testgrid.appspot.com/sig-storage-csi-ci) have the +naming convention `-on-`. + +1. Jobs should be actively monitored to find and fix failures in sidecars and + infrastructure changes early in the development cycle. Test failures are sent + to kubernetes-sig-storage-test-failures@googlegroups.com. +1. "-on-master" jobs are the closest reflection to the new Kubernetes version. +1. Fixes to our prow.sh CI script can be tested in the [CSI hostpath + repo](https://github.com/kubernetes-csi/csi-driver-host-path) by modifying + [prow.sh](https://github.com/kubernetes-csi/csi-driver-host-path/blob/master/release-tools/prow.sh) + along with any overrides in + [.prow.sh](https://github.com/kubernetes-csi/csi-driver-host-path/blob/master/.prow.sh) + to mirror the failing environment. Once e2e tests are passing (verify-unit tests + will fail), then the prow.sh changes can be submitted to [csi-release-tools](https://github.com/kubernetes-csi/csi-release-tools). +1. Changes can then be updated in all the sidecar repos and hostpath driver repo + by following the [update + instructions](https://github.com/kubernetes-csi/csi-release-tools/blob/master/README.md#sharing-and-updating). +1. New pull and CI jobs are configured by + [gen-jobs.sh](https://github.com/kubernetes/test-infra/blob/master/config/jobs/kubernetes-csi/gen-jobs.sh). + New pull jobs that have been unverified should be initially made optional. + [Example](https://github.com/kubernetes/test-infra/pull/15055) +1. Once new pull and CI jobs have been verified, and the new Kubernetes version + is released, we can make the optional jobs required, and also remove the + Kubernetes versions that are no longer supported. + +## Release Process +1. Identify all issues and ongoing PRs that should go into the release, and + drive them to resolution. +1. Download [K8s release notes + generator](https://github.com/kubernetes/release/tree/master/cmd/release-notes) +1. Generate release notes for the release. Replace arguments with the relevant + information. + ``` + GITHUB_TOKEN= ./release-notes --start-sha=0ed6978fd199e3ca10326b82b4b8b8e916211c9b --end-sha=3cb3d2f18ed8cb40371c6d8886edcabd1f27e7b9 \ + --github-org=kubernetes-csi --github-repo=external-attacher -branch=master -output out.md + ``` + * `--start-sha` should point to the last release from the same branch. For + example: + * `1.X-1.0` tag when releasing `1.X.0` + * `1.X.Y-1` tag when releasing `1.X.Y` +1. Compare the generated output to the new commits for the release to check if + any notable change missed a release note. +1. Reword release notes as needed. Make sure to check notes for breaking + changes and deprecations. +1. If release is a new major/minor version, create a new `CHANGELOG-..md` + file. Otherwise, add the release notes to the top of the existing CHANGELOG + file for that minor version. +1. Submit a PR for the CHANGELOG changes. +1. Submit a PR for README changes, in particular, Compatibility, Feature status, + and any other sections that may need updating. +1. Check that all [canary CI + jobs](https://k8s-testgrid.appspot.com/sig-storage-csi-ci) are passing, + and that test coverage is adequate for the changes that are going into the release. +1. Make sure that no new PRs have merged in the meantime, and no PRs are in + flight and soon to be merged. +1. Create a new release following a previous release as a template. Be sure to select the correct + branch. This requires Github release permissions as required by the prerequisites. + [external-provisioner example](https://github.com/kubernetes-csi/external-provisioner/releases/new) +1. If release was a new major/minor version, create a new `release-` + branch at that commit. +1. Update [kubernetes-csi/docs](https://github.com/kubernetes-csi/docs) sidecar + and feature pages with the new released version. +1. After all the sidecars have been released, update + CSI hostpath driver with the new sidecars in the [CSI repo](https://github.com/kubernetes-csi/csi-driver-host-path/tree/master/deploy) + and [k/k + in-tree](https://github.com/kubernetes/kubernetes/tree/master/test/e2e/testing-manifests/storage-csi/hostpath/hostpath) From fa90abd07862e6fa0a7b66ea2435b9db364ce6c9 Mon Sep 17 00:00:00 2001 From: wangzheng03 Date: Sun, 19 Jan 2020 09:42:26 +0800 Subject: [PATCH 66/71] fix incorrect link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 08c82884..60eab2a9 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ on what is enabled in Prow, see https://github.com/kubernetes/test-infra/tree/master/config/jobs/kubernetes-csi Test results for periodic jobs are visible in -https://testgrid.k8s.io/sig-storage-csi +https://testgrid.k8s.io/sig-storage-csi-ci It is possible to reproduce the Prow testing locally on a suitable machine: - Linux host From 84f78b120e891b76b3e368cb0d20502baec026ea Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 10 Feb 2020 13:08:33 +0100 Subject: [PATCH 67/71] prow.sh: generic driver installation This relies on a slightly different deployment script: a "deploy.sh" must exist which knows that it has to dump a test driver configurion into the file pointed to with CSI_PROW_TEST_DRIVER, if that env variable is set. That way, we no longer need to know what capabilities the installed driver has. --- prow.sh | 104 +++++++++++++++++++++++--------------------------------- 1 file changed, 43 insertions(+), 61 deletions(-) diff --git a/prow.sh b/prow.sh index 118c5bd1..bb80741e 100755 --- a/prow.sh +++ b/prow.sh @@ -157,7 +157,9 @@ csi_prow_kubernetes_version_suffix="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | tr # the caller. configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory" -# The hostpath deployment script is searched for in several places. +# By default, this script tests sidecars with the CSI hostpath driver, +# using the install_csi_driver function. That function depends on +# a deployment script that it searches for in several places: # # - The "deploy" directory in the current repository: this is useful # for the situation that a component becomes incompatible with the @@ -165,11 +167,11 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip # own example until the shared one can be updated; it's also how # csi-driver-host-path itself provides the example. # -# - CSI_PROW_HOSTPATH_VERSION of the CSI_PROW_HOSTPATH_REPO is checked +# - CSI_PROW_DRIVER_VERSION of the CSI_PROW_DRIVER_REPO is checked # out: this allows other repos to reference a version of the example # that is known to be compatible. # -# - The csi-driver-host-path/deploy directory has multiple sub-directories, +# - The /deploy directory can have multiple sub-directories, # each with different deployments (stable set of images for Kubernetes 1.13, # stable set of images for Kubernetes 1.14, canary for latest Kubernetes, etc.). # This is necessary because there may be incompatible changes in the @@ -186,16 +188,26 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip # "none" disables the deployment of the hostpath driver. # # When no deploy script is found (nothing in `deploy` directory, -# CSI_PROW_HOSTPATH_REPO=none), nothing gets deployed. -configvar CSI_PROW_HOSTPATH_VERSION "v1.3.0-rc3" "hostpath driver" -configvar CSI_PROW_HOSTPATH_REPO https://github.com/kubernetes-csi/csi-driver-host-path "hostpath repo" +# CSI_PROW_DRIVER_REPO=none), nothing gets deployed. +# +# If the deployment script is called with CSI_PROW_TEST_DRIVER= as +# environment variable, then it must write a suitable test driver configuration +# into that file in addition to installing the driver. +configvar CSI_PROW_DRIVER_VERSION "v1.3.0-rc4" "CSI driver version" +configvar CSI_PROW_DRIVER_REPO https://github.com/kubernetes-csi/csi-driver-host-path "CSI driver repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" -configvar CSI_PROW_HOSTPATH_DRIVER_NAME "hostpath.csi.k8s.io" "the hostpath driver name" -# If CSI_PROW_HOSTPATH_CANARY is set (typically to "canary", but also -# "1.0-canary"), then all image versions are replaced with that -# version tag. -configvar CSI_PROW_HOSTPATH_CANARY "" "hostpath image" +# The install_csi_driver function may work also for other CSI drivers, +# as long as they follow the conventions of the CSI hostpath driver. +# If they don't, then a different install function can be provided in +# a .prow.sh file and this config variable can be overridden. +configvar CSI_PROW_DRIVER_INSTALL "install_csi_driver" "name of the shell function which installs the CSI driver" + +# If CSI_PROW_DRIVER_CANARY is set (typically to "canary", but also +# version tag. Usually empty. CSI_PROW_HOSTPATH_CANARY is +# accepted as alternative name because some test-infra jobs +# still use that name. +configvar CSI_PROW_DRIVER_CANARY "${CSI_PROW_HOSTPATH_CANARY}" "driver image override for canary images" # The E2E testing can come from an arbitrary repo. The expectation is that # the repo supports "go test ./test/e2e -args --storage.testdriver" (https://github.com/kubernetes/kubernetes/pull/72836) @@ -613,7 +625,7 @@ find_deployment () { # Fixed deployment name? Use it if it exists, otherwise fail. if [ "${CSI_PROW_DEPLOYMENT}" ]; then - file="$dir/${CSI_PROW_DEPLOYMENT}/deploy-hostpath.sh" + file="$dir/${CSI_PROW_DEPLOYMENT}/deploy.sh" if ! [ -e "$file" ]; then return 1 fi @@ -623,9 +635,9 @@ find_deployment () { # Ignore: See if you can use ${variable//search/replace} instead. # shellcheck disable=SC2001 - file="$dir/kubernetes-$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')/deploy-hostpath.sh" + file="$dir/kubernetes-$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')/deploy.sh" if ! [ -e "$file" ]; then - file="$dir/kubernetes-latest/deploy-hostpath.sh" + file="$dir/kubernetes-latest/deploy.sh" if ! [ -e "$file" ]; then return 1 fi @@ -633,12 +645,11 @@ find_deployment () { echo "$file" } -# This installs the hostpath driver example. CSI_PROW_HOSTPATH_CANARY overrides all -# image versions with that canary version. The parameters of install_hostpath can be -# used to override registry and/or tag of individual images (CSI_PROVISIONER_REGISTRY=localhost:9000 -# CSI_PROVISIONER_TAG=latest). -install_hostpath () { - local images deploy_hostpath +# This installs the CSI driver. It's called with a list of env variables +# that override the default images. CSI_PROW_DRIVER_CANARY overrides all +# image versions with that canary version. +install_csi_driver () { + local images deploy_driver images="$*" if [ "${CSI_PROW_DEPLOYMENT}" = "none" ]; then @@ -654,31 +665,31 @@ install_hostpath () { done fi - if deploy_hostpath="$(find_deployment "$(pwd)/deploy")"; then + if deploy_driver="$(find_deployment "$(pwd)/deploy")"; then : - elif [ "${CSI_PROW_HOSTPATH_REPO}" = "none" ]; then + elif [ "${CSI_PROW_DRIVER_REPO}" = "none" ]; then return 1 else - git_checkout "${CSI_PROW_HOSTPATH_REPO}" "${CSI_PROW_WORK}/hostpath" "${CSI_PROW_HOSTPATH_VERSION}" --depth=1 || die "checking out hostpath repo failed" - if deploy_hostpath="$(find_deployment "${CSI_PROW_WORK}/hostpath/deploy")"; then + git_checkout "${CSI_PROW_DRIVER_REPO}" "${CSI_PROW_WORK}/csi-driver" "${CSI_PROW_DRIVER_VERSION}" --depth=1 || die "checking out CSI driver repo failed" + if deploy_driver="$(find_deployment "${CSI_PROW_WORK}/csi-driver/deploy")"; then : else - die "deploy-hostpath.sh not found in ${CSI_PROW_HOSTPATH_REPO} ${CSI_PROW_HOSTPATH_VERSION}. To disable E2E testing, set CSI_PROW_HOSTPATH_REPO=none" + die "deploy.sh not found in ${CSI_PROW_DRIVER_REPO} ${CSI_PROW_DRIVER_VERSION}. To disable E2E testing, set CSI_PROW_DRIVER_REPO=none" fi fi - if [ "${CSI_PROW_HOSTPATH_CANARY}" != "stable" ]; then - images="$images IMAGE_TAG=${CSI_PROW_HOSTPATH_CANARY}" + if [ "${CSI_PROW_DRIVER_CANARY}" != "stable" ]; then + images="$images IMAGE_TAG=${CSI_PROW_DRIVER_CANARY}" fi # Ignore: Double quote to prevent globbing and word splitting. # It's intentional here for $images. # shellcheck disable=SC2086 - if ! run env $images "${deploy_hostpath}"; then + if ! run env "CSI_PROW_TEST_DRIVER=${CSI_PROW_WORK}/test-driver.yaml" $images "${deploy_driver}"; then # Collect information about failed deployment before failing. collect_cluster_info (start_loggers >/dev/null; wait) info "For container output see job artifacts." - die "deploying the hostpath driver with ${deploy_hostpath} failed" + die "deploying the CSI driver with ${deploy_driver} failed" fi } @@ -804,33 +815,6 @@ install_sanity () ( run_with_go "${CSI_PROW_GO_VERSION_SANITY}" go test -c -o "${CSI_PROW_WORK}/csi-sanity" "${CSI_PROW_SANITY_IMPORT_PATH}/cmd/csi-sanity" || die "building csi-sanity failed" ) -# The default implementation of this function generates a external -# driver test configuration for the hostpath driver. -# -# The content depends on both what the E2E suite expects and what the -# installed hostpath driver supports. Generating it here seems prone -# to breakage, but it is uncertain where a better place might be. -generate_test_driver () { - cat <"${CSI_PROW_WORK}/test-driver.yaml" || die "generating test-driver.yaml failed" - # Rename, merge and filter JUnit files. Necessary in case that we run the E2E suite again # and to avoid the large number of "skipped" tests that we get from using # the full Kubernetes E2E testsuite while only running a few tests. @@ -1063,7 +1045,7 @@ main () { cmds="$(grep '^\s*CMDS\s*=' Makefile | sed -e 's/\s*CMDS\s*=//')" # Get the image that was just built (if any) from the # top-level Makefile CMDS variable and set the - # deploy-hostpath.sh env variables for it. We also need to + # deploy.sh env variables for it. We also need to # side-load those images into the cluster. for i in $cmds; do e=$(echo "$i" | tr '[:lower:]' '[:upper:]' | tr - _) @@ -1101,7 +1083,7 @@ main () { fi # Installing the driver might be disabled. - if install_hostpath "$images"; then + if ${CSI_PROW_DRIVER_INSTALL} "$images"; then collect_cluster_info if sanity_enabled; then @@ -1158,7 +1140,7 @@ main () { fi # Installing the driver might be disabled. - if install_hostpath "$images"; then + if ${CSI_PROW_DRIVER_INSTALL} "$images"; then collect_cluster_info if tests_enabled "parallel-alpha"; then From 5f74333a466f0767cec812945c970c943a59cf7f Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 10 Feb 2020 11:06:31 +0100 Subject: [PATCH 68/71] prow.sh: also configure feature gates for kubelet That this hasn't been done before is an oversight. Apparently it hasn't been a problem because there never have been feature gates that mattered? --- prow.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/prow.sh b/prow.sh index 118c5bd1..6778e8b8 100755 --- a/prow.sh +++ b/prow.sh @@ -568,6 +568,13 @@ kubeadmConfigPatches: nodeRegistration: kubeletExtraArgs: "feature-gates": "$gates" +- | + apiVersion: kubelet.config.k8s.io/v1beta1 + kind: KubeletConfiguration + metadata: + name: config + featureGates: +$(list_gates "$gates") - | apiVersion: kubeproxy.config.k8s.io/v1alpha1 kind: KubeProxyConfiguration From 679bb9dc07a49467356283b89a16fee9de7baca6 Mon Sep 17 00:00:00 2001 From: Jan Wozniak Date: Wed, 12 Feb 2020 15:43:33 +0100 Subject: [PATCH 69/71] Add dependency on k8s.io/utils/mount --- go.mod | 3 +-- go.sum | 10 ++++++++-- pkg/nfs/nfs.go | 2 +- pkg/nfs/nodeserver.go | 2 +- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index dcdfc6f9..f722f085 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,6 @@ require ( github.com/onsi/ginkgo v1.8.0 github.com/onsi/gomega v1.5.0 github.com/opencontainers/go-digest v1.0.0-rc1 // indirect - github.com/spf13/afero v1.2.2 // indirect github.com/spf13/cobra v0.0.3 golang.org/x/net v0.0.0-20190415100556-4a65cf94b679 google.golang.org/grpc v1.20.0 @@ -24,5 +23,5 @@ require ( k8s.io/kube-aggregator v0.0.0-20190415133304-80ce4e5a0cbc // indirect k8s.io/kube-openapi v0.0.0-20190401085232-94e1e7b7574c // indirect k8s.io/kubernetes v1.14.1 - k8s.io/utils v0.0.0-20190308190857-21c4ce38f2a7 // indirect + k8s.io/utils v0.0.0-20200124190032-861946025e34 ) diff --git a/go.sum b/go.sum index 0d3ac9c2..9df72a5f 100644 --- a/go.sum +++ b/go.sum @@ -26,6 +26,7 @@ github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7 h1:u9SHYsPQNyt5t github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea h1:n2Ltr3SrfQlf/9nOna1DoGKxLx3qTSI8Ttl6Xrqp6mw= github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda h1:NyywMz59neOoVRFDz+ccfKWxn784fiHMDnZSy6T+JXY= @@ -160,10 +161,13 @@ github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU= github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8 h1:ndzgwNDnKIqyCvHTXaCqh9KlOWKvBry6nuXMJmonVsE= @@ -268,6 +272,8 @@ k8s.io/csi-translation-lib v0.0.0-20190415134207-82f1dfd98d10/go.mod h1:L35fSWHA k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog v0.0.0-20190306015804-8e90cee79f82 h1:SHucoAy7lRb+w5oC/hbXyZg+zX+Wftn6hD4tGzHCVqA= k8s.io/klog v0.0.0-20190306015804-8e90cee79f82/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE= +k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-aggregator v0.0.0-20190415133304-80ce4e5a0cbc h1:My3QUoguktKeT5UYet1WNr9Mw4noRGlCHHsKnGzHsBI= k8s.io/kube-aggregator v0.0.0-20190415133304-80ce4e5a0cbc/go.mod h1:T/aMpr/6zvgjgJIumGJ+E8XhJCDwJH0hRfExeZJ1IJM= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI= @@ -277,8 +283,8 @@ k8s.io/kube-openapi v0.0.0-20190401085232-94e1e7b7574c/go.mod h1:BXM9ceUBTj2QnfH k8s.io/kubernetes v1.14.1 h1:I9F52h5sqVxBmoSsBlNQ0YygNcukDilkpGxUbJRoBoY= k8s.io/kubernetes v1.14.1/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= -k8s.io/utils v0.0.0-20190308190857-21c4ce38f2a7 h1:8r+l4bNWjRlsFYlQJnKJ2p7s1YQPj4XyXiJVqDHRx7c= -k8s.io/utils v0.0.0-20190308190857-21c4ce38f2a7/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= +k8s.io/utils v0.0.0-20200124190032-861946025e34 h1:HjlUD6M0K3P8nRXmr2B9o4F9dUy9TCj/aEpReeyi6+k= +k8s.io/utils v0.0.0-20200124190032-861946025e34/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= diff --git a/pkg/nfs/nfs.go b/pkg/nfs/nfs.go index 6dfd0b7c..d637b0d2 100644 --- a/pkg/nfs/nfs.go +++ b/pkg/nfs/nfs.go @@ -19,7 +19,7 @@ package nfs import ( "github.com/container-storage-interface/spec/lib/go/csi" "github.com/golang/glog" - "k8s.io/kubernetes/pkg/util/mount" + "k8s.io/utils/mount" ) type nfsDriver struct { diff --git a/pkg/nfs/nodeserver.go b/pkg/nfs/nodeserver.go index d4fcdee5..439d3c8b 100644 --- a/pkg/nfs/nodeserver.go +++ b/pkg/nfs/nodeserver.go @@ -27,7 +27,7 @@ import ( "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "k8s.io/kubernetes/pkg/util/mount" + "k8s.io/utils/mount" ) type nodeServer struct { From 40f35a984c03896579d2c130b3c958bb718988c4 Mon Sep 17 00:00:00 2001 From: Jan Wozniak Date: Wed, 12 Feb 2020 15:44:54 +0100 Subject: [PATCH 70/71] go mod vendor --- vendor/github.com/ghodss/yaml/.travis.yml | 5 +- vendor/github.com/ghodss/yaml/yaml.go | 46 +- vendor/github.com/ghodss/yaml/yaml_go110.go | 14 + vendor/github.com/pkg/errors/.travis.yml | 12 +- vendor/github.com/pkg/errors/README.md | 4 +- vendor/github.com/pkg/errors/errors.go | 43 +- vendor/github.com/pkg/errors/stack.go | 51 +- .../bitbucket.org/ww/goautoneg/autoneg.go | 6 +- .../prometheus/common/model/metric.go | 1 + .../prometheus/procfs/MAINTAINERS.md | 3 +- vendor/github.com/prometheus/procfs/Makefile | 10 +- .../prometheus/procfs/Makefile.common | 89 +- .../prometheus/procfs/fixtures.ttar | 1426 +---------------- vendor/github.com/prometheus/procfs/fs.go | 36 + vendor/github.com/prometheus/procfs/go.mod | 2 - vendor/github.com/prometheus/procfs/go.sum | 2 - .../prometheus/procfs/internal/util/parse.go | 59 + .../procfs/internal/util/sysreadfile_linux.go | 45 + .../prometheus/procfs/mountstats.go | 10 - .../github.com/prometheus/procfs/nfs/nfs.go | 263 +++ .../github.com/prometheus/procfs/nfs/parse.go | 317 ++++ .../prometheus/procfs/nfs/parse_nfs.go | 67 + .../prometheus/procfs/nfs/parse_nfsd.go | 89 + .../github.com/prometheus/procfs/proc_psi.go | 110 -- .../github.com/prometheus/procfs/proc_stat.go | 4 +- .../github.com/prometheus/procfs/xfs/parse.go | 330 ++++ .../github.com/prometheus/procfs/xfs/xfs.go | 163 ++ vendor/github.com/stretchr/testify/LICENSE | 35 +- .../stretchr/testify/assert/assertions.go | 32 +- .../stretchr/testify/require/requirements.go | 2 +- vendor/k8s.io/klog/README.md | 22 +- vendor/k8s.io/klog/SECURITY_CONTACTS | 4 +- vendor/k8s.io/klog/klog.go | 46 +- vendor/k8s.io/klog/klog_file.go | 2 +- .../utils/io/{consistentread.go => read.go} | 19 + vendor/k8s.io/utils/mount/OWNERS | 14 + vendor/k8s.io/utils/mount/doc.go | 18 + vendor/k8s.io/utils/mount/fake_mounter.go | 207 +++ vendor/k8s.io/utils/mount/mount.go | 300 ++++ .../k8s.io/utils/mount/mount_helper_common.go | 99 ++ .../k8s.io/utils/mount/mount_helper_unix.go | 158 ++ .../utils/mount/mount_helper_windows.go | 101 ++ vendor/k8s.io/utils/mount/mount_linux.go | 536 +++++++ .../k8s.io/utils/mount/mount_unsupported.go | 72 + vendor/k8s.io/utils/mount/mount_windows.go | 280 ++++ vendor/k8s.io/utils/net/ipnet.go | 100 ++ vendor/k8s.io/utils/net/net.go | 154 +- vendor/k8s.io/utils/pointer/pointer.go | 2 +- vendor/k8s.io/utils/trace/trace.go | 51 +- vendor/modules.txt | 20 +- 50 files changed, 3836 insertions(+), 1645 deletions(-) create mode 100644 vendor/github.com/ghodss/yaml/yaml_go110.go delete mode 100644 vendor/github.com/prometheus/procfs/go.sum create mode 100644 vendor/github.com/prometheus/procfs/internal/util/parse.go create mode 100644 vendor/github.com/prometheus/procfs/internal/util/sysreadfile_linux.go create mode 100644 vendor/github.com/prometheus/procfs/nfs/nfs.go create mode 100644 vendor/github.com/prometheus/procfs/nfs/parse.go create mode 100644 vendor/github.com/prometheus/procfs/nfs/parse_nfs.go create mode 100644 vendor/github.com/prometheus/procfs/nfs/parse_nfsd.go delete mode 100644 vendor/github.com/prometheus/procfs/proc_psi.go create mode 100644 vendor/github.com/prometheus/procfs/xfs/parse.go create mode 100644 vendor/github.com/prometheus/procfs/xfs/xfs.go rename vendor/k8s.io/utils/io/{consistentread.go => read.go} (77%) create mode 100644 vendor/k8s.io/utils/mount/OWNERS create mode 100644 vendor/k8s.io/utils/mount/doc.go create mode 100644 vendor/k8s.io/utils/mount/fake_mounter.go create mode 100644 vendor/k8s.io/utils/mount/mount.go create mode 100644 vendor/k8s.io/utils/mount/mount_helper_common.go create mode 100644 vendor/k8s.io/utils/mount/mount_helper_unix.go create mode 100644 vendor/k8s.io/utils/mount/mount_helper_windows.go create mode 100644 vendor/k8s.io/utils/mount/mount_linux.go create mode 100644 vendor/k8s.io/utils/mount/mount_unsupported.go create mode 100644 vendor/k8s.io/utils/mount/mount_windows.go diff --git a/vendor/github.com/ghodss/yaml/.travis.yml b/vendor/github.com/ghodss/yaml/.travis.yml index 0e9d6edc..930860e0 100644 --- a/vendor/github.com/ghodss/yaml/.travis.yml +++ b/vendor/github.com/ghodss/yaml/.travis.yml @@ -1,7 +1,8 @@ language: go go: - - 1.3 - - 1.4 + - "1.3" + - "1.4" + - "1.10" script: - go test - go build diff --git a/vendor/github.com/ghodss/yaml/yaml.go b/vendor/github.com/ghodss/yaml/yaml.go index 4fb4054a..6e7f14fc 100644 --- a/vendor/github.com/ghodss/yaml/yaml.go +++ b/vendor/github.com/ghodss/yaml/yaml.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "io" "reflect" "strconv" @@ -26,15 +27,19 @@ func Marshal(o interface{}) ([]byte, error) { return y, nil } -// Converts YAML to JSON then uses JSON to unmarshal into an object. -func Unmarshal(y []byte, o interface{}) error { +// JSONOpt is a decoding option for decoding from JSON format. +type JSONOpt func(*json.Decoder) *json.Decoder + +// Unmarshal converts YAML to JSON then uses JSON to unmarshal into an object, +// optionally configuring the behavior of the JSON unmarshal. +func Unmarshal(y []byte, o interface{}, opts ...JSONOpt) error { vo := reflect.ValueOf(o) - j, err := yamlToJSON(y, &vo) + j, err := yamlToJSON(y, &vo, yaml.Unmarshal) if err != nil { return fmt.Errorf("error converting YAML to JSON: %v", err) } - err = json.Unmarshal(j, o) + err = jsonUnmarshal(bytes.NewReader(j), o, opts...) if err != nil { return fmt.Errorf("error unmarshaling JSON: %v", err) } @@ -42,6 +47,21 @@ func Unmarshal(y []byte, o interface{}) error { return nil } +// jsonUnmarshal unmarshals the JSON byte stream from the given reader into the +// object, optionally applying decoder options prior to decoding. We are not +// using json.Unmarshal directly as we want the chance to pass in non-default +// options. +func jsonUnmarshal(r io.Reader, o interface{}, opts ...JSONOpt) error { + d := json.NewDecoder(r) + for _, opt := range opts { + d = opt(d) + } + if err := d.Decode(&o); err != nil { + return fmt.Errorf("while decoding JSON: %v", err) + } + return nil +} + // Convert JSON to YAML. func JSONToYAML(j []byte) ([]byte, error) { // Convert the JSON to an object. @@ -60,8 +80,8 @@ func JSONToYAML(j []byte) ([]byte, error) { return yaml.Marshal(jsonObj) } -// Convert YAML to JSON. Since JSON is a subset of YAML, passing JSON through -// this method should be a no-op. +// YAMLToJSON converts YAML to JSON. Since JSON is a subset of YAML, +// passing JSON through this method should be a no-op. // // Things YAML can do that are not supported by JSON: // * In YAML you can have binary and null keys in your maps. These are invalid @@ -70,14 +90,22 @@ func JSONToYAML(j []byte) ([]byte, error) { // use binary data with this library, encode the data as base64 as usual but do // not use the !!binary tag in your YAML. This will ensure the original base64 // encoded data makes it all the way through to the JSON. +// +// For strict decoding of YAML, use YAMLToJSONStrict. func YAMLToJSON(y []byte) ([]byte, error) { - return yamlToJSON(y, nil) + return yamlToJSON(y, nil, yaml.Unmarshal) } -func yamlToJSON(y []byte, jsonTarget *reflect.Value) ([]byte, error) { +// YAMLToJSONStrict is like YAMLToJSON but enables strict YAML decoding, +// returning an error on any duplicate field names. +func YAMLToJSONStrict(y []byte) ([]byte, error) { + return yamlToJSON(y, nil, yaml.UnmarshalStrict) +} + +func yamlToJSON(y []byte, jsonTarget *reflect.Value, yamlUnmarshal func([]byte, interface{}) error) ([]byte, error) { // Convert the YAML to an object. var yamlObj interface{} - err := yaml.Unmarshal(y, &yamlObj) + err := yamlUnmarshal(y, &yamlObj) if err != nil { return nil, err } diff --git a/vendor/github.com/ghodss/yaml/yaml_go110.go b/vendor/github.com/ghodss/yaml/yaml_go110.go new file mode 100644 index 00000000..ab3e06a2 --- /dev/null +++ b/vendor/github.com/ghodss/yaml/yaml_go110.go @@ -0,0 +1,14 @@ +// This file contains changes that are only compatible with go 1.10 and onwards. + +// +build go1.10 + +package yaml + +import "encoding/json" + +// DisallowUnknownFields configures the JSON decoder to error out if unknown +// fields come along, instead of dropping them by default. +func DisallowUnknownFields(d *json.Decoder) *json.Decoder { + d.DisallowUnknownFields() + return d +} diff --git a/vendor/github.com/pkg/errors/.travis.yml b/vendor/github.com/pkg/errors/.travis.yml index d4b92663..588ceca1 100644 --- a/vendor/github.com/pkg/errors/.travis.yml +++ b/vendor/github.com/pkg/errors/.travis.yml @@ -1,14 +1,10 @@ language: go go_import_path: github.com/pkg/errors go: - - 1.4.x - - 1.5.x - - 1.6.x - - 1.7.x - - 1.8.x - - 1.9.x - - 1.10.x - - 1.11.x + - 1.4.3 + - 1.5.4 + - 1.6.2 + - 1.7.1 - tip script: diff --git a/vendor/github.com/pkg/errors/README.md b/vendor/github.com/pkg/errors/README.md index 6483ba2a..273db3c9 100644 --- a/vendor/github.com/pkg/errors/README.md +++ b/vendor/github.com/pkg/errors/README.md @@ -1,4 +1,4 @@ -# errors [![Travis-CI](https://travis-ci.org/pkg/errors.svg)](https://travis-ci.org/pkg/errors) [![AppVeyor](https://ci.appveyor.com/api/projects/status/b98mptawhudj53ep/branch/master?svg=true)](https://ci.appveyor.com/project/davecheney/errors/branch/master) [![GoDoc](https://godoc.org/github.com/pkg/errors?status.svg)](http://godoc.org/github.com/pkg/errors) [![Report card](https://goreportcard.com/badge/github.com/pkg/errors)](https://goreportcard.com/report/github.com/pkg/errors) [![Sourcegraph](https://sourcegraph.com/github.com/pkg/errors/-/badge.svg)](https://sourcegraph.com/github.com/pkg/errors?badge) +# errors [![Travis-CI](https://travis-ci.org/pkg/errors.svg)](https://travis-ci.org/pkg/errors) [![AppVeyor](https://ci.appveyor.com/api/projects/status/b98mptawhudj53ep/branch/master?svg=true)](https://ci.appveyor.com/project/davecheney/errors/branch/master) [![GoDoc](https://godoc.org/github.com/pkg/errors?status.svg)](http://godoc.org/github.com/pkg/errors) [![Report card](https://goreportcard.com/badge/github.com/pkg/errors)](https://goreportcard.com/report/github.com/pkg/errors) Package errors provides simple error handling primitives. @@ -47,6 +47,6 @@ We welcome pull requests, bug fixes and issue reports. With that said, the bar f Before proposing a change, please discuss your change by raising an issue. -## License +## Licence BSD-2-Clause diff --git a/vendor/github.com/pkg/errors/errors.go b/vendor/github.com/pkg/errors/errors.go index 7421f326..842ee804 100644 --- a/vendor/github.com/pkg/errors/errors.go +++ b/vendor/github.com/pkg/errors/errors.go @@ -6,7 +6,7 @@ // return err // } // -// which when applied recursively up the call stack results in error reports +// which applied recursively up the call stack results in error reports // without context or debugging information. The errors package allows // programmers to add context to the failure path in their code in a way // that does not destroy the original value of the error. @@ -15,17 +15,16 @@ // // The errors.Wrap function returns a new error that adds context to the // original error by recording a stack trace at the point Wrap is called, -// together with the supplied message. For example +// and the supplied message. For example // // _, err := ioutil.ReadAll(r) // if err != nil { // return errors.Wrap(err, "read failed") // } // -// If additional control is required, the errors.WithStack and -// errors.WithMessage functions destructure errors.Wrap into its component -// operations: annotating an error with a stack trace and with a message, -// respectively. +// If additional control is required the errors.WithStack and errors.WithMessage +// functions destructure errors.Wrap into its component operations of annotating +// an error with a stack trace and an a message, respectively. // // Retrieving the cause of an error // @@ -39,7 +38,7 @@ // } // // can be inspected by errors.Cause. errors.Cause will recursively retrieve -// the topmost error that does not implement causer, which is assumed to be +// the topmost error which does not implement causer, which is assumed to be // the original cause. For example: // // switch err := errors.Cause(err).(type) { @@ -49,16 +48,16 @@ // // unknown error // } // -// Although the causer interface is not exported by this package, it is -// considered a part of its stable public interface. +// causer interface is not exported by this package, but is considered a part +// of stable public API. // // Formatted printing of errors // // All error values returned from this package implement fmt.Formatter and can -// be formatted by the fmt package. The following verbs are supported: +// be formatted by the fmt package. The following verbs are supported // // %s print the error. If the error has a Cause it will be -// printed recursively. +// printed recursively // %v see %s // %+v extended format. Each Frame of the error's StackTrace will // be printed in detail. @@ -66,13 +65,13 @@ // Retrieving the stack trace of an error or wrapper // // New, Errorf, Wrap, and Wrapf record a stack trace at the point they are -// invoked. This information can be retrieved with the following interface: +// invoked. This information can be retrieved with the following interface. // // type stackTracer interface { // StackTrace() errors.StackTrace // } // -// The returned errors.StackTrace type is defined as +// Where errors.StackTrace is defined as // // type StackTrace []Frame // @@ -86,8 +85,8 @@ // } // } // -// Although the stackTracer interface is not exported by this package, it is -// considered a part of its stable public interface. +// stackTracer interface is not exported by this package, but is considered a part +// of stable public API. // // See the documentation for Frame.Format for more details. package errors @@ -193,7 +192,7 @@ func Wrap(err error, message string) error { } // Wrapf returns an error annotating err with a stack trace -// at the point Wrapf is called, and the format specifier. +// at the point Wrapf is call, and the format specifier. // If err is nil, Wrapf returns nil. func Wrapf(err error, format string, args ...interface{}) error { if err == nil { @@ -221,18 +220,6 @@ func WithMessage(err error, message string) error { } } -// WithMessagef annotates err with the format specifier. -// If err is nil, WithMessagef returns nil. -func WithMessagef(err error, format string, args ...interface{}) error { - if err == nil { - return nil - } - return &withMessage{ - cause: err, - msg: fmt.Sprintf(format, args...), - } -} - type withMessage struct { cause error msg string diff --git a/vendor/github.com/pkg/errors/stack.go b/vendor/github.com/pkg/errors/stack.go index 2874a048..6b1f2891 100644 --- a/vendor/github.com/pkg/errors/stack.go +++ b/vendor/github.com/pkg/errors/stack.go @@ -46,8 +46,7 @@ func (f Frame) line() int { // // Format accepts flags that alter the printing of some verbs, as follows: // -// %+s function name and path of source file relative to the compile time -// GOPATH separated by \n\t (\n\t) +// %+s path of source file relative to the compile time GOPATH // %+v equivalent to %+s:%d func (f Frame) Format(s fmt.State, verb rune) { switch verb { @@ -80,14 +79,6 @@ func (f Frame) Format(s fmt.State, verb rune) { // StackTrace is stack of Frames from innermost (newest) to outermost (oldest). type StackTrace []Frame -// Format formats the stack of Frames according to the fmt.Formatter interface. -// -// %s lists source files for each Frame in the stack -// %v lists the source file and line number for each Frame in the stack -// -// Format accepts flags that alter the printing of some verbs, as follows: -// -// %+v Prints filename, function, and line number for each Frame in the stack. func (st StackTrace) Format(s fmt.State, verb rune) { switch verb { case 'v': @@ -145,3 +136,43 @@ func funcname(name string) string { i = strings.Index(name, ".") return name[i+1:] } + +func trimGOPATH(name, file string) string { + // Here we want to get the source file path relative to the compile time + // GOPATH. As of Go 1.6.x there is no direct way to know the compiled + // GOPATH at runtime, but we can infer the number of path segments in the + // GOPATH. We note that fn.Name() returns the function name qualified by + // the import path, which does not include the GOPATH. Thus we can trim + // segments from the beginning of the file path until the number of path + // separators remaining is one more than the number of path separators in + // the function name. For example, given: + // + // GOPATH /home/user + // file /home/user/src/pkg/sub/file.go + // fn.Name() pkg/sub.Type.Method + // + // We want to produce: + // + // pkg/sub/file.go + // + // From this we can easily see that fn.Name() has one less path separator + // than our desired output. We count separators from the end of the file + // path until it finds two more than in the function name and then move + // one character forward to preserve the initial path segment without a + // leading separator. + const sep = "/" + goal := strings.Count(name, sep) + 2 + i := len(file) + for n := 0; n < goal; n++ { + i = strings.LastIndex(file[:i], sep) + if i == -1 { + // not enough separators found, set i so that the slice expression + // below leaves file unmodified + i = -len(sep) + break + } + } + // get back to 0 or trim the leading separator + file = file[i+len(sep):] + return file +} diff --git a/vendor/github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg/autoneg.go b/vendor/github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg/autoneg.go index 26e92288..648b38cb 100644 --- a/vendor/github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg/autoneg.go +++ b/vendor/github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg/autoneg.go @@ -1,12 +1,12 @@ /* -Copyright (c) 2011, Open Knowledge Foundation Ltd. -All rights reserved. - HTTP Content-Type Autonegotiation. The functions in this package implement the behaviour specified in http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html +Copyright (c) 2011, Open Knowledge Foundation Ltd. +All rights reserved. + Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/vendor/github.com/prometheus/common/model/metric.go b/vendor/github.com/prometheus/common/model/metric.go index 00804b7f..f7250909 100644 --- a/vendor/github.com/prometheus/common/model/metric.go +++ b/vendor/github.com/prometheus/common/model/metric.go @@ -21,6 +21,7 @@ import ( ) var ( + separator = []byte{0} // MetricNameRE is a regular expression matching valid metric // names. Note that the IsValidMetricName function performs the same // check but faster than a match with this regular expression. diff --git a/vendor/github.com/prometheus/procfs/MAINTAINERS.md b/vendor/github.com/prometheus/procfs/MAINTAINERS.md index f1d3b993..35993c41 100644 --- a/vendor/github.com/prometheus/procfs/MAINTAINERS.md +++ b/vendor/github.com/prometheus/procfs/MAINTAINERS.md @@ -1,2 +1 @@ -* Tobias Schmidt @grobie -* Johannes 'fish' Ziemke @discordianfish +* Tobias Schmidt diff --git a/vendor/github.com/prometheus/procfs/Makefile b/vendor/github.com/prometheus/procfs/Makefile index 314d1ba5..947d7d8f 100644 --- a/vendor/github.com/prometheus/procfs/Makefile +++ b/vendor/github.com/prometheus/procfs/Makefile @@ -17,12 +17,14 @@ include Makefile.common ./ttar -C $(dir $*) -x -f $*.ttar touch $@ -update_fixtures: - rm -vf fixtures/.unpacked - ./ttar -c -f fixtures.ttar fixtures/ +update_fixtures: fixtures.ttar sysfs/fixtures.ttar + +%fixtures.ttar: %/fixtures + rm -v $(dir $*)fixtures/.unpacked + ./ttar -C $(dir $*) -c -f $*fixtures.ttar fixtures/ .PHONY: build build: .PHONY: test -test: fixtures/.unpacked common-test +test: fixtures/.unpacked sysfs/fixtures/.unpacked common-test diff --git a/vendor/github.com/prometheus/procfs/Makefile.common b/vendor/github.com/prometheus/procfs/Makefile.common index ed29aa82..741579e6 100644 --- a/vendor/github.com/prometheus/procfs/Makefile.common +++ b/vendor/github.com/prometheus/procfs/Makefile.common @@ -29,15 +29,12 @@ GO ?= go GOFMT ?= $(GO)fmt FIRST_GOPATH := $(firstword $(subst :, ,$(shell $(GO) env GOPATH))) GOOPTS ?= -GOHOSTOS ?= $(shell $(GO) env GOHOSTOS) -GOHOSTARCH ?= $(shell $(GO) env GOHOSTARCH) GO_VERSION ?= $(shell $(GO) version) GO_VERSION_NUMBER ?= $(word 3, $(GO_VERSION)) PRE_GO_111 ?= $(shell echo $(GO_VERSION_NUMBER) | grep -E 'go1\.(10|[0-9])\.') -GOVENDOR := -GO111MODULE := +unexport GOVENDOR ifeq (, $(PRE_GO_111)) ifneq (,$(wildcard go.mod)) # Enforce Go modules support just in case the directory is inside GOPATH (and for Travis CI). @@ -58,51 +55,32 @@ $(warning Some recipes may not work as expected as the current Go runtime is '$( # This repository isn't using Go modules (yet). GOVENDOR := $(FIRST_GOPATH)/bin/govendor endif + + unexport GO111MODULE endif PROMU := $(FIRST_GOPATH)/bin/promu +STATICCHECK := $(FIRST_GOPATH)/bin/staticcheck pkgs = ./... -ifeq (arm, $(GOHOSTARCH)) - GOHOSTARM ?= $(shell GOARM= $(GO) env GOARM) - GO_BUILD_PLATFORM ?= $(GOHOSTOS)-$(GOHOSTARCH)v$(GOHOSTARM) -else - GO_BUILD_PLATFORM ?= $(GOHOSTOS)-$(GOHOSTARCH) -endif +GO_VERSION ?= $(shell $(GO) version) +GO_BUILD_PLATFORM ?= $(subst /,-,$(lastword $(GO_VERSION))) -PROMU_VERSION ?= 0.3.0 +PROMU_VERSION ?= 0.2.0 PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz -STATICCHECK := -# staticcheck only supports linux, freebsd, darwin and windows platforms on i386/amd64 -# windows isn't included here because of the path separator being different. -ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux freebsd darwin)) - ifeq ($(GOHOSTARCH),$(filter $(GOHOSTARCH),amd64 i386)) - STATICCHECK := $(FIRST_GOPATH)/bin/staticcheck - STATICCHECK_VERSION ?= 2019.1 - STATICCHECK_URL := https://github.com/dominikh/go-tools/releases/download/$(STATICCHECK_VERSION)/staticcheck_$(GOHOSTOS)_$(GOHOSTARCH) - endif -endif - PREFIX ?= $(shell pwd) BIN_DIR ?= $(shell pwd) DOCKER_IMAGE_TAG ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD)) DOCKER_REPO ?= prom -ifeq ($(GOHOSTARCH),amd64) - ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux freebsd darwin windows)) - # Only supported on amd64 - test-flags := -race - endif -endif +.PHONY: all +all: precheck style staticcheck unused build test # This rule is used to forward a target like "build" to "common-build". This # allows a new "build" target to be defined in a Makefile which includes this # one and override "common-build" without override warnings. %: common-% ; -.PHONY: common-all -common-all: precheck style check_license staticcheck unused build test - .PHONY: common-style common-style: @echo ">> checking code style" @@ -124,15 +102,6 @@ common-check_license: exit 1; \ fi -.PHONY: common-deps -common-deps: - @echo ">> getting dependencies" -ifdef GO111MODULE - GO111MODULE=$(GO111MODULE) $(GO) mod download -else - $(GO) get $(GOOPTS) -t ./... -endif - .PHONY: common-test-short common-test-short: @echo ">> running short tests" @@ -141,12 +110,12 @@ common-test-short: .PHONY: common-test common-test: @echo ">> running all tests" - GO111MODULE=$(GO111MODULE) $(GO) test $(test-flags) $(GOOPTS) $(pkgs) + GO111MODULE=$(GO111MODULE) $(GO) test -race $(GOOPTS) $(pkgs) .PHONY: common-format common-format: @echo ">> formatting code" - GO111MODULE=$(GO111MODULE) $(GO) fmt $(pkgs) + GO111MODULE=$(GO111MODULE) $(GO) fmt $(GOOPTS) $(pkgs) .PHONY: common-vet common-vet: @@ -155,18 +124,12 @@ common-vet: .PHONY: common-staticcheck common-staticcheck: $(STATICCHECK) -ifdef STATICCHECK @echo ">> running staticcheck" - chmod +x $(STATICCHECK) ifdef GO111MODULE -# 'go list' needs to be executed before staticcheck to prepopulate the modules cache. -# Otherwise staticcheck might fail randomly for some reason not yet explained. - GO111MODULE=$(GO111MODULE) $(GO) list -e -compiled -test=true -export=false -deps=true -find=false -tags= -- ./... > /dev/null - GO111MODULE=$(GO111MODULE) $(STATICCHECK) -ignore "$(STATICCHECK_IGNORE)" $(pkgs) + GO111MODULE=$(GO111MODULE) $(STATICCHECK) -ignore "$(STATICCHECK_IGNORE)" -checks "SA*" $(pkgs) else $(STATICCHECK) -ignore "$(STATICCHECK_IGNORE)" $(pkgs) endif -endif .PHONY: common-unused common-unused: $(GOVENDOR) @@ -177,9 +140,8 @@ else ifdef GO111MODULE @echo ">> running check for unused/missing packages in go.mod" GO111MODULE=$(GO111MODULE) $(GO) mod tidy -ifeq (,$(wildcard vendor)) @git diff --exit-code -- go.sum go.mod -else +ifneq (,$(wildcard vendor)) @echo ">> running check for unused packages in vendor/" GO111MODULE=$(GO111MODULE) $(GO) mod vendor @git diff --exit-code -- go.sum go.mod vendor/ @@ -213,21 +175,29 @@ common-docker-tag-latest: promu: $(PROMU) $(PROMU): - $(eval PROMU_TMP := $(shell mktemp -d)) - curl -s -L $(PROMU_URL) | tar -xvzf - -C $(PROMU_TMP) - mkdir -p $(FIRST_GOPATH)/bin - cp $(PROMU_TMP)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM)/promu $(FIRST_GOPATH)/bin/promu - rm -r $(PROMU_TMP) + curl -s -L $(PROMU_URL) | tar -xvz -C /tmp + mkdir -v -p $(FIRST_GOPATH)/bin + cp -v /tmp/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM)/promu $(PROMU) .PHONY: proto proto: @echo ">> generating code from proto files" @./scripts/genproto.sh -ifdef STATICCHECK +.PHONY: $(STATICCHECK) $(STATICCHECK): - mkdir -p $(FIRST_GOPATH)/bin - curl -s -L $(STATICCHECK_URL) > $(STATICCHECK) +ifdef GO111MODULE +# Get staticcheck from a temporary directory to avoid modifying the local go.{mod,sum}. +# See https://github.com/golang/go/issues/27643. +# For now, we are using the next branch of staticcheck because master isn't compatible yet with Go modules. + tmpModule=$$(mktemp -d 2>&1) && \ + mkdir -p $${tmpModule}/staticcheck && \ + cd "$${tmpModule}"/staticcheck && \ + GO111MODULE=on $(GO) mod init example.com/staticcheck && \ + GO111MODULE=on GOOS= GOARCH= $(GO) get -u honnef.co/go/tools/cmd/staticcheck@next && \ + rm -rf $${tmpModule}; +else + GOOS= GOARCH= GO111MODULE=off $(GO) get -u honnef.co/go/tools/cmd/staticcheck endif ifdef GOVENDOR @@ -242,6 +212,7 @@ precheck:: define PRECHECK_COMMAND_template = precheck:: $(1)_precheck + PRECHECK_COMMAND_$(1) ?= $(1) $$(strip $$(PRECHECK_OPTIONS_$(1))) .PHONY: $(1)_precheck $(1)_precheck: diff --git a/vendor/github.com/prometheus/procfs/fixtures.ttar b/vendor/github.com/prometheus/procfs/fixtures.ttar index f7f84ef3..13c831ef 100644 --- a/vendor/github.com/prometheus/procfs/fixtures.ttar +++ b/vendor/github.com/prometheus/procfs/fixtures.ttar @@ -1,48 +1,45 @@ # Archive created by ttar -c -f fixtures.ttar fixtures/ Directory: fixtures -Mode: 775 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/proc Mode: 755 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/proc/26231 +Directory: fixtures/26231 Mode: 755 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26231/cmdline +Path: fixtures/26231/cmdline Lines: 1 vimNULLBYTEtest.goNULLBYTE+10NULLBYTEEOF Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26231/comm +Path: fixtures/26231/comm Lines: 1 vim Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26231/cwd +Path: fixtures/26231/cwd SymlinkTo: /usr/bin # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26231/exe +Path: fixtures/26231/exe SymlinkTo: /usr/bin/vim # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/proc/26231/fd +Directory: fixtures/26231/fd Mode: 755 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26231/fd/0 +Path: fixtures/26231/fd/0 SymlinkTo: ../../symlinktargets/abc # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26231/fd/1 +Path: fixtures/26231/fd/1 SymlinkTo: ../../symlinktargets/def # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26231/fd/10 +Path: fixtures/26231/fd/10 SymlinkTo: ../../symlinktargets/xyz # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26231/fd/2 +Path: fixtures/26231/fd/2 SymlinkTo: ../../symlinktargets/ghi # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26231/fd/3 +Path: fixtures/26231/fd/3 SymlinkTo: ../../symlinktargets/uvw # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26231/io +Path: fixtures/26231/io Lines: 7 rchar: 750339 wchar: 818609 @@ -53,7 +50,7 @@ write_bytes: 2048 cancelled_write_bytes: -1024 Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26231/limits +Path: fixtures/26231/limits Lines: 17 Limit Soft Limit Hard Limit Units Max cpu time unlimited unlimited seconds @@ -74,7 +71,7 @@ Max realtime priority 0 0 Max realtime timeout unlimited unlimited us Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26231/mountstats +Path: fixtures/26231/mountstats Lines: 19 device rootfs mounted on / with fstype rootfs device sysfs mounted on /sys with fstype sysfs @@ -97,10 +94,10 @@ device 192.168.1.1:/srv/test mounted on /mnt/nfs/test with fstype nfs4 statvers= Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/proc/26231/net +Directory: fixtures/26231/net Mode: 755 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26231/net/dev +Path: fixtures/26231/net/dev Lines: 4 Inter-| Receive | Transmit face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed @@ -108,57 +105,57 @@ Inter-| Receive | Transmit eth0: 438 5 0 0 0 0 0 0 648 8 0 0 0 0 0 0 Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/proc/26231/ns +Directory: fixtures/26231/ns Mode: 755 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26231/ns/mnt +Path: fixtures/26231/ns/mnt SymlinkTo: mnt:[4026531840] # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26231/ns/net +Path: fixtures/26231/ns/net SymlinkTo: net:[4026531993] # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26231/root +Path: fixtures/26231/root SymlinkTo: / # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26231/stat +Path: fixtures/26231/stat Lines: 1 26231 (vim) R 5392 7446 5392 34835 7446 4218880 32533 309516 26 82 1677 44 158 99 20 0 1 0 82375 56274944 1981 18446744073709551615 4194304 6294284 140736914091744 140736914087944 139965136429984 0 0 12288 1870679807 0 0 0 17 0 0 0 31 0 0 8391624 8481048 16420864 140736914093252 140736914093279 140736914093279 140736914096107 0 Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/proc/26232 +Directory: fixtures/26232 Mode: 755 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26232/cmdline +Path: fixtures/26232/cmdline Lines: 0 Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26232/comm +Path: fixtures/26232/comm Lines: 1 ata_sff Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26232/cwd +Path: fixtures/26232/cwd SymlinkTo: /does/not/exist # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/proc/26232/fd +Directory: fixtures/26232/fd Mode: 755 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26232/fd/0 +Path: fixtures/26232/fd/0 SymlinkTo: ../../symlinktargets/abc # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26232/fd/1 +Path: fixtures/26232/fd/1 SymlinkTo: ../../symlinktargets/def # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26232/fd/2 +Path: fixtures/26232/fd/2 SymlinkTo: ../../symlinktargets/ghi # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26232/fd/3 +Path: fixtures/26232/fd/3 SymlinkTo: ../../symlinktargets/uvw # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26232/fd/4 +Path: fixtures/26232/fd/4 SymlinkTo: ../../symlinktargets/xyz # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26232/limits +Path: fixtures/26232/limits Lines: 17 Limit Soft Limit Hard Limit Units Max cpu time unlimited unlimited seconds @@ -179,98 +176,71 @@ Max realtime priority 0 0 Max realtime timeout unlimited unlimited us Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26232/root +Path: fixtures/26232/root SymlinkTo: /does/not/exist # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26232/stat +Path: fixtures/26232/stat Lines: 1 33 (ata_sff) S 2 0 0 0 -1 69238880 0 0 0 0 0 0 0 0 0 -20 1 0 5 0 0 18446744073709551615 0 0 0 0 0 0 0 2147483647 0 18446744073709551615 0 0 17 1 0 0 0 0 0 0 0 0 0 0 0 0 0 Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/proc/26233 +Directory: fixtures/26233 Mode: 755 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/26233/cmdline +Path: fixtures/26233/cmdline Lines: 1 com.github.uiautomatorNULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTEEOF Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/proc/584 +Directory: fixtures/584 Mode: 755 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/584/stat +Path: fixtures/584/stat Lines: 2 1020 ((a b ) ( c d) ) R 28378 1020 28378 34842 1020 4218880 286 0 0 0 0 0 0 0 20 0 1 0 10839175 10395648 155 18446744073709551615 4194304 4238788 140736466511168 140736466511168 140609271124624 0 0 0 0 0 0 0 17 5 0 0 0 0 0 6336016 6337300 25579520 140736466515030 140736466515061 140736466515061 140736466518002 0 #!/bin/cat /proc/self/stat Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/buddyinfo +Directory: fixtures/buddyinfo +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: fixtures/buddyinfo/short +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/buddyinfo/short/buddyinfo +Lines: 3 +Node 0, zone +Node 0, zone +Node 0, zone +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: fixtures/buddyinfo/sizemismatch +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/buddyinfo/sizemismatch/buddyinfo +Lines: 3 +Node 0, zone DMA 1 0 1 0 2 1 1 0 1 1 3 +Node 0, zone DMA32 759 572 791 475 194 45 12 0 0 0 0 0 +Node 0, zone Normal 4381 1093 185 1530 567 102 4 0 0 0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: fixtures/buddyinfo/valid +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/buddyinfo/valid/buddyinfo Lines: 3 Node 0, zone DMA 1 0 1 0 2 1 1 0 1 1 3 Node 0, zone DMA32 759 572 791 475 194 45 12 0 0 0 0 Node 0, zone Normal 4381 1093 185 1530 567 102 4 0 0 0 0 Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/diskstats -Lines: 49 - 1 0 ram0 0 0 0 0 0 0 0 0 0 0 0 - 1 1 ram1 0 0 0 0 0 0 0 0 0 0 0 - 1 2 ram2 0 0 0 0 0 0 0 0 0 0 0 - 1 3 ram3 0 0 0 0 0 0 0 0 0 0 0 - 1 4 ram4 0 0 0 0 0 0 0 0 0 0 0 - 1 5 ram5 0 0 0 0 0 0 0 0 0 0 0 - 1 6 ram6 0 0 0 0 0 0 0 0 0 0 0 - 1 7 ram7 0 0 0 0 0 0 0 0 0 0 0 - 1 8 ram8 0 0 0 0 0 0 0 0 0 0 0 - 1 9 ram9 0 0 0 0 0 0 0 0 0 0 0 - 1 10 ram10 0 0 0 0 0 0 0 0 0 0 0 - 1 11 ram11 0 0 0 0 0 0 0 0 0 0 0 - 1 12 ram12 0 0 0 0 0 0 0 0 0 0 0 - 1 13 ram13 0 0 0 0 0 0 0 0 0 0 0 - 1 14 ram14 0 0 0 0 0 0 0 0 0 0 0 - 1 15 ram15 0 0 0 0 0 0 0 0 0 0 0 - 7 0 loop0 0 0 0 0 0 0 0 0 0 0 0 - 7 1 loop1 0 0 0 0 0 0 0 0 0 0 0 - 7 2 loop2 0 0 0 0 0 0 0 0 0 0 0 - 7 3 loop3 0 0 0 0 0 0 0 0 0 0 0 - 7 4 loop4 0 0 0 0 0 0 0 0 0 0 0 - 7 5 loop5 0 0 0 0 0 0 0 0 0 0 0 - 7 6 loop6 0 0 0 0 0 0 0 0 0 0 0 - 7 7 loop7 0 0 0 0 0 0 0 0 0 0 0 - 8 0 sda 25354637 34367663 1003346126 18492372 28444756 11134226 505697032 63877960 0 9653880 82621804 - 8 1 sda1 250 0 2000 36 0 0 0 0 0 36 36 - 8 2 sda2 246 0 1968 32 0 0 0 0 0 32 32 - 8 3 sda3 340 13 2818 52 11 8 152 8 0 56 60 - 8 4 sda4 25353629 34367650 1003337964 18492232 27448755 11134218 505696880 61593380 0 7576432 80332428 - 252 0 dm-0 59910002 0 1003337218 46229572 39231014 0 505696880 1158557800 0 11325968 1206301256 - 252 1 dm-1 388 0 3104 84 74 0 592 0 0 76 84 - 252 2 dm-2 11571 0 308350 6536 153522 0 5093416 122884 0 65400 129416 - 252 3 dm-3 3870 0 3870 104 0 0 0 0 0 16 104 - 252 4 dm-4 392 0 1034 28 38 0 137 16 0 24 44 - 252 5 dm-5 3729 0 84279 924 98918 0 1151688 104684 0 58848 105632 - 179 0 mmcblk0 192 3 1560 156 0 0 0 0 0 136 156 - 179 1 mmcblk0p1 17 3 160 24 0 0 0 0 0 24 24 - 179 2 mmcblk0p2 95 0 760 68 0 0 0 0 0 68 68 - 2 0 fd0 2 0 16 80 0 0 0 0 0 80 80 - 254 0 vda 1775784 15386 32670882 8655768 6038856 20711856 213637440 2069221364 0 41614592 2077872228 - 254 1 vda1 668 85 5984 956 207 4266 35784 32772 0 8808 33720 - 254 2 vda2 1774936 15266 32663262 8654692 5991028 20707590 213601656 2069152216 0 41607628 2077801992 - 11 0 sr0 0 0 0 0 0 0 0 0 0 0 0 - 259 0 nvme0n1 47114 4 4643973 21650 1078320 43950 39451633 1011053 0 222766 1032546 - 259 1 nvme0n1p1 1140 0 9370 16 1 0 1 0 0 16 16 - 259 2 nvme0n1p2 45914 4 4631243 21626 1036885 43950 39451632 919480 0 131580 940970 - 8 0 sdb 326552 841 9657779 84 41822 2895 1972905 5007 0 60730 67070 68851 0 1925173784 11130 - 8 1 sdb1 231 3 34466 4 24 23 106 0 0 64 64 0 0 0 0 - 8 2 sdb2 326310 838 9622281 67 40726 2872 1972799 4924 0 58250 64567 68851 0 1925173784 11130 -Mode: 664 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/proc/fs +Directory: fixtures/fs Mode: 755 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/proc/fs/xfs +Directory: fixtures/fs/xfs Mode: 755 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/fs/xfs/stat +Path: fixtures/fs/xfs/stat Lines: 23 extent_alloc 92447 97589 92448 93751 abt 0 0 0 0 @@ -297,7 +267,7 @@ xpc 399724544 92823103 86219234 debug 0 Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/mdstat +Path: fixtures/mdstat Lines: 26 Personalities : [linear] [multipath] [raid0] [raid1] [raid6] [raid5] [raid4] [raid10] md3 : active raid6 sda1[8] sdh1[7] sdg1[6] sdf1[5] sde1[11] sdd1[3] sdc1[10] sdb1[9] @@ -327,10 +297,10 @@ md7 : active raid6 sdb1[0] sde1[3] sdd1[2] sdc1[1] unused devices: Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/proc/net +Directory: fixtures/net Mode: 755 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/net/dev +Path: fixtures/net/dev Lines: 6 Inter-| Receive | Transmit face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed @@ -340,7 +310,7 @@ docker0: 2568 38 0 0 0 0 0 0 438 eth0: 874354587 1036395 0 0 0 0 0 0 563352563 732147 0 0 0 0 0 0 Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/net/ip_vs +Path: fixtures/net/ip_vs Lines: 21 IP Virtual Server version 1.2.1 (size=4096) Prot LocalAddress:Port Scheduler Flags @@ -365,7 +335,7 @@ FWM 10001000 wlc -> C0A83215:0CEA Route 0 0 2 Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/net/ip_vs_stats +Path: fixtures/net/ip_vs_stats Lines: 6 Total Incoming Outgoing Incoming Outgoing Conns Packets Packets Bytes Bytes @@ -375,10 +345,10 @@ Lines: 6 4 1FB3C 0 1282A8F 0 Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/proc/net/rpc +Directory: fixtures/net/rpc Mode: 755 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/net/rpc/nfs +Path: fixtures/net/rpc/nfs Lines: 5 net 18628 0 18628 6 rpc 4329785 0 4338291 @@ -387,7 +357,7 @@ proc3 22 1 4084749 29200 94754 32580 186 47747 7981 8639 0 6356 0 6962 0 7958 0 proc4 61 1 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/net/rpc/nfsd +Path: fixtures/net/rpc/nfsd Lines: 11 rc 0 6 18622 fh 0 0 0 0 0 @@ -402,7 +372,7 @@ proc4 2 2 10853 proc4ops 72 0 0 0 1098 2 0 0 0 0 8179 5896 0 0 0 0 5900 0 0 2 0 2 0 9609 0 2 150 1272 0 0 0 1236 0 0 0 0 3 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/net/xfrm_stat +Path: fixtures/net/xfrm_stat Lines: 28 XfrmInError 1 XfrmInBufferError 2 @@ -434,30 +404,10 @@ XfrmOutStateInvalid 28765 XfrmAcquireError 24532 Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/proc/pressure -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/pressure/cpu -Lines: 1 -some avg10=0.10 avg60=2.00 avg300=3.85 total=15 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/pressure/io -Lines: 2 -some avg10=0.10 avg60=2.00 avg300=3.85 total=15 -full avg10=0.20 avg60=3.00 avg300=4.95 total=25 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/pressure/memory -Lines: 2 -some avg10=0.10 avg60=2.00 avg300=3.85 total=15 -full avg10=0.20 avg60=3.00 avg300=4.95 total=25 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/self +Path: fixtures/self SymlinkTo: 26231 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/stat +Path: fixtures/stat Lines: 16 cpu 301854 612 111922 8979004 3552 2 3944 0 0 0 cpu0 44490 19 21045 1087069 220 1 3410 0 0 0 @@ -477,1238 +427,36 @@ procs_blocked 1 softirq 5057579 250191 1481983 1647 211099 186066 0 1783454 622196 12499 508444 Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/proc/symlinktargets +Directory: fixtures/symlinktargets Mode: 755 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/symlinktargets/README +Path: fixtures/symlinktargets/README Lines: 2 This directory contains some empty files that are the symlinks the files in the "fd" directory point to. They are otherwise ignored by the tests Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/symlinktargets/abc +Path: fixtures/symlinktargets/abc Lines: 0 Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/symlinktargets/def +Path: fixtures/symlinktargets/def Lines: 0 Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/symlinktargets/ghi +Path: fixtures/symlinktargets/ghi Lines: 0 Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/symlinktargets/uvw +Path: fixtures/symlinktargets/uvw Lines: 0 Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/proc/symlinktargets/xyz +Path: fixtures/symlinktargets/xyz Lines: 0 Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/block -Mode: 775 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/block/dm-0 -Mode: 775 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/block/dm-0/stat -Lines: 1 -6447303 0 710266738 1529043 953216 0 31201176 4557464 0 796160 6088971 -Mode: 664 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/block/sda -Mode: 775 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/block/sda/stat -Lines: 1 -9652963 396792 759304206 412943 8422549 6731723 286915323 13947418 0 5658367 19174573 1 2 3 12 -Mode: 664 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/class -Mode: 775 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/class/net -Mode: 775 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/class/net/eth0 -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/net/eth0/addr_assign_type -Lines: 1 -3 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/net/eth0/addr_len -Lines: 1 -6 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/net/eth0/address -Lines: 1 -01:01:01:01:01:01 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/net/eth0/broadcast -Lines: 1 -ff:ff:ff:ff:ff:ff -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/net/eth0/carrier -Lines: 1 -1 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/net/eth0/carrier_changes -Lines: 1 -2 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/net/eth0/carrier_down_count -Lines: 1 -1 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/net/eth0/carrier_up_count -Lines: 1 -1 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/net/eth0/dev_id -Lines: 1 -0x20 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/net/eth0/dormant -Lines: 1 -1 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/net/eth0/duplex -Lines: 1 -full -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/net/eth0/flags -Lines: 1 -0x1303 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/net/eth0/ifalias +Path: fixtures/.unpacked Lines: 0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/net/eth0/ifindex -Lines: 1 -2 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/net/eth0/iflink -Lines: 1 -2 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/net/eth0/link_mode -Lines: 1 -1 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/net/eth0/mtu -Lines: 1 -1500 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/net/eth0/name_assign_type -Lines: 1 -2 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/net/eth0/netdev_group -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/net/eth0/operstate -Lines: 1 -up -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/net/eth0/phys_port_id -Lines: 0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/net/eth0/phys_port_name -Lines: 0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/net/eth0/phys_switch_id -Lines: 0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/net/eth0/speed -Lines: 1 -1000 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/net/eth0/tx_queue_len -Lines: 1 -1000 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/net/eth0/type -Lines: 1 -1 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/class/power_supply -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/class/power_supply/AC -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/power_supply/AC/online -Lines: 1 -0 -Mode: 444 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/power_supply/AC/type -Lines: 1 -Mains -Mode: 444 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/power_supply/AC/uevent -Lines: 2 -POWER_SUPPLY_NAME=AC -POWER_SUPPLY_ONLINE=0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/class/power_supply/BAT0 -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/power_supply/BAT0/alarm -Lines: 1 -2503000 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/power_supply/BAT0/capacity -Lines: 1 -98 -Mode: 444 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/power_supply/BAT0/capacity_level -Lines: 1 -Normal -Mode: 444 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/power_supply/BAT0/charge_start_threshold -Lines: 1 -95 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/power_supply/BAT0/charge_stop_threshold -Lines: 1 -100 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/power_supply/BAT0/cycle_count -Lines: 1 -0 -Mode: 444 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/power_supply/BAT0/energy_full -Lines: 1 -50060000 -Mode: 444 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/power_supply/BAT0/energy_full_design -Lines: 1 -47520000 -Mode: 444 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/power_supply/BAT0/energy_now -Lines: 1 -49450000 -Mode: 444 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/power_supply/BAT0/manufacturer -Lines: 1 -LGC -Mode: 444 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/power_supply/BAT0/model_name -Lines: 1 -LNV-45N1 -Mode: 444 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/power_supply/BAT0/power_now -Lines: 1 -4830000 -Mode: 444 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/power_supply/BAT0/present -Lines: 1 -1 -Mode: 444 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/power_supply/BAT0/serial_number -Lines: 1 -38109 -Mode: 444 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/power_supply/BAT0/status -Lines: 1 -Discharging -Mode: 444 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/power_supply/BAT0/technology -Lines: 1 -Li-ion -Mode: 444 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/power_supply/BAT0/type -Lines: 1 -Battery -Mode: 444 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/power_supply/BAT0/uevent -Lines: 16 -POWER_SUPPLY_NAME=BAT0 -POWER_SUPPLY_STATUS=Discharging -POWER_SUPPLY_PRESENT=1 -POWER_SUPPLY_TECHNOLOGY=Li-ion -POWER_SUPPLY_CYCLE_COUNT=0 -POWER_SUPPLY_VOLTAGE_MIN_DESIGN=10800000 -POWER_SUPPLY_VOLTAGE_NOW=12229000 -POWER_SUPPLY_POWER_NOW=4830000 -POWER_SUPPLY_ENERGY_FULL_DESIGN=47520000 -POWER_SUPPLY_ENERGY_FULL=50060000 -POWER_SUPPLY_ENERGY_NOW=49450000 -POWER_SUPPLY_CAPACITY=98 -POWER_SUPPLY_CAPACITY_LEVEL=Normal -POWER_SUPPLY_MODEL_NAME=LNV-45N1 -POWER_SUPPLY_MANUFACTURER=LGC -POWER_SUPPLY_SERIAL_NUMBER=38109 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/power_supply/BAT0/voltage_min_design -Lines: 1 -10800000 -Mode: 444 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/power_supply/BAT0/voltage_now -Lines: 1 -12229000 -Mode: 444 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/class/thermal -Mode: 775 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/class/thermal/thermal_zone0 -Mode: 775 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/thermal/thermal_zone0/policy -Lines: 1 -step_wise Mode: 664 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/thermal/thermal_zone0/temp -Lines: 1 -49925 -Mode: 664 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/thermal/thermal_zone0/type -Lines: 1 -bcm2835_thermal -Mode: 664 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/class/thermal/thermal_zone1 -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/thermal/thermal_zone1/mode -Lines: 1 -enabled -Mode: 664 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/thermal/thermal_zone1/passive -Lines: 1 -0 -Mode: 664 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/thermal/thermal_zone1/policy -Lines: 1 -step_wise -Mode: 664 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/thermal/thermal_zone1/temp -Lines: 1 -44000 -Mode: 664 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/class/thermal/thermal_zone1/type -Lines: 1 -acpitz -Mode: 664 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices/pci0000:00 -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices/pci0000:00/0000:00:0d.0 -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4 -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3 -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0 -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0 -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/dirty_data -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_day -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_day/bypassed -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_day/cache_bypass_hits -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_day/cache_bypass_misses -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_day/cache_hit_ratio -Lines: 1 -100 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_day/cache_hits -Lines: 1 -289 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_day/cache_miss_collisions -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_day/cache_misses -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_day/cache_readaheads -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_five_minute -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_five_minute/bypassed -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_five_minute/cache_bypass_hits -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_five_minute/cache_bypass_misses -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_five_minute/cache_hit_ratio -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_five_minute/cache_hits -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_five_minute/cache_miss_collisions -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_five_minute/cache_misses -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_five_minute/cache_readaheads -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_hour -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_hour/bypassed -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_hour/cache_bypass_hits -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_hour/cache_bypass_misses -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_hour/cache_hit_ratio -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_hour/cache_hits -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_hour/cache_miss_collisions -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_hour/cache_misses -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_hour/cache_readaheads -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_total -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_total/bypassed -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_total/cache_bypass_hits -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_total/cache_bypass_misses -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_total/cache_hit_ratio -Lines: 1 -100 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_total/cache_hits -Lines: 1 -546 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_total/cache_miss_collisions -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_total/cache_misses -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata4/host3/target3:0:0/3:0:0:0/block/sdb/bcache/stats_total/cache_readaheads -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata5 -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata5/host4 -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata5/host4/target4:0:0 -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata5/host4/target4:0:0/4:0:0:0 -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata5/host4/target4:0:0/4:0:0:0/block -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata5/host4/target4:0:0/4:0:0:0/block/sdc -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata5/host4/target4:0:0/4:0:0:0/block/sdc/bcache -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata5/host4/target4:0:0/4:0:0:0/block/sdc/bcache/io_errors -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata5/host4/target4:0:0/4:0:0:0/block/sdc/bcache/metadata_written -Lines: 1 -512 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata5/host4/target4:0:0/4:0:0:0/block/sdc/bcache/priority_stats -Lines: 5 -Unused: 99% -Metadata: 0% -Average: 10473 -Sectors per Q: 64 -Quantiles: [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 20946 20946 20946 20946 20946 20946 20946 20946 20946 20946 20946 20946 20946 20946 20946 20946] -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/pci0000:00/0000:00:0d.0/ata5/host4/target4:0:0/4:0:0:0/block/sdc/bcache/written -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices/system -Mode: 775 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices/system/cpu -Mode: 775 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices/system/cpu/cpu0 -Mode: 775 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/system/cpu/cpu0/cpufreq -SymlinkTo: ../cpufreq/policy0 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices/system/cpu/cpu1 -Mode: 775 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices/system/cpu/cpu1/cpufreq -Mode: 775 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/system/cpu/cpu1/cpufreq/cpuinfo_cur_freq -Lines: 1 -1200195 -Mode: 400 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/system/cpu/cpu1/cpufreq/cpuinfo_max_freq -Lines: 1 -3300000 -Mode: 664 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/system/cpu/cpu1/cpufreq/cpuinfo_min_freq -Lines: 1 -1200000 -Mode: 664 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/system/cpu/cpu1/cpufreq/cpuinfo_transition_latency -Lines: 1 -4294967295 -Mode: 664 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/system/cpu/cpu1/cpufreq/related_cpus -Lines: 1 -1 -Mode: 664 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/system/cpu/cpu1/cpufreq/scaling_available_governors -Lines: 1 -performance powersave -Mode: 664 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/system/cpu/cpu1/cpufreq/scaling_driver -Lines: 1 -intel_pstate -Mode: 664 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/system/cpu/cpu1/cpufreq/scaling_governor -Lines: 1 -powersave -Mode: 664 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/system/cpu/cpu1/cpufreq/scaling_max_freq -Lines: 1 -3300000 -Mode: 664 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/system/cpu/cpu1/cpufreq/scaling_min_freq -Lines: 1 -1200000 -Mode: 664 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/system/cpu/cpu1/cpufreq/scaling_setspeed -Lines: 1 - -Mode: 664 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices/system/cpu/cpufreq -Mode: 775 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices/system/cpu/cpufreq/policy0 -Mode: 775 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/system/cpu/cpufreq/policy0/affected_cpus -Lines: 1 -0 -Mode: 444 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/system/cpu/cpufreq/policy0/cpuinfo_max_freq -Lines: 1 -2400000 -Mode: 444 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/system/cpu/cpufreq/policy0/cpuinfo_min_freq -Lines: 1 -800000 -Mode: 444 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/system/cpu/cpufreq/policy0/cpuinfo_transition_latency -Lines: 1 -0 -Mode: 444 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/system/cpu/cpufreq/policy0/related_cpus -Lines: 1 -0 -Mode: 444 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/system/cpu/cpufreq/policy0/scaling_available_governors -Lines: 1 -performance powersave -Mode: 444 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/system/cpu/cpufreq/policy0/scaling_cur_freq -Lines: 1 -1219917 -Mode: 444 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/system/cpu/cpufreq/policy0/scaling_driver -Lines: 1 -intel_pstate -Mode: 444 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/system/cpu/cpufreq/policy0/scaling_governor -Lines: 1 -powersave -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/system/cpu/cpufreq/policy0/scaling_max_freq -Lines: 1 -2400000 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/system/cpu/cpufreq/policy0/scaling_min_freq -Lines: 1 -800000 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/devices/system/cpu/cpufreq/policy0/scaling_setspeed -Lines: 1 - -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/devices/system/cpu/cpufreq/policy1 -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/fs -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/fs/bcache -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74 -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/average_key_size -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0 -Mode: 777 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/dirty_data -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_day -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_day/bypassed -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_day/cache_bypass_hits -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_day/cache_bypass_misses -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_day/cache_hit_ratio -Lines: 1 -100 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_day/cache_hits -Lines: 1 -289 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_day/cache_miss_collisions -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_day/cache_misses -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_day/cache_readaheads -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_five_minute -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_five_minute/bypassed -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_five_minute/cache_bypass_hits -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_five_minute/cache_bypass_misses -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_five_minute/cache_hit_ratio -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_five_minute/cache_hits -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_five_minute/cache_miss_collisions -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_five_minute/cache_misses -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_five_minute/cache_readaheads -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_hour -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_hour/bypassed -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_hour/cache_bypass_hits -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_hour/cache_bypass_misses -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_hour/cache_hit_ratio -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_hour/cache_hits -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_hour/cache_miss_collisions -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_hour/cache_misses -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_hour/cache_readaheads -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_total -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_total/bypassed -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_total/cache_bypass_hits -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_total/cache_bypass_misses -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_total/cache_hit_ratio -Lines: 1 -100 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_total/cache_hits -Lines: 1 -546 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_total/cache_miss_collisions -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_total/cache_misses -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/bdev0/stats_total/cache_readaheads -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/btree_cache_size -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/cache0 -Mode: 777 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/cache0/io_errors -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/cache0/metadata_written -Lines: 1 -512 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/cache0/priority_stats -Lines: 5 -Unused: 99% -Metadata: 0% -Average: 10473 -Sectors per Q: 64 -Quantiles: [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 20946 20946 20946 20946 20946 20946 20946 20946 20946 20946 20946 20946 20946 20946 20946 20946] -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/cache0/written -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/cache_available_percent -Lines: 1 -100 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/congested -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/internal -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/internal/active_journal_entries -Lines: 1 -1 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/internal/btree_nodes -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/internal/btree_read_average_duration_us -Lines: 1 -1305 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/internal/cache_read_races -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/root_usage_percent -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_day -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_day/bypassed -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_day/cache_bypass_hits -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_day/cache_bypass_misses -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_day/cache_hit_ratio -Lines: 1 -100 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_day/cache_hits -Lines: 1 -289 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_day/cache_miss_collisions -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_day/cache_misses -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_day/cache_readaheads -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_five_minute -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_five_minute/bypassed -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_five_minute/cache_bypass_hits -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_five_minute/cache_bypass_misses -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_five_minute/cache_hit_ratio -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_five_minute/cache_hits -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_five_minute/cache_miss_collisions -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_five_minute/cache_misses -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_five_minute/cache_readaheads -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_hour -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_hour/bypassed -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_hour/cache_bypass_hits -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_hour/cache_bypass_misses -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_hour/cache_hit_ratio -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_hour/cache_hits -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_hour/cache_miss_collisions -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_hour/cache_misses -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_hour/cache_readaheads -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_total -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_total/bypassed -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_total/cache_bypass_hits -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_total/cache_bypass_misses -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_total/cache_hit_ratio -Lines: 1 -100 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_total/cache_hits -Lines: 1 -546 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_total/cache_miss_collisions -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_total/cache_misses -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/stats_total/cache_readaheads -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/bcache/deaddd54-c735-46d5-868e-f331c5fd7c74/tree_depth -Lines: 1 -0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/fs/xfs -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/fs/xfs/sda1 -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/fs/xfs/sda1/stats -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/xfs/sda1/stats/stats -Lines: 1 -extent_alloc 1 0 0 0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/fs/xfs/sdb1 -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directory: fixtures/sys/fs/xfs/sdb1/stats -Mode: 755 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Path: fixtures/sys/fs/xfs/sdb1/stats/stats -Lines: 1 -extent_alloc 2 0 0 0 -Mode: 644 -# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/vendor/github.com/prometheus/procfs/fs.go b/vendor/github.com/prometheus/procfs/fs.go index f7a151cc..b6c6b2ce 100644 --- a/vendor/github.com/prometheus/procfs/fs.go +++ b/vendor/github.com/prometheus/procfs/fs.go @@ -17,6 +17,9 @@ import ( "fmt" "os" "path" + + "github.com/prometheus/procfs/nfs" + "github.com/prometheus/procfs/xfs" ) // FS represents the pseudo-filesystem proc, which provides an interface to @@ -44,3 +47,36 @@ func NewFS(mountPoint string) (FS, error) { func (fs FS) Path(p ...string) string { return path.Join(append([]string{string(fs)}, p...)...) } + +// XFSStats retrieves XFS filesystem runtime statistics. +func (fs FS) XFSStats() (*xfs.Stats, error) { + f, err := os.Open(fs.Path("fs/xfs/stat")) + if err != nil { + return nil, err + } + defer f.Close() + + return xfs.ParseStats(f) +} + +// NFSClientRPCStats retrieves NFS client RPC statistics. +func (fs FS) NFSClientRPCStats() (*nfs.ClientRPCStats, error) { + f, err := os.Open(fs.Path("net/rpc/nfs")) + if err != nil { + return nil, err + } + defer f.Close() + + return nfs.ParseClientRPCStats(f) +} + +// NFSdServerRPCStats retrieves NFS daemon RPC statistics. +func (fs FS) NFSdServerRPCStats() (*nfs.ServerRPCStats, error) { + f, err := os.Open(fs.Path("net/rpc/nfsd")) + if err != nil { + return nil, err + } + defer f.Close() + + return nfs.ParseServerRPCStats(f) +} diff --git a/vendor/github.com/prometheus/procfs/go.mod b/vendor/github.com/prometheus/procfs/go.mod index 8a1b839f..e89ee6c9 100644 --- a/vendor/github.com/prometheus/procfs/go.mod +++ b/vendor/github.com/prometheus/procfs/go.mod @@ -1,3 +1 @@ module github.com/prometheus/procfs - -require golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 diff --git a/vendor/github.com/prometheus/procfs/go.sum b/vendor/github.com/prometheus/procfs/go.sum deleted file mode 100644 index 7827dd3d..00000000 --- a/vendor/github.com/prometheus/procfs/go.sum +++ /dev/null @@ -1,2 +0,0 @@ -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/vendor/github.com/prometheus/procfs/internal/util/parse.go b/vendor/github.com/prometheus/procfs/internal/util/parse.go new file mode 100644 index 00000000..2ff228e9 --- /dev/null +++ b/vendor/github.com/prometheus/procfs/internal/util/parse.go @@ -0,0 +1,59 @@ +// Copyright 2018 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "io/ioutil" + "strconv" + "strings" +) + +// ParseUint32s parses a slice of strings into a slice of uint32s. +func ParseUint32s(ss []string) ([]uint32, error) { + us := make([]uint32, 0, len(ss)) + for _, s := range ss { + u, err := strconv.ParseUint(s, 10, 32) + if err != nil { + return nil, err + } + + us = append(us, uint32(u)) + } + + return us, nil +} + +// ParseUint64s parses a slice of strings into a slice of uint64s. +func ParseUint64s(ss []string) ([]uint64, error) { + us := make([]uint64, 0, len(ss)) + for _, s := range ss { + u, err := strconv.ParseUint(s, 10, 64) + if err != nil { + return nil, err + } + + us = append(us, u) + } + + return us, nil +} + +// ReadUintFromFile reads a file and attempts to parse a uint64 from it. +func ReadUintFromFile(path string) (uint64, error) { + data, err := ioutil.ReadFile(path) + if err != nil { + return 0, err + } + return strconv.ParseUint(strings.TrimSpace(string(data)), 10, 64) +} diff --git a/vendor/github.com/prometheus/procfs/internal/util/sysreadfile_linux.go b/vendor/github.com/prometheus/procfs/internal/util/sysreadfile_linux.go new file mode 100644 index 00000000..df0d567b --- /dev/null +++ b/vendor/github.com/prometheus/procfs/internal/util/sysreadfile_linux.go @@ -0,0 +1,45 @@ +// Copyright 2018 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !windows + +package util + +import ( + "bytes" + "os" + "syscall" +) + +// SysReadFile is a simplified ioutil.ReadFile that invokes syscall.Read directly. +// https://github.com/prometheus/node_exporter/pull/728/files +func SysReadFile(file string) (string, error) { + f, err := os.Open(file) + if err != nil { + return "", err + } + defer f.Close() + + // On some machines, hwmon drivers are broken and return EAGAIN. This causes + // Go's ioutil.ReadFile implementation to poll forever. + // + // Since we either want to read data or bail immediately, do the simplest + // possible read using syscall directly. + b := make([]byte, 128) + n, err := syscall.Read(int(f.Fd()), b) + if err != nil { + return "", err + } + + return string(bytes.TrimSpace(b[:n])), nil +} diff --git a/vendor/github.com/prometheus/procfs/mountstats.go b/vendor/github.com/prometheus/procfs/mountstats.go index fc385afc..7a8a1e09 100644 --- a/vendor/github.com/prometheus/procfs/mountstats.go +++ b/vendor/github.com/prometheus/procfs/mountstats.go @@ -69,8 +69,6 @@ type MountStats interface { type MountStatsNFS struct { // The version of statistics provided. StatVersion string - // The optional mountaddr of the NFS mount. - MountAddress string // The age of the NFS mount. Age time.Duration // Statistics related to byte counters for various operations. @@ -319,7 +317,6 @@ func parseMount(ss []string) (*Mount, error) { func parseMountStatsNFS(s *bufio.Scanner, statVersion string) (*MountStatsNFS, error) { // Field indicators for parsing specific types of data const ( - fieldOpts = "opts:" fieldAge = "age:" fieldBytes = "bytes:" fieldEvents = "events:" @@ -341,13 +338,6 @@ func parseMountStatsNFS(s *bufio.Scanner, statVersion string) (*MountStatsNFS, e } switch ss[0] { - case fieldOpts: - for _, opt := range strings.Split(ss[1], ",") { - split := strings.Split(opt, "=") - if len(split) == 2 && split[0] == "mountaddr" { - stats.MountAddress = split[1] - } - } case fieldAge: // Age integer is in seconds d, err := time.ParseDuration(ss[1] + "s") diff --git a/vendor/github.com/prometheus/procfs/nfs/nfs.go b/vendor/github.com/prometheus/procfs/nfs/nfs.go new file mode 100644 index 00000000..651bf681 --- /dev/null +++ b/vendor/github.com/prometheus/procfs/nfs/nfs.go @@ -0,0 +1,263 @@ +// Copyright 2018 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package nfs implements parsing of /proc/net/rpc/nfsd. +// Fields are documented in https://www.svennd.be/nfsd-stats-explained-procnetrpcnfsd/ +package nfs + +// ReplyCache models the "rc" line. +type ReplyCache struct { + Hits uint64 + Misses uint64 + NoCache uint64 +} + +// FileHandles models the "fh" line. +type FileHandles struct { + Stale uint64 + TotalLookups uint64 + AnonLookups uint64 + DirNoCache uint64 + NoDirNoCache uint64 +} + +// InputOutput models the "io" line. +type InputOutput struct { + Read uint64 + Write uint64 +} + +// Threads models the "th" line. +type Threads struct { + Threads uint64 + FullCnt uint64 +} + +// ReadAheadCache models the "ra" line. +type ReadAheadCache struct { + CacheSize uint64 + CacheHistogram []uint64 + NotFound uint64 +} + +// Network models the "net" line. +type Network struct { + NetCount uint64 + UDPCount uint64 + TCPCount uint64 + TCPConnect uint64 +} + +// ClientRPC models the nfs "rpc" line. +type ClientRPC struct { + RPCCount uint64 + Retransmissions uint64 + AuthRefreshes uint64 +} + +// ServerRPC models the nfsd "rpc" line. +type ServerRPC struct { + RPCCount uint64 + BadCnt uint64 + BadFmt uint64 + BadAuth uint64 + BadcInt uint64 +} + +// V2Stats models the "proc2" line. +type V2Stats struct { + Null uint64 + GetAttr uint64 + SetAttr uint64 + Root uint64 + Lookup uint64 + ReadLink uint64 + Read uint64 + WrCache uint64 + Write uint64 + Create uint64 + Remove uint64 + Rename uint64 + Link uint64 + SymLink uint64 + MkDir uint64 + RmDir uint64 + ReadDir uint64 + FsStat uint64 +} + +// V3Stats models the "proc3" line. +type V3Stats struct { + Null uint64 + GetAttr uint64 + SetAttr uint64 + Lookup uint64 + Access uint64 + ReadLink uint64 + Read uint64 + Write uint64 + Create uint64 + MkDir uint64 + SymLink uint64 + MkNod uint64 + Remove uint64 + RmDir uint64 + Rename uint64 + Link uint64 + ReadDir uint64 + ReadDirPlus uint64 + FsStat uint64 + FsInfo uint64 + PathConf uint64 + Commit uint64 +} + +// ClientV4Stats models the nfs "proc4" line. +type ClientV4Stats struct { + Null uint64 + Read uint64 + Write uint64 + Commit uint64 + Open uint64 + OpenConfirm uint64 + OpenNoattr uint64 + OpenDowngrade uint64 + Close uint64 + Setattr uint64 + FsInfo uint64 + Renew uint64 + SetClientID uint64 + SetClientIDConfirm uint64 + Lock uint64 + Lockt uint64 + Locku uint64 + Access uint64 + Getattr uint64 + Lookup uint64 + LookupRoot uint64 + Remove uint64 + Rename uint64 + Link uint64 + Symlink uint64 + Create uint64 + Pathconf uint64 + StatFs uint64 + ReadLink uint64 + ReadDir uint64 + ServerCaps uint64 + DelegReturn uint64 + GetACL uint64 + SetACL uint64 + FsLocations uint64 + ReleaseLockowner uint64 + Secinfo uint64 + FsidPresent uint64 + ExchangeID uint64 + CreateSession uint64 + DestroySession uint64 + Sequence uint64 + GetLeaseTime uint64 + ReclaimComplete uint64 + LayoutGet uint64 + GetDeviceInfo uint64 + LayoutCommit uint64 + LayoutReturn uint64 + SecinfoNoName uint64 + TestStateID uint64 + FreeStateID uint64 + GetDeviceList uint64 + BindConnToSession uint64 + DestroyClientID uint64 + Seek uint64 + Allocate uint64 + DeAllocate uint64 + LayoutStats uint64 + Clone uint64 +} + +// ServerV4Stats models the nfsd "proc4" line. +type ServerV4Stats struct { + Null uint64 + Compound uint64 +} + +// V4Ops models the "proc4ops" line: NFSv4 operations +// Variable list, see: +// v4.0 https://tools.ietf.org/html/rfc3010 (38 operations) +// v4.1 https://tools.ietf.org/html/rfc5661 (58 operations) +// v4.2 https://tools.ietf.org/html/draft-ietf-nfsv4-minorversion2-41 (71 operations) +type V4Ops struct { + //Values uint64 // Variable depending on v4.x sub-version. TODO: Will this always at least include the fields in this struct? + Op0Unused uint64 + Op1Unused uint64 + Op2Future uint64 + Access uint64 + Close uint64 + Commit uint64 + Create uint64 + DelegPurge uint64 + DelegReturn uint64 + GetAttr uint64 + GetFH uint64 + Link uint64 + Lock uint64 + Lockt uint64 + Locku uint64 + Lookup uint64 + LookupRoot uint64 + Nverify uint64 + Open uint64 + OpenAttr uint64 + OpenConfirm uint64 + OpenDgrd uint64 + PutFH uint64 + PutPubFH uint64 + PutRootFH uint64 + Read uint64 + ReadDir uint64 + ReadLink uint64 + Remove uint64 + Rename uint64 + Renew uint64 + RestoreFH uint64 + SaveFH uint64 + SecInfo uint64 + SetAttr uint64 + Verify uint64 + Write uint64 + RelLockOwner uint64 +} + +// ClientRPCStats models all stats from /proc/net/rpc/nfs. +type ClientRPCStats struct { + Network Network + ClientRPC ClientRPC + V2Stats V2Stats + V3Stats V3Stats + ClientV4Stats ClientV4Stats +} + +// ServerRPCStats models all stats from /proc/net/rpc/nfsd. +type ServerRPCStats struct { + ReplyCache ReplyCache + FileHandles FileHandles + InputOutput InputOutput + Threads Threads + ReadAheadCache ReadAheadCache + Network Network + ServerRPC ServerRPC + V2Stats V2Stats + V3Stats V3Stats + ServerV4Stats ServerV4Stats + V4Ops V4Ops +} diff --git a/vendor/github.com/prometheus/procfs/nfs/parse.go b/vendor/github.com/prometheus/procfs/nfs/parse.go new file mode 100644 index 00000000..95a83cc5 --- /dev/null +++ b/vendor/github.com/prometheus/procfs/nfs/parse.go @@ -0,0 +1,317 @@ +// Copyright 2018 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nfs + +import ( + "fmt" +) + +func parseReplyCache(v []uint64) (ReplyCache, error) { + if len(v) != 3 { + return ReplyCache{}, fmt.Errorf("invalid ReplyCache line %q", v) + } + + return ReplyCache{ + Hits: v[0], + Misses: v[1], + NoCache: v[2], + }, nil +} + +func parseFileHandles(v []uint64) (FileHandles, error) { + if len(v) != 5 { + return FileHandles{}, fmt.Errorf("invalid FileHandles, line %q", v) + } + + return FileHandles{ + Stale: v[0], + TotalLookups: v[1], + AnonLookups: v[2], + DirNoCache: v[3], + NoDirNoCache: v[4], + }, nil +} + +func parseInputOutput(v []uint64) (InputOutput, error) { + if len(v) != 2 { + return InputOutput{}, fmt.Errorf("invalid InputOutput line %q", v) + } + + return InputOutput{ + Read: v[0], + Write: v[1], + }, nil +} + +func parseThreads(v []uint64) (Threads, error) { + if len(v) != 2 { + return Threads{}, fmt.Errorf("invalid Threads line %q", v) + } + + return Threads{ + Threads: v[0], + FullCnt: v[1], + }, nil +} + +func parseReadAheadCache(v []uint64) (ReadAheadCache, error) { + if len(v) != 12 { + return ReadAheadCache{}, fmt.Errorf("invalid ReadAheadCache line %q", v) + } + + return ReadAheadCache{ + CacheSize: v[0], + CacheHistogram: v[1:11], + NotFound: v[11], + }, nil +} + +func parseNetwork(v []uint64) (Network, error) { + if len(v) != 4 { + return Network{}, fmt.Errorf("invalid Network line %q", v) + } + + return Network{ + NetCount: v[0], + UDPCount: v[1], + TCPCount: v[2], + TCPConnect: v[3], + }, nil +} + +func parseServerRPC(v []uint64) (ServerRPC, error) { + if len(v) != 5 { + return ServerRPC{}, fmt.Errorf("invalid RPC line %q", v) + } + + return ServerRPC{ + RPCCount: v[0], + BadCnt: v[1], + BadFmt: v[2], + BadAuth: v[3], + BadcInt: v[4], + }, nil +} + +func parseClientRPC(v []uint64) (ClientRPC, error) { + if len(v) != 3 { + return ClientRPC{}, fmt.Errorf("invalid RPC line %q", v) + } + + return ClientRPC{ + RPCCount: v[0], + Retransmissions: v[1], + AuthRefreshes: v[2], + }, nil +} + +func parseV2Stats(v []uint64) (V2Stats, error) { + values := int(v[0]) + if len(v[1:]) != values || values != 18 { + return V2Stats{}, fmt.Errorf("invalid V2Stats line %q", v) + } + + return V2Stats{ + Null: v[1], + GetAttr: v[2], + SetAttr: v[3], + Root: v[4], + Lookup: v[5], + ReadLink: v[6], + Read: v[7], + WrCache: v[8], + Write: v[9], + Create: v[10], + Remove: v[11], + Rename: v[12], + Link: v[13], + SymLink: v[14], + MkDir: v[15], + RmDir: v[16], + ReadDir: v[17], + FsStat: v[18], + }, nil +} + +func parseV3Stats(v []uint64) (V3Stats, error) { + values := int(v[0]) + if len(v[1:]) != values || values != 22 { + return V3Stats{}, fmt.Errorf("invalid V3Stats line %q", v) + } + + return V3Stats{ + Null: v[1], + GetAttr: v[2], + SetAttr: v[3], + Lookup: v[4], + Access: v[5], + ReadLink: v[6], + Read: v[7], + Write: v[8], + Create: v[9], + MkDir: v[10], + SymLink: v[11], + MkNod: v[12], + Remove: v[13], + RmDir: v[14], + Rename: v[15], + Link: v[16], + ReadDir: v[17], + ReadDirPlus: v[18], + FsStat: v[19], + FsInfo: v[20], + PathConf: v[21], + Commit: v[22], + }, nil +} + +func parseClientV4Stats(v []uint64) (ClientV4Stats, error) { + values := int(v[0]) + if len(v[1:]) != values { + return ClientV4Stats{}, fmt.Errorf("invalid ClientV4Stats line %q", v) + } + + // This function currently supports mapping 59 NFS v4 client stats. Older + // kernels may emit fewer stats, so we must detect this and pad out the + // values to match the expected slice size. + if values < 59 { + newValues := make([]uint64, 60) + copy(newValues, v) + v = newValues + } + + return ClientV4Stats{ + Null: v[1], + Read: v[2], + Write: v[3], + Commit: v[4], + Open: v[5], + OpenConfirm: v[6], + OpenNoattr: v[7], + OpenDowngrade: v[8], + Close: v[9], + Setattr: v[10], + FsInfo: v[11], + Renew: v[12], + SetClientID: v[13], + SetClientIDConfirm: v[14], + Lock: v[15], + Lockt: v[16], + Locku: v[17], + Access: v[18], + Getattr: v[19], + Lookup: v[20], + LookupRoot: v[21], + Remove: v[22], + Rename: v[23], + Link: v[24], + Symlink: v[25], + Create: v[26], + Pathconf: v[27], + StatFs: v[28], + ReadLink: v[29], + ReadDir: v[30], + ServerCaps: v[31], + DelegReturn: v[32], + GetACL: v[33], + SetACL: v[34], + FsLocations: v[35], + ReleaseLockowner: v[36], + Secinfo: v[37], + FsidPresent: v[38], + ExchangeID: v[39], + CreateSession: v[40], + DestroySession: v[41], + Sequence: v[42], + GetLeaseTime: v[43], + ReclaimComplete: v[44], + LayoutGet: v[45], + GetDeviceInfo: v[46], + LayoutCommit: v[47], + LayoutReturn: v[48], + SecinfoNoName: v[49], + TestStateID: v[50], + FreeStateID: v[51], + GetDeviceList: v[52], + BindConnToSession: v[53], + DestroyClientID: v[54], + Seek: v[55], + Allocate: v[56], + DeAllocate: v[57], + LayoutStats: v[58], + Clone: v[59], + }, nil +} + +func parseServerV4Stats(v []uint64) (ServerV4Stats, error) { + values := int(v[0]) + if len(v[1:]) != values || values != 2 { + return ServerV4Stats{}, fmt.Errorf("invalid V4Stats line %q", v) + } + + return ServerV4Stats{ + Null: v[1], + Compound: v[2], + }, nil +} + +func parseV4Ops(v []uint64) (V4Ops, error) { + values := int(v[0]) + if len(v[1:]) != values || values < 39 { + return V4Ops{}, fmt.Errorf("invalid V4Ops line %q", v) + } + + stats := V4Ops{ + Op0Unused: v[1], + Op1Unused: v[2], + Op2Future: v[3], + Access: v[4], + Close: v[5], + Commit: v[6], + Create: v[7], + DelegPurge: v[8], + DelegReturn: v[9], + GetAttr: v[10], + GetFH: v[11], + Link: v[12], + Lock: v[13], + Lockt: v[14], + Locku: v[15], + Lookup: v[16], + LookupRoot: v[17], + Nverify: v[18], + Open: v[19], + OpenAttr: v[20], + OpenConfirm: v[21], + OpenDgrd: v[22], + PutFH: v[23], + PutPubFH: v[24], + PutRootFH: v[25], + Read: v[26], + ReadDir: v[27], + ReadLink: v[28], + Remove: v[29], + Rename: v[30], + Renew: v[31], + RestoreFH: v[32], + SaveFH: v[33], + SecInfo: v[34], + SetAttr: v[35], + Verify: v[36], + Write: v[37], + RelLockOwner: v[38], + } + + return stats, nil +} diff --git a/vendor/github.com/prometheus/procfs/nfs/parse_nfs.go b/vendor/github.com/prometheus/procfs/nfs/parse_nfs.go new file mode 100644 index 00000000..c0d3a5ad --- /dev/null +++ b/vendor/github.com/prometheus/procfs/nfs/parse_nfs.go @@ -0,0 +1,67 @@ +// Copyright 2018 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nfs + +import ( + "bufio" + "fmt" + "io" + "strings" + + "github.com/prometheus/procfs/internal/util" +) + +// ParseClientRPCStats returns stats read from /proc/net/rpc/nfs +func ParseClientRPCStats(r io.Reader) (*ClientRPCStats, error) { + stats := &ClientRPCStats{} + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := scanner.Text() + parts := strings.Fields(scanner.Text()) + // require at least + if len(parts) < 2 { + return nil, fmt.Errorf("invalid NFS metric line %q", line) + } + + values, err := util.ParseUint64s(parts[1:]) + if err != nil { + return nil, fmt.Errorf("error parsing NFS metric line: %s", err) + } + + switch metricLine := parts[0]; metricLine { + case "net": + stats.Network, err = parseNetwork(values) + case "rpc": + stats.ClientRPC, err = parseClientRPC(values) + case "proc2": + stats.V2Stats, err = parseV2Stats(values) + case "proc3": + stats.V3Stats, err = parseV3Stats(values) + case "proc4": + stats.ClientV4Stats, err = parseClientV4Stats(values) + default: + return nil, fmt.Errorf("unknown NFS metric line %q", metricLine) + } + if err != nil { + return nil, fmt.Errorf("errors parsing NFS metric line: %s", err) + } + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("error scanning NFS file: %s", err) + } + + return stats, nil +} diff --git a/vendor/github.com/prometheus/procfs/nfs/parse_nfsd.go b/vendor/github.com/prometheus/procfs/nfs/parse_nfsd.go new file mode 100644 index 00000000..57bb4a35 --- /dev/null +++ b/vendor/github.com/prometheus/procfs/nfs/parse_nfsd.go @@ -0,0 +1,89 @@ +// Copyright 2018 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nfs + +import ( + "bufio" + "fmt" + "io" + "strings" + + "github.com/prometheus/procfs/internal/util" +) + +// ParseServerRPCStats returns stats read from /proc/net/rpc/nfsd +func ParseServerRPCStats(r io.Reader) (*ServerRPCStats, error) { + stats := &ServerRPCStats{} + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := scanner.Text() + parts := strings.Fields(scanner.Text()) + // require at least + if len(parts) < 2 { + return nil, fmt.Errorf("invalid NFSd metric line %q", line) + } + label := parts[0] + + var values []uint64 + var err error + if label == "th" { + if len(parts) < 3 { + return nil, fmt.Errorf("invalid NFSd th metric line %q", line) + } + values, err = util.ParseUint64s(parts[1:3]) + } else { + values, err = util.ParseUint64s(parts[1:]) + } + if err != nil { + return nil, fmt.Errorf("error parsing NFSd metric line: %s", err) + } + + switch metricLine := parts[0]; metricLine { + case "rc": + stats.ReplyCache, err = parseReplyCache(values) + case "fh": + stats.FileHandles, err = parseFileHandles(values) + case "io": + stats.InputOutput, err = parseInputOutput(values) + case "th": + stats.Threads, err = parseThreads(values) + case "ra": + stats.ReadAheadCache, err = parseReadAheadCache(values) + case "net": + stats.Network, err = parseNetwork(values) + case "rpc": + stats.ServerRPC, err = parseServerRPC(values) + case "proc2": + stats.V2Stats, err = parseV2Stats(values) + case "proc3": + stats.V3Stats, err = parseV3Stats(values) + case "proc4": + stats.ServerV4Stats, err = parseServerV4Stats(values) + case "proc4ops": + stats.V4Ops, err = parseV4Ops(values) + default: + return nil, fmt.Errorf("unknown NFSd metric line %q", metricLine) + } + if err != nil { + return nil, fmt.Errorf("errors parsing NFSd metric line: %s", err) + } + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("error scanning NFSd file: %s", err) + } + + return stats, nil +} diff --git a/vendor/github.com/prometheus/procfs/proc_psi.go b/vendor/github.com/prometheus/procfs/proc_psi.go deleted file mode 100644 index 4f11cdbd..00000000 --- a/vendor/github.com/prometheus/procfs/proc_psi.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2019 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package procfs - -// The PSI / pressure interface is described at -// https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/accounting/psi.txt -// Each resource (cpu, io, memory, ...) is exposed as a single file. -// Each file may contain up to two lines, one for "some" pressure and one for "full" pressure. -// Each line contains several averages (over n seconds) and a total in µs. -// -// Example io pressure file: -// > some avg10=0.06 avg60=0.21 avg300=0.99 total=8537362 -// > full avg10=0.00 avg60=0.13 avg300=0.96 total=8183134 - -import ( - "fmt" - "io" - "io/ioutil" - "os" - "strings" -) - -const lineFormat = "avg10=%f avg60=%f avg300=%f total=%d" - -// PSILine is a single line of values as returned by /proc/pressure/* -// The Avg entries are averages over n seconds, as a percentage -// The Total line is in microseconds -type PSILine struct { - Avg10 float64 - Avg60 float64 - Avg300 float64 - Total uint64 -} - -// PSIStats represent pressure stall information from /proc/pressure/* -// Some indicates the share of time in which at least some tasks are stalled -// Full indicates the share of time in which all non-idle tasks are stalled simultaneously -type PSIStats struct { - Some *PSILine - Full *PSILine -} - -// NewPSIStatsForResource reads pressure stall information for the specified -// resource. At time of writing this can be either "cpu", "memory" or "io". -func NewPSIStatsForResource(resource string) (PSIStats, error) { - fs, err := NewFS(DefaultMountPoint) - if err != nil { - return PSIStats{}, err - } - - return fs.NewPSIStatsForResource(resource) -} - -// NewPSIStatsForResource reads pressure stall information from /proc/pressure/ -func (fs FS) NewPSIStatsForResource(resource string) (PSIStats, error) { - file, err := os.Open(fs.Path(fmt.Sprintf("%s/%s", "pressure", resource))) - if err != nil { - return PSIStats{}, fmt.Errorf("psi_stats: unavailable for %s", resource) - } - - defer file.Close() - return parsePSIStats(resource, file) -} - -// parsePSIStats parses the specified file for pressure stall information -func parsePSIStats(resource string, file io.Reader) (PSIStats, error) { - psiStats := PSIStats{} - stats, err := ioutil.ReadAll(file) - if err != nil { - return psiStats, fmt.Errorf("psi_stats: unable to read data for %s", resource) - } - - for _, l := range strings.Split(string(stats), "\n") { - prefix := strings.Split(l, " ")[0] - switch prefix { - case "some": - psi := PSILine{} - _, err := fmt.Sscanf(l, fmt.Sprintf("some %s", lineFormat), &psi.Avg10, &psi.Avg60, &psi.Avg300, &psi.Total) - if err != nil { - return PSIStats{}, err - } - psiStats.Some = &psi - case "full": - psi := PSILine{} - _, err := fmt.Sscanf(l, fmt.Sprintf("full %s", lineFormat), &psi.Avg10, &psi.Avg60, &psi.Avg300, &psi.Total) - if err != nil { - return PSIStats{}, err - } - psiStats.Full = &psi - default: - // If we encounter a line with an unknown prefix, ignore it and move on - // Should new measurement types be added in the future we'll simply ignore them instead - // of erroring on retrieval - continue - } - } - - return psiStats, nil -} diff --git a/vendor/github.com/prometheus/procfs/proc_stat.go b/vendor/github.com/prometheus/procfs/proc_stat.go index e7c626a8..3cf2a9f1 100644 --- a/vendor/github.com/prometheus/procfs/proc_stat.go +++ b/vendor/github.com/prometheus/procfs/proc_stat.go @@ -95,7 +95,7 @@ type ProcStat struct { // in clock ticks. Starttime uint64 // Virtual memory size in bytes. - VSize uint + VSize int // Resident set size in pages. RSS int @@ -164,7 +164,7 @@ func (p Proc) NewStat() (ProcStat, error) { } // VirtualMemory returns the virtual memory size in bytes. -func (s ProcStat) VirtualMemory() uint { +func (s ProcStat) VirtualMemory() int { return s.VSize } diff --git a/vendor/github.com/prometheus/procfs/xfs/parse.go b/vendor/github.com/prometheus/procfs/xfs/parse.go new file mode 100644 index 00000000..2bc0ef34 --- /dev/null +++ b/vendor/github.com/prometheus/procfs/xfs/parse.go @@ -0,0 +1,330 @@ +// Copyright 2017 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package xfs + +import ( + "bufio" + "fmt" + "io" + "strings" + + "github.com/prometheus/procfs/internal/util" +) + +// ParseStats parses a Stats from an input io.Reader, using the format +// found in /proc/fs/xfs/stat. +func ParseStats(r io.Reader) (*Stats, error) { + const ( + // Fields parsed into stats structures. + fieldExtentAlloc = "extent_alloc" + fieldAbt = "abt" + fieldBlkMap = "blk_map" + fieldBmbt = "bmbt" + fieldDir = "dir" + fieldTrans = "trans" + fieldIg = "ig" + fieldLog = "log" + fieldRw = "rw" + fieldAttr = "attr" + fieldIcluster = "icluster" + fieldVnodes = "vnodes" + fieldBuf = "buf" + fieldXpc = "xpc" + + // Unimplemented at this time due to lack of documentation. + fieldPushAil = "push_ail" + fieldXstrat = "xstrat" + fieldAbtb2 = "abtb2" + fieldAbtc2 = "abtc2" + fieldBmbt2 = "bmbt2" + fieldIbt2 = "ibt2" + fieldFibt2 = "fibt2" + fieldQm = "qm" + fieldDebug = "debug" + ) + + var xfss Stats + + s := bufio.NewScanner(r) + for s.Scan() { + // Expect at least a string label and a single integer value, ex: + // - abt 0 + // - rw 1 2 + ss := strings.Fields(string(s.Bytes())) + if len(ss) < 2 { + continue + } + label := ss[0] + + // Extended precision counters are uint64 values. + if label == fieldXpc { + us, err := util.ParseUint64s(ss[1:]) + if err != nil { + return nil, err + } + + xfss.ExtendedPrecision, err = extendedPrecisionStats(us) + if err != nil { + return nil, err + } + + continue + } + + // All other counters are uint32 values. + us, err := util.ParseUint32s(ss[1:]) + if err != nil { + return nil, err + } + + switch label { + case fieldExtentAlloc: + xfss.ExtentAllocation, err = extentAllocationStats(us) + case fieldAbt: + xfss.AllocationBTree, err = btreeStats(us) + case fieldBlkMap: + xfss.BlockMapping, err = blockMappingStats(us) + case fieldBmbt: + xfss.BlockMapBTree, err = btreeStats(us) + case fieldDir: + xfss.DirectoryOperation, err = directoryOperationStats(us) + case fieldTrans: + xfss.Transaction, err = transactionStats(us) + case fieldIg: + xfss.InodeOperation, err = inodeOperationStats(us) + case fieldLog: + xfss.LogOperation, err = logOperationStats(us) + case fieldRw: + xfss.ReadWrite, err = readWriteStats(us) + case fieldAttr: + xfss.AttributeOperation, err = attributeOperationStats(us) + case fieldIcluster: + xfss.InodeClustering, err = inodeClusteringStats(us) + case fieldVnodes: + xfss.Vnode, err = vnodeStats(us) + case fieldBuf: + xfss.Buffer, err = bufferStats(us) + } + if err != nil { + return nil, err + } + } + + return &xfss, s.Err() +} + +// extentAllocationStats builds an ExtentAllocationStats from a slice of uint32s. +func extentAllocationStats(us []uint32) (ExtentAllocationStats, error) { + if l := len(us); l != 4 { + return ExtentAllocationStats{}, fmt.Errorf("incorrect number of values for XFS extent allocation stats: %d", l) + } + + return ExtentAllocationStats{ + ExtentsAllocated: us[0], + BlocksAllocated: us[1], + ExtentsFreed: us[2], + BlocksFreed: us[3], + }, nil +} + +// btreeStats builds a BTreeStats from a slice of uint32s. +func btreeStats(us []uint32) (BTreeStats, error) { + if l := len(us); l != 4 { + return BTreeStats{}, fmt.Errorf("incorrect number of values for XFS btree stats: %d", l) + } + + return BTreeStats{ + Lookups: us[0], + Compares: us[1], + RecordsInserted: us[2], + RecordsDeleted: us[3], + }, nil +} + +// BlockMappingStat builds a BlockMappingStats from a slice of uint32s. +func blockMappingStats(us []uint32) (BlockMappingStats, error) { + if l := len(us); l != 7 { + return BlockMappingStats{}, fmt.Errorf("incorrect number of values for XFS block mapping stats: %d", l) + } + + return BlockMappingStats{ + Reads: us[0], + Writes: us[1], + Unmaps: us[2], + ExtentListInsertions: us[3], + ExtentListDeletions: us[4], + ExtentListLookups: us[5], + ExtentListCompares: us[6], + }, nil +} + +// DirectoryOperationStats builds a DirectoryOperationStats from a slice of uint32s. +func directoryOperationStats(us []uint32) (DirectoryOperationStats, error) { + if l := len(us); l != 4 { + return DirectoryOperationStats{}, fmt.Errorf("incorrect number of values for XFS directory operation stats: %d", l) + } + + return DirectoryOperationStats{ + Lookups: us[0], + Creates: us[1], + Removes: us[2], + Getdents: us[3], + }, nil +} + +// TransactionStats builds a TransactionStats from a slice of uint32s. +func transactionStats(us []uint32) (TransactionStats, error) { + if l := len(us); l != 3 { + return TransactionStats{}, fmt.Errorf("incorrect number of values for XFS transaction stats: %d", l) + } + + return TransactionStats{ + Sync: us[0], + Async: us[1], + Empty: us[2], + }, nil +} + +// InodeOperationStats builds an InodeOperationStats from a slice of uint32s. +func inodeOperationStats(us []uint32) (InodeOperationStats, error) { + if l := len(us); l != 7 { + return InodeOperationStats{}, fmt.Errorf("incorrect number of values for XFS inode operation stats: %d", l) + } + + return InodeOperationStats{ + Attempts: us[0], + Found: us[1], + Recycle: us[2], + Missed: us[3], + Duplicate: us[4], + Reclaims: us[5], + AttributeChange: us[6], + }, nil +} + +// LogOperationStats builds a LogOperationStats from a slice of uint32s. +func logOperationStats(us []uint32) (LogOperationStats, error) { + if l := len(us); l != 5 { + return LogOperationStats{}, fmt.Errorf("incorrect number of values for XFS log operation stats: %d", l) + } + + return LogOperationStats{ + Writes: us[0], + Blocks: us[1], + NoInternalBuffers: us[2], + Force: us[3], + ForceSleep: us[4], + }, nil +} + +// ReadWriteStats builds a ReadWriteStats from a slice of uint32s. +func readWriteStats(us []uint32) (ReadWriteStats, error) { + if l := len(us); l != 2 { + return ReadWriteStats{}, fmt.Errorf("incorrect number of values for XFS read write stats: %d", l) + } + + return ReadWriteStats{ + Read: us[0], + Write: us[1], + }, nil +} + +// AttributeOperationStats builds an AttributeOperationStats from a slice of uint32s. +func attributeOperationStats(us []uint32) (AttributeOperationStats, error) { + if l := len(us); l != 4 { + return AttributeOperationStats{}, fmt.Errorf("incorrect number of values for XFS attribute operation stats: %d", l) + } + + return AttributeOperationStats{ + Get: us[0], + Set: us[1], + Remove: us[2], + List: us[3], + }, nil +} + +// InodeClusteringStats builds an InodeClusteringStats from a slice of uint32s. +func inodeClusteringStats(us []uint32) (InodeClusteringStats, error) { + if l := len(us); l != 3 { + return InodeClusteringStats{}, fmt.Errorf("incorrect number of values for XFS inode clustering stats: %d", l) + } + + return InodeClusteringStats{ + Iflush: us[0], + Flush: us[1], + FlushInode: us[2], + }, nil +} + +// VnodeStats builds a VnodeStats from a slice of uint32s. +func vnodeStats(us []uint32) (VnodeStats, error) { + // The attribute "Free" appears to not be available on older XFS + // stats versions. Therefore, 7 or 8 elements may appear in + // this slice. + l := len(us) + if l != 7 && l != 8 { + return VnodeStats{}, fmt.Errorf("incorrect number of values for XFS vnode stats: %d", l) + } + + s := VnodeStats{ + Active: us[0], + Allocate: us[1], + Get: us[2], + Hold: us[3], + Release: us[4], + Reclaim: us[5], + Remove: us[6], + } + + // Skip adding free, unless it is present. The zero value will + // be used in place of an actual count. + if l == 7 { + return s, nil + } + + s.Free = us[7] + return s, nil +} + +// BufferStats builds a BufferStats from a slice of uint32s. +func bufferStats(us []uint32) (BufferStats, error) { + if l := len(us); l != 9 { + return BufferStats{}, fmt.Errorf("incorrect number of values for XFS buffer stats: %d", l) + } + + return BufferStats{ + Get: us[0], + Create: us[1], + GetLocked: us[2], + GetLockedWaited: us[3], + BusyLocked: us[4], + MissLocked: us[5], + PageRetries: us[6], + PageFound: us[7], + GetRead: us[8], + }, nil +} + +// ExtendedPrecisionStats builds an ExtendedPrecisionStats from a slice of uint32s. +func extendedPrecisionStats(us []uint64) (ExtendedPrecisionStats, error) { + if l := len(us); l != 3 { + return ExtendedPrecisionStats{}, fmt.Errorf("incorrect number of values for XFS extended precision stats: %d", l) + } + + return ExtendedPrecisionStats{ + FlushBytes: us[0], + WriteBytes: us[1], + ReadBytes: us[2], + }, nil +} diff --git a/vendor/github.com/prometheus/procfs/xfs/xfs.go b/vendor/github.com/prometheus/procfs/xfs/xfs.go new file mode 100644 index 00000000..d86794b7 --- /dev/null +++ b/vendor/github.com/prometheus/procfs/xfs/xfs.go @@ -0,0 +1,163 @@ +// Copyright 2017 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package xfs provides access to statistics exposed by the XFS filesystem. +package xfs + +// Stats contains XFS filesystem runtime statistics, parsed from +// /proc/fs/xfs/stat. +// +// The names and meanings of each statistic were taken from +// http://xfs.org/index.php/Runtime_Stats and xfs_stats.h in the Linux +// kernel source. Most counters are uint32s (same data types used in +// xfs_stats.h), but some of the "extended precision stats" are uint64s. +type Stats struct { + // The name of the filesystem used to source these statistics. + // If empty, this indicates aggregated statistics for all XFS + // filesystems on the host. + Name string + + ExtentAllocation ExtentAllocationStats + AllocationBTree BTreeStats + BlockMapping BlockMappingStats + BlockMapBTree BTreeStats + DirectoryOperation DirectoryOperationStats + Transaction TransactionStats + InodeOperation InodeOperationStats + LogOperation LogOperationStats + ReadWrite ReadWriteStats + AttributeOperation AttributeOperationStats + InodeClustering InodeClusteringStats + Vnode VnodeStats + Buffer BufferStats + ExtendedPrecision ExtendedPrecisionStats +} + +// ExtentAllocationStats contains statistics regarding XFS extent allocations. +type ExtentAllocationStats struct { + ExtentsAllocated uint32 + BlocksAllocated uint32 + ExtentsFreed uint32 + BlocksFreed uint32 +} + +// BTreeStats contains statistics regarding an XFS internal B-tree. +type BTreeStats struct { + Lookups uint32 + Compares uint32 + RecordsInserted uint32 + RecordsDeleted uint32 +} + +// BlockMappingStats contains statistics regarding XFS block maps. +type BlockMappingStats struct { + Reads uint32 + Writes uint32 + Unmaps uint32 + ExtentListInsertions uint32 + ExtentListDeletions uint32 + ExtentListLookups uint32 + ExtentListCompares uint32 +} + +// DirectoryOperationStats contains statistics regarding XFS directory entries. +type DirectoryOperationStats struct { + Lookups uint32 + Creates uint32 + Removes uint32 + Getdents uint32 +} + +// TransactionStats contains statistics regarding XFS metadata transactions. +type TransactionStats struct { + Sync uint32 + Async uint32 + Empty uint32 +} + +// InodeOperationStats contains statistics regarding XFS inode operations. +type InodeOperationStats struct { + Attempts uint32 + Found uint32 + Recycle uint32 + Missed uint32 + Duplicate uint32 + Reclaims uint32 + AttributeChange uint32 +} + +// LogOperationStats contains statistics regarding the XFS log buffer. +type LogOperationStats struct { + Writes uint32 + Blocks uint32 + NoInternalBuffers uint32 + Force uint32 + ForceSleep uint32 +} + +// ReadWriteStats contains statistics regarding the number of read and write +// system calls for XFS filesystems. +type ReadWriteStats struct { + Read uint32 + Write uint32 +} + +// AttributeOperationStats contains statistics regarding manipulation of +// XFS extended file attributes. +type AttributeOperationStats struct { + Get uint32 + Set uint32 + Remove uint32 + List uint32 +} + +// InodeClusteringStats contains statistics regarding XFS inode clustering +// operations. +type InodeClusteringStats struct { + Iflush uint32 + Flush uint32 + FlushInode uint32 +} + +// VnodeStats contains statistics regarding XFS vnode operations. +type VnodeStats struct { + Active uint32 + Allocate uint32 + Get uint32 + Hold uint32 + Release uint32 + Reclaim uint32 + Remove uint32 + Free uint32 +} + +// BufferStats contains statistics regarding XFS read/write I/O buffers. +type BufferStats struct { + Get uint32 + Create uint32 + GetLocked uint32 + GetLockedWaited uint32 + BusyLocked uint32 + MissLocked uint32 + PageRetries uint32 + PageFound uint32 + GetRead uint32 +} + +// ExtendedPrecisionStats contains high precision counters used to track the +// total number of bytes read, written, or flushed, during XFS operations. +type ExtendedPrecisionStats struct { + FlushBytes uint64 + WriteBytes uint64 + ReadBytes uint64 +} diff --git a/vendor/github.com/stretchr/testify/LICENSE b/vendor/github.com/stretchr/testify/LICENSE index 473b670a..f38ec595 100644 --- a/vendor/github.com/stretchr/testify/LICENSE +++ b/vendor/github.com/stretchr/testify/LICENSE @@ -1,22 +1,21 @@ -Copyright (c) 2012 - 2013 Mat Ryer and Tyler Bunnell +MIT License -Please consider promoting this project if you find it useful. +Copyright (c) 2012-2018 Mat Ryer and Tyler Bunnell -Permission is hereby granted, free of charge, to any person -obtaining a copy of this software and associated documentation -files (the "Software"), to deal in the Software without restriction, -including without limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of the Software, -and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included -in all copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT -OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE -OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/stretchr/testify/assert/assertions.go b/vendor/github.com/stretchr/testify/assert/assertions.go index 5bdec56c..9bd4a80e 100644 --- a/vendor/github.com/stretchr/testify/assert/assertions.go +++ b/vendor/github.com/stretchr/testify/assert/assertions.go @@ -39,7 +39,7 @@ type ValueAssertionFunc func(TestingT, interface{}, ...interface{}) bool // for table driven tests. type BoolAssertionFunc func(TestingT, bool, ...interface{}) bool -// ValuesAssertionFunc is a common function prototype when validating an error value. Can be useful +// ErrorAssertionFunc is a common function prototype when validating an error value. Can be useful // for table driven tests. type ErrorAssertionFunc func(TestingT, error, ...interface{}) bool @@ -179,7 +179,11 @@ func messageFromMsgAndArgs(msgAndArgs ...interface{}) string { return "" } if len(msgAndArgs) == 1 { - return msgAndArgs[0].(string) + msg := msgAndArgs[0] + if msgAsStr, ok := msg.(string); ok { + return msgAsStr + } + return fmt.Sprintf("%+v", msg) } if len(msgAndArgs) > 1 { return fmt.Sprintf(msgAndArgs[0].(string), msgAndArgs[1:]...) @@ -415,6 +419,17 @@ func NotNil(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { return Fail(t, "Expected value not to be nil.", msgAndArgs...) } +// containsKind checks if a specified kind in the slice of kinds. +func containsKind(kinds []reflect.Kind, kind reflect.Kind) bool { + for i := 0; i < len(kinds); i++ { + if kind == kinds[i] { + return true + } + } + + return false +} + // isNil checks if a specified object is nil or not, without Failing. func isNil(object interface{}) bool { if object == nil { @@ -423,7 +438,14 @@ func isNil(object interface{}) bool { value := reflect.ValueOf(object) kind := value.Kind() - if kind >= reflect.Chan && kind <= reflect.Slice && value.IsNil() { + isNilableKind := containsKind( + []reflect.Kind{ + reflect.Chan, reflect.Func, + reflect.Interface, reflect.Map, + reflect.Ptr, reflect.Slice}, + kind) + + if isNilableKind && value.IsNil() { return true } @@ -1327,7 +1349,7 @@ func typeAndKind(v interface{}) (reflect.Type, reflect.Kind) { } // diff returns a diff of both values as long as both are of the same type and -// are a struct, map, slice or array. Otherwise it returns an empty string. +// are a struct, map, slice, array or string. Otherwise it returns an empty string. func diff(expected interface{}, actual interface{}) string { if expected == nil || actual == nil { return "" @@ -1345,7 +1367,7 @@ func diff(expected interface{}, actual interface{}) string { } var e, a string - if ek != reflect.String { + if et != reflect.TypeOf("") { e = spewConfig.Sdump(expected) a = spewConfig.Sdump(actual) } else { diff --git a/vendor/github.com/stretchr/testify/require/requirements.go b/vendor/github.com/stretchr/testify/require/requirements.go index 690583a8..6b85c5ec 100644 --- a/vendor/github.com/stretchr/testify/require/requirements.go +++ b/vendor/github.com/stretchr/testify/require/requirements.go @@ -22,7 +22,7 @@ type ValueAssertionFunc func(TestingT, interface{}, ...interface{}) // for table driven tests. type BoolAssertionFunc func(TestingT, bool, ...interface{}) -// ValuesAssertionFunc is a common function prototype when validating an error value. Can be useful +// ErrorAssertionFunc is a common function prototype when validating an error value. Can be useful // for table driven tests. type ErrorAssertionFunc func(TestingT, error, ...interface{}) diff --git a/vendor/k8s.io/klog/README.md b/vendor/k8s.io/klog/README.md index 6cb6d168..bee306f3 100644 --- a/vendor/k8s.io/klog/README.md +++ b/vendor/k8s.io/klog/README.md @@ -1,7 +1,27 @@ klog ==== -klog is a permanant fork of https://github.com/golang/glog. original README from glog is below +klog is a permanent fork of https://github.com/golang/glog. + +## Why was klog created? + +The decision to create klog was one that wasn't made lightly, but it was necessary due to some +drawbacks that are present in [glog](https://github.com/golang/glog). Ultimately, the fork was created due to glog not being under active development; this can be seen in the glog README: + +> The code in this repo [...] is not itself under development + +This makes us unable to solve many use cases without a fork. The factors that contributed to needing feature development are listed below: + + * `glog` [presents a lot "gotchas"](https://github.com/kubernetes/kubernetes/issues/61006) and introduces challenges in containerized environments, all of which aren't well documented. + * `glog` doesn't provide an easy way to test logs, which detracts from the stability of software using it + * A long term goal is to implement a logging interface that allows us to add context, change output format, etc. + +Historical context is available here: + + * https://github.com/kubernetes/kubernetes/issues/61006 + * https://github.com/kubernetes/kubernetes/issues/70264 + * https://groups.google.com/forum/#!msg/kubernetes-sig-architecture/wCWiWf3Juzs/hXRVBH90CgAJ + * https://groups.google.com/forum/#!msg/kubernetes-dev/7vnijOMhLS0/1oRiNtigBgAJ ---- diff --git a/vendor/k8s.io/klog/SECURITY_CONTACTS b/vendor/k8s.io/klog/SECURITY_CONTACTS index 520ddb52..6128a586 100644 --- a/vendor/k8s.io/klog/SECURITY_CONTACTS +++ b/vendor/k8s.io/klog/SECURITY_CONTACTS @@ -1,10 +1,10 @@ # Defined below are the security contacts for this repo. # -# They are the contact point for the Product Security Team to reach out +# They are the contact point for the Product Security Committee to reach out # to for triaging and handling of incoming issues. # # The below names agree to abide by the -# [Embargo Policy](https://github.com/kubernetes/sig-release/blob/master/security-release-process-documentation/security-release-process.md#embargo-policy) +# [Embargo Policy](https://git.k8s.io/security/private-distributors-list.md#embargo-policy) # and will be removed and replaced if they violate that agreement. # # DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE diff --git a/vendor/k8s.io/klog/klog.go b/vendor/k8s.io/klog/klog.go index 398cd1c5..887ea62d 100644 --- a/vendor/k8s.io/klog/klog.go +++ b/vendor/k8s.io/klog/klog.go @@ -78,6 +78,7 @@ import ( "fmt" "io" stdLog "log" + "math" "os" "path/filepath" "runtime" @@ -410,10 +411,14 @@ func InitFlags(flagset *flag.FlagSet) { } flagset.StringVar(&logging.logDir, "log_dir", "", "If non-empty, write log files in this directory") flagset.StringVar(&logging.logFile, "log_file", "", "If non-empty, use this log file") + flagset.Uint64Var(&logging.logFileMaxSizeMB, "log_file_max_size", 1800, + "Defines the maximum size a log file can grow to. Unit is megabytes. "+ + "If the value is 0, the maximum file size is unlimited.") flagset.BoolVar(&logging.toStderr, "logtostderr", true, "log to standard error instead of files") flagset.BoolVar(&logging.alsoToStderr, "alsologtostderr", false, "log to standard error as well as files") flagset.Var(&logging.verbosity, "v", "number for the log level verbosity") flagset.BoolVar(&logging.skipHeaders, "skip_headers", false, "If true, avoid header prefixes in the log messages") + flagset.BoolVar(&logging.skipLogHeaders, "skip_log_headers", false, "If true, avoid headers when openning log files") flagset.Var(&logging.stderrThreshold, "stderrthreshold", "logs at or above this threshold go to stderr") flagset.Var(&logging.vmodule, "vmodule", "comma-separated list of pattern=N settings for file-filtered logging") flagset.Var(&logging.traceLocation, "log_backtrace_at", "when logging hits line file:N, emit a stack trace") @@ -471,8 +476,15 @@ type loggingT struct { // with the log-dir option. logFile string + // When logFile is specified, this limiter makes sure the logFile won't exceeds a certain size. When exceeds, the + // logFile will be cleaned up. If this value is 0, no size limitation will be applied to logFile. + logFileMaxSizeMB uint64 + // If true, do not add the prefix headers, useful when used with SetOutput skipHeaders bool + + // If true, do not add the headers to log files + skipLogHeaders bool } // buffer holds a byte Buffer for reuse. The zero value is ready for use. @@ -861,17 +873,32 @@ func (l *loggingT) exit(err error) { type syncBuffer struct { logger *loggingT *bufio.Writer - file *os.File - sev severity - nbytes uint64 // The number of bytes written to this file + file *os.File + sev severity + nbytes uint64 // The number of bytes written to this file + maxbytes uint64 // The max number of bytes this syncBuffer.file can hold before cleaning up. } func (sb *syncBuffer) Sync() error { return sb.file.Sync() } +// CalculateMaxSize returns the real max size in bytes after considering the default max size and the flag options. +func CalculateMaxSize() uint64 { + if logging.logFile != "" { + if logging.logFileMaxSizeMB == 0 { + // If logFileMaxSizeMB is zero, we don't have limitations on the log size. + return math.MaxUint64 + } + // Flag logFileMaxSizeMB is in MB for user convenience. + return logging.logFileMaxSizeMB * 1024 * 1024 + } + // If "log_file" flag is not specified, the target file (sb.file) will be cleaned up when reaches a fixed size. + return MaxSize +} + func (sb *syncBuffer) Write(p []byte) (n int, err error) { - if sb.nbytes+uint64(len(p)) >= MaxSize { + if sb.nbytes+uint64(len(p)) >= sb.maxbytes { if err := sb.rotateFile(time.Now(), false); err != nil { sb.logger.exit(err) } @@ -886,7 +913,7 @@ func (sb *syncBuffer) Write(p []byte) (n int, err error) { // rotateFile closes the syncBuffer's file and starts a new one. // The startup argument indicates whether this is the initial startup of klog. -// If startup is true, existing files are opened for apending instead of truncated. +// If startup is true, existing files are opened for appending instead of truncated. func (sb *syncBuffer) rotateFile(now time.Time, startup bool) error { if sb.file != nil { sb.Flush() @@ -901,6 +928,10 @@ func (sb *syncBuffer) rotateFile(now time.Time, startup bool) error { sb.Writer = bufio.NewWriterSize(sb.file, bufferSize) + if sb.logger.skipLogHeaders { + return nil + } + // Write header. var buf bytes.Buffer fmt.Fprintf(&buf, "Log file created at: %s\n", now.Format("2006/01/02 15:04:05")) @@ -925,8 +956,9 @@ func (l *loggingT) createFiles(sev severity) error { // has already been created, we can stop. for s := sev; s >= infoLog && l.file[s] == nil; s-- { sb := &syncBuffer{ - logger: l, - sev: s, + logger: l, + sev: s, + maxbytes: CalculateMaxSize(), } if err := sb.rotateFile(now, true); err != nil { return err diff --git a/vendor/k8s.io/klog/klog_file.go b/vendor/k8s.io/klog/klog_file.go index 1c4897f4..e4010ad4 100644 --- a/vendor/k8s.io/klog/klog_file.go +++ b/vendor/k8s.io/klog/klog_file.go @@ -98,7 +98,7 @@ var onceLogDirs sync.Once // successfully, create also attempts to update the symlink for that tag, ignoring // errors. // The startup argument indicates whether this is the initial startup of klog. -// If startup is true, existing files are opened for apending instead of truncated. +// If startup is true, existing files are opened for appending instead of truncated. func create(tag string, t time.Time, startup bool) (f *os.File, filename string, err error) { if logging.logFile != "" { f, err := openOrCreate(logging.logFile, startup) diff --git a/vendor/k8s.io/utils/io/consistentread.go b/vendor/k8s.io/utils/io/read.go similarity index 77% rename from vendor/k8s.io/utils/io/consistentread.go rename to vendor/k8s.io/utils/io/read.go index 1d79bed3..16a638d7 100644 --- a/vendor/k8s.io/utils/io/consistentread.go +++ b/vendor/k8s.io/utils/io/read.go @@ -18,10 +18,15 @@ package io import ( "bytes" + "errors" "fmt" + "io" "io/ioutil" ) +// ErrLimitReached means that the read limit is reached. +var ErrLimitReached = errors.New("the read limit is reached") + // ConsistentRead repeatedly reads a file until it gets the same content twice. // This is useful when reading files in /proc that are larger than page size // and kernel may modify them between individual read() syscalls. @@ -53,3 +58,17 @@ func consistentReadSync(filename string, attempts int, sync func(int)) ([]byte, } return nil, fmt.Errorf("could not get consistent content of %s after %d attempts", filename, attempts) } + +// ReadAtMost reads up to `limit` bytes from `r`, and reports an error +// when `limit` bytes are read. +func ReadAtMost(r io.Reader, limit int64) ([]byte, error) { + limitedReader := &io.LimitedReader{R: r, N: limit} + data, err := ioutil.ReadAll(limitedReader) + if err != nil { + return data, err + } + if limitedReader.N <= 0 { + return data, ErrLimitReached + } + return data, nil +} diff --git a/vendor/k8s.io/utils/mount/OWNERS b/vendor/k8s.io/utils/mount/OWNERS new file mode 100644 index 00000000..5fa5cc24 --- /dev/null +++ b/vendor/k8s.io/utils/mount/OWNERS @@ -0,0 +1,14 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +reviewers: + - jingxu97 + - saad-ali + - jsafrane + - msau42 + - andyzhangx + - gnufied +approvers: + - jingxu97 + - saad-ali + - jsafrane + diff --git a/vendor/k8s.io/utils/mount/doc.go b/vendor/k8s.io/utils/mount/doc.go new file mode 100644 index 00000000..c81b426c --- /dev/null +++ b/vendor/k8s.io/utils/mount/doc.go @@ -0,0 +1,18 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package mount defines an interface to mounting filesystems. +package mount // import "k8s.io/utils/mount" diff --git a/vendor/k8s.io/utils/mount/fake_mounter.go b/vendor/k8s.io/utils/mount/fake_mounter.go new file mode 100644 index 00000000..d3a82f41 --- /dev/null +++ b/vendor/k8s.io/utils/mount/fake_mounter.go @@ -0,0 +1,207 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "os" + "path/filepath" + "sync" + + "k8s.io/klog" +) + +// FakeMounter implements mount.Interface for tests. +type FakeMounter struct { + MountPoints []MountPoint + log []FakeAction + // Error to return for a path when calling IsLikelyNotMountPoint + MountCheckErrors map[string]error + // Some tests run things in parallel, make sure the mounter does not produce + // any golang's DATA RACE warnings. + mutex sync.Mutex + UnmountFunc UnmountFunc +} + +// UnmountFunc is a function callback to be executed during the Unmount() call. +type UnmountFunc func(path string) error + +var _ Interface = &FakeMounter{} + +const ( + // FakeActionMount is the string for specifying mount as FakeAction.Action + FakeActionMount = "mount" + // FakeActionUnmount is the string for specifying unmount as FakeAction.Action + FakeActionUnmount = "unmount" +) + +// FakeAction objects are logged every time a fake mount or unmount is called. +type FakeAction struct { + Action string // "mount" or "unmount" + Target string // applies to both mount and unmount actions + Source string // applies only to "mount" actions + FSType string // applies only to "mount" actions +} + +// NewFakeMounter returns a FakeMounter struct that implements Interface and is +// suitable for testing purposes. +func NewFakeMounter(mps []MountPoint) *FakeMounter { + return &FakeMounter{ + MountPoints: mps, + } +} + +// ResetLog clears all the log entries in FakeMounter +func (f *FakeMounter) ResetLog() { + f.mutex.Lock() + defer f.mutex.Unlock() + + f.log = []FakeAction{} +} + +// GetLog returns the slice of FakeActions taken by the mounter +func (f *FakeMounter) GetLog() []FakeAction { + f.mutex.Lock() + defer f.mutex.Unlock() + + return f.log +} + +// Mount records the mount event and updates the in-memory mount points for FakeMounter +func (f *FakeMounter) Mount(source string, target string, fstype string, options []string) error { + f.mutex.Lock() + defer f.mutex.Unlock() + + opts := []string{} + + for _, option := range options { + // find 'bind' option + if option == "bind" { + // This is a bind-mount. In order to mimic linux behaviour, we must + // use the original device of the bind-mount as the real source. + // E.g. when mounted /dev/sda like this: + // $ mount /dev/sda /mnt/test + // $ mount -o bind /mnt/test /mnt/bound + // then /proc/mount contains: + // /dev/sda /mnt/test + // /dev/sda /mnt/bound + // (and not /mnt/test /mnt/bound) + // I.e. we must use /dev/sda as source instead of /mnt/test in the + // bind mount. + for _, mnt := range f.MountPoints { + if source == mnt.Path { + source = mnt.Device + break + } + } + } + // reuse MountPoint.Opts field to mark mount as readonly + opts = append(opts, option) + } + + // If target is a symlink, get its absolute path + absTarget, err := filepath.EvalSymlinks(target) + if err != nil { + absTarget = target + } + f.MountPoints = append(f.MountPoints, MountPoint{Device: source, Path: absTarget, Type: fstype, Opts: opts}) + klog.V(5).Infof("Fake mounter: mounted %s to %s", source, absTarget) + f.log = append(f.log, FakeAction{Action: FakeActionMount, Target: absTarget, Source: source, FSType: fstype}) + return nil +} + +// Unmount records the unmount event and updates the in-memory mount points for FakeMounter +func (f *FakeMounter) Unmount(target string) error { + f.mutex.Lock() + defer f.mutex.Unlock() + + // If target is a symlink, get its absolute path + absTarget, err := filepath.EvalSymlinks(target) + if err != nil { + absTarget = target + } + + newMountpoints := []MountPoint{} + for _, mp := range f.MountPoints { + if mp.Path == absTarget { + if f.UnmountFunc != nil { + err := f.UnmountFunc(absTarget) + if err != nil { + return err + } + } + klog.V(5).Infof("Fake mounter: unmounted %s from %s", mp.Device, absTarget) + // Don't copy it to newMountpoints + continue + } + newMountpoints = append(newMountpoints, MountPoint{Device: mp.Device, Path: mp.Path, Type: mp.Type}) + } + f.MountPoints = newMountpoints + f.log = append(f.log, FakeAction{Action: FakeActionUnmount, Target: absTarget}) + delete(f.MountCheckErrors, target) + return nil +} + +// List returns all the in-memory mountpoints for FakeMounter +func (f *FakeMounter) List() ([]MountPoint, error) { + f.mutex.Lock() + defer f.mutex.Unlock() + + return f.MountPoints, nil +} + +// IsLikelyNotMountPoint determines whether a path is a mountpoint by checking +// if the absolute path to file is in the in-memory mountpoints +func (f *FakeMounter) IsLikelyNotMountPoint(file string) (bool, error) { + f.mutex.Lock() + defer f.mutex.Unlock() + + err := f.MountCheckErrors[file] + if err != nil { + return false, err + } + + _, err = os.Stat(file) + if err != nil { + return true, err + } + + // If file is a symlink, get its absolute path + absFile, err := filepath.EvalSymlinks(file) + if err != nil { + absFile = file + } + + for _, mp := range f.MountPoints { + if mp.Path == absFile { + klog.V(5).Infof("isLikelyNotMountPoint for %s: mounted %s, false", file, mp.Path) + return false, nil + } + } + klog.V(5).Infof("isLikelyNotMountPoint for %s: true", file) + return true, nil +} + +// GetMountRefs finds all mount references to the path, returns a +// list of paths. +func (f *FakeMounter) GetMountRefs(pathname string) ([]string, error) { + realpath, err := filepath.EvalSymlinks(pathname) + if err != nil { + // Ignore error in FakeMounter, because we actually didn't create files. + realpath = pathname + } + return getMountRefsByDev(f, realpath) +} diff --git a/vendor/k8s.io/utils/mount/mount.go b/vendor/k8s.io/utils/mount/mount.go new file mode 100644 index 00000000..46d77327 --- /dev/null +++ b/vendor/k8s.io/utils/mount/mount.go @@ -0,0 +1,300 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// TODO(thockin): This whole pkg is pretty linux-centric. As soon as we have +// an alternate platform, we will need to abstract further. + +package mount + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + utilexec "k8s.io/utils/exec" +) + +const ( + // Default mount command if mounter path is not specified. + defaultMountCommand = "mount" +) + +// Interface defines the set of methods to allow for mount operations on a system. +type Interface interface { + // Mount mounts source to target as fstype with given options. + Mount(source string, target string, fstype string, options []string) error + // Unmount unmounts given target. + Unmount(target string) error + // List returns a list of all mounted filesystems. This can be large. + // On some platforms, reading mounts directly from the OS is not guaranteed + // consistent (i.e. it could change between chunked reads). This is guaranteed + // to be consistent. + List() ([]MountPoint, error) + // IsLikelyNotMountPoint uses heuristics to determine if a directory + // is not a mountpoint. + // It should return ErrNotExist when the directory does not exist. + // IsLikelyNotMountPoint does NOT properly detect all mountpoint types + // most notably linux bind mounts and symbolic link. For callers that do not + // care about such situations, this is a faster alternative to calling List() + // and scanning that output. + IsLikelyNotMountPoint(file string) (bool, error) + // GetMountRefs finds all mount references to pathname, returning a slice of + // paths. Pathname can be a mountpoint path or a normal directory + // (for bind mount). On Linux, pathname is excluded from the slice. + // For example, if /dev/sdc was mounted at /path/a and /path/b, + // GetMountRefs("/path/a") would return ["/path/b"] + // GetMountRefs("/path/b") would return ["/path/a"] + // On Windows there is no way to query all mount points; as long as pathname is + // a valid mount, it will be returned. + GetMountRefs(pathname string) ([]string, error) +} + +// Compile-time check to ensure all Mounter implementations satisfy +// the mount interface. +var _ Interface = &Mounter{} + +// MountPoint represents a single line in /proc/mounts or /etc/fstab. +type MountPoint struct { + Device string + Path string + Type string + Opts []string + Freq int + Pass int +} + +type MountErrorType string + +const ( + FilesystemMismatch MountErrorType = "FilesystemMismatch" + HasFilesystemErrors MountErrorType = "HasFilesystemErrors" + UnformattedReadOnly MountErrorType = "UnformattedReadOnly" + FormatFailed MountErrorType = "FormatFailed" + GetDiskFormatFailed MountErrorType = "GetDiskFormatFailed" + UnknownMountError MountErrorType = "UnknownMountError" +) + +type MountError struct { + Type MountErrorType + Message string +} + +func (mountError MountError) String() string { + return mountError.Message +} + +func (mountError MountError) Error() string { + return mountError.Message +} + +func NewMountError(mountErrorValue MountErrorType, format string, args ...interface{}) error { + mountError := MountError{ + Type: mountErrorValue, + Message: fmt.Sprintf(format, args...), + } + return mountError +} + +// SafeFormatAndMount probes a device to see if it is formatted. +// Namely it checks to see if a file system is present. If so it +// mounts it otherwise the device is formatted first then mounted. +type SafeFormatAndMount struct { + Interface + Exec utilexec.Interface +} + +// FormatAndMount formats the given disk, if needed, and mounts it. +// That is if the disk is not formatted and it is not being mounted as +// read-only it will format it first then mount it. Otherwise, if the +// disk is already formatted or it is being mounted as read-only, it +// will be mounted without formatting. +func (mounter *SafeFormatAndMount) FormatAndMount(source string, target string, fstype string, options []string) error { + return mounter.formatAndMount(source, target, fstype, options) +} + +// getMountRefsByDev finds all references to the device provided +// by mountPath; returns a list of paths. +// Note that mountPath should be path after the evaluation of any symblolic links. +func getMountRefsByDev(mounter Interface, mountPath string) ([]string, error) { + mps, err := mounter.List() + if err != nil { + return nil, err + } + + // Finding the device mounted to mountPath. + diskDev := "" + for i := range mps { + if mountPath == mps[i].Path { + diskDev = mps[i].Device + break + } + } + + // Find all references to the device. + var refs []string + for i := range mps { + if mps[i].Device == diskDev || mps[i].Device == mountPath { + if mps[i].Path != mountPath { + refs = append(refs, mps[i].Path) + } + } + } + return refs, nil +} + +// GetDeviceNameFromMount given a mnt point, find the device from /proc/mounts +// returns the device name, reference count, and error code. +func GetDeviceNameFromMount(mounter Interface, mountPath string) (string, int, error) { + mps, err := mounter.List() + if err != nil { + return "", 0, err + } + + // Find the device name. + // FIXME if multiple devices mounted on the same mount path, only the first one is returned. + device := "" + // If mountPath is symlink, need get its target path. + slTarget, err := filepath.EvalSymlinks(mountPath) + if err != nil { + slTarget = mountPath + } + for i := range mps { + if mps[i].Path == slTarget { + device = mps[i].Device + break + } + } + + // Find all references to the device. + refCount := 0 + for i := range mps { + if mps[i].Device == device { + refCount++ + } + } + return device, refCount, nil +} + +// IsNotMountPoint determines if a directory is a mountpoint. +// It should return ErrNotExist when the directory does not exist. +// IsNotMountPoint is more expensive than IsLikelyNotMountPoint. +// IsNotMountPoint detects bind mounts in linux. +// IsNotMountPoint enumerates all the mountpoints using List() and +// the list of mountpoints may be large, then it uses +// isMountPointMatch to evaluate whether the directory is a mountpoint. +func IsNotMountPoint(mounter Interface, file string) (bool, error) { + // IsLikelyNotMountPoint provides a quick check + // to determine whether file IS A mountpoint. + notMnt, notMntErr := mounter.IsLikelyNotMountPoint(file) + if notMntErr != nil && os.IsPermission(notMntErr) { + // We were not allowed to do the simple stat() check, e.g. on NFS with + // root_squash. Fall back to /proc/mounts check below. + notMnt = true + notMntErr = nil + } + if notMntErr != nil { + return notMnt, notMntErr + } + // identified as mountpoint, so return this fact. + if notMnt == false { + return notMnt, nil + } + + // Resolve any symlinks in file, kernel would do the same and use the resolved path in /proc/mounts. + resolvedFile, err := filepath.EvalSymlinks(file) + if err != nil { + return true, err + } + + // check all mountpoints since IsLikelyNotMountPoint + // is not reliable for some mountpoint types. + mountPoints, mountPointsErr := mounter.List() + if mountPointsErr != nil { + return notMnt, mountPointsErr + } + for _, mp := range mountPoints { + if isMountPointMatch(mp, resolvedFile) { + notMnt = false + break + } + } + return notMnt, nil +} + +// MakeBindOpts detects whether a bind mount is being requested and makes the remount options to +// use in case of bind mount, due to the fact that bind mount doesn't respect mount options. +// The list equals: +// options - 'bind' + 'remount' (no duplicate) +func MakeBindOpts(options []string) (bool, []string, []string) { + // Because we have an FD opened on the subpath bind mount, the "bind" option + // needs to be included, otherwise the mount target will error as busy if you + // remount as readonly. + // + // As a consequence, all read only bind mounts will no longer change the underlying + // volume mount to be read only. + bindRemountOpts := []string{"bind", "remount"} + bind := false + bindOpts := []string{"bind"} + + // _netdev is a userspace mount option and does not automatically get added when + // bind mount is created and hence we must carry it over. + if checkForNetDev(options) { + bindOpts = append(bindOpts, "_netdev") + } + + for _, option := range options { + switch option { + case "bind": + bind = true + break + case "remount": + break + default: + bindRemountOpts = append(bindRemountOpts, option) + } + } + + return bind, bindOpts, bindRemountOpts +} + +func checkForNetDev(options []string) bool { + for _, option := range options { + if option == "_netdev" { + return true + } + } + return false +} + +// PathWithinBase checks if give path is within given base directory. +func PathWithinBase(fullPath, basePath string) bool { + rel, err := filepath.Rel(basePath, fullPath) + if err != nil { + return false + } + if StartsWithBackstep(rel) { + // Needed to escape the base path. + return false + } + return true +} + +// StartsWithBackstep checks if the given path starts with a backstep segment. +func StartsWithBackstep(rel string) bool { + // normalize to / and check for ../ + return rel == ".." || strings.HasPrefix(filepath.ToSlash(rel), "../") +} diff --git a/vendor/k8s.io/utils/mount/mount_helper_common.go b/vendor/k8s.io/utils/mount/mount_helper_common.go new file mode 100644 index 00000000..81f91a8b --- /dev/null +++ b/vendor/k8s.io/utils/mount/mount_helper_common.go @@ -0,0 +1,99 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "fmt" + "os" + + "k8s.io/klog" +) + +// CleanupMountPoint unmounts the given path and deletes the remaining directory +// if successful. If extensiveMountPointCheck is true IsNotMountPoint will be +// called instead of IsLikelyNotMountPoint. IsNotMountPoint is more expensive +// but properly handles bind mounts within the same fs. +func CleanupMountPoint(mountPath string, mounter Interface, extensiveMountPointCheck bool) error { + pathExists, pathErr := PathExists(mountPath) + if !pathExists { + klog.Warningf("Warning: Unmount skipped because path does not exist: %v", mountPath) + return nil + } + corruptedMnt := IsCorruptedMnt(pathErr) + if pathErr != nil && !corruptedMnt { + return fmt.Errorf("Error checking path: %v", pathErr) + } + return doCleanupMountPoint(mountPath, mounter, extensiveMountPointCheck, corruptedMnt) +} + +// doCleanupMountPoint unmounts the given path and +// deletes the remaining directory if successful. +// if extensiveMountPointCheck is true +// IsNotMountPoint will be called instead of IsLikelyNotMountPoint. +// IsNotMountPoint is more expensive but properly handles bind mounts within the same fs. +// if corruptedMnt is true, it means that the mountPath is a corrupted mountpoint, and the mount point check +// will be skipped +func doCleanupMountPoint(mountPath string, mounter Interface, extensiveMountPointCheck bool, corruptedMnt bool) error { + if !corruptedMnt { + var notMnt bool + var err error + if extensiveMountPointCheck { + notMnt, err = IsNotMountPoint(mounter, mountPath) + } else { + notMnt, err = mounter.IsLikelyNotMountPoint(mountPath) + } + + if err != nil { + return err + } + + if notMnt { + klog.Warningf("Warning: %q is not a mountpoint, deleting", mountPath) + return os.Remove(mountPath) + } + } + + // Unmount the mount path + klog.V(4).Infof("%q is a mountpoint, unmounting", mountPath) + if err := mounter.Unmount(mountPath); err != nil { + return err + } + + notMnt, mntErr := mounter.IsLikelyNotMountPoint(mountPath) + if mntErr != nil { + return mntErr + } + if notMnt { + klog.V(4).Infof("%q is unmounted, deleting the directory", mountPath) + return os.Remove(mountPath) + } + return fmt.Errorf("Failed to unmount path %v", mountPath) +} + +// PathExists returns true if the specified path exists. +// TODO: clean this up to use pkg/util/file/FileExists +func PathExists(path string) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return true, nil + } else if os.IsNotExist(err) { + return false, nil + } else if IsCorruptedMnt(err) { + return true, err + } + return false, err +} diff --git a/vendor/k8s.io/utils/mount/mount_helper_unix.go b/vendor/k8s.io/utils/mount/mount_helper_unix.go new file mode 100644 index 00000000..1e14d8c9 --- /dev/null +++ b/vendor/k8s.io/utils/mount/mount_helper_unix.go @@ -0,0 +1,158 @@ +// +build !windows + +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "fmt" + "os" + "strconv" + "strings" + "syscall" + + utilio "k8s.io/utils/io" +) + +const ( + // At least number of fields per line in /proc//mountinfo. + expectedAtLeastNumFieldsPerMountInfo = 10 + // How many times to retry for a consistent read of /proc/mounts. + maxListTries = 3 +) + +// IsCorruptedMnt return true if err is about corrupted mount point +func IsCorruptedMnt(err error) bool { + if err == nil { + return false + } + var underlyingError error + switch pe := err.(type) { + case nil: + return false + case *os.PathError: + underlyingError = pe.Err + case *os.LinkError: + underlyingError = pe.Err + case *os.SyscallError: + underlyingError = pe.Err + } + + return underlyingError == syscall.ENOTCONN || underlyingError == syscall.ESTALE || underlyingError == syscall.EIO || underlyingError == syscall.EACCES +} + +// MountInfo represents a single line in /proc//mountinfo. +type MountInfo struct { + // Unique ID for the mount (maybe reused after umount). + ID int + // The ID of the parent mount (or of self for the root of this mount namespace's mount tree). + ParentID int + // Major indicates one half of the device ID which identifies the device class + // (parsed from `st_dev` for files on this filesystem). + Major int + // Minor indicates one half of the device ID which identifies a specific + // instance of device (parsed from `st_dev` for files on this filesystem). + Minor int + // The pathname of the directory in the filesystem which forms the root of this mount. + Root string + // Mount source, filesystem-specific information. e.g. device, tmpfs name. + Source string + // Mount point, the pathname of the mount point. + MountPoint string + // Optional fieds, zero or more fields of the form "tag[:value]". + OptionalFields []string + // The filesystem type in the form "type[.subtype]". + FsType string + // Per-mount options. + MountOptions []string + // Per-superblock options. + SuperOptions []string +} + +// ParseMountInfo parses /proc/xxx/mountinfo. +func ParseMountInfo(filename string) ([]MountInfo, error) { + content, err := utilio.ConsistentRead(filename, maxListTries) + if err != nil { + return []MountInfo{}, err + } + contentStr := string(content) + infos := []MountInfo{} + + for _, line := range strings.Split(contentStr, "\n") { + if line == "" { + // the last split() item is empty string following the last \n + continue + } + // See `man proc` for authoritative description of format of the file. + fields := strings.Fields(line) + if len(fields) < expectedAtLeastNumFieldsPerMountInfo { + return nil, fmt.Errorf("wrong number of fields in (expected at least %d, got %d): %s", expectedAtLeastNumFieldsPerMountInfo, len(fields), line) + } + id, err := strconv.Atoi(fields[0]) + if err != nil { + return nil, err + } + parentID, err := strconv.Atoi(fields[1]) + if err != nil { + return nil, err + } + mm := strings.Split(fields[2], ":") + if len(mm) != 2 { + return nil, fmt.Errorf("parsing '%s' failed: unexpected minor:major pair %s", line, mm) + } + major, err := strconv.Atoi(mm[0]) + if err != nil { + return nil, fmt.Errorf("parsing '%s' failed: unable to parse major device id, err:%v", mm[0], err) + } + minor, err := strconv.Atoi(mm[1]) + if err != nil { + return nil, fmt.Errorf("parsing '%s' failed: unable to parse minor device id, err:%v", mm[1], err) + } + + info := MountInfo{ + ID: id, + ParentID: parentID, + Major: major, + Minor: minor, + Root: fields[3], + MountPoint: fields[4], + MountOptions: strings.Split(fields[5], ","), + } + // All fields until "-" are "optional fields". + i := 6 + for ; i < len(fields) && fields[i] != "-"; i++ { + info.OptionalFields = append(info.OptionalFields, fields[i]) + } + // Parse the rest 3 fields. + i++ + if len(fields)-i < 3 { + return nil, fmt.Errorf("expect 3 fields in %s, got %d", line, len(fields)-i) + } + info.FsType = fields[i] + info.Source = fields[i+1] + info.SuperOptions = strings.Split(fields[i+2], ",") + infos = append(infos, info) + } + return infos, nil +} + +// isMountPointMatch returns true if the path in mp is the same as dir. +// Handles case where mountpoint dir has been renamed due to stale NFS mount. +func isMountPointMatch(mp MountPoint, dir string) bool { + deletedDir := fmt.Sprintf("%s\\040(deleted)", dir) + return ((mp.Path == dir) || (mp.Path == deletedDir)) +} diff --git a/vendor/k8s.io/utils/mount/mount_helper_windows.go b/vendor/k8s.io/utils/mount/mount_helper_windows.go new file mode 100644 index 00000000..516f970d --- /dev/null +++ b/vendor/k8s.io/utils/mount/mount_helper_windows.go @@ -0,0 +1,101 @@ +// +build windows + +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "fmt" + "os" + "strconv" + "strings" + "syscall" + + "k8s.io/klog" +) + +// following failure codes are from https://docs.microsoft.com/en-us/windows/desktop/debug/system-error-codes--1300-1699- +// ERROR_BAD_NETPATH = 53 +// ERROR_NETWORK_BUSY = 54 +// ERROR_UNEXP_NET_ERR = 59 +// ERROR_NETNAME_DELETED = 64 +// ERROR_NETWORK_ACCESS_DENIED = 65 +// ERROR_BAD_DEV_TYPE = 66 +// ERROR_BAD_NET_NAME = 67 +// ERROR_SESSION_CREDENTIAL_CONFLICT = 1219 +// ERROR_LOGON_FAILURE = 1326 +var errorNoList = [...]int{53, 54, 59, 64, 65, 66, 67, 1219, 1326} + +// IsCorruptedMnt return true if err is about corrupted mount point +func IsCorruptedMnt(err error) bool { + if err == nil { + return false + } + + var underlyingError error + switch pe := err.(type) { + case nil: + return false + case *os.PathError: + underlyingError = pe.Err + case *os.LinkError: + underlyingError = pe.Err + case *os.SyscallError: + underlyingError = pe.Err + } + + if ee, ok := underlyingError.(syscall.Errno); ok { + for _, errno := range errorNoList { + if int(ee) == errno { + klog.Warningf("IsCorruptedMnt failed with error: %v, error code: %v", err, errno) + return true + } + } + } + + return false +} + +// NormalizeWindowsPath makes sure the given path is a valid path on Windows +// systems by making sure all instances of `/` are replaced with `\\`, and the +// path beings with `c:` +func NormalizeWindowsPath(path string) string { + normalizedPath := strings.Replace(path, "/", "\\", -1) + if strings.HasPrefix(normalizedPath, "\\") { + normalizedPath = "c:" + normalizedPath + } + return normalizedPath +} + +// ValidateDiskNumber : disk number should be a number in [0, 99] +func ValidateDiskNumber(disk string) error { + diskNum, err := strconv.Atoi(disk) + if err != nil { + return fmt.Errorf("wrong disk number format: %q, err:%v", disk, err) + } + + if diskNum < 0 || diskNum > 99 { + return fmt.Errorf("disk number out of range: %q", disk) + } + + return nil +} + +// isMountPointMatch determines if the mountpoint matches the dir +func isMountPointMatch(mp MountPoint, dir string) bool { + return mp.Path == dir +} diff --git a/vendor/k8s.io/utils/mount/mount_linux.go b/vendor/k8s.io/utils/mount/mount_linux.go new file mode 100644 index 00000000..4dc696d6 --- /dev/null +++ b/vendor/k8s.io/utils/mount/mount_linux.go @@ -0,0 +1,536 @@ +// +build linux + +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + + "k8s.io/klog" + utilexec "k8s.io/utils/exec" + utilio "k8s.io/utils/io" +) + +const ( + // Number of fields per line in /proc/mounts as per the fstab man page. + expectedNumFieldsPerLine = 6 + // Location of the mount file to use + procMountsPath = "/proc/mounts" + // Location of the mountinfo file + procMountInfoPath = "/proc/self/mountinfo" + // 'fsck' found errors and corrected them + fsckErrorsCorrected = 1 + // 'fsck' found errors but exited without correcting them + fsckErrorsUncorrected = 4 +) + +// Mounter provides the default implementation of mount.Interface +// for the linux platform. This implementation assumes that the +// kubelet is running in the host's root mount namespace. +type Mounter struct { + mounterPath string + withSystemd bool +} + +// New returns a mount.Interface for the current system. +// It provides options to override the default mounter behavior. +// mounterPath allows using an alternative to `/bin/mount` for mounting. +func New(mounterPath string) Interface { + return &Mounter{ + mounterPath: mounterPath, + withSystemd: detectSystemd(), + } +} + +// Mount mounts source to target as fstype with given options. 'source' and 'fstype' must +// be an empty string in case it's not required, e.g. for remount, or for auto filesystem +// type, where kernel handles fstype for you. The mount 'options' is a list of options, +// currently come from mount(8), e.g. "ro", "remount", "bind", etc. If no more option is +// required, call Mount with an empty string list or nil. +func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { + // Path to mounter binary if containerized mounter is needed. Otherwise, it is set to empty. + // All Linux distros are expected to be shipped with a mount utility that a support bind mounts. + mounterPath := "" + bind, bindOpts, bindRemountOpts := MakeBindOpts(options) + if bind { + err := mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindOpts) + if err != nil { + return err + } + return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindRemountOpts) + } + // The list of filesystems that require containerized mounter on GCI image cluster + fsTypesNeedMounter := map[string]struct{}{ + "nfs": {}, + "glusterfs": {}, + "ceph": {}, + "cifs": {}, + } + if _, ok := fsTypesNeedMounter[fstype]; ok { + mounterPath = mounter.mounterPath + } + return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, options) +} + +// doMount runs the mount command. mounterPath is the path to mounter binary if containerized mounter is used. +func (mounter *Mounter) doMount(mounterPath string, mountCmd string, source string, target string, fstype string, options []string) error { + mountArgs := MakeMountArgs(source, target, fstype, options) + if len(mounterPath) > 0 { + mountArgs = append([]string{mountCmd}, mountArgs...) + mountCmd = mounterPath + } + + if mounter.withSystemd { + // Try to run mount via systemd-run --scope. This will escape the + // service where kubelet runs and any fuse daemons will be started in a + // specific scope. kubelet service than can be restarted without killing + // these fuse daemons. + // + // Complete command line (when mounterPath is not used): + // systemd-run --description=... --scope -- mount -t + // + // Expected flow: + // * systemd-run creates a transient scope (=~ cgroup) and executes its + // argument (/bin/mount) there. + // * mount does its job, forks a fuse daemon if necessary and finishes. + // (systemd-run --scope finishes at this point, returning mount's exit + // code and stdout/stderr - thats one of --scope benefits). + // * systemd keeps the fuse daemon running in the scope (i.e. in its own + // cgroup) until the fuse daemon dies (another --scope benefit). + // Kubelet service can be restarted and the fuse daemon survives. + // * When the fuse daemon dies (e.g. during unmount) systemd removes the + // scope automatically. + // + // systemd-mount is not used because it's too new for older distros + // (CentOS 7, Debian Jessie). + mountCmd, mountArgs = AddSystemdScope("systemd-run", target, mountCmd, mountArgs) + } else { + // No systemd-run on the host (or we failed to check it), assume kubelet + // does not run as a systemd service. + // No code here, mountCmd and mountArgs are already populated. + } + + klog.V(4).Infof("Mounting cmd (%s) with arguments (%s)", mountCmd, mountArgs) + command := exec.Command(mountCmd, mountArgs...) + output, err := command.CombinedOutput() + if err != nil { + args := strings.Join(mountArgs, " ") + klog.Errorf("Mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s\n", err, mountCmd, args, string(output)) + return fmt.Errorf("mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s", + err, mountCmd, args, string(output)) + } + return err +} + +// detectSystemd returns true if OS runs with systemd as init. When not sure +// (permission errors, ...), it returns false. +// There may be different ways how to detect systemd, this one makes sure that +// systemd-runs (needed by Mount()) works. +func detectSystemd() bool { + if _, err := exec.LookPath("systemd-run"); err != nil { + klog.V(2).Infof("Detected OS without systemd") + return false + } + // Try to run systemd-run --scope /bin/true, that should be enough + // to make sure that systemd is really running and not just installed, + // which happens when running in a container with a systemd-based image + // but with different pid 1. + cmd := exec.Command("systemd-run", "--description=Kubernetes systemd probe", "--scope", "true") + output, err := cmd.CombinedOutput() + if err != nil { + klog.V(2).Infof("Cannot run systemd-run, assuming non-systemd OS") + klog.V(4).Infof("systemd-run failed with: %v", err) + klog.V(4).Infof("systemd-run output: %s", string(output)) + return false + } + klog.V(2).Infof("Detected OS with systemd") + return true +} + +// MakeMountArgs makes the arguments to the mount(8) command. +// Implementation is shared with NsEnterMounter +func MakeMountArgs(source, target, fstype string, options []string) []string { + // Build mount command as follows: + // mount [-t $fstype] [-o $options] [$source] $target + mountArgs := []string{} + if len(fstype) > 0 { + mountArgs = append(mountArgs, "-t", fstype) + } + if len(options) > 0 { + mountArgs = append(mountArgs, "-o", strings.Join(options, ",")) + } + if len(source) > 0 { + mountArgs = append(mountArgs, source) + } + mountArgs = append(mountArgs, target) + + return mountArgs +} + +// AddSystemdScope adds "system-run --scope" to given command line +// implementation is shared with NsEnterMounter +func AddSystemdScope(systemdRunPath, mountName, command string, args []string) (string, []string) { + descriptionArg := fmt.Sprintf("--description=Kubernetes transient mount for %s", mountName) + systemdRunArgs := []string{descriptionArg, "--scope", "--", command} + return systemdRunPath, append(systemdRunArgs, args...) +} + +// Unmount unmounts the target. +func (mounter *Mounter) Unmount(target string) error { + klog.V(4).Infof("Unmounting %s", target) + command := exec.Command("umount", target) + output, err := command.CombinedOutput() + if err != nil { + return fmt.Errorf("unmount failed: %v\nUnmounting arguments: %s\nOutput: %s", err, target, string(output)) + } + return nil +} + +// List returns a list of all mounted filesystems. +func (*Mounter) List() ([]MountPoint, error) { + return ListProcMounts(procMountsPath) +} + +// IsLikelyNotMountPoint determines if a directory is not a mountpoint. +// It is fast but not necessarily ALWAYS correct. If the path is in fact +// a bind mount from one part of a mount to another it will not be detected. +// It also can not distinguish between mountpoints and symbolic links. +// mkdir /tmp/a /tmp/b; mount --bind /tmp/a /tmp/b; IsLikelyNotMountPoint("/tmp/b") +// will return true. When in fact /tmp/b is a mount point. If this situation +// is of interest to you, don't use this function... +func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { + stat, err := os.Stat(file) + if err != nil { + return true, err + } + rootStat, err := os.Stat(filepath.Dir(strings.TrimSuffix(file, "/"))) + if err != nil { + return true, err + } + // If the directory has a different device as parent, then it is a mountpoint. + if stat.Sys().(*syscall.Stat_t).Dev != rootStat.Sys().(*syscall.Stat_t).Dev { + return false, nil + } + + return true, nil +} + +// GetMountRefs finds all mount references to pathname, returns a +// list of paths. Path could be a mountpoint or a normal +// directory (for bind mount). +func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { + pathExists, pathErr := PathExists(pathname) + if !pathExists { + return []string{}, nil + } else if IsCorruptedMnt(pathErr) { + klog.Warningf("GetMountRefs found corrupted mount at %s, treating as unmounted path", pathname) + return []string{}, nil + } else if pathErr != nil { + return nil, fmt.Errorf("error checking path %s: %v", pathname, pathErr) + } + realpath, err := filepath.EvalSymlinks(pathname) + if err != nil { + return nil, err + } + return SearchMountPoints(realpath, procMountInfoPath) +} + +// checkAndRepairFileSystem checks and repairs filesystems using command fsck. +func (mounter *SafeFormatAndMount) checkAndRepairFilesystem(source string) error { + klog.V(4).Infof("Checking for issues with fsck on disk: %s", source) + args := []string{"-a", source} + out, err := mounter.Exec.Command("fsck", args...).CombinedOutput() + if err != nil { + ee, isExitError := err.(utilexec.ExitError) + switch { + case err == utilexec.ErrExecutableNotFound: + klog.Warningf("'fsck' not found on system; continuing mount without running 'fsck'.") + case isExitError && ee.ExitStatus() == fsckErrorsCorrected: + klog.Infof("Device %s has errors which were corrected by fsck.", source) + case isExitError && ee.ExitStatus() == fsckErrorsUncorrected: + return NewMountError(HasFilesystemErrors, "'fsck' found errors on device %s but could not correct them: %s", source, string(out)) + case isExitError && ee.ExitStatus() > fsckErrorsUncorrected: + klog.Infof("`fsck` error %s", string(out)) + } + } + return nil +} + +// checkAndRepairXfsFilesystem checks and repairs xfs filesystem using command xfs_repair. +func (mounter *SafeFormatAndMount) checkAndRepairXfsFilesystem(source string) error { + klog.V(4).Infof("Checking for issues with xfs_repair on disk: %s", source) + + args := []string{source} + checkArgs := []string{"-n", source} + + // check-only using "xfs_repair -n", if the exit status is not 0, perform a "xfs_repair" + _, err := mounter.Exec.Command("xfs_repair", checkArgs...).CombinedOutput() + if err != nil { + if err == utilexec.ErrExecutableNotFound { + klog.Warningf("'xfs_repair' not found on system; continuing mount without running 'xfs_repair'.") + return nil + } else { + klog.Warningf("Filesystem corruption was detected for %s, running xfs_repair to repair", source) + out, err := mounter.Exec.Command("xfs_repair", args...).CombinedOutput() + if err != nil { + return NewMountError(HasFilesystemErrors, "'xfs_repair' found errors on device %s but could not correct them: %s\n", source, out) + } else { + klog.Infof("Device %s has errors which were corrected by xfs_repair.", source) + return nil + } + } + } + return nil +} + +// formatAndMount uses unix utils to format and mount the given disk +func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error { + readOnly := false + for _, option := range options { + if option == "ro" { + readOnly = true + break + } + } + + options = append(options, "defaults") + mountErrorValue := UnknownMountError + + // Check if the disk is already formatted + existingFormat, err := mounter.GetDiskFormat(source) + if err != nil { + return NewMountError(GetDiskFormatFailed, "failed to get disk format of disk %s: %v", source, err) + } + + // Use 'ext4' as the default + if len(fstype) == 0 { + fstype = "ext4" + } + + if existingFormat == "" { + // Do not attempt to format the disk if mounting as readonly, return an error to reflect this. + if readOnly { + return NewMountError(UnformattedReadOnly, "cannot mount unformatted disk %s as we are manipulating it in read-only mode", source) + } + + // Disk is unformatted so format it. + args := []string{source} + if fstype == "ext4" || fstype == "ext3" { + args = []string{ + "-F", // Force flag + "-m0", // Zero blocks reserved for super-user + source, + } + } + + klog.Infof("Disk %q appears to be unformatted, attempting to format as type: %q with options: %v", source, fstype, args) + output, err := mounter.Exec.Command("mkfs."+fstype, args...).CombinedOutput() + if err != nil { + detailedErr := fmt.Sprintf("format of disk %q failed: type:(%q) target:(%q) options:(%q) errcode:(%v) output:(%v) ", source, fstype, target, options, err, string(output)) + klog.Error(detailedErr) + return NewMountError(FormatFailed, detailedErr) + } + + klog.Infof("Disk successfully formatted (mkfs): %s - %s %s", fstype, source, target) + } else { + if fstype != existingFormat { + // Verify that the disk is formatted with filesystem type we are expecting + mountErrorValue = FilesystemMismatch + klog.Warningf("Configured to mount disk %s as %s but current format is %s, things might break", source, existingFormat, fstype) + } + + if !readOnly { + // Run check tools on the disk to fix repairable issues, only do this for formatted volumes requested as rw. + var err error + switch existingFormat { + case "xfs": + err = mounter.checkAndRepairXfsFilesystem(source) + default: + err = mounter.checkAndRepairFilesystem(source) + } + + if err != nil { + return err + } + } + } + + // Mount the disk + klog.V(4).Infof("Attempting to mount disk %s in %s format at %s", source, fstype, target) + if err := mounter.Interface.Mount(source, target, fstype, options); err != nil { + return NewMountError(mountErrorValue, err.Error()) + } + + return nil +} + +// GetDiskFormat uses 'blkid' to see if the given disk is unformatted +func (mounter *SafeFormatAndMount) GetDiskFormat(disk string) (string, error) { + args := []string{"-p", "-s", "TYPE", "-s", "PTTYPE", "-o", "export", disk} + klog.V(4).Infof("Attempting to determine if disk %q is formatted using blkid with args: (%v)", disk, args) + dataOut, err := mounter.Exec.Command("blkid", args...).CombinedOutput() + output := string(dataOut) + klog.V(4).Infof("Output: %q, err: %v", output, err) + + if err != nil { + if exit, ok := err.(utilexec.ExitError); ok { + if exit.ExitStatus() == 2 { + // Disk device is unformatted. + // For `blkid`, if the specified token (TYPE/PTTYPE, etc) was + // not found, or no (specified) devices could be identified, an + // exit code of 2 is returned. + return "", nil + } + } + klog.Errorf("Could not determine if disk %q is formatted (%v)", disk, err) + return "", err + } + + var fstype, pttype string + + lines := strings.Split(output, "\n") + for _, l := range lines { + if len(l) <= 0 { + // Ignore empty line. + continue + } + cs := strings.Split(l, "=") + if len(cs) != 2 { + return "", fmt.Errorf("blkid returns invalid output: %s", output) + } + // TYPE is filesystem type, and PTTYPE is partition table type, according + // to https://www.kernel.org/pub/linux/utils/util-linux/v2.21/libblkid-docs/. + if cs[0] == "TYPE" { + fstype = cs[1] + } else if cs[0] == "PTTYPE" { + pttype = cs[1] + } + } + + if len(pttype) > 0 { + klog.V(4).Infof("Disk %s detected partition table type: %s", disk, pttype) + // Returns a special non-empty string as filesystem type, then kubelet + // will not format it. + return "unknown data, probably partitions", nil + } + + return fstype, nil +} + +// ListProcMounts is shared with NsEnterMounter +func ListProcMounts(mountFilePath string) ([]MountPoint, error) { + content, err := utilio.ConsistentRead(mountFilePath, maxListTries) + if err != nil { + return nil, err + } + return parseProcMounts(content) +} + +func parseProcMounts(content []byte) ([]MountPoint, error) { + out := []MountPoint{} + lines := strings.Split(string(content), "\n") + for _, line := range lines { + if line == "" { + // the last split() item is empty string following the last \n + continue + } + fields := strings.Fields(line) + if len(fields) != expectedNumFieldsPerLine { + return nil, fmt.Errorf("wrong number of fields (expected %d, got %d): %s", expectedNumFieldsPerLine, len(fields), line) + } + + mp := MountPoint{ + Device: fields[0], + Path: fields[1], + Type: fields[2], + Opts: strings.Split(fields[3], ","), + } + + freq, err := strconv.Atoi(fields[4]) + if err != nil { + return nil, err + } + mp.Freq = freq + + pass, err := strconv.Atoi(fields[5]) + if err != nil { + return nil, err + } + mp.Pass = pass + + out = append(out, mp) + } + return out, nil +} + +// SearchMountPoints finds all mount references to the source, returns a list of +// mountpoints. +// The source can be a mount point or a normal directory (bind mount). We +// didn't support device because there is no use case by now. +// Some filesystems may share a source name, e.g. tmpfs. And for bind mounting, +// it's possible to mount a non-root path of a filesystem, so we need to use +// root path and major:minor to represent mount source uniquely. +// This implementation is shared between Linux and NsEnterMounter +func SearchMountPoints(hostSource, mountInfoPath string) ([]string, error) { + mis, err := ParseMountInfo(mountInfoPath) + if err != nil { + return nil, err + } + + mountID := 0 + rootPath := "" + major := -1 + minor := -1 + + // Finding the underlying root path and major:minor if possible. + // We need search in backward order because it's possible for later mounts + // to overlap earlier mounts. + for i := len(mis) - 1; i >= 0; i-- { + if hostSource == mis[i].MountPoint || PathWithinBase(hostSource, mis[i].MountPoint) { + // If it's a mount point or path under a mount point. + mountID = mis[i].ID + rootPath = filepath.Join(mis[i].Root, strings.TrimPrefix(hostSource, mis[i].MountPoint)) + major = mis[i].Major + minor = mis[i].Minor + break + } + } + + if rootPath == "" || major == -1 || minor == -1 { + return nil, fmt.Errorf("failed to get root path and major:minor for %s", hostSource) + } + + var refs []string + for i := range mis { + if mis[i].ID == mountID { + // Ignore mount entry for mount source itself. + continue + } + if mis[i].Root == rootPath && mis[i].Major == major && mis[i].Minor == minor { + refs = append(refs, mis[i].MountPoint) + } + } + + return refs, nil +} diff --git a/vendor/k8s.io/utils/mount/mount_unsupported.go b/vendor/k8s.io/utils/mount/mount_unsupported.go new file mode 100644 index 00000000..d13ec5c3 --- /dev/null +++ b/vendor/k8s.io/utils/mount/mount_unsupported.go @@ -0,0 +1,72 @@ +// +build !linux,!windows + +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "errors" +) + +// Mounter implements mount.Interface for unsupported platforms +type Mounter struct { + mounterPath string +} + +var errUnsupported = errors.New("util/mount on this platform is not supported") + +// New returns a mount.Interface for the current system. +// It provides options to override the default mounter behavior. +// mounterPath allows using an alternative to `/bin/mount` for mounting. +func New(mounterPath string) Interface { + return &Mounter{ + mounterPath: mounterPath, + } +} + +// Mount always returns an error on unsupported platforms +func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { + return errUnsupported +} + +// Unmount always returns an error on unsupported platforms +func (mounter *Mounter) Unmount(target string) error { + return errUnsupported +} + +// List always returns an error on unsupported platforms +func (mounter *Mounter) List() ([]MountPoint, error) { + return []MountPoint{}, errUnsupported +} + +// IsLikelyNotMountPoint always returns an error on unsupported platforms +func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { + return true, errUnsupported +} + +// GetMountRefs always returns an error on unsupported platforms +func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { + return nil, errUnsupported +} + +func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error { + return mounter.Interface.Mount(source, target, fstype, options) +} + +func (mounter *SafeFormatAndMount) diskLooksUnformatted(disk string) (bool, error) { + return true, errUnsupported +} diff --git a/vendor/k8s.io/utils/mount/mount_windows.go b/vendor/k8s.io/utils/mount/mount_windows.go new file mode 100644 index 00000000..9371ff43 --- /dev/null +++ b/vendor/k8s.io/utils/mount/mount_windows.go @@ -0,0 +1,280 @@ +// +build windows + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "k8s.io/klog" + utilexec "k8s.io/utils/exec" + "k8s.io/utils/keymutex" + utilpath "k8s.io/utils/path" +) + +// Mounter provides the default implementation of mount.Interface +// for the windows platform. This implementation assumes that the +// kubelet is running in the host's root mount namespace. +type Mounter struct { + mounterPath string +} + +// New returns a mount.Interface for the current system. +// It provides options to override the default mounter behavior. +// mounterPath allows using an alternative to `/bin/mount` for mounting. +func New(mounterPath string) Interface { + return &Mounter{ + mounterPath: mounterPath, + } +} + +// acquire lock for smb mount +var getSMBMountMutex = keymutex.NewHashed(0) + +// Mount : mounts source to target with given options. +// currently only supports cifs(smb), bind mount(for disk) +func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { + target = NormalizeWindowsPath(target) + + if source == "tmpfs" { + klog.V(3).Infof("mounting source (%q), target (%q), with options (%q)", source, target, options) + return os.MkdirAll(target, 0755) + } + + parentDir := filepath.Dir(target) + if err := os.MkdirAll(parentDir, 0755); err != nil { + return err + } + + klog.V(4).Infof("mount options(%q) source:%q, target:%q, fstype:%q, begin to mount", + options, source, target, fstype) + bindSource := source + + // tell it's going to mount azure disk or azure file according to options + if bind, _, _ := MakeBindOpts(options); bind { + // mount azure disk + bindSource = NormalizeWindowsPath(source) + } else { + if len(options) < 2 { + klog.Warningf("mount options(%q) command number(%d) less than 2, source:%q, target:%q, skip mounting", + options, len(options), source, target) + return nil + } + + // currently only cifs mount is supported + if strings.ToLower(fstype) != "cifs" { + return fmt.Errorf("only cifs mount is supported now, fstype: %q, mounting source (%q), target (%q), with options (%q)", fstype, source, target, options) + } + + // lock smb mount for the same source + getSMBMountMutex.LockKey(source) + defer getSMBMountMutex.UnlockKey(source) + + if output, err := newSMBMapping(options[0], options[1], source); err != nil { + if isSMBMappingExist(source) { + klog.V(2).Infof("SMB Mapping(%s) already exists, now begin to remove and remount", source) + if output, err := removeSMBMapping(source); err != nil { + return fmt.Errorf("Remove-SmbGlobalMapping failed: %v, output: %q", err, output) + } + if output, err := newSMBMapping(options[0], options[1], source); err != nil { + return fmt.Errorf("New-SmbGlobalMapping remount failed: %v, output: %q", err, output) + } + } else { + return fmt.Errorf("New-SmbGlobalMapping failed: %v, output: %q", err, output) + } + } + } + + if output, err := exec.Command("cmd", "/c", "mklink", "/D", target, bindSource).CombinedOutput(); err != nil { + klog.Errorf("mklink failed: %v, source(%q) target(%q) output: %q", err, bindSource, target, string(output)) + return err + } + + return nil +} + +// do the SMB mount with username, password, remotepath +// return (output, error) +func newSMBMapping(username, password, remotepath string) (string, error) { + if username == "" || password == "" || remotepath == "" { + return "", fmt.Errorf("invalid parameter(username: %s, password: %s, remoteapth: %s)", username, password, remotepath) + } + + // use PowerShell Environment Variables to store user input string to prevent command line injection + // https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_environment_variables?view=powershell-5.1 + cmdLine := `$PWord = ConvertTo-SecureString -String $Env:smbpassword -AsPlainText -Force` + + `;$Credential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $Env:smbuser, $PWord` + + `;New-SmbGlobalMapping -RemotePath $Env:smbremotepath -Credential $Credential` + cmd := exec.Command("powershell", "/c", cmdLine) + cmd.Env = append(os.Environ(), + fmt.Sprintf("smbuser=%s", username), + fmt.Sprintf("smbpassword=%s", password), + fmt.Sprintf("smbremotepath=%s", remotepath)) + + output, err := cmd.CombinedOutput() + return string(output), err +} + +// check whether remotepath is already mounted +func isSMBMappingExist(remotepath string) bool { + cmd := exec.Command("powershell", "/c", `Get-SmbGlobalMapping -RemotePath $Env:smbremotepath`) + cmd.Env = append(os.Environ(), fmt.Sprintf("smbremotepath=%s", remotepath)) + _, err := cmd.CombinedOutput() + return err == nil +} + +// remove SMB mapping +func removeSMBMapping(remotepath string) (string, error) { + cmd := exec.Command("powershell", "/c", `Remove-SmbGlobalMapping -RemotePath $Env:smbremotepath -Force`) + cmd.Env = append(os.Environ(), fmt.Sprintf("smbremotepath=%s", remotepath)) + output, err := cmd.CombinedOutput() + return string(output), err +} + +// Unmount unmounts the target. +func (mounter *Mounter) Unmount(target string) error { + klog.V(4).Infof("azureMount: Unmount target (%q)", target) + target = NormalizeWindowsPath(target) + if output, err := exec.Command("cmd", "/c", "rmdir", target).CombinedOutput(); err != nil { + klog.Errorf("rmdir failed: %v, output: %q", err, string(output)) + return err + } + return nil +} + +// List returns a list of all mounted filesystems. todo +func (mounter *Mounter) List() ([]MountPoint, error) { + return []MountPoint{}, nil +} + +// IsLikelyNotMountPoint determines if a directory is not a mountpoint. +func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { + stat, err := os.Lstat(file) + if err != nil { + return true, err + } + // If current file is a symlink, then it is a mountpoint. + if stat.Mode()&os.ModeSymlink != 0 { + target, err := os.Readlink(file) + if err != nil { + return true, fmt.Errorf("readlink error: %v", err) + } + exists, err := utilpath.Exists(utilpath.CheckFollowSymlink, target) + if err != nil { + return true, err + } + return !exists, nil + } + + return true, nil +} + +// GetMountRefs : empty implementation here since there is no place to query all mount points on Windows +func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { + windowsPath := NormalizeWindowsPath(pathname) + pathExists, pathErr := PathExists(windowsPath) + if !pathExists { + return []string{}, nil + } else if IsCorruptedMnt(pathErr) { + klog.Warningf("GetMountRefs found corrupted mount at %s, treating as unmounted path", windowsPath) + return []string{}, nil + } else if pathErr != nil { + return nil, fmt.Errorf("error checking path %s: %v", windowsPath, pathErr) + } + return []string{pathname}, nil +} + +func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error { + // Try to mount the disk + klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, source, target) + + if err := ValidateDiskNumber(source); err != nil { + klog.Errorf("diskMount: formatAndMount failed, err: %v", err) + return err + } + + if len(fstype) == 0 { + // Use 'NTFS' as the default + fstype = "NTFS" + } + + // format disk if it is unformatted(raw) + cmd := fmt.Sprintf("Get-Disk -Number %s | Where partitionstyle -eq 'raw' | Initialize-Disk -PartitionStyle MBR -PassThru"+ + " | New-Partition -AssignDriveLetter -UseMaximumSize | Format-Volume -FileSystem %s -Confirm:$false", source, fstype) + if output, err := mounter.Exec.Command("powershell", "/c", cmd).CombinedOutput(); err != nil { + return fmt.Errorf("diskMount: format disk failed, error: %v, output: %q", err, string(output)) + } + klog.V(4).Infof("diskMount: Disk successfully formatted, disk: %q, fstype: %q", source, fstype) + + driveLetter, err := getDriveLetterByDiskNumber(source, mounter.Exec) + if err != nil { + return err + } + driverPath := driveLetter + ":" + target = NormalizeWindowsPath(target) + klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, driverPath, target) + if output, err := mounter.Exec.Command("cmd", "/c", "mklink", "/D", target, driverPath).CombinedOutput(); err != nil { + klog.Errorf("mklink failed: %v, output: %q", err, string(output)) + return err + } + return nil +} + +// Get drive letter according to windows disk number +func getDriveLetterByDiskNumber(diskNum string, exec utilexec.Interface) (string, error) { + cmd := fmt.Sprintf("(Get-Partition -DiskNumber %s).DriveLetter", diskNum) + output, err := exec.Command("powershell", "/c", cmd).CombinedOutput() + if err != nil { + return "", fmt.Errorf("azureMount: Get Drive Letter failed: %v, output: %q", err, string(output)) + } + if len(string(output)) < 1 { + return "", fmt.Errorf("azureMount: Get Drive Letter failed, output is empty") + } + return string(output)[:1], nil +} + +// getAllParentLinks walks all symbolic links and return all the parent targets recursively +func getAllParentLinks(path string) ([]string, error) { + const maxIter = 255 + links := []string{} + for { + links = append(links, path) + if len(links) > maxIter { + return links, fmt.Errorf("unexpected length of parent links: %v", links) + } + + fi, err := os.Lstat(path) + if err != nil { + return links, fmt.Errorf("Lstat: %v", err) + } + if fi.Mode()&os.ModeSymlink == 0 { + break + } + + path, err = os.Readlink(path) + if err != nil { + return links, fmt.Errorf("Readlink error: %v", err) + } + } + + return links, nil +} diff --git a/vendor/k8s.io/utils/net/ipnet.go b/vendor/k8s.io/utils/net/ipnet.go index abbb37a5..c2e844bf 100644 --- a/vendor/k8s.io/utils/net/ipnet.go +++ b/vendor/k8s.io/utils/net/ipnet.go @@ -17,6 +17,7 @@ limitations under the License. package net import ( + "fmt" "net" "strings" ) @@ -119,3 +120,102 @@ func (s IPNetSet) Equal(s2 IPNetSet) bool { func (s IPNetSet) Len() int { return len(s) } + +// IPSet maps string to net.IP +type IPSet map[string]net.IP + +// ParseIPSet parses string slice to IPSet +func ParseIPSet(items ...string) (IPSet, error) { + ipset := make(IPSet) + for _, item := range items { + ip := net.ParseIP(strings.TrimSpace(item)) + if ip == nil { + return nil, fmt.Errorf("error parsing IP %q", item) + } + + ipset[ip.String()] = ip + } + + return ipset, nil +} + +// Insert adds items to the set. +func (s IPSet) Insert(items ...net.IP) { + for _, item := range items { + s[item.String()] = item + } +} + +// Delete removes all items from the set. +func (s IPSet) Delete(items ...net.IP) { + for _, item := range items { + delete(s, item.String()) + } +} + +// Has returns true if and only if item is contained in the set. +func (s IPSet) Has(item net.IP) bool { + _, contained := s[item.String()] + return contained +} + +// HasAll returns true if and only if all items are contained in the set. +func (s IPSet) HasAll(items ...net.IP) bool { + for _, item := range items { + if !s.Has(item) { + return false + } + } + return true +} + +// Difference returns a set of objects that are not in s2 +// For example: +// s1 = {a1, a2, a3} +// s2 = {a1, a2, a4, a5} +// s1.Difference(s2) = {a3} +// s2.Difference(s1) = {a4, a5} +func (s IPSet) Difference(s2 IPSet) IPSet { + result := make(IPSet) + for k, i := range s { + _, found := s2[k] + if found { + continue + } + result[k] = i + } + return result +} + +// StringSlice returns a []string with the String representation of each element in the set. +// Order is undefined. +func (s IPSet) StringSlice() []string { + a := make([]string, 0, len(s)) + for k := range s { + a = append(a, k) + } + return a +} + +// IsSuperset returns true if and only if s1 is a superset of s2. +func (s IPSet) IsSuperset(s2 IPSet) bool { + for k := range s2 { + _, found := s[k] + if !found { + return false + } + } + return true +} + +// Equal returns true if and only if s1 is equal (as a set) to s2. +// Two sets are equal if their membership is identical. +// (In practice, this means same elements, order doesn't matter) +func (s IPSet) Equal(s2 IPSet) bool { + return len(s) == len(s2) && s.IsSuperset(s2) +} + +// Len returns the size of the set. +func (s IPSet) Len() int { + return len(s) +} diff --git a/vendor/k8s.io/utils/net/net.go b/vendor/k8s.io/utils/net/net.go index 178d7052..2690aa0c 100644 --- a/vendor/k8s.io/utils/net/net.go +++ b/vendor/k8s.io/utils/net/net.go @@ -16,7 +16,102 @@ limitations under the License. package net -import "net" +import ( + "errors" + "fmt" + "math" + "math/big" + "net" + "strconv" +) + +// ParseCIDRs parses a list of cidrs and return error if any is invalid. +// order is maintained +func ParseCIDRs(cidrsString []string) ([]*net.IPNet, error) { + cidrs := make([]*net.IPNet, 0, len(cidrsString)) + for _, cidrString := range cidrsString { + _, cidr, err := net.ParseCIDR(cidrString) + if err != nil { + return nil, fmt.Errorf("failed to parse cidr value:%q with error:%v", cidrString, err) + } + cidrs = append(cidrs, cidr) + } + return cidrs, nil +} + +// IsDualStackIPs returns if a slice of ips is: +// - all are valid ips +// - at least one ip from each family (v4 or v6) +func IsDualStackIPs(ips []net.IP) (bool, error) { + v4Found := false + v6Found := false + for _, ip := range ips { + if ip == nil { + return false, fmt.Errorf("ip %v is invalid", ip) + } + + if v4Found && v6Found { + continue + } + + if IsIPv6(ip) { + v6Found = true + continue + } + + v4Found = true + } + + return (v4Found && v6Found), nil +} + +// IsDualStackIPStrings returns if +// - all are valid ips +// - at least one ip from each family (v4 or v6) +func IsDualStackIPStrings(ips []string) (bool, error) { + parsedIPs := make([]net.IP, 0, len(ips)) + for _, ip := range ips { + parsedIP := net.ParseIP(ip) + parsedIPs = append(parsedIPs, parsedIP) + } + return IsDualStackIPs(parsedIPs) +} + +// IsDualStackCIDRs returns if +// - all are valid cidrs +// - at least one cidr from each family (v4 or v6) +func IsDualStackCIDRs(cidrs []*net.IPNet) (bool, error) { + v4Found := false + v6Found := false + for _, cidr := range cidrs { + if cidr == nil { + return false, fmt.Errorf("cidr %v is invalid", cidr) + } + + if v4Found && v6Found { + continue + } + + if IsIPv6(cidr.IP) { + v6Found = true + continue + } + v4Found = true + } + + return v4Found && v6Found, nil +} + +// IsDualStackCIDRStrings returns if +// - all are valid cidrs +// - at least one cidr from each family (v4 or v6) +func IsDualStackCIDRStrings(cidrs []string) (bool, error) { + parsedCIDRs, err := ParseCIDRs(cidrs) + if err != nil { + return false, err + } + return IsDualStackCIDRs(parsedCIDRs) +} // IsIPv6 returns if netIP is IPv6. func IsIPv6(netIP net.IP) bool { @@ -35,3 +130,60 @@ func IsIPv6CIDRString(cidr string) bool { ip, _, _ := net.ParseCIDR(cidr) return IsIPv6(ip) } + +// IsIPv6CIDR returns if a cidr is ipv6 +func IsIPv6CIDR(cidr *net.IPNet) bool { + ip := cidr.IP + return IsIPv6(ip) +} + +// ParsePort parses a string representing an IP port. If the string is not a +// valid port number, this returns an error. +func ParsePort(port string, allowZero bool) (int, error) { + portInt, err := strconv.ParseUint(port, 10, 16) + if err != nil { + return 0, err + } + if portInt == 0 && !allowZero { + return 0, errors.New("0 is not a valid port number") + } + return int(portInt), nil +} + +// BigForIP creates a big.Int based on the provided net.IP +func BigForIP(ip net.IP) *big.Int { + b := ip.To4() + if b == nil { + b = ip.To16() + } + return big.NewInt(0).SetBytes(b) +} + +// AddIPOffset adds the provided integer offset to a base big.Int representing a +// net.IP +func AddIPOffset(base *big.Int, offset int) net.IP { + return net.IP(big.NewInt(0).Add(base, big.NewInt(int64(offset))).Bytes()) +} + +// RangeSize returns the size of a range in valid addresses. +// returns the size of the subnet (or math.MaxInt64 if the range size would overflow int64) +func RangeSize(subnet *net.IPNet) int64 { + ones, bits := subnet.Mask.Size() + if bits == 32 && (bits-ones) >= 31 || bits == 128 && (bits-ones) >= 127 { + return 0 + } + // this checks that we are not overflowing an int64 + if bits-ones >= 63 { + return math.MaxInt64 + } + return int64(1) << uint(bits-ones) +} + +// GetIndexedIP returns a net.IP that is subnet.IP + index in the contiguous IP space. +func GetIndexedIP(subnet *net.IPNet, index int) (net.IP, error) { + ip := AddIPOffset(BigForIP(subnet.IP), index) + if !subnet.Contains(ip) { + return nil, fmt.Errorf("can't generate IP with index %d from subnet. subnet too small. subnet: %q", index, subnet) + } + return ip, nil +} diff --git a/vendor/k8s.io/utils/pointer/pointer.go b/vendor/k8s.io/utils/pointer/pointer.go index a11a540f..5365a113 100644 --- a/vendor/k8s.io/utils/pointer/pointer.go +++ b/vendor/k8s.io/utils/pointer/pointer.go @@ -56,7 +56,7 @@ func Int64Ptr(i int64) *int64 { return &i } -// Int32PtrDerefOr dereference the int32 ptr and returns it i not nil, +// Int32PtrDerefOr dereference the int32 ptr and returns it if not nil, // else returns def. func Int32PtrDerefOr(ptr *int32, def int32) int32 { if ptr != nil { diff --git a/vendor/k8s.io/utils/trace/trace.go b/vendor/k8s.io/utils/trace/trace.go index 3a1ecfc7..3b424104 100644 --- a/vendor/k8s.io/utils/trace/trace.go +++ b/vendor/k8s.io/utils/trace/trace.go @@ -25,31 +25,55 @@ import ( "k8s.io/klog" ) +// Field is a key value pair that provides additional details about the trace. +type Field struct { + Key string + Value interface{} +} + +func (f Field) format() string { + return fmt.Sprintf("%s:%v", f.Key, f.Value) +} + +func writeFields(b *bytes.Buffer, l []Field) { + for i, f := range l { + b.WriteString(f.format()) + if i < len(l)-1 { + b.WriteString(",") + } + } +} + type traceStep struct { stepTime time.Time msg string + fields []Field } // Trace keeps track of a set of "steps" and allows us to log a specific // step if it took longer than its share of the total allowed time type Trace struct { name string + fields []Field startTime time.Time steps []traceStep } -// New creates a Trace with the specified name -func New(name string) *Trace { - return &Trace{name, time.Now(), nil} +// New creates a Trace with the specified name. The name identifies the operation to be traced. The +// Fields add key value pairs to provide additional details about the trace, such as operation inputs. +func New(name string, fields ...Field) *Trace { + return &Trace{name: name, startTime: time.Now(), fields: fields} } -// Step adds a new step with a specific message -func (t *Trace) Step(msg string) { +// Step adds a new step with a specific message. Call this at the end of an execution step to record +// how long it took. The Fields add key value pairs to provide additional details about the trace +// step. +func (t *Trace) Step(msg string, fields ...Field) { if t.steps == nil { // traces almost always have less than 6 steps, do this to avoid more than a single allocation t.steps = make([]traceStep, 0, 6) } - t.steps = append(t.steps, traceStep{time.Now(), msg}) + t.steps = append(t.steps, traceStep{stepTime: time.Now(), msg: msg, fields: fields}) } // Log is used to dump all the steps in the Trace @@ -64,12 +88,23 @@ func (t *Trace) logWithStepThreshold(stepThreshold time.Duration) { endTime := time.Now() totalTime := endTime.Sub(t.startTime) - buffer.WriteString(fmt.Sprintf("Trace[%d]: %q (started: %v) (total time: %v):\n", tracenum, t.name, t.startTime, totalTime)) + buffer.WriteString(fmt.Sprintf("Trace[%d]: %q ", tracenum, t.name)) + if len(t.fields) > 0 { + writeFields(&buffer, t.fields) + buffer.WriteString(" ") + } + buffer.WriteString(fmt.Sprintf("(started: %v) (total time: %v):\n", t.startTime, totalTime)) lastStepTime := t.startTime for _, step := range t.steps { stepDuration := step.stepTime.Sub(lastStepTime) if stepThreshold == 0 || stepDuration > stepThreshold || klog.V(4) { - buffer.WriteString(fmt.Sprintf("Trace[%d]: [%v] [%v] %v\n", tracenum, step.stepTime.Sub(t.startTime), stepDuration, step.msg)) + buffer.WriteString(fmt.Sprintf("Trace[%d]: [%v] [%v] ", tracenum, step.stepTime.Sub(t.startTime), stepDuration)) + buffer.WriteString(step.msg) + if len(step.fields) > 0 { + buffer.WriteString(" ") + writeFields(&buffer, step.fields) + } + buffer.WriteString("\n") } lastStepTime = step.stepTime } diff --git a/vendor/modules.txt b/vendor/modules.txt index 98522435..4da98c2d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -21,7 +21,7 @@ github.com/emicklei/go-restful/log github.com/evanphx/json-patch # github.com/fatih/camelcase v1.0.0 github.com/fatih/camelcase -# github.com/ghodss/yaml v1.0.0 +# github.com/ghodss/yaml v0.0.0-20180820084758-c7ce16629ff4 github.com/ghodss/yaml # github.com/go-openapi/jsonpointer v0.19.0 github.com/go-openapi/jsonpointer @@ -127,26 +127,29 @@ github.com/opencontainers/go-digest github.com/pborman/uuid # github.com/peterbourgon/diskv v2.0.1+incompatible github.com/peterbourgon/diskv -# github.com/pkg/errors v0.8.1 +# github.com/pkg/errors v0.8.0 github.com/pkg/errors # github.com/pmezard/go-difflib v1.0.0 github.com/pmezard/go-difflib/difflib # github.com/prometheus/client_golang v0.9.2 github.com/prometheus/client_golang/prometheus github.com/prometheus/client_golang/prometheus/internal -# github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 +# github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 github.com/prometheus/client_model/go -# github.com/prometheus/common v0.3.0 +# github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 github.com/prometheus/common/expfmt github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg github.com/prometheus/common/model -# github.com/prometheus/procfs v0.0.0-20190412120340-e22ddced7142 +# github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a github.com/prometheus/procfs +github.com/prometheus/procfs/internal/util +github.com/prometheus/procfs/nfs +github.com/prometheus/procfs/xfs # github.com/spf13/cobra v0.0.3 github.com/spf13/cobra # github.com/spf13/pflag v1.0.1 github.com/spf13/pflag -# github.com/stretchr/testify v1.2.2 +# github.com/stretchr/testify v1.3.0 github.com/stretchr/testify/assert github.com/stretchr/testify/require # golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 @@ -564,7 +567,7 @@ k8s.io/component-base/config # k8s.io/csi-translation-lib v0.0.0-20190415134207-82f1dfd98d10 k8s.io/csi-translation-lib k8s.io/csi-translation-lib/plugins -# k8s.io/klog v0.0.0-20190306015804-8e90cee79f82 +# k8s.io/klog v0.3.0 k8s.io/klog # k8s.io/kube-aggregator v0.0.0-20190415133304-80ce4e5a0cbc k8s.io/kube-aggregator/pkg/apis/apiregistration @@ -779,12 +782,13 @@ k8s.io/kubernetes/test/e2e/storage/utils k8s.io/kubernetes/test/utils k8s.io/kubernetes/test/utils/image k8s.io/kubernetes/third_party/forked/golang/expansion -# k8s.io/utils v0.0.0-20190308190857-21c4ce38f2a7 +# k8s.io/utils v0.0.0-20200124190032-861946025e34 k8s.io/utils/buffer k8s.io/utils/exec k8s.io/utils/integer k8s.io/utils/io k8s.io/utils/keymutex +k8s.io/utils/mount k8s.io/utils/net k8s.io/utils/nsenter k8s.io/utils/path From fdb32183fea99d452a2e110ef9ca6bfd6f3e9fa3 Mon Sep 17 00:00:00 2001 From: Jan Wozniak Date: Thu, 13 Feb 2020 11:21:45 +0100 Subject: [PATCH 71/71] Change 'make test-vet' to call 'go vet' --- build.make | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.make b/build.make index a9b9d25d..3bf3391c 100644 --- a/build.make +++ b/build.make @@ -113,7 +113,7 @@ test-go: test: test-vet test-vet: @ echo; echo "### $@:" - go test $(GOFLAGS_VENDOR) `go list $(GOFLAGS_VENDOR) ./... | grep -v vendor $(TEST_VET_FILTER_CMD)` + go vet $(GOFLAGS_VENDOR) `go list $(GOFLAGS_VENDOR) ./... | grep -v vendor $(TEST_VET_FILTER_CMD)` .PHONY: test-fmt test: test-fmt