diff --git a/.ci/helm.sh b/.ci/helm.sh index 55df7ca..a967f38 100644 --- a/.ci/helm.sh +++ b/.ci/helm.sh @@ -29,6 +29,28 @@ NAMESPACE=pulsar CLUSTER=pulsar-ci CLUSTER_ID=$(uuidgen) +# brew package 'coreutils' is required on MacOSX +# coreutils includes the 'timeout' command +if [[ "$OSTYPE" == "darwin"* ]]; then + brew_gnubin_packages=(coreutils) + if ! type -P brew &>/dev/null; then + echo "On MacOSX, you must install required binaries with the following command:" + echo "brew install" "${brew_gnubin_packages[@]}" + exit 1 + fi + for dep in "${brew_gnubin_packages[@]}"; do + path_element="$(brew --prefix)/opt/${dep}/libexec/gnubin" + if [ ! -d "${path_element}" ]; then + echo "'${path_element}' is missing. Quick fix: 'brew install ${dep}'." + echo "On MacOSX, you must install required binaries with the following command:" + echo "brew install" "${brew_gnubin_packages[@]}" + exit 1 + fi + PATH="${path_element}:$PATH" + done + export PATH +fi + function ci::create_cluster() { echo "Creating a kind cluster ..." ${CHARTS_HOME}/hack/kind-cluster-build.sh --name pulsar-ci-${CLUSTER_ID} -c 1 -v 10 @@ -65,11 +87,19 @@ function ci::install_cert_manager() { echo ${WC}; sleep 15 ${KUBECTL} get pods -n cert-manager + ${KUBECTL} get events --sort-by=.lastTimestamp -A | tail -n 30 || true WC=$(${KUBECTL} get pods -n cert-manager --field-selector=status.phase=Running | wc -l) done echo "Successfully installed the cert manager." } +function ci::print_pod_logs() { + echo "Logs for all pulsar containers:" + for pod in $(${KUBECTL} get pods -n ${NAMESPACE} -l app=pulsar -o=name); do + ${KUBECTL} logs -n ${NAMESPACE} "$pod" --all-containers=true --ignore-errors=true --prefix=true --tail=100 || true + done; +} + function ci::install_pulsar_chart() { local value_file=$1 local extra_opts=$2 @@ -87,10 +117,20 @@ function ci::install_pulsar_chart() { echo "wait until broker is alive" WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep ${CLUSTER}-broker | wc -l) + counter=1 while [[ ${WC} -lt 1 ]]; do + ((counter++)) echo ${WC}; sleep 15 ${KUBECTL} get pods -n ${NAMESPACE} + ${KUBECTL} get events --sort-by=.lastTimestamp -A | tail -n 30 || true + if [[ $((counter % 20)) -eq 0 ]]; then + ci::print_pod_logs + if [[ $counter -gt 100 ]]; then + echo >&2 "Timeout waiting..." + exit 1 + fi + fi WC=$(${KUBECTL} get pods -n ${NAMESPACE} | grep ${CLUSTER}-broker | wc -l) if [[ ${WC} -gt 1 ]]; then ${KUBECTL} describe pod -n ${NAMESPACE} pulsar-ci-broker-0 @@ -98,17 +138,27 @@ function ci::install_pulsar_chart() { fi WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep ${CLUSTER}-broker | wc -l) done - ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-broker; do sleep 3; done' - ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until [ "$(curl -L http://pulsar-ci-broker:8080/status.html)" == "OK" ]; do sleep 3; done' + timeout 300s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-broker; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; } + timeout 120s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until [ "$(curl -L http://pulsar-ci-broker:8080/status.html)" == "OK" ]; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; } WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep ${CLUSTER}-proxy | wc -l) + counter=1 while [[ ${WC} -lt 1 ]]; do + ((counter++)) echo ${WC}; sleep 15 ${KUBECTL} get pods -n ${NAMESPACE} + ${KUBECTL} get events --sort-by=.lastTimestamp -A | tail -n 30 || true + if [[ $((counter % 8)) -eq 0 ]]; then + ci::print_pod_logs + if [[ $counter -gt 16 ]]; then + echo >&2 "Timeout waiting..." + exit 1 + fi + fi WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep ${CLUSTER}-proxy | wc -l) done - ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-proxy; do sleep 3; done' + timeout 300s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-proxy; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; } # ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until [ "$(curl -L http://pulsar-ci-proxy:8080/status.html)" == "OK" ]; do sleep 3; done' } @@ -131,6 +181,7 @@ function ci::wait_function_running() { echo ${num_running} sleep 15 ${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running + ${KUBECTL} get events --sort-by=.lastTimestamp -A | tail -n 30 || true num_running=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'bin/pulsar-admin functions status --tenant pulsar-ci --namespace test --name test-function | bin/jq .numRunning') done }