Add testing of upgrading of latest released chart version to CI (#432)
* Add upgrade test to CI - install latest released chart version and then upgrade to the local chart version * Add metallb for loadbalancer support so that "--wait" will work * Improve script logging * produce messages with old version of pulsar and consume with new version * Add 10 second delay * Remove duplication * Improve logging * Fix quotes * Before trying produce-consume test, verify that the Admin API is available * Add 2 more upgrade scenarios: TLS & PSP
This commit is contained in:
Lari Hotari
GitHub
parent
f20756abb8
commit
a8c7745b3e
@ -38,14 +38,39 @@ if [[ "x${SYMMETRIC}" == "xtrue" ]]; then
|
||||
extra_opts="-s"
|
||||
fi
|
||||
|
||||
# install pulsar chart
|
||||
ci::install_pulsar_chart ${PULSAR_HOME}/.ci/values-common.yaml ${PULSAR_HOME}/${VALUES_FILE} ${extra_opts}
|
||||
install_type="install"
|
||||
test_action="produce-consume"
|
||||
if [[ "$UPGRADE_FROM_VERSION" != "" ]]; then
|
||||
# install older version of pulsar chart
|
||||
PULSAR_CHART_VERSION="$UPGRADE_FROM_VERSION"
|
||||
ci::install_pulsar_chart install ${PULSAR_HOME}/.ci/values-common.yaml ${PULSAR_HOME}/${VALUES_FILE} ${extra_opts}
|
||||
install_type="upgrade"
|
||||
echo "Wait 10 seconds"
|
||||
sleep 10
|
||||
# test that we can access the admin api
|
||||
ci::test_pulsar_admin_api_access
|
||||
# produce messages with old version of pulsar and consume with new version
|
||||
ci::test_pulsar_producer_consumer "produce"
|
||||
test_action="consume"
|
||||
fi
|
||||
|
||||
PULSAR_CHART_VERSION="local"
|
||||
# install (or upgrade) pulsar chart
|
||||
ci::install_pulsar_chart ${install_type} ${PULSAR_HOME}/.ci/values-common.yaml ${PULSAR_HOME}/${VALUES_FILE} ${extra_opts}
|
||||
|
||||
echo "Wait 10 seconds"
|
||||
sleep 10
|
||||
|
||||
# check pulsar environment
|
||||
ci::check_pulsar_environment
|
||||
|
||||
# test that we can access the admin api
|
||||
ci::test_pulsar_admin_api_access
|
||||
# test producer/consumer
|
||||
ci::test_pulsar_producer_consumer
|
||||
ci::test_pulsar_producer_consumer "${test_action}"
|
||||
|
||||
if [[ "x${FUNCTION}" == "xtrue" ]]; then
|
||||
# install cert manager
|
||||
# test functions
|
||||
ci::test_pulsar_function
|
||||
fi
|
||||
|
||||
|
||||
133
.ci/helm.sh
133
.ci/helm.sh
@ -20,13 +20,15 @@
|
||||
BINDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
|
||||
PULSAR_HOME="$(cd "${BINDIR}/.." && pwd)"
|
||||
CHARTS_HOME=${PULSAR_HOME}
|
||||
PULSAR_CHART_LOCAL=${CHARTS_HOME}/charts/pulsar
|
||||
PULSAR_CHART_VERSION=${PULSAR_CHART_VERSION:-"local"}
|
||||
OUTPUT_BIN=${CHARTS_HOME}/output/bin
|
||||
KIND_BIN=$OUTPUT_BIN/kind
|
||||
HELM=${OUTPUT_BIN}/helm
|
||||
KUBECTL=${OUTPUT_BIN}/kubectl
|
||||
NAMESPACE=pulsar
|
||||
CLUSTER=pulsar-ci
|
||||
CLUSTER_ID=$(uuidgen)
|
||||
: ${CLUSTER_ID:=$(uuidgen)}
|
||||
K8S_LOGS_DIR="${K8S_LOGS_DIR:-/tmp/k8s-logs}"
|
||||
export PATH="$OUTPUT_BIN:$PATH"
|
||||
|
||||
@ -100,10 +102,13 @@ function ci::collect_k8s_logs() {
|
||||
}
|
||||
|
||||
function ci::install_pulsar_chart() {
|
||||
local common_value_file=$1
|
||||
local value_file=$2
|
||||
local extra_opts=$3
|
||||
local install_type=$1
|
||||
local common_value_file=$2
|
||||
local value_file=$3
|
||||
local extra_opts=$4
|
||||
local install_args
|
||||
|
||||
if [[ "${install_type}" == "install" ]]; then
|
||||
echo "Installing the pulsar chart"
|
||||
${KUBECTL} create namespace ${NAMESPACE}
|
||||
ci::install_cert_manager
|
||||
@ -111,12 +116,43 @@ function ci::install_pulsar_chart() {
|
||||
${CHARTS_HOME}/scripts/pulsar/prepare_helm_release.sh -k ${CLUSTER} -n ${NAMESPACE} ${extra_opts}
|
||||
sleep 10
|
||||
|
||||
echo ${HELM} dependency update ${CHARTS_HOME}/charts/pulsar
|
||||
${HELM} dependency update ${CHARTS_HOME}/charts/pulsar
|
||||
echo ${HELM} install --set initialize=true --values ${common_value_file} --values ${value_file} ${CLUSTER} ${CHARTS_HOME}/charts/pulsar
|
||||
${HELM} template --values ${common_value_file} --values ${value_file} ${CLUSTER} ${CHARTS_HOME}/charts/pulsar
|
||||
${HELM} install --set initialize=true --values ${common_value_file} --values ${value_file} --namespace=${NAMESPACE} ${CLUSTER} ${CHARTS_HOME}/charts/pulsar
|
||||
# install metallb for loadbalancer support
|
||||
# following instructions from https://kind.sigs.k8s.io/docs/user/loadbalancer/
|
||||
${KUBECTL} apply -f https://raw.githubusercontent.com/metallb/metallb/v0.13.12/config/manifests/metallb-native.yaml
|
||||
# wait until metallb is ready
|
||||
${KUBECTL} wait --namespace metallb-system \
|
||||
--for=condition=ready pod \
|
||||
--selector=app=metallb \
|
||||
--timeout=90s
|
||||
# configure metallb
|
||||
${KUBECTL} apply -f ${BINDIR}/metallb/metallb-config.yaml
|
||||
|
||||
install_args=""
|
||||
else
|
||||
install_args="--wait --wait-for-jobs --timeout 300s --debug"
|
||||
fi
|
||||
|
||||
CHART_ARGS=""
|
||||
if [[ "${PULSAR_CHART_VERSION}" == "local" ]]; then
|
||||
set -x
|
||||
${HELM} dependency update ${PULSAR_CHART_LOCAL}
|
||||
set +x
|
||||
CHART_ARGS="${PULSAR_CHART_LOCAL}"
|
||||
else
|
||||
set -x
|
||||
${HELM} repo add apache https://pulsar.apache.org/charts
|
||||
set +x
|
||||
CHART_ARGS="apache/pulsar --dependency-update"
|
||||
if [[ "${PULSAR_CHART_VERSION}" != "latest" ]]; then
|
||||
CHART_ARGS="${CHART_ARGS} --version ${PULSAR_CHART_VERSION}"
|
||||
fi
|
||||
fi
|
||||
set -x
|
||||
${HELM} template --values ${common_value_file} --values ${value_file} ${CLUSTER} ${CHART_ARGS}
|
||||
${HELM} ${install_type} --values ${common_value_file} --values ${value_file} --namespace=${NAMESPACE} ${CLUSTER} ${CHART_ARGS} ${install_args}
|
||||
set +x
|
||||
|
||||
if [[ "${install_type}" == "install" ]]; then
|
||||
echo "wait until broker is alive"
|
||||
WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep ${CLUSTER}-broker | wc -l)
|
||||
counter=1
|
||||
@ -141,7 +177,7 @@ function ci::install_pulsar_chart() {
|
||||
WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep ${CLUSTER}-broker | wc -l)
|
||||
done
|
||||
timeout 300s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-broker; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
|
||||
timeout 120s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until [ "$(curl -L http://pulsar-ci-broker:8080/status.html)" == "OK" ]; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
|
||||
timeout 120s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until [ "$(curl -s -L http://pulsar-ci-broker:8080/status.html)" == "OK" ]; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
|
||||
|
||||
WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep ${CLUSTER}-proxy | wc -l)
|
||||
counter=1
|
||||
@ -161,7 +197,15 @@ function ci::install_pulsar_chart() {
|
||||
WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep ${CLUSTER}-proxy | wc -l)
|
||||
done
|
||||
timeout 300s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-proxy; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
|
||||
# ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until [ "$(curl -L http://pulsar-ci-proxy:8080/status.html)" == "OK" ]; do sleep 3; done'
|
||||
echo "Install complete"
|
||||
else
|
||||
echo "wait until broker is alive"
|
||||
timeout 300s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-broker; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
|
||||
timeout 120s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until [ "$(curl -s -L http://pulsar-ci-broker:8080/status.html)" == "OK" ]; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
|
||||
echo "wait until proxy is alive"
|
||||
timeout 300s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-proxy; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
|
||||
echo "Upgrade complete"
|
||||
fi
|
||||
}
|
||||
|
||||
helm_values_cached=""
|
||||
@ -173,28 +217,66 @@ function ci::helm_values_for_deployment() {
|
||||
printf "%s" "${helm_values_cached}"
|
||||
}
|
||||
|
||||
function ci::test_pulsar_producer_consumer() {
|
||||
sleep 120
|
||||
function ci::check_pulsar_environment() {
|
||||
echo "Wait until pulsar-ci-broker is ready"
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-broker; do sleep 3; done'
|
||||
echo "Wait until pulsar-ci-proxy is ready"
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-proxy; do sleep 3; done'
|
||||
echo "bookie-0 disk usage"
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-bookie-0 -- df -h
|
||||
echo "bookie-0 bookkeeper.conf"
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-bookie-0 -- cat conf/bookkeeper.conf
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/bookkeeper shell listbookies -rw
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/bookkeeper shell listbookies -ro
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin tenants create pulsar-ci
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin namespaces create pulsar-ci/test
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin topics create pulsar-ci/test/test-topic
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin topics create-subscription -s test pulsar-ci/test/test-topic
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client produce -m "test-message" pulsar-ci/test/test-topic
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client consume -s test pulsar-ci/test/test-topic
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin topics create-subscription -s test2 pulsar-ci/test/test-topic
|
||||
echo "bookie-0 bookies list (rw)"
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/bookkeeper shell listbookies -rw | grep ListBookiesCommand
|
||||
echo "bookie-0 bookies list (ro)"
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/bookkeeper shell listbookies -ro | grep ListBookiesCommand
|
||||
}
|
||||
|
||||
# function to retry a given commend 3 times with a backoff of 10 seconds in between
|
||||
function ci::retry() {
|
||||
local n=1
|
||||
local max=3
|
||||
local delay=10
|
||||
while true; do
|
||||
"$@" && break || {
|
||||
if [[ $n -lt $max ]]; then
|
||||
((n++))
|
||||
echo "::warning::Command failed. Attempt $n/$max:"
|
||||
sleep $delay
|
||||
else
|
||||
fail "::error::The command has failed after $n attempts."
|
||||
fi
|
||||
}
|
||||
done
|
||||
}
|
||||
|
||||
function ci::test_pulsar_admin_api_access() {
|
||||
ci::retry ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin tenants list
|
||||
}
|
||||
|
||||
function ci::test_pulsar_producer_consumer() {
|
||||
action="${1:-"produce-consume"}"
|
||||
echo "Testing with ${action}"
|
||||
if [[ "$(ci::helm_values_for_deployment | yq .tls.proxy.enabled)" == "true" ]]; then
|
||||
PROXY_URL="pulsar+ssl://pulsar-ci-proxy:6651"
|
||||
else
|
||||
PROXY_URL="pulsar://pulsar-ci-proxy:6650"
|
||||
fi
|
||||
set -x
|
||||
if [[ "${action}" == "produce" || "${action}" == "produce-consume" ]]; then
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin tenants create pulsar-ci
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin namespaces create pulsar-ci/test
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin topics create pulsar-ci/test/test-topic
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin topics create-subscription -s test pulsar-ci/test/test-topic
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client produce -m "test-message" pulsar-ci/test/test-topic
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin topics create-subscription -s test2 pulsar-ci/test/test-topic
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client --url "${PROXY_URL}" produce -m "test-message2" pulsar-ci/test/test-topic
|
||||
fi
|
||||
if [[ "${action}" == "consume" || "${action}" == "produce-consume" ]]; then
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client consume -s test pulsar-ci/test/test-topic
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client --url "${PROXY_URL}" consume -s test2 pulsar-ci/test/test-topic
|
||||
fi
|
||||
set +x
|
||||
}
|
||||
|
||||
function ci::wait_function_running() {
|
||||
@ -219,14 +301,7 @@ function ci::wait_message_processed() {
|
||||
}
|
||||
|
||||
function ci::test_pulsar_function() {
|
||||
sleep 120
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-broker; do sleep 3; done'
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-proxy; do sleep 3; done'
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-bookie-0 -- df -h
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/bookkeeper shell listbookies -rw
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/bookkeeper shell listbookies -ro
|
||||
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin functions create --tenant pulsar-ci --namespace test --name test-function --inputs "pulsar-ci/test/test_input" --output "pulsar-ci/test/test_output" --parallelism 1 --classname org.apache.pulsar.functions.api.examples.ExclamationFunction --jar /pulsar/examples/api-examples.jar
|
||||
|
||||
# wait until the function is running
|
||||
# TODO: re-enable function test
|
||||
# ci::wait_function_running
|
||||
|
||||
33
.ci/metallb/metallb-config.yaml
Normal file
33
.ci/metallb/metallb-config.yaml
Normal file
@ -0,0 +1,33 @@
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
|
||||
apiVersion: metallb.io/v1beta1
|
||||
kind: IPAddressPool
|
||||
metadata:
|
||||
name: example
|
||||
namespace: metallb-system
|
||||
spec:
|
||||
addresses:
|
||||
- 172.19.255.200-172.19.255.250
|
||||
---
|
||||
apiVersion: metallb.io/v1beta1
|
||||
kind: L2Advertisement
|
||||
metadata:
|
||||
name: empty
|
||||
namespace: metallb-system
|
||||
31
.github/workflows/pulsar-helm-chart-ci.yaml
vendored
31
.github/workflows/pulsar-helm-chart-ci.yaml
vendored
@ -136,7 +136,7 @@ jobs:
|
||||
action: wait
|
||||
|
||||
install-chart-tests:
|
||||
name: ${{ matrix.testScenario.name }} - k8s ${{ matrix.k8sVersion.version }} - Install
|
||||
name: ${{ matrix.testScenario.name }} - k8s ${{ matrix.k8sVersion.version }} - ${{ matrix.testScenario.type || 'install' }}
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: ${{ matrix.testScenario.timeout || 45 }}
|
||||
needs: ['preconditions', 'ct-lint']
|
||||
@ -151,9 +151,10 @@ jobs:
|
||||
- version: "1.27.3"
|
||||
kind_image_tag: v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72
|
||||
testScenario:
|
||||
- name: Basic
|
||||
values_file: .ci/clusters/values-basic.yaml
|
||||
shortname: basic
|
||||
- name: Upgrade latest released version
|
||||
values_file: .ci/clusters/values-upgrade.yaml
|
||||
shortname: upgrade
|
||||
type: upgrade
|
||||
- name: Use Pulsar Image
|
||||
values_file: .ci/clusters/values-pulsar-image.yaml
|
||||
shortname: pulsar-image
|
||||
@ -181,6 +182,23 @@ jobs:
|
||||
- name: PSP
|
||||
values_file: .ci/clusters/values-psp.yaml
|
||||
shortname: psp
|
||||
include:
|
||||
- k8sVersion:
|
||||
version: "1.21.14"
|
||||
kind_image_tag: v1.21.14@sha256:8a4e9bb3f415d2bb81629ce33ef9c76ba514c14d707f9797a01e3216376ba093
|
||||
testScenario:
|
||||
name: "Upgrade TLS"
|
||||
values_file: .ci/clusters/values-tls.yaml
|
||||
shortname: tls
|
||||
type: upgrade
|
||||
- k8sVersion:
|
||||
version: "1.21.14"
|
||||
kind_image_tag: v1.21.14@sha256:8a4e9bb3f415d2bb81629ce33ef9c76ba514c14d707f9797a01e3216376ba093
|
||||
testScenario:
|
||||
name: "Upgrade PSP"
|
||||
values_file: .ci/clusters/values-psp.yaml
|
||||
shortname: psp
|
||||
type: upgrade
|
||||
env:
|
||||
k8sVersion: ${{ matrix.k8sVersion.kind_image_tag }}
|
||||
KUBECTL_VERSION: ${{ matrix.k8sVersion.version }}
|
||||
@ -234,13 +252,16 @@ jobs:
|
||||
with:
|
||||
limit-access-to-actor: true
|
||||
|
||||
- name: Run chart-testing (install)
|
||||
- name: Run chart-testing (${{ matrix.testScenario.type || 'install' }})
|
||||
run: |
|
||||
case "${{ matrix.testScenario.shortname }}" in
|
||||
"jwt-symmetric")
|
||||
export SYMMETRIC=true
|
||||
;;
|
||||
esac
|
||||
if [[ "${{ matrix.testScenario.type || 'install' }}" == "upgrade" ]]; then
|
||||
export UPGRADE_FROM_VERSION=latest
|
||||
fi
|
||||
.ci/chart_test.sh ${{ matrix.testScenario.values_file }}
|
||||
|
||||
- name: Collect k8s logs on failure
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user