Add testing of upgrading of latest released chart version to CI (#432)

* Add upgrade test to CI

- install latest released chart version and then upgrade to the local chart version

* Add metallb for loadbalancer support so that "--wait" will work

* Improve script logging

* produce messages with old version of pulsar and consume with new version

* Add 10 second delay

* Remove duplication

* Improve logging

* Fix quotes

* Before trying produce-consume test, verify that the Admin API is available

* Add 2 more upgrade scenarios: TLS & PSP

.ci/chart_test.sh

@@ -38,14 +38,39 @@ if [[ "x${SYMMETRIC}" == "xtrue" ]]; then
     extra_opts="-s"
 fi
 
-# install pulsar chart
+install_type="install"
-ci::install_pulsar_chart ${PULSAR_HOME}/.ci/values-common.yaml ${PULSAR_HOME}/${VALUES_FILE} ${extra_opts}
+test_action="produce-consume"
+if [[ "$UPGRADE_FROM_VERSION" != "" ]]; then
+    # install older version of pulsar chart
+    PULSAR_CHART_VERSION="$UPGRADE_FROM_VERSION"
+    ci::install_pulsar_chart install ${PULSAR_HOME}/.ci/values-common.yaml ${PULSAR_HOME}/${VALUES_FILE} ${extra_opts}    
+    install_type="upgrade"
+    echo "Wait 10 seconds"
+    sleep 10
+    # test that we can access the admin api
+    ci::test_pulsar_admin_api_access
+    # produce messages with old version of pulsar and consume with new version
+    ci::test_pulsar_producer_consumer "produce"
+    test_action="consume"
+fi
 
+PULSAR_CHART_VERSION="local"
+# install (or upgrade) pulsar chart
+ci::install_pulsar_chart ${install_type} ${PULSAR_HOME}/.ci/values-common.yaml ${PULSAR_HOME}/${VALUES_FILE} ${extra_opts}
+
+echo "Wait 10 seconds"
+sleep 10
+
+# check pulsar environment
+ci::check_pulsar_environment
+
+# test that we can access the admin api
+ci::test_pulsar_admin_api_access
 # test producer/consumer
-ci::test_pulsar_producer_consumer
+ci::test_pulsar_producer_consumer "${test_action}"
 
 if [[ "x${FUNCTION}" == "xtrue" ]]; then
-    # install cert manager
+    # test functions
     ci::test_pulsar_function
 fi
 

.ci/helm.sh

@@ -20,13 +20,15 @@
 BINDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
 PULSAR_HOME="$(cd "${BINDIR}/.." && pwd)"
 CHARTS_HOME=${PULSAR_HOME}
+PULSAR_CHART_LOCAL=${CHARTS_HOME}/charts/pulsar
+PULSAR_CHART_VERSION=${PULSAR_CHART_VERSION:-"local"}
 OUTPUT_BIN=${CHARTS_HOME}/output/bin
 KIND_BIN=$OUTPUT_BIN/kind
 HELM=${OUTPUT_BIN}/helm
 KUBECTL=${OUTPUT_BIN}/kubectl
 NAMESPACE=pulsar
 CLUSTER=pulsar-ci
-CLUSTER_ID=$(uuidgen)
+: ${CLUSTER_ID:=$(uuidgen)}
 K8S_LOGS_DIR="${K8S_LOGS_DIR:-/tmp/k8s-logs}"
 export PATH="$OUTPUT_BIN:$PATH"
 
@@ -100,10 +102,13 @@ function ci::collect_k8s_logs() {
 }
 
 function ci::install_pulsar_chart() {
-    local common_value_file=$1
+    local install_type=$1
-    local value_file=$2
+    local common_value_file=$2
-    local extra_opts=$3
+    local value_file=$3
+    local extra_opts=$4
+    local install_args
 
+    if [[ "${install_type}" == "install" ]]; then
       echo "Installing the pulsar chart"
       ${KUBECTL} create namespace ${NAMESPACE}
       ci::install_cert_manager
@@ -111,12 +116,43 @@ function ci::install_pulsar_chart() {
       ${CHARTS_HOME}/scripts/pulsar/prepare_helm_release.sh -k ${CLUSTER} -n ${NAMESPACE} ${extra_opts}
       sleep 10
 
-    echo ${HELM} dependency update ${CHARTS_HOME}/charts/pulsar
+      # install metallb for loadbalancer support
-    ${HELM} dependency update ${CHARTS_HOME}/charts/pulsar
+      # following instructions from https://kind.sigs.k8s.io/docs/user/loadbalancer/
-    echo ${HELM} install --set initialize=true --values ${common_value_file} --values ${value_file} ${CLUSTER} ${CHARTS_HOME}/charts/pulsar
+      ${KUBECTL} apply -f https://raw.githubusercontent.com/metallb/metallb/v0.13.12/config/manifests/metallb-native.yaml
-    ${HELM} template --values ${common_value_file} --values ${value_file} ${CLUSTER} ${CHARTS_HOME}/charts/pulsar
+      # wait until metallb is ready
-    ${HELM} install --set initialize=true --values ${common_value_file} --values ${value_file} --namespace=${NAMESPACE} ${CLUSTER} ${CHARTS_HOME}/charts/pulsar
+      ${KUBECTL} wait --namespace metallb-system \
+                --for=condition=ready pod \
+                --selector=app=metallb \
+                --timeout=90s
+      # configure metallb
+      ${KUBECTL} apply -f ${BINDIR}/metallb/metallb-config.yaml
 
+      install_args=""
+    else 
+      install_args="--wait --wait-for-jobs --timeout 300s --debug"
+    fi
+
+    CHART_ARGS=""
+    if [[ "${PULSAR_CHART_VERSION}" == "local" ]]; then
+      set -x
+      ${HELM} dependency update ${PULSAR_CHART_LOCAL}
+      set +x
+      CHART_ARGS="${PULSAR_CHART_LOCAL}"
+    else
+      set -x
+      ${HELM} repo add apache https://pulsar.apache.org/charts
+      set +x
+      CHART_ARGS="apache/pulsar --dependency-update"
+      if [[ "${PULSAR_CHART_VERSION}" != "latest" ]]; then
+        CHART_ARGS="${CHART_ARGS} --version ${PULSAR_CHART_VERSION}"
+      fi
+    fi
+    set -x
+    ${HELM} template --values ${common_value_file} --values ${value_file} ${CLUSTER} ${CHART_ARGS}
+    ${HELM} ${install_type} --values ${common_value_file} --values ${value_file} --namespace=${NAMESPACE} ${CLUSTER} ${CHART_ARGS} ${install_args}
+    set +x
+
+    if [[ "${install_type}" == "install" ]]; then
       echo "wait until broker is alive"
       WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep ${CLUSTER}-broker | wc -l)
       counter=1
@@ -141,7 +177,7 @@ function ci::install_pulsar_chart() {
         WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep ${CLUSTER}-broker | wc -l)
       done
       timeout 300s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-broker; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
-    timeout 120s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until [ "$(curl -L http://pulsar-ci-broker:8080/status.html)" == "OK" ]; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
+      timeout 120s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until [ "$(curl -s -L http://pulsar-ci-broker:8080/status.html)" == "OK" ]; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
 
       WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep ${CLUSTER}-proxy | wc -l)
       counter=1
@@ -161,7 +197,15 @@ function ci::install_pulsar_chart() {
         WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep ${CLUSTER}-proxy | wc -l)
       done
       timeout 300s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-proxy; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
-    # ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until [ "$(curl -L http://pulsar-ci-proxy:8080/status.html)" == "OK" ]; do sleep 3; done'
+      echo "Install complete"
+    else
+      echo "wait until broker is alive"
+      timeout 300s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-broker; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
+      timeout 120s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until [ "$(curl -s -L http://pulsar-ci-broker:8080/status.html)" == "OK" ]; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
+      echo "wait until proxy is alive"
+      timeout 300s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-proxy; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
+      echo "Upgrade complete"
+    fi
 }
 
 helm_values_cached=""
@@ -173,28 +217,66 @@ function ci::helm_values_for_deployment() {
     printf "%s" "${helm_values_cached}"
 }
 
-function ci::test_pulsar_producer_consumer() {
+function ci::check_pulsar_environment() {
-    sleep 120
+    echo "Wait until pulsar-ci-broker is ready"
     ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-broker; do sleep 3; done'
+    echo "Wait until pulsar-ci-proxy is ready"
     ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-proxy; do sleep 3; done'
+    echo "bookie-0 disk usage"
     ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-bookie-0 -- df -h
+    echo "bookie-0 bookkeeper.conf"
     ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-bookie-0 -- cat conf/bookkeeper.conf
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/bookkeeper shell listbookies -rw
+    echo "bookie-0 bookies list (rw)"
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/bookkeeper shell listbookies -ro
+    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/bookkeeper shell listbookies -rw | grep ListBookiesCommand
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin tenants create pulsar-ci
+    echo "bookie-0 bookies list (ro)"
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin namespaces create pulsar-ci/test
+    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/bookkeeper shell listbookies -ro | grep ListBookiesCommand
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin topics create pulsar-ci/test/test-topic
+}
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin topics create-subscription -s test pulsar-ci/test/test-topic
+
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client produce -m "test-message" pulsar-ci/test/test-topic
+# function to retry a given commend 3 times with a backoff of 10 seconds in between
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client consume -s test pulsar-ci/test/test-topic
+function ci::retry() {
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin topics create-subscription -s test2 pulsar-ci/test/test-topic
+  local n=1
+  local max=3
+  local delay=10
+  while true; do
+    "$@" && break || {
+      if [[ $n -lt $max ]]; then
+        ((n++))
+        echo "::warning::Command failed. Attempt $n/$max:"
+        sleep $delay
+      else
+        fail "::error::The command has failed after $n attempts."
+      fi
+    }
+  done
+}
+
+function ci::test_pulsar_admin_api_access() {
+  ci::retry ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin tenants list
+}
+
+function ci::test_pulsar_producer_consumer() {
+    action="${1:-"produce-consume"}"
+    echo "Testing with ${action}"
     if [[ "$(ci::helm_values_for_deployment | yq .tls.proxy.enabled)" == "true" ]]; then
       PROXY_URL="pulsar+ssl://pulsar-ci-proxy:6651"
     else
       PROXY_URL="pulsar://pulsar-ci-proxy:6650"
     fi
+    set -x
+    if [[ "${action}" == "produce" || "${action}" == "produce-consume" ]]; then
+      ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin tenants create pulsar-ci
+      ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin namespaces create pulsar-ci/test
+      ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin topics create pulsar-ci/test/test-topic
+      ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin topics create-subscription -s test pulsar-ci/test/test-topic
+      ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client produce -m "test-message" pulsar-ci/test/test-topic
+      ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin topics create-subscription -s test2 pulsar-ci/test/test-topic
       ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client --url "${PROXY_URL}" produce -m "test-message2" pulsar-ci/test/test-topic
+    fi
+    if [[ "${action}" == "consume" || "${action}" == "produce-consume" ]]; then
+      ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client consume -s test pulsar-ci/test/test-topic
       ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client --url "${PROXY_URL}" consume -s test2 pulsar-ci/test/test-topic
+    fi
+    set +x
 }
 
 function ci::wait_function_running() {
@@ -219,14 +301,7 @@ function ci::wait_message_processed() {
 }
 
 function ci::test_pulsar_function() {
-    sleep 120
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-broker; do sleep 3; done'
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-proxy; do sleep 3; done'
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-bookie-0 -- df -h
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/bookkeeper shell listbookies -rw
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/bookkeeper shell listbookies -ro
     ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin functions create --tenant pulsar-ci --namespace test --name test-function --inputs "pulsar-ci/test/test_input" --output "pulsar-ci/test/test_output" --parallelism 1 --classname org.apache.pulsar.functions.api.examples.ExclamationFunction --jar /pulsar/examples/api-examples.jar
-
     # wait until the function is running
     # TODO: re-enable function test
     # ci::wait_function_running

.ci/metallb/metallb-config.yaml

@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+apiVersion: metallb.io/v1beta1
+kind: IPAddressPool
+metadata:
+  name: example
+  namespace: metallb-system
+spec:
+  addresses:
+  - 172.19.255.200-172.19.255.250
+---
+apiVersion: metallb.io/v1beta1
+kind: L2Advertisement
+metadata:
+  name: empty
+  namespace: metallb-system

.github/workflows/pulsar-helm-chart-ci.yaml

@@ -136,7 +136,7 @@ jobs:
           action: wait
 
   install-chart-tests:
-    name: ${{ matrix.testScenario.name }} - k8s ${{ matrix.k8sVersion.version }} - Install
+    name: ${{ matrix.testScenario.name }} - k8s ${{ matrix.k8sVersion.version }} - ${{ matrix.testScenario.type || 'install' }}
     runs-on: ubuntu-22.04
     timeout-minutes: ${{ matrix.testScenario.timeout || 45 }}
     needs: ['preconditions', 'ct-lint']
@@ -151,9 +151,10 @@ jobs:
           - version: "1.27.3"
             kind_image_tag: v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72
         testScenario:
-          - name: Basic
+          - name: Upgrade latest released version
-            values_file: .ci/clusters/values-basic.yaml
+            values_file: .ci/clusters/values-upgrade.yaml
-            shortname: basic
+            shortname: upgrade
+            type: upgrade
           - name: Use Pulsar Image
             values_file: .ci/clusters/values-pulsar-image.yaml
             shortname: pulsar-image
@@ -181,6 +182,23 @@ jobs:
           - name: PSP
             values_file: .ci/clusters/values-psp.yaml
             shortname: psp
+        include:
+          - k8sVersion:
+              version: "1.21.14"
+              kind_image_tag: v1.21.14@sha256:8a4e9bb3f415d2bb81629ce33ef9c76ba514c14d707f9797a01e3216376ba093
+            testScenario:
+              name: "Upgrade TLS"
+              values_file: .ci/clusters/values-tls.yaml
+              shortname: tls
+              type: upgrade
+          - k8sVersion:
+              version: "1.21.14"
+              kind_image_tag: v1.21.14@sha256:8a4e9bb3f415d2bb81629ce33ef9c76ba514c14d707f9797a01e3216376ba093
+            testScenario:
+              name: "Upgrade PSP"
+              values_file: .ci/clusters/values-psp.yaml
+              shortname: psp
+              type: upgrade
     env:
       k8sVersion: ${{ matrix.k8sVersion.kind_image_tag }}
       KUBECTL_VERSION: ${{ matrix.k8sVersion.version }}
@@ -234,13 +252,16 @@ jobs:
         with:
           limit-access-to-actor: true
 
-      - name: Run chart-testing (install)
+      - name: Run chart-testing (${{ matrix.testScenario.type || 'install' }})
         run: |
           case "${{ matrix.testScenario.shortname }}" in
             "jwt-symmetric")
               export SYMMETRIC=true
               ;;
           esac
+          if [[ "${{ matrix.testScenario.type || 'install' }}" == "upgrade" ]]; then
+            export UPGRADE_FROM_VERSION=latest
+          fi
           .ci/chart_test.sh ${{ matrix.testScenario.values_file }}
 
       - name: Collect k8s logs on failure