[CI] Collect and upload k8s logs on failure (#215)

.ci/chart_test.sh

@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -20,8 +21,8 @@
 set -e
 
 
-BINDIR=`dirname "$0"`
-PULSAR_HOME=`cd ${BINDIR}/..;pwd`
+BINDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+PULSAR_HOME="$(cd "${BINDIR}/.." && pwd)"
 VALUES_FILE=$1
 TLS=${TLS:-"false"}
 SYMMETRIC=${SYMMETRIC:-"false"}

.ci/ct.sh

@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -17,8 +18,6 @@
 # under the License.
 #
 
-#!/usr/bin/env bash
-
 set -o errexit
 set -o nounset
 set -o pipefail
@@ -164,4 +163,4 @@ docker_exec() {
     docker exec --interactive ct "$@"
 }
 
-main "$@"
+main "$@"

.ci/helm.sh

@@ -17,9 +17,8 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-
-BINDIR=`dirname "$0"`
-PULSAR_HOME=`cd ${BINDIR}/..;pwd`
+BINDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+PULSAR_HOME="$(cd "${BINDIR}/.." && pwd)"
 CHARTS_HOME=${PULSAR_HOME}
 OUTPUT_BIN=${CHARTS_HOME}/output/bin
 KIND_BIN=$OUTPUT_BIN/kind
@@ -28,6 +27,7 @@ KUBECTL=${OUTPUT_BIN}/kubectl
 NAMESPACE=pulsar
 CLUSTER=pulsar-ci
 CLUSTER_ID=$(uuidgen)
+K8S_LOGS_DIR="${K8S_LOGS_DIR:-/tmp/k8s-logs}"
 export PATH="$OUTPUT_BIN:$PATH"
 
 # brew package 'coreutils' is required on MacOSX
@@ -86,6 +86,18 @@ function ci::print_pod_logs() {
     done;
 }
 
+function ci::collect_k8s_logs() {
+    mkdir -p "${K8S_LOGS_DIR}" && cd "${K8S_LOGS_DIR}"
+    echo "Collecting k8s logs to ${K8S_LOGS_DIR}"
+    for k8sobject in $(${KUBECTL} get pods,jobs -n ${NAMESPACE} -l app=pulsar -o=name); do
+      ${KUBECTL} logs -n ${NAMESPACE} "$k8sobject" --all-containers=true --ignore-errors=true --prefix=true > "${k8sobject}.$$.log.txt"
+      ${KUBECTL} logs -n ${NAMESPACE} "$k8sobject" --all-containers=true --ignore-errors=true --prefix=true --previous=true > "${k8sobject}.previous.$$.log.txt"
+    done;
+    ${KUBECTL} get events --sort-by=.lastTimestamp -A > events.$$.log.txt
+    ${KUBECTL} get events --sort-by=.lastTimestamp -A -o yaml > events.$$.log.yaml
+    ${KUBECTL} get -n ${NAMESPACE} all -o yaml > k8s_resources.$$.yaml
+}
+
 function ci::install_pulsar_chart() {
     local value_file=$1
     local extra_opts=$2

.ci/lint.sh

@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -19,7 +20,7 @@
 
 set -e
 
-BINDIR=`dirname "$0"`
-CI_HOME=`cd ${BINDIR};pwd`
+BINDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+CI_HOME="${BINDIR}"
 
 ${CI_HOME}/ct.sh -c lint

.ci/release.sh

@@ -18,8 +18,9 @@
 # under the License.
 #
 
-BINDIR=`dirname "$0"`
-CHARTS_HOME=`cd ${BINDIR}/..;pwd`
+
+BINDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+CHARTS_HOME="$(cd "${BINDIR}/.." && pwd)"
 CHARTS_PKGS=${CHARTS_HOME}/.chart-packages
 CHARTS_INDEX=${CHARTS_HOME}/.chart-index
 CHARTS_REPO=${CHARTS_REPO:-"https://pulsar.apache.org/charts/"}

.github/workflows/pulsar.yml

@@ -49,3 +49,19 @@ jobs:
           .ci/chart_test.sh .ci/clusters/values-local-pv.yaml
         # Only build a kind cluster if there are chart changes to test.
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+
+      - name: Collect k8s logs on failure
+        if: ${{ cancelled() || failure() }}
+        continue-on-error: true
+        shell: bash
+        run: |
+          source .ci/helm.sh
+          ci::collect_k8s_logs
+
+      - name: Upload k8s logs on failure
+        uses: actions/upload-artifact@v2
+        if: ${{ cancelled() || failure() }}
+        continue-on-error: true
+        with:
+          name: k8s-logs
+          path: /tmp/k8s-logs

.github/workflows/pulsar_bk_tls.yml

@@ -48,3 +48,19 @@ jobs:
           .ci/chart_test.sh .ci/clusters/values-bk-tls.yaml
         # Only build a kind cluster if there are chart changes to test.
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+
+      - name: Collect k8s logs on failure
+        if: ${{ cancelled() || failure() }}
+        continue-on-error: true
+        shell: bash
+        run: |
+          source .ci/helm.sh
+          ci::collect_k8s_logs
+
+      - name: Upload k8s logs on failure
+        uses: actions/upload-artifact@v2
+        if: ${{ cancelled() || failure() }}
+        continue-on-error: true
+        with:
+          name: k8s-logs
+          path: /tmp/k8s-logs

.github/workflows/pulsar_broker_tls.yml

@@ -49,3 +49,19 @@ jobs:
           .ci/chart_test.sh .ci/clusters/values-broker-tls.yaml
         # Only build a kind cluster if there are chart changes to test.
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+
+      - name: Collect k8s logs on failure
+        if: ${{ cancelled() || failure() }}
+        continue-on-error: true
+        shell: bash
+        run: |
+          source .ci/helm.sh
+          ci::collect_k8s_logs
+
+      - name: Upload k8s logs on failure
+        uses: actions/upload-artifact@v2
+        if: ${{ cancelled() || failure() }}
+        continue-on-error: true
+        with:
+          name: k8s-logs
+          path: /tmp/k8s-logs

.github/workflows/pulsar_function.yml

@@ -51,3 +51,19 @@ jobs:
           FUNCTION: "true"
         # Only build a kind cluster if there are chart changes to test.
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+
+      - name: Collect k8s logs on failure
+        if: ${{ cancelled() || failure() }}
+        continue-on-error: true
+        shell: bash
+        run: |
+          source .ci/helm.sh
+          ci::collect_k8s_logs
+
+      - name: Upload k8s logs on failure
+        uses: actions/upload-artifact@v2
+        if: ${{ cancelled() || failure() }}
+        continue-on-error: true
+        with:
+          name: k8s-logs
+          path: /tmp/k8s-logs

.github/workflows/pulsar_image.yml

@@ -49,3 +49,19 @@ jobs:
           .ci/chart_test.sh .ci/clusters/values-pulsar-image.yaml
         # Only build a kind cluster if there are chart changes to test.
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+
+      - name: Collect k8s logs on failure
+        if: ${{ cancelled() || failure() }}
+        continue-on-error: true
+        shell: bash
+        run: |
+          source .ci/helm.sh
+          ci::collect_k8s_logs
+
+      - name: Upload k8s logs on failure
+        uses: actions/upload-artifact@v2
+        if: ${{ cancelled() || failure() }}
+        continue-on-error: true
+        with:
+          name: k8s-logs
+          path: /tmp/k8s-logs

.github/workflows/pulsar_jwt_asymmetric.yml

@@ -51,3 +51,19 @@ jobs:
           SYMMETRIC: "false"
         # Only build a kind cluster if there are chart changes to test.
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+
+      - name: Collect k8s logs on failure
+        if: ${{ cancelled() || failure() }}
+        continue-on-error: true
+        shell: bash
+        run: |
+          source .ci/helm.sh
+          ci::collect_k8s_logs
+
+      - name: Upload k8s logs on failure
+        uses: actions/upload-artifact@v2
+        if: ${{ cancelled() || failure() }}
+        continue-on-error: true
+        with:
+          name: k8s-logs
+          path: /tmp/k8s-logs

.github/workflows/pulsar_jwt_symmetric.yml

@@ -51,3 +51,19 @@ jobs:
           SYMMETRIC: "true"
         # Only build a kind cluster if there are chart changes to test.
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+
+      - name: Collect k8s logs on failure
+        if: ${{ cancelled() || failure() }}
+        continue-on-error: true
+        shell: bash
+        run: |
+          source .ci/helm.sh
+          ci::collect_k8s_logs
+
+      - name: Upload k8s logs on failure
+        uses: actions/upload-artifact@v2
+        if: ${{ cancelled() || failure() }}
+        continue-on-error: true
+        with:
+          name: k8s-logs
+          path: /tmp/k8s-logs

.github/workflows/pulsar_tls.yml

@@ -49,3 +49,19 @@ jobs:
           .ci/chart_test.sh .ci/clusters/values-tls.yaml
         # Only build a kind cluster if there are chart changes to test.
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+
+      - name: Collect k8s logs on failure
+        if: ${{ cancelled() || failure() }}
+        continue-on-error: true
+        shell: bash
+        run: |
+          source .ci/helm.sh
+          ci::collect_k8s_logs
+
+      - name: Upload k8s logs on failure
+        uses: actions/upload-artifact@v2
+        if: ${{ cancelled() || failure() }}
+        continue-on-error: true
+        with:
+          name: k8s-logs
+          path: /tmp/k8s-logs

.github/workflows/pulsar_zk_tls.yml

@@ -49,3 +49,19 @@ jobs:
           .ci/chart_test.sh .ci/clusters/values-zk-tls.yaml
         # Only build a kind cluster if there are chart changes to test.
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+
+      - name: Collect k8s logs on failure
+        if: ${{ cancelled() || failure() }}
+        continue-on-error: true
+        shell: bash
+        run: |
+          source .ci/helm.sh
+          ci::collect_k8s_logs
+
+      - name: Upload k8s logs on failure
+        uses: actions/upload-artifact@v2
+        if: ${{ cancelled() || failure() }}
+        continue-on-error: true
+        with:
+          name: k8s-logs
+          path: /tmp/k8s-logs

.github/workflows/pulsar_zkbk_tls.yml

@@ -49,3 +49,19 @@ jobs:
           .ci/chart_test.sh .ci/clusters/values-zkbk-tls.yaml
         # Only build a kind cluster if there are chart changes to test.
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+
+      - name: Collect k8s logs on failure
+        if: ${{ cancelled() || failure() }}
+        continue-on-error: true
+        shell: bash
+        run: |
+          source .ci/helm.sh
+          ci::collect_k8s_logs
+
+      - name: Upload k8s logs on failure
+        uses: actions/upload-artifact@v2
+        if: ${{ cancelled() || failure() }}
+        continue-on-error: true
+        with:
+          name: k8s-logs
+          path: /tmp/k8s-logs

.github/workflows/release.yml

@@ -46,7 +46,7 @@ jobs:
         id: check_changes
         run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
 
-      - name: Install chart
+      - name: Release chart
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         env:
           GITHUB_TOKEN: ${{ secrets.PULSARBOT_TOKEN }}

.github/workflows/verify_release.yml

@@ -46,7 +46,7 @@ jobs:
         id: check_changes
         run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
 
-      - name: Install chart
+      - name: Verify release
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}