Upgrade kube-prometheus-stack to 56.x.x version (#445)

* Upgrade to kube-prometheus-stack 56.x.x * Add CI test case for kube-prometheus-stack upgrade * Add "--force-conflicts" flag
2024-01-26 03:07:10 -08:00 · 2024-01-26 03:07:10 -08:00 · 72a8fb6b3e
commit 72a8fb6b3e
parent 727dccb013
6 changed files with 93 additions and 2 deletions
--- a/.ci/chart_test.sh
+++ b/.ci/chart_test.sh
@ -52,6 +52,11 @@ if [[ "$UPGRADE_FROM_VERSION" != "" ]]; then
    # produce messages with old version of pulsar and consume with new version
    ci::test_pulsar_producer_consumer "produce"
    test_action="consume"
+
+    if [[ "$(ci::helm_values_for_deployment | yq .kube-prometheus-stack.enabled)" == "true" ]]; then
+        echo "Upgrade Prometheus Operator CRDs before upgrading the deployment"
+        ${PULSAR_HOME}/scripts/kube-prometheus-stack/upgrade_prometheus_operator_crds.sh
+    fi
 fi

 PULSAR_CHART_VERSION="local"
--- a/.ci/clusters/values-prometheus-grafana.yaml
+++ b/.ci/clusters/values-prometheus-grafana.yaml
@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+kube-prometheus-stack:
+  enabled: true
+  prometheus:
+    enabled: true
+  grafana:
+    enabled: true
+    adminPassword: pulsar-ci-admin
+  alertmanager:
+    enabled: false
+  prometheus-node-exporter:
+    enabled: true
--- a/.github/workflows/pulsar-helm-chart-ci.yaml
+++ b/.github/workflows/pulsar-helm-chart-ci.yaml
@ -199,6 +199,15 @@ jobs:
              values_file: .ci/clusters/values-psp.yaml
              shortname: psp
              type: upgrade
+          - k8sVersion:
+              version: "1.21.14"
+              kind_image_tag: v1.21.14@sha256:8a4e9bb3f415d2bb81629ce33ef9c76ba514c14d707f9797a01e3216376ba093
+            testScenario:
+              name: "Upgrade kube-prometheus-stack"
+              values_file: .ci/clusters/values-prometheus-grafana.yaml
+              shortname: prometheus-grafana
+              type: upgrade
+              upgradeFromVersion: 3.2.0
          - k8sVersion:
              version: "1.21.14"
              kind_image_tag: v1.21.14@sha256:8a4e9bb3f415d2bb81629ce33ef9c76ba514c14d707f9797a01e3216376ba093
@ -270,7 +279,7 @@ jobs:
              ;;
          esac
          if [[ "${{ matrix.testScenario.type || 'install' }}" == "upgrade" ]]; then
-            export UPGRADE_FROM_VERSION=latest
+            export UPGRADE_FROM_VERSION="${{ matrix.testScenario.upgradeFromVersion || 'latest' }}"
          fi
          .ci/chart_test.sh ${{ matrix.testScenario.values_file }}

--- a/README.md
+++ b/README.md
@ -204,6 +204,19 @@ helm upgrade -f pulsar.yaml \

 For more detailed information, see our [Upgrading](http://pulsar.apache.org/docs/helm-upgrade/) guide.

+## Upgrading from Helm Chart version 3.0.0-3.2.x to 3.3.0 version and above
+
+The kube-prometheus-stack version has been upgraded to 56.x.x in Pulsar Helm Chart version 3.3.0 .
+Before running "helm upgrade", you should first upgrade the Prometheus Operator CRDs as [instructed
+in kube-prometheus-stack upgrade notes](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack#from-55x-to-56x). 
+
+There's a script to run the required commands:
+```shell
+./scripts/kube-prometheus-stack/upgrade_prometeheus_operator_crds.sh
+```
+
+After, this you can proceed with `helm upgrade`.
+
 ## Upgrading to Apache Pulsar 2.10.0 and above (or Helm Chart version 3.0.0 and above)

 The 2.10.0+ Apache Pulsar docker image is a non-root container, by default. That complicates an upgrade to 2.10.0
--- a/charts/pulsar/Chart.yaml
+++ b/charts/pulsar/Chart.yaml
@ -33,6 +33,6 @@ maintainers:
    email: dev@pulsar.apache.org
 dependencies:
  - name: kube-prometheus-stack
-    version: 41.x.x
+    version: 56.x.x
    repository: https://prometheus-community.github.io/helm-charts
    condition: kube-prometheus-stack.enabled
--- a/scripts/kube-prometheus-stack/upgrade_prometheus_operator_crds.sh
+++ b/scripts/kube-prometheus-stack/upgrade_prometheus_operator_crds.sh
@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# This script is used to upgrade the Prometheus Operator CRDs before running "helm upgrade"
+# source: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack#from-55x-to-56x
+# "Run these commands to update the CRDs before applying the upgrade."
+# "--force-conflicts" is required to upgrade the CRDs. Following instructions from https://github.com/prometheus-community/helm-charts/issues/2489
+kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml
+kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml
+kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml
+kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml
+kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml
+kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml
+kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml
+kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml
+kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
+kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml