Feat/cacerts (#619)

* extra volume mounts for oxia coordinator

* .

* .

.ci/auth/keycloak/1-client-template.json

@@ -0,0 +1,73 @@
+{
+  "clientId": $ARGS.named.CLIENT_ID,
+  "enabled": true,
+  "clientAuthenticatorType": "client-secret",
+  "secret": $ARGS.named.CLIENT_SECRET,
+  "standardFlowEnabled" : false,
+  "implicitFlowEnabled" : false,
+  "serviceAccountsEnabled": true,
+  "protocol": "openid-connect",
+  "attributes": {
+    "realm_client": "false",
+    "oidc.ciba.grant.enabled": "false",
+    "client.secret.creation.time": "1735689600",
+    "backchannel.logout.session.required": "true",
+    "standard.token.exchange.enabled": "false",
+    "frontchannel.logout.session.required": "true",
+    "oauth2.device.authorization.grant.enabled": "false",
+    "display.on.consent.screen": "false",
+    "backchannel.logout.revoke.offline.tokens": "false"
+  },
+  "protocolMappers": [
+    {
+      "name": "sub",
+      "protocol": "openid-connect",
+      "protocolMapper": "oidc-hardcoded-claim-mapper",
+      "consentRequired": false,
+      "config": {
+        "introspection.token.claim": "true",
+        "claim.value": $ARGS.named.SUB_CLAIM_VALUE,
+        "userinfo.token.claim": "true",
+        "id.token.claim": "true",
+        "lightweight.claim": "false",
+        "access.token.claim": "true",
+        "claim.name": "sub",
+        "jsonType.label": "String",
+        "access.tokenResponse.claim": "false"
+      }
+    },
+    {
+      "name": "nbf",
+      "protocol": "openid-connect",
+      "protocolMapper": "oidc-hardcoded-claim-mapper",
+      "consentRequired": false,
+      "config": {
+        "introspection.token.claim": "true",
+        "claim.value": "1735689600",
+        "userinfo.token.claim": "true",
+        "id.token.claim": "true",
+        "lightweight.claim": "false",
+        "access.token.claim": "true",
+        "claim.name": "nbf",
+        "jsonType.label": "long",
+        "access.tokenResponse.claim": "false"
+      }
+    }
+  ],
+  "defaultClientScopes": [
+    "web-origins",
+    "service_account",
+    "acr",
+    "profile",
+    "roles",
+    "basic",
+    "email"
+  ],
+  "optionalClientScopes": [
+    "address",
+    "phone",
+    "organization",
+    "offline_access",
+    "microprofile-jwt"
+  ]
+}

.ci/auth/keycloak/README.md

@@ -0,0 +1,26 @@
+# Keycloak
+
+Keycloak is used to validate OIDC configuration.
+
+To create the pulsar realm configuration, we use :
+
+* `0-realm-pulsar-partial-export.json` : after creating pulsar realm in Keycloack UI, this file is the result of the partial export in Keycloak UI without options.
+* `1-client-template.json` : this is the template to create pulsar clients.
+
+To create the final `realm-pulsar.json`, merge files with `jq` command :
+
+* create a client with `CLIENT_ID`, `CLIENT_SECRET` and `SUB_CLAIM_VALUE` :
+
+```
+CLIENT_ID=xx
+CLIENT_SECRET=yy
+SUB_CLAIM_VALUE=zz
+
+jq -n --arg CLIENT_ID "$CLIENT_ID" --arg CLIENT_SECRET "$CLIENT_SECRET" --arg SUB_CLAIM_VALUE "$SUB_CLAIM_VALUE" 1-client-template.json > client.json
+```
+
+* then merge the realm and the client :
+
+```
+jq '.clients += [input]' 0-realm-pulsar-partial-export.json client.json > realm-pulsar.json
+```

.ci/auth/keycloak/values.yaml

@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+tls:
+  enabled: false
+# This block sets up an example Pulsar Realm
+# https://www.keycloak.org/server/importExport#_importing_a_realm_from_a_directory
+extraEnvVars:
+  - name: KEYCLOAK_EXTRA_ARGS
+    value: "--import-realm"
+extraVolumes:
+  - name: realm-config
+    secret:
+      secretName: keycloak-ci-realm-config
+extraVolumeMounts:
+  - name: realm-config
+    mountPath: "/opt/bitnami/keycloak/data/import"
+    readOnly: true

.ci/auth/oauth2/credentials_file.json

@@ -0,0 +1,5 @@
+{
+    "type": "client_credentials",
+    "client_id": $ARGS.named.CLIENT_ID,
+    "client_secret": $ARGS.named.CLIENT_SECRET
+}

.ci/chart_test.sh

@@ -27,27 +27,90 @@ VALUES_FILE=$1
 TLS=${TLS:-"false"}
 SYMMETRIC=${SYMMETRIC:-"false"}
 FUNCTION=${FUNCTION:-"false"}
+MANAGER=${MANAGER:-"false"}
+ALLOW_LOADBALANCERS=${ALLOW_LOADBALANCERS:-"false"}
 
 source ${PULSAR_HOME}/.ci/helm.sh
 
 # create cluster
 ci::create_cluster
 
-extra_opts=""
+ci::helm_repo_add
+
+extra_opts=()
+
+# Add any arguments after $1 to extra_opts
+shift # Remove $1 from the argument list
+while [[ $# -gt 0 ]]; do
+    extra_opts+=("$1")
+    shift
+done
+
 if [[ "x${SYMMETRIC}" == "xtrue" ]]; then
-    extra_opts="-s"
+    extra_opts+=("-s")
 fi
 
-# install pulsar chart
-ci::install_pulsar_chart ${PULSAR_HOME}/${VALUES_FILE} ${extra_opts}
+if [[ "x${EXTRA_SUPERUSERS}" != "x" ]]; then
+    extra_opts+=("--pulsar-superusers" "proxy-admin,broker-admin,admin,${EXTRA_SUPERUSERS}")
+fi
 
+install_type="install"
+test_action="produce-consume"
+if [[ "$UPGRADE_FROM_VERSION" != "" ]]; then
+    ALLOW_LOADBALANCERS="true"
+    # install older version of pulsar chart
+    PULSAR_CHART_VERSION="$UPGRADE_FROM_VERSION"
+
+    # Install Prometheus Operator CRDs using the upgrade script since kube-prometheus-stack is now disabled before the upgrade
+    ${PULSAR_HOME}/scripts/kube-prometheus-stack/upgrade_prometheus_operator_crds.sh
+
+    ci::install_pulsar_chart install ${PULSAR_HOME}/.ci/values-common.yaml ${PULSAR_HOME}/${VALUES_FILE} --set kube-prometheus-stack.enabled=false "${extra_opts[@]}"
+    install_type="upgrade"
+    echo "Wait 10 seconds"
+    sleep 10
+    # check pulsar environment
+    ci::check_pulsar_environment
+    # test that we can access the admin api
+    ci::test_pulsar_admin_api_access
+    # produce messages with old version of pulsar and consume with new version
+    ci::test_pulsar_producer_consumer "produce"
+    test_action="consume"
+
+    if [[ "$(ci::helm_values_for_deployment | yq .victoria-metrics-k8s-stack.enabled)" == "true" ]]; then
+        echo "Upgrade Victoria Metrics Operator CRDs before upgrading the deployment"
+        ${PULSAR_HOME}/scripts/victoria-metrics-k8s-stack/upgrade_vm_operator_crds.sh
+    fi
+fi
+
+PULSAR_CHART_VERSION="local"
+# install (or upgrade) pulsar chart
+ci::install_pulsar_chart ${install_type} ${PULSAR_HOME}/.ci/values-common.yaml ${PULSAR_HOME}/${VALUES_FILE} "${extra_opts[@]}"
+
+echo "Wait 10 seconds"
+sleep 10
+
+# check that there aren't any loadbalancers if ALLOW_LOADBALANCERS is false
+if [[ "${ALLOW_LOADBALANCERS}" == "false" ]]; then
+    ci::check_loadbalancers
+fi
+
+# check pulsar environment
+ci::check_pulsar_environment
+
+# test that we can access the admin api
+ci::test_pulsar_admin_api_access
 # test producer/consumer
-ci::test_pulsar_producer_consumer
+ci::test_pulsar_producer_consumer "${test_action}"
 
-if [[ "x${FUNCTION}" == "xtrue" ]]; then
-    # install cert manager
+if [[ "$(ci::helm_values_for_deployment | yq .components.functions)" == "true" ]]; then
+    # test functions
     ci::test_pulsar_function
 fi
 
+if [[ "$(ci::helm_values_for_deployment | yq .components.pulsar_manager)" == "true" ]]; then
+    # test manager
+    ci::test_pulsar_manager
+fi
+
 # delete the cluster
 ci::delete_cluster

.ci/clusters/values-bk-tls.yaml

@@ -17,71 +17,6 @@
 # under the License.
 #
 
-kube-prometheus-stack:
-  enabled: false
-  prometheusOperator:
-    enabled: false
-  grafana:
-    enabled: false
-  alertmanager:
-    enabled: false
-  prometheus:
-    enabled: false
-
-# disabled AntiAffinity
-affinity:
-  anti_affinity: false
-
-# disable auto recovery and pulsar manager
-components:
-  autorecovery: false
-  pulsar_manager: false
-
-zookeeper:
-  replicaCount: 1
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-
-bookkeeper:
-  replicaCount: 3
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-  configData:
-    diskUsageThreshold: "0.999"
-    diskUsageWarnThreshold: "0.999"
-    PULSAR_PREFIX_diskUsageThreshold: "0.999"
-    PULSAR_PREFIX_diskUsageWarnThreshold: "0.999"
-
-broker:
-  replicaCount: 1
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-  configData:
-    ## Enable `autoSkipNonRecoverableData` since bookkeeper is running
-    ## without persistence
-    autoSkipNonRecoverableData: "true"
-    # storage settings
-    managedLedgerDefaultEnsembleSize: "1"
-    managedLedgerDefaultWriteQuorum: "1"
-    managedLedgerDefaultAckQuorum: "1"
-
-autorecovery:
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-
-proxy:
-  replicaCount: 1
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-
-toolset:
-  useProxy: false
-
 # enable TLS
 tls:
   enabled: true

.ci/clusters/values-broker-tls.yaml

@@ -17,71 +17,6 @@
 # under the License.
 #
 
-kube-prometheus-stack:
-  enabled: false
-  prometheusOperator:
-    enabled: false
-  grafana:
-    enabled: false
-  alertmanager:
-    enabled: false
-  prometheus:
-    enabled: false
-
-# disabled AntiAffinity
-affinity:
-  anti_affinity: false
-
-# disable auto recovery and pulsar manager
-components:
-  autorecovery: false
-  pulsar_manager: false
-
-zookeeper:
-  replicaCount: 1
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-
-bookkeeper:
-  replicaCount: 3
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-  configData:
-    diskUsageThreshold: "0.999"
-    diskUsageWarnThreshold: "0.999"
-    PULSAR_PREFIX_diskUsageThreshold: "0.999"
-    PULSAR_PREFIX_diskUsageWarnThreshold: "0.999"
-
-broker:
-  replicaCount: 1
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-  configData:
-    ## Enable `autoSkipNonRecoverableData` since bookkeeper is running
-    ## without persistence
-    autoSkipNonRecoverableData: "true"
-    # storage settings
-    managedLedgerDefaultEnsembleSize: "1"
-    managedLedgerDefaultWriteQuorum: "1"
-    managedLedgerDefaultAckQuorum: "1"
-
-autorecovery:
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-
-proxy:
-  replicaCount: 1
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-
-toolset:
-  useProxy: false
-
 # enable TLS
 tls:
   enabled: true

.ci/clusters/values-cacerts.yaml

@@ -0,0 +1,105 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+# enable TLS with cacerts
+tls:
+  enabled: true
+  proxy:
+    enabled: true
+    cacerts:
+      enabled: true
+      certs:
+        - name: common-cacert
+          existingSecret: "pulsar-ci-common-cacert"
+          secretKeys:
+            - ca.crt
+  broker:
+    enabled: true
+    cacerts:
+      enabled: true
+      certs:
+        - name: common-cacert
+          existingSecret: "pulsar-ci-common-cacert"
+          secretKeys:
+            - ca.crt
+  bookie:
+    enabled: true
+    cacerts:
+      enabled: true
+      certs:
+        - name: common-cacert
+          existingSecret: "pulsar-ci-common-cacert"
+          secretKeys:
+            - ca.crt
+  zookeeper:
+    enabled: true
+    cacerts:
+      enabled: true
+      certs:
+        - name: common-cacert
+          existingSecret: "pulsar-ci-common-cacert"
+          secretKeys:
+            - ca.crt
+  toolset:
+    cacerts:
+      enabled: true
+      certs:
+        - name: common-cacert
+          existingSecret: "pulsar-ci-common-cacert"
+          secretKeys:
+            - ca.crt
+  autorecovery:
+    cacerts:
+      enabled: true
+      certs:
+        - name: common-cacert
+          existingSecret: "pulsar-ci-common-cacert"
+          secretKeys:
+            - ca.crt
+
+# enable cert-manager
+certs:
+  internal_issuer:
+    enabled: true
+    type: selfsigning
+
+# deploy cacerts
+extraDeploy:
+  - |
+    apiVersion: "{{ .Values.certs.internal_issuer.apiVersion }}"
+    kind: Certificate
+    metadata:
+      name: "{{ template "pulsar.fullname" . }}-common-cacert"
+      namespace: {{ template "pulsar.namespace" . }}
+      labels:
+        {{- include "pulsar.standardLabels" . | nindent 4 }}
+    spec:
+      secretName: "{{ template "pulsar.fullname" . }}-common-cacert"
+      commonName: "common-cacert"
+      duration: "{{ .Values.certs.internal_issuer.duration }}"
+      renewBefore: "{{ .Values.certs.internal_issuer.renewBefore }}"
+      usages:
+        - server auth
+        - client auth
+      isCA: true
+      issuerRef:
+        name: "{{ template "pulsar.fullname" . }}-{{ .Values.certs.internal_issuer.component }}"
+        kind: Issuer
+        group: cert-manager.io

.ci/clusters/values-jwt-asymmetric.yaml

@@ -17,77 +17,13 @@
 # under the License.
 #
 
-kube-prometheus-stack:
-  enabled: false
-  prometheusOperator:
-    enabled: false
-  grafana:
-    enabled: false
-  alertmanager:
-    enabled: false
-  prometheus:
-    enabled: false
-
-# disabled AntiAffinity
-affinity:
-  anti_affinity: false
-
-# disable auto recovery and pulsar manager
-components:
-  autorecovery: false
-  pulsar_manager: false
-
-zookeeper:
-  replicaCount: 1
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-
-bookkeeper:
-  replicaCount: 2
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-  configData:
-    diskUsageThreshold: "0.999"
-    diskUsageWarnThreshold: "0.999"
-    PULSAR_PREFIX_diskUsageThreshold: "0.999"
-    PULSAR_PREFIX_diskUsageWarnThreshold: "0.999"
-
-broker:
-  replicaCount: 1
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-  configData:
-    ## Enable `autoSkipNonRecoverableData` since bookkeeper is running
-    ## without persistence
-    autoSkipNonRecoverableData: "true"
-    # storage settings
-    managedLedgerDefaultEnsembleSize: "1"
-    managedLedgerDefaultWriteQuorum: "1"
-    managedLedgerDefaultAckQuorum: "1"
-
-autorecovery:
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-
-proxy:
-  replicaCount: 1
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-
-toolset:
-  useProxy: false
 
 auth:
   authentication:
     enabled: true
-    provider: "jwt"
     jwt:
       # Enable JWT authentication
+      enabled: true
       # If the token is generated by a secret key, set the usingSecretKey as true.
       # If the token is generated by a private key, set the usingSecretKey as false.
       usingSecretKey: false
@@ -100,3 +36,9 @@ auth:
     proxy: "proxy-admin"
     # pulsar-admin client to broker/proxy communication
     client: "admin"
+    # pulsar-manager to broker communication
+    manager: "manager-admin"
+
+components:
+  pulsar_manager: true
+

.ci/clusters/values-jwt-symmetric.yaml

@@ -17,77 +17,13 @@
 # under the License.
 #
 
-kube-prometheus-stack:
-  enabled: false
-  prometheusOperator:
-    enabled: false
-  grafana:
-    enabled: false
-  alertmanager:
-    enabled: false
-  prometheus:
-    enabled: false
-
-# disabled AntiAffinity
-affinity:
-  anti_affinity: false
-
-# disable auto recovery and pulsar manager
-components:
-  autorecovery: false
-  pulsar_manager: false
-
-zookeeper:
-  replicaCount: 1
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-
-bookkeeper:
-  replicaCount: 3
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-  configData:
-    diskUsageThreshold: "0.999"
-    diskUsageWarnThreshold: "0.999"
-    PULSAR_PREFIX_diskUsageThreshold: "0.999"
-    PULSAR_PREFIX_diskUsageWarnThreshold: "0.999"
-
-broker:
-  replicaCount: 1
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-  configData:
-    ## Enable `autoSkipNonRecoverableData` since bookkeeper is running
-    ## without persistence
-    autoSkipNonRecoverableData: "true"
-    # storage settings
-    managedLedgerDefaultEnsembleSize: "1"
-    managedLedgerDefaultWriteQuorum: "1"
-    managedLedgerDefaultAckQuorum: "1"
-
-autorecovery:
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-
-proxy:
-  replicaCount: 1
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-
-toolset:
-  useProxy: false
 
 auth:
   authentication:
     enabled: true
-    provider: "jwt"
     jwt:
       # Enable JWT authentication
+      enabled: true
       # If the token is generated by a secret key, set the usingSecretKey as true.
       # If the token is generated by a private key, set the usingSecretKey as false.
       usingSecretKey: true
@@ -100,3 +36,8 @@ auth:
     proxy: "proxy-admin"
     # pulsar-admin client to broker/proxy communication
     client: "admin"
+    # pulsar manager to broker
+    manager: "manager-admin"
+
+components:
+  pulsar_manager: true

.ci/clusters/values-local-pv.yaml

@@ -1,83 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-kube-prometheus-stack:
-  enabled: false
-  prometheusOperator:
-    enabled: false
-  grafana:
-    enabled: false
-  alertmanager:
-    enabled: false
-  prometheus:
-    enabled: false
-
-# disabled AntiAffinity
-affinity:
-  anti_affinity: false
-
-# disable auto recovery and pulsar manager
-components:
-  autorecovery: false
-  pulsar_manager: false
-
-zookeeper:
-  replicaCount: 1
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-
-bookkeeper:
-  replicaCount: 3
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-  configData:
-    diskUsageThreshold: "0.999"
-    diskUsageWarnThreshold: "0.999"
-    PULSAR_PREFIX_diskUsageThreshold: "0.999"
-    PULSAR_PREFIX_diskUsageWarnThreshold: "0.999"
-
-broker:
-  replicaCount: 1
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-  configData:
-    ## Enable `autoSkipNonRecoverableData` since bookkeeper is running
-    ## without persistence
-    autoSkipNonRecoverableData: "true"
-    # storage settings
-    managedLedgerDefaultEnsembleSize: "1"
-    managedLedgerDefaultWriteQuorum: "1"
-    managedLedgerDefaultAckQuorum: "1"
-
-autorecovery:
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-
-proxy:
-  replicaCount: 1
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-
-toolset:
-  useProxy: false

.ci/clusters/values-openid.yaml

@@ -0,0 +1,94 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Mount crendentials to each component
+proxy:
+  configData:
+    # Authentication settings of the broker itself. Used when the broker connects to other brokers, or when the proxy connects to brokers, either in same or other clusters
+    brokerClientAuthenticationPlugin: "org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2"
+    brokerClientAuthenticationParameters: '{"privateKey":"file:///pulsar/auth/proxy/credentials_file.json","audience":"account","issuerUrl":"http://keycloak-ci-headless:8080/realms/pulsar"}'
+  extraVolumes:
+    - name: pulsar-proxy-credentials
+      secret:
+        secretName: pulsar-proxy-credentials
+  extraVolumeMounts:
+    - name: pulsar-proxy-credentials
+      mountPath: "/pulsar/auth/proxy"
+      readOnly: true
+
+broker:
+  configData:
+    # Authentication settings of the broker itself. Used when the broker connects to other brokers, or when the proxy connects to brokers, either in same or other clusters
+    brokerClientAuthenticationPlugin: "org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2"
+    brokerClientAuthenticationParameters: '{"privateKey":"file:///pulsar/auth/broker/credentials_file.json","audience":"account","issuerUrl":"http://keycloak-ci-headless:8080/realms/pulsar"}'
+  extraVolumes:
+    - name: pulsar-broker-credentials
+      secret:
+        secretName: pulsar-broker-credentials
+  extraVolumeMounts:
+    - name: pulsar-broker-credentials
+      mountPath: "/pulsar/auth/broker"
+      readOnly: true
+
+toolset:
+  configData:
+    authPlugin: "org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2"
+    authParams: '{"privateKey":"file:///pulsar/auth/admin/credentials_file.json","audience":"account","issuerUrl":"http://keycloak-ci-headless:8080/realms/pulsar"}'
+  extraVolumes:
+    - name: pulsar-admin-credentials
+      secret:
+        secretName: pulsar-admin-credentials
+  extraVolumeMounts:
+    - name: pulsar-admin-credentials
+      mountPath: "/pulsar/auth/admin"
+      readOnly: true
+
+auth:
+  authentication:
+    enabled: true
+    openid:
+      # Enable openid authentication
+      enabled: true
+      # https://pulsar.apache.org/docs/next/security-openid-connect/#enable-openid-connect-authentication-in-the-broker-and-proxy
+      openIDAllowedTokenIssuers:
+        - http://keycloak-ci-headless:8080/realms/pulsar
+      openIDAllowedAudiences:
+        - account
+      #openIDTokenIssuerTrustCertsFilePath:
+      openIDRoleClaim: "sub"
+      openIDAcceptedTimeLeewaySeconds: "0"
+      openIDCacheSize: "5"
+      openIDCacheRefreshAfterWriteSeconds: "64800"
+      openIDCacheExpirationSeconds: "86400"
+      openIDHttpConnectionTimeoutMillis: "10000"
+      openIDHttpReadTimeoutMillis: "10000"
+      openIDKeyIdCacheMissRefreshSeconds: "300"
+      openIDRequireIssuersUseHttps: "false"
+      openIDFallbackDiscoveryMode: "DISABLED"
+  authorization:
+    enabled: true
+  superUsers:
+    # broker to broker communication
+    broker: "broker-admin"
+    # proxy to broker communication
+    proxy: "proxy-admin"
+    # pulsar-admin client to broker/proxy communication
+    client: "admin"
+    # pulsar manager to broker
+    manager: "manager-admin"

.ci/clusters/values-oxia.yaml

@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+components:
+  zookeeper: false
+  oxia: true
+  # disable functions for oxia tests since there's no support for Oxia in 
+  # BookKeeperPackagesStorage which requires Zookeeper
+  functions: false
+
+oxia:
+  initialShardCount: 3
+  replicationFactor: 3
+  server:
+    replicas: 3
+    cpuLimit: 333m
+    memoryLimit: 200Mi
+    dbCacheSizeMb: 100
+    storageSize: 1Gi

.ci/clusters/values-pulsar-image.yaml

@@ -1,86 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-kube-prometheus-stack:
-  enabled: false
-  prometheusOperator:
-    enabled: false
-  grafana:
-    enabled: false
-  alertmanager:
-    enabled: false
-  prometheus:
-    enabled: false
-
-volumes:
-  persistence: false
-
-# disabled AntiAffinity
-affinity:
-  anti_affinity: false
-
-# disable auto recovery and pulsar manager
-components:
-  autorecovery: false
-  pulsar_manager: false
-
-zookeeper:
-  replicaCount: 1
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-
-bookkeeper:
-  replicaCount: 3
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-  configData:
-    diskUsageThreshold: "0.999"
-    diskUsageWarnThreshold: "0.999"
-    PULSAR_PREFIX_diskUsageThreshold: "0.999"
-    PULSAR_PREFIX_diskUsageWarnThreshold: "0.999"
-
-broker:
-  replicaCount: 1
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-  configData:
-    ## Enable `autoSkipNonRecoverableData` since bookkeeper is running
-    ## without persistence
-    autoSkipNonRecoverableData: "true"
-    # storage settings
-    managedLedgerDefaultEnsembleSize: "1"
-    managedLedgerDefaultWriteQuorum: "1"
-    managedLedgerDefaultAckQuorum: "1"
-
-autorecovery:
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-
-proxy:
-  replicaCount: 1
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-
-toolset:
-  useProxy: false

.ci/clusters/values-pulsar-manager.yaml

@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+components:
+  pulsar_manager: true

.ci/clusters/values-pulsar-previous-lts.yaml

@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+defaultPulsarImageTag: 3.0.12

.ci/clusters/values-tls.yaml

@@ -17,70 +17,6 @@
 # under the License.
 #
 
-kube-prometheus-stack:
-  enabled: false
-  prometheusOperator:
-    enabled: false
-  grafana:
-    enabled: false
-  alertmanager:
-    enabled: false
-  prometheus:
-    enabled: false
-
-# disabled AntiAffinity
-affinity:
-  anti_affinity: false
-
-# disable auto recovery and pulsar manager
-components:
-  autorecovery: false
-  pulsar_manager: false
-
-zookeeper:
-  replicaCount: 1
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-
-bookkeeper:
-  replicaCount: 3
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-  configData:
-    diskUsageThreshold: "0.999"
-    diskUsageWarnThreshold: "0.999"
-    PULSAR_PREFIX_diskUsageThreshold: "0.999"
-    PULSAR_PREFIX_diskUsageWarnThreshold: "0.999"
-
-broker:
-  replicaCount: 1
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-  configData:
-    ## Enable `autoSkipNonRecoverableData` since bookkeeper is running
-    ## without persistence
-    autoSkipNonRecoverableData: "true"
-    # storage settings
-    managedLedgerDefaultEnsembleSize: "1"
-    managedLedgerDefaultWriteQuorum: "1"
-    managedLedgerDefaultAckQuorum: "1"
-
-autorecovery:
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-
-proxy:
-  replicaCount: 1
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-
-toolset:
-  useProxy: false
 
 # enable TLS
 tls:

.ci/clusters/values-upgrade.yaml

@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+

.ci/clusters/values-victoria-metrics-grafana.yaml

@@ -0,0 +1,60 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+victoria-metrics-k8s-stack:
+  enabled: true
+  victoria-metrics-operator:
+    enabled: true
+  vmsingle:
+    enabled: true
+  vmagent:
+    enabled: true
+  grafana:
+    enabled: true
+    adminPassword: pulsar-ci-admin
+  prometheus-node-exporter:
+    enabled: true
+
+zookeeper:
+  podMonitor:
+    enabled: true
+
+bookkeeper:
+  podMonitor:
+    enabled: true
+
+broker:
+  podMonitor:
+    enabled: true
+    
+autorecovery:
+  podMonitor:
+    enabled: true
+
+proxy:
+  podMonitor:
+    enabled: true
+
+oxia:
+  coordinator:
+    podMonitor:
+      enabled: true
+  server:
+    podMonitor:
+      enabled: true

.ci/clusters/values-zk-tls.yaml

@@ -17,71 +17,6 @@
 # under the License.
 #
 
-kube-prometheus-stack:
-  enabled: false
-  prometheusOperator:
-    enabled: false
-  grafana:
-    enabled: false
-  alertmanager:
-    enabled: false
-  prometheus:
-    enabled: false
-
-# disabled AntiAffinity
-affinity:
-  anti_affinity: false
-
-# disable auto recovery and pulsar manager
-components:
-  autorecovery: false
-  pulsar_manager: false
-
-zookeeper:
-  replicaCount: 1
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-
-bookkeeper:
-  replicaCount: 3
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-  configData:
-    diskUsageThreshold: "0.999"
-    diskUsageWarnThreshold: "0.999"
-    PULSAR_PREFIX_diskUsageThreshold: "0.999"
-    PULSAR_PREFIX_diskUsageWarnThreshold: "0.999"
-
-broker:
-  replicaCount: 1
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-  configData:
-    ## Enable `autoSkipNonRecoverableData` since bookkeeper is running
-    ## without persistence
-    autoSkipNonRecoverableData: "true"
-    # storage settings
-    managedLedgerDefaultEnsembleSize: "1"
-    managedLedgerDefaultWriteQuorum: "1"
-    managedLedgerDefaultAckQuorum: "1"
-
-autorecovery:
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-
-proxy:
-  replicaCount: 1
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-
-toolset:
-  useProxy: false
-
 # enable TLS
 tls:
   enabled: true

.ci/clusters/values-zkbk-tls.yaml

@@ -17,71 +17,6 @@
 # under the License.
 #
 
-kube-prometheus-stack:
-  enabled: false
-  prometheusOperator:
-    enabled: false
-  grafana:
-    enabled: false
-  alertmanager:
-    enabled: false
-  prometheus:
-    enabled: false
-
-# disabled AntiAffinity
-affinity:
-  anti_affinity: false
-
-# disable auto recovery and pulsar manager
-components:
-  autorecovery: false
-  pulsar_manager: false
-
-zookeeper:
-  replicaCount: 1
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-
-bookkeeper:
-  replicaCount: 3
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-  configData:
-    diskUsageThreshold: "0.999"
-    diskUsageWarnThreshold: "0.999"
-    PULSAR_PREFIX_diskUsageThreshold: "0.999"
-    PULSAR_PREFIX_diskUsageWarnThreshold: "0.999"
-
-broker:
-  replicaCount: 1
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-  configData:
-    ## Enable `autoSkipNonRecoverableData` since bookkeeper is running
-    ## without persistence
-    autoSkipNonRecoverableData: "true"
-    # storage settings
-    managedLedgerDefaultEnsembleSize: "1"
-    managedLedgerDefaultWriteQuorum: "1"
-    managedLedgerDefaultAckQuorum: "1"
-
-autorecovery:
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-
-proxy:
-  replicaCount: 1
-  # Disable pod monitor since we're disabling CRD installation
-  podMonitor:
-    enabled: false
-
-toolset:
-  useProxy: false
-
 # enable TLS
 tls:
   enabled: true

.ci/configure_ci_runner_for_debugging.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+# this script is used to install tools for the GitHub Actions CI runner while debugging with ssh
+
+if [[ -z "${GITHUB_ACTIONS}" ]]; then
+    echo "Error: This script is intended to run only in GitHub Actions environment"
+    exit 1
+fi
+
+cat >> $HOME/.bashrc <<'EOF'
+function use_kind_kubeconfig() {
+    export KUBECONFIG=$(ls $HOME/kind/pulsar-ci-*/kubeconfig.yaml)
+}
+
+function kubectl() {
+    # use kind environment's kubeconfig
+    if [ -z "$KUBECONFIG" ]; then
+        use_kind_kubeconfig
+    fi
+    command kubectl "$@"
+}
+
+function k9s() {
+    # use kind environment's kubeconfig
+    if [ -z "$KUBECONFIG" ]; then
+        use_kind_kubeconfig
+    fi
+    # install k9s on the fly
+    if [ ! -x /usr/local/bin/k9s ]; then
+        echo "Installing k9s..."
+        curl -L -s https://github.com/derailed/k9s/releases/download/v0.40.5/k9s_Linux_amd64.tar.gz | sudo tar xz -C /usr/local/bin k9s
+    fi
+    command k9s "$@"
+}
+
+alias k=kubectl
+EOF
+cat >> $HOME/.bash_profile <<'EOF'
+if [ -f ~/.bashrc ]; then
+    source ~/.bashrc
+fi
+EOF

.ci/helm.sh

@@ -20,13 +20,15 @@
 BINDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
 PULSAR_HOME="$(cd "${BINDIR}/.." && pwd)"
 CHARTS_HOME=${PULSAR_HOME}
+PULSAR_CHART_LOCAL=${CHARTS_HOME}/charts/pulsar
+PULSAR_CHART_VERSION=${PULSAR_CHART_VERSION:-"local"}
 OUTPUT_BIN=${CHARTS_HOME}/output/bin
 KIND_BIN=$OUTPUT_BIN/kind
 HELM=${OUTPUT_BIN}/helm
 KUBECTL=${OUTPUT_BIN}/kubectl
 NAMESPACE=pulsar
 CLUSTER=pulsar-ci
-CLUSTER_ID=$(uuidgen)
+: ${CLUSTER_ID:=$(uuidgen)}
 K8S_LOGS_DIR="${K8S_LOGS_DIR:-/tmp/k8s-logs}"
 export PATH="$OUTPUT_BIN:$PATH"
 
@@ -79,9 +81,17 @@ function ci::install_cert_manager() {
     echo "Successfully installed the cert manager."
 }
 
+function ci::helm_repo_add() {
+    echo "Adding the helm repo ..."
+    ${HELM} repo add prometheus-community https://prometheus-community.github.io/helm-charts
+    ${HELM} repo add vm https://victoriametrics.github.io/helm-charts/
+    ${HELM} repo update
+    echo "Successfully added the helm repo."
+}
+
 function ci::print_pod_logs() {
-    echo "Logs for all pulsar containers:"
-    for k8sobject in $(${KUBECTL} get pods,jobs -n ${NAMESPACE} -l app=pulsar -o=name); do
+    echo "Logs for all containers:"
+    for k8sobject in $(${KUBECTL} get pods,jobs -n ${NAMESPACE} -o=name); do
       ${KUBECTL} logs -n ${NAMESPACE} "$k8sobject" --all-containers=true --ignore-errors=true --prefix=true --tail=100 || true
     done;
 }
@@ -89,7 +99,7 @@ function ci::print_pod_logs() {
 function ci::collect_k8s_logs() {
     mkdir -p "${K8S_LOGS_DIR}" && cd "${K8S_LOGS_DIR}"
     echo "Collecting k8s logs to ${K8S_LOGS_DIR}"
-    for k8sobject in $(${KUBECTL} get pods,jobs -n ${NAMESPACE} -l app=pulsar -o=name); do
+    for k8sobject in $(${KUBECTL} get pods,jobs -n ${NAMESPACE} -o=name); do
       filebase="${k8sobject//\//_}"
       ${KUBECTL} logs -n ${NAMESPACE} "$k8sobject" --all-containers=true --ignore-errors=true --prefix=true > "${filebase}.$$.log.txt" || true
       ${KUBECTL} logs -n ${NAMESPACE} "$k8sobject" --all-containers=true --ignore-errors=true --prefix=true --previous=true > "${filebase}.previous.$$.log.txt" || true
@@ -100,22 +110,75 @@ function ci::collect_k8s_logs() {
 }
 
 function ci::install_pulsar_chart() {
-    local value_file=$1
-    local extra_opts=$2
+    local install_type=$1
+    local common_value_file=$2
+    local value_file=$3
+    shift 3
+    local extra_values=()
+    local extra_opts=()
+    local values_next=false
+    for arg in "$@"; do
+        if [[ "$arg" == "--values" || "$arg" == "--set" ]]; then
+            extra_values+=("$arg")
+            values_next=true
+        elif [[ "$values_next" == true ]]; then
+            extra_values+=("$arg")
+            values_next=false
+        else
+            extra_opts+=("$arg")
+        fi
+    done
+    local install_args
 
+    if [[ "${install_type}" == "install" ]]; then
       echo "Installing the pulsar chart"
       ${KUBECTL} create namespace ${NAMESPACE}
       ci::install_cert_manager
-    echo ${CHARTS_HOME}/scripts/pulsar/prepare_helm_release.sh -k ${CLUSTER} -n ${NAMESPACE} ${extra_opts}
-    ${CHARTS_HOME}/scripts/pulsar/prepare_helm_release.sh -k ${CLUSTER} -n ${NAMESPACE} ${extra_opts}
+      echo ${CHARTS_HOME}/scripts/pulsar/prepare_helm_release.sh -k ${CLUSTER} -n ${NAMESPACE} "${extra_opts[@]}"
+      ${CHARTS_HOME}/scripts/pulsar/prepare_helm_release.sh -k ${CLUSTER} -n ${NAMESPACE} "${extra_opts[@]}"
       sleep 10
 
-    echo ${HELM} dependency update ${CHARTS_HOME}/charts/pulsar
-    ${HELM} dependency update ${CHARTS_HOME}/charts/pulsar
-    echo ${HELM} install --set initialize=true --values ${value_file} ${CLUSTER} ${CHARTS_HOME}/charts/pulsar
-    ${HELM} template --values ${value_file} ${CLUSTER} ${CHARTS_HOME}/charts/pulsar
-    ${HELM} install --set initialize=true --values ${value_file} --namespace=${NAMESPACE} ${CLUSTER} ${CHARTS_HOME}/charts/pulsar
+      # install metallb for loadbalancer support
+      # following instructions from https://kind.sigs.k8s.io/docs/user/loadbalancer/
+      ${KUBECTL} apply -f https://raw.githubusercontent.com/metallb/metallb/v0.13.12/config/manifests/metallb-native.yaml
+      # wait until metallb is ready
+      ${KUBECTL} wait --namespace metallb-system \
+                --for=condition=ready pod \
+                --selector=app=metallb \
+                --timeout=120s
+      # configure metallb
+      ${KUBECTL} apply -f ${BINDIR}/metallb/metallb-config.yaml
+      install_args=""
 
+      # create auth resources
+      if [[ "x${AUTHENTICATION_PROVIDER}" == "xopenid" ]]; then
+          ci::create_openid_resources
+      fi
+    else
+      install_args="--wait --wait-for-jobs --timeout 360s --debug"
+    fi
+
+    CHART_ARGS=""
+    if [[ "${PULSAR_CHART_VERSION}" == "local" ]]; then
+      set -x
+      ${HELM} dependency update ${PULSAR_CHART_LOCAL}
+      set +x
+      CHART_ARGS="${PULSAR_CHART_LOCAL}"
+    else
+      set -x
+      ${HELM} repo add apache https://pulsar.apache.org/charts
+      set +x
+      CHART_ARGS="apache/pulsar --dependency-update"
+      if [[ "${PULSAR_CHART_VERSION}" != "latest" ]]; then
+        CHART_ARGS="${CHART_ARGS} --version ${PULSAR_CHART_VERSION}"
+      fi
+    fi
+    set -x
+    ${HELM} template --values ${common_value_file} --values ${value_file} "${extra_values[@]}" ${CLUSTER} ${CHART_ARGS}
+    ${HELM} ${install_type} --values ${common_value_file} --values ${value_file} "${extra_values[@]}" --namespace=${NAMESPACE} ${CLUSTER} ${CHART_ARGS} ${install_args}
+    set +x
+
+    if [[ "${install_type}" == "install" ]]; then
       echo "wait until broker is alive"
       WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep ${CLUSTER}-broker | wc -l)
       counter=1
@@ -140,7 +203,7 @@ function ci::install_pulsar_chart() {
         WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep ${CLUSTER}-broker | wc -l)
       done
       timeout 300s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-broker; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
-    timeout 120s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until [ "$(curl -L http://pulsar-ci-broker:8080/status.html)" == "OK" ]; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
+      timeout 120s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until [ "$(curl -s -L http://pulsar-ci-broker:8080/status.html)" == "OK" ]; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
 
       WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep ${CLUSTER}-proxy | wc -l)
       counter=1
@@ -160,58 +223,365 @@ function ci::install_pulsar_chart() {
         WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep ${CLUSTER}-proxy | wc -l)
       done
       timeout 300s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-proxy; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
-    # ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until [ "$(curl -L http://pulsar-ci-proxy:8080/status.html)" == "OK" ]; do sleep 3; done'
+      echo "Install complete"
+    else
+      echo "wait until broker is alive"
+      timeout 300s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-broker; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
+      timeout 120s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until [ "$(curl -s -L http://pulsar-ci-broker:8080/status.html)" == "OK" ]; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
+      echo "wait until proxy is alive"
+      timeout 300s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-proxy; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
+      echo "Upgrade complete"
+    fi
+}
+
+helm_values_cached=""
+
+function ci::helm_values_for_deployment() {
+    if [[ -z "${helm_values_cached}" ]]; then
+        helm_values_cached=$(helm get values -n ${NAMESPACE} ${CLUSTER} -a -o yaml)
+    fi
+    printf "%s" "${helm_values_cached}"
+}
+
+function ci::check_pulsar_environment() {
+    echo "Wait until pulsar-ci-broker is ready"
+    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-broker; do sleep 3; done'
+    echo "Wait until pulsar-ci-proxy is ready"
+    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-proxy; do sleep 3; done'
+    echo "bookie-0 disk usage"
+    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-bookie-0 -- df -h
+    echo "bookie-0 bookkeeper.conf"
+    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-bookie-0 -- cat conf/bookkeeper.conf
+    echo "bookie-0 bookies list (rw)"
+    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/bookkeeper shell listbookies -rw | grep ListBookiesCommand
+    echo "bookie-0 bookies list (ro)"
+    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/bookkeeper shell listbookies -ro | grep ListBookiesCommand
+}
+
+# function to retry a given commend 3 times with a backoff of 10 seconds in between
+function ci::retry() {
+  local n=1
+  local max=3
+  local delay=10
+  while true; do
+    "$@" && break || {
+      if [[ $n -lt $max ]]; then
+        ((n++))
+        echo "::warning::Command failed. Attempt $n/$max:"
+        sleep $delay
+      else
+        fail "::error::The command has failed after $n attempts."
+      fi
+    }
+  done
+}
+
+function ci::test_pulsar_admin_api_access() {
+  echo "Test pulsar admin api access"
+  ci::retry ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin tenants list
+}
+
+function ci::test_create_test_namespace() {
+    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin tenants create pulsar-ci
+    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin namespaces create pulsar-ci/test
 }
 
 function ci::test_pulsar_producer_consumer() {
-    sleep 120
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-broker; do sleep 3; done'
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-proxy; do sleep 3; done'
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-bookie-0 -- df -h
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-bookie-0 -- cat conf/bookkeeper.conf
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/bookkeeper shell listbookies -rw
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/bookkeeper shell listbookies -ro
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin tenants create pulsar-ci
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin namespaces create pulsar-ci/test
+    action="${1:-"produce-consume"}"
+    echo "Testing with ${action}"
+    if [[ "$(ci::helm_values_for_deployment | yq .tls.proxy.enabled)" == "true" ]]; then
+      PROXY_URL="pulsar+ssl://pulsar-ci-proxy:6651"
+    else
+      PROXY_URL="pulsar://pulsar-ci-proxy:6650"
+    fi
+    set -x
+    if [[ "${action}" == "produce" || "${action}" == "produce-consume" ]]; then
+      ci::test_create_test_namespace
       ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin topics create pulsar-ci/test/test-topic
       ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin topics create-subscription -s test pulsar-ci/test/test-topic
       ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client produce -m "test-message" pulsar-ci/test/test-topic
+      ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin topics create-subscription -s test2 pulsar-ci/test/test-topic
+      ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client --url "${PROXY_URL}" produce -m "test-message2" pulsar-ci/test/test-topic
+    fi
+    if [[ "${action}" == "consume" || "${action}" == "produce-consume" ]]; then
       ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client consume -s test pulsar-ci/test/test-topic
+      ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client --url "${PROXY_URL}" consume -s test2 pulsar-ci/test/test-topic
+    fi
+    set +x
 }
 
 function ci::wait_function_running() {
-    num_running=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'bin/pulsar-admin functions status --tenant pulsar-ci --namespace test --name test-function | bin/jq .numRunning')
+    num_running=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'bin/pulsar-admin functions status --tenant pulsar-ci --namespace test --name test-function' | jq .numRunning)
+    counter=1
     while [[ ${num_running} -lt 1 ]]; do
-      echo ${num_running}
+      ((counter++))
+      if [[ $counter -gt 6 ]]; then
+        echo >&2 "Timeout waiting..."
+        return 1
+      fi
+      echo "Waiting 15 seconds for function to be running"
       sleep 15
-      ${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running
+      ${KUBECTL} get pods -n ${NAMESPACE} -l component=function || true
       ${KUBECTL} get events --sort-by=.lastTimestamp -A | tail -n 30 || true
-      num_running=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'bin/pulsar-admin functions status --tenant pulsar-ci --namespace test --name test-function | bin/jq .numRunning')
+      podname=$(${KUBECTL} get pods -l component=function -n ${NAMESPACE} --no-headers -o custom-columns=":metadata.name") || true
+      if [[ -n "$podname" ]]; then
+        echo "Function pod is $podname"
+        ${KUBECTL} describe pod -n ${NAMESPACE} $podname
+        echo "Function pod logs"
+        ${KUBECTL} logs -n ${NAMESPACE} $podname
+      fi
+      num_running=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'bin/pulsar-admin functions status --tenant pulsar-ci --namespace test --name test-function' | jq .numRunning)
     done
 }
 
 function ci::wait_message_processed() {
-    num_processed=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'bin/pulsar-admin functions stats --tenant pulsar-ci --namespace test --name test-function | bin/jq .processedSuccessfullyTotal')
+    num_processed=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'bin/pulsar-admin functions stats --tenant pulsar-ci --namespace test --name test-function' | jq .processedSuccessfullyTotal)
+    podname=$(${KUBECTL} get pods -l component=function -n ${NAMESPACE} --no-headers -o custom-columns=":metadata.name")
+    counter=1
     while [[ ${num_processed} -lt 1 ]]; do
-      echo ${num_processed}
+      ((counter++))
+      if [[ $counter -gt 6 ]]; then
+        echo >&2 "Timeout waiting..."
+        return 1
+      fi
+      echo "Waiting 15 seconds for message to be processed"
       sleep 15
+      echo "Function pod is $podname"
+      ${KUBECTL} describe pod -n ${NAMESPACE} $podname
+      echo "Function pod logs"
+      ${KUBECTL} logs -n ${NAMESPACE} $podname
       ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin functions stats --tenant pulsar-ci --namespace test --name test-function
-      num_processed=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'bin/pulsar-admin functions stats --tenant pulsar-ci --namespace test --name test-function | bin/jq .processedSuccessfullyTotal')
+      num_processed=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'bin/pulsar-admin functions stats --tenant pulsar-ci --namespace test --name test-function' | jq .processedSuccessfullyTotal)
     done
 }
 
 function ci::test_pulsar_function() {
-    sleep 120
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-broker; do sleep 3; done'
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-proxy; do sleep 3; done'
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-bookie-0 -- df -h
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/bookkeeper shell listbookies -rw
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/bookkeeper shell listbookies -ro
+    echo "Testing functions"
+    echo "Creating function"
     ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin functions create --tenant pulsar-ci --namespace test --name test-function --inputs "pulsar-ci/test/test_input" --output "pulsar-ci/test/test_output" --parallelism 1 --classname org.apache.pulsar.functions.api.examples.ExclamationFunction --jar /pulsar/examples/api-examples.jar
-
+    echo "Creating subscription for output topic"
+    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin topics create-subscription -s test pulsar-ci/test/test_output
+    echo "Waiting for function to be ready"
     # wait until the function is running
-    # TODO: re-enable function test
-    # ci::wait_function_running
-    # ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client produce -m "hello pulsar function!" pulsar-ci/test/test_input
-    # ci::wait_message_processed
+    ci::wait_function_running
+    echo "Sending input message"
+    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client produce -m 'hello pulsar function!' pulsar-ci/test/test_input
+    echo "Waiting for message to be processed"
+    ci::wait_message_processed
+    echo "Consuming output message"
+    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client consume -s test pulsar-ci/test/test_output
 }
+
+function ci::test_pulsar_manager() {
+  echo "Testing pulsar manager"
+
+  until ${KUBECTL} get jobs -n ${NAMESPACE} ${CLUSTER}-pulsar-manager-init -o json | jq -r '.status.conditions[] | select (.type | test("Complete")).status' | grep True; do sleep 3; done
+  ${KUBECTL} describe job -n ${NAMESPACE} ${CLUSTER}-pulsar-manager-init
+  ${KUBECTL} logs -n ${NAMESPACE} job.batch/${CLUSTER}-pulsar-manager-init
+  ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-manager-0 -- cat /pulsar-manager/pulsar-manager.log
+  echo "Checking Podname"
+  podname=$(${KUBECTL} get pods -n ${NAMESPACE} -l component=pulsar-manager --no-headers -o custom-columns=":metadata.name")
+  echo "Getting pulsar manager UI password"
+  PASSWORD=$(${KUBECTL} get secret -n ${NAMESPACE} -l component=pulsar-manager -o=jsonpath="{.items[0].data.UI_PASSWORD}" | base64 --decode)
+
+  echo "Getting CSRF_TOKEN"
+  CSRF_TOKEN=$(${KUBECTL} exec -n ${NAMESPACE} ${podname} -- curl http://127.0.0.1:7750/pulsar-manager/csrf-token)
+
+  echo "Performing login"
+  ${KUBECTL} exec -n ${NAMESPACE} ${podname} -- curl -X POST http://127.0.0.1:9527/pulsar-manager/login \
+                                                 -H 'Accept: application/json, text/plain, */*' \
+                                                 -H 'Content-Type: application/json' \
+                                                 -H "X-XSRF-TOKEN: $CSRF_TOKEN" \
+                                                 -H "Cookie: XSRF-TOKEN=$CSRF_TOKEN" \
+                                                 -sS -D headers.txt \
+                                                 -d '{"username": "pulsar", "password": "'${PASSWORD}'"}'
+  LOGIN_TOKEN=$(${KUBECTL} exec -n ${NAMESPACE} ${podname} -- grep "token:" headers.txt | sed 's/^.*: //')
+  LOGIN_JSESSIONID=$(${KUBECTL} exec -n ${NAMESPACE} ${podname} -- grep -o "JSESSIONID=[a-zA-Z0-9_]*" headers.txt | sed 's/^.*=//')
+
+  echo "Checking environment"
+  envs=$(${KUBECTL} exec -n ${NAMESPACE} ${podname} -- curl -X GET http://127.0.0.1:9527/pulsar-manager/environments \
+                  -H 'Content-Type: application/json' \
+                  -H "token: $LOGIN_TOKEN" \
+                  -H "X-XSRF-TOKEN: $CSRF_TOKEN" \
+                  -H "username: pulsar" \
+                  -H "Cookie: XSRF-TOKEN=$CSRF_TOKEN; JSESSIONID=$LOGIN_JSESSIONID;")
+  echo "$envs"
+  number_of_envs=$(echo $envs | jq '.total')
+  if [ "$number_of_envs" -ne 1 ]; then
+    echo "Error: Did not find expected environment"
+    exit 1
+  fi
+
+  # Force manager to query broker for tenant info. This will require use of the manager's JWT, if JWT authentication is enabled.
+  echo "Checking tenants"
+  pulsar_env=$(echo $envs | jq -r '.data[0].name')
+  tenants=$(${KUBECTL} exec -n ${NAMESPACE} ${podname} -- curl -X GET http://127.0.0.1:9527/pulsar-manager/admin/v2/tenants \
+                  -H 'Content-Type: application/json' \
+                  -H "token: $LOGIN_TOKEN" \
+                  -H "X-XSRF-TOKEN: $CSRF_TOKEN" \
+                  -H "username: pulsar" \
+                  -H "tenant: pulsar" \
+                  -H "environment: ${pulsar_env}" \
+                  -H "Cookie: XSRF-TOKEN=$CSRF_TOKEN; JSESSIONID=$LOGIN_JSESSIONID;")
+  echo "$tenants"
+  number_of_tenants=$(echo $tenants | jq '.total')
+  if [ "$number_of_tenants" -lt 1 ]; then
+    echo "Error: Found no tenants!"
+    exit 1
+  fi
+}
+
+function ci::check_loadbalancers() {
+  (
+  set +e
+  ${KUBECTL} get services -n ${NAMESPACE} | grep LoadBalancer
+  if [ $? -eq 0 ]; then
+    echo "Error: Found service with type LoadBalancer. This is not allowed because of security reasons."
+    exit 1
+  fi
+  exit 0
+  )
+}
+
+function ci::validate_kustomize_yaml() {
+  # if kustomize is not installed, install kustomize to a temp directory
+  if ! command -v kustomize &> /dev/null; then
+    KUSTOMIZE_VERSION=5.6.0
+    KUSTOMIZE_DIR=$(mktemp -d)
+    echo "Installing kustomize ${KUSTOMIZE_VERSION} to ${KUSTOMIZE_DIR}"
+    curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash -s ${KUSTOMIZE_VERSION} ${KUSTOMIZE_DIR}
+    export PATH=${KUSTOMIZE_DIR}:$PATH
+  fi
+  # prevent regression of https://github.com/apache/pulsar-helm-chart/issues/569
+  local kustomize_yaml_dir=$(mktemp -d)
+  cp ${PULSAR_HOME}/.ci/kustomization.yaml ${kustomize_yaml_dir}
+  PULSAR_HOME=${PULSAR_HOME} yq -i '.helmGlobals.chartHome = env(PULSAR_HOME) + "/charts"' ${kustomize_yaml_dir}/kustomization.yaml
+  failures=0
+  # validate zookeeper init
+  echo "Validating kustomize yaml output with zookeeper init"
+  _ci::validate_kustomize_yaml ${kustomize_yaml_dir} || ((failures++))
+  # validate oxia init
+  yq -i '.helmCharts[0].valuesInline.components += {"zookeeper": false, "oxia": true}' ${kustomize_yaml_dir}/kustomization.yaml
+  echo "Validating kustomize yaml output with oxia init"
+  _ci::validate_kustomize_yaml ${kustomize_yaml_dir} || ((failures++)) 
+  if [ $failures -gt 0 ]; then
+    exit 1
+  fi
+}
+
+function _ci::validate_kustomize_yaml() {
+  local kustomize_yaml_dir=$1
+  kustomize build --enable-helm --helm-kube-version 1.23.0 --load-restrictor=LoadRestrictionsNone ${kustomize_yaml_dir} | yq 'select(.spec.template.spec.containers[0].args != null) | .spec.template.spec.containers[0].args' | \
+  awk '{
+    if (prev_line ~ /\\$/ && $0 ~ /^$/) {
+      print "Found issue: backslash at end of line followed by empty line. Must use pipe character for multiline strings to support kustomize due to kubernetes-sigs/kustomize#4201.";
+      print "Line: " prev_line;
+      has_issue = 1;
+    }
+    prev_line = $0;
+  }
+  END {
+    if (!has_issue) {
+      print "No issues found: no backslash followed by empty line";
+      exit 0;
+    }
+    exit 1;
+  }'
+}
+
+# Create all resources needed for openid authentication
+function ci::create_openid_resources() {
+
+  echo "Creating openid resources"
+
+  cp ${PULSAR_HOME}/.ci/auth/keycloak/0-realm-pulsar-partial-export.json /tmp/realm-pulsar.json
+
+  for component in broker proxy admin manager; do
+
+    echo "Creating openid resources for ${component}"
+
+    local client_id=pulsar-${component}
+
+    # Github action hang up when read string from /dev/urandom, so use python to generate a random string
+    local client_secret=$(python -c "import secrets; import string; length = 32; random_string = ''.join(secrets.choice(string.ascii_letters + string.digits) for _ in range(length)); print(random_string);")
+
+    if [[ "${component}" == "admin" ]]; then
+      local sub_claim_value="admin"
+    else
+      local sub_claim_value="${component}-admin"
+    fi
+
+    # Create the client credentials file
+    jq -n --arg CLIENT_ID $client_id --arg CLIENT_SECRET "$client_secret" -f ${PULSAR_HOME}/.ci/auth/oauth2/credentials_file.json > /tmp/${component}-credentials_file.json
+
+    # Create the secret for the client credentials
+    local secret_name="pulsar-${component}-credentials"
+    ${KUBECTL} create secret generic ${secret_name} --from-file=credentials_file.json=/tmp/${component}-credentials_file.json -n ${NAMESPACE}
+
+    # Create the keycloak client file
+    jq -n --arg CLIENT_ID $client_id --arg CLIENT_SECRET "$client_secret" --arg SUB_CLAIM_VALUE "$sub_claim_value" -f ${PULSAR_HOME}/.ci/auth/keycloak/1-client-template.json > /tmp/${component}-keycloak-client.json
+
+    # Merge the keycloak client file with the realm
+    jq '.clients += [input]' /tmp/realm-pulsar.json /tmp/${component}-keycloak-client.json > /tmp/realm-pulsar.json.tmp
+    mv /tmp/realm-pulsar.json.tmp /tmp/realm-pulsar.json
+
+  done
+
+  echo "Create keycloak realm configuration"
+  ${KUBECTL} create secret generic keycloak-ci-realm-config --from-file=realm-pulsar.json=/tmp/realm-pulsar.json -n ${NAMESPACE}
+
+  echo "Installing keycloak helm chart"
+  ${HELM} install keycloak-ci oci://registry-1.docker.io/bitnamicharts/keycloak --version 24.6.4 --values ${PULSAR_HOME}/.ci/auth/keycloak/values.yaml -n ${NAMESPACE}
+
+  echo "Wait until keycloak is running"
+  WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep keycloak-ci-0 | wc -l)
+  counter=1
+  while [[ ${WC} -lt 1 ]]; do
+    ((counter++))
+    echo ${WC};
+    sleep 15
+    ${KUBECTL} get pods,jobs -n ${NAMESPACE}
+    ${KUBECTL} get events --sort-by=.lastTimestamp -A | tail -n 30 || true
+    if [[ $((counter % 20)) -eq 0 ]]; then
+      ci::print_pod_logs
+      if [[ $counter -gt 100 ]]; then
+        echo >&2 "Timeout waiting..."
+        exit 1
+      fi
+    fi
+    WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep keycloak-ci-0 | wc -l)
+  done
+
+  echo "Wait until keycloak is ready"
+  ${KUBECTL} wait --for=condition=Ready pod/keycloak-ci-0 -n ${NAMESPACE} --timeout 180s
+
+  echo "Check keycloack realm pulsar issuer url"
+  ${KUBECTL} exec -n ${NAMESPACE} keycloak-ci-0 -c keycloak -- bash -c 'curl -sSL http://keycloak-ci-headless:8080/realms/pulsar'
+
+}
+
+# lists all available functions in this tool
+function ci::list_functions() {
+  declare -F | awk '{print $NF}' | sort | grep -E '^ci::' | sed 's/^ci:://'
+}
+
+# Only run this section if the script is being executed directly (not sourced)
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+  if [ -z "$1" ]; then
+    echo "usage: $0 [function_name]"
+    echo "Available functions:"
+    ci::list_functions
+    exit 1
+  fi
+  ci_function_name="ci::$1"
+  shift
+  if [[ "$(LC_ALL=C type -t "${ci_function_name}")" == "function" ]]; then
+    eval "$ci_function_name" "$@"
+    exit $?
+  else
+    echo "Invalid ci function"
+    echo "Available functions:"
+    ci::list_functions
+    exit 1
+  fi
+fi

.ci/kustomization.yaml

@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+helmGlobals:
+  chartHome: ../charts
+helmCharts:
+  - name: pulsar
+    releaseName: pulsar
+    valuesInline:
+      victoria-metrics-k8s-stack:
+        enabled: false
+      components:
+        pulsar_manager: true
+        zookeeper: true

.ci/metallb/metallb-config.yaml

@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+apiVersion: metallb.io/v1beta1
+kind: IPAddressPool
+metadata:
+  name: example
+  namespace: metallb-system
+spec:
+  addresses:
+  - 172.19.255.200-172.19.255.250
+---
+apiVersion: metallb.io/v1beta1
+kind: L2Advertisement
+metadata:
+  name: empty
+  namespace: metallb-system

.ci/values-common.yaml

@@ -17,15 +17,35 @@
 # under the License.
 #
 
-kube-prometheus-stack:
+victoria-metrics-k8s-stack:
   enabled: false
-  prometheusOperator:
+  victoria-metrics-operator:
     enabled: false
-  grafana:
+  vmsingle:
+    enabled: false
+  vmagent:
+    enabled: false
+  vmalert:
     enabled: false
   alertmanager:
     enabled: false
-  prometheus:
+  grafana:
+    enabled: false
+  prometheus-node-exporter:
+    enabled: false
+  kube-state-metrics:
+    enabled: false
+  kubelet:
+    enabled: false
+  kubeApiServer:
+    enabled: false
+  kubeControllerManager:
+    enabled: false
+  coreDns:
+    enabled: false
+  kubeEtcd:
+    enabled: false
+  kubeScheduler:
     enabled: false
 
 # disabled AntiAffinity
@@ -36,6 +56,8 @@ affinity:
 components:
   autorecovery: false
   pulsar_manager: false
+  # enable functions by default in CI
+  functions: true
 
 zookeeper:
   replicaCount: 1
@@ -53,6 +75,12 @@ bookkeeper:
     diskUsageWarnThreshold: "0.999"
     PULSAR_PREFIX_diskUsageThreshold: "0.999"
     PULSAR_PREFIX_diskUsageWarnThreshold: "0.999"
+    # minimal memory use for bookkeeper
+    # https://bookkeeper.apache.org/docs/reference/config#db-ledger-storage-settings
+    dbStorage_writeCacheMaxSizeMb: "32"
+    dbStorage_readAheadCacheMaxSizeMb: "32"
+    dbStorage_rocksDB_writeBufferSizeMB: "8"
+    dbStorage_rocksDB_blockCacheSize: "8388608"    
 
 broker:
   replicaCount: 1
@@ -84,3 +112,11 @@ proxy:
 
 toolset:
   useProxy: false
+
+oxia:
+  coordinator:
+    podMonitor:
+      enabled: false
+  server:
+    podMonitor:
+      enabled: false

.github/actions/chart-testing-action/action.yml

@@ -39,15 +39,15 @@ inputs:
   version:
     description: "The chart-testing version to install"
     required: false
-    default: v3.10.1
+    default: v3.12.0
   yamllint_version:
     description: "The yamllint version to install"
     required: false
-    default: '1.33.0'
+    default: '1.35.1'
   yamale_version:
     description:  "The yamale version to install"
     required: false
-    default: '4.0.4'
+    default: '6.0.0'
 runs:
   using: composite
   steps:

.github/actions/chart-testing-action/ct.sh

@@ -35,9 +35,20 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-DEFAULT_CHART_TESTING_VERSION=v3.7.1
-DEFAULT_YAMLLINT_VERSION=1.27.1
-DEFAULT_YAMALE_VERSION=3.0.4
+DEFAULT_CHART_TESTING_VERSION=v3.12.0
+DEFAULT_YAMLLINT_VERSION=1.35.1
+DEFAULT_YAMALE_VERSION=6.0.0
+
+ARCH=$(uname -m)
+case $ARCH in
+    x86) ARCH="386";;
+    x86_64) ARCH="amd64";;
+    i686) ARCH="386";;
+    i386) ARCH="386";;
+    arm64) ARCH="arm64";;
+    aarch64) ARCH="arm64";;
+esac
+OS=$(uname|tr '[:upper:]' '[:lower:]')
 
 show_help() {
 cat << EOF
@@ -109,31 +120,35 @@ install_chart_testing() {
         exit 1
     fi
 
-    local arch
-    arch=$(uname -m)
-    local cache_dir="$RUNNER_TOOL_CACHE/ct/$version/$arch"
+    local cache_dir="$RUNNER_TOOL_CACHE/ct/$version/${ARCH}"
     local venv_dir="$cache_dir/venv"
 
     if [[ ! -d "$cache_dir" ]]; then
         mkdir -p "$cache_dir"
 
         echo "Installing chart-testing..."
-        curl -sSLo ct.tar.gz "https://github.com/helm/chart-testing/releases/download/$version/chart-testing_${version#v}_linux_amd64.tar.gz"
+        curl -sSLo ct.tar.gz "https://github.com/helm/chart-testing/releases/download/$version/chart-testing_${version#v}_${OS}_${ARCH}.tar.gz"
         tar -xzf ct.tar.gz -C "$cache_dir"
         rm -f ct.tar.gz
 
+        # if uv (https://docs.astral.sh/uv/) is not installed, install it
+        if ! command -v uv &> /dev/null; then
+            echo 'Installing uv...'
+            curl -LsSf https://astral.sh/uv/install.sh | sh
+        fi
+
         echo 'Creating virtual Python environment...'
-        python3 -m venv "$venv_dir"
+        uv venv "$venv_dir"
 
         echo 'Activating virtual environment...'
         # shellcheck disable=SC1090
         source "$venv_dir/bin/activate"
 
         echo 'Installing yamllint...'
-        pip3 install "yamllint==${yamllint_version}"
+        uv pip install "yamllint==${yamllint_version}"
 
         echo 'Installing Yamale...'
-        pip3 install "yamale==${yamale_version}"
+        uv pip install "yamale==${yamale_version}"
     fi
 
     # https://github.com/helm/chart-testing-action/issues/62

.github/actions/tune-runner-vm/action.yml

@@ -53,8 +53,8 @@ runs:
             # tune filesystem mount options, https://www.kernel.org/doc/Documentation/filesystems/ext4.txt
             # commit=999999, effectively disables automatic syncing to disk (default is every 5 seconds)
             # nobarrier/barrier=0, loosen data consistency on system crash (no negative impact to empheral CI nodes)
-            sudo mount -o remount,nodiscard,commit=999999,barrier=0 /
-            sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt
+            sudo mount -o remount,nodiscard,commit=999999,barrier=0 / || true
+            sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt || true
             # disable discard/trim at device level since remount with nodiscard doesn't seem to be effective
             # https://www.spinics.net/lists/linux-ide/msg52562.html
             for i in /sys/block/sd*/queue/discard_max_bytes; do
@@ -77,12 +77,6 @@ runs:
             # stop Azure Linux agent to save RAM
             sudo systemctl stop walinuxagent.service || true
           
-            # enable docker experimental mode which is
-            # required for using "docker build --squash" / "-Ddocker.squash=true"
-            daemon_json="$(sudo cat /etc/docker/daemon.json  | jq '.experimental = true')"
-            echo "$daemon_json" | sudo tee /etc/docker/daemon.json
-            # restart docker daemon
-            sudo systemctl restart docker
             echo '::endgroup::'
 
             # show memory

.github/workflows/pulsar-helm-chart-ci.yaml

@@ -19,25 +19,30 @@
 
 name: Pulsar Helm Chart CI
 on:
+  push:
+    branches:
+      - master
   pull_request:
     branches:
       - master
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
+ 
   preconditions:
     name: Preconditions
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     if: (github.event_name != 'schedule') || (github.repository == 'apache/pulsar-helm-chart')
     outputs:
       docs_only: ${{ steps.check_changes.outputs.docs_only }}
 
     steps:
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Detect changed files
         id:   changes
@@ -58,18 +63,18 @@ jobs:
   license-check:
     needs: preconditions
     name: License Check
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 10
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
     steps:
       - name: Set up Go 1.12
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: 1.12
         id: go
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Check license
         run: |
@@ -79,14 +84,14 @@ jobs:
   ct-lint:
     needs: ['preconditions', 'license-check']
     name: chart-testing lint
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 45
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
-    outputs:
-      no_chart_changes: ${{ steps.ct-lint.outputs.no_chart_changes }}
     steps:
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
@@ -101,15 +106,19 @@ jobs:
 
       - name: Set up Helm
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
-        uses: azure/setup-helm@v3
+        uses: azure/setup-helm@v4
         with:
-          version: v3.12.3
+          version: v3.16.4
 
       - name: Set up Python
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: '3.12'
+
+      - name: Install uv, a fast modern package manager for Python
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+        run: curl -LsSf https://astral.sh/uv/install.sh | sh
 
       - name: Set up chart-testing
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
@@ -121,16 +130,46 @@ jobs:
         run: |
           ct lint --check-version-increment=false \
             --validate-maintainers=false \
-            --target-branch ${{ github.event.repository.default_branch }} \
-            | tee /tmp/ct-lint.log || { 
-            if grep -q "No chart changes detected." /tmp/ct-lint.log; then
-              echo no_chart_changes=true >> $GITHUB_OUTPUT
-              exit 0
+            --target-branch ${{ github.event.repository.default_branch }}
+
+      - name: Run kubeconform check for helm template with every major k8s version 1.25.0-1.32.0
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+        run: |
+          PULSAR_CHART_HOME=$(pwd)
+          source ${PULSAR_CHART_HOME}/hack/common.sh
+          source ${PULSAR_CHART_HOME}/.ci/helm.sh
+          hack::ensure_kubectl
+          hack::ensure_helm
+          hack::ensure_kubeconform
+          ci::helm_repo_add
+          helm dependency build charts/pulsar
+          validate_helm_template_with_k8s_version() {
+            local kube_version=$1
+            shift
+            echo -n "Validating helm template with kubeconform for k8s version $kube_version"
+            if [ $# -gt 0 ]; then
+              echo " Extra args: $*"
             else
-              echo no_chart_changes=false >> $GITHUB_OUTPUT
-              exit 1
+              echo ""
             fi
+            helm template charts/pulsar --set victoria-metrics-k8s-stack.enabled=false --set components.pulsar_manager=true --kube-version $kube_version "$@" | \
+              kubeconform -schema-location default -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' -strict -kubernetes-version $kube_version -summary
           }
+          set -o pipefail
+          for k8s_version_part in {25..32}; do
+            k8s_version="1.${k8s_version_part}.0"
+            echo "Validating default values with k8s version $k8s_version"
+            validate_helm_template_with_k8s_version $k8s_version
+            for config in .ci/clusters/*.yaml; do
+              echo "Validating $config with k8s version $k8s_version"
+              validate_helm_template_with_k8s_version $k8s_version --values .ci/values-common.yaml --values $config
+            done
+          done
+
+      - name: Validate kustomize yaml for extra new lines in pulsar-init commands
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+        run: |
+          ./.ci/helm.sh validate_kustomize_yaml
 
       - name: Wait for ssh connection when build fails
         # ssh access is enabled for builds in own forks
@@ -141,30 +180,29 @@ jobs:
           action: wait
 
   install-chart-tests:
-    name: ${{ matrix.testScenario.name }} - k8s ${{ matrix.k8sVersion.version }} - Install
-    runs-on: ubuntu-22.04
+    name: ${{ matrix.testScenario.name }} - k8s ${{ matrix.k8sVersion.version }} - ${{ matrix.testScenario.type || 'install' }}
+    runs-on: ubuntu-24.04
     timeout-minutes: ${{ matrix.testScenario.timeout || 45 }}
     needs: ['preconditions', 'ct-lint']
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
     strategy:
       fail-fast: false
       matrix:
-        # see https://github.com/kubernetes-sigs/kind/releases/tag/v0.20.0 for the list of supported k8s versions for kind 0.20.0
+        # see https://github.com/kubernetes-sigs/kind/releases/tag/v0.27.0 for the list of supported k8s versions for kind 0.27.0
+        # docker images are available at https://hub.docker.com/r/kindest/node/tags
         k8sVersion:
-          - version: "1.21.14"
-            kind_image_tag: v1.21.14@sha256:8a4e9bb3f415d2bb81629ce33ef9c76ba514c14d707f9797a01e3216376ba093
-          - version: "1.27.3"
-            kind_image_tag: v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72
+          - version: "1.25.16"
+            kind_image_tag: v1.25.16@sha256:6110314339b3b44d10da7d27881849a87e092124afab5956f2e10ecdb463b025
+          - version: "1.32.2"
+            kind_image_tag: v1.32.2@sha256:f226345927d7e348497136874b6d207e0b32cc52154ad8323129352923a3142f
         testScenario:
-          - name: Basic
-            values_file: .ci/clusters/values-local-pv.yaml
-            shortname: basic
-          - name: Pulsar Function
-            values_file: .ci/clusters/values-function.yaml
-            shortname: function
-          - name: Use Pulsar Image
-            values_file: .ci/clusters/values-pulsar-image.yaml
-            shortname: pulsar-image
+          - name: Upgrade latest released version
+            values_file: .ci/clusters/values-upgrade.yaml
+            shortname: upgrade
+            type: upgrade
+          - name: Use previous LTS Pulsar Image
+            values_file: .ci/clusters/values-pulsar-previous-lts.yaml
+            shortname: pulsar-previous-lts
           - name: JWT Asymmetric Keys
             values_file: .ci/clusters/values-jwt-asymmetric.yaml
             shortname: jwt-asymmetric
@@ -186,50 +224,59 @@ jobs:
           - name: ZK & BK TLS Only
             values_file: .ci/clusters/values-zkbk-tls.yaml
             shortname: zkbk-tls
+          - name: Pulsar Manager
+            values_file: .ci/clusters/values-pulsar-manager.yaml
+            shortname: pulsar-manager
+          - name: Oxia
+            values_file: .ci/clusters/values-oxia.yaml
+            shortname: oxia
+          - name: OpenID
+            values_file: .ci/clusters/values-openid.yaml
+            shortname: openid
+          - name: CA certificates
+            values_file: .ci/clusters/values-cacerts.yaml
+            shortname: cacerts
+        include:
+          - k8sVersion:
+              version: "1.25.16"
+              kind_image_tag: v1.25.16@sha256:6110314339b3b44d10da7d27881849a87e092124afab5956f2e10ecdb463b025
+            testScenario:
+              name: "Upgrade TLS"
+              values_file: .ci/clusters/values-tls.yaml
+              shortname: tls
+              type: upgrade
+          - k8sVersion:
+              version: "1.25.16"
+              kind_image_tag: v1.25.16@sha256:6110314339b3b44d10da7d27881849a87e092124afab5956f2e10ecdb463b025
+            testScenario:
+              name: "Upgrade victoria-metrics-k8s-stack for previous LTS"
+              values_file: .ci/clusters/values-victoria-metrics-grafana.yaml --values .ci/clusters/values-pulsar-previous-lts.yaml
+              shortname: victoria-metrics-grafana
+              type: upgrade
+              upgradeFromVersion: 3.2.0
+          - k8sVersion:
+              version: "1.25.16"
+              kind_image_tag: v1.25.16@sha256:6110314339b3b44d10da7d27881849a87e092124afab5956f2e10ecdb463b025
+            testScenario:
+              name: "TLS with helm 3.12.0"
+              values_file: .ci/clusters/values-tls.yaml
+              shortname: tls
+              type: install
+            helmVersion: 3.12.0
     env:
       k8sVersion: ${{ matrix.k8sVersion.kind_image_tag }}
       KUBECTL_VERSION: ${{ matrix.k8sVersion.version }}
+      HELM_VERSION: ${{ matrix.helmVersion || '3.14.4' }}
     steps:
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
       - name: Setup debugging tools for ssh access
         if: ${{ github.repository != 'apache/pulsar-helm-chart' && github.event_name == 'pull_request' }}
-        run: |
-          cat >> $HOME/.bashrc <<'EOF'
-          function use_kind_kubeconfig() {
-            export KUBECONFIG=$(ls $HOME/kind/pulsar-ci-*/kubeconfig.yaml)
-          }
-
-          function kubectl() {
-            # use kind environment's kubeconfig
-            if [ -z "$KUBECONFIG" ]; then
-              use_kind_kubeconfig
-            fi
-            command kubectl "$@"
-          }
-
-          function k9s() {
-            # use kind environment's kubeconfig
-            if [ -z "$KUBECONFIG" ]; then
-              use_kind_kubeconfig
-            fi
-            # install k9s on the fly
-            if [ ! -x /usr/local/bin/k9s ]; then
-              echo "Installing k9s..."
-              curl -L -s https://github.com/derailed/k9s/releases/download/v0.29.1/k9s_Linux_amd64.tar.gz | sudo tar xz -C /usr/local/bin k9s
-            fi
-            command k9s "$@"
-          }
-          EOF
-          cat >> $HOME/.bash_profile <<'EOF'
-          if [ -f ~/.bashrc ]; then
-            source ~/.bashrc
-          fi
-          EOF
+        run: .ci/configure_ci_runner_for_debugging.sh
 
       - name: Setup ssh access to build runner VM
         # ssh access is enabled for builds in own forks
@@ -239,13 +286,23 @@ jobs:
         with:
           limit-access-to-actor: true
 
-      - name: Run chart-testing (install)
+      - name: Run chart-testing (${{ matrix.testScenario.type || 'install' }}) with helm ${{ env.HELM_VERSION }}
         run: |
           case "${{ matrix.testScenario.shortname }}" in
             "jwt-symmetric")
               export SYMMETRIC=true
+              export EXTRA_SUPERUSERS=manager-admin
+              ;;
+            "jwt-asymmetric")
+              export EXTRA_SUPERUSERS=manager-admin
+              ;;
+            "openid")
+              export AUTHENTICATION_PROVIDER=openid
               ;;
           esac
+          if [[ "${{ matrix.testScenario.type || 'install' }}" == "upgrade" ]]; then
+            export UPGRADE_FROM_VERSION="${{ matrix.testScenario.upgradeFromVersion || 'latest' }}"
+          fi
           .ci/chart_test.sh ${{ matrix.testScenario.values_file }}
 
       - name: Collect k8s logs on failure
@@ -258,7 +315,7 @@ jobs:
           ci::collect_k8s_logs
 
       - name: Upload k8s logs on failure
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         if: ${{ cancelled() || failure() }}
         continue-on-error: true
         with:
@@ -280,7 +337,7 @@ jobs:
   pulsar-helm-chart-ci-checks-completed:
     name: "CI checks completed"
     if: ${{ always() && ((github.event_name != 'schedule') || (github.repository == 'apache/pulsar-helm-chart')) }}
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 10
     needs: [
       'preconditions',

.gitignore

@@ -17,5 +17,3 @@ charts/**/*.lock
 PRIVATEKEY
 PUBLICKEY
 .vagrant/
-pulsarctl-*-*.tar.gz
-pulsarctl-*-*/

README.md

@@ -27,6 +27,113 @@ Read [Deploying Pulsar on Kubernetes](http://pulsar.apache.org/docs/deploy-kuber
 
 > :warning: This helm chart is updated outside of the regular Pulsar release cycle and might lag behind a bit. It only supports basic Kubernetes features now. Currently, it can be used as no more than a template and starting point for a Kubernetes deployment. In many cases, it would require some customizations.
 
+## Important Security Advisory for Helm Chart Usage
+
+### Notice of Default Configuration
+
+This Helm chart's default configuration DOES NOT meet production security requirements.
+Users MUST review and customize security settings for their specific environment.
+
+IMPORTANT: This Helm chart provides a starting point for Pulsar deployments but requires
+significant security customization before use in production environments. We strongly
+recommend implementing:
+
+1. Authentication and authorization for all components
+2. TLS encryption for all communication channels
+3. Proper network isolation and access controls
+4. Regular security updates and vulnerability assessments
+
+As an open source project, we welcome contributions to improve security features.
+Please consider submitting pull requests to address security gaps or enhance
+existing security implementations.
+
+### Pulsar Proxy Security Considerations
+
+As per the [Pulsar Proxy documentation](https://pulsar.apache.org/docs/3.1.x/administration-proxy/), it is explicitly stated that the Pulsar proxy is not designed for exposure to the public internet. The design assumes that deployments will be protected by network perimeter security measures. It is crucial to understand that relying solely on the default configuration can expose your deployment to significant security vulnerabilities.
+
+### Upgrading
+
+#### To 4.1.0
+
+This version introduces `OpenID` authentication. Setting `auth.authentication.provider` is no longer supported, you need to enable the provider with `auth.authentication.<provider>.enabled`.
+
+#### To 4.0.0
+
+The default service type for the Pulsar proxy has changed from `LoadBalancer` to `ClusterIP` for security reasons. This limits access to within the Kubernetes environment by default.
+
+### External Access Recommendations
+
+If you need to expose the Pulsar Proxy outside the cluster:
+
+1. **USE INTERNAL LOAD BALANCERS ONLY**
+   - Set type to LoadBalancer only in secured environments with proper network controls
+   - Add cloud provider-specific annotations for internal load balancers:
+     - Kubernetes documentation about internal load balancers:
+        - [Internal load balancer](https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer)
+     - See cloud provider documentation:
+       - AWS / EKS: [AWS Load Balancer Controller / Service Annotations](https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/guide/service/annotations/)
+       - Azure / AKS: [Use an internal load balancer with Azure Kubernetes Service (AKS)](https://learn.microsoft.com/en-us/azure/aks/internal-lb)
+       - GCP / GKE: [LoadBalancer service parameters](https://cloud.google.com/kubernetes-engine/docs/concepts/service-load-balancer-parameters)
+     - Examples (verify correctness for your environment):
+       - AWS / EKS:  `service.beta.kubernetes.io/aws-load-balancer-internal: "true"`
+       - Azure / AKS: `service.beta.kubernetes.io/azure-load-balancer-internal: "true"`
+       - GCP / GKE:   `networking.gke.io/load-balancer-type: "Internal"`
+
+2. **IMPLEMENT AUTHENTICATION AND AUTHORIZATION**
+   - Configure all clients to authenticate properly
+   - Set up appropriate authorization policies
+
+3. **USE TLS FOR ALL CONNECTIONS**
+   - Enable TLS for client-to-proxy connections
+   - Enable TLS for proxy-to-broker connections
+   - Enable TLS for all internal cluster communications
+   - Note: TLS alone is NOT sufficient as a security solution. Even with TLS enabled, clusters exposed to untrusted networks remain vulnerable to denial-of-service attacks, authentication bypass attempts, and protocol-level exploits.
+
+4. **NETWORK SECURITY**
+   - Use private networks (VPCs)
+   - Configure firewalls, security groups, and IP restrictions
+
+5. **CLIENT IP ADDRESS BASED ACCESS RESTRICTIONS**
+
+   - When using a LoadBalancer service type, restrict access to specific IP ranges by configuring `proxy.service.loadBalancerSourceRanges` in your values.yaml:
+     ```yaml
+     proxy:
+       service:
+         loadBalancerSourceRanges:
+           - 10.0.0.0/8     # Private network range
+           - 172.16.0.0/12  # Private network range
+           - 192.168.0.0/16 # Private network range
+     ```
+   - This feature:
+     - Provides an additional defense layer by filtering traffic at the load balancer level
+     - Only allows connections from specified CIDR blocks
+     - Works only with LoadBalancer service type and when your cloud provider supports the `loadBalancerSourceRanges` parameter
+   - Important: This should be implemented alongside other security measures (internal load balancer, authentication, TLS, network policies) as part of a defense-in-depth strategy,
+     not as a standalone security solution
+
+### Alternative for External Access
+
+As an alternative method for external access, Pulsar has support for [SNI proxy routing](https://pulsar.apache.org/docs/next/concepts-proxy-sni-routing/). SNI Proxy routing is supported with proxy servers such as Apache Traffic Server, HAProxy and Nginx.
+
+Note: This option isn't currently implemented in the Apache Pulsar Helm chart.
+
+**IMPORTANT**: Pulsar binary protocol cannot be exposed outside of the Kubernetes cluster using Kubernetes Ingress. Kubernetes Ingress works for the Admin REST API and topic lookups, but clients would be connecting to the advertised listener addresses returned by the brokers and it would only work when clients can connect directly to brokers. This is not a supported secure option for exposing Pulsar to untrusted networks.
+
+### General Recommendations
+
+- **Network Perimeter Security:** It is imperative to implement robust network perimeter security to safeguard your deployment. The absence of such security measures can lead to unauthorized access and potential data breaches.
+- **Restricted Access:** For environments where security is less critical, such as certain development or testing scenarios, the use of `loadBalancerSourceRanges` may be employed to restrict access to specified IP addresses or ranges. This, however, should not be considered a substitute for comprehensive security measures in production environments.
+
+### User Responsibility
+
+The user assumes full responsibility for the security and integrity of their deployment. This includes, but is not limited to, the proper configuration of security features and adherence to best practices for securing network access. The providers of this Helm chart disclaim all warranties, whether express or implied, including any warranties of merchantability, fitness for a particular purpose, and non-infringement of third-party rights.
+
+### No Security Guarantees
+
+The providers of this Helm chart make no guarantees regarding the security of the chart under any circumstances. It is the user's responsibility to ensure that their deployment is secure and complies with all relevant security standards and regulations.
+
+By using this Helm chart, the user acknowledges the risks associated with its default configuration and the necessity for proper security customization. The user further agrees that the providers of the Helm chart shall not be liable for any security breaches or incidents resulting from the use of the chart.
+
 ## Features
 
 This Helm Chart includes all the components of Apache Pulsar for a complete experience.
@@ -40,7 +147,7 @@ This Helm Chart includes all the components of Apache Pulsar for a complete expe
 - [x] Management & monitoring components:
     - [x] Pulsar Manager
     - [x] Optional PodMonitors for each component (enabled by default)
-    - [x] [Kube-Prometheus-Stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) (as of 3.0.0)
+    - [x] [victoria-metrics-k8s-stack](hhttps://github.com/VictoriaMetrics/helm-charts/tree/master/charts/victoria-metrics-k8s-stack) (as of 4.0.0)
 
 It includes support for:
 
@@ -53,9 +160,10 @@ It includes support for:
         - [x] Broker
         - [x] Toolset
         - [x] Bookie
-        - [x] ZooKeeper
+        - [x] ZooKeeper (requires the `AdditionalCertificateOutputFormats=true` feature gate to be enabled in the cert-manager deployment when using cert-manager versions below 1.15.0)
     - [x] Authentication
         - [x] JWT
+        - [x] OpenID
         - [ ] Mutal TLS
         - [ ] Kerberos
     - [x] Authorization
@@ -64,7 +172,7 @@ It includes support for:
     - [x] Non-persistence storage
     - [x] Persistence Volume
     - [x] Local Persistent Volumes
-    - [ ] Tiered Storage
+    - [x] Tiered Storage
 - [x] Functions
     - [x] Kubernetes Runtime
     - [x] Process Runtime
@@ -76,9 +184,9 @@ It includes support for:
 
 In order to use this chart to deploy Apache Pulsar on Kubernetes, the followings are required.
 
-1. kubectl 1.21 or higher, compatible with your cluster ([+/- 1 minor release from your cluster](https://kubernetes.io/docs/tasks/tools/install-kubectl/#before-you-begin))
-2. Helm v3 (3.0.2 or higher)
-3. A Kubernetes cluster, version 1.21 or higher.
+1. kubectl 1.25 or higher, compatible with your cluster ([+/- 1 minor release from your cluster](https://kubernetes.io/docs/tasks/tools/install-kubectl/#before-you-begin))
+2. Helm v3 (3.12.0 or higher)
+3. A Kubernetes cluster, version 1.25 or higher.
 
 ## Environment setup
 
@@ -93,26 +201,62 @@ Before proceeding to deploying Pulsar, you need to prepare your environment.
 To add this chart to your local Helm repository:
 
 ```bash
-helm repo add apache https://pulsar.apache.org/charts
+helm repo add apachepulsar https://pulsar.apache.org/charts
+helm repo update
 ```
 
 ## Kubernetes cluster preparation
 
-You need a Kubernetes cluster whose version is 1.21 or higher in order to use this chart, due to the usage of certain Kubernetes features.
+You need a Kubernetes cluster whose version is 1.25 or higher in order to use this chart, due to the usage of certain Kubernetes features.
 
 We provide some instructions to guide you through the preparation: http://pulsar.apache.org/docs/helm-prepare/
 
 ## Deploy Pulsar to Kubernetes
 
 1. Configure your values file. The best way to know which values are available is to read the [values.yaml](./charts/pulsar/values.yaml).
+   A best practice is to start with an empty values file and only set the keys that differ from the default configuration.
+
+   Anti-affinity rules for Zookeeper and Bookie components require at least one node per replica. For Kubernetes clusters with less than 3 nodes,
+   you must disable this feature by adding this to your initial values.yaml file:
+
+    ```yaml
+    affinity:
+      anti_affinity: false
+    ```
 
 2. Install the chart:
 
     ```bash
-    helm install <release-name> -n <namespace> -f your-values.yaml apache/pulsar
+    helm install -n <namespace> --create-namespace <release-name> -f your-values.yaml apachepulsar/pulsar
     ```
 
-3. Access the Pulsar cluster
+3. Observe the deployment progress
+
+    Watching events to view progress of deployment:
+
+    ```shell
+    kubectl get -n <namespace> events -o wide --watch
+    ```
+
+    Watching state of deployed Kubernetes objects, updated every 2 seconds:
+
+    ```shell
+    watch kubectl get -n <namespace> all
+    ```
+
+    Waiting until Pulsar Proxy is available:
+
+    ```shell
+    kubectl wait --timeout=600s --for=condition=ready pod -n <namespace> -l component=proxy
+    ```
+
+    Watching state with k9s (https://k9scli.io/topics/install/):
+
+    ```shell
+    k9s -n <namespace>
+    ```
+
+4. Access the Pulsar cluster
 
     The default values will create a `ClusterIP` for the proxy you can use to interact with the cluster. To find the IP address of proxy use:
 
@@ -139,35 +283,102 @@ You can also checkout out the example values file for different deployments.
 - [Deploy a Pulsar cluster with JWT authentication using symmetric key](examples/values-jwt-symmetric.yaml)
 - [Deploy a Pulsar cluster with JWT authentication using asymmetric key](examples/values-jwt-asymmetric.yaml)
 
-## Disabling Kube-Prometheus-Stack CRDs
+## Disabling victoria-metrics-k8s-stack components
 
-In order to disable the kube-prometheus-stack fully, it is necessary to add the following to your `values.yaml`:
+In order to disable the victoria-metrics-k8s-stack, you can add the following to your `values.yaml`.
+Victoria Metrics components can also be disabled and enabled individually if you only need specific monitoring features.
 
 ```yaml
-kube-prometheus-stack:
+# disable VictoriaMetrics and related components
+victoria-metrics-k8s-stack:
   enabled: false
-  prometheusOperator:
+  victoria-metrics-operator:
+    enabled: false
+  vmsingle:
+    enabled: false
+  vmagent:
+    enabled: false
+  kube-state-metrics:
+    enabled: false
+  prometheus-node-exporter:
     enabled: false
   grafana:
     enabled: false
-  alertmanager:
+
+Additionally, you'll need to set each component's `podMonitor` property to `false`. 
+
+```yaml
+# disable pod monitors
+autorecovery:
+  podMonitor:
     enabled: false
-  prometheus:
+bookkeeper:
+  podMonitor:
+    enabled: false
+oxia:
+  server:
+    podMonitor:
+      enabled: false
+  coordinator:
+    podMonitor:
+      enabled: false
+broker:
+  podMonitor:
+    enabled: false
+proxy:
+  podMonitor:
+    enabled: false
+zookeeper:
+  podMonitor:
     enabled: false
 ```
 
-Otherwise, the helm chart installation will attempt to install the CRDs for the kube-prometheus-stack. Additionally,
-you'll need to disable each of the component's `PodMonitors`. This is shown in some [examples](./examples) and is
-verified in some [tests](./.ci/clusters).
+This is shown in some [examples/values-disable-monitoring.yaml](examples/values-disable-monitoring.yaml).
+
+## Pulsar Manager
+
+The Pulsar Manager can be deployed alongside the pulsar cluster instance.
+Depending on the given settings it uses an existing Secret within the given namespace or creates a new one, with random
+passwords for both, the UI and the internal database.
+
+To forward the UI use (assumes you did not change the namespace):
+
+```
+kubectl port-forward $(kubectl get pods -l component=pulsar-manager -o jsonpath='{.items[0].metadata.name}') 9527:9527
+```
+
+And then opening the browser to http://localhost:9527
+
+The default user is `pulsar` and you can find out the password with this command
+
+```
+kubectl get secret -l component=pulsar-manager -o=jsonpath="{.items[0].data.UI_PASSWORD}" | base64 --decode
+```
 
 ## Grafana Dashboards
 
-The Apache Pulsar Helm Chart uses the `kube-prometheus-stack` Helm Chart to deploy Grafana. Dashboards are loaded via a Kubernetes `ConfigMap`. Please see their documentation for loading those dashboards.
+The Apache Pulsar Helm Chart uses the `victoria-metrics-k8s-stack` Helm Chart to deploy Grafana.
 
-The `apache/pulsar` GitHub repo contains some dashboards [here](https://github.com/apache/pulsar/tree/master/grafana).
+There are several ways to configure Grafana dashboards. The default [`values.yaml`](charts/pulsar/values.yaml) comes with examples of Pulsar dashboards which get downloaded from the Apache-2.0 licensed [lhotari/pulsar-grafana-dashboards OSS project](https://github.com/lhotari/pulsar-grafana-dashboards) by URL.
 
-### Third Party Dashboards
+Dashboards can be configured in [`values.yaml`](charts/pulsar/values.yaml) or by adding `ConfigMap` items with the label `grafana_dashboard: "1"`.
+In [`values.yaml`](charts/pulsar/values.yaml), it's possible to include dashboards by URL or by grafana.com dashboard id (`gnetId` and `revision`).
+Please see the [Grafana Helm chart documentation for importing dashboards](https://github.com/grafana/helm-charts/blob/main/charts/grafana/README.md#import-dashboards).
 
+You can connect to Grafana by forwarding port 3000
+```
+kubectl port-forward $(kubectl get pods -l app.kubernetes.io/name=grafana -o jsonpath='{.items[0].metadata.name}') 3000:3000
+```
+And then opening the browser to http://localhost:3000 . The default user is `admin`.
+
+You can find out the password with this command
+```
+kubectl get secret -l app.kubernetes.io/name=grafana -o=jsonpath="{.items[0].data.admin-password}" | base64 --decode
+```
+
+### Pulsar Grafana Dashboards
+
+* The `apache/pulsar` GitHub repo contains some Grafana dashboards [here](https://github.com/apache/pulsar/tree/master/grafana).
 * StreamNative provides Grafana Dashboards for Apache Pulsar in this [GitHub repository](https://github.com/streamnative/apache-pulsar-grafana-dashboard).
 * DataStax provides Grafana Dashboards for Apache Pulsar in this [GitHub repository](https://github.com/datastax/pulsar-helm-chart/tree/master/helm-chart-sources/pulsar/grafana-dashboards).
 
@@ -179,21 +390,58 @@ Once your Pulsar Chart is installed, configuration changes and chart
 updates should be done using `helm upgrade`.
 
 ```bash
-helm repo add apache https://pulsar.apache.org/charts
+helm repo add apachepulsar https://pulsar.apache.org/charts
 helm repo update
-helm get values <pulsar-release-name> > pulsar.yaml
-helm upgrade -f pulsar.yaml \
-    <pulsar-release-name> apache/pulsar
+# If you are using the provided victoria-metrics-k8s-stack for monitoring, this installs or upgrades the required CRDs
+./scripts/victoria-metrics-k8s-stack/upgrade_vm_operator_crds.sh
+# get the existing values.yaml used for the most recent deployment
+helm get values -n <namespace> <pulsar-release-name> > values.yaml
+# upgrade the deployment
+helm upgrade -n <namespace> -f values.yaml <pulsar-release-name> apachepulsar/pulsar
 ```
 
 For more detailed information, see our [Upgrading](http://pulsar.apache.org/docs/helm-upgrade/) guide.
 
+## Upgrading to Helm chart version 4.2.0 (not released yet)
+
+### TLS configuration for ZooKeeper has changed
+
+The TLS configuration for ZooKeeper has been changed to fix certificate and private key expiration issues.
+This change impacts configurations that have `tls.enabled` and `tls.zookeeper.enabled` set in `values.yaml`.
+The revised solution requires the `AdditionalCertificateOutputFormats=true` feature gate to be enabled in the `cert-manager` deployment when using cert-manager versions below 1.15.0.
+If you installed `cert-manager` using `./scripts/cert-manager/install-cert-manager.sh`, you can re-run the updated script to set the feature gate. The script currently installs or upgrades cert-manager LTS version 1.12.17, where the feature gate must be explicitly enabled.
+
+## Upgrading from Helm Chart versions before 4.0.0 to 4.0.0 version and above
+
+### Pulsar Proxy service's default type has been changed from `LoadBalancer` to `ClusterIP`
+
+Please check the section "External Access Recommendations" for guidance and also check the security advisory section.
+You will need to configure keys under `proxy.service` in your `values.yaml` to preserve existing functionality since the default has been changed.
+
+### kube-prometheus-stack replaced with victoria-metrics-k8s-stack
+
+The `kube-prometheus-stack` was replaced with `victoria-metrics-k8s-stack` in Pulsar Helm chart version 4.0.0. The trigger for the change was incompatibilities discovered in testing with most recent `kube-prometheus-stack` and Prometheus 3.2.1 which failed to scrape Pulsar metrics in certain cases without providing proper error messages or debug information at debug level logging.
+
+[Victoria Metrics](https://docs.victoriametrics.com/) is Apache 2.0 Licensed OSS and it's a fully compatible drop-in replacement for Prometheus which is fast and efficient.
+
+Before upgrading to Pulsar Helm Chart version 4.0.0, it is recommended to disable kube-prometheus-stack in the original Helm chart version that
+is used:
+
+```shell
+# get the existing values.yaml used for the most recent deployment
+helm get values -n <namespace> <pulsar-release-name> > values.yaml
+# disable kube-prometheus-stack in the currently used version before upgrading to Pulsar Helm chart 4.0.0
+helm upgrade -n <namespace> -f values.yaml --version <your-current-chart-version> --set kube-prometheus-stack.enabled=false  <pulsar-release-name> apachepulsar/pulsar
+```
+
+After, this you can proceed with `helm upgrade`.
+
 ## Upgrading to Apache Pulsar 2.10.0 and above (or Helm Chart version 3.0.0 and above)
 
 The 2.10.0+ Apache Pulsar docker image is a non-root container, by default. That complicates an upgrade to 2.10.0
 because the existing files are owned by the root user but are not writable by the root group. In order to leverage this
 new security feature, the Bookkeeper and Zookeeper StatefulSet [securityContexts](https://kubernetes.io/docs/tasks/configure-pod-container/security-context)
-are configurable in the `values.yaml`. They default to:
+are configurable in the [`values.yaml`](charts/pulsar/values.yaml). They default to:
 
 ```yaml
   securityContext:
@@ -238,15 +486,47 @@ Caused by: org.rocksdb.RocksDBException: while open a file for lock: /pulsar/dat
     ... 13 more
 ```
 
+### Recovering from `helm upgrade` error "unable to build kubernetes objects from current release manifest"
+
+Example of the error message:
+
+```bash
+Error: UPGRADE FAILED: unable to build kubernetes objects from current release manifest:
+[resource mapping not found for name: "pulsar-bookie" namespace: "pulsar" from "":
+no matches for kind "PodDisruptionBudget" in version "policy/v1beta1" ensure CRDs are installed first,
+resource mapping not found for name: "pulsar-broker" namespace: "pulsar" from "":
+no matches for kind "PodDisruptionBudget" in version "policy/v1beta1" ensure CRDs are installed first,
+resource mapping not found for name: "pulsar-zookeeper" namespace: "pulsar" from "":
+no matches for kind "PodDisruptionBudget" in version "policy/v1beta1" ensure CRDs are installed first]
+```
+
+Helm documentation [explains issues with managing releases deployed using outdated APIs](https://helm.sh/docs/topics/kubernetes_apis/#helm-users) when the Kubernetes cluster has been upgraded
+to a version where these APIs are removed. This happens regardless of whether the chart in the upgrade includes supported API versions.
+In this case, you can use the following workaround:
+
+1. Install the [Helm mapkubeapis plugin](https://github.com/helm/helm-mapkubeapis):
+
+    ```bash
+    helm plugin install https://github.com/helm/helm-mapkubeapis
+    ```
+
+2. Run the `helm mapkubeapis` command with the appropriate namespace and release name. In this example, we use the namespace "pulsar" and release name "pulsar":
+
+    ```bash
+    helm mapkubeapis --namespace pulsar pulsar
+    ```
+
+This workaround addresses the issue by updating in-place Helm release metadata that contains deprecated or removed Kubernetes APIs to a new instance with supported Kubernetes APIs and should allow for a successful Helm upgrade.
+
 ## Uninstall
 
 To uninstall the Pulsar Chart, run the following command:
 
 ```bash
-helm delete <pulsar-release-name>
+helm uninstall <pulsar-release-name>
 ```
 
-For the purposes of continuity, these charts have some Kubernetes objects that are not removed when performing `helm delete`.
+For the purposes of continuity, these charts have some Kubernetes objects that are not removed when performing `helm uninstall`.
 These items we require you to *conciously* remove them, as they affect re-deployment should you choose to.
 
 * PVCs for stateful data, which you must *consciously* remove
@@ -261,6 +541,36 @@ We've done our best to make these charts as seamless as possible,
 occasionally troubles do surface outside of our control. We've collected
 tips and tricks for troubleshooting common issues. Please examine these first before raising an [issue](https://github.com/apache/pulsar-helm-chart/issues/new/choose), and feel free to add to them by raising a [Pull Request](https://github.com/apache/pulsar-helm-chart/compare)!
 
+### VictoriaMetrics Troubleshooting
+
+In example commands, k8s is namespace `pulsar` replace with your deployment namespace.
+
+#### VictoriaMetrics Web UI
+
+Connecting to `vmsingle` pod for web UI.
+
+```shell
+kubectl port-forward -n pulsar $(kubectl get pods -n pulsar -l app.kubernetes.io/name=vmsingle -o jsonpath='{.items[0].metadata.name}') 8429:8429
+```
+
+Now you can access the UI at http://localhost:8429 and http://localhost:8429/vmui (for similar UI as in Prometheus)
+
+#### VictoriaMetrics Scraping debugging UI - Active Targets
+
+Connection to `vmagent` pod for debugging targets.
+
+```shell
+kubectl port-forward -n pulsar $(kubectl get pods -n pulsar -l app.kubernetes.io/name=vmagent -o jsonpath='{.items[0].metadata.name}') 8429:8429
+```
+
+Now you can access the UI at http://localhost:8429
+
+Active Targets UI
+- http://localhost:8429/targets
+
+Scraping Configuration
+- http://localhost:8429/config
+
 ## Release Process
 
 See [RELEASE.md](RELEASE.md)

RELEASE.md

@@ -23,7 +23,7 @@ This document details the steps for releasing the Apache Pulsar Helm Chart.
 
 ## Prerequisites
 
-- Helm version >= 3.0.2
+- Helm version >= 3.12.0
 - Helm gpg plugin (one option: https://github.com/technosophos/helm-gpg)
 
 ## Build Release Notes
@@ -42,35 +42,44 @@ official Apache releases must not include the rcN suffix.
 
     ```shell
     # Set Version
-    export VERSION=3.0.0-candidate-1
-    export VERSION_WITHOUT_RC=${VERSION%-candidate-*}
+    export VERSION_RC=3.0.0-candidate-1
+    export VERSION_WITHOUT_RC=${VERSION_RC%-candidate-*}
+    # set your ASF user id
+    export APACHE_USER=<your ASF userid>
+    ```
 
-    # Clone and set PULSAR_REPO_ROOT
-    git clone https://github.com/apache/pulsar-helm-chart.git pulsar
+- Clone clean repository and set PULSAR_REPO_ROOT
+
+    ```shell
+    git clone https://github.com/apache/pulsar-helm-chart.git
     cd pulsar-helm-chart
     export PULSAR_REPO_ROOT=$(pwd)
     ```
 
-- We currently release Helm Chart from `master` branch:
+- Alternatively (not recommended), go to your already checked out pulsar-helm-chart directory and ensure that it's clean
 
     ```shell
     git checkout master
-    ```
-
-- Clean the checkout: the sdist step below will
-
-    ```shell
+    git fetch origin
+    git reset --hard origin/master
+    # clean the checkout
     git clean -fdX .
+    export PULSAR_REPO_ROOT=$(pwd)
     ```
 
 - Update Helm Chart version in `Chart.yaml`, example: `version: 1.0.0` (without
   the RC tag). Verify that the `appVersion` matches the `values.yaml` versions for Pulsar components.
 
+    ```shell
+    yq -i '.version=strenv(VERSION_WITHOUT_RC)' charts/pulsar/Chart.yaml
+    ```
+
 - Add and commit the version change.
 
     ```shell
     git add charts/pulsar/Chart.yaml
     git commit -m "Chart: Bump version to $VERSION_WITHOUT_RC"
+    git push origin master
     ```
 
   Note: You will tag this commit, you do not need to open a PR for it.
@@ -78,7 +87,7 @@ official Apache releases must not include the rcN suffix.
 - Tag your release
 
     ```shell
-    git tag -s pulsar-${VERSION} -m "Apache Pulsar Helm Chart $VERSION"
+    git tag -u $APACHE_USER@apache.org -s pulsar-${VERSION_RC} -m "Apache Pulsar Helm Chart $VERSION_RC"
     ```
 
 - Tarball the repo
@@ -87,7 +96,7 @@ official Apache releases must not include the rcN suffix.
      in the file produced.
 
     ```shell
-    git archive --format=tar.gz pulsar-${VERSION} --prefix=pulsar-chart-${VERSION_WITHOUT_RC}/ \
+    git archive --format=tar.gz pulsar-${VERSION_RC} --prefix=pulsar-chart-${VERSION_WITHOUT_RC}/ \
         -o pulsar-chart-${VERSION_WITHOUT_RC}-source.tar.gz .
     ```
 
@@ -106,7 +115,7 @@ official Apache releases must not include the rcN suffix.
     http://www.apache.org/dev/openpgp.html#key-gen-generate-key)
 
     ```shell
-    helm gpg sign -u <apache_id>@apache.org pulsar-${VERSION_WITHOUT_RC}.tgz
+    helm gpg sign -u $APACHE_USER@apache.org pulsar-${VERSION_WITHOUT_RC}.tgz
     ```
 
     Warning: you need the `helm gpg` plugin to sign the chart. It can be found at: https://github.com/technosophos/helm-gpg
@@ -114,10 +123,14 @@ official Apache releases must not include the rcN suffix.
     This should also generate a provenance file (Example: `pulsar-1.0.0.tgz.prov`) as described in
     https://helm.sh/docs/topics/provenance/, which can be used to verify integrity of the Helm chart.
 
-    Verify the signed chart (with example output shown):
+    Verify the signed chart:
 
     ```shell
-    $ helm gpg verify pulsar-${VERSION_WITHOUT_RC}.tgz
+    helm gpg verify pulsar-${VERSION_WITHOUT_RC}.tgz
+    ```
+
+    Example output:
+    ```
     gpg: Signature made Thu Oct 20 16:36:24 2022 CDT
     gpg:                using RSA key BD4291E509D771B79E7BD1F5C5724B3F5588C4EB
     gpg:                issuer "mmarshall@apache.org"
@@ -135,74 +148,93 @@ official Apache releases must not include the rcN suffix.
 - Move the artifacts to ASF dev dist repo, generate convenience `index.yaml` & publish them
 
   ```shell
-  # First clone the repo
-  svn checkout https://dist.apache.org/repos/dist/dev/pulsar pulsar-dist-dev
-
   # Create new folder for the release
-  cd pulsar-dist-dev/helm-chart
-  svn mkdir ${VERSION}
+  svn mkdir --username $APACHE_USER -m "Add directory for pulsar-helm-chart $VERSION_RC release" https://dist.apache.org/repos/dist/dev/pulsar/helm-chart/$VERSION_RC
+  # checkout the directory
+  svn co --username $APACHE_USER https://dist.apache.org/repos/dist/dev/pulsar/helm-chart/$VERSION_RC helm-chart-$VERSION_RC
 
   # Move the artifacts to svn folder
-  mv ${PULSAR_REPO_ROOT}/pulsar-${VERSION_WITHOUT_RC}.tgz* ${VERSION}/
-  mv ${PULSAR_REPO_ROOT}/pulsar-chart-${VERSION_WITHOUT_RC}-source.tar.gz* ${VERSION}/
-  cd ${VERSION}
+  mv ${PULSAR_REPO_ROOT}/pulsar-${VERSION_WITHOUT_RC}.tgz* helm-chart-${VERSION_RC}/
+  mv ${PULSAR_REPO_ROOT}/pulsar-chart-${VERSION_WITHOUT_RC}-source.tar.gz* helm-chart-${VERSION_RC}/
+  cd helm-chart-${VERSION_RC}/
 
   ###### Generate index.yaml file - Start
   # Download the latest index.yaml on Pulsar Website
-  curl https://dist.apache.org/repos/dist/release/pulsar/helm-chart/index.yaml --output index.yaml
+  curl https://pulsar.apache.org/charts/index.yaml --output index.yaml
 
   # Replace the URLs from "https://downloads.apache.org" to "https://archive.apache.org"
   # as the downloads.apache.org only contains latest releases.
   sed -i 's|https://downloads.apache.org/pulsar/helm-chart/|https://archive.apache.org/dist/pulsar/helm-chart/|' index.yaml
 
   # Generate / Merge the new version with existing index.yaml
-  helm repo index --merge ./index.yaml . --url "https://dist.apache.org/repos/dist/dev/pulsar/helm-chart/${VERSION}"
+  helm repo index --merge ./index.yaml . --url "https://dist.apache.org/repos/dist/dev/pulsar/helm-chart/${VERSION_RC}"
 
   ###### Generate index.yaml file - End
 
   # Commit the artifacts
   svn add *
-  svn commit -m "Add artifacts for Helm Chart ${VERSION}"
+  svn commit -m "Add artifacts for Helm Chart ${VERSION_RC}"
   ```
 
 - Remove old Helm Chart versions from the dev repo 
 
+  First check if this is required by viewing the versions available at https://dist.apache.org/repos/dist/dev/pulsar/helm-chart
+
   ```shell
-  cd ..
-  export PREVIOUS_VERSION=3.0.0-candidate-1
-  svn rm ${PREVIOUS_VERSION}
-  svn commit -m "Remove old Helm Chart release: ${PREVIOUS_VERSION}"
+  export PREVIOUS_VERSION_RC=3.0.0-candidate-1
+  svn rm --username $APACHE_USER -m "Remove old Helm Chart release: ${PREVIOUS_VERSION_RC}" https://dist.apache.org/repos/dist/dev/pulsar/helm-chart/${PREVIOUS_VERSION_RC}
   ```
 
 - Push Tag for the release candidate
 
   ```shell
   cd ${PULSAR_REPO_ROOT}
-  git push upstream tag pulsar-${VERSION}
+  git push origin tag pulsar-${VERSION_RC}
   ```
 
+## Create release notes for the release candidate in GitHub UI
+
+```shell 
+# open this URL and create release notes by clicking "Create release from tag"
+echo https://github.com/apache/pulsar-helm-chart/releases/tag/pulsar-${VERSION_RC}
+```
+
+1. Open the above URL in a browser and create release notes by clicking "Create release from tag".
+2. Find "Previous tag: auto" in the UI above the text box and choose the previous release there.
+3. Click "Generate release notes".
+4. Review the generated release notes.
+5. Select "Set as a pre-release"
+6. Click "Publish release".  
+
 ## Prepare Vote email on the Apache Pulsar release candidate
 
+
 - Send out a vote to the dev@pulsar.apache.org mailing list:
 
+> [!TIP]
+> The template output will get copied to the clipboard using pbpaste. On Linux, you can install xsel and add `alias pbcopy='xsel --clipboard --input'` to the shell. 
+
 Subject:
 
 ```shell
-cat <<EOF
-[VOTE] Release Apache Pulsar Helm Chart ${VERSION_WITHOUT_RC} based on ${VERSION}
+tee >(pbcopy) <<EOF
+[VOTE] Release Apache Pulsar Helm Chart ${VERSION_WITHOUT_RC} based on ${VERSION_RC}
 EOF
 ```
 
 Body:
 
 ```shell
-cat <<EOF
+tee >(pbcopy) <<EOF
 Hello Apache Pulsar Community,
 
 This is a call for the vote to release the Apache Pulsar Helm Chart version ${VERSION_WITHOUT_RC}.
 
+Release notes for $VERSION_RC:
+https://github.com/apache/pulsar-helm-chart/releases/tag/pulsar-$VERSION_RC
+
 The release candidate is available at:
-https://dist.apache.org/repos/dist/dev/pulsar/helm-chart/$VERSION/
+https://dist.apache.org/repos/dist/dev/pulsar/helm-chart/$VERSION_RC/
 
 pulsar-chart-${VERSION_WITHOUT_RC}-source.tar.gz - is the "main source release".
 pulsar-${VERSION_WITHOUT_RC}.tgz - is the binary Helm Chart release.
@@ -211,14 +243,22 @@ Public keys are available at: https://www.apache.org/dist/pulsar/KEYS
 
 For convenience "index.yaml" has been uploaded (though excluded from voting), so you can also run the below commands.
 
-helm repo add apache-pulsar-dist-dev https://dist.apache.org/repos/dist/dev/pulsar/helm-chart/$VERSION/
+helm repo add --force-update apache-pulsar-dist-dev \\
+  https://dist.apache.org/repos/dist/dev/pulsar/helm-chart/$VERSION_RC/
 helm repo update
-helm install pulsar apache-pulsar-dist-dev/pulsar
+helm install pulsar apache-pulsar-dist-dev/pulsar \\
+ --version ${VERSION_WITHOUT_RC} --set affinity.anti_affinity=false \\
+ --wait --timeout 10m --debug
+
+ For observing the deployment progress, you can use the k9s tool to view the cluster state changes in a different terminal window.
+ The k9s tool is available at https://k9scli.io/topics/install/.
  
 pulsar-${VERSION_WITHOUT_RC}.tgz.prov - is also uploaded for verifying Chart Integrity, though it is not strictly required for releasing the artifact based on ASF Guidelines. 
 
 You can optionally verify this file using this helm plugin https://github.com/technosophos/helm-gpg, or by using helm --verify (https://helm.sh/docs/helm/helm_verify/).
 
+helm fetch --prov apache-pulsar-dist-dev/pulsar
+helm plugin install https://github.com/technosophos/helm-gpg
 helm gpg verify pulsar-${VERSION_WITHOUT_RC}.tgz
 
 The vote will be open for at least 72 hours.
@@ -265,7 +305,7 @@ The legal checks include:
 ## SVN check
 
 The files should be present in the sub-folder of
-[Pulsar dist](https://dist.apache.org/repos/dist/dev/pulsar/)
+[Pulsar dist](https://dist.apache.org/repos/dist/dev/pulsar/helm-chart)
 
 The following files should be present (7 files):
 
@@ -276,7 +316,7 @@ The following files should be present (7 files):
 As a PMC member you should be able to clone the SVN repository:
 
 ```shell
-svn co https://dist.apache.org/repos/dist/dev/pulsar
+svn co https://dist.apache.org/repos/dist/dev/pulsar/helm-chart
 ```
 
 Or update it if you already checked it out:
@@ -368,9 +408,16 @@ Checking pulsar-chart-1.0.0-source.tar.gz.sha512
 Contributors can run below commands to test the Helm Chart
 
 ```shell
-helm repo add apache-pulsar-dist-dev https://dist.apache.org/repos/dist/dev/pulsar/helm-chart/1.0.1-candidate-1/
+export VERSION_RC=3.0.0-candidate-1
+export VERSION_WITHOUT_RC=${VERSION_RC%-candidate-*}
+```
+
+```shell
+helm repo add --force-update \
+ apache-pulsar-dist-dev https://dist.apache.org/repos/dist/dev/pulsar/helm-chart/$VERSION_RC/
 helm repo update
-helm install pulsar apache-pulsar-dist-dev/pulsar
+helm install pulsar apache-pulsar-dist-dev/pulsar \
+ --version ${VERSION_WITHOUT_RC} --set affinity.anti_affinity=false
 ```
 
 You can then perform any other verifications to check that it works as you expected by
@@ -385,18 +432,18 @@ Once the vote has been passed, you will need to send a result vote to [dev@pulsa
 Subject:
 
 ```shell
-cat <<EOF
-[RESULT][VOTE] Release Apache Pulsar Helm Chart ${VERSION_WITHOUT_RC} based on ${VERSION}
+tee >(pbcopy) <<EOF
+[RESULT][VOTE] Release Apache Pulsar Helm Chart ${VERSION_WITHOUT_RC} based on ${VERSION_RC}
 EOF
 ```
 
 Message:
 
 ```shell
-cat <<EOF
+tee >(pbcopy) <<EOF
 Hello all,
 
-The vote to release Apache Pulsar Helm Chart version ${VERSION_WITHOUT_RC} based on ${VERSION} is now closed.
+The vote to release Apache Pulsar Helm Chart version ${VERSION_WITHOUT_RC} based on ${VERSION_RC} is now closed.
 
 The vote PASSED with X binding "+1", Y non-binding "+1" and 0 "-1" votes:
 
@@ -417,30 +464,23 @@ EOF
 
 ## Publish release to SVN
 
-You need to migrate the RC artifacts that passed to this repository:
-https://dist.apache.org/repos/dist/release/pulsar/helm-chart/
-(The migration should include renaming the files so that they no longer have the RC number in their filenames.)
+Set environment variables
+```shell
+export VERSION_RC=3.0.0-candidate-1
+export VERSION_WITHOUT_RC=${VERSION_RC%-candidate-*}
+export APACHE_USER=<your ASF userid>
+```
 
-The best way of doing this is to svn cp between the two repos (this avoids having to upload
-the binaries again, and gives a clearer history in the svn commit logs):
+Migrating the approved RC artifacts to the release directory:
+https://dist.apache.org/repos/dist/release/pulsar/helm-chart/
+
+svn commands for handling this:
 
 ```shell
-# First clone the repo
-export RC=3.0.0-candidate-1
-export VERSION=${RC%-candidate-*}
-svn checkout https://dist.apache.org/repos/dist/release/pulsar pulsar-dist-release
-
-# Create new folder for the release
-cd pulsar-dist-release/helm-chart
-export PULSAR_SVN_RELEASE_HELM=$(pwd)
-svn mkdir ${VERSION}
-cd ${VERSION}
-
-# Move the artifacts to svn folder, remove index.yaml, and commit
-for f in ../../../pulsar-dist-dev/helm-chart/$RC/*; do cp $f $(basename $f); done
-rm index.yaml
-svn add pulsar-*
-svn commit -m "Release Pulsar Helm Chart ${VERSION} from ${RC}"
+svn rm --username $APACHE_USER -m "Remove temporary index.yaml file" https://dist.apache.org/repos/dist/dev/pulsar/helm-chart/${VERSION_RC}/index.yaml
+svn move --username $APACHE_USER -m "Release Pulsar Helm Chart ${VERSION_WITHOUT_RC} from ${VERSION_RC}" \
+  https://dist.apache.org/repos/dist/dev/pulsar/helm-chart/${VERSION_RC} \
+  https://dist.apache.org/repos/dist/release/pulsar/helm-chart/${VERSION_WITHOUT_RC}
 ```
 
 Verify that the packages appear in [Pulsar Helm Chart](https://dist.apache.org/repos/dist/release/pulsar/helm-chart/).
@@ -450,10 +490,8 @@ Verify that the packages appear in [Pulsar Helm Chart](https://dist.apache.org/r
 Create and push the release tag:
 
 ```shell
-cd "${PULSAR_REPO_ROOT}"
-git checkout pulsar-${RC}
-git tag -s pulsar-${VERSION} -m "Apache Pulsar Helm Chart ${VERSION}"
-git push upstream pulsar-${VERSION}
+git tag -u $APACHE_USER@apache.org pulsar-$VERSION_WITHOUT_RC $(git rev-parse pulsar-$VERSION_RC^{}) -m "Apache Pulsar Helm Chart ${VERSION_WITHOUT_RC}"
+git push origin pulsar-${VERSION_WITHOUT_RC}
 ```
 
 ## Update index.yaml
@@ -464,20 +502,65 @@ file at `pulsar.apache.org/charts/index.yaml`.
 Then, run the following command from within `github.com/apache/pulsar-site` in the git repo.
 
 ```shell
-# Run on a branch based on main branch
-cd site2/website-next/static/charts
-cp ${PULSAR_SVN_RELEASE_HELM}/${VERSION}/pulsar-${VERSION}.tgz .
-helm repo index --merge ./index.yaml . --url "https://downloads.apache.org/pulsar/helm-chart/${VERSION}"
+# checkout pulsar-site
+git clone https://github.com/apache/pulsar-site
+cd pulsar-site
 ```
 
-Verify that the updated `index.yaml` file has the most recent version. Then run:
+```shell
+# Run on a branch based on main branch
+cd static/charts
+# need the chart file temporarily to update the index
+wget https://dist.apache.org/repos/dist/release/pulsar/helm-chart/${VERSION_WITHOUT_RC}/pulsar-${VERSION_WITHOUT_RC}.tgz
+# store the license header temporarily
+head -n 17 index.yaml > license_header.txt
+# update the index
+helm repo index --merge ./index.yaml . --url "https://downloads.apache.org/pulsar/helm-chart/${VERSION_WITHOUT_RC}"
+# restore the license header
+mv index.yaml index.yaml.new
+cat license_header.txt index.yaml.new > index.yaml
+rm license_header.txt index.yaml.new
+# remove the temp file
+rm pulsar-${VERSION_WITHOUT_RC}.tgz
+```
+
+Verify that the updated `index.yaml` file has the most recent version. 
+
+Wait until the file is available:
+
+```shell
+while ! curl -fIL https://downloads.apache.org/pulsar/helm-chart/${VERSION_WITHOUT_RC}/pulsar-${VERSION_WITHOUT_RC}.tgz; do
+  echo "Waiting for pulsar-${VERSION_WITHOUT_RC}.tgz to become available..."
+  sleep 10
+done
+```
+
+Then run:
 
 ```shell
 git add index.yaml
-git commit -m "Adding Pulsar Helm Chart ${VERSION} to index.yaml"
+git commit -m "Adding Pulsar Helm Chart ${VERSION_WITHOUT_RC} to index.yaml"
 ```
 
-Then open a PR.
+Then commit the change.
+```
+git push origin main
+```
+
+
+## Create release notes for the tag in GitHub UI
+
+```shell 
+# open this URL and create release notes by clicking "Create release from tag"
+echo https://github.com/apache/pulsar-helm-chart/releases/tag/pulsar-${VERSION_WITHOUT_RC}
+```
+
+1. Open the above URL in a browser and create release notes by clicking "Create release from tag".
+2. Find "Previous tag: auto" in the UI above the text box and choose the previous release there.
+3. Click "Generate release notes".
+4. Review the generated release notes.
+5. Click "Publish release".
+
 
 ## Notify developers of release
 
@@ -489,26 +572,31 @@ the artifacts have been published:
 Subject:
 
 ```shell
-cat <<EOF
-[ANNOUNCE] Apache Pulsar Helm Chart version ${VERSION} Released
+tee >(pbcopy) <<EOF
+[ANNOUNCE] Apache Pulsar Helm Chart version ${VERSION_WITHOUT_RC} Released
 EOF
 ```
 
 Body:
 
 ```shell
-cat <<EOF
-Dear Pulsar community,
+tee >(pbcopy) <<EOF
+Dear community,
 
-The Apache Pulsar team is pleased to announce the release of Apache Pulsar Helm
-Chart $VERSION.
+The Apache Pulsar team is pleased to announce the release of the Apache
+Pulsar Helm Chart $VERSION_WITHOUT_RC.
 
-The source release, as well as the "binary" Helm Chart release, are available:
+The official source release, as well as the binary Helm Chart release,
+are available at
+https://downloads.apache.org/pulsar/helm-chart/$VERSION_WITHOUT_RC/.
 
-Official Sources: https://pulsar.apache.org/download/
-ArtifactHub: https://artifacthub.io/packages/helm/apache/pulsar/$VERSION
-Docs: https://pulsar.apache.org/docs/helm-overview
-Release Notes: https://github.com/apache/pulsar-helm-chart/releases/tag/pulsar-$VERSION
+The helm chart index at https://pulsar.apache.org/charts/ has been
+updated and the release is also available directly via helm.
+
+Release Notes:
+https://github.com/apache/pulsar-helm-chart/releases/tag/pulsar-$VERSION_WITHOUT_RC
+Docs: https://github.com/apache/pulsar-helm-chart#readme and https://pulsar.apache.org/docs/helm-overview
+ArtifactHub: https://artifacthub.io/packages/helm/apache/pulsar/$VERSION_WITHOUT_RC
 
 Thanks to all the contributors who made this possible.
 
@@ -518,9 +606,10 @@ The Apache Pulsar Team
 EOF
 ```
 
-Send the same email to announce@apache.org, except change the opening line to `Dear community,`.
+
+Send the same email to announce@apache.org.
 It is more reliable to send it via the web ui at https://lists.apache.org/list.html?announce@apache.org
-(press "c" to compose a new thread)
+(press "c" to compose a new thread).
 
 ## Create release on GitHub
 
@@ -535,30 +624,29 @@ Close the milestone on GitHub. Create the next one if it hasn't been already.
 Post this in the #announce channel:
 
 ```shell
-cat <<EOF
-We've just released Apache Pulsar Helm Chart ${VERSION} 🎉
+tee >(pbcopy) <<EOF
+We've just released Apache Pulsar Helm Chart ${VERSION_WITHOUT_RC} 🎉
 
-Official Sources: https://pulsar.apache.org/download/
-ArtifactHub: https://artifacthub.io/packages/helm/apache/pulsar/$VERSION
-Docs: https://pulsar.apache.org/docs/helm-overview
-Release Notes: https://pulsar.apache.org/docs/helm-chart/$VERSION/release_notes.html
+The official source release, as well as the binary Helm Chart release,
+are available at
+https://downloads.apache.org/pulsar/helm-chart/$VERSION_WITHOUT_RC/.
+
+The helm chart index at https://pulsar.apache.org/charts/ has been
+updated and the release is also available directly via helm.
+
+Release Notes:
+https://github.com/apache/pulsar-helm-chart/releases/tag/pulsar-$VERSION_WITHOUT_RC
+Docs: https://github.com/apache/pulsar-helm-chart#readme and https://pulsar.apache.org/docs/helm-overview
+ArtifactHub: https://artifacthub.io/packages/helm/apache/pulsar/$VERSION_WITHOUT_RC
 
 Thanks to all the contributors who made this possible.
 EOF
 ```
 
-## Remove old releases
+## Maintaining svn https://dist.apache.org/repos/dist/release/pulsar/helm-chart/ content
 
-We should keep the old version a little longer than a day. We updated the `index.yaml` earlier so that it points to the
-older releases. The short delay will help avoid errors for users who haven't run ``helm repo update`` to get the latest
-`index.yaml`.
+The chart references the files in https://downloads.apache.org/pulsar/helm-chart/ which are maintained
+by SVN directory https://dist.apache.org/repos/dist/release/pulsar/helm-chart/.
 
-It is probably ok if we leave last 2 versions on release svn repo too.
-
-```shell
-# https://www.apache.org/legal/release-policy.html#when-to-archive
-cd pulsar-dist-release/helm-chart
-export PREVIOUS_VERSION=1.0.0
-svn rm ${PREVIOUS_VERSION}
-svn commit -m "Remove old Helm Chart release: ${PREVIOUS_VERSION}"
-```
+If you remove releases from this directory, the URLs in index.yaml should be updated point to the 
+https://archive.apache.org/dist/pulsar/helm-chart/ URL base instead of https://downloads.apache.org/pulsar/helm-chart/.

charts/pulsar/Chart.yaml

@@ -18,10 +18,11 @@
 #
 
 apiVersion: v2
-appVersion: "3.0.2"
+appVersion: "4.0.5"
 description: Apache Pulsar Helm chart for Kubernetes
 name: pulsar
-version: 3.1.0
+version: 4.1.0
+kubeVersion: ">=1.25.0-0"
 home: https://pulsar.apache.org
 sources:
   - https://github.com/apache/pulsar
@@ -31,7 +32,7 @@ maintainers:
   - name: The Apache Pulsar Team
     email: dev@pulsar.apache.org
 dependencies:
-  - name: kube-prometheus-stack
-    version: 41.x.x
-    repository: https://prometheus-community.github.io/helm-charts
-    condition: kube-prometheus-stack.enabled
+  - name: victoria-metrics-k8s-stack
+    version: 0.38.x
+    repository: https://victoriametrics.github.io/helm-charts/
+    condition: victoria-metrics-k8s-stack.enabled

charts/pulsar/templates/NOTES.txt

@@ -0,0 +1,185 @@
+======================================================================================
+                           APACHE PULSAR HELM CHART
+======================================================================================
+
+======================================================================================
+                           SECURITY ADVISORY
+======================================================================================
+
+This Helm chart's default configuration DOES NOT meet production security requirements.
+Users MUST review and customize security settings for their specific environment.
+
+IMPORTANT: This Helm chart provides a starting point for Pulsar deployments but requires
+significant security customization before use in production environments. We strongly
+recommend implementing:
+
+1. Proper network isolation and access controls
+2. Authentication and authorization for all components
+3. TLS encryption for all communication channels
+4. Regular security updates and vulnerability assessments
+
+As an open source project, we welcome contributions to improve security features.
+Please consider submitting pull requests to address security gaps or enhance
+existing security implementations.
+
+---------------------------------------------------------------------------------------
+
+SECURITY NOTICE: The Pulsar proxy is not designed for direct public internet exposure.
+It lacks security features required for untrusted networks and should only be deployed
+within secured environments with proper network controls.
+
+IMPORTANT CHANGE IN v4.0.0: Default service type changed from LoadBalancer to ClusterIP
+for security reasons. This limits access to within the Kubernetes environment by default.
+
+---------------------------------------------------------------------------------------
+IF YOU NEED EXTERNAL ACCESS FOR YOUR PULSAR CLUSTER:
+---------------------------------------------------------------------------------------
+
+Note: This information might be outdated. Please go to https://github.com/apache/pulsar-helm-chart for updated information.
+
+If you need to expose the Pulsar Proxy outside the cluster using a LoadBalancer service type:
+
+1. USE INTERNAL LOAD BALANCERS ONLY
+   - Set type to LoadBalancer only in secured environments with proper network controls
+   - Add cloud provider-specific annotations for internal load balancers
+   - See cloud provider documentation:
+     * AWS / EKS: https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/guide/service/annotations/
+     * Azure / AKS: https://learn.microsoft.com/en-us/azure/aks/internal-lb
+     * GCP / GKE: https://cloud.google.com/kubernetes-engine/docs/concepts/service-load-balancer-parameters
+   - Examples (verify correctness for your environment):
+     * AWS / EKS:  service.beta.kubernetes.io/aws-load-balancer-internal: "true"
+     * Azure / AKS: service.beta.kubernetes.io/azure-load-balancer-internal: "true"
+     * GCP / GKE:   networking.gke.io/load-balancer-type: "Internal"
+
+2. IMPLEMENT AUTHENTICATION AND AUTHORIZATION
+   - Configure all clients to authenticate properly
+   - Set up appropriate authorization policies
+
+3. USE TLS FOR ALL CONNECTIONS
+   - Enable TLS for client-to-proxy connections
+   - Enable TLS for proxy-to-broker connections
+   - Enable TLS for all internal cluster communications (brokers, zookeepers, bookies)
+   - Note: TLS alone is NOT sufficient as a security solution in Pulsar. Even with TLS enabled,
+     clusters exposed to untrusted networks remain vulnerable to denial-of-service attacks, 
+     authentication bypass attempts, and protocol-level exploits. Always implement defense-in-depth
+     security measures and limit exposure to trusted networks only.
+
+4. NETWORK SECURITY
+   - Use private networks (VPCs)
+   - Configure firewalls, security groups, and IP restrictions appropriately
+   - In addition, consider using loadBalancerSourceRanges to limit access to specific IP ranges
+
+5. CLIENT IP ADDRESS BASED ACCESS RESTRICTIONS
+   - When using a LoadBalancer service type, restrict access to specific IP ranges by configuring 
+     `proxy.service.loadBalancerSourceRanges` in your values.yaml
+   - Important: This should be implemented alongside other security measures (internal load balancer,
+     authentication, TLS, network policies) as part of a defense-in-depth strategy,
+     not as a standalone security solution
+
+---------------------------------------------------------------------------------------
+ALTERNATIVE FOR EXTERNAL ACCESS
+---------------------------------------------------------------------------------------
+
+As an alternative method for external access, Pulsar has support for SNI proxy routing:
+https://pulsar.apache.org/docs/next/concepts-proxy-sni-routing/
+SNI Proxy routing is supported with proxy servers such as Apache Traffic Server, HAProxy and Nginx.
+
+Note: This option isn't currently implemented in the Apache Pulsar Helm chart.
+
+IMPORTANT: Pulsar binary protocol cannot be exposed outside of the Kubernetes cluster
+using Kubernetes Ingress. Kubernetes Ingress works for the Admin REST API and topic lookups,
+but clients would be connecting to the advertised listener addresses returned by the brokers and it 
+would only work when clients can connect directly to brokers. This is not a supported secure option 
+for exposing Pulsar to untrusted networks.
+
+{{- if .Values.useReleaseStatus }}
+
+======================================================================================
+                           🚀 QUICK START 🚀
+======================================================================================
+
+Watching events to view progress of deployment:
+kubectl get -n {{ .Values.namespace | default .Release.Namespace }} events -o wide --watch
+
+Watching state of deployed Kubernetes objects, updated every 2 seconds:
+watch kubectl get -n {{ .Values.namespace | default .Release.Namespace }} all
+
+{{- if .Values.components.proxy }}
+
+Waiting until Pulsar Proxy is available:
+kubectl wait --timeout=600s --for=condition=ready pod -n {{ .Values.namespace | default .Release.Namespace }} -l component=proxy
+{{- end }}
+
+Watching state with k9s (https://k9scli.io/topics/install/):
+k9s -n {{ .Values.namespace | default .Release.Namespace }}
+
+{{- if and .Values.affinity.anti_affinity (or (gt (int .Values.bookkeeper.replicaCount) 1) (gt (int .Values.zookeeper.replicaCount) 1)) }}
+
+======================================================================================
+                      ⚠️  NOTICE FOR DEV K8S CLUSTER USERS  ⚠️
+======================================================================================
+
+Please note that anti-affinity rules for Zookeeper and Bookie components require at least 
+one node per replica. There are currently {{ .Values.bookkeeper.replicaCount }} bookies and {{ .Values.zookeeper.replicaCount }} zookeepers configured.
+
+For Kubernetes clusters with fewer than 3 nodes, such as single-node Kubernetes clusters in 
+development environments like minikube, Docker Desktop, Rancher Desktop (k3s), or Podman 
+Desktop, you must disable the anti-affinity feature by either:
+
+Adding to your values.yaml:
+affinity:
+  anti_affinity: false
+
+Or adding "--set affinity.anti_affinity=false" to the helm command line.
+
+After making the changes to your values yaml file, redeploy with "helm upgrade":
+helm upgrade -n {{ .Release.Namespace }} -f your_values_file.yaml {{ .Release.Name }} apachepulsar/pulsar
+
+These configuration instructions can be omitted for Kubernetes clusters with 3 or more nodes.
+{{- end }}
+{{- end }}
+{{- if and (eq .Values.proxy.service.type "LoadBalancer") (not .Values.proxy.service.annotations) }}
+
+======================================================================================
+                      ⚠️ 🚨 INSECURE CONFIGURATION DETECTED 🚨 ⚠️
+======================================================================================
+WARNING: You are using a LoadBalancer service type without internal load balancer
+annotations. This is potentially an insecure configuration. Please carefully review
+the security recommendations above and visit https://github.com/apache/pulsar-helm-chart
+for more information.
+======================================================================================
+{{- end }}
+
+======================================================================================
+                           DISCLAIMER
+======================================================================================
+
+The providers of this Helm chart make no guarantees regarding the security of the chart under
+any circumstances. It is the user's responsibility to ensure that their deployment is secure
+and complies with all relevant security standards and regulations.
+
+By using this Helm chart, the user acknowledges the risks associated with its default
+configuration and the necessity for proper security customization. The user further 
+agrees that the providers of the Helm chart shall not be liable for any security breaches
+or incidents resulting from the use of the chart.
+
+The user assumes full responsibility for the security and integrity of their deployment.
+This includes, but is not limited to, the proper configuration of security features and 
+adherence to best practices for securing network access. The providers of this Helm chart
+disclaim all warranties, whether express or implied, including any warranties of
+merchantability, fitness for a particular purpose, and non-infringement of third-party rights.
+
+======================================================================================
+                           RESOURCES
+======================================================================================
+
+- 🖥️ Install k9s terminal interface for viewing and managing k8s clusters: https://k9scli.io/topics/install/
+- ❓ Usage Questions: https://github.com/apache/pulsar/discussions/categories/q-a
+- 🐛 Report Issues: https://github.com/apache/pulsar-helm-chart/issues
+- 🔒 Security Issues: https://pulsar.apache.org/security/
+- 📚 Documentation: https://github.com/apache/pulsar-helm-chart
+
+🌟 Please contribute to improve the Apache Pulsar Helm chart and its documentation:
+- 🤝 Contribute: https://github.com/apache/pulsar-helm-chart
+
+Thank you for installing Apache Pulsar Helm chart version {{ .Chart.Version }}.

charts/pulsar/templates/_autorecovery.tpl

@@ -36,7 +36,7 @@ Define autorecovery zookeeper client tls settings
 */}}
 {{- define "pulsar.autorecovery.zookeeper.tls.settings" -}}
 {{- if and .Values.tls.enabled .Values.tls.zookeeper.enabled }}
-/pulsar/keytool/keytool.sh autorecovery {{ template "pulsar.autorecovery.hostname" . }} true;
+{{- include "pulsar.component.zookeeper.tls.settings" (dict "component" "autorecovery" "isClient" true "isCacerts" .Values.tls.autorecovery.cacerts.enabled) -}}
 {{- end }}
 {{- end }}
 
@@ -51,11 +51,21 @@ Define autorecovery tls certs mounts
 - name: ca
   mountPath: "/pulsar/certs/ca"
   readOnly: true
-{{- if .Values.tls.zookeeper.enabled }}
-- name: keytool
-  mountPath: "/pulsar/keytool/keytool.sh"
-  subPath: keytool.sh
 {{- end }}
+{{- if .Values.tls.autorecovery.cacerts.enabled }}
+- mountPath: "/pulsar/certs/cacerts"
+  name: autorecovery-cacerts
+{{- range $cert := .Values.tls.autorecovery.cacerts.certs }}
+- name: {{ $cert.name }}
+  mountPath: "/pulsar/certs/{{ $cert.name }}"
+  readOnly: true
+{{- end }}
+- name: certs-scripts
+  mountPath: "/pulsar/bin/certs-combine-pem.sh"
+  subPath: certs-combine-pem.sh
+- name: certs-scripts
+  mountPath: "/pulsar/bin/certs-combine-pem-infinity.sh"
+  subPath: certs-combine-pem-infinity.sh
 {{- end }}
 {{- end }}
 
@@ -72,18 +82,32 @@ Define autorecovery tls certs volumes
       path: tls.crt
     - key: tls.key
       path: tls.key
+    - key: tls-combined.pem
+      path: tls-combined.pem
 - name: ca
   secret:
-    secretName: "{{ .Release.Name }}-{{ .Values.tls.ca_suffix }}"
+    secretName: "{{ template "pulsar.certs.issuers.ca.secretName" . }}"
     items:
     - key: ca.crt
       path: ca.crt
-{{- if .Values.tls.zookeeper.enabled }}
-- name: keytool
-  configMap:
-    name: "{{ template "pulsar.fullname" . }}-keytool-configmap"
-    defaultMode: 0755
 {{- end }}
+{{- if .Values.tls.autorecovery.cacerts.enabled }}
+- name: autorecovery-cacerts
+  emptyDir: {}
+{{- range $cert := .Values.tls.autorecovery.cacerts.certs }}
+- name: {{ $cert.name }}
+  secret:
+    secretName: "{{ $cert.existingSecret }}"
+    items:
+    {{- range $key := $cert.secretKeys }}
+      - key: {{ $key }}
+        path: {{ $key }}
+    {{- end }}
+{{- end }}
+- name: certs-scripts
+  configMap:
+    name: "{{ template "pulsar.fullname" . }}-certs-scripts"
+    defaultMode: 0755
 {{- end }}
 {{- end }}
 
@@ -92,8 +116,9 @@ Define autorecovery init container : verify cluster id
 */}}
 {{- define "pulsar.autorecovery.init.verify_cluster_id" -}}
 bin/apply-config-from-env.py conf/bookkeeper.conf;
-{{- include "pulsar.autorecovery.zookeeper.tls.settings" . -}}
-until bin/bookkeeper shell whatisinstanceid; do
+export BOOKIE_MEM="-Xmx128M";
+{{- include "pulsar.autorecovery.zookeeper.tls.settings" . }}
+until timeout 15 bin/bookkeeper shell whatisinstanceid; do
   sleep 3;
 done;
 {{- end }}

charts/pulsar/templates/_bookkeeper.tpl

@@ -37,7 +37,7 @@ Define bookie zookeeper client tls settings
 */}}
 {{- define "pulsar.bookkeeper.zookeeper.tls.settings" -}}
 {{- if and .Values.tls.enabled .Values.tls.zookeeper.enabled }}
-/pulsar/keytool/keytool.sh bookie {{ template "pulsar.bookkeeper.hostname" . }} true;
+{{- include "pulsar.component.zookeeper.tls.settings" (dict "component" "bookie" "isClient" true "isCacerts" .Values.tls.bookie.cacerts.enabled) -}}
 {{- end }}
 {{- end }}
 
@@ -45,18 +45,30 @@ Define bookie zookeeper client tls settings
 Define bookie tls certs mounts
 */}}
 {{- define "pulsar.bookkeeper.certs.volumeMounts" -}}
-{{- if and .Values.tls.enabled (or .Values.tls.bookie.enabled .Values.tls.zookeeper.enabled) }}
+{{- if .Values.tls.enabled }}
+{{- if or .Values.tls.bookie.enabled .Values.tls.zookeeper.enabled }}
 - name: bookie-certs
   mountPath: "/pulsar/certs/bookie"
   readOnly: true
+{{- end }}
 - name: ca
   mountPath: "/pulsar/certs/ca"
   readOnly: true
-{{- if .Values.tls.zookeeper.enabled }}
-- name: keytool
-  mountPath: "/pulsar/keytool/keytool.sh"
-  subPath: keytool.sh
 {{- end }}
+{{- if .Values.tls.bookie.cacerts.enabled }}
+- mountPath: "/pulsar/certs/cacerts"
+  name: bookie-cacerts
+{{- range $cert := .Values.tls.bookie.cacerts.certs }}
+- name: {{ $cert.name }}
+  mountPath: "/pulsar/certs/{{ $cert.name }}"
+  readOnly: true
+{{- end }}
+- name: certs-scripts
+  mountPath: "/pulsar/bin/certs-combine-pem.sh"
+  subPath: certs-combine-pem.sh
+- name: certs-scripts
+  mountPath: "/pulsar/bin/certs-combine-pem-infinity.sh"
+  subPath: certs-combine-pem-infinity.sh
 {{- end }}
 {{- end }}
 
@@ -64,7 +76,8 @@ Define bookie tls certs mounts
 Define bookie tls certs volumes
 */}}
 {{- define "pulsar.bookkeeper.certs.volumes" -}}
-{{- if and .Values.tls.enabled (or .Values.tls.bookie.enabled .Values.tls.zookeeper.enabled) }}
+{{- if .Values.tls.enabled }}
+{{- if or .Values.tls.bookie.enabled .Values.tls.zookeeper.enabled }}
 - name: bookie-certs
   secret:
     secretName: "{{ .Release.Name }}-{{ .Values.tls.bookie.cert_name }}"
@@ -73,18 +86,35 @@ Define bookie tls certs volumes
       path: tls.crt
     - key: tls.key
       path: tls.key
+{{- if .Values.tls.zookeeper.enabled }}
+    - key: tls-combined.pem
+      path: tls-combined.pem
+{{- end }}
+{{- end }}
 - name: ca
   secret:
-    secretName: "{{ .Release.Name }}-{{ .Values.tls.ca_suffix }}"
+    secretName: "{{ template "pulsar.certs.issuers.ca.secretName" . }}"
     items:
     - key: ca.crt
       path: ca.crt
-{{- if .Values.tls.zookeeper.enabled }}
-- name: keytool
-  configMap:
-    name: "{{ template "pulsar.fullname" . }}-keytool-configmap"
-    defaultMode: 0755
 {{- end }}
+{{- if .Values.tls.bookie.cacerts.enabled }}
+- name: bookie-cacerts
+  emptyDir: {}
+{{- range $cert := .Values.tls.bookie.cacerts.certs }}
+- name: {{ $cert.name }}
+  secret:
+    secretName: "{{ $cert.existingSecret }}"
+    items:
+    {{- range $key := $cert.secretKeys }}
+      - key: {{ $key }}
+        path: {{ $key }}
+    {{- end }}
+{{- end }}
+- name: certs-scripts
+  configMap:
+    name: "{{ template "pulsar.fullname" . }}-certs-scripts"
+    defaultMode: 0755
 {{- end }}
 {{- end }}
 
@@ -92,8 +122,31 @@ Define bookie tls certs volumes
 Define bookie common config
 */}}
 {{- define "pulsar.bookkeeper.config.common" -}}
-zkServers: "{{ template "pulsar.zookeeper.connect" . }}"
-zkLedgersRootPath: "{{ .Values.metadataPrefix }}/ledgers"
+{{/*
+Configure BookKeeper's metadata store (available since BookKeeper 4.7.0 / BP-29)
+https://bookkeeper.apache.org/bps/BP-29-metadata-store-api-module/
+https://bookkeeper.apache.org/docs/deployment/manual#cluster-metadata-setup
+*/}}
+# Set empty values for zkServers and zkLedgersRootPath since we're using the metadataServiceUri to configure BookKeeper's metadata store
+zkServers: ""
+zkLedgersRootPath: ""
+{{- if .Values.components.zookeeper }}
+{{- if (and (hasKey .Values.pulsar_metadata "bookkeeper") .Values.pulsar_metadata.bookkeeper.usePulsarMetadataBookieDriver) }}
+# there's a bug when using PulsarMetadataBookieDriver since it always appends /ledgers to the metadataServiceUri
+# Possibly a bug in org.apache.pulsar.metadata.bookkeeper.AbstractMetadataDriver#resolveLedgersRootPath in Pulsar code base
+metadataServiceUri: "metadata-store:zk:{{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }}"
+{{- else }}
+# use zk+hierarchical:// when using BookKeeper's built-in metadata driver
+metadataServiceUri: "zk+hierarchical://{{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }}/ledgers"
+{{- end }}
+{{- else if .Values.components.oxia }}
+metadataServiceUri: "{{ template "pulsar.oxia.metadata.url.bookkeeper" . }}"
+{{- end }}
+{{- /* metadataStoreSessionTimeoutMillis maps to zkTimeout in bookkeeper.conf for both zookeeper and oxia metadata stores */}}
+{{- if (and (hasKey .Values.pulsar_metadata "bookkeeper") (hasKey .Values.pulsar_metadata.bookkeeper "metadataStoreSessionTimeoutMillis")) }}
+zkTimeout: "{{ .Values.pulsar_metadata.bookkeeper.metadataStoreSessionTimeoutMillis }}"
+{{- end }}
+
 # enable bookkeeper http server
 httpServerEnabled: "true"
 httpServerPort: "{{ .Values.bookkeeper.ports.http }}"
@@ -113,7 +166,7 @@ PULSAR_PREFIX_tlsCertificatePath: /pulsar/certs/bookie/tls.crt
 PULSAR_PREFIX_tlsKeyStoreType: PEM
 PULSAR_PREFIX_tlsKeyStore: /pulsar/certs/bookie/tls.key
 PULSAR_PREFIX_tlsTrustStoreType: PEM
-PULSAR_PREFIX_tlsTrustStore: /pulsar/certs/ca/ca.crt 
+PULSAR_PREFIX_tlsTrustStore: {{ ternary "/pulsar/certs/cacerts/ca-combined.pem" "/pulsar/certs/ca/ca.crt" .Values.tls.bookie.cacerts.enabled | quote }}
 {{- end }}
 {{- end }}
 
@@ -123,8 +176,9 @@ Define bookie init container : verify cluster id
 {{- define "pulsar.bookkeeper.init.verify_cluster_id" -}}
 {{- if not (and .Values.volumes.persistence .Values.bookkeeper.volumes.persistence) }}
 bin/apply-config-from-env.py conf/bookkeeper.conf;
-{{- include "pulsar.bookkeeper.zookeeper.tls.settings" . -}}
-until bin/bookkeeper shell whatisinstanceid; do
+export BOOKIE_MEM="-Xmx128M";
+{{- include "pulsar.bookkeeper.zookeeper.tls.settings" . }}
+until timeout 15 bin/bookkeeper shell whatisinstanceid; do
   sleep 3;
 done;
 bin/bookkeeper shell bookieformat -nonInteractive -force -deleteCookie || true
@@ -132,8 +186,9 @@ bin/bookkeeper shell bookieformat -nonInteractive -force -deleteCookie || true
 {{- if and .Values.volumes.persistence .Values.bookkeeper.volumes.persistence }}
 set -e;
 bin/apply-config-from-env.py conf/bookkeeper.conf;
-{{- include "pulsar.bookkeeper.zookeeper.tls.settings" . -}}
-until bin/bookkeeper shell whatisinstanceid; do
+export BOOKIE_MEM="-Xmx128M";
+{{- include "pulsar.bookkeeper.zookeeper.tls.settings" . }}
+until timeout 15 bin/bookkeeper shell whatisinstanceid; do
   sleep 3;
 done;
 {{- end }}

charts/pulsar/templates/_broker.tpl

@@ -43,7 +43,7 @@ Define broker zookeeper client tls settings
 */}}
 {{- define "pulsar.broker.zookeeper.tls.settings" -}}
 {{- if and .Values.tls.enabled .Values.tls.zookeeper.enabled }}
-/pulsar/keytool/keytool.sh broker {{ template "pulsar.broker.hostname" . }} true;
+{{- include "pulsar.component.zookeeper.tls.settings" (dict "component" "broker" "isClient" true "isCacerts" .Values.tls.broker.cacerts.enabled) -}}
 {{- end }}
 {{- end }}
 
@@ -51,18 +51,30 @@ Define broker zookeeper client tls settings
 Define broker tls certs mounts
 */}}
 {{- define "pulsar.broker.certs.volumeMounts" -}}
-{{- if and .Values.tls.enabled (or .Values.tls.broker.enabled (or .Values.tls.bookie.enabled .Values.tls.zookeeper.enabled)) }}
+{{- if .Values.tls.enabled }}
+{{- if or .Values.tls.broker.enabled (or .Values.tls.bookie.enabled .Values.tls.zookeeper.enabled) }}
 - name: broker-certs
   mountPath: "/pulsar/certs/broker"
   readOnly: true
+{{- end }}
 - name: ca
   mountPath: "/pulsar/certs/ca"
   readOnly: true
-{{- if .Values.tls.zookeeper.enabled }}
-- name: keytool
-  mountPath: "/pulsar/keytool/keytool.sh"
-  subPath: keytool.sh
 {{- end }}
+{{- if .Values.tls.broker.cacerts.enabled }}
+- mountPath: "/pulsar/certs/cacerts"
+  name: broker-cacerts
+{{- range $cert := .Values.tls.broker.cacerts.certs }}
+- name: {{ $cert.name }}
+  mountPath: "/pulsar/certs/{{ $cert.name }}"
+  readOnly: true
+{{- end }}
+- name: certs-scripts
+  mountPath: "/pulsar/bin/certs-combine-pem.sh"
+  subPath: certs-combine-pem.sh
+- name: certs-scripts
+  mountPath: "/pulsar/bin/certs-combine-pem-infinity.sh"
+  subPath: certs-combine-pem-infinity.sh
 {{- end }}
 {{- end }}
 
@@ -70,7 +82,8 @@ Define broker tls certs mounts
 Define broker tls certs volumes
 */}}
 {{- define "pulsar.broker.certs.volumes" -}}
-{{- if and .Values.tls.enabled (or .Values.tls.broker.enabled (or .Values.tls.bookie.enabled .Values.tls.zookeeper.enabled)) }}
+{{- if .Values.tls.enabled }}
+{{- if or .Values.tls.broker.enabled (or .Values.tls.bookie.enabled .Values.tls.zookeeper.enabled) }}
 - name: broker-certs
   secret:
     secretName: "{{ .Release.Name }}-{{ .Values.tls.broker.cert_name }}"
@@ -79,17 +92,34 @@ Define broker tls certs volumes
       path: tls.crt
     - key: tls.key
       path: tls.key
+{{- if .Values.tls.zookeeper.enabled }}
+    - key: tls-combined.pem
+      path: tls-combined.pem
+{{- end }}
+{{- end }}
 - name: ca
   secret:
-    secretName: "{{ .Release.Name }}-{{ .Values.tls.ca_suffix }}"
+    secretName: "{{ template "pulsar.certs.issuers.ca.secretName" . }}"
     items:
     - key: ca.crt
       path: ca.crt
-{{- if .Values.tls.zookeeper.enabled }}
-- name: keytool
+{{- end }}
+{{- if .Values.tls.broker.cacerts.enabled }}
+- name: broker-cacerts
+  emptyDir: {}
+{{- range $cert := .Values.tls.broker.cacerts.certs }}
+- name: {{ $cert.name }}
+  secret:
+    secretName: "{{ $cert.existingSecret }}"
+    items:
+    {{- range $key := $cert.secretKeys }}
+      - key: {{ $key }}
+        path: {{ $key }}
+    {{- end }}
+{{- end }}
+- name: certs-scripts
   configMap:
-    name: "{{ template "pulsar.fullname" . }}-keytool-configmap"
+    name: "{{ template "pulsar.fullname" . }}-certs-scripts"
     defaultMode: 0755
 {{- end }}
 {{- end }}
-{{- end }}

charts/pulsar/templates/_certs.tpl

@@ -0,0 +1,132 @@
+{{/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+*/}}
+
+{{/*
+Define the pulsar certs ca issuer name
+*/}}
+{{- define "pulsar.certs.issuers.ca.name" -}}
+{{- if .Values.certs.internal_issuer.enabled -}}
+{{- if and (eq .Values.certs.internal_issuer.type "selfsigning") .Values.certs.issuers.selfsigning.name -}}
+{{- .Values.certs.issuers.selfsigning.name -}}
+{{- else if and (eq .Values.certs.internal_issuer.type "ca") .Values.certs.issuers.ca.name -}}
+{{- .Values.certs.issuers.ca.name -}}
+{{- else -}}
+{{- template "pulsar.fullname" . }}-{{ .Values.certs.internal_issuer.component }}-ca-issuer
+{{- end -}}
+{{- else -}}
+{{- if .Values.certs.issuers.ca.name -}}
+{{- .Values.certs.issuers.ca.name -}}
+{{- else -}}
+{{- fail "certs.issuers.ca.name is required when TLS is enabled and certs.internal_issuer.enabled is false" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Define the pulsar certs ca issuer secret name
+*/}}
+{{- define "pulsar.certs.issuers.ca.secretName" -}}
+{{- if .Values.certs.internal_issuer.enabled -}}
+{{- if and (eq .Values.certs.internal_issuer.type "selfsigning") .Values.certs.issuers.selfsigning.secretName -}}
+{{- .Values.certs.issuers.selfsigning.secretName -}}
+{{- else if and (eq .Values.certs.internal_issuer.type "ca") .Values.certs.issuers.ca.secretName -}}
+{{- .Values.certs.issuers.ca.secretName -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name .Values.tls.ca_suffix -}}
+{{- end -}}
+{{- else -}}
+{{- if .Values.certs.issuers.ca.secretName -}}
+{{- .Values.certs.issuers.ca.secretName -}}
+{{- else -}}
+{{- fail "certs.issuers.ca.secretName is required when TLS is enabled and certs.internal_issuer.enabled is false" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Common certificate template
+Usage: {{- include "pulsar.cert.template" (dict "root" . "componentConfig" .Values.proxy "tlsConfig" .Values.tls.proxy) -}}
+*/}}
+{{- define "pulsar.cert.template" -}}
+{{- if eq .root.Values.certs.internal_issuer.apiVersion "cert-manager.io/v1beta1" -}}
+{{- fail "cert-manager.io/v1beta1 is no longer supported. Please set certs.internal_issuer.apiVersion to cert-manager.io/v1" -}}
+{{- end -}}
+apiVersion: "{{ .root.Values.certs.internal_issuer.apiVersion }}"
+kind: Certificate
+metadata:
+  name: "{{ template "pulsar.fullname" .root }}-{{ .tlsConfig.cert_name }}"
+  namespace: {{ template "pulsar.namespace" .root }}
+  labels:
+    {{- include "pulsar.standardLabels" .root | nindent 4 }}
+spec:
+  # Secret names are always required.
+  secretName: "{{ .root.Release.Name }}-{{ .tlsConfig.cert_name }}"
+{{- if .root.Values.tls.zookeeper.enabled }}
+  additionalOutputFormats:
+    - type: CombinedPEM
+{{- end }}
+  duration: "{{ .root.Values.tls.common.duration }}"
+  renewBefore: "{{ .root.Values.tls.common.renewBefore }}"
+  subject:
+    organizations:
+{{ toYaml .root.Values.tls.common.organization | indent 4 }}
+  # The use of the common name field has been deprecated since 2000 and is
+  # discouraged from being used.
+  commonName: "{{ template "pulsar.fullname" .root }}-{{ .componentConfig.component }}"
+  isCA: false
+  privateKey:
+    size: {{ .root.Values.tls.common.keySize }}
+    algorithm: {{ .root.Values.tls.common.keyAlgorithm }}
+    encoding: {{ .root.Values.tls.common.keyEncoding }}
+  usages:
+    - server auth
+    - client auth
+  # At least one of a DNS Name, USI SAN, or IP address is required.
+  dnsNames:
+{{- if .tlsConfig.dnsNames }}
+{{ toYaml .tlsConfig.dnsNames | indent 4 }}
+{{- end }}
+    - {{ printf "*.%s-%s.%s.svc.%s" (include "pulsar.fullname" .root) .componentConfig.component (include "pulsar.namespace" .root) .root.Values.clusterDomain | quote }}
+    - {{ printf "%s-%s" (include "pulsar.fullname" .root) .componentConfig.component | quote }}
+  # Issuer references are always required.
+  issuerRef:
+    name: "{{ template "pulsar.certs.issuers.ca.name" .root }}"
+    # We can reference ClusterIssuers by changing the kind here.
+    # The default value is Issuer (i.e. a locally namespaced Issuer)
+    kind: Issuer
+    # This is optional since cert-manager will default to this value however
+    # if you are using an external issuer, change this to that issuer group.
+    group: cert-manager.io
+{{- end -}}
+
+{{/*
+CA certificates template
+Usage: {{ include "pulsar.certs.cacerts" (dict "certs" .Values.tls.<component>.cacerts.certs) }}
+*/}}
+{{- define "pulsar.certs.cacerts" -}}
+{{- $certs := .certs -}}
+{{- $cacerts := list -}}
+{{- $cacerts = print "/pulsar/certs/ca/ca.crt" | append $cacerts -}}
+{{- range $cert := $certs -}}
+{{- range $key := $cert.secretKeys -}}
+{{- $cacerts = print "/pulsar/certs/" $cert.name "/" $key | append $cacerts -}}
+{{- end -}}
+{{- end -}}
+{{ join " " $cacerts }}
+{{- end -}}

charts/pulsar/templates/_helpers.tpl

@@ -126,5 +126,13 @@ imagePullSecrets:
 Create full image name
 */}}
 {{- define "pulsar.imageFullName" -}}
-{{- printf "%s:%s" .image.repository (.image.tag | default .root.Values.defaultPulsarImageTag | default .root.Chart.AppVersion) -}}
+{{- printf "%s:%s" (.image.repository | default .root.Values.defaultPulsarImageRepository) (.image.tag | default .root.Values.defaultPulsarImageTag | default .root.Chart.AppVersion) -}}
 {{- end -}}
+
+{{/*
+Lookup pull policy, default to defaultPullPolicy
+*/}}
+{{- define "pulsar.imagePullPolicy" -}}
+{{- printf "%s" (.image.pullPolicy | default .root.Values.defaultPullPolicy) -}}
+{{- end -}}
+

charts/pulsar/templates/_monitor.tpl

@@ -0,0 +1,97 @@
+{{/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+*/}}
+
+{{- define "pulsar.podMonitor" -}}
+{{- $root := index . 0 }}
+{{- $component := index . 1 }}
+{{- $matchLabel := index . 2 }}
+{{- $portName := "http" }}
+{{- if gt (len .) 3 }}
+{{- $portName = index . 3 }}
+{{- end }}
+
+{{/* Extract component parts for nested values */}}
+{{- $componentParts := splitList "." $component }}
+{{- $valuesPath := $root.Values }}
+{{- range $componentParts }}
+  {{- $valuesPath = index $valuesPath . }}
+{{- end }}
+
+{{- if index $root.Values "victoria-metrics-k8s-stack" "enabled" }}
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+{{- else }}
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+{{- end }}
+metadata:
+  name: {{ template "pulsar.fullname" $root }}-{{ replace "." "-" $component }}
+  labels:
+    {{- include "pulsar.standardLabels" $root | nindent 4 }}
+spec:
+  jobLabel: {{ replace "." "-" $component }}
+  podMetricsEndpoints:
+    - port: {{ $portName }}
+      path: /metrics
+      scheme: http
+      interval: {{ $valuesPath.podMonitor.interval }}
+      scrapeTimeout: {{ $valuesPath.podMonitor.scrapeTimeout }}
+      # Set honor labels to true to allow overriding namespace label with Pulsar's namespace label
+      honorLabels: true
+      {{- if index $root.Values "victoria-metrics-k8s-stack" "enabled" }}
+      relabelConfigs:
+      {{- else }}
+      relabelings:
+      {{- end }}
+        - action: labelmap
+          regex: __meta_kubernetes_pod_label_(.+)
+        - sourceLabels: [__meta_kubernetes_namespace]
+          action: replace
+          targetLabel: kubernetes_namespace
+        - sourceLabels: [__meta_kubernetes_pod_label_component]
+          action: replace
+          targetLabel: job
+        - sourceLabels: [__meta_kubernetes_pod_name]
+          action: replace
+          targetLabel: kubernetes_pod_name
+      {{- if or $valuesPath.podMonitor.metricRelabelings (and $valuesPath.podMonitor.dropUnderscoreCreatedMetrics (index $valuesPath.podMonitor.dropUnderscoreCreatedMetrics "enabled")) }}
+      {{- if index $root.Values "victoria-metrics-k8s-stack" "enabled" }}
+      metricRelabelConfigs:
+      {{- else }}
+      metricRelabelings:
+      {{- end }}
+      {{- if and $valuesPath.podMonitor.dropUnderscoreCreatedMetrics (index $valuesPath.podMonitor.dropUnderscoreCreatedMetrics "enabled") }}
+        # Drop metrics that end with _created, auto-created by metrics library to match OpenMetrics format
+        - sourceLabels: [__name__]
+          {{- if and (hasKey $valuesPath.podMonitor.dropUnderscoreCreatedMetrics "excludePatterns") $valuesPath.podMonitor.dropUnderscoreCreatedMetrics.excludePatterns }}
+          regex: "(?!{{ $valuesPath.podMonitor.dropUnderscoreCreatedMetrics.excludePatterns | join "|" }}).*_created$"
+          {{- else }}
+          regex: ".*_created$"
+          {{- end }}
+          action: drop
+      {{- end }}
+      {{- with $valuesPath.podMonitor.metricRelabelings }}
+{{ toYaml . | indent 8 }}
+      {{- end }}
+      {{- end }}
+  selector:
+    matchLabels:
+      {{- include "pulsar.matchLabels" $root | nindent 6 }}
+      {{ $matchLabel }}
+{{- end -}}

charts/pulsar/templates/_oxia.tpl

@@ -0,0 +1,122 @@
+{{/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+*/}}
+
+{{/*
+Probe
+*/}}
+{{- define "oxia-cluster.probe" -}}
+exec:
+  command: ["oxia", "health", "--port={{ . }}"]
+initialDelaySeconds: 10
+timeoutSeconds: 10
+{{- end }}
+
+
+{{/*
+Probe
+*/}}
+{{- define "oxia-cluster.readiness-probe" -}}
+exec:
+  command: ["oxia", "health", "--port={{ . }}", "--service=oxia-readiness"]
+initialDelaySeconds: 10
+timeoutSeconds: 10
+{{- end }}
+
+{{/*
+Probe
+*/}}
+{{- define "oxia-cluster.startup-probe" -}}
+exec:
+  command: ["oxia", "health", "--port={{ . }}"]
+initialDelaySeconds: 60
+timeoutSeconds: 10
+{{- end }}
+
+{{/*
+Define the pulsar oxia
+*/}}
+{{- define "pulsar.oxia.server.service" -}}
+{{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-svc
+{{- end }}
+
+{{/*
+oxia url for broker metadata
+*/}}
+{{- define "pulsar.oxia.metadata.url.broker" -}}
+{{- if .Values.components.oxia -}}
+oxia://{{ template "pulsar.oxia.server.service" . }}:{{ .Values.oxia.server.ports.public }}/broker
+{{- end -}}
+{{- end -}}
+
+{{/*
+oxia url for bookkeeper metadata
+*/}}
+{{- define "pulsar.oxia.metadata.url.bookkeeper" -}}
+{{- if .Values.components.oxia -}}
+metadata-store:oxia://{{ template "pulsar.oxia.server.service" . }}:{{ .Values.oxia.server.ports.public }}/bookkeeper
+{{- end -}}
+{{- end -}}
+
+{{/*
+Define coordinator configmap
+*/}}
+{{- define "oxia.coordinator.config.yaml" -}}
+namespaces:
+  - name: default
+    initialShardCount: {{ .Values.oxia.initialShardCount }}
+    replicationFactor: {{ .Values.oxia.replicationFactor }}
+  - name: broker
+    initialShardCount: {{ .Values.oxia.initialShardCount }}
+    replicationFactor: {{ .Values.oxia.replicationFactor }}
+  - name: bookkeeper
+    initialShardCount: {{ .Values.oxia.initialShardCount }}
+    replicationFactor: {{ .Values.oxia.replicationFactor }}
+servers:
+  {{- $servicename := printf "%s-%s-svc" (include "pulsar.fullname" .) .Values.oxia.component }}
+  {{- $fqdnSuffix := printf "%s.svc.cluster.local" (include "pulsar.namespace" .) }}
+  {{- $podnamePrefix := printf "%s-%s-server-" (include "pulsar.fullname" .) .Values.oxia.component }}
+  {{- range until (int .Values.oxia.server.replicas) }}
+  {{- $podnameIndex := . }}
+  {{- $podname := printf "%s%d.%s" $podnamePrefix $podnameIndex $servicename }}
+  {{- $podnameFQDN := printf "%s.%s" $podname $fqdnSuffix }}
+  - public: {{ $podnameFQDN }}:{{ $.Values.oxia.server.ports.public }}
+    internal: {{ $podname }}:{{ $.Values.oxia.server.ports.internal }}
+  {{- end }}
+{{- end }}
+
+{{/*
+Define coordinator entrypoint
+*/}}
+{{- define "oxia.coordinator.entrypoint" -}}
+- "oxia"
+- "coordinator"
+{{- if .Values.oxia.coordinator.customConfigMapName }}
+- "--conf=configmap:{{ template "pulsar.namespace" . }}/{{ .Values.oxia.coordinator.customConfigMapName }}"
+{{- else }}
+- "--conf=configmap:{{ template "pulsar.namespace" . }}/{{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator"
+{{- end }}
+- "--log-json"
+- "--metadata=configmap"
+- "--k8s-namespace={{ template "pulsar.namespace" . }}"
+- "--k8s-configmap-name={{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator-status"
+{{- if .Values.oxia.pprofEnabled }}
+- "--profile"
+{{- end}}
+{{- end}}
+

charts/pulsar/templates/_proxy.tpl

@@ -0,0 +1,95 @@
+{{/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+*/}}
+
+{{/*
+Define proxy tls certs mounts
+*/}}
+{{- define "pulsar.proxy.certs.volumeMounts" -}}
+{{- if .Values.tls.enabled }}
+{{- if .Values.tls.proxy.enabled }}
+- mountPath: "/pulsar/certs/proxy"
+  name: proxy-certs
+  readOnly: true
+{{- end }}
+- mountPath: "/pulsar/certs/ca"
+  name: ca
+  readOnly: true
+{{- end }}
+{{- if .Values.tls.proxy.cacerts.enabled }}
+- mountPath: "/pulsar/certs/cacerts"
+  name: proxy-cacerts
+{{- range $cert := .Values.tls.proxy.cacerts.certs }}
+- name: {{ $cert.name }}
+  mountPath: "/pulsar/certs/{{ $cert.name }}"
+  readOnly: true
+{{- end }}
+- name: certs-scripts
+  mountPath: "/pulsar/bin/certs-combine-pem.sh"
+  subPath: certs-combine-pem.sh
+- name: certs-scripts
+  mountPath: "/pulsar/bin/certs-combine-pem-infinity.sh"
+  subPath: certs-combine-pem-infinity.sh
+{{- end }}
+{{- end }}
+
+{{/*
+Define proxy tls certs volumes
+*/}}
+{{- define "pulsar.proxy.certs.volumes" -}}
+{{- if .Values.tls.enabled }}
+{{- if .Values.tls.proxy.enabled }}
+- name: proxy-certs
+  secret:
+    secretName: "{{ .Release.Name }}-{{ .Values.tls.proxy.cert_name }}"
+    items:
+      - key: tls.crt
+        path: tls.crt
+      - key: tls.key
+        path: tls.key
+      {{- if .Values.tls.zookeeper.enabled }}
+      - key: tls-combined.pem
+        path: tls-combined.pem
+      {{- end }}
+{{- end }}
+- name: ca
+  secret:
+    secretName: "{{ template "pulsar.certs.issuers.ca.secretName" . }}"
+    items:
+      - key: ca.crt
+        path: ca.crt
+{{- end }}
+{{- if .Values.tls.proxy.cacerts.enabled }}
+- name: proxy-cacerts
+  emptyDir: {}
+{{- range $cert := .Values.tls.proxy.cacerts.certs }}
+- name: {{ $cert.name }}
+  secret:
+    secretName: "{{ $cert.existingSecret }}"
+    items:
+    {{- range $key := $cert.secretKeys }}
+      - key: {{ $key }}
+        path: {{ $key }}
+    {{- end }}
+{{- end }}
+- name: certs-scripts
+  configMap:
+    name: "{{ template "pulsar.fullname" . }}-certs-scripts"
+    defaultMode: 0755
+{{- end }}
+{{- end }}

charts/pulsar/templates/_toolset.tpl

@@ -36,7 +36,7 @@ Define toolset zookeeper client tls settings
 */}}
 {{- define "pulsar.toolset.zookeeper.tls.settings" -}}
 {{- if and .Values.tls.enabled .Values.tls.zookeeper.enabled -}}
-/pulsar/keytool/keytool.sh toolset {{ template "pulsar.toolset.hostname" . }} true;
+{{- include "pulsar.component.zookeeper.tls.settings" (dict "component" "toolset" "isClient" true "isCacerts" .Values.tls.toolset.cacerts.enabled) -}}
 {{- end -}}
 {{- end }}
 
@@ -44,18 +44,30 @@ Define toolset zookeeper client tls settings
 Define toolset tls certs mounts
 */}}
 {{- define "pulsar.toolset.certs.volumeMounts" -}}
-{{- if and .Values.tls.enabled .Values.tls.zookeeper.enabled }}
+{{- if .Values.tls.enabled }}
+{{- if .Values.tls.zookeeper.enabled }}
 - name: toolset-certs
   mountPath: "/pulsar/certs/toolset"
   readOnly: true
+{{- end }}
 - name: ca
   mountPath: "/pulsar/certs/ca"
   readOnly: true
-{{- if .Values.tls.zookeeper.enabled }}
-- name: keytool
-  mountPath: "/pulsar/keytool/keytool.sh"
-  subPath: keytool.sh
 {{- end }}
+{{- if .Values.tls.toolset.cacerts.enabled }}
+- mountPath: "/pulsar/certs/cacerts"
+  name: toolset-cacerts
+{{- range $cert := .Values.tls.toolset.cacerts.certs }}
+- name: {{ $cert.name }}
+  mountPath: "/pulsar/certs/{{ $cert.name }}"
+  readOnly: true
+{{- end }}
+- name: certs-scripts
+  mountPath: "/pulsar/bin/certs-combine-pem.sh"
+  subPath: certs-combine-pem.sh
+- name: certs-scripts
+  mountPath: "/pulsar/bin/certs-combine-pem-infinity.sh"
+  subPath: certs-combine-pem-infinity.sh
 {{- end }}
 {{- end }}
 
@@ -63,7 +75,8 @@ Define toolset tls certs mounts
 Define toolset tls certs volumes
 */}}
 {{- define "pulsar.toolset.certs.volumes" -}}
-{{- if and .Values.tls.enabled .Values.tls.zookeeper.enabled }}
+{{- if .Values.tls.enabled  }}
+{{- if .Values.tls.zookeeper.enabled }}
 - name: toolset-certs
   secret:
     secretName: "{{ .Release.Name }}-{{ .Values.tls.toolset.cert_name }}"
@@ -72,17 +85,32 @@ Define toolset tls certs volumes
       path: tls.crt
     - key: tls.key
       path: tls.key
+    - key: tls-combined.pem
+      path: tls-combined.pem
+{{- end }}
 - name: ca
   secret:
-    secretName: "{{ .Release.Name }}-{{ .Values.tls.ca_suffix }}"
+    secretName: "{{ template "pulsar.certs.issuers.ca.secretName" . }}"
     items:
     - key: ca.crt
       path: ca.crt
-{{- if .Values.tls.zookeeper.enabled }}
-- name: keytool
+{{- end }}
+{{- if .Values.tls.toolset.cacerts.enabled }}
+- name: toolset-cacerts
+  emptyDir: {}
+{{- range $cert := .Values.tls.toolset.cacerts.certs }}
+- name: {{ $cert.name }}
+  secret:
+    secretName: "{{ $cert.existingSecret }}"
+    items:
+    {{- range $key := $cert.secretKeys }}
+      - key: {{ $key }}
+        path: {{ $key }}
+    {{- end }}
+{{- end }}
+- name: certs-scripts
   configMap:
-    name: "{{ template "pulsar.fullname" . }}-keytool-configmap"
+    name: "{{ template "pulsar.fullname" . }}-certs-scripts"
     defaultMode: 0755
 {{- end }}
 {{- end }}
-{{- end }}

charts/pulsar/templates/_tplvalues.tpl

@@ -0,0 +1,37 @@
+{{/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+*/}}
+
+{{/*
+Renders a value that contains template perhaps with scope if the scope is present.
+Usage:
+{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ ) }}
+{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ "scope" $app ) }}
+*/}}
+{{- define "common.tplvalues.render" -}}
+{{- $value := typeIs "string" .value | ternary .value (.value | toYaml) }}
+{{- if contains "{{" (toJson .value) }}
+  {{- if .scope }}
+      {{- tpl (cat "{{- with $.RelativeScope -}}" $value "{{- end }}") (merge (dict "RelativeScope" .scope) .context) }}
+  {{- else }}
+    {{- tpl $value .context }}
+  {{- end }}
+{{- else }}
+    {{- $value }}
+{{- end }}
+{{- end -}}

charts/pulsar/templates/_values_validation.tpl

@@ -0,0 +1,25 @@
+{{/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+*/}}
+
+{{/*
+Check deprecated setting auth.authentication.provider since 4.1.0
+*/}}
+{{- if (and .Values.auth.authentication.enabled (not (empty .Values.auth.authentication.provider))) }}
+  {{- fail "ERROR: Setting auth.authentication.provider is no longer supported. For details, see the migration guide in README.md." }}
+{{- end }}

charts/pulsar/templates/_zookeeper.tpl

@@ -53,6 +53,93 @@ Define zookeeper tls settings
 */}}
 {{- define "pulsar.zookeeper.tls.settings" -}}
 {{- if and .Values.tls.enabled .Values.tls.zookeeper.enabled }}
-/pulsar/keytool/keytool.sh zookeeper {{ template "pulsar.zookeeper.hostname" . }} false;
+{{- include "pulsar.component.zookeeper.tls.settings" (dict "component" "zookeeper" "isClient" false "isCacerts" .Values.tls.zookeeper.cacerts.enabled) -}}
+{{- end }}
+{{- end }}
+
+{{- define "pulsar.component.zookeeper.tls.settings" }}
+{{- $component := .component -}}
+{{- $isClient := .isClient -}}
+{{- $keyFile := printf "/pulsar/certs/%s/tls-combined.pem" $component -}}
+{{- $caFile := ternary "/pulsar/certs/cacerts/ca-combined.pem" "/pulsar/certs/ca/ca.crt" .isCacerts -}}
+{{- if $isClient }}
+echo $'\n' >> conf/pulsar_env.sh
+echo "PULSAR_EXTRA_OPTS=\"\${PULSAR_EXTRA_OPTS} -Dzookeeper.clientCnxnSocket=org.apache.zookeeper.ClientCnxnSocketNetty -Dzookeeper.client.secure=true -Dzookeeper.client.certReload=true -Dzookeeper.ssl.keyStore.location={{- $keyFile }} -Dzookeeper.ssl.keyStore.type=PEM -Dzookeeper.ssl.trustStore.location={{- $caFile }} -Dzookeeper.ssl.trustStore.type=PEM\"" >> conf/pulsar_env.sh
+echo $'\n' >> conf/bkenv.sh
+echo "BOOKIE_EXTRA_OPTS=\"\${BOOKIE_EXTRA_OPTS} -Dzookeeper.clientCnxnSocket=org.apache.zookeeper.ClientCnxnSocketNetty -Dzookeeper.client.secure=true -Dzookeeper.client.certReload=true -Dzookeeper.ssl.keyStore.location={{- $keyFile }} -Dzookeeper.ssl.keyStore.type=PEM -Dzookeeper.ssl.trustStore.location={{- $caFile }} -Dzookeeper.ssl.trustStore.type=PEM\"" >> conf/bkenv.sh
+{{- else }}
+echo $'\n' >> conf/pulsar_env.sh
+echo "PULSAR_EXTRA_OPTS=\"\${PULSAR_EXTRA_OPTS} -Dzookeeper.ssl.keyStore.location={{- $keyFile }} -Dzookeeper.ssl.keyStore.type=PEM -Dzookeeper.ssl.trustStore.location={{- $caFile }} -Dzookeeper.ssl.trustStore.type=PEM\"" >> conf/pulsar_env.sh
+{{- end }}
+{{- end }}
+
+{{/*
+Define zookeeper tls certs mounts
+*/}}
+{{- define "pulsar.zookeeper.certs.volumeMounts" -}}
+{{- if and .Values.tls.enabled .Values.tls.zookeeper.enabled }}
+- mountPath: "/pulsar/certs/zookeeper"
+  name: zookeeper-certs
+  readOnly: true
+- mountPath: "/pulsar/certs/ca"
+  name: ca
+  readOnly: true
+{{- end }}
+{{- if .Values.tls.zookeeper.cacerts.enabled }}
+- mountPath: "/pulsar/certs/cacerts"
+  name: zookeeper-cacerts
+{{- range $cert := .Values.tls.zookeeper.cacerts.certs }}
+- name: {{ $cert.name }}
+  mountPath: "/pulsar/certs/{{ $cert.name }}"
+  readOnly: true
+{{- end }}
+- name: certs-scripts
+  mountPath: "/pulsar/bin/certs-combine-pem.sh"
+  subPath: certs-combine-pem.sh
+- name: certs-scripts
+  mountPath: "/pulsar/bin/certs-combine-pem-infinity.sh"
+  subPath: certs-combine-pem-infinity.sh
+{{- end }}
+{{- end }}
+
+{{/*
+Define zookeeper tls certs volumes
+*/}}
+{{- define "pulsar.zookeeper.certs.volumes" -}}
+{{- if and .Values.tls.enabled .Values.tls.zookeeper.enabled }}
+- name: zookeeper-certs
+  secret:
+    secretName: "{{ .Release.Name }}-{{ .Values.tls.zookeeper.cert_name }}"
+    items:
+      - key: tls.crt
+        path: tls.crt
+      - key: tls.key
+        path: tls.key
+      - key: tls-combined.pem
+        path: tls-combined.pem
+- name: ca
+  secret:
+    secretName: "{{ template "pulsar.certs.issuers.ca.secretName" . }}"
+    items:
+      - key: ca.crt
+        path: ca.crt
+{{- end }}
+{{- if .Values.tls.zookeeper.cacerts.enabled }}
+- name: zookeeper-cacerts
+  emptyDir: {}
+{{- range $cert := .Values.tls.zookeeper.cacerts.certs }}
+- name: {{ $cert.name }}
+  secret:
+    secretName: "{{ $cert.existingSecret }}"
+    items:
+    {{- range $key := $cert.secretKeys }}
+      - key: {{ $key }}
+        path: {{ $key }}
+    {{- end }}
+{{- end }}
+- name: certs-scripts
+  configMap:
+    name: "{{ template "pulsar.fullname" . }}-certs-scripts"
+    defaultMode: 0755
 {{- end }}
 {{- end }}

charts/pulsar/templates/autorecovery-configmap.yaml

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-{{- if or .Values.components.autorecovery .Values.extra.autoRecovery  }}
+{{- if .Values.components.autorecovery }}
 apiVersion: v1
 kind: ConfigMap
 metadata:

charts/pulsar/templates/autorecovery-podmonitor.yaml

@@ -17,39 +17,7 @@
 # under the License.
 #
 
-# deploy broker PodMonitor only when `$.Values.broker.podMonitor.enabled` is true
+# deploy autorecovery PodMonitor only when `$.Values.autorecovery.podMonitor.enabled` is true
 {{- if $.Values.autorecovery.podMonitor.enabled }}
-apiVersion: monitoring.coreos.com/v1
-kind: PodMonitor
-metadata:
-  name: {{ template "pulsar.name" . }}-recovery
-  labels:
-    app: {{ template "pulsar.name" . }}
-    chart: {{ template "pulsar.chart" . }}
-    release: {{ .Release.Name }}
-    heritage: {{ .Release.Service }}
-spec:
-  jobLabel: recovery
-  podMetricsEndpoints:
-    - port: http
-      path: /metrics
-      scheme: http
-      interval: {{ $.Values.autorecovery.podMonitor.interval }}
-      scrapeTimeout: {{ $.Values.autorecovery.podMonitor.scrapeTimeout }}
-      relabelings:
-        - action: labelmap
-          regex: __meta_kubernetes_pod_label_(.+)
-        - sourceLabels: [__meta_kubernetes_namespace]
-          action: replace
-          targetLabel: kubernetes_namespace
-        - sourceLabels: [__meta_kubernetes_pod_label_component]
-          action: replace
-          targetLabel: job
-        - sourceLabels: [__meta_kubernetes_pod_name]
-          action: replace
-          targetLabel: kubernetes_pod_name
-  selector:
-    matchLabels:
-      {{- include "pulsar.matchLabels" . | nindent 6 }}
-      component: {{ .Values.autorecovery.component }}
+{{- include "pulsar.podMonitor" (list . "autorecovery" (printf "component: %s" .Values.autorecovery.component)) }}
 {{- end }}

charts/pulsar/templates/autorecovery-rbac.yaml

@@ -1,92 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-{{- if and .Values.rbac.enabled .Values.rbac.psp }}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.autorecovery.component }}"
-  namespace: {{ template "pulsar.namespace" . }}
-rules:
-  - apiGroups:
-      - policy
-    resourceNames:
-      - "{{ template "pulsar.fullname" . }}-{{ .Values.autorecovery.component }}"
-    resources:
-      - podsecuritypolicies
-    verbs:
-      - use
----
-
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.autorecovery.component }}"
-  namespace: {{ template "pulsar.namespace" . }}
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.autorecovery.component }}"
-  namespace: {{ template "pulsar.namespace" . }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.autorecovery.component }}"
-subjects:
-- kind: ServiceAccount
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.autorecovery.component }}"
-  namespace: {{ template "pulsar.namespace" . }}
----
-
-apiVersion: policy/v1beta1
-kind: PodSecurityPolicy
-metadata:
-{{- if .Values.rbac.limit_to_namespace }}
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.autorecovery.component }}-{{ template "pulsar.namespace" . }}"
-{{- else}}
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.autorecovery.component }}"
-{{- end}}
-spec:
-  readOnlyRootFilesystem: false
-  privileged: false
-  allowPrivilegeEscalation: false
-  runAsUser:
-    rule: 'RunAsAny'
-  supplementalGroups:
-    ranges:
-    - max: 65535
-      min: 1
-    rule: MustRunAs
-  fsGroup:
-    rule: 'MustRunAs'
-    ranges:
-      - min: 1
-        max: 65535
-  seLinux:
-    rule: 'RunAsAny'
-  volumes:
-  - configMap
-  - emptyDir
-  - projected
-  - secret
-  - downwardAPI
-  - persistentVolumeClaim
-{{- end }}

charts/pulsar/templates/autorecovery-service-account.yaml

@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+{{- if .Values.components.autorecovery }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: "{{ template "pulsar.fullname" . }}-{{ .Values.autorecovery.component }}"
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.autorecovery.component }}
+{{- with .Values.autorecovery.service_account.annotations }}
+  annotations:
+{{ toYaml . | indent 4 }}
+{{- end }}
+{{- end }}

charts/pulsar/templates/autorecovery-service.yaml

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-{{- if or .Values.components.autorecovery .Values.extra.autoRecovery }}
+{{- if .Values.components.autorecovery }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -26,6 +26,10 @@ metadata:
   labels:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
     component: {{ .Values.autorecovery.component }}
+{{- with .Values.autorecovery.service.annotations }}
+  annotations:
+{{ toYaml . | indent 4 }}
+{{- end }}
 spec:
   ports:
   - name: http

charts/pulsar/templates/autorecovery-statefulset.yaml

@@ -17,12 +17,13 @@
 # under the License.
 #
 
-{{- if or .Values.components.autorecovery .Values.extra.autoRecovery }}
+{{- if .Values.components.autorecovery }}
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: "{{ template "pulsar.fullname" . }}-{{ .Values.autorecovery.component }}"
   namespace: {{ template "pulsar.namespace" . }}
+  annotations: {{ .Values.autorecovery.appAnnotations | toYaml | nindent 4 }}
   labels:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
     component: {{ .Values.autorecovery.component }}
@@ -43,8 +44,10 @@ spec:
         {{- include "pulsar.template.labels" . | nindent 8 }}
         component: {{ .Values.autorecovery.component }}
       annotations:
+        {{- if not .Values.autorecovery.podMonitor.enabled }}
         prometheus.io/scrape: "true"
         prometheus.io/port: "{{ .Values.autorecovery.ports.http }}"
+        {{- end }}
         {{- if .Values.autorecovery.restartPodsOnConfigMapChange }}
         checksum/config: {{ include (print $.Template.BasePath "/autorecovery-configmap.yaml") . | sha256sum }}
         {{- end }}
@@ -61,6 +64,10 @@ spec:
 {{- with .Values.autorecovery.tolerations }}
 {{ toYaml . | indent 8 }}
 {{- end }}
+    {{- end }}
+    {{- if .Values.autorecovery.topologySpreadConstraints }}
+      topologySpreadConstraints:
+        {{- toYaml .Values.autorecovery.topologySpreadConstraints | nindent 8 }}
     {{- end }}
       affinity:
         {{- if and .Values.affinity.anti_affinity .Values.autorecovery.affinity.anti_affinity}}
@@ -104,39 +111,59 @@ spec:
         {{ end }}
         {{- end }}
       terminationGracePeriodSeconds: {{ .Values.autorecovery.gracePeriod }}
-    {{- if and .Values.rbac.enabled  .Values.rbac.psp }}
       serviceAccountName: "{{ template "pulsar.fullname" . }}-{{ .Values.autorecovery.component }}"
-    {{- end}}
       initContainers:
+      {{- if .Values.tls.autorecovery.cacerts.enabled }}
+      - name: cacerts
+        image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.autorecovery "root" .) }}"
+        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.autorecovery "root" .) }}"
+        resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
+        command: ["sh", "-c"]
+        args:
+        - |
+          bin/certs-combine-pem.sh /pulsar/certs/cacerts/ca-combined.pem {{ template "pulsar.certs.cacerts" (dict "certs" .Values.tls.autorecovery.cacerts.certs) }}
+        volumeMounts:
+        {{- include "pulsar.autorecovery.certs.volumeMounts" . | nindent 8 }}
+      {{- end }}
+      {{- if and .Values.autorecovery.waitBookkeeperTimeout (gt (.Values.autorecovery.waitBookkeeperTimeout | int) 0) }}
       # This initContainer will wait for bookkeeper initnewcluster to complete
       # before deploying the bookies
       - name: pulsar-bookkeeper-verify-clusterid
         image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.autorecovery "root" .) }}"
-        imagePullPolicy: {{ .Values.images.autorecovery.pullPolicy }}
-        command: ["sh", "-c"]
+        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.autorecovery "root" .) }}"
+        resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
+        command: ["timeout", "{{ .Values.autorecovery.waitBookkeeperTimeout }}", "sh", "-c"]
         args:
-        - >
+        - |
           {{- include "pulsar.autorecovery.init.verify_cluster_id" . | nindent 10 }}
         envFrom:
         - configMapRef:
             name: "{{ template "pulsar.fullname" . }}-{{ .Values.autorecovery.component }}"
         volumeMounts:
+        {{- if .Values.autorecovery.extraVolumeMounts }}
+{{ toYaml .Values.autorecovery.extraVolumeMounts | indent 8 }}
+        {{- end }}
         {{- include "pulsar.autorecovery.certs.volumeMounts" . | nindent 8 }}
+      {{- end }}
+      {{- if .Values.autorecovery.initContainers }}
+        {{- toYaml .Values.autorecovery.initContainers | nindent 6 }}
+      {{- end }}
       containers:
       - name: "{{ template "pulsar.fullname" . }}-{{ .Values.autorecovery.component }}"
         image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.autorecovery "root" .) }}"
-        imagePullPolicy: {{ .Values.images.autorecovery.pullPolicy }}
+        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.autorecovery "root" .) }}"
       {{- if .Values.autorecovery.resources }}
         resources:
 {{ toYaml .Values.autorecovery.resources | indent 10 }}
-      {{- end }}
-      {{- if and .Values.rbac.enabled .Values.rbac.psp }}
-        securityContext:
-          readOnlyRootFilesystem: false
       {{- end }}
         command: ["sh", "-c"]
         args:
-        - >
+        - |
+          {{- if .Values.tls.autorecovery.cacerts.enabled }}
+          cd /pulsar/certs/cacerts;
+          nohup /pulsar/bin/certs-combine-pem-infinity.sh /pulsar/certs/cacerts/ca-combined.pem {{ template "pulsar.certs.cacerts" (dict "certs" .Values.tls.autorecovery.cacerts.certs) }} > /pulsar/certs/cacerts/certs-combine-pem-infinity.log 2>&1 &
+          cd /pulsar;
+          {{- end }}
           bin/apply-config-from-env.py conf/bookkeeper.conf;
           {{- include "pulsar.autorecovery.zookeeper.tls.settings" . | nindent 10 }}
           OPTS="${OPTS} -Dlog4j2.formatMsgNoLookups=true" exec bin/bookkeeper autorecovery
@@ -150,6 +177,9 @@ spec:
         {{- include "pulsar.autorecovery.certs.volumeMounts" . | nindent 8 }}
       volumes:
       {{- include "pulsar.autorecovery.certs.volumes" . | nindent 6 }}
+      {{- if .Values.autorecovery.extraVolumes }}
+{{ toYaml .Values.autorecovery.extraVolumes | indent 6 }}
+      {{- end }}
       {{- include "pulsar.imagePullSecrets" . | nindent 6}}
 {{- end }}
 

charts/pulsar/templates/bookkeeper-cluster-initialize.yaml

@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-{{- if or .Release.IsInstall .Values.initialize }}
+{{- if or (and .Values.useReleaseStatus .Release.IsInstall) .Values.initialize }}
 {{- if .Values.components.bookkeeper }}
 apiVersion: batch/v1
 kind: Job
@@ -29,28 +29,49 @@ metadata:
 spec:
 # This feature was previously behind a feature gate for several Kubernetes versions and will default to true in 1.23 and beyond
 # https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/
-{{- if .Values.job.ttl.enabled }}
-  ttlSecondsAfterFinished: {{ .Values.job.ttl.secondsAfterFinished }}
+{{- if and .Values.job.ttl.enabled (semverCompare ">=1.23-0" .Capabilities.KubeVersion.Version) }}
+  ttlSecondsAfterFinished: {{ .Values.job.ttl.secondsAfterFinished | default 600 }}
 {{- end }}
   template:
+    metadata:
+      labels:
+        {{- include "pulsar.template.labels" . | nindent 8 }}
+        component: {{ .Values.bookkeeper.component }}-init
     spec:
-    {{- if and .Values.rbac.enabled .Values.rbac.psp }}
+    {{- include "pulsar.imagePullSecrets" . | nindent 6 }}
       serviceAccountName: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
-    {{- end }}
-      nodeSelector:
       {{- if .Values.pulsar_metadata.nodeSelector }}
+      nodeSelector:
 {{ toYaml .Values.pulsar_metadata.nodeSelector | indent 8 }}
+      {{- end }}
+      {{- with .Values.pulsar_metadata.tolerations }}
+      tolerations:
+{{ toYaml .Values.pulsar_metadata.tolerations | indent 8 }}
       {{- end }}
       initContainers:
-      - name: wait-zookeeper-ready
+      {{- if .Values.tls.bookie.cacerts.enabled }}
+      - name: cacerts
         image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.bookie "root" .) }}"
-        imagePullPolicy: {{ .Values.images.bookie.pullPolicy }}
+        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.bookie "root" .) }}"
+        resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
         command: ["sh", "-c"]
         args:
-          - >-
+          - |
+            bin/certs-combine-pem.sh /pulsar/certs/cacerts/ca-combined.pem {{ template "pulsar.certs.cacerts" (dict "certs" .Values.tls.bookie.cacerts.certs) }}
+        volumeMounts:
+        {{- include "pulsar.toolset.certs.volumeMounts" . | nindent 8 }}
+      {{- end }}
+      {{- if and .Values.components.zookeeper .Values.bookkeeper.metadata.waitZookeeperTimeout (gt (.Values.bookkeeper.metadata.waitZookeeperTimeout | int) 0) }}
+      - name: wait-zookeeper-ready
+        image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.bookie "root" .) }}"
+        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.bookie "root" .) }}"
+        resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
+        command: ["timeout", "{{ .Values.bookkeeper.metadata.waitZookeeperTimeout }}", "sh", "-c"]
+        args:
+          - |
             {{- if $zk:=.Values.pulsar_metadata.userProvidedZookeepers }}
             export PULSAR_MEM="-Xmx128M";
-            until bin/pulsar zookeeper-shell -server {{ $zk }} ls {{ or .Values.metadataPrefix "/" }}; do
+            until timeout 15 bin/pulsar zookeeper-shell -server {{ $zk }} ls {{ or .Values.metadataPrefix "/" }}; do
               echo "user provided zookeepers {{ $zk }} are unreachable... check in 3 seconds ..." && sleep 3;
             done;
             {{ else }}
@@ -58,35 +79,44 @@ spec:
               sleep 3;
             done;
             {{- end}}
+      {{- end}}
+      {{- if and .Values.components.oxia .Values.bookkeeper.metadata.waitOxiaTimeout (gt (.Values.bookkeeper.metadata.waitOxiaTimeout | int) 0) }}
+      - name: wait-oxia-ready
+        image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.bookie "root" .) }}"
+        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.bookie "root" .) }}"
+        resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
+        command: ["timeout", "{{ .Values.bookkeeper.metadata.waitOxiaTimeout }}", "sh", "-c"]
+        args:
+          - |
+            until nslookup {{ template "pulsar.oxia.server.service" . }}; do
+              sleep 3;
+            done;
+      {{- end }}
       containers:
       - name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-init"
         image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.bookie "root" .) }}"
-        imagePullPolicy: {{ .Values.images.bookie.pullPolicy }}
+        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.bookie "root" .) }}"
       {{- if .Values.bookkeeper.metadata.resources }}
         resources:
 {{ toYaml .Values.bookkeeper.metadata.resources | indent 10 }}
       {{- end }}
-        command: ["sh", "-c"]
+        command: ["timeout", "{{ .Values.bookkeeper.metadata.initTimeout | default 60 }}", "sh", "-c"]
         args:
-          - >
+          - |
             bin/apply-config-from-env.py conf/bookkeeper.conf;
             {{- include "pulsar.toolset.zookeeper.tls.settings" . | nindent 12 }}
             export BOOKIE_MEM="-Xmx128M";
-            if bin/bookkeeper shell whatisinstanceid; then
+            if timeout 15 bin/bookkeeper shell whatisinstanceid; then
                 echo "bookkeeper cluster already initialized";
             else
-                {{- if not (eq .Values.metadataPrefix "") }}
-                bin/bookkeeper org.apache.zookeeper.ZooKeeperMain -server {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }} create {{ .Values.metadataPrefix }} && echo 'created for pulsar cluster "{{ template "pulsar.cluster.name" . }}"' &&
+                {{- if and .Values.components.zookeeper (not (eq .Values.metadataPrefix "")) }}
+                bin/pulsar zookeeper-shell -server {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }} create {{ .Values.metadataPrefix }} && echo 'created for pulsar cluster "{{ template "pulsar.cluster.name" . }}"' &&
                 {{- end }}
                 bin/bookkeeper shell initnewcluster;
             fi
             {{- if .Values.extraInitCommand }}
               {{ .Values.extraInitCommand }}
             {{- end }}
-      {{- if and .Values.rbac.enabled .Values.rbac.psp }}
-        securityContext:
-          readOnlyRootFilesystem: false
-      {{- end }}
         envFrom:
         - configMapRef:
             name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"

charts/pulsar/templates/bookkeeper-podmonitor.yaml

@@ -19,37 +19,5 @@
 
 # deploy bookkeeper PodMonitor only when `$.Values.bookkeeper.podMonitor.enabled` is true
 {{- if $.Values.bookkeeper.podMonitor.enabled }}
-apiVersion: monitoring.coreos.com/v1
-kind: PodMonitor
-metadata:
-  name: {{ template "pulsar.fullname" . }}-bookie
-  labels:
-    app: {{ template "pulsar.name" . }}
-    chart: {{ template "pulsar.chart" . }}
-    release: {{ .Release.Name }}
-    heritage: {{ .Release.Service }}
-spec:
-  jobLabel: bookie
-  podMetricsEndpoints:
-    - port: http
-      path: /metrics
-      scheme: http
-      interval: {{ $.Values.bookkeeper.podMonitor.interval }}
-      scrapeTimeout: {{ $.Values.bookkeeper.podMonitor.scrapeTimeout }}
-      relabelings:
-        - action: labelmap
-          regex: __meta_kubernetes_pod_label_(.+)
-        - sourceLabels: [__meta_kubernetes_namespace]
-          action: replace
-          targetLabel: kubernetes_namespace
-        - sourceLabels: [__meta_kubernetes_pod_label_component]
-          action: replace
-          targetLabel: job
-        - sourceLabels: [__meta_kubernetes_pod_name]
-          action: replace
-          targetLabel: kubernetes_pod_name
-  selector:
-    matchLabels:
-      {{- include "pulsar.matchLabels" . | nindent 6 }}
-      component: bookie
+{{- include "pulsar.podMonitor" (list . "bookkeeper" (printf "component: %s" .Values.bookkeeper.component)) }}
 {{- end }}

charts/pulsar/templates/bookkeeper-rbac.yaml

@@ -1,92 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-{{- if and .Values.rbac.enabled .Values.rbac.psp }}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
-  namespace: {{ template "pulsar.namespace" . }}
-rules:
-  - apiGroups:
-      - policy
-    resourceNames:
-      - "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
-    resources:
-      - podsecuritypolicies
-    verbs:
-      - use
----
-
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
-  namespace: {{ template "pulsar.namespace" . }}
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
-  namespace: {{ template "pulsar.namespace" . }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
-subjects:
-- kind: ServiceAccount
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
-  namespace: {{ template "pulsar.namespace" . }}
----
-
-apiVersion: policy/v1beta1
-kind: PodSecurityPolicy
-metadata:
-{{- if .Values.rbac.limit_to_namespace }}
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ template "pulsar.namespace" . }}"
-{{- else}}
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
-{{- end}}
-spec:
-  readOnlyRootFilesystem: false
-  privileged: false
-  allowPrivilegeEscalation: false
-  runAsUser:
-    rule: 'RunAsAny'
-  supplementalGroups:
-    ranges:
-    - max: 65535
-      min: 1
-    rule: MustRunAs
-  fsGroup:
-    rule: 'MustRunAs'
-    ranges:
-      - min: 1
-        max: 65535
-  seLinux:
-    rule: 'RunAsAny'
-  volumes:
-  - configMap
-  - emptyDir
-  - projected
-  - secret
-  - downwardAPI
-  - persistentVolumeClaim
-  {{- end}}

charts/pulsar/templates/bookkeeper-service-account.yaml

@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+{{- if .Values.components.bookkeeper }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.bookkeeper.component }}
+{{- with .Values.bookkeeper.service_account.annotations }}
+  annotations:
+{{ toYaml . | indent 4 }}
+{{- end }}
+{{- end }}

charts/pulsar/templates/bookkeeper-service.yaml

@@ -26,9 +26,9 @@ metadata:
   labels:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
     component: {{ .Values.bookkeeper.component }}
-{{- if .Values.bookkeeper.service.annotations }}
+{{- with .Values.bookkeeper.service.annotations }}
   annotations:
-{{ toYaml .Values.bookkeeper.service.annotations | indent 4 }}
+{{ toYaml . | indent 4 }}
 {{- end }}
 spec:
   ports:

charts/pulsar/templates/bookkeeper-statefulset.yaml

@@ -23,6 +23,7 @@ kind: StatefulSet
 metadata:
   name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
   namespace: {{ template "pulsar.namespace" . }}
+  annotations: {{ .Values.bookkeeper.appAnnotations | toYaml | nindent 4 }}
   labels:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
     component: {{ .Values.bookkeeper.component }}
@@ -42,8 +43,10 @@ spec:
         {{- include "pulsar.template.labels" . | nindent 8 }}
         component: {{ .Values.bookkeeper.component }}
       annotations:
+        {{- if not .Values.bookkeeper.podMonitor.enabled }}
         prometheus.io/scrape: "true"
         prometheus.io/port: "{{ .Values.bookkeeper.ports.http }}"
+        {{- end }}
         {{- if .Values.bookkeeper.restartPodsOnConfigMapChange }}
         checksum/config: {{ include (print $.Template.BasePath "/bookkeeper-configmap.yaml") . | sha256sum }}
         {{- end }}
@@ -58,11 +61,15 @@ spec:
     {{- if .Values.bookkeeper.tolerations }}
       tolerations:
 {{ toYaml .Values.bookkeeper.tolerations | indent 8 }}
+    {{- end }}
+    {{- if .Values.bookkeeper.topologySpreadConstraints }}
+      topologySpreadConstraints:
+        {{- toYaml .Values.bookkeeper.topologySpreadConstraints | nindent 8 }}
     {{- end }}
       affinity:
         {{- if and .Values.affinity.anti_affinity .Values.bookkeeper.affinity.anti_affinity}}
         podAntiAffinity:
-          {{ if eq .Values.bookkeeper.affinity.type "requiredDuringSchedulingIgnoredDuringExecution"}}
+          {{- if eq .Values.bookkeeper.affinity.type "requiredDuringSchedulingIgnoredDuringExecution"}}
           {{ .Values.bookkeeper.affinity.type }}:
           - labelSelector:
               matchExpressions:
@@ -79,7 +86,7 @@ spec:
                 values:
                 - {{ .Values.bookkeeper.component }}
             topologyKey: {{ .Values.bookkeeper.affinity.anti_affinity_topology_key }}
-        {{ else }}
+        {{- else }}
           {{ .Values.bookkeeper.affinity.type }}:
             - weight: 100
               podAffinityTerm:
@@ -101,37 +108,49 @@ spec:
         {{ end }}
         {{- end }}
       terminationGracePeriodSeconds: {{ .Values.bookkeeper.gracePeriod }}
-    {{- if and .Values.rbac.enabled .Values.rbac.psp }}
       serviceAccountName: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
-    {{- end}}
       {{- if .Values.bookkeeper.securityContext }}
       securityContext:
 {{ toYaml .Values.bookkeeper.securityContext | indent 8 }}
       {{- end }}
+      {{- if and .Values.bookkeeper.waitMetadataTimeout (gt (.Values.bookkeeper.waitMetadataTimeout | int) 0) }}
       initContainers:
+      {{- if .Values.tls.bookie.cacerts.enabled }}
+      - name: cacerts
+        image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.bookie "root" .) }}"
+        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.bookie "root" .) }}"
+        resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
+        command: ["sh", "-c"]
+        args:
+        - |
+          bin/certs-combine-pem.sh /pulsar/certs/cacerts/ca-combined.pem {{ template "pulsar.certs.cacerts" (dict "certs" .Values.tls.bookie.cacerts.certs) }}
+        volumeMounts:
+        {{- include "pulsar.bookkeeper.certs.volumeMounts" . | nindent 8 }}
+      {{- end }}
       # This initContainer will wait for bookkeeper initnewcluster to complete
       # before deploying the bookies
       - name: pulsar-bookkeeper-verify-clusterid
         image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.bookie "root" .) }}"
-        imagePullPolicy: {{ .Values.images.bookie.pullPolicy }}
-        command: ["sh", "-c"]
+        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.bookie "root" .) }}"
+        resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
+        command: ["timeout", "{{ .Values.bookkeeper.waitMetadataTimeout }}", "sh", "-c"]
         args:
         # only reformat bookie if bookkeeper is running without persistence
-        - >
+        - |
           {{- include "pulsar.bookkeeper.init.verify_cluster_id" . | nindent 10 }}
         envFrom:
         - configMapRef:
             name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
-      {{- if and .Values.rbac.enabled .Values.rbac.psp }}
-        securityContext:
-          readOnlyRootFilesystem: false
-      {{- end}}
         volumeMounts:
         {{- include "pulsar.bookkeeper.certs.volumeMounts" . | nindent 8 }}
+      {{- end }}
+      {{- if .Values.bookkeeper.initContainers }}
+        {{- toYaml .Values.bookkeeper.initContainers | nindent 6 }}
+      {{- end }}
       containers:
       - name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
         image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.bookie "root" .) }}"
-        imagePullPolicy: {{ .Values.images.bookie.pullPolicy }}
+        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.bookie "root" .) }}"
         {{- if .Values.bookkeeper.probe.liveness.enabled }}
         livenessProbe:
           httpGet:
@@ -168,17 +187,34 @@ spec:
       {{- end }}
         command: ["sh", "-c"]
         args:
-        - >
+        - |
+          # set required environment variables to use rocksdb config files provided in the Pulsar image
+          export PULSAR_PREFIX_defaultRocksdbConf=${PULSAR_PREFIX_defaultRocksdbConf:-conf/default_rocksdb.conf}
+          export PULSAR_PREFIX_entryLocationRocksdbConf=${PULSAR_PREFIX_entryLocationRocksdbConf:-conf/entry_location_rocksdb.conf}
+          export PULSAR_PREFIX_ledgerMetadataRocksdbConf=${PULSAR_PREFIX_ledgerMetadataRocksdbConf:-conf/ledger_metadata_rocksdb.conf}
+          if [ -x bin/update-rocksdb-conf-from-env.py ] && [ -f "${PULSAR_PREFIX_entryLocationRocksdbConf}" ]; then
+            echo "Updating ${PULSAR_PREFIX_entryLocationRocksdbConf} from environment variables starting with dbStorage_rocksDB_*"
+            bin/update-rocksdb-conf-from-env.py "${PULSAR_PREFIX_entryLocationRocksdbConf}"
+          else
+            # Ensure that Bookkeeper will not load RocksDB config from existing files and fallback to use default RocksDB config
+            # See https://github.com/apache/bookkeeper/pull/3523 as reference
+            export PULSAR_PREFIX_defaultRocksdbConf=conf/non_existing_default_rocksdb.conf
+            export PULSAR_PREFIX_entryLocationRocksdbConf=conf/non_existing_entry_location_rocksdb.conf
+            export PULSAR_PREFIX_ledgerMetadataRocksdbConf=conf/non_existing_ledger_metadata_rocksdb.conf
+            # Ensure that Bookkeeper will use RocksDB format_version 5 (this currently applies only to the entry location rocksdb due to a bug in Bookkeeper)
+            export PULSAR_PREFIX_dbStorage_rocksDB_format_version=${PULSAR_PREFIX_dbStorage_rocksDB_format_version:-5}
+          fi
           {{- if .Values.bookkeeper.additionalCommand }}
           {{ .Values.bookkeeper.additionalCommand }}
           {{- end }}
+          {{- if .Values.tls.bookie.cacerts.enabled }}
+          cd /pulsar/certs/cacerts;
+          nohup /pulsar/bin/certs-combine-pem-infinity.sh /pulsar/certs/cacerts/ca-combined.pem {{ template "pulsar.certs.cacerts" (dict "certs" .Values.tls.bookie.cacerts.certs) }} > /pulsar/certs/cacerts/certs-combine-pem-infinity.log 2>&1 &
+          cd /pulsar;
+          {{- end }}
           bin/apply-config-from-env.py conf/bookkeeper.conf;
           {{- include "pulsar.bookkeeper.zookeeper.tls.settings" . | nindent 10 }}
           OPTS="${OPTS} -Dlog4j2.formatMsgNoLookups=true" exec bin/pulsar bookie;
-      {{- if and .Values.rbac.enabled .Values.rbac.psp }}
-        securityContext:
-          readOnlyRootFilesystem: false
-      {{- end}}
         ports:
         - name: "{{ .Values.tcpPrefix }}bookie"
           containerPort: {{ .Values.bookkeeper.ports.bookie }}
@@ -227,10 +263,10 @@ spec:
         emptyDir: {}
       {{- end }}
       {{- include "pulsar.bookkeeper.certs.volumes" . | nindent 6 }}
-      {{- include "pulsar.imagePullSecrets" . | nindent 6}}
       {{- if .Values.bookkeeper.extraVolumes }}
 {{ toYaml .Values.bookkeeper.extraVolumes | indent 6 }}
       {{- end }}
+      {{- include "pulsar.imagePullSecrets" . | nindent 6}}
 {{- if and (and .Values.persistence .Values.volumes.persistence) .Values.bookkeeper.volumes.persistence}}
   volumeClaimTemplates:
   {{- if .Values.bookkeeper.volumes.useSingleCommonVolume }}

charts/pulsar/templates/broker-cluster-role-binding.yaml

@@ -63,12 +63,22 @@ rules:
   resources:
   - configmaps
   verbs: ["get", "list", "watch"]
-- apiGroups: ["", "extensions", "apps"]
+- apiGroups: [""]
   resources:
     - pods
     - services
-    - deployments
     - secrets
+  verbs:
+    - list
+    - watch
+    - get
+    - update
+    - create
+    - delete
+    - patch
+- apiGroups: ["apps"]
+  resources:
+    - deployments
     - statefulsets
   verbs:
     - list

charts/pulsar/templates/broker-configmap.yaml

@@ -28,27 +28,130 @@ metadata:
     component: {{ .Values.broker.component }}
 data:
   # Metadata settings
-  zookeeperServers: "{{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }}"
+  {{- if .Values.components.zookeeper }}
+  metadataStoreUrl: "zk:{{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }}"
+  {{- $configMetadataStoreUrl := "" }}
   {{- if .Values.pulsar_metadata.configurationStore }}
-  configurationStoreServers: "{{ template "pulsar.configurationStore.connect" . }}{{ .Values.pulsar_metadata.configurationStoreMetadataPrefix }}"
+  {{- $configMetadataStoreUrl = printf "zk:%s%s" (include "pulsar.configurationStore.connect" .) .Values.pulsar_metadata.configurationStoreMetadataPrefix }}
+  {{- else }}
+  {{- $configMetadataStoreUrl = printf "zk:%s%s" (include "pulsar.zookeeper.connect" .) .Values.metadataPrefix }}
   {{- end }}
-  {{- if not .Values.pulsar_metadata.configurationStore }}
-  configurationStoreServers: "{{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }}"
+  configurationMetadataStoreUrl: "{{ $configMetadataStoreUrl }}"
+  {{- if .Values.pulsar_metadata.bookkeeper.usePulsarMetadataClientDriver }}
+  bookkeeperMetadataServiceUri: "metadata-store:{{ $configMetadataStoreUrl }}/ledgers"
+  {{- else }}
+  bookkeeperMetadataServiceUri: "zk+hierarchical://{{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }}/ledgers"
+  {{- end }}
+  {{- end }}
+  {{- if .Values.components.oxia }}
+  metadataStoreUrl: "{{ template "pulsar.oxia.metadata.url.broker" . }}"
+  configurationMetadataStoreUrl: "{{ template "pulsar.oxia.metadata.url.broker" . }}"
+  bookkeeperMetadataServiceUri: "{{ template "pulsar.oxia.metadata.url.bookkeeper" . }}"
+  {{- end }}
+
+  {{- if hasKey .Values.pulsar_metadata "metadataStoreAllowReadOnlyOperations" }}
+  PULSAR_PREFIX_metadataStoreAllowReadOnlyOperations: "{{ .Values.pulsar_metadata.metadataStoreAllowReadOnlyOperations }}"
+  {{- end }}
+  {{- if hasKey .Values.pulsar_metadata "metadataStoreSessionTimeoutMillis" }}
+  metadataStoreSessionTimeoutMillis: "{{ .Values.pulsar_metadata.metadataStoreSessionTimeoutMillis }}"
+  {{- end }}
+  {{- if hasKey .Values.pulsar_metadata "metadataStoreOperationTimeoutSeconds" }}
+  metadataStoreOperationTimeoutSeconds: "{{ .Values.pulsar_metadata.metadataStoreOperationTimeoutSeconds }}"
+  {{- end }}
+  {{- if hasKey .Values.pulsar_metadata "metadataStoreCacheExpirySeconds" }}
+  metadataStoreCacheExpirySeconds: "{{ .Values.pulsar_metadata.metadataStoreCacheExpirySeconds }}"
+  {{- end }}
+  {{- if hasKey .Values.pulsar_metadata "metadataStoreBatchingEnabled" }}
+  metadataStoreBatchingEnabled: "{{ .Values.pulsar_metadata.metadataStoreBatchingEnabled }}"
+  {{- end }}
+  {{- if hasKey .Values.pulsar_metadata "metadataStoreBatchingMaxDelayMillis" }}
+  metadataStoreBatchingMaxDelayMillis: "{{ .Values.pulsar_metadata.metadataStoreBatchingMaxDelayMillis }}"
+  {{- end }}
+  {{- if hasKey .Values.pulsar_metadata "metadataStoreBatchingMaxOperations" }}
+  metadataStoreBatchingMaxOperations: "{{ .Values.pulsar_metadata.metadataStoreBatchingMaxOperations }}"
+  {{- end }}
+  {{- if hasKey .Values.pulsar_metadata "metadataStoreBatchingMaxSizeKb" }}
+  metadataStoreBatchingMaxSizeKb: "{{ .Values.pulsar_metadata.metadataStoreBatchingMaxSizeKb }}"
   {{- end }}
 
   # Broker settings
   clusterName: {{ template "pulsar.cluster.name" . }}
+  
+  # Enable all metrics by default
   exposeTopicLevelMetricsInPrometheus: "true"
+  exposeConsumerLevelMetricsInPrometheus: "true"
+  exposeProducerLevelMetricsInPrometheus: "true"
+  exposeManagedLedgerMetricsInPrometheus: "true"
+  exposeManagedCursorMetricsInPrometheus: "true"
+  exposeBundlesMetricsInPrometheus: "true"
+  exposePublisherStats: "true"
+  exposePreciseBacklogInPrometheus: "true"
+  replicationMetricsEnabled: "true"
+  splitTopicAndPartitionLabelInPrometheus: "true"
+  aggregatePublisherStatsByProducerName: "true"
+  bookkeeperClientExposeStatsToPrometheus: "true"
+
   numHttpServerThreads: "8"
-  zooKeeperSessionTimeoutMillis: "30000"
-  statusFilePath: "{{ template "pulsar.home" . }}/status"
+  statusFilePath: "{{ template "pulsar.home" . }}/logs/status"
+
+  # Tiered storage settings
+  {{- if .Values.broker.storageOffload.driver }}
+  {{- if eq .Values.broker.storageOffload.driver "aws-s3" }}
+  managedLedgerOffloadDriver: "{{ .Values.broker.storageOffload.driver }}" 
+  s3ManagedLedgerOffloadBucket: "{{ .Values.broker.storageOffload.bucket }}" 
+  s3ManagedLedgerOffloadRegion: "{{ .Values.broker.storageOffload.region }}" 
+  {{- if .Values.broker.storageOffload.managedLedgerOffloadAutoTriggerSizeThresholdBytes }}
+  PULSAR_PREFIX_managedLedgerOffloadThresholdInBytes: "{{ .Values.broker.storageOffload.managedLedgerOffloadAutoTriggerSizeThresholdBytes }}" 
+  {{- end }} 
+  {{- if .Values.broker.storageOffload.managedLedgerOffloadDeletionLagMs }}
+  PULSAR_PREFIX_managedLedgerOffloadDeletionLagInMillis: "{{ .Values.broker.storageOffload.managedLedgerOffloadDeletionLagMs }}" 
+  {{- end }}
+  {{- if .Values.broker.storageOffload.maxBlockSizeInBytes }}
+  s3ManagedLedgerOffloadMaxBlockSizeInBytes: "{{ .Values.broker.storageOffload.maxBlockSizeInBytes }}" 
+  {{- end }}
+  {{- if .Values.broker.storageOffload.readBufferSizeInBytes }}
+  s3ManagedLedgerOffloadReadBufferSizeInBytes: "{{ .Values.broker.storageOffload.readBufferSizeInBytes }}" 
+  {{- end }}
+  {{- end }}
+  {{- if eq .Values.broker.storageOffload.driver "google-cloud-storage" }}
+  managedLedgerOffloadDriver: "{{ .Values.broker.storageOffload.driver }}" 
+  gcsManagedLedgerOffloadBucket: "{{ .Values.broker.storageOffload.bucket }}"
+  gcsManagedLedgerOffloadRegion: "{{ .Values.broker.storageOffload.region }}"
+  gcsManagedLedgerOffloadServiceAccountKeyFile: "/pulsar/gcp-service-account/{{ .Values.broker.storageOffload.gcsServiceAccountJsonFile }}"
+  {{- if .Values.broker.storageOffload.managedLedgerOffloadAutoTriggerSizeThresholdBytes }}
+  PULSAR_PREFIX_managedLedgerOffloadThresholdInBytes: "{{ .Values.broker.storageOffload.managedLedgerOffloadAutoTriggerSizeThresholdBytes }}" 
+  {{- end }} 
+  {{- if .Values.broker.storageOffload.managedLedgerOffloadDeletionLagMs }}
+  PULSAR_PREFIX_managedLedgerOffloadDeletionLagInMillis: "{{ .Values.broker.storageOffload.managedLedgerOffloadDeletionLagMs }}" 
+  {{- end }}
+  {{- if .Values.broker.storageOffload.maxBlockSizeInBytes }}
+  gcsManagedLedgerOffloadMaxBlockSizeInBytes: "{{ .Values.broker.storageOffload.maxBlockSizeInBytes }}" 
+  {{- end }}
+  {{- if .Values.broker.storageOffload.readBufferSizeInBytes }}
+  gcsManagedLedgerOffloadReadBufferSizeInBytes: "{{ .Values.broker.storageOffload.readBufferSizeInBytes }}" 
+  {{- end }}
+  {{- end }}
+  {{- if eq .Values.broker.storageOffload.driver "azureblob" }}
+  managedLedgerOffloadDriver: "{{ .Values.broker.storageOffload.driver }}" 
+  managedLedgerOffloadBucket: "{{ .Values.broker.storageOffload.bucket }}"
+  {{- if .Values.broker.storageOffload.managedLedgerOffloadAutoTriggerSizeThresholdBytes }}
+  PULSAR_PREFIX_managedLedgerOffloadThresholdInBytes: "{{ .Values.broker.storageOffload.managedLedgerOffloadAutoTriggerSizeThresholdBytes }}" 
+  {{- end }} 
+  {{- if .Values.broker.storageOffload.managedLedgerOffloadDeletionLagMs }}
+  PULSAR_PREFIX_managedLedgerOffloadDeletionLagInMillis: "{{ .Values.broker.storageOffload.managedLedgerOffloadDeletionLagMs }}" 
+  {{- end }}
+  {{- if .Values.broker.storageOffload.maxBlockSizeInBytes }}
+  managedLedgerOffloadMaxBlockSizeInBytes: "{{ .Values.broker.storageOffload.maxBlockSizeInBytes }}" 
+  {{- end }}
+  {{- end }}
+  {{- end }}
 
   # Function Worker Settings
   # function worker configuration
-  {{- if not (or .Values.components.functions .Values.extra.functionsAsPods) }}
+  {{- if not .Values.components.functions }}
   functionsWorkerEnabled: "false"
   {{- end }}
-  {{- if or .Values.components.functions .Values.extra.functionsAsPods }}
+  {{- if .Values.components.functions }}
   functionsWorkerEnabled: "true"
   {{- if .Values.functions.useBookieAsStateStore }}
   PF_stateStorageServiceUrl: "bk://{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}:{{ .Values.bookkeeper.ports.statestore }}"
@@ -62,36 +165,32 @@ data:
   PF_functionRuntimeFactoryConfigs_pulsarRootDir: {{ template "pulsar.home" . }}
   PF_kubernetesContainerFactory_pulsarRootDir: {{ template "pulsar.home" . }}
   PF_functionRuntimeFactoryConfigs_pulsarDockerImageName: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.functions "root" .) }}"
+  PF_functionRuntimeFactoryConfigs_imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.functions "root" .) }}"
   PF_functionRuntimeFactoryConfigs_submittingInsidePod: "true"
   PF_functionRuntimeFactoryConfigs_installUserCodeDependencies: "true"
   PF_functionRuntimeFactoryConfigs_jobNamespace: {{ template "pulsar.namespace" . }}
   PF_functionRuntimeFactoryConfigs_expectedMetricsCollectionInterval: "30"
-  {{- if not (and .Values.tls.enabled .Values.tls.broker.enabled) }}
+  {{- if not (and .Values.tls.enabled .Values.tls.broker.enabled .Values.tls.function_instance.enabled) }}
   PF_functionRuntimeFactoryConfigs_pulsarAdminUrl: "http://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.http }}/"
   PF_functionRuntimeFactoryConfigs_pulsarServiceUrl: "pulsar://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.pulsar }}/"
-  {{- end }}
-  {{- if and .Values.tls.enabled .Values.tls.broker.enabled }}
+  {{- else }}
   PF_functionRuntimeFactoryConfigs_pulsarAdminUrl: "https://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.https }}/"
   PF_functionRuntimeFactoryConfigs_pulsarServiceUrl: "pulsar+ssl://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.pulsarssl }}/"
   {{- end }}
-  PF_functionRuntimeFactoryConfigs_changeConfigMap: "{{ template "pulsar.fullname" . }}-{{ .Values.functions.component }}-config"
-  PF_functionRuntimeFactoryConfigs_changeConfigMapNamespace: {{ template "pulsar.namespace" . }}
   # support version < 2.5.0
   PF_kubernetesContainerFactory_pulsarDockerImageName: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.functions "root" .) }}"
+  PF_kubernetesContainerFactory_imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.functions "root" .) }}"
   PF_kubernetesContainerFactory_submittingInsidePod: "true"
   PF_kubernetesContainerFactory_installUserCodeDependencies: "true"
   PF_kubernetesContainerFactory_jobNamespace: {{ template "pulsar.namespace" . }}
   PF_kubernetesContainerFactory_expectedMetricsCollectionInterval: "30"
-  {{- if not (and .Values.tls.enabled .Values.tls.broker.enabled) }}
+  {{- if not (and .Values.tls.enabled .Values.tls.broker.enabled .Values.tls.function_instance.enabled) }}
   PF_kubernetesContainerFactory_pulsarAdminUrl: "http://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.http }}/"
   PF_kubernetesContainerFactory_pulsarServiceUrl: "pulsar://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.pulsar }}/"
-  {{- end }}
-  {{- if and .Values.tls.enabled .Values.tls.broker.enabled }}
+  {{- else }}
   PF_kubernetesContainerFactory_pulsarAdminUrl: "https://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.https }}/"
   PF_kubernetesContainerFactory_pulsarServiceUrl: "pulsar+ssl://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.pulsarssl }}/"
   {{- end }}
-  PF_kubernetesContainerFactory_changeConfigMap: "{{ template "pulsar.fullname" . }}-{{ .Values.functions.component }}-config"
-  PF_kubernetesContainerFactory_changeConfigMapNamespace: {{ template "pulsar.namespace" . }}
   {{- end }}
 
   # prometheus needs to access /metrics endpoint
@@ -105,7 +204,7 @@ data:
   # TLS Settings
   tlsCertificateFilePath: "/pulsar/certs/broker/tls.crt"
   tlsKeyFilePath: "/pulsar/certs/broker/tls.key"
-  tlsTrustCertsFilePath: "/pulsar/certs/ca/ca.crt"
+  tlsTrustCertsFilePath: {{ ternary "/pulsar/certs/cacerts/ca-combined.pem" "/pulsar/certs/ca/ca.crt" .Values.tls.broker.cacerts.enabled | quote }}
   {{- end }}
 
   # Authentication Settings
@@ -113,11 +212,19 @@ data:
   authenticationEnabled: "true"
   {{- if .Values.auth.authorization.enabled }}
   authorizationEnabled: "true"
-  superUserRoles: {{ .Values.auth.superUsers | values | sortAlpha | join "," }}
+  superUserRoles: {{ .Values.auth.superUsers | values | compact | sortAlpha | join "," }}
+  {{- if .Values.auth.useProxyRoles }}
+  proxyRoles: {{ .Values.auth.superUsers.proxy }}
   {{- end }}
-  {{- if eq .Values.auth.authentication.provider "jwt" }}
+  {{- end }}
+  {{- if and .Values.auth.authentication.enabled .Values.auth.authentication.jwt.enabled  }}
   # token authentication configuration
+  {{- if and .Values.auth.authentication.enabled .Values.auth.authentication.jwt.enabled .Values.auth.authentication.openid.enabled }}
+  authenticationProviders: "org.apache.pulsar.broker.authentication.AuthenticationProviderToken,org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID"
+  {{- end }}
+  {{- if and .Values.auth.authentication.enabled .Values.auth.authentication.jwt.enabled ( not .Values.auth.authentication.openid.enabled ) }}
   authenticationProviders: "org.apache.pulsar.broker.authentication.AuthenticationProviderToken"
+  {{- end }}
   brokerClientAuthenticationParameters: "file:///pulsar/tokens/broker/token"
   brokerClientAuthenticationPlugin: "org.apache.pulsar.client.impl.auth.AuthenticationToken"
   {{- if .Values.auth.authentication.jwt.usingSecretKey }}
@@ -126,6 +233,25 @@ data:
   tokenPublicKey: "file:///pulsar/keys/token/public.key"
   {{- end }}
   {{- end }}
+  {{- if and .Values.auth.authentication.enabled .Values.auth.authentication.openid.enabled  }}
+  # openid authentication configuration
+  {{- if and .Values.auth.authentication.enabled .Values.auth.authentication.openid.enabled ( not .Values.auth.authentication.jwt.enabled ) }}
+  authenticationProviders: "org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID"
+  {{- end }}
+  PULSAR_PREFIX_openIDAllowedTokenIssuers: {{ .Values.auth.authentication.openid.openIDAllowedTokenIssuers | uniq | compact | sortAlpha | join "," | quote }}
+  PULSAR_PREFIX_openIDAllowedAudiences: {{ .Values.auth.authentication.openid.openIDAllowedAudiences | uniq | compact | sortAlpha | join "," | quote }}
+  PULSAR_PREFIX_openIDTokenIssuerTrustCertsFilePath: {{ .Values.auth.authentication.openid.openIDTokenIssuerTrustCertsFilePath | quote }}
+  PULSAR_PREFIX_openIDRoleClaim: {{  .Values.auth.authentication.openid.openIDRoleClaim | quote }}
+  PULSAR_PREFIX_openIDAcceptedTimeLeewaySeconds: {{ .Values.auth.authentication.openid.openIDAcceptedTimeLeewaySeconds | quote }}
+  PULSAR_PREFIX_openIDCacheSize: {{ .Values.auth.authentication.openid.openIDCacheSize | quote }}
+  PULSAR_PREFIX_openIDCacheRefreshAfterWriteSeconds: {{ .Values.auth.authentication.openid.openIDCacheRefreshAfterWriteSeconds | quote }}
+  PULSAR_PREFIX_openIDCacheExpirationSeconds: {{ .Values.auth.authentication.openid.openIDCacheExpirationSeconds | quote }}
+  PULSAR_PREFIX_openIDHttpConnectionTimeoutMillis: {{ .Values.auth.authentication.openid.openIDHttpConnectionTimeoutMillis | quote }}
+  PULSAR_PREFIX_openIDHttpReadTimeoutMillis: {{ .Values.auth.authentication.openid.openIDHttpReadTimeoutMillis | quote }}
+  PULSAR_PREFIX_openIDKeyIdCacheMissRefreshSeconds: {{ .Values.auth.authentication.openid.openIDKeyIdCacheMissRefreshSeconds | quote }}
+  PULSAR_PREFIX_openIDRequireIssuersUseHttps: {{ .Values.auth.authentication.openid.openIDRequireIssuersUseHttps | quote }}
+  PULSAR_PREFIX_openIDFallbackDiscoveryMode: {{ .Values.auth.authentication.openid.openIDFallbackDiscoveryMode | quote }}
+  {{- end }}
   {{- end }}
 
   {{- if and .Values.tls.enabled .Values.tls.bookie.enabled }}
@@ -134,13 +260,13 @@ data:
   bookkeeperTLSKeyFileType: "PEM"
   bookkeeperTLSKeyFilePath: "/pulsar/certs/broker/tls.key"
   bookkeeperTLSCertificateFilePath: "/pulsar/certs/broker/tls.crt"
-  bookkeeperTLSTrustCertsFilePath: "/pulsar/certs/ca/ca.crt"
+  bookkeeperTLSTrustCertsFilePath: {{ ternary "/pulsar/certs/cacerts/ca-combined.pem" "/pulsar/certs/ca/ca.crt" .Values.tls.broker.cacerts.enabled | quote }}
   bookkeeperTLSTrustCertTypes: "PEM"
   PULSAR_PREFIX_bookkeeperTLSClientAuthentication: "true"
   PULSAR_PREFIX_bookkeeperTLSKeyFileType: "PEM"
   PULSAR_PREFIX_bookkeeperTLSKeyFilePath: "/pulsar/certs/broker/tls.key"
   PULSAR_PREFIX_bookkeeperTLSCertificateFilePath: "/pulsar/certs/broker/tls.crt"
-  PULSAR_PREFIX_bookkeeperTLSTrustCertsFilePath: "/pulsar/certs/ca/ca.crt"
+  PULSAR_PREFIX_bookkeeperTLSTrustCertsFilePath: {{ ternary "/pulsar/certs/cacerts/ca-combined.pem" "/pulsar/certs/ca/ca.crt" .Values.tls.broker.cacerts.enabled | quote }}
   PULSAR_PREFIX_bookkeeperTLSTrustCertTypes: "PEM"
   # https://github.com/apache/bookkeeper/pull/2300
   bookkeeperUseV2WireProtocol: "false"

charts/pulsar/templates/broker-hpa.yaml

@@ -26,6 +26,7 @@ apiVersion: autoscaling/v2
 kind: HorizontalPodAutoscaler
 metadata:
   name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}"
+  namespace: {{ template "pulsar.namespace" . }}
 spec:
   maxReplicas: {{ .Values.broker.autoscaling.maxReplicas }}
   {{- with .Values.broker.autoscaling.metrics }}

charts/pulsar/templates/broker-podmonitor.yaml

@@ -19,37 +19,5 @@
 
 # deploy broker PodMonitor only when `$.Values.broker.podMonitor.enabled` is true
 {{- if $.Values.broker.podMonitor.enabled }}
-apiVersion: monitoring.coreos.com/v1
-kind: PodMonitor
-metadata:
-  name: {{ template "pulsar.fullname" . }}-broker
-  labels:
-    app: {{ template "pulsar.name" . }}
-    chart: {{ template "pulsar.chart" . }}
-    release: {{ .Release.Name }}
-    heritage: {{ .Release.Service }}
-spec:
-  jobLabel: broker
-  podMetricsEndpoints:
-    - port: http
-      path: /metrics
-      scheme: http
-      interval: {{ $.Values.broker.podMonitor.interval }}
-      scrapeTimeout: {{ $.Values.broker.podMonitor.scrapeTimeout }}
-      relabelings:
-        - action: labelmap
-          regex: __meta_kubernetes_pod_label_(.+)
-        - sourceLabels: [__meta_kubernetes_namespace]
-          action: replace
-          targetLabel: kubernetes_namespace
-        - sourceLabels: [__meta_kubernetes_pod_label_component]
-          action: replace
-          targetLabel: job
-        - sourceLabels: [__meta_kubernetes_pod_name]
-          action: replace
-          targetLabel: kubernetes_pod_name
-  selector:
-    matchLabels:
-      {{- include "pulsar.matchLabels" . | nindent 6 }}
-      component: broker
+{{- include "pulsar.podMonitor" (list . "broker" (printf "component: %s" .Values.broker.component)) }}
 {{- end }}

charts/pulsar/templates/broker-rbac.yaml

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-{{- if or .Values.components.functions .Values.extra.functionsAsPods }}
+{{- if .Values.components.functions }}
 apiVersion: rbac.authorization.k8s.io/v1
 {{- if .Values.functions.rbac.limit_to_namespace }}
 kind: Role
@@ -44,13 +44,6 @@ rules:
   - '*'
 ---
 
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.functions.component }}"
-  namespace: {{ template "pulsar.namespace" . }}
----
-
 apiVersion: rbac.authorization.k8s.io/v1
 {{- if .Values.functions.rbac.limit_to_namespace }}
 kind: RoleBinding
@@ -75,73 +68,3 @@ subjects:
   name: "{{ template "pulsar.fullname" . }}-{{ .Values.functions.component }}"
   namespace: {{ template "pulsar.namespace" . }}
 {{- end }}
-
-{{- if and .Values.rbac.enabled .Values.rbac.psp }}
----
-
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}-psp"
-  namespace: {{ template "pulsar.namespace" . }}
-rules:
-  - apiGroups:
-      - policy
-    resourceNames:
-      - "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}"
-    resources:
-      - podsecuritypolicies
-    verbs:
-      - use
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}-psp"
-  namespace: {{ template "pulsar.namespace" . }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}-psp"
-subjects:
-- kind: ServiceAccount
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}-acct"
-  namespace: {{ template "pulsar.namespace" . }}
----
-
-apiVersion: policy/v1beta1
-kind: PodSecurityPolicy
-metadata:
-{{- if .Values.rbac.limit_to_namespace }}
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}-{{ template "pulsar.namespace" . }}"
-{{- else}}
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}"
-{{- end}}
-spec:
-  readOnlyRootFilesystem: false
-  privileged: false
-  allowPrivilegeEscalation: false
-  runAsUser:
-    rule: 'RunAsAny'
-  supplementalGroups:
-    ranges:
-    - max: 65535
-      min: 1
-    rule: MustRunAs
-  fsGroup:
-    rule: 'MustRunAs'
-    ranges:
-      - min: 1
-        max: 65535
-  seLinux:
-    rule: 'RunAsAny'
-  volumes:
-  - configMap
-  - emptyDir
-  - projected
-  - secret
-  - downwardAPI
-  - persistentVolumeClaim
-{{- end}}

charts/pulsar/templates/broker-service-account.yaml

@@ -26,8 +26,25 @@ metadata:
   labels:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
     component: {{ .Values.broker.component }}
-  annotations:
 {{- with .Values.broker.service_account.annotations }}
+  annotations:
 {{ toYaml . | indent 4 }}
 {{- end }}
+---
+{{- end }}
+
+{{- if .Values.components.functions }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: "{{ template "pulsar.fullname" . }}-{{ .Values.functions.component }}"
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.functions.component }}
+{{- with .Values.functions.service_account.annotations }}
+  annotations:
+{{ toYaml . | indent 4 }}
+{{- end }}
+---
 {{- end }}

charts/pulsar/templates/broker-service.yaml

@@ -26,9 +26,12 @@ metadata:
   labels:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
     component: {{ .Values.broker.component }}
+{{- with .Values.broker.service.annotations }}
   annotations:
-{{ toYaml .Values.broker.service.annotations | indent 4 }}
+{{ toYaml . | indent 4 }}
+{{- end }}
 spec:
+  type: ClusterIP
   ports:
   # prometheus needs to access /metrics endpoint
   - name: http
@@ -43,7 +46,7 @@ spec:
   - name: "{{ .Values.tlsPrefix }}pulsarssl"
     port: {{ .Values.broker.ports.pulsarssl }}
   {{- end }}
-  clusterIP: None
+  clusterIP: "{{ .Values.broker.service.clusterIP }}"
   selector:
     {{- include "pulsar.matchLabels" . | nindent 4 }}
     component: {{ .Values.broker.component }}

charts/pulsar/templates/broker-statefulset.yaml

@@ -21,8 +21,11 @@
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}"
-  namespace: {{ template "pulsar.namespace" . }}
+  {{- $stsName := printf "%s-%s" (include "pulsar.fullname" .) .Values.broker.component }}
+  name: {{ $stsName | quote }}
+  {{- $namespace := include "pulsar.namespace" . }}
+  namespace: {{ $namespace | quote }}
+  annotations: {{ .Values.broker.appAnnotations | toYaml | nindent 4 }}
   labels:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
     component: {{ .Values.broker.component }}
@@ -37,15 +40,33 @@ spec:
       component: {{ .Values.broker.component }}
   updateStrategy:
     type: RollingUpdate
+  {{- /*
+  When functions are enabled, podManagementPolicy must be OrderedReady to ensure that other started brokers are available via DNS
+  for the function worker to connect to.
+  Since podManagementPolicy is immutable, this rule is only applied when the broker is first installed.
+  */}}
+  {{- $stsObj := lookup "apps/v1" "StatefulSet" $namespace $stsName }}
+  {{- if $stsObj }}
+  podManagementPolicy: {{ $stsObj.spec.podManagementPolicy }}
+  {{- else }}
+  {{- if .Values.broker.podManagementPolicy }}
+  podManagementPolicy: {{ .Values.broker.podManagementPolicy }}
+  {{- else if not .Values.components.functions }}
   podManagementPolicy: Parallel
+  {{- else }}
+  podManagementPolicy: OrderedReady
+  {{- end }}
+  {{- end }}
   template:
     metadata:
       labels:
         {{- include "pulsar.template.labels" . | nindent 8 }}
         component: {{ .Values.broker.component }}
       annotations:
+        {{- if not .Values.broker.podMonitor.enabled }}
         prometheus.io/scrape: "true"
         prometheus.io/port: "{{ .Values.broker.ports.http }}"
+        {{- end }}
         {{- if .Values.broker.restartPodsOnConfigMapChange }}
         checksum/config: {{ include (print $.Template.BasePath "/broker-configmap.yaml") . | sha256sum }}
         {{- end }}
@@ -61,11 +82,15 @@ spec:
     {{- if .Values.broker.tolerations }}
       tolerations:
 {{ toYaml .Values.broker.tolerations | indent 8 }}
+    {{- end }}
+    {{- if .Values.broker.topologySpreadConstraints }}
+      topologySpreadConstraints:
+        {{- toYaml .Values.broker.topologySpreadConstraints | nindent 8 }}
     {{- end }}
       affinity:
         {{- if and .Values.affinity.anti_affinity .Values.broker.affinity.anti_affinity}}
         podAntiAffinity:
-          {{ if eq .Values.broker.affinity.type "requiredDuringSchedulingIgnoredDuringExecution"}}
+          {{- if eq .Values.broker.affinity.type "requiredDuringSchedulingIgnoredDuringExecution"}}
           {{ .Values.broker.affinity.type }}:
           - labelSelector:
               matchExpressions:
@@ -82,7 +107,7 @@ spec:
                 values:
                 - {{ .Values.broker.component }}
             topologyKey: {{ .Values.broker.affinity.anti_affinity_topology_key }}
-        {{ else }}
+        {{- else }}
           {{ .Values.broker.affinity.type }}:
             - weight: 100
               podAffinityTerm:
@@ -101,46 +126,71 @@ spec:
                       values:
                       - {{ .Values.broker.component }}
                 topologyKey: {{ .Values.broker.affinity.anti_affinity_topology_key }}
-        {{ end }}
+        {{- end }}
         {{- end }}
       terminationGracePeriodSeconds: {{ .Values.broker.gracePeriod }}
       initContainers:
+      {{- if .Values.tls.broker.cacerts.enabled }}
+      - name: cacerts
+        image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.broker "root" .) }}"
+        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.broker "root" .) }}"
+        resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
+        command: ["sh", "-c"]
+        args:
+          - |
+            bin/certs-combine-pem.sh /pulsar/certs/cacerts/ca-combined.pem {{ template "pulsar.certs.cacerts" (dict "certs" .Values.tls.broker.cacerts.certs) }}
+        volumeMounts:
+        {{- include "pulsar.broker.certs.volumeMounts" . | nindent 8 }}
+      {{- end }}
+      {{- if and .Values.components.zookeeper .Values.broker.waitZookeeperTimeout (gt (.Values.broker.waitZookeeperTimeout | int) 0) }}
       # This init container will wait for zookeeper to be ready before
       # deploying the bookies
       - name: wait-zookeeper-ready
         image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.broker "root" .) }}"
-        imagePullPolicy: {{ .Values.images.broker.pullPolicy }}
-        command: ["sh", "-c"]
+        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.broker "root" .) }}"
+        resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
+        command: ["timeout", "{{ .Values.broker.waitZookeeperTimeout }}", "sh", "-c"]
         args:
-          - >-
+          - |
             {{- include "pulsar.broker.zookeeper.tls.settings" . | nindent 12 }}
-            export BOOKIE_MEM="-Xmx128M";
+            export PULSAR_MEM="-Xmx128M";
             {{- if .Values.pulsar_metadata.configurationStore }}
-            until bin/bookkeeper org.apache.zookeeper.ZooKeeperMain -server {{ template "pulsar.configurationStore.connect" . }} get {{ .Values.configurationStoreMetadataPrefix }}/admin/clusters/{{ template "pulsar.cluster.name" . }}; do
+            until timeout 15 bin/pulsar zookeeper-shell -server {{ template "pulsar.configurationStore.connect" . }} get {{ .Values.pulsar_metadata.configurationStoreMetadataPrefix }}/admin/clusters/{{ template "pulsar.cluster.name" . }}; do
             {{- end }}
             {{- if not .Values.pulsar_metadata.configurationStore }}
-            until bin/bookkeeper org.apache.zookeeper.ZooKeeperMain -server {{ template "pulsar.zookeeper.connect" . }} get {{ .Values.metadataPrefix }}/admin/clusters/{{ template "pulsar.cluster.name" . }}; do
+            until timeout 15 bin/pulsar zookeeper-shell -server {{ template "pulsar.zookeeper.connect" . }} get {{ .Values.metadataPrefix }}/admin/clusters/{{ template "pulsar.cluster.name" . }}; do
             {{- end }}
               echo "pulsar cluster {{ template "pulsar.cluster.name" . }} isn't initialized yet ... check in 3 seconds ..." && sleep 3;
             done;
-      {{- if and .Values.rbac.enabled .Values.rbac.psp }}
-        securityContext:
-          readOnlyRootFilesystem: false
-      {{- end }}
         volumeMounts:
         {{- include "pulsar.broker.certs.volumeMounts" . | nindent 8 }}
+      {{- end }}
+      {{- if and .Values.components.oxia .Values.broker.waitOxiaTimeout (gt (.Values.broker.waitOxiaTimeout | int) 0) }}
+      - name: wait-oxia-ready
+        image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.broker "root" .) }}"
+        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.broker "root" .) }}"
+        resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
+        command: ["timeout", "{{ .Values.broker.waitOxiaTimeout }}", "sh", "-c"]
+        args:
+          - |
+            until nslookup {{ template "pulsar.oxia.server.service" . }}; do
+              sleep 3;
+            done;
+      {{- end }}
+      {{- if and .Values.broker.waitBookkeeperTimeout (gt (.Values.broker.waitBookkeeperTimeout | int) 0) }}
       # This init container will wait for bookkeeper to be ready before
       # deploying the broker
       - name: wait-bookkeeper-ready
         image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.broker "root" .) }}"
-        imagePullPolicy: {{ .Values.images.broker.pullPolicy }}
-        command: ["sh", "-c"]
+        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.broker "root" .) }}"
+        resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
+        command: ["timeout", "{{ .Values.broker.waitBookkeeperTimeout }}", "sh", "-c"]
         args:
-          - >
+          - |
             {{- include "pulsar.broker.zookeeper.tls.settings" . | nindent 12 }}
             bin/apply-config-from-env.py conf/bookkeeper.conf;
             export BOOKIE_MEM="-Xmx128M";
-            until bin/bookkeeper shell whatisinstanceid; do
+            until timeout 15 bin/bookkeeper shell whatisinstanceid; do
               echo "bookkeeper cluster is not initialized yet. backoff for 3 seconds ...";
               sleep 3;
             done;
@@ -155,16 +205,16 @@ spec:
         envFrom:
           - configMapRef:
               name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
-      {{- if and .Values.rbac.enabled .Values.rbac.psp }}
-        securityContext:
-          readOnlyRootFilesystem: false
-      {{- end }}
         volumeMounts:
         {{- include "pulsar.broker.certs.volumeMounts" . | nindent 10 }}
+      {{- end }}
+      {{- if .Values.broker.initContainers }}
+        {{- toYaml .Values.broker.initContainers | nindent 6 }}
+      {{- end }}
       containers:
       - name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}"
         image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.broker "root" .) }}"
-        imagePullPolicy: {{ .Values.images.broker.pullPolicy }}
+        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.broker "root" .) }}"
         {{- if .Values.broker.probe.liveness.enabled }}
         livenessProbe:
           httpGet:
@@ -201,20 +251,27 @@ spec:
       {{- end }}
         command: ["sh", "-c"]
         args:
-        - >
+        - |
         {{- if .Values.broker.additionalCommand }}
           {{ .Values.broker.additionalCommand }}
+        {{- end }}
+          {{- if .Values.tls.broker.cacerts.enabled }}
+          cd /pulsar/certs/cacerts;
+          nohup /pulsar/bin/certs-combine-pem-infinity.sh /pulsar/certs/cacerts/ca-combined.pem {{ template "pulsar.certs.cacerts" (dict "certs" .Values.tls.broker.cacerts.certs) }} > /pulsar/certs/cacerts/certs-combine-pem-infinity.log 2>&1 &
+          cd /pulsar;
           {{- end }}
           bin/apply-config-from-env.py conf/broker.conf;
           bin/gen-yml-from-env.py conf/functions_worker.yml;
-          echo "OK" > status;
+          echo "OK" > "${statusFilePath:-status}";
+          {{- if .Values.components.zookeeper }}
           {{- include "pulsar.broker.zookeeper.tls.settings" . | nindent 10 }}
-          bin/pulsar zookeeper-shell -server {{ template "pulsar.zookeeper.connect" . }} get {{ template "pulsar.broker.znode" . }};
+          timeout 15 bin/pulsar zookeeper-shell -server {{ template "pulsar.zookeeper.connect" . }} get {{ template "pulsar.broker.znode" . }};
           while [ $? -eq 0 ]; do
             echo "broker {{ template "pulsar.broker.hostname" . }} znode still exists ... check in 10 seconds ...";
             sleep 10;
-            bin/pulsar zookeeper-shell -server {{ template "pulsar.zookeeper.connect" . }} get {{ template "pulsar.broker.znode" . }};
+            timeout 15 bin/pulsar zookeeper-shell -server {{ template "pulsar.zookeeper.connect" . }} get {{ template "pulsar.broker.znode" . }};
           done;
+          {{- end }}
           cat conf/pulsar_env.sh;
           OPTS="${OPTS} -Dlog4j2.formatMsgNoLookups=true" exec bin/pulsar broker;
         ports:
@@ -231,16 +288,12 @@ spec:
         - name: "{{ .Values.tlsPrefix }}pulsarssl"
           containerPort: {{ .Values.broker.ports.pulsarssl }}
         {{- end }}
-{{- if .Values.broker.extreEnvs }}
-        env:
-{{ toYaml .Values.broker.extreEnvs | indent 8 }}
-{{- end }}
         envFrom:
         - configMapRef:
             name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}"
         volumeMounts:
           {{- if .Values.auth.authentication.enabled }}
-          {{- if eq .Values.auth.authentication.provider "jwt" }}
+          {{- if .Values.auth.authentication.jwt.enabled }}
           - mountPath: "/pulsar/keys"
             name: token-keys
             readOnly: true
@@ -249,20 +302,51 @@ spec:
             readOnly: true
           {{- end }}
           {{- end }}
+          {{- if .Values.broker.storageOffload.driver }}
+          {{- if eq .Values.broker.storageOffload.driver "google-cloud-storage" }}
+          - name: gcp-service-account
+            readOnly: true
+            mountPath: /pulsar/gcp-service-account
+          {{- end }}
+          {{- end }}
           {{- if .Values.broker.extraVolumeMounts }}
 {{ toYaml .Values.broker.extraVolumeMounts | indent 10 }}
           {{- end }}
           {{- include "pulsar.broker.certs.volumeMounts" . | nindent 10 }}
-      {{- if and .Values.rbac.enabled .Values.rbac.psp }}
-        securityContext:
-          readOnlyRootFilesystem: false
+        env:
+          {{- if and (and .Values.broker.storageOffload (eq .Values.broker.storageOffload.driver "aws-s3")) .Values.broker.storageOffload.secret }}
+          - name: AWS_ACCESS_KEY_ID
+            valueFrom:
+              secretKeyRef:
+                name: {{ .Values.broker.storageOffload.secret }}
+                key: AWS_ACCESS_KEY_ID
+          - name: AWS_SECRET_ACCESS_KEY
+            valueFrom:
+              secretKeyRef:
+                name: {{ .Values.broker.storageOffload.secret }}
+                key: AWS_SECRET_ACCESS_KEY
+          {{- end }}
+          {{- if and .Values.broker.storageOffload (eq .Values.broker.storageOffload.driver "azureblob") }}
+          - name: AZURE_STORAGE_ACCOUNT
+            valueFrom:
+              secretKeyRef:
+                name: {{ .Values.broker.storageOffload.secret }}
+                key: AZURE_STORAGE_ACCOUNT
+          - name: AZURE_STORAGE_ACCESS_KEY
+            valueFrom:
+              secretKeyRef:
+                name: {{ .Values.broker.storageOffload.secret }}
+                key: AZURE_STORAGE_ACCESS_KEY
+          {{- end }}
+        {{- if .Values.broker.extraEnvs }}
+{{- toYaml .Values.broker.extraEnvs | nindent 10 }}
         {{- end }}
       volumes:
       {{- if .Values.broker.extraVolumes }}
 {{ toYaml .Values.broker.extraVolumes | indent 6 }}
       {{- end }}
       {{- if .Values.auth.authentication.enabled }}
-      {{- if eq .Values.auth.authentication.provider "jwt" }}
+      {{- if .Values.auth.authentication.jwt.enabled }}
       - name: token-keys
         secret:
           {{- if not .Values.auth.authentication.jwt.usingSecretKey }}
@@ -287,6 +371,13 @@ spec:
               path: broker/token
       {{- end}}
       {{- end}}
+      {{- if .Values.broker.storageOffload.driver }}
+      {{- if eq .Values.broker.storageOffload.driver "google-cloud-storage" }}
+      - name: gcp-service-account
+        secret:
+          secretName: {{ .Values.broker.storageOffload.gcsServiceAccountSecret }}
+      {{- end }}
+      {{- end }}
       {{- include "pulsar.broker.certs.volumes" . | nindent 6 }}
       {{- include "pulsar.imagePullSecrets" . | nindent 6}}
 {{- end }}

charts/pulsar/templates/certs-scripts-configmap.yaml

@@ -0,0 +1,82 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "{{ template "pulsar.fullname" . }}-certs-scripts"
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: certs-scripts 
+data:
+    certs-combine-pem.sh: |
+      #!/bin/bash
+      # This script combines all certificates into a single file.
+      # Usage: certs-combine-pem.sh <output_file> <cert1> <cert2> ...
+      set -eu -o pipefail
+
+      if [ "$#" -lt 2 ]; then
+        echo "Usage: $0 <output_file> <cert1> <cert2> ..."
+        exit 1
+      fi
+
+      OUTPUT_FILE="$1"
+      shift
+
+      OUTPUT_FILE_TMP="${OUTPUT_FILE}.tmp"
+      rm -f "$OUTPUT_FILE_TMP"
+
+      for CERT in "$@"; do
+        if [ -f "$CERT" ]; then
+          echo "# $CERT" >> "$OUTPUT_FILE_TMP"
+          cat "$CERT" >> "$OUTPUT_FILE_TMP"
+        else
+          echo "Certificate file '$CERT' does not exist, skipping"
+        fi
+      done
+
+      if [ ! -f "$OUTPUT_FILE" ]; then
+        touch "$OUTPUT_FILE"
+      fi
+
+      if diff -q "$OUTPUT_FILE" "$OUTPUT_FILE_TMP" > /dev/null; then
+        # No changes detected, skipping update
+        rm -f "$OUTPUT_FILE_TMP"
+      else
+        # Update $OUTPUT_FILE with new certificates
+        mv "$OUTPUT_FILE_TMP" "$OUTPUT_FILE"
+      fi
+
+    certs-combine-pem-infinity.sh: |
+      #!/bin/bash
+      # This script combines all certificates into a single file, every minutes.
+      # Usage: certs-combine-pem-infinity.sh <output_file> <cert1> <cert2> ...
+      set -eu -o pipefail
+
+      if [ "$#" -lt 2 ]; then
+        echo "Usage: $0 <output_file> <cert1> <cert2> ..."
+        exit 1
+      fi
+
+      while true; do
+        /pulsar/bin/certs-combine-pem.sh "$@"
+        sleep 60
+      done
+    

charts/pulsar/templates/check_helm_version.yaml

@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+{{- if semverCompare "<3.12.0-0" .Capabilities.HelmVersion.Version -}}
+{{- fail "Your Helm version is not supported. Please upgrade to Helm 3.12.0 or later. The recommended version is currently 3.14.4 or newer. You can find more about Helm releases and installation at https://github.com/helm/helm/releases. " -}}
+{{- end -}}

charts/pulsar/templates/dashboard-deployment.yaml

@@ -1,67 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-{{- if .Values.extra.dashboard }}
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.dashboard.component }}"
-  namespace: {{ template "pulsar.namespace" . }}
-  labels:
-    {{- include "pulsar.standardLabels" . | nindent 4 }}
-    component: {{ .Values.dashboard.component }}
-spec:
-  replicas: {{ .Values.dashboard.replicaCount }}
-  selector:
-    matchLabels:
-      {{- include "pulsar.matchLabels" . | nindent 6 }}
-      component: {{ .Values.dashboard.component }}
-  template:
-    metadata:
-      labels:
-        {{- include "pulsar.template.labels" . | nindent 8 }}
-        component: {{ .Values.dashboard.component }}
-
-      annotations:
-{{ toYaml .Values.dashboard.annotations | indent 8 }}
-    spec:
-    {{- if .Values.dashboard.nodeSelector }}
-      nodeSelector:
-{{ toYaml .Values.dashboard.nodeSelector | indent 8 }}
-    {{- end }}
-    {{- if .Values.dashboard.tolerations }}
-      tolerations:
-{{ toYaml .Values.dashboard.tolerations | indent 8 }}
-    {{- end }}
-      terminationGracePeriodSeconds: {{ .Values.dashboard.gracePeriod }}
-      containers:
-      - name: "{{ template "pulsar.fullname" . }}-{{ .Values.dashboard.component }}"
-        image: "{{ .Values.dashboard.image.repository }}:{{ .Values.dashboard.image.tag }}"
-        imagePullPolicy: {{ .Values.dashboard.image.pullPolicy }}
-      {{- if .Values.dashboard.resources }}
-        resources:
-{{ toYaml .Values.dashboard.resources | indent 10 }}
-      {{- end }}
-        ports:
-        - name: http
-          containerPort: 80
-        env:
-        - name: SERVICE_URL
-          value: http://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:8080/
-{{- end }}

charts/pulsar/templates/dashboard-ingress.yaml

@@ -1,68 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-{{- if .Values.extra.dashboard }}
-{{- if .Values.dashboard.ingress.enabled }}
-{{- if semverCompare "<1.19-0" .Capabilities.KubeVersion.Version }}
-apiVersion: extensions/v1beta1
-{{- else }}
-apiVersion: networking.k8s.io/v1
-{{- end }}
-kind: Ingress
-metadata:
-  labels:
-    {{- include "pulsar.standardLabels" . | nindent 4 }}
-    component: {{ .Values.dashboard.component }}
-  annotations:
-{{- with .Values.dashboard.ingress.annotations }}
-{{ toYaml . | indent 4 }}
-{{- end }}
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.dashboard.component }}"
-  namespace: {{ template "pulsar.namespace" . }}
-spec:
-  {{- with .Values.dashboard.ingress.ingressClassName }}
-  ingressClassName: {{ . }}
-  {{- end }}
-{{- if .Values.dashboard.ingress.tls.enabled }}
-  tls:
-    - hosts:
-        - {{ .Values.dashboard.ingress.hostname }}
-      {{- with .Values.dashboard.ingress.tls.secretName }}
-      secretName: {{ . }}
-      {{- end }}
-{{- end }}
-  rules:
-    - host: {{ required "Dashboard ingress hostname not provided" .Values.dashboard.ingress.hostname }}
-      http:
-        paths:
-          - path: {{ .Values.dashboard.ingress.path }}
-            {{- if semverCompare "<1.19-0" .Capabilities.KubeVersion.Version }}
-            backend:
-              serviceName: "{{ template "pulsar.fullname" . }}-{{ .Values.dashboard.component }}"
-              servicePort: {{ .Values.dashboard.ingress.port }}
-            {{- else }}
-            pathType: ImplementationSpecific
-            backend:
-              service:
-                name: "{{ template "pulsar.fullname" . }}-{{ .Values.dashboard.component }}"
-                port:
-                  number: {{ .Values.dashboard.ingress.port }}
-            {{- end }}
-{{- end }}
-{{- end }}

charts/pulsar/templates/extra-list.yaml

@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+{{- range .Values.extraDeploy }}
+---
+{{ include "common.tplvalues.render" (dict "value" . "context" $) }}
+{{- end }}

charts/pulsar/templates/keytool.yaml

@@ -1,105 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-# script to process key/cert to keystore and truststore
-{{- if .Values.tls.zookeeper.enabled }}
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: "{{ template "pulsar.fullname" . }}-keytool-configmap"
-  namespace: {{ template "pulsar.namespace" . }}
-  labels:
-    {{- include "pulsar.standardLabels" . | nindent 4 }}
-    component: keytool 
-data:
-  keytool.sh: |
-    #!/bin/bash
-    component=$1
-    name=$2
-    isClient=$3
-    crtFile=/pulsar/certs/${component}/tls.crt
-    keyFile=/pulsar/certs/${component}/tls.key
-    caFile=/pulsar/certs/ca/ca.crt
-    p12File=/pulsar/${component}.p12
-    keyStoreFile=/pulsar/${component}.keystore.jks
-    trustStoreFile=/pulsar/${component}.truststore.jks
-    
-    function checkFile() {
-        local file=$1
-        local len=$(wc -c ${file} | awk '{print $1}')
-        echo "processing ${file} : len = ${len}"
-        if [ ! -f ${file} ]; then
-            echo "${file} is not found"
-            return -1
-        fi
-        if [ $len -le 0 ]; then
-            echo "${file} is empty"
-            return -1
-        fi
-    }
-
-    function ensureFileNotEmpty() {
-        local file=$1
-        until checkFile ${file}; do
-            echo "file isn't initialized yet ... check in 3 seconds ..." && sleep 3;
-        done;
-    }
-    
-    ensureFileNotEmpty ${crtFile}
-    ensureFileNotEmpty ${keyFile}
-    ensureFileNotEmpty ${caFile}
-    
-    PASSWORD=$(head /dev/urandom | base64 | head -c 24)
-    
-    openssl pkcs12 \
-        -export \
-        -in ${crtFile} \
-        -inkey ${keyFile} \
-        -out ${p12File} \
-        -name ${name} \
-        -passout "pass:${PASSWORD}"
-    
-    keytool -importkeystore \
-        -srckeystore ${p12File} \
-        -srcstoretype PKCS12 -srcstorepass "${PASSWORD}" \
-        -alias ${name} \
-        -destkeystore ${keyStoreFile} \
-        -deststorepass "${PASSWORD}"
-    
-    keytool -import \
-        -file ${caFile} \
-        -storetype JKS \
-        -alias ${name} \
-        -keystore ${trustStoreFile} \
-        -storepass "${PASSWORD}" \
-        -trustcacerts -noprompt
-    
-    ensureFileNotEmpty ${keyStoreFile}
-    ensureFileNotEmpty ${trustStoreFile}
-    
-    if [[ "x${isClient}" == "xtrue" ]]; then
-        echo $'\n' >> conf/pulsar_env.sh
-        echo "PULSAR_EXTRA_OPTS=\"\${PULSAR_EXTRA_OPTS} -Dzookeeper.clientCnxnSocket=org.apache.zookeeper.ClientCnxnSocketNetty -Dzookeeper.client.secure=true -Dzookeeper.ssl.keyStore.location=${keyStoreFile} -Dzookeeper.ssl.keyStore.password=${PASSWORD} -Dzookeeper.ssl.trustStore.location=${trustStoreFile} -Dzookeeper.ssl.trustStore.password=${PASSWORD}\"" >> conf/pulsar_env.sh
-        echo $'\n' >> conf/bkenv.sh
-        echo "BOOKIE_EXTRA_OPTS=\"\${BOOKIE_EXTRA_OPTS} -Dzookeeper.clientCnxnSocket=org.apache.zookeeper.ClientCnxnSocketNetty -Dzookeeper.client.secure=true -Dzookeeper.ssl.keyStore.location=${keyStoreFile} -Dzookeeper.ssl.keyStore.password=${PASSWORD} -Dzookeeper.ssl.trustStore.location=${trustStoreFile} -Dzookeeper.ssl.trustStore.password=${PASSWORD}\"" >> conf/bkenv.sh
-    else
-        echo $'\n' >> conf/pulsar_env.sh
-        echo "PULSAR_EXTRA_OPTS=\"\${PULSAR_EXTRA_OPTS} -Dzookeeper.ssl.keyStore.location=${keyStoreFile} -Dzookeeper.ssl.keyStore.password=${PASSWORD} -Dzookeeper.ssl.trustStore.location=${trustStoreFile} -Dzookeeper.ssl.trustStore.password=${PASSWORD}\"" >> conf/pulsar_env.sh
-    fi
-{{- end }}

charts/pulsar/templates/oxia-coordinator-configmap.yaml

@@ -16,17 +16,17 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-
-{{- if .Values.components.functions }}
-## function config map
+{{- if and .Values.components.oxia (not .Values.oxia.coordinator.customConfigMapName) }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.functions.component }}-config"
+  name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator
   namespace: {{ template "pulsar.namespace" . }}
   labels:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
-    component: {{ .Values.functions.component }}
+    component: {{ .Values.oxia.component }}-coordinator
 data:
-  pulsarDockerImageName: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.functions "root" .) }}"
+  config.yaml: |
+    {{- include "oxia.coordinator.config.yaml" . | nindent 4 }}
+
 {{- end }}

charts/pulsar/templates/oxia-coordinator-deployment.yaml

@@ -0,0 +1,95 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+{{- if .Values.components.oxia }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.oxia.component }}-coordinator
+  annotations: {{ .Values.oxia.coordinator.appAnnotations | toYaml | nindent 4 }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      {{- include "pulsar.matchLabels" . | nindent 6 }}
+      component: {{ .Values.oxia.component }}-coordinator
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        {{- include "pulsar.template.labels" . | nindent 8 }}
+        component: {{ .Values.oxia.component }}-coordinator
+      annotations:
+        {{- if not .Values.oxia.coordinator.podMonitor.enabled }}
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "{{ .Values.oxia.coordinator.ports.metrics }}"
+        {{- end }}
+{{- with .Values.oxia.coordinator.annotations }}
+{{ toYaml . | indent 8 }}
+{{- end }}
+    spec:
+    {{- if .Values.oxia.coordinator.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.oxia.coordinator.nodeSelector | indent 8 }}
+    {{- end }}
+    {{- if .Values.oxia.coordinator.tolerations }}
+      tolerations:
+{{ toYaml .Values.oxia.coordinator.tolerations | indent 8 }}
+    {{- end }}    
+      serviceAccountName: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator
+      containers:
+        - command:
+{{- if .Values.oxia.coordinator.entrypoint }}
+{{ toYaml .Values.oxia.coordinator.entrypoint | indent 12 }}
+{{- else }}
+            {{- include "oxia.coordinator.entrypoint" . | nindent 12 }}
+{{- end }}
+          image: "{{ .Values.images.oxia.repository }}:{{ .Values.images.oxia.tag }}"
+          imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.oxia "root" .) }}"
+          name: coordinator
+          ports:
+            {{- range $key, $value := .Values.oxia.coordinator.ports }}
+            - containerPort: {{ $value | int }}
+              name: {{ $key }}
+            {{- end}}
+          resources:
+            limits:
+              cpu: {{ .Values.oxia.coordinator.cpuLimit }}
+              memory: {{ .Values.oxia.coordinator.memoryLimit }}
+      {{- if .Values.oxia.coordinator.extraVolumeMounts }}
+          volumeMounts:
+        {{- toYaml .Values.oxia.coordinator.extraVolumeMounts | nindent 12 }}
+      {{- end }}
+          livenessProbe:
+            {{- include "oxia-cluster.probe" .Values.oxia.coordinator.ports.internal | nindent 12 }}
+          readinessProbe:
+            {{- include "oxia-cluster.probe" .Values.oxia.coordinator.ports.internal | nindent 12 }}
+      {{- if .Values.oxia.coordinator.extraContainers }}
+        {{- toYaml .Values.oxia.coordinator.extraContainers | nindent 8 }}
+      {{- end }}
+      {{- if .Values.oxia.coordinator.extraVolumes }}
+      volumes:
+{{- toYaml .Values.oxia.coordinator.extraVolumes | nindent 8 }}
+      {{- end }}
+{{- end }}

charts/pulsar/templates/oxia-coordinator-podmonitor.yaml

@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# deploy oxia-coordinator PodMonitor only when `$.Values.oxia.coordinator.podMonitor.enabled` is true
+{{- if and $.Values.components.oxia $.Values.oxia.coordinator.podMonitor.enabled }}
+{{- include "pulsar.podMonitor" (list . "oxia.coordinator" (printf "component: %s-coordinator" .Values.oxia.component) "metrics") }}
+{{- end }}

charts/pulsar/templates/oxia-coordinator-role.yaml

@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+{{- if .Values.components.oxia }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.oxia.component }}-coordinator
+rules:
+  - apiGroups: [ "" ]
+    resources: [ "configmaps" ]
+    verbs: [ "*" ]
+{{- end }}

charts/pulsar/templates/oxia-coordinator-rolebinding.yaml

@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+{{- if .Values.components.oxia }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.oxia.component }}-coordinator
+subjects:
+  - kind: ServiceAccount
+    name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator
+    namespace: {{ template "pulsar.namespace" . }}
+roleRef:
+  apiGroup: ""
+  kind: Role
+  name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator
+{{- end }}

charts/pulsar/templates/oxia-coordinator-service.yaml

@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+{{- if .Values.components.oxia }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.oxia.component }}-coordinator
+{{- with .Values.oxia.coordinator.service.annotations }}
+  annotations:
+{{ toYaml . | indent 4 }}
+{{- end }}
+spec:
+  ports:
+    {{- range $key, $value := .Values.oxia.coordinator.ports }}
+    - name: {{ $key }}
+      port: {{ $value }}
+      targetPort: {{ $key }}
+    {{- end}}
+  selector:
+    {{- include "pulsar.matchLabels" . | nindent 4 }}
+    component: {{ .Values.oxia.component }}-coordinator
+{{- end }}

charts/pulsar/templates/oxia-coordinator-serviceaccount.yaml

@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+{{- if .Values.components.oxia }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.oxia.component }}-coordinator
+{{- with .Values.oxia.coordinator.service_account.annotations }}
+  annotations:
+{{ toYaml . | indent 4 }}
+{{- end }}
+{{- if .Values.images.imagePullSecrets }}
+imagePullSecrets:
+  - name: {{ .Values.images.imagePullSecrets.secretName }}
+{{- end}}
+{{- end}}

charts/pulsar/templates/oxia-server-podmonitor.yaml

@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# deploy oxia-server PodMonitor only when `$.Values.oxia.server.podMonitor.enabled` is true
+{{- if and $.Values.components.oxia $.Values.oxia.server.podMonitor.enabled }}
+{{- include "pulsar.podMonitor" (list . "oxia.server" (printf "component: %s-server" .Values.oxia.component) "metrics") }}
+{{- end }}

charts/pulsar/templates/oxia-server-service-public.yaml

@@ -17,22 +17,27 @@
 # under the License.
 #
 
-{{- if .Values.extra.dashboard }}
+{{- if .Values.components.oxia }}
 apiVersion: v1
 kind: Service
 metadata:
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.dashboard.component }}"
+  name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}
   namespace: {{ template "pulsar.namespace" . }}
   labels:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
-    component: {{ .Values.dashboard.component }}
+    component: {{ .Values.oxia.component }}-server
+{{- with .Values.oxia.server.service.public.annotations }}
   annotations:
-{{ toYaml .Values.dashboard.service.annotations | indent 4 }}
+{{ toYaml . | indent 4 }}
+{{- end }}
 spec:
   ports:
-{{ toYaml .Values.dashboard.service.ports | indent 2 }}
-  clusterIP: None
+    {{- range $key, $value := .Values.oxia.server.ports }}
+    - name: {{ $key }}
+      port: {{ $value }}
+      targetPort: {{ $key }}
+    {{- end}}
   selector:
     {{- include "pulsar.matchLabels" . | nindent 4 }}
-    component: {{ .Values.dashboard.component }}
+    component: {{ .Values.oxia.component }}-server
 {{- end}}

charts/pulsar/templates/oxia-server-service.yaml

@@ -0,0 +1,45 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+{{- if .Values.components.oxia }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-svc
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.oxia.component }}-server
+{{- with .Values.oxia.server.service.internal.annotations }}
+  annotations:
+{{ toYaml . | indent 4 }}
+{{- end }}
+spec:
+  clusterIP: None
+  publishNotReadyAddresses: true
+  ports:
+    {{- range $key, $value := .Values.oxia.server.ports }}
+    - name: {{ $key }}
+      port: {{ $value }}
+      targetPort: {{ $key }}
+    {{- end}}
+  selector:
+    {{- include "pulsar.matchLabels" . | nindent 4 }}
+    component: {{ .Values.oxia.component }}-server
+{{- end}}

charts/pulsar/templates/oxia-server-serviceaccount.yaml

@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+{{- if .Values.components.oxia }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.oxia.component }}-server
+{{- with .Values.oxia.server.service_account.annotations }}
+  annotations:
+{{ toYaml . | indent 4 }}
+{{- end }}
+{{- if .Values.images.imagePullSecrets }}
+imagePullSecrets:
+  - name: {{ .Values.images.imagePullSecrets.secretName }}
+{{- end}}
+{{- end}}

charts/pulsar/templates/oxia-server-statefulset.yaml

@@ -0,0 +1,153 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+{{- if .Values.components.oxia }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-server
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.oxia.component }}-server
+  annotations: {{ .Values.oxia.server.appAnnotations | toYaml | nindent 4 }}
+spec:
+  replicas: {{ .Values.oxia.server.replicas }}
+  selector:
+    matchLabels:
+      {{- include "pulsar.matchLabels" . | nindent 6 }}
+      component: {{ .Values.oxia.component }}-server
+  serviceName: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-svc
+  podManagementPolicy: Parallel
+  template:
+    metadata:
+      labels:
+        {{- include "pulsar.template.labels" . | nindent 8 }}
+        component: {{ .Values.oxia.component }}-server
+      annotations:
+        {{- if not .Values.oxia.server.podMonitor.enabled }}
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "{{ .Values.oxia.server.ports.metrics }}"
+        {{- end }}
+{{- with .Values.oxia.server.annotations }}
+{{ toYaml . | indent 8 }}
+{{- end }}
+    spec:
+    {{- if .Values.oxia.server.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.oxia.server.nodeSelector | indent 8 }}
+    {{- end }}
+    {{- if .Values.oxia.server.tolerations }}
+      tolerations:
+{{ toYaml .Values.oxia.server.tolerations | indent 8 }}
+    {{- end }}
+    {{- if .Values.oxia.server.topologySpreadConstraints }}
+      topologySpreadConstraints:
+        {{- toYaml .Values.oxia.server.topologySpreadConstraints | nindent 8 }}
+    {{- end }}
+      affinity:
+        {{- if and .Values.affinity.anti_affinity .Values.oxia.server.affinity.anti_affinity}}
+        podAntiAffinity:
+          {{ if eq .Values.oxia.server.affinity.type "requiredDuringSchedulingIgnoredDuringExecution"}}
+          {{ .Values.oxia.server.affinity.type }}:
+          - labelSelector:
+              matchExpressions:
+              - key: "app"
+                operator: In
+                values:
+                - "{{ template "pulsar.name" . }}"
+              - key: "release"
+                operator: In
+                values:
+                - {{ .Release.Name }}
+              - key: "component"
+                operator: In
+                values:
+                - {{ .Values.oxia.component }}-server
+            topologyKey: {{ .Values.oxia.server.affinity.anti_affinity_topology_key }}
+        {{ else }}
+          {{ .Values.oxia.server.affinity.type }}:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchExpressions:
+                    - key: "app"
+                      operator: In
+                      values:
+                      - "{{ template "pulsar.name" . }}"
+                    - key: "release"
+                      operator: In
+                      values:
+                      - {{ .Release.Name }}
+                    - key: "component" 
+                      operator: In
+                      values:
+                      - {{ .Values.oxia.component }}-server
+                topologyKey: {{ .Values.oxia.server.affinity.anti_affinity_topology_key }}
+        {{ end }}
+        {{- end }}    
+      serviceAccountName: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}
+      {{- if .Values.oxia.server.securityContext }}
+      securityContext:
+{{ toYaml .Values.oxia.server.securityContext | indent 8 }}
+      {{- end }}
+      containers:
+        - command:
+            - "oxia"
+            - "server"
+            - "--log-json"
+            - "--data-dir=/data/db"
+            - "--wal-dir=/data/wal"
+            - "--db-cache-size-mb={{ .Values.oxia.server.dbCacheSizeMb }}"
+            {{- if .Values.oxia.pprofEnabled }}
+            - "--profile"
+            {{- end}}
+          image: "{{ .Values.images.oxia.repository }}:{{ .Values.images.oxia.tag }}"
+          imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.oxia "root" .) }}"
+          name: server
+          ports:
+            {{- range $key, $value := .Values.oxia.server.ports }}
+            - containerPort: {{ $value | int }}
+              name: {{ $key }}
+            {{- end}}
+          resources:
+            limits:
+              cpu: {{ .Values.oxia.server.cpuLimit }}
+              memory: {{ .Values.oxia.server.memoryLimit }}
+          volumeMounts:
+            - name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-data
+              mountPath: /data
+          livenessProbe:
+            {{- include "oxia-cluster.probe" .Values.oxia.server.ports.internal | nindent 12 }}
+          readinessProbe:
+            {{- include "oxia-cluster.readiness-probe" .Values.oxia.server.ports.internal | nindent 12 }}
+          startupProbe:
+            {{- include "oxia-cluster.startup-probe" .Values.oxia.server.ports.internal | nindent 12 }}
+  volumeClaimTemplates:
+    - metadata:
+        name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-data
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        {{- if .Values.oxia.server.storageClassName }}
+        storageClassName: {{ .Values.oxia.server.storageClassName }}
+        {{- end}}
+        resources:
+          requests:
+            storage: {{ .Values.oxia.server.storageSize }}
+{{- end}}

charts/pulsar/templates/proxy-configmap.yaml

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-{{- if or .Values.components.proxy .Values.extra.proxy }}
+{{- if .Values.components.proxy }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -28,7 +28,7 @@ metadata:
     component: {{ .Values.proxy.component }}
 data:
   clusterName: {{ template "pulsar.cluster.name" . }}
-  statusFilePath: "{{ template "pulsar.home" . }}/status"
+  statusFilePath: "{{ template "pulsar.home" . }}/logs/status"
   # prometheus needs to access /metrics endpoint
   webServicePort: "{{ .Values.proxy.ports.containerPorts.http }}"
   {{- if or (not .Values.tls.enabled) (not .Values.tls.proxy.enabled) }}
@@ -42,14 +42,14 @@ data:
   webServicePortTls: "{{ .Values.proxy.ports.containerPorts.https }}"
   tlsCertificateFilePath: "/pulsar/certs/proxy/tls.crt"
   tlsKeyFilePath: "/pulsar/certs/proxy/tls.key"
-  tlsTrustCertsFilePath: "/pulsar/certs/ca/ca.crt"
+  tlsTrustCertsFilePath: {{ ternary "/pulsar/certs/cacerts/ca-combined.pem" "/pulsar/certs/ca/ca.crt" .Values.tls.proxy.cacerts.enabled | quote }}
   {{- if and .Values.tls.enabled .Values.tls.broker.enabled }}
   # if broker enables TLS, configure proxy to talk to broker using TLS
   brokerServiceURLTLS: pulsar+ssl://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.pulsarssl }}
   brokerWebServiceURLTLS: https://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.https }}
   tlsEnabledWithBroker: "true"
   tlsCertRefreshCheckDurationSec: "300"
-  brokerClientTrustCertsFilePath: "/pulsar/certs/ca/ca.crt"
+  brokerClientTrustCertsFilePath: {{ ternary "/pulsar/certs/cacerts/ca-combined.pem" "/pulsar/certs/ca/ca.crt" .Values.tls.proxy.cacerts.enabled | quote }}
   {{- end }}
   {{- if not (and .Values.tls.enabled .Values.tls.broker.enabled) }}
   brokerServiceURL: pulsar://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.pulsar }}
@@ -64,11 +64,20 @@ data:
   # disable authorization on proxy and forward authorization credentials to broker
   authorizationEnabled: "false"
   forwardAuthorizationCredentials: "true"
-  superUserRoles: {{ .Values.auth.superUsers | values | sortAlpha | join "," }}
+  {{- if .Values.auth.useProxyRoles }}
+  superUserRoles: {{ omit .Values.auth.superUsers "proxy" | values | compact | sortAlpha | join "," }}
+  {{- else }}
+  superUserRoles: {{ .Values.auth.superUsers | values | compact | sortAlpha | join "," }}
   {{- end }}
-  {{- if eq .Values.auth.authentication.provider "jwt" }}
+  {{- end }}
+  {{- if and .Values.auth.authentication.enabled .Values.auth.authentication.jwt.enabled  }}
   # token authentication configuration
+  {{- if and .Values.auth.authentication.enabled .Values.auth.authentication.jwt.enabled .Values.auth.authentication.openid.enabled }}
+  authenticationProviders: "org.apache.pulsar.broker.authentication.AuthenticationProviderToken,org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID"
+  {{- end }}
+  {{- if and .Values.auth.authentication.enabled .Values.auth.authentication.jwt.enabled ( not .Values.auth.authentication.openid.enabled ) }}
   authenticationProviders: "org.apache.pulsar.broker.authentication.AuthenticationProviderToken"
+  {{- end }}
   brokerClientAuthenticationParameters: "file:///pulsar/tokens/proxy/token"
   brokerClientAuthenticationPlugin: "org.apache.pulsar.client.impl.auth.AuthenticationToken"
   {{- if .Values.auth.authentication.jwt.usingSecretKey }}
@@ -77,6 +86,25 @@ data:
   tokenPublicKey: "file:///pulsar/keys/token/public.key"
   {{- end }}
   {{- end }}
+  {{- if and .Values.auth.authentication.enabled .Values.auth.authentication.openid.enabled  }}
+  # openid authentication configuration
+  {{- if and .Values.auth.authentication.enabled .Values.auth.authentication.openid.enabled ( not .Values.auth.authentication.jwt.enabled ) }}
+  authenticationProviders: "org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID"
+  {{- end }}
+  PULSAR_PREFIX_openIDAllowedTokenIssuers: {{ .Values.auth.authentication.openid.openIDAllowedTokenIssuers | uniq | compact | sortAlpha | join "," | quote }}
+  PULSAR_PREFIX_openIDAllowedAudiences: {{ .Values.auth.authentication.openid.openIDAllowedAudiences | uniq | compact | sortAlpha | join "," | quote }}
+  PULSAR_PREFIX_openIDTokenIssuerTrustCertsFilePath: {{ .Values.auth.authentication.openid.openIDTokenIssuerTrustCertsFilePath | quote }}
+  PULSAR_PREFIX_openIDRoleClaim: {{  .Values.auth.authentication.openid.openIDRoleClaim | quote }}
+  PULSAR_PREFIX_openIDAcceptedTimeLeewaySeconds: {{ .Values.auth.authentication.openid.openIDAcceptedTimeLeewaySeconds | quote }}
+  PULSAR_PREFIX_openIDCacheSize: {{ .Values.auth.authentication.openid.openIDCacheSize | quote }}
+  PULSAR_PREFIX_openIDCacheRefreshAfterWriteSeconds: {{ .Values.auth.authentication.openid.openIDCacheRefreshAfterWriteSeconds | quote }}
+  PULSAR_PREFIX_openIDCacheExpirationSeconds: {{ .Values.auth.authentication.openid.openIDCacheExpirationSeconds | quote }}
+  PULSAR_PREFIX_openIDHttpConnectionTimeoutMillis: {{ .Values.auth.authentication.openid.openIDHttpConnectionTimeoutMillis | quote }}
+  PULSAR_PREFIX_openIDHttpReadTimeoutMillis: {{ .Values.auth.authentication.openid.openIDHttpReadTimeoutMillis | quote }}
+  PULSAR_PREFIX_openIDKeyIdCacheMissRefreshSeconds: {{ .Values.auth.authentication.openid.openIDKeyIdCacheMissRefreshSeconds | quote }}
+  PULSAR_PREFIX_openIDRequireIssuersUseHttps: {{ .Values.auth.authentication.openid.openIDRequireIssuersUseHttps | quote }}
+  PULSAR_PREFIX_openIDFallbackDiscoveryMode: {{ .Values.auth.authentication.openid.openIDFallbackDiscoveryMode | quote }}
+  {{- end }}
   {{- end }}
 {{ toYaml .Values.proxy.configData | indent 2 }}
 {{- end }}

charts/pulsar/templates/proxy-hpa.yaml

@@ -26,6 +26,9 @@ apiVersion: autoscaling/v2
 kind: HorizontalPodAutoscaler
 metadata:
   name: "{{ template "pulsar.fullname" . }}-{{ .Values.proxy.component }}"
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
 spec:
   maxReplicas: {{ .Values.proxy.autoscaling.maxReplicas }}
   {{- with .Values.proxy.autoscaling.metrics }}

charts/pulsar/templates/proxy-ingress.yaml

@@ -59,7 +59,7 @@ spec:
               servicePort: {{ .Values.proxy.ports.http }}
               {{- end }}
             {{- else }}
-            pathType: ImplementationSpecific
+            pathType: {{ .Values.proxy.ingress.pathType }}
             backend:
               service:
                 name: "{{ template "pulsar.fullname" . }}-{{ .Values.proxy.component }}"

charts/pulsar/templates/proxy-pdb.yaml

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-{{- if or .Values.components.proxy .Values.extra.proxy }}
+{{- if .Values.components.proxy }}
 {{- if .Values.proxy.pdb.usePolicy }}
 # pdb version detection
 {{- if semverCompare "<1.21-0" .Capabilities.KubeVersion.Version }}

charts/pulsar/templates/proxy-podmonitor.yaml

@@ -19,37 +19,5 @@
 
 # deploy proxy PodMonitor only when `$.Values.proxy.podMonitor.enabled` is true
 {{- if $.Values.proxy.podMonitor.enabled }}
-apiVersion: monitoring.coreos.com/v1
-kind: PodMonitor
-metadata:
-  name: {{ template "pulsar.fullname" . }}-proxy
-  labels:
-    app: {{ template "pulsar.name" . }}
-    chart: {{ template "pulsar.chart" . }}
-    release: {{ .Release.Name }}
-    heritage: {{ .Release.Service }}
-spec:
-  jobLabel: proxy
-  podMetricsEndpoints:
-    - port: http
-      path: /metrics
-      scheme: http
-      interval: {{ $.Values.proxy.podMonitor.interval }}
-      scrapeTimeout: {{ $.Values.proxy.podMonitor.scrapeTimeout }}
-      relabelings:
-        - action: labelmap
-          regex: __meta_kubernetes_pod_label_(.+)
-        - sourceLabels: [__meta_kubernetes_namespace]
-          action: replace
-          targetLabel: kubernetes_namespace
-        - sourceLabels: [__meta_kubernetes_pod_label_component]
-          action: replace
-          targetLabel: job
-        - sourceLabels: [__meta_kubernetes_pod_name]
-          action: replace
-          targetLabel: kubernetes_pod_name
-  selector:
-    matchLabels:
-      {{- include "pulsar.matchLabels" . | nindent 6 }}
-      component: proxy
+{{- include "pulsar.podMonitor" (list . "proxy" (printf "component: %s" .Values.proxy.component) "sts-http") }}
 {{- end }}

charts/pulsar/templates/proxy-rbac.yaml

@@ -1,92 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-{{- if and .Values.rbac.enabled .Values.rbac.psp }}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.proxy.component }}"
-  namespace: {{ template "pulsar.namespace" . }}
-rules:
-  - apiGroups:
-      - policy
-    resourceNames:
-      - "{{ template "pulsar.fullname" . }}-{{ .Values.proxy.component }}"
-    resources:
-      - podsecuritypolicies
-    verbs:
-      - use
----
-
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.proxy.component }}"
-  namespace: {{ template "pulsar.namespace" . }}
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.proxy.component }}"
-  namespace: {{ template "pulsar.namespace" . }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.proxy.component }}"
-subjects:
-- kind: ServiceAccount
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.proxy.component }}"
-  namespace: {{ template "pulsar.namespace" . }}
----
-
-apiVersion: policy/v1beta1
-kind: PodSecurityPolicy
-metadata:
-{{- if .Values.rbac.limit_to_namespace }}
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.proxy.component }}-{{ template "pulsar.namespace" . }}"
-{{- else}}
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.proxy.component }}"
-{{- end}}
-spec:
-  readOnlyRootFilesystem: false
-  privileged: false
-  allowPrivilegeEscalation: false
-  runAsUser:
-    rule: 'RunAsAny'
-  supplementalGroups:
-    ranges:
-    - max: 65535
-      min: 1
-    rule: MustRunAs
-  fsGroup:
-    rule: 'MustRunAs'
-    ranges:
-      - min: 1
-        max: 65535
-  seLinux:
-    rule: 'RunAsAny'
-  volumes:
-  - configMap
-  - emptyDir
-  - projected
-  - secret
-  - downwardAPI
-  - persistentVolumeClaim
-  {{- end}}

charts/pulsar/templates/proxy-service-account.yaml

@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+{{- if .Values.components.proxy }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: "{{ template "pulsar.fullname" . }}-{{ .Values.proxy.component }}"
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.proxy.component }}
+  annotations:
+{{- with .Values.proxy.service_account.annotations }}
+{{ toYaml . | indent 4 }}
+{{- end }}
+{{- end }}

charts/pulsar/templates/proxy-service.yaml

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-{{- if or .Values.components.proxy .Values.extra.proxy }}
+{{- if .Values.components.proxy }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -32,26 +32,50 @@ metadata:
   {{- end }}
 spec:
   type: {{ .Values.proxy.service.type }}
+  {{- with .Values.proxy.service.loadBalancerIP }}
+  loadBalancerIP: {{ . }}
+  {{- end }}
+  {{- with .Values.proxy.service.loadBalancerClass }}
+  loadBalancerClass: {{ . }}
+  {{- end }}
+  {{- if .Values.proxy.service.externalTrafficPolicy }}
+  externalTrafficPolicy: {{ .Values.proxy.service.externalTrafficPolicy }}
+  {{- end }}
+  {{- if .Values.proxy.service.loadBalancerSourceRanges }}
+  loadBalancerSourceRanges: {{ toYaml .Values.proxy.service.loadBalancerSourceRanges | nindent 4 }}
+  {{- end }}
   ports:
     {{- if or (not .Values.tls.enabled) (not .Values.tls.proxy.enabled) }}
     - name: http
       port: {{ .Values.proxy.ports.http }}
       protocol: TCP
       targetPort: sts-http
+      {{- if and (eq .Values.proxy.service.type "NodePort") (ne .Values.proxy.service.nodePorts.http "") }}
+      nodePort: {{ .Values.proxy.service.nodePorts.http }}
+      {{- end}}
     - name: "{{ .Values.tcpPrefix }}pulsar"
       port: {{ .Values.proxy.ports.pulsar }}
       protocol: TCP
       targetPort: "sts-{{ .Values.tcpPrefix }}pulsar"
+      {{- if and (eq .Values.proxy.service.type "NodePort") (ne .Values.proxy.service.nodePorts.pulsar "") }}
+      nodePort: {{ .Values.proxy.service.nodePorts.pulsar }}
+      {{- end}}
     {{- end }}
     {{- if and .Values.tls.enabled .Values.tls.proxy.enabled }}
     - name: https
       port: {{ .Values.proxy.ports.https }}
       protocol: TCP
       targetPort: sts-https
+      {{- if and (eq .Values.proxy.service.type "NodePort") (ne .Values.proxy.service.nodePorts.https "") }}
+      nodePort: {{ .Values.proxy.service.nodePorts.https }}
+      {{- end}}
     - name: "{{ .Values.tlsPrefix }}pulsarssl"
       port: {{ .Values.proxy.ports.pulsarssl }}
       protocol: TCP
       targetPort: "sts-{{ .Values.tlsPrefix }}pulsarssl"
+      {{- if and (eq .Values.proxy.service.type "NodePort") (ne .Values.proxy.service.nodePorts.pulsarssl "") }}
+      nodePort: {{ .Values.proxy.service.nodePorts.pulsarssl }}
+      {{- end}}
     {{- end }}
   selector:
     {{- include "pulsar.matchLabels" . | nindent 4 }}

charts/pulsar/templates/proxy-statefulset.yaml

@@ -17,12 +17,13 @@
 # under the License.
 #
 
-{{- if or .Values.components.proxy .Values.extra.proxy }}
+{{- if .Values.components.proxy }}
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: "{{ template "pulsar.fullname" . }}-{{ .Values.proxy.component }}"
   namespace: {{ template "pulsar.namespace" . }}
+  annotations: {{ .Values.proxy.appAnnotations | toYaml | nindent 4 }}
   labels:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
     component: {{ .Values.proxy.component }}
@@ -44,8 +45,10 @@ spec:
         {{- include "pulsar.template.labels" . | nindent 8 }}
         component: {{ .Values.proxy.component }}
       annotations:
+        {{- if not .Values.proxy.podMonitor.enabled }}
         prometheus.io/scrape: "true"
-        prometheus.io/port: "{{ .Values.proxy.ports.http }}"
+        prometheus.io/port: "{{ .Values.proxy.ports.containerPorts.http }}"
+        {{- end }}
         {{- if .Values.proxy.restartPodsOnConfigMapChange }}
         checksum/config: {{ include (print $.Template.BasePath "/proxy-configmap.yaml") . | sha256sum }}
         {{- end }}
@@ -60,6 +63,10 @@ spec:
     {{- if .Values.proxy.tolerations }}
       tolerations:
 {{ toYaml .Values.proxy.tolerations | indent 8 }}
+    {{- end }}
+    {{- if .Values.proxy.topologySpreadConstraints }}
+      topologySpreadConstraints:
+        {{- toYaml .Values.proxy.topologySpreadConstraints | nindent 8 }}
     {{- end }}
       affinity:
         {{- if and .Values.affinity.anti_affinity .Values.proxy.affinity.anti_affinity}}
@@ -103,36 +110,67 @@ spec:
         {{ end }}
         {{- end }}
       terminationGracePeriodSeconds: {{ .Values.proxy.gracePeriod }}
-    {{- if and .Values.rbac.enabled .Values.rbac.psp }}
       serviceAccountName: "{{ template "pulsar.fullname" . }}-{{ .Values.proxy.component }}"
-    {{- end}}
       initContainers:
+      {{- if .Values.tls.proxy.cacerts.enabled }}
+      - name: combine-certs
+        image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.proxy "root" .) }}"
+        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.proxy "root" .) }}"
+        resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
+        command: ["sh", "-c"]
+        args:
+          - |
+            bin/certs-combine-pem.sh /pulsar/certs/cacerts/ca-combined.pem {{ template "pulsar.certs.cacerts" (dict "certs" .Values.tls.proxy.cacerts.certs) }}
+        volumeMounts:
+        {{- include "pulsar.proxy.certs.volumeMounts" . | nindent 8 }}
+      {{- end }}
+      {{- if and .Values.components.zookeeper .Values.proxy.waitZookeeperTimeout (gt (.Values.proxy.waitZookeeperTimeout | int) 0) }}
       # This init container will wait for zookeeper to be ready before
       # deploying the bookies
       - name: wait-zookeeper-ready
         image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.proxy "root" .) }}"
-        imagePullPolicy: {{ .Values.images.proxy.pullPolicy }}
-        command: ["sh", "-c"]
+        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.proxy "root" .) }}"
+        resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
+        command: ["timeout", "{{ .Values.proxy.waitZookeeperTimeout }}", "sh", "-c"]
         args:
-          - >-
+          - |
             export PULSAR_MEM="-Xmx128M";
             {{- if $zk:=.Values.pulsar_metadata.userProvidedZookeepers }}
-            until bin/pulsar zookeeper-shell -server {{ $zk }} ls {{ or .Values.metadataPrefix "/" }}; do
+            until timeout 15 bin/pulsar zookeeper-shell -server {{ $zk }} ls {{ or .Values.metadataPrefix "/" }}; do
               echo "user provided zookeepers {{ $zk }} are unreachable... check in 3 seconds ..." && sleep 3;
             done; 
-            {{ else }}
-            until bin/pulsar zookeeper-shell -server {{ template "pulsar.configurationStore.service" . }} get {{ .Values.metadataPrefix }}/admin/clusters/{{ template "pulsar.cluster.name" . }}; do
+            {{- else if .Values.pulsar_metadata.configurationStore }}
+            until timeout 15 bin/pulsar zookeeper-shell -server {{ template "pulsar.configurationStore.service" . }} get {{ .Values.pulsar_metadata.configurationStoreMetadataPrefix }}/admin/clusters/{{ template "pulsar.cluster.name" . }}; do
+              echo "pulsar cluster {{ template "pulsar.cluster.name" . }} isn't initialized yet ... check in 3 seconds ..." && sleep 3;
+            done;
+            {{- else }}
+            until timeout 15 bin/pulsar zookeeper-shell -server {{ template "pulsar.zookeeper.service" . }} get {{ .Values.metadataPrefix }}/admin/clusters/{{ template "pulsar.cluster.name" . }}; do
+              echo "pulsar cluster {{ template "pulsar.cluster.name" . }} isn't initialized yet ... check in 3 seconds ..." && sleep 3;
+            done;
+            {{- end}}
+      {{- end}}
+      {{- if and .Values.components.oxia .Values.proxy.waitOxiaTimeout (gt (.Values.proxy.waitOxiaTimeout | int) 0) }}
+      - name: wait-oxia-ready
+        image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.proxy "root" .) }}"
+        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.proxy "root" .) }}"
+        resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
+        command: ["timeout", "{{ .Values.proxy.waitOxiaTimeout }}", "sh", "-c"]
+        args:
+          - |
+            until nslookup {{ template "pulsar.oxia.server.service" . }}; do
               sleep 3;
             done;
       {{- end }}      
+      {{- if and .Values.proxy.waitBrokerTimeout (gt (.Values.proxy.waitBrokerTimeout | int) 0) }}
       # This init container will wait for at least one broker to be ready before
       # deploying the proxy
       - name: wait-broker-ready
         image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.proxy "root" .) }}"
-        imagePullPolicy: {{ .Values.images.proxy.pullPolicy }}
-        command: ["sh", "-c"]
+        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.proxy "root" .) }}"
+        resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
+        command: ["timeout", "{{ .Values.proxy.waitBrokerTimeout }}", "sh", "-c"]
         args:
-          - >-
+          - |
             set -e;
             brokerServiceNumber="$(nslookup -timeout=10 {{ template "pulsar.fullname" . }}-{{ .Values.broker.component }} | grep Name | wc -l)";
             until [ ${brokerServiceNumber} -ge 1 ]; do
@@ -140,10 +178,14 @@ spec:
               sleep 10;
               brokerServiceNumber="$(nslookup -timeout=10 {{ template "pulsar.fullname" . }}-{{ .Values.broker.component }} | grep Name | wc -l)";
             done;
+      {{- end}}
+      {{- if .Values.proxy.initContainers }}
+        {{- toYaml .Values.proxy.initContainers | nindent 6 }}
+      {{- end }}
       containers:
       - name: "{{ template "pulsar.fullname" . }}-{{ .Values.proxy.component }}"
         image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.proxy "root" .) }}"
-        imagePullPolicy: {{ .Values.images.proxy.pullPolicy }}
+        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.proxy "root" .) }}"
         {{- if .Values.proxy.probe.liveness.enabled }}
         livenessProbe:
           httpGet:
@@ -180,12 +222,17 @@ spec:
       {{- end }}
         command: ["sh", "-c"]
         args:
-        - >
+        - |
         {{- if .Values.proxy.additionalCommand }}
           {{ .Values.proxy.additionalCommand }}
+        {{- end }}
+          {{- if .Values.tls.proxy.cacerts.enabled }}
+          cd /pulsar/certs/cacerts;
+          nohup /pulsar/bin/certs-combine-pem-infinity.sh /pulsar/certs/cacerts/ca-combined.pem {{ template "pulsar.certs.cacerts" (dict "certs" .Values.tls.proxy.cacerts.certs) }} > /pulsar/certs/cacerts/certs-combine-pem-infinity.log 2>&1 &
+          cd /pulsar;
           {{- end }}
           bin/apply-config-from-env.py conf/proxy.conf &&
-          echo "OK" > status &&
+          echo "OK" > "${statusFilePath:-status}" &&
           OPTS="${OPTS} -Dlog4j2.formatMsgNoLookups=true" exec bin/pulsar proxy
         ports:
         # prometheus needs to access /metrics endpoint
@@ -201,13 +248,9 @@ spec:
         - name: "sts-{{ .Values.tlsPrefix }}pulsarssl"
           containerPort: {{ .Values.proxy.ports.pulsarssl }}
         {{- end }}
-      {{- if and .Values.rbac.enabled .Values.rbac.psp }}
-        securityContext:
-          readOnlyRootFilesystem: false
-      {{- end }}
-{{- if .Values.proxy.extreEnvs }}
+{{- if .Values.proxy.extraEnvs }}
         env:
-{{ toYaml .Values.proxy.extreEnvs | indent 8 }}
+{{ toYaml .Values.proxy.extraEnvs | indent 8 }}
 {{- end }}
         envFrom:
         - configMapRef:
@@ -215,7 +258,7 @@ spec:
         {{- if or .Values.proxy.extraVolumeMounts .Values.auth.authentication.enabled (and .Values.tls.enabled (or .Values.tls.proxy.enabled .Values.tls.broker.enabled)) }}
         volumeMounts:
           {{- if .Values.auth.authentication.enabled }}
-          {{- if eq .Values.auth.authentication.provider "jwt" }}
+          {{- if .Values.auth.authentication.jwt.enabled }}
           - mountPath: "/pulsar/keys"
             name: token-keys
             readOnly: true
@@ -224,16 +267,7 @@ spec:
             readOnly: true
           {{- end }}
           {{- end }}
-          {{- if .Values.tls.proxy.enabled }}
-          - mountPath: "/pulsar/certs/proxy"
-            name: proxy-certs
-            readOnly: true
-          {{- end}}
-          {{- if .Values.tls.enabled }}
-          - mountPath: "/pulsar/certs/ca"
-            name: ca
-            readOnly: true
-          {{- end}}
+          {{- include "pulsar.proxy.certs.volumeMounts" . | nindent 10 }}
           {{- if .Values.proxy.extraVolumeMounts }}
 {{ toYaml .Values.proxy.extraVolumeMounts | indent 10 }}
           {{- end }}
@@ -245,7 +279,7 @@ spec:
 {{ toYaml .Values.proxy.extraVolumes | indent 8 }}
         {{- end }}
         {{- if .Values.auth.authentication.enabled }}
-        {{- if eq .Values.auth.authentication.provider "jwt" }}
+        {{- if .Values.auth.authentication.jwt.enabled }}
         - name: token-keys
           secret:
             {{- if not .Values.auth.authentication.jwt.usingSecretKey }}
@@ -270,21 +304,6 @@ spec:
                 path: proxy/token
         {{- end}}
         {{- end}}
-        {{- if .Values.tls.proxy.enabled }}
-        - name: ca
-          secret:
-            secretName: "{{ .Release.Name }}-{{ .Values.tls.ca_suffix }}"
-            items:
-              - key: ca.crt
-                path: ca.crt
-        - name: proxy-certs
-          secret:
-            secretName: "{{ .Release.Name }}-{{ .Values.tls.proxy.cert_name }}"
-            items:
-              - key: tls.crt
-                path: tls.crt
-              - key: tls.key
-                path: tls.key
-        {{- end}}
+        {{- include "pulsar.proxy.certs.volumes" . | nindent 8 }}
       {{- end}}
 {{- end }}

charts/pulsar/templates/pulsar-cluster-initialize.yaml

@@ -17,12 +17,12 @@
 # under the License.
 #
 
-{{- if or .Release.IsInstall .Values.initialize }}
+{{- if or (and .Values.useReleaseStatus .Release.IsInstall) .Values.initialize }}
 {{- if .Values.components.broker }}
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_metadata.component }}"
+  name: {{ template "pulsar.fullname" . }}-{{ .Values.pulsar_metadata.component }}
   namespace: {{ template "pulsar.namespace" . }}
   labels:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
@@ -30,75 +30,112 @@ metadata:
 spec:
 # This feature was previously behind a feature gate for several Kubernetes versions and will default to true in 1.23 and beyond
 # https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/
-{{- if .Values.job.ttl.enabled }}
-  ttlSecondsAfterFinished: {{ .Values.job.ttl.secondsAfterFinished }}
+{{- if and .Values.job.ttl.enabled (semverCompare ">=1.23-0" .Capabilities.KubeVersion.Version) }}
+  ttlSecondsAfterFinished: {{ .Values.job.ttl.secondsAfterFinished | default 600 }}
 {{- end }}
   template:
+    metadata:
+      labels:
+        {{- include "pulsar.template.labels" . | nindent 8 }}
+        component: {{ .Values.pulsar_metadata.component }}
     spec:
+    {{- include "pulsar.imagePullSecrets" . | nindent 6 }}
     {{- if .Values.pulsar_metadata.nodeSelector }}
       nodeSelector:
 {{ toYaml .Values.pulsar_metadata.nodeSelector | indent 8 }}
     {{- end }}
       initContainers:
-      {{- if .Values.pulsar_metadata.configurationStore }}
-      - name: wait-cs-ready
+      {{- if .Values.tls.toolset.cacerts.enabled }}
+      - name: cacerts
         image: "{{ template "pulsar.imageFullName" (dict "image" .Values.pulsar_metadata.image "root" .) }}"
-        imagePullPolicy: {{ .Values.pulsar_metadata.image.pullPolicy }}
+        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.pulsar_metadata.image "root" .) }}"
+        resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
         command: ["sh", "-c"]
         args:
-          - >-
+          - |
+            bin/certs-combine-pem.sh /pulsar/certs/cacerts/ca-combined.pem {{ template "pulsar.certs.cacerts" (dict "certs" .Values.tls.toolset.cacerts.certs) }}
+        volumeMounts:
+        {{- include "pulsar.toolset.certs.volumeMounts" . | nindent 8 }}
+      {{- end }}
+      {{- if and .Values.components.zookeeper .Values.pulsar_metadata.waitZookeeperTimeout (gt (.Values.pulsar_metadata.waitZookeeperTimeout | int) 0) }}
+      {{- if .Values.pulsar_metadata.configurationStore }}
+      - name: wait-zk-cs-ready
+        image: "{{ template "pulsar.imageFullName" (dict "image" .Values.pulsar_metadata.image "root" .) }}"
+        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.pulsar_metadata.image "root" .) }}"
+        resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
+        command: ["timeout", "{{ .Values.pulsar_metadata.waitZookeeperTimeout }}", "sh", "-c"]
+        args:
+          - |
             until nslookup {{ .Values.pulsar_metadata.configurationStore}}; do
               sleep 3;
             done;
       {{- end }}
-      - name: wait-zookeeper-ready
+      - name: wait-zk-metastore-ready
         image: "{{ template "pulsar.imageFullName" (dict "image" .Values.pulsar_metadata.image "root" .) }}"
-        imagePullPolicy: {{ .Values.pulsar_metadata.image.pullPolicy }}
-        command: ["sh", "-c"]
+        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.pulsar_metadata.image "root" .) }}"
+        resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
+        command: ["timeout", "{{ .Values.pulsar_metadata.waitZookeeperTimeout }}", "sh", "-c"]
         args:
-          - >-
+          - |
             {{- if $zk := .Values.pulsar_metadata.userProvidedZookeepers }}
             export PULSAR_MEM="-Xmx128M";
-            until bin/pulsar zookeeper-shell -server {{ $zk }} ls {{ or .Values.metadataPrefix "/" }}; do
+            until timeout 15 bin/pulsar zookeeper-shell -server {{ $zk }} ls {{ or .Values.metadataPrefix "/" }}; do
               echo "user provided zookeepers {{ $zk }} are unreachable... check in 3 seconds ..." && sleep 3;
             done;
-            {{ else }}
+            {{ else if .Values.components.zookeeper }}
             until nslookup {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-{{ add (.Values.zookeeper.replicaCount | int) -1 }}.{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}.{{ template "pulsar.namespace" . }}; do
               sleep 3;
             done;
             {{- end }}
+      {{- end }}
+      {{- if and .Values.components.oxia .Values.pulsar_metadata.waitOxiaTimeout (gt (.Values.pulsar_metadata.waitOxiaTimeout | int) 0) }}
+      - name: wait-oxia-ready
+        image: "{{ template "pulsar.imageFullName" (dict "image" .Values.pulsar_metadata.image "root" .) }}"
+        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.pulsar_metadata.image "root" .) }}"
+        resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
+        command: ["timeout", "{{ .Values.pulsar_metadata.waitOxiaTimeout }}", "sh", "-c"]
+        args:
+          - |
+            until nslookup {{ template "pulsar.oxia.server.service" . }}; do
+              sleep 3;
+            done;
+      {{- end }}
+      {{- if and .Values.pulsar_metadata.waitBookkeeperTimeout (gt (.Values.pulsar_metadata.waitBookkeeperTimeout | int) 0) }}
       # This initContainer will wait for bookkeeper initnewcluster to complete
       # before initializing pulsar metadata
       - name: pulsar-bookkeeper-verify-clusterid
-        image: "{{ template "pulsar.imageFullName" (dict "image" .Values.pulsar_metadata.image "root" .) }}"
-        imagePullPolicy: {{ .Values.pulsar_metadata.image.pullPolicy }}
-        command: ["sh", "-c"]
+        image: {{ template "pulsar.imageFullName" (dict "image" .Values.pulsar_metadata.image "root" .) }}
+        imagePullPolicy: {{ template "pulsar.imagePullPolicy" (dict "image" .Values.pulsar_metadata.image "root" .) }}
+        resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
+        command: ["timeout", "{{ .Values.pulsar_metadata.waitBookkeeperTimeout }}", "sh", "-c"]
         args:
-        - >
+        - |
           bin/apply-config-from-env.py conf/bookkeeper.conf;
           echo Default BOOKIE_MEM settings are set very high, which can cause the init container to fail.;
           echo Setting the memory to a lower value to avoid OOM as operations below are not memory intensive.;
           export BOOKIE_MEM="-Xmx128M";
           {{- include "pulsar.toolset.zookeeper.tls.settings" . | nindent 10 }}
-          until bin/bookkeeper shell whatisinstanceid; do
+          until timeout 15 bin/bookkeeper shell whatisinstanceid; do
             sleep 3;
           done;
         envFrom:
         - configMapRef:
-            name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
+            name: {{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}
         volumeMounts:
         {{- include "pulsar.toolset.certs.volumeMounts" . | nindent 8 }}
+      {{- end }}
       containers:
-      - name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_metadata.component }}"
-        image: "{{ template "pulsar.imageFullName" (dict "image" .Values.pulsar_metadata.image "root" .) }}"
-        imagePullPolicy: {{ .Values.pulsar_metadata.image.pullPolicy }}
+      - name: {{ template "pulsar.fullname" . }}-{{ .Values.pulsar_metadata.component }}
+        image: {{ template "pulsar.imageFullName" (dict "image" .Values.pulsar_metadata.image "root" .) }}
+        imagePullPolicy: {{ template "pulsar.imagePullPolicy" (dict "image" .Values.pulsar_metadata.image "root" .) }}
       {{- if .Values.pulsar_metadata.resources }}
         resources:
 {{ toYaml .Values.pulsar_metadata.resources | indent 10 }}
       {{- end }}
-        command: ["sh", "-c"]
+        command: ["timeout", "{{ .Values.pulsar_metadata.initTimeout | default 60 }}", "sh", "-c"]
+        {{- if .Values.components.zookeeper }}
         args:
-          - |
+          - | # Use the pipe character for the YAML multiline string. Workaround for kubernetes-sigs/kustomize#4201
             {{- include "pulsar.toolset.zookeeper.tls.settings" . | nindent 12 }}
             export PULSAR_MEM="-Xmx128M";
             bin/pulsar initialize-cluster-metadata \
@@ -106,8 +143,7 @@ spec:
               --zookeeper {{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }} \
             {{- if .Values.pulsar_metadata.configurationStore }}
               --configuration-store {{ template "pulsar.configurationStore.connect" . }}{{ .Values.pulsar_metadata.configurationStoreMetadataPrefix }} \
-              {{- end }}
-              {{- if not .Values.pulsar_metadata.configurationStore }}
+            {{- else }}
               --configuration-store {{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }} \
             {{- end }}
               --web-service-url http://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}.{{ template "pulsar.namespace" . }}.svc.{{ .Values.clusterDomain }}:{{ .Values.broker.ports.http }}/ \
@@ -117,14 +153,34 @@ spec:
             {{- if .Values.extraInitCommand }}
               {{ .Values.extraInitCommand }}
             {{- end }}
+        {{- else if .Values.components.oxia }}
+        args:
+          - | # Use the pipe character for the YAML multiline string. Workaround for kubernetes-sigs/kustomize#4201
+            export PULSAR_MEM="-Xmx128M";
+            bin/pulsar initialize-cluster-metadata \
+              --cluster {{ template "pulsar.cluster.name" . }} \
+              --metadata-store "{{ template "pulsar.oxia.metadata.url.broker" . }}" \
+              --configuration-store "{{ template "pulsar.oxia.metadata.url.broker" . }}" \
+              --web-service-url http://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}.{{ template "pulsar.namespace" . }}.svc.{{ .Values.clusterDomain }}:{{ .Values.broker.ports.http }}/ \
+              --web-service-url-tls https://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}.{{ template "pulsar.namespace" . }}.svc.{{ .Values.clusterDomain }}:{{ .Values.broker.ports.https }}/ \
+              --broker-service-url pulsar://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}.{{ template "pulsar.namespace" . }}.svc.{{ .Values.clusterDomain }}:{{ .Values.broker.ports.pulsar }}/ \
+              --broker-service-url-tls pulsar+ssl://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}.{{ template "pulsar.namespace" . }}.svc.{{ .Values.clusterDomain }}:{{ .Values.broker.ports.pulsarssl }}/ ;
+            {{- if .Values.extraInitCommand }}
+              {{ .Values.extraInitCommand }}
+            {{- end }}
+        {{- end }}
         volumeMounts:
-        {{- include "pulsar.toolset.certs.volumeMounts" . | nindent 8 }}
+        {{- include "pulsar.toolset.certs.volumeMounts" . | nindent 10 }}
       volumes:
-      {{- include "pulsar.toolset.certs.volumes" . | nindent 6 }}
+      {{- include "pulsar.toolset.certs.volumes" . | nindent 8 }}
       restartPolicy: OnFailure
       {{- if .Values.pulsar_metadata.nodeSelector }}
       nodeSelector:
 {{ toYaml .Values.pulsar_metadata.nodeSelector | indent 8 }}
       {{- end }}
+      {{- if .Values.pulsar_metadata.tolerations }}
+      tolerations:
+{{ toYaml .Values.pulsar_metadata.tolerations | indent 8 }}
+      {{- end }}
 {{- end }}
 {{- end }}

charts/pulsar/templates/pulsar-manager-admin-secret.yaml

@@ -17,23 +17,36 @@
 # under the License.
 #
 
-{{- if and (or .Values.components.pulsar_manager .Values.extra.pulsar_manager) (not .Values.pulsar_manager.existingSecretName) }}
+{{- if and .Values.components.pulsar_manager ( not .Values.pulsar_manager.admin.existingSecret ) }}
 apiVersion: v1
 kind: Secret
 metadata:
   name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_manager.component }}-secret"
   namespace: {{ template "pulsar.namespace" . }}
   labels:
-    app: {{ template "pulsar.name" . }}
-    chart: {{ template "pulsar.chart" . }}
-    release: {{ .Release.Name }}
-    heritage: {{ .Release.Service }}
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
     component: {{ .Values.pulsar_manager.component }}
-    cluster: {{ template "pulsar.fullname" . }}
+    "helm.sh/resource-policy": "keep"  # do not remove when uninstalling to keep it for next install
 type: Opaque
 data:
-  {{- if .Values.pulsar_manager.admin}}
-  PULSAR_MANAGER_ADMIN_PASSWORD: {{ .Values.pulsar_manager.admin.password | default "pulsar" | b64enc }}
-  PULSAR_MANAGER_ADMIN_USER: {{ .Values.pulsar_manager.admin.user | default "pulsar" | b64enc }}
-  {{- end }}
+  {{/*  https://itnext.io/manage-auto-generated-secrets-in-your-helm-charts-5aee48ba6918 */}}
+  {{- $namespace := include "pulsar.namespace" . -}}
+  {{- $fullname := include "pulsar.fullname" . -}}
+  {{- $secretName := printf "%s-%s-secret" $fullname .Values.pulsar_manager.component -}}
+  {{- $secretObj := lookup "v1" "Secret" $namespace $secretName | default dict }}
+  {{- $secretData := (get $secretObj "data") | default dict }}
+
+  {{- $ui_user := ((get $secretData "UI_USERNAME") | b64dec) | default (.Values.pulsar_manager.admin.ui_username) | default ("pulsar") | b64enc }}
+  {{- $ui_password := ((get $secretData "UI_PASSWORD") | b64dec) | default (.Values.pulsar_manager.admin.ui_password) | default (randAlphaNum 32) | b64enc }}
+  UI_USERNAME: {{ $ui_user | quote }}
+  UI_PASSWORD: {{ $ui_password | quote }}
+
+  {{- $db_user := ((get $secretData "DB_USERNAME") | b64dec) | default (.Values.pulsar_manager.admin.db_username) | default ("pulsar") | b64enc }}
+  {{- $db_password := ((get $secretData "DB_PASSWORD") | b64dec) | default (.Values.pulsar_manager.admin.db_password) | default (randAlphaNum 32) | b64enc }}
+  DB_USERNAME: {{ $db_user | quote }}
+  DB_PASSWORD: {{ $db_password | quote }}
+
 {{- end }}
+
+
+

charts/pulsar/templates/pulsar-manager-cluster-initialize.yaml

@@ -0,0 +1,188 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+{{- if or (and .Values.useReleaseStatus .Release.IsInstall) .Values.initialize }}
+{{- if .Values.components.pulsar_manager }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_manager.component }}-init"
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.pulsar_manager.component }}-init
+spec:
+  {{- if and .Values.job.ttl.enabled (semverCompare ">=1.23-0" .Capabilities.KubeVersion.Version) }}
+  ttlSecondsAfterFinished: {{ .Values.job.ttl.secondsAfterFinished | default 600 }}
+  {{- end }}
+  template:
+    metadata:
+      labels:
+        {{- include "pulsar.template.labels" . | nindent 8 }}
+        component: {{ .Values.pulsar_manager.component }}-init
+    spec:
+    {{- include "pulsar.imagePullSecrets" . | nindent 6 }}
+      nodeSelector:
+      {{- if .Values.pulsar_metadata.nodeSelector }}
+      {{ toYaml .Values.pulsar_metadata.nodeSelector | indent 8 }}
+      {{- end }}
+      tolerations:
+      {{- if .Values.pulsar_metadata.tolerations }}
+      {{ toYaml .Values.pulsar_metadata.tolerations | indent 8 }}
+      {{- end }}
+      restartPolicy: OnFailure
+      initContainers:
+        - name: wait-pulsar-manager-ready
+          image: "{{ template "pulsar.imageFullName" (dict "image" .Values.pulsar_metadata.image "root" .) }}"
+          imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.pulsar_metadata.image "root" .) }}"
+          resources: {{ toYaml .Values.initContainer.resources | nindent 12 }}
+          command: [ "sh", "-c" ]
+          args:
+            - |
+              ADMIN_URL={{ template "pulsar.fullname" . }}-{{ .Values.pulsar_manager.component }}-admin:{{ .Values.pulsar_manager.adminService.port }}
+              until $(curl -sS --fail -X GET http://${ADMIN_URL} > /dev/null 2>&1); do
+                  sleep 3;
+              done;
+        # This init container will wait for at least one broker to be ready before
+        # initializing the pulsar-manager
+        {{- if .Values.components.broker }}
+        - name: wait-broker-ready
+          image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.proxy "root" .) }}"
+          imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.pulsar_metadata.image "root" .) }}"
+          resources: {{ toYaml .Values.initContainer.resources | nindent 12 }}
+          command: [ "sh", "-c" ]
+          args:
+            - |
+              set -e;
+              brokerServiceNumber="$(nslookup -timeout=10 {{ template "pulsar.fullname" . }}-{{ .Values.broker.component }} | grep Name | wc -l)";
+              until [ ${brokerServiceNumber} -ge 1 ]; do
+                echo "pulsar cluster {{ template "pulsar.cluster.name" . }} isn't initialized yet ... check in 10 seconds ...";
+                sleep 10;
+                brokerServiceNumber="$(nslookup -timeout=10 {{ template "pulsar.fullname" . }}-{{ .Values.broker.component }} | grep Name | wc -l)";
+              done;
+        {{- end }}
+      containers:
+        - name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_manager.component }}-init"
+          image: "{{ template "pulsar.imageFullName" (dict "image" .Values.pulsar_metadata.image "root" .) }}"
+          imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.pulsar_metadata.image "root" .) }}"
+          {{- if .Values.pulsar_metadata.resources }}
+          resources: {{ toYaml .Values.pulsar_metadata.resources | nindent 12 }}
+          {{- end }}
+          command: [ "sh", "-c" ]
+          args:
+            - |
+              cd /tmp
+              ADMIN_URL={{ template "pulsar.fullname" . }}-{{ .Values.pulsar_manager.component }}-admin:{{ .Values.pulsar_manager.adminService.port }}
+              CSRF_TOKEN=$(curl http://${ADMIN_URL}/pulsar-manager/csrf-token)
+              UI_URL={{ template "pulsar.fullname" . }}-{{ .Values.pulsar_manager.component }}:{{ .Values.pulsar_manager.service.port }}
+
+              {{/* check if account is already existing */}}
+              LOGIN_REPLY=$(curl -v \
+                -X POST http://${UI_URL}/pulsar-manager/login \
+                -H 'Accept: application/json, text/plain, */*' \
+                -H 'Content-Type: application/json' \
+                -H "X-XSRF-TOKEN: $CSRF_TOKEN" \
+                -H "Cookie: XSRF-TOKEN=$CSRF_TOKEN" \
+                -sS -D headers.txt \
+                -d '{"username": "'${USERNAME}'", "password": "'${PASSWORD}'"}')
+              echo "$LOGIN_REPLY"
+
+              if [ -n "$(echo "$LOGIN_REPLY" | grep 'success')" ]; then
+                echo "account already exists"
+              else
+                echo "creating account"
+                {{/* set admin credentials */}}
+                curl -v \
+                  -X PUT http://${ADMIN_URL}/pulsar-manager/users/superuser \
+                  -H "X-XSRF-TOKEN: $CSRF_TOKEN" \
+                  -H "Cookie: XSRF-TOKEN=$CSRF_TOKEN;" \
+                  -H 'Content-Type: application/json' \
+                  -d '{"name": "'"${USERNAME}"'", "password": "'"${PASSWORD}"'", "description": "Helm-managed Admin Account", "email": "'"${USERNAME}"'@pulsar.org"}'
+                {{/* login as admin */}}
+                LOGIN_REPLY=$(curl -v \
+                  -X POST http://${UI_URL}/pulsar-manager/login \
+                  -H 'Accept: application/json, text/plain, */*' \
+                  -H 'Content-Type: application/json' \
+                  -H "X-XSRF-TOKEN: $CSRF_TOKEN" \
+                  -H "Cookie: XSRF-TOKEN=$CSRF_TOKEN" \
+                  -sS -D headers.txt \
+                  -d '{"username": "'${USERNAME}'", "password": "'${PASSWORD}'"}')
+                echo "$LOGIN_REPLY"
+              fi
+
+              {{- if .Values.components.broker }}
+              LOGIN_TOKEN=$(grep "token:" headers.txt | sed 's/^.*: //')
+              LOGIN_JSESSSIONID=$(grep -o "JSESSIONID=[a-zA-Z0-9_]*" headers.txt | sed 's/^.*=//')
+
+              {{/* create environment */}}
+              {{- if or (not .Values.tls.enabled) (not .Values.tls.broker.enabled) }}
+              BROKER_URL="http://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.http }}"
+              {{- else }}
+              BROKER_URL="https://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.https }}"
+              {{- end }}
+              BOOKIE_URL="http://{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}:{{ .Values.bookkeeper.ports.http }}"
+              echo '{ "name": "{{ template "pulsar.fullname" . }}", "broker": "'$BROKER_URL'", "bookie": "'$BOOKIE_URL'"}'
+
+              ENVIRONMENT_REPLY=$(curl -v \
+                -X PUT http://${UI_URL}/pulsar-manager/environments/environment \
+                -H 'Content-Type: application/json' \
+                -H "token: $LOGIN_TOKEN" \
+                -H "X-XSRF-TOKEN: $CSRF_TOKEN" \
+                -H "username: $USERNAME" \
+                -H "Cookie: XSRF-TOKEN=$CSRF_TOKEN; JSESSIONID=$LOGIN_JSESSSIONID;" \
+                -d '{ "name": "{{ template "pulsar.fullname" . }}", "broker": "'$BROKER_URL'", "bookie": "'$BOOKIE_URL'"}')
+              echo "$ENVIRONMENT_REPLY"
+
+              if [ -n "$(echo "$ENVIRONMENT_REPLY" | grep -e 'success' -e 'exist')" ]; then
+                echo "Successfully created / found existing environment"
+                exit 0
+              else
+                echo "Error creating environment"
+                exit 1
+              fi
+              {{- else }}
+              if [ -n "$(echo "$LOGIN_REPLY" | grep 'success')" ]; then
+                echo "Successfully created / found existing account"
+                exit 0
+              else
+                echo "Error creating account"
+                exit 1
+              fi
+              {{- end }}
+          env:
+            - name: USERNAME
+              valueFrom:
+                secretKeyRef:
+                  {{- if .Values.pulsar_manager.admin.existingSecret }}
+                  name: {{ .Values.pulsar_manager.admin.existingSecret | quote }}
+                  {{- else }}
+                  name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_manager.component }}-secret"
+                  {{- end }}
+                  key: UI_USERNAME
+            - name: PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  {{- if .Values.pulsar_manager.admin.existingSecret }}
+                  name: {{ .Values.pulsar_manager.admin.existingSecret | quote }}
+                  {{- else }}
+                  name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_manager.component }}-secret"
+                  {{- end }}
+                  key: UI_PASSWORD
+{{- end }}
+{{- end }}

charts/pulsar/templates/pulsar-manager-configmap.yaml

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-{{- if or .Values.components.pulsar_manager .Values.extra.pulsar_manager }}
+{{- if .Values.components.pulsar_manager }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -27,5 +27,18 @@ metadata:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
     component: {{ .Values.pulsar_manager.component }}
 data:
+  PULSAR_CLUSTER: {{ template "pulsar.fullname" . }}
+  PULSAR_MANAGER_OPTS: "-Dlog4j2.formatMsgNoLookups=true"
+  {{- if .Values.auth.authentication.enabled }}
+  # auth
+  {{- if .Values.auth.authentication.jwt.enabled }}
+  {{- if .Values.auth.authentication.jwt.usingSecretKey }}
+  SECRET_KEY: "file:///pulsar-manager/keys/token/secret.key"
+  {{- else }}
+  PRIVATE_KEY: "file:///pulsar-manager/keys/token/private.key"
+  PUBLIC_KEY: "file:///pulsar-manager/keys/token/public.key"
+  {{- end }}
+  {{- end }}
+  {{- end }}
 {{ toYaml .Values.pulsar_manager.configData | indent 2}}
 {{- end }}

charts/pulsar/templates/pulsar-manager-deployment.yaml

@@ -1,101 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-{{- if or .Values.components.pulsar_manager .Values.extra.pulsar_manager }}
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_manager.component }}"
-  namespace: {{ template "pulsar.namespace" . }}
-  labels:
-    {{- include "pulsar.standardLabels" . | nindent 4 }}
-    component: {{ .Values.pulsar_manager.component }}
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      {{- include "pulsar.matchLabels" . | nindent 6 }}
-      component: {{ .Values.pulsar_manager.component }}
-  template:
-    metadata:
-      labels:
-        {{- include "pulsar.template.labels" . | nindent 8 }}
-        component: {{ .Values.pulsar_manager.component }}
-      annotations:
-        {{- if .Values.pulsar_manager.restartPodsOnConfigMapChange }}
-        checksum/config: {{ include (print $.Template.BasePath "/pulsar-manager-configmap.yaml") . | sha256sum }}
-        {{- end }}
-{{- with .Values.pulsar_manager.annotations }}
-{{ toYaml . | indent 8 }}
-{{- end }}
-    spec:
-    {{- if .Values.pulsar_manager.nodeSelector }}
-      nodeSelector:
-{{ toYaml .Values.pulsar_manager.nodeSelector | indent 8 }}
-    {{- end }}
-    {{- if .Values.pulsar_manager.tolerations }}
-      tolerations:
-{{ toYaml .Values.pulsar_manager.tolerations | indent 8 }}
-    {{- end }}
-      terminationGracePeriodSeconds: {{ .Values.pulsar_manager.gracePeriod }}
-      containers:
-        - name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_manager.component }}"
-          image: "{{ .Values.images.pulsar_manager.repository }}:{{ .Values.images.pulsar_manager.tag }}"
-          imagePullPolicy: {{ .Values.images.pulsar_manager.pullPolicy }}
-        {{- if .Values.pulsar_manager.resources }}
-          resources:
-{{ toYaml .Values.pulsar_manager.resources | indent 12 }}
-        {{- end }}
-          ports:
-          - containerPort: {{ .Values.pulsar_manager.service.targetPort }}
-          volumeMounts:
-          - name: pulsar-manager-data
-            mountPath: /data
-          envFrom:
-          - configMapRef:
-              name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_manager.component }}"
-          env:
-          - name: PULSAR_CLUSTER
-            value: {{ template "pulsar.fullname" . }}
-          - name: USERNAME
-            valueFrom:
-              secretKeyRef:
-                key: PULSAR_MANAGER_ADMIN_USER
-                {{- if .Values.pulsar_manager.existingSecretName }}
-                name: "{{ .Values.pulsar_manager.existingSecretName }}"
-                {{- else }}
-                name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_manager.component }}-secret"
-                {{- end }}
-          - name: PASSWORD
-            valueFrom:
-              secretKeyRef:
-                key: PULSAR_MANAGER_ADMIN_PASSWORD
-                {{- if .Values.pulsar_manager.existingSecretName }}
-                name: "{{ .Values.pulsar_manager.existingSecretName }}"
-                {{- else }}
-                name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_manager.component }}-secret"
-                {{- end }}
-          - name: PULSAR_MANAGER_OPTS
-            value: "$(PULSAR_MANAGER_OPTS) -Dlog4j2.formatMsgNoLookups=true"
-        {{- include "pulsar.imagePullSecrets" . | nindent 6}}
-      volumes:
-        - name: pulsar-manager-data
-          emptyDir: {}
-
-{{- end }}