Feature/pulsar manager initialize (#457)

* add better pulsar manager integration and init along with tests & docs * fix pulsar manager startup args * update pulsar manager service to ClusterIP + remove duplicate
2024-02-14 19:13:54 +01:00 · 2024-02-14 19:13:54 +01:00 · d0b784a953
commit d0b784a953
parent 1f20887f09
11 changed files with 295 additions and 41 deletions
--- a/.ci/chart_test.sh
+++ b/.ci/chart_test.sh
@ -81,5 +81,10 @@ if [[ "$(ci::helm_values_for_deployment | yq .components.functions)" == "true" ]
    ci::test_pulsar_function
 fi

+if [[ "$(ci::helm_values_for_deployment | yq .components.pulsar_manager)" == "true" ]]; then
+    # test manager
+    ci::test_pulsar_manager
+fi
+
 # delete the cluster
 ci::delete_cluster
--- a/.ci/clusters/values-pulsar-manager.yaml
+++ b/.ci/clusters/values-pulsar-manager.yaml
@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+components:
+  pulsar_manager: true
--- a/.ci/helm.sh
+++ b/.ci/helm.sh
@ -133,9 +133,8 @@ function ci::install_pulsar_chart() {
                --timeout=90s
      # configure metallb
      ${KUBECTL} apply -f ${BINDIR}/metallb/metallb-config.yaml
-
      install_args=""
-    else 
+    else
      install_args="--wait --wait-for-jobs --timeout 300s --debug"
    fi

@ -351,3 +350,43 @@ function ci::test_pulsar_function() {
    echo "Consuming output message"
    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client consume -s test pulsar-ci/test/test_output
 }
+
+function ci::test_pulsar_manager() {
+  echo "Testing pulsar manager"
+
+  until ${KUBECTL} get jobs -n ${NAMESPACE} ${CLUSTER}-pulsar-manager-init -o json | jq -r '.status.conditions[] | select (.type | test("Complete")).status' | grep True; do sleep 3; done
+
+
+  echo "Checking Podname"
+  podname=$(${KUBECTL} get pods -n ${NAMESPACE} -l component=pulsar-manager --no-headers -o custom-columns=":metadata.name")
+  echo "Getting pulsar manager UI password"
+  PASSWORD=$(${KUBECTL} get secret -n ${NAMESPACE} -l component=pulsar-manager -o=jsonpath="{.items[0].data.UI_PASSWORD}" | base64 --decode)
+
+  echo "Getting CSRF_TOKEN"
+  CSRF_TOKEN=$(${KUBECTL} exec -n ${NAMESPACE} ${podname} -- curl http://127.0.0.1:7750/pulsar-manager/csrf-token)
+
+  echo "Performing login"
+  ${KUBECTL} exec -n ${NAMESPACE} ${podname} -- curl -X POST http://127.0.0.1:9527/pulsar-manager/login \
+                                                 -H 'Accept: application/json, text/plain, */*' \
+                                                 -H 'Content-Type: application/json' \
+                                                 -H "X-XSRF-TOKEN: $CSRF_TOKEN" \
+                                                 -H "Cookie: XSRF-TOKEN=$CSRF_TOKEN" \
+                                                 -sS -D headers.txt \
+                                                 -d '{"username": "pulsar", "password": "'${PASSWORD}'"}'
+  LOGIN_TOKEN=$(${KUBECTL} exec -n ${NAMESPACE} ${podname} -- grep "token:" headers.txt | sed 's/^.*: //')
+  LOGIN_JSESSSIONID=$(${KUBECTL} exec -n ${NAMESPACE} ${podname} -- grep -o "JSESSIONID=[a-zA-Z0-9_]*" headers.txt | sed 's/^.*=//')
+
+  echo "Checking environment"
+  envs=$(${KUBECTL} exec -n ${NAMESPACE} ${podname} -- curl -X GET http://localhost:9527/pulsar-manager/environments \
+                  -H 'Content-Type: application/json' \
+                  -H "token: $LOGIN_TOKEN" \
+                  -H "X-XSRF-TOKEN: $CSRF_TOKEN" \
+                  -H "username: pulsar" \
+                  -H "Cookie: XSRF-TOKEN=$CSRF_TOKEN; JSESSIONID=$LOGIN_JSESSSIONID;")
+  number_of_envs=$(echo $envs | jq '.total')
+  if [ "$number_of_envs" -ne 1 ]; then
+    echo "Error: Did not find expected environment"
+    exit 1
+  fi
+}
+
--- a/.github/workflows/pulsar-helm-chart-ci.yaml
+++ b/.github/workflows/pulsar-helm-chart-ci.yaml
@ -205,6 +205,9 @@ jobs:
          - name: PSP
            values_file: .ci/clusters/values-psp.yaml
            shortname: psp
+          - name: Pulsar Manager
+            values_file: .ci/clusters/values-pulsar-manager.yaml
+            shortname: pulsar-manager
        include:
          - k8sVersion:
              version: "1.21.14"
--- a/README.md
+++ b/README.md
@ -160,6 +160,26 @@ Otherwise, the helm chart installation will attempt to install the CRDs for the
 you'll need to disable each of the component's `PodMonitors`. This is shown in some [examples](./examples) and is
 verified in some [tests](./.ci/clusters).

+## Pulsar Manager
+
+The Pulsar Manager can be deployed alongside the pulsar cluster instance.
+Depending on the given settings it uses an existing Secret within the given namespace or creates a new one, with random
+passwords for both, the UI and the internal database.
+
+To forward the UI use (assumes you did not change the namespace):
+
+```
+kubectl port-forward $(kubectl get pods -l component=pulsar-manager -o jsonpath='{.items[0].metadata.name}') 9527:9527
+```
+
+And then opening the browser to http://localhost:9527
+
+The default user is `pulsar` and you can find out the password with this command
+
+```
+kubectl get secret -l component=pulsar-manager -o=jsonpath="{.items[0].data.UI_PASSWORD}" | base64 --decode
+```
+
 ## Grafana Dashboards

 The Apache Pulsar Helm Chart uses the `kube-prometheus-stack` Helm Chart to deploy Grafana.
--- a/charts/pulsar/templates/pulsar-manager-admin-secret.yaml
+++ b/charts/pulsar/templates/pulsar-manager-admin-secret.yaml
@ -17,7 +17,7 @@
 # under the License.
 #

-{{- if and (or .Values.components.pulsar_manager .Values.extra.pulsar_manager) (not .Values.pulsar_manager.existingSecretName) }}
+{{- if and (or .Values.components.pulsar_manager .Values.extra.pulsar_manager) }}
 apiVersion: v1
 kind: Secret
 metadata:
@ -30,10 +30,27 @@ metadata:
    heritage: {{ .Release.Service }}
    component: {{ .Values.pulsar_manager.component }}
    cluster: {{ template "pulsar.fullname" . }}
+    "helm.sh/resource-policy": "keep"  # do not remove when uninstalling to keep it for next install
 type: Opaque
 data:
-  {{- if .Values.pulsar_manager.admin}}
-  PULSAR_MANAGER_ADMIN_PASSWORD: {{ .Values.pulsar_manager.admin.password | default "pulsar" | b64enc }}
-  PULSAR_MANAGER_ADMIN_USER: {{ .Values.pulsar_manager.admin.user | default "pulsar" | b64enc }}
-  {{- end }}
+  {{/*  https://itnext.io/manage-auto-generated-secrets-in-your-helm-charts-5aee48ba6918 */}}
+  {{- $namespace := include "pulsar.namespace" . -}}
+  {{- $fullname := include "pulsar.fullname" . -}}
+  {{- $secretName := printf "%s-%s-secret" $fullname .Values.pulsar_manager.component -}}
+  {{- $secretObj := lookup "v1" "Secret" $namespace $secretName | default dict }}
+  {{- $secretData := (get $secretObj "data") | default dict }}
+
+  {{- $ui_user := (get $secretData "UI_USERNAME") | default (.Values.pulsar_manager.admin.ui_username) | default ("pulsar") | b64enc }}
+  {{- $ui_password := (get $secretData "UI_PASSWORD") | default (.Values.pulsar_manager.admin.ui_password) | default (randAlphaNum 32) | b64enc }}
+  UI_USERNAME: {{ $ui_user | quote }}
+  UI_PASSWORD: {{ $ui_password | quote }}
+
+  {{- $db_user := (get $secretData "DB_USERNAME") | default (.Values.pulsar_manager.admin.db_username) | default ("pulsar") | b64enc }}
+  {{- $db_password := (get $secretData "DB_PASSWORD") | default (.Values.pulsar_manager.admin.db_password) | default (randAlphaNum 32) | b64enc }}
+  DB_USERNAME: {{ $db_user | quote }}
+  DB_PASSWORD: {{ $db_password | quote }}
+
 {{- end }}
+
+
+
--- a/charts/pulsar/templates/pulsar-manager-cluster-initialize.yaml
+++ b/charts/pulsar/templates/pulsar-manager-cluster-initialize.yaml
@ -0,0 +1,135 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+{{- if or .Release.IsInstall .Values.initialize }}
+{{- if or .Values.components.pulsar_manager .Values.extra.pulsar_manager }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_manager.component }}-init"
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.pulsar_manager.component }}-init
+spec:
+  {{- if or .Values.job.ttl.enabled (semverCompare ">=1.23-0" .Capabilities.KubeVersion.Version) }}
+  ttlSecondsAfterFinished: {{ .Values.job.ttl.secondsAfterFinished | default 600 }}
+  {{- end }}
+  template:
+    spec:
+      nodeSelector:
+      {{- if .Values.pulsar_metadata.nodeSelector }}
+      {{ toYaml .Values.pulsar_metadata.nodeSelector | indent 8 }}
+      {{- end }}
+      tolerations:
+      {{- if .Values.pulsar_metadata.tolerations }}
+      {{ toYaml .Values.pulsar_metadata.tolerations | indent 8 }}
+      {{- end }}
+      restartPolicy: OnFailure
+      initContainers:
+        - name: wait-pulsar-manager-ready
+          image: "{{ template "pulsar.imageFullName" (dict "image" .Values.pulsar_metadata.image "root" .) }}"
+          imagePullPolicy: {{ .Values.pulsar_metadata.image.pullPolicy }}
+          resources: {{ toYaml .Values.initContainer.resources | nindent 12 }}
+          command: [ "sh", "-c" ]
+          args:
+            - |
+              ADMIN_URL={{ template "pulsar.fullname" . }}-{{ .Values.pulsar_manager.component }}-admin:{{ .Values.pulsar_manager.adminService.port }}
+              until $(curl -sS --fail -X GET http://${ADMIN_URL} > /dev/null 2>&1); do
+                  sleep 3;
+              done;
+        # This init container will wait for at least one broker to be ready before
+        # initializing the pulsar-manager
+        - name: wait-broker-ready
+          image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.proxy "root" .) }}"
+          imagePullPolicy: {{ .Values.images.proxy.pullPolicy }}
+          resources: {{ toYaml .Values.initContainer.resources | nindent 12 }}
+          command: [ "sh", "-c" ]
+          args:
+            - >-
+              set -e;
+              brokerServiceNumber="$(nslookup -timeout=10 {{ template "pulsar.fullname" . }}-{{ .Values.broker.component }} | grep Name | wc -l)";
+              until [ ${brokerServiceNumber} -ge 1 ]; do
+                echo "pulsar cluster {{ template "pulsar.cluster.name" . }} isn't initialized yet ... check in 10 seconds ...";
+                sleep 10;
+                brokerServiceNumber="$(nslookup -timeout=10 {{ template "pulsar.fullname" . }}-{{ .Values.broker.component }} | grep Name | wc -l)";
+              done;
+      containers:
+        - name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_manager.component }}-init"
+          image: "{{ template "pulsar.imageFullName" (dict "image" .Values.pulsar_metadata.image "root" .) }}"
+          imagePullPolicy: {{ .Values.pulsar_metadata.image.pullPolicy }}
+          {{- if .Values.pulsar_metadata.resources }}
+          resources: {{ toYaml .Values.pulsar_metadata.resources | nindent 12 }}
+          {{- end }}
+          command: [ "sh", "-c" ]
+          args:
+            - |
+              ADMIN_URL={{ template "pulsar.fullname" . }}-{{ .Values.pulsar_manager.component }}-admin:{{ .Values.pulsar_manager.adminService.port }}
+              CSRF_TOKEN=$(curl http://${ADMIN_URL}/pulsar-manager/csrf-token)
+              {{/* set admin credentials */}}
+              curl -v \
+                -X PUT http://${ADMIN_URL}/pulsar-manager/users/superuser \
+                -H "X-XSRF-TOKEN: $CSRF_TOKEN" \
+                -H "Cookie: XSRF-TOKEN=$CSRF_TOKEN;" \
+                -H 'Content-Type: application/json' \
+                -d '{"name": "'"${USERNAME}"'", "password": "'"${PASSWORD}"'", "description": "Helm-managed Admin Account", "email": "'"${USERNAME}"'@pulsar.org"}'
+
+              UI_URL={{ template "pulsar.fullname" . }}-{{ .Values.pulsar_manager.component }}:{{ .Values.pulsar_manager.service.port }}
+              {{/* login as admin */}}
+              curl -v \
+                -X POST http://${UI_URL}/pulsar-manager/login \
+                -H 'Accept: application/json, text/plain, */*' \
+                -H 'Content-Type: application/json' \
+                -H "X-XSRF-TOKEN: $CSRF_TOKEN" \
+                -H "Cookie: XSRF-TOKEN=$CSRF_TOKEN" \
+                -sS -D headers.txt \
+                -d '{"username": "'${USERNAME}'", "password": "'${PASSWORD}'"}'
+
+              LOGIN_TOKEN=$(grep "token:" headers.txt | sed 's/^.*: //')
+              LOGIN_JSESSSIONID=$(grep -o "JSESSIONID=[a-zA-Z0-9_]*" headers.txt | sed 's/^.*=//')
+
+              {{/* create environment */}}
+              {{- if or (not .Values.tls.enabled) (not .Values.tls.broker.enabled) }}
+              BROKER_URL="http://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.http }}"
+              {{- else }}
+              BROKER_URL="https://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.https }}"
+              {{- end }}
+              BOOKIE_URL="http://{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}:{{ .Values.bookkeeper.ports.http }}"
+
+              curl -v \
+                -X PUT http://${UI_URL}/pulsar-manager/environments/environment \
+                -H 'Content-Type: application/json' \
+                -H "token: $LOGIN_TOKEN" \
+                -H "X-XSRF-TOKEN: $CSRF_TOKEN" \
+                -H "username: $USERNAME" \
+                -H "Cookie: XSRF-TOKEN=$CSRF_TOKEN; JSESSIONID=$LOGIN_JSESSSIONID;" \
+                -d '{ "name": "{{ template "pulsar.fullname" . }}", "broker": "'$BROKER_URL'", "bookie": "'$BOOKIE_URL'"}'
+          env:
+            - name: USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_manager.component }}-secret"
+                  key: UI_USERNAME
+            - name: PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_manager.component }}-secret"
+                  key: UI_PASSWORD
+{{- end }}
+{{- end }}
--- a/charts/pulsar/templates/pulsar-manager-deployment.yaml
+++ b/charts/pulsar/templates/pulsar-manager-deployment.yaml
@ -64,7 +64,7 @@ spec:
        {{- end }}
          ports:
          - containerPort: {{ .Values.pulsar_manager.service.targetPort }}
-          - containerPort: {{ .Values.pulsar_manager.service.adminTargetPort }}
+          - containerPort: {{ .Values.pulsar_manager.adminService.targetPort }}
          volumeMounts:
          - name: pulsar-manager-data
            mountPath: /data
@ -77,21 +77,13 @@ spec:
          - name: USERNAME
            valueFrom:
              secretKeyRef:
-                key: PULSAR_MANAGER_ADMIN_USER
-                {{- if .Values.pulsar_manager.existingSecretName }}
-                name: "{{ .Values.pulsar_manager.existingSecretName }}"
-                {{- else }}
                name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_manager.component }}-secret"
-                {{- end }}
+                key: DB_USERNAME
          - name: PASSWORD
            valueFrom:
              secretKeyRef:
-                key: PULSAR_MANAGER_ADMIN_PASSWORD
-                {{- if .Values.pulsar_manager.existingSecretName }}
-                name: "{{ .Values.pulsar_manager.existingSecretName }}"
-                {{- else }}
                name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_manager.component }}-secret"
-                {{- end }}
+                key: DB_PASSWORD
          - name: PULSAR_MANAGER_OPTS
            value: "$(PULSAR_MANAGER_OPTS) -Dlog4j2.formatMsgNoLookups=true"
        {{- include "pulsar.imagePullSecrets" . | nindent 6}}
--- a/charts/pulsar/templates/pulsar-manager-service.yaml
+++ b/charts/pulsar/templates/pulsar-manager-service.yaml
@ -41,15 +41,31 @@ spec:
      port: {{ .Values.pulsar_manager.service.port }}
      targetPort: {{ .Values.pulsar_manager.service.targetPort }}
      protocol: TCP
-    - name: admin
-      port: {{ .Values.pulsar_manager.service.adminPort }}
-      targetPort: {{ .Values.pulsar_manager.service.adminTargetPort }}
+  selector:
+    {{- include "pulsar.matchLabels" . | nindent 4 }}
+    component: {{ .Values.pulsar_manager.component }}
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_manager.component }}-admin"
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.pulsar_manager.component }}
+  annotations:
+{{ toYaml .Values.pulsar_manager.adminService.annotations | indent 4 }}
+spec:
+  type: {{ .Values.pulsar_manager.adminService.type }}
+  ports:
+    - port: {{ .Values.pulsar_manager.adminService.port }}
+      targetPort: {{ .Values.pulsar_manager.adminService.targetPort }}
      protocol: TCP
  selector:
    {{- include "pulsar.matchLabels" . | nindent 4 }}
    component: {{ .Values.pulsar_manager.component }}
-{{- if .Values.pulsar_manager.service.loadBalancerSourceRanges }}
-  loadBalancerSourceRanges:
-{{ toYaml .Values.pulsar_manager.service.loadBalancerSourceRanges | indent 4 }}
-{{- end }}
+
 {{- end }}
+
--- a/charts/pulsar/values.yaml
+++ b/charts/pulsar/values.yaml
@ -1314,17 +1314,18 @@ pulsar_manager:
    ## If you enabled authentication support
    ## JWT_TOKEN: <token>
    ## SECRET_KEY: data:base64,<secret key>
+
+    # the pulsar manager image relies on these variables, if they are not set the backend will keep crashing
+    # however, feel free to overwrite them
+    SPRING_CONFIGURATION_FILE: "/pulsar-manager/pulsar-manager/application.properties"
+    PULSAR_MANAGER_OPTS: " -Dlog4j2.formatMsgNoLookups=true"
  ## Pulsar manager service
  ## templates/pulsar-manager-service.yaml
  ##
  service:
-    type: LoadBalancer
-
+    type: ClusterIP
    port: 9527
    targetPort: 9527
-    adminPort: 7750
-    adminTargetPort: 7750
-
    annotations: {}
    ## Set external traffic policy to: "Local" to preserve source IP on providers supporting it.
    ## Ref: https://kubernetes.io/docs/tutorials/services/source-ip/#source-ip-for-services-with-typeloadbalancer
@ -1332,6 +1333,11 @@ pulsar_manager:
    ## Restrict traffic through the load balancer to specified IPs on providers supporting it.
    # loadBalancerSourceRanges:
    #   - 10.0.0.0/8
+  adminService:
+    type: ClusterIP
+    port: 7750
+    targetPort: 7750
+    annotations: {}
  ## Pulsar manager ingress
  ## templates/pulsar-manager-ingress.yaml
  ##
@ -1348,15 +1354,21 @@ pulsar_manager:
    hostname: ""
    path: "/"

-  ## If set use existing secret with specified name to set pulsar admin credentials.
-  existingSecretName:
+  ## On first install, the helm chart tries to reuse an existing secret with matching name by default
+  ## if this should fail it uses the given username and password to create a new secret
+  ## if either are missing the default value of "pulsar" is used for the username or a random password is generated
+  ## And decode any key by using:
+  ## kubectl get secret -l component=pulsar-manager -o=jsonpath="{.items[0].data.UI_PASSWORD}" | base64 --decode
  admin:
-    user: pulsar
-    password: pulsar
+    ui_username: "pulsar"
+    ui_password: ""  # leave empty for random password
+    db_username: "pulsar"
+    db_password: ""  # leave empty for random password

 # These are jobs where job ttl configuration is used
 # pulsar-helm-chart/charts/pulsar/templates/pulsar-cluster-initialize.yaml
 # pulsar-helm-chart/charts/pulsar/templates/bookkeeper-cluster-initialize.yaml
+# pulsar-helm-chart/charts/pulsar/templates/pulsar-manager-cluster-initialize.yaml
 job:
  ttl:
    enabled: false
--- a/examples/values-minikube.yaml
+++ b/examples/values-minikube.yaml
@ -55,10 +55,4 @@ broker:
    managedLedgerDefaultAckQuorum: "1"

 proxy:
-  replicaCount: 1
-
-pulsar_manager:
-  configData:
-    ENV_SPRING_CONFIGURATION_FILE: "/pulsar-manager/pulsar-manager/application.properties"
-    SPRING_CONFIGURATION_FILE: "/pulsar-manager/pulsar-manager/application.properties"
-    PULSAR_MANAGER_OPTS: " -Dlog4j2.formatMsgNoLookups=true"
+  replicaCount: 1