Add support for using Oxia as the metadata store for Pulsar and BookKeeper (#544)

Co-authored-by: Lari Hotari <lhotari@apache.org>

.ci/clusters/values-oxia.yaml

@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+components:
+  zookeeper: false
+  oxia: true
+  # disable functions for oxia tests since there's no support for Oxia in 
+  # BookKeeperPackagesStorage which requires Zookeeper
+  functions: false
+
+oxia:
+  initialShardCount: 1
+  replicationFactor: 1
+  server:
+    replicas: 1
+    memoryLimit: 256Mi
+    dbCacheSizeMb: 128
+    storageSize: 1Gi

.ci/values-common.yaml

@@ -86,3 +86,11 @@ proxy:
 
 toolset:
   useProxy: false
+
+oxia:
+  coordinator:
+    podMonitor:
+      enabled: false
+  server:
+    podMonitor:
+      enabled: false

.github/workflows/pulsar-helm-chart-ci.yaml

@@ -140,15 +140,22 @@ jobs:
           helm dependency build charts/pulsar
           validate_helm_template_with_k8s_version() {
             local kube_version=$1
-            echo "Validating helm template with kubeconform for k8s version $kube_version"
+            shift
-            helm template charts/pulsar --set kube-prometheus-stack.enabled=false --set components.pulsar_manager=true --kube-version $kube_version | \
+            echo -n "Validating helm template with kubeconform for k8s version $kube_version"
+            if [ $# -gt 0 ]; then
+              echo " Extra args: $*"
+            else
+              echo ""
+            fi
+            helm template charts/pulsar --set kube-prometheus-stack.enabled=false --set components.pulsar_manager=true --kube-version $kube_version "$@" | \
               kubeconform -schema-location default -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' -strict -kubernetes-version $kube_version -summary
           }
           set -o pipefail
           for k8s_version_part in {23..30}; do
             k8s_version="1.${k8s_version_part}.0"
-            echo "Validating helm template with kubeconform for k8s version $k8s_version"
             validate_helm_template_with_k8s_version $k8s_version
+            echo "Validating with Oxia enabled"
+            validate_helm_template_with_k8s_version $k8s_version --set components.zookeeper=false --set components.oxia=true
           done
       - name: Wait for ssh connection when build fails
         # ssh access is enabled for builds in own forks
@@ -208,6 +215,9 @@ jobs:
           - name: Pulsar Manager
             values_file: .ci/clusters/values-pulsar-manager.yaml
             shortname: pulsar-manager
+          - name: Oxia
+            values_file: .ci/clusters/values-oxia.yaml
+            shortname: oxia
         include:
           - k8sVersion:
               version: "1.23.17"

charts/pulsar/templates/_autorecovery.tpl

@@ -92,6 +92,7 @@ Define autorecovery init container : verify cluster id
 */}}
 {{- define "pulsar.autorecovery.init.verify_cluster_id" -}}
 bin/apply-config-from-env.py conf/bookkeeper.conf;
+export BOOKIE_MEM="-Xmx128M";
 {{- include "pulsar.autorecovery.zookeeper.tls.settings" . -}}
 until timeout 15 bin/bookkeeper shell whatisinstanceid; do
   sleep 3;

charts/pulsar/templates/_bookkeeper.tpl

@@ -92,8 +92,12 @@ Define bookie tls certs volumes
 Define bookie common config
 */}}
 {{- define "pulsar.bookkeeper.config.common" -}}
+{{- if .Values.components.zookeeper }}
 zkServers: "{{ template "pulsar.zookeeper.connect" . }}"
 zkLedgersRootPath: "{{ .Values.metadataPrefix }}/ledgers"
+{{- else if .Values.components.oxia }}
+metadataServiceUri: "{{ template "pulsar.oxia.metadata.url.bookkeeper" . }}"
+{{- end }}
 # enable bookkeeper http server
 httpServerEnabled: "true"
 httpServerPort: "{{ .Values.bookkeeper.ports.http }}"
@@ -123,6 +127,7 @@ Define bookie init container : verify cluster id
 {{- define "pulsar.bookkeeper.init.verify_cluster_id" -}}
 {{- if not (and .Values.volumes.persistence .Values.bookkeeper.volumes.persistence) }}
 bin/apply-config-from-env.py conf/bookkeeper.conf;
+export BOOKIE_MEM="-Xmx128M";
 {{- include "pulsar.bookkeeper.zookeeper.tls.settings" . -}}
 until timeout 15 bin/bookkeeper shell whatisinstanceid; do
   sleep 3;
@@ -132,6 +137,7 @@ bin/bookkeeper shell bookieformat -nonInteractive -force -deleteCookie || true
 {{- if and .Values.volumes.persistence .Values.bookkeeper.volumes.persistence }}
 set -e;
 bin/apply-config-from-env.py conf/bookkeeper.conf;
+export BOOKIE_MEM="-Xmx128M";
 {{- include "pulsar.bookkeeper.zookeeper.tls.settings" . -}}
 until timeout 15 bin/bookkeeper shell whatisinstanceid; do
   sleep 3;

charts/pulsar/templates/_helpers.tpl

@@ -135,3 +135,4 @@ Lookup pull policy, default to defaultPullPolicy
 {{- define "pulsar.imagePullPolicy" -}}
 {{- printf "%s" (.image.pullPolicy | default .root.Values.defaultPullPolicy) -}}
 {{- end -}}
+

charts/pulsar/templates/_oxia.tpl

@@ -0,0 +1,110 @@
+{{/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+*/}}
+
+{{/*
+Probe
+*/}}
+{{- define "oxia-cluster.probe" -}}
+exec:
+  command: ["oxia", "health", "--port={{ . }}"]
+initialDelaySeconds: 10
+timeoutSeconds: 10
+{{- end }}
+
+
+{{/*
+Probe
+*/}}
+{{- define "oxia-cluster.readiness-probe" -}}
+exec:
+  command: ["oxia", "health", "--port={{ . }}", "--service=oxia-readiness"]
+initialDelaySeconds: 10
+timeoutSeconds: 10
+{{- end }}
+
+{{/*
+Probe
+*/}}
+{{- define "oxia-cluster.startup-probe" -}}
+exec:
+  command: ["oxia", "health", "--port={{ . }}"]
+initialDelaySeconds: 60
+timeoutSeconds: 10
+{{- end }}
+
+{{/*
+Define the pulsar oxia
+*/}}
+{{- define "pulsar.oxia.server.service" -}}
+{{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-svc
+{{- end }}
+
+{{/*
+oxia url for broker metadata
+*/}}
+{{- define "pulsar.oxia.metadata.url.broker" -}}
+{{- if .Values.components.oxia -}}
+oxia://{{ template "pulsar.oxia.server.service" . }}:{{ .Values.oxia.server.ports.public }}/broker
+{{- end -}}
+{{- end -}}
+
+{{/*
+oxia url for bookkeeper metadata
+*/}}
+{{- define "pulsar.oxia.metadata.url.bookkeeper" -}}
+{{- if .Values.components.oxia -}}
+metadata-store:oxia://{{ template "pulsar.oxia.server.service" . }}:{{ .Values.oxia.server.ports.public }}/bookkeeper
+{{- end -}}
+{{- end -}}
+
+{{/*
+Define coordinator configmap
+*/}}
+{{- define "oxia.coordinator.config.yaml" -}}
+namespaces:
+  - name: default
+    initialShardCount: {{ .Values.oxia.initialShardCount }}
+    replicationFactor: {{ .Values.oxia.replicationFactor }}
+  - name: broker
+    initialShardCount: {{ .Values.oxia.initialShardCount }}
+    replicationFactor: {{ .Values.oxia.replicationFactor }}
+  - name: bookkeeper
+    initialShardCount: {{ .Values.oxia.initialShardCount }}
+    replicationFactor: {{ .Values.oxia.replicationFactor }}
+servers:
+  - public:  {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-svc.{{ template "pulsar.namespace" . }}.svc.cluster.local:{{ .Values.oxia.server.ports.public }}
+    internal:  {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-svc.{{ template "pulsar.namespace" . }}.svc:{{ .Values.oxia.server.ports.internal }}
+{{- end }}
+
+{{/*
+Define coordinator entrypoint
+*/}}
+{{- define "oxia.coordinator.entrypoint" -}}
+- "oxia"
+- "coordinator"
+- "--conf=configmap:{{ template "pulsar.namespace" . }}/{{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator"
+- "--log-json"
+- "--metadata=configmap"
+- "--k8s-namespace={{ template "pulsar.namespace" . }}"
+- "--k8s-configmap-name={{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator-status"
+{{- if .Values.oxia.pprofEnabled }}
+- "--profile"
+{{- end}}
+{{- end}}
+

charts/pulsar/templates/_zookeeper.tpl

@@ -56,3 +56,4 @@ Define zookeeper tls settings
 /pulsar/keytool/keytool.sh zookeeper {{ template "pulsar.zookeeper.hostname" . }} false;
 {{- end }}
 {{- end }}
+

charts/pulsar/templates/autorecovery-service-account.yaml

@@ -26,8 +26,8 @@ metadata:
   labels:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
     component: {{ .Values.autorecovery.component }}
-  annotations:
 {{- with .Values.autorecovery.service_account.annotations }}
+  annotations:
 {{ toYaml . | indent 4 }}
 {{- end }}
 {{- end }}

charts/pulsar/templates/autorecovery-statefulset.yaml

@@ -110,7 +110,7 @@ spec:
       terminationGracePeriodSeconds: {{ .Values.autorecovery.gracePeriod }}
       serviceAccountName: "{{ template "pulsar.fullname" . }}-{{ .Values.autorecovery.component }}"
       initContainers:
-      {{- if and .Values.autorecovery.waitBookkeeperTimeout (not (eq (.Values.autorecovery.waitBookkeeperTimeout | toString) "0")) }}
+      {{- if and .Values.autorecovery.waitBookkeeperTimeout (gt (.Values.autorecovery.waitBookkeeperTimeout | int) 0) }}
       # This initContainer will wait for bookkeeper initnewcluster to complete
       # before deploying the bookies
       - name: pulsar-bookkeeper-verify-clusterid

charts/pulsar/templates/bookkeeper-cluster-initialize.yaml

@@ -36,16 +36,16 @@ spec:
     spec:
     {{- include "pulsar.imagePullSecrets" . | nindent 6 }}
       serviceAccountName: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
+      {{- with .Values.pulsar_metadata.nodeSelector }}
       nodeSelector:
-      {{- if .Values.pulsar_metadata.nodeSelector }}
 {{ toYaml .Values.pulsar_metadata.nodeSelector | indent 8 }}
       {{- end }}
+      {{- with .Values.pulsar_metadata.tolerations }}
       tolerations:
-      {{- if .Values.pulsar_metadata.tolerations }}
 {{ toYaml .Values.pulsar_metadata.tolerations | indent 8 }}
       {{- end }}
       initContainers:
-      {{- if and .Values.bookkeeper.metadata.waitZookeeperTimeout (not (eq (.Values.bookkeeper.metadata.waitZookeeperTimeout | toString) "0")) }}
+      {{- if and .Values.components.zookeeper .Values.bookkeeper.metadata.waitZookeeperTimeout (gt (.Values.bookkeeper.metadata.waitZookeeperTimeout | int) 0) }}
       - name: wait-zookeeper-ready
         image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.bookie "root" .) }}"
         imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.bookie "root" .) }}"
@@ -64,6 +64,18 @@ spec:
             done;
             {{- end}}
       {{- end}}
+      {{- if and .Values.components.oxia .Values.bookkeeper.metadata.waitOxiaTimeout (gt (.Values.bookkeeper.metadata.waitOxiaTimeout | int) 0) }}
+      - name: wait-oxia-ready
+        image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.bookie "root" .) }}"
+        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.bookie "root" .) }}"
+        resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
+        command: ["timeout", "{{ .Values.bookkeeper.metadata.waitOxiaTimeout }}", "sh", "-c"]
+        args:
+          - >-
+            until nslookup {{ template "pulsar.oxia.server.service" . }}; do
+              sleep 3;
+            done;
+      {{- end }}
       containers:
       - name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-init"
         image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.bookie "root" .) }}"
@@ -81,7 +93,7 @@ spec:
             if timeout 15 bin/bookkeeper shell whatisinstanceid; then
                 echo "bookkeeper cluster already initialized";
             else
-                {{- if not (eq .Values.metadataPrefix "") }}
+                {{- if and .Values.components.zookeeper (not (eq .Values.metadataPrefix "")) }}
                 bin/pulsar zookeeper-shell -server {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }} create {{ .Values.metadataPrefix }} && echo 'created for pulsar cluster "{{ template "pulsar.cluster.name" . }}"' &&
                 {{- end }}
                 bin/bookkeeper shell initnewcluster;

charts/pulsar/templates/bookkeeper-service-account.yaml

@@ -26,8 +26,8 @@ metadata:
   labels:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
     component: {{ .Values.bookkeeper.component }}
-  annotations:
 {{- with .Values.bookkeeper.service_account.annotations }}
+  annotations:
 {{ toYaml . | indent 4 }}
 {{- end }}
 {{- end }}

charts/pulsar/templates/bookkeeper-statefulset.yaml

@@ -66,7 +66,7 @@ spec:
       affinity:
         {{- if and .Values.affinity.anti_affinity .Values.bookkeeper.affinity.anti_affinity}}
         podAntiAffinity:
-          {{ if eq .Values.bookkeeper.affinity.type "requiredDuringSchedulingIgnoredDuringExecution"}}
+          {{- if eq .Values.bookkeeper.affinity.type "requiredDuringSchedulingIgnoredDuringExecution"}}
           {{ .Values.bookkeeper.affinity.type }}:
           - labelSelector:
               matchExpressions:
@@ -83,7 +83,7 @@ spec:
                 values:
                 - {{ .Values.bookkeeper.component }}
             topologyKey: {{ .Values.bookkeeper.affinity.anti_affinity_topology_key }}
-        {{ else }}
+        {{- else }}
           {{ .Values.bookkeeper.affinity.type }}:
             - weight: 100
               podAffinityTerm:
@@ -110,7 +110,7 @@ spec:
       securityContext:
 {{ toYaml .Values.bookkeeper.securityContext | indent 8 }}
       {{- end }}
-      {{- if and .Values.bookkeeper.waitMetadataTimeout (not (eq (.Values.bookkeeper.waitMetadataTimeout | toString) "0")) }}
+      {{- if and .Values.bookkeeper.waitMetadataTimeout (gt (.Values.bookkeeper.waitMetadataTimeout | int) 0) }}
       initContainers:
       # This initContainer will wait for bookkeeper initnewcluster to complete
       # before deploying the bookies

charts/pulsar/templates/broker-configmap.yaml

@@ -28,6 +28,7 @@ metadata:
     component: {{ .Values.broker.component }}
 data:
   # Metadata settings
+  {{- if .Values.components.zookeeper }}
   zookeeperServers: "{{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }}"
   {{- if .Values.pulsar_metadata.configurationStore }}
   configurationStoreServers: "{{ template "pulsar.configurationStore.connect" . }}{{ .Values.pulsar_metadata.configurationStoreMetadataPrefix }}"
@@ -35,6 +36,12 @@ data:
   {{- if not .Values.pulsar_metadata.configurationStore }}
   configurationStoreServers: "{{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }}"
   {{- end }}
+  {{- end }}
+  {{- if .Values.components.oxia }}
+  metadataStoreUrl: "{{ template "pulsar.oxia.metadata.url.broker" . }}"
+  configurationMetadataStoreUrl: "{{ template "pulsar.oxia.metadata.url.broker" . }}"
+  bookkeeperMetadataServiceUri: "{{ template "pulsar.oxia.metadata.url.bookkeeper" . }}"
+  {{- end }}
 
   # Broker settings
   clusterName: {{ template "pulsar.cluster.name" . }}

charts/pulsar/templates/broker-service-account.yaml

@@ -26,8 +26,8 @@ metadata:
   labels:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
     component: {{ .Values.broker.component }}
-  annotations:
 {{- with .Values.broker.service_account.annotations }}
+  annotations:
 {{ toYaml . | indent 4 }}
 {{- end }}
 ---
@@ -42,8 +42,8 @@ metadata:
   labels:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
     component: {{ .Values.functions.component }}
-  annotations:
 {{- with .Values.functions.service_account.annotations }}
+  annotations:
 {{ toYaml . | indent 4 }}
 {{- end }}
 ---

charts/pulsar/templates/broker-service.yaml

@@ -26,8 +26,10 @@ metadata:
   labels:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
     component: {{ .Values.broker.component }}
+{{- with .Values.broker.service_account.annotations }}
   annotations:
-{{ toYaml .Values.broker.service.annotations | indent 4 }}
+{{ toYaml . | indent 4 }}
+{{- end }}
 spec:
   type: ClusterIP
   ports:

charts/pulsar/templates/broker-statefulset.yaml

@@ -87,7 +87,7 @@ spec:
       affinity:
         {{- if and .Values.affinity.anti_affinity .Values.broker.affinity.anti_affinity}}
         podAntiAffinity:
-          {{ if eq .Values.broker.affinity.type "requiredDuringSchedulingIgnoredDuringExecution"}}
+          {{- if eq .Values.broker.affinity.type "requiredDuringSchedulingIgnoredDuringExecution"}}
           {{ .Values.broker.affinity.type }}:
           - labelSelector:
               matchExpressions:
@@ -104,7 +104,7 @@ spec:
                 values:
                 - {{ .Values.broker.component }}
             topologyKey: {{ .Values.broker.affinity.anti_affinity_topology_key }}
-        {{ else }}
+        {{- else }}
           {{ .Values.broker.affinity.type }}:
             - weight: 100
               podAffinityTerm:
@@ -123,11 +123,11 @@ spec:
                       values:
                       - {{ .Values.broker.component }}
                 topologyKey: {{ .Values.broker.affinity.anti_affinity_topology_key }}
-        {{ end }}
+        {{- end }}
         {{- end }}
       terminationGracePeriodSeconds: {{ .Values.broker.gracePeriod }}
       initContainers:
-      {{- if and .Values.broker.waitZookeeperTimeout (not (eq (.Values.broker.waitZookeeperTimeout | toString) "0")) }}
+      {{- if and .Values.components.zookeeper .Values.broker.waitZookeeperTimeout (gt (.Values.broker.waitZookeeperTimeout | int) 0) }}
       # This init container will wait for zookeeper to be ready before
       # deploying the bookies
       - name: wait-zookeeper-ready
@@ -154,7 +154,19 @@ spec:
         volumeMounts:
         {{- include "pulsar.broker.certs.volumeMounts" . | nindent 8 }}
       {{- end }}
-      {{- if and .Values.broker.waitBookkeeperTimeout (not (eq (.Values.broker.waitBookkeeperTimeout | toString) "0")) }}
+      {{- if and .Values.components.oxia .Values.broker.waitOxiaTimeout (gt (.Values.broker.waitOxiaTimeout | int) 0) }}
+      - name: wait-oxia-ready
+        image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.broker "root" .) }}"
+        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.broker "root" .) }}"
+        resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
+        command: ["timeout", "{{ .Values.broker.waitOxiaTimeout }}", "sh", "-c"]
+        args:
+          - >-
+            until nslookup {{ template "pulsar.oxia.server.service" . }}; do
+              sleep 3;
+            done;
+      {{- end }}
+      {{- if and .Values.broker.waitBookkeeperTimeout (gt (.Values.broker.waitBookkeeperTimeout | int) 0) }}
       # This init container will wait for bookkeeper to be ready before
       # deploying the broker
       - name: wait-bookkeeper-ready
@@ -239,6 +251,7 @@ spec:
           bin/apply-config-from-env.py conf/broker.conf;
           bin/gen-yml-from-env.py conf/functions_worker.yml;
           echo "OK" > "${statusFilePath:-status}";
+          {{- if .Values.components.zookeeper }}
           {{- include "pulsar.broker.zookeeper.tls.settings" . | nindent 10 }}
           timeout 15 bin/pulsar zookeeper-shell -server {{ template "pulsar.zookeeper.connect" . }} get {{ template "pulsar.broker.znode" . }};
           while [ $? -eq 0 ]; do
@@ -246,6 +259,7 @@ spec:
             sleep 10;
             timeout 15 bin/pulsar zookeeper-shell -server {{ template "pulsar.zookeeper.connect" . }} get {{ template "pulsar.broker.znode" . }};
           done;
+          {{- end }}
           cat conf/pulsar_env.sh;
           OPTS="${OPTS} -Dlog4j2.formatMsgNoLookups=true" exec bin/pulsar broker;
         ports:

charts/pulsar/templates/oxia-coordinator-configmap.yaml

@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+{{- if .Values.components.oxia }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.oxia.component }}-coordinator
+data:
+  config.yaml: |
+    {{- include "oxia.coordinator.config.yaml" . | nindent 4 }}
+
+{{- end }}

charts/pulsar/templates/oxia-coordinator-deployment.yaml

@@ -0,0 +1,74 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+{{- if .Values.components.oxia }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.oxia.component }}-coordinator
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      {{- include "pulsar.matchLabels" . | nindent 6 }}
+      component: {{ .Values.oxia.component }}-coordinator
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        {{- include "pulsar.template.labels" . | nindent 8 }}
+        component: {{ .Values.oxia.component }}-coordinator
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "{{ .Values.oxia.coordinator.ports.metrics }}"
+    spec:
+    {{- if .Values.oxia.server.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.oxia.server.nodeSelector | indent 8 }}
+    {{- end }}
+    {{- if .Values.oxia.server.tolerations }}
+      tolerations:
+{{ toYaml .Values.oxia.server.tolerations | indent 8 }}
+    {{- end }}    
+      serviceAccountName: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator
+      containers:
+        - command:
+            {{- include "oxia.coordinator.entrypoint" . | nindent 12 }}
+          image: "{{ .Values.images.oxia.repository }}:{{ .Values.images.oxia.tag }}"
+          imagePullPolicy: {{ .Values.images.oxia.pullPolicy }}
+          name: coordinator
+          ports:
+            {{- range $key, $value := .Values.oxia.coordinator.ports }}
+            - containerPort: {{ $value | int }}
+              name: {{ $key }}
+            {{- end}}
+          resources:
+            limits:
+              cpu: {{ .Values.oxia.coordinator.cpuLimit }}
+              memory: {{ .Values.oxia.coordinator.memoryLimit }}
+          livenessProbe:
+            {{- include "oxia-cluster.probe" .Values.oxia.coordinator.ports.internal | nindent 12 }}
+          readinessProbe:
+            {{- include "oxia-cluster.probe" .Values.oxia.coordinator.ports.internal | nindent 12 }}
+{{- end }}

charts/pulsar/templates/oxia-coordinator-podmonitor.yaml

@@ -0,0 +1,58 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# deploy oxia-coordinator PodMonitor only when `$.Values.oxia.podMonitor.enabled` is true
+{{- if and $.Values.components.oxia $.Values.oxia.coordinator.podMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: {{ template "pulsar.fullname" . }}-oxia-coordinator
+  labels:
+    app: {{ template "pulsar.name" . }}
+    chart: {{ template "pulsar.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  jobLabel: oxia-coordinator
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics
+      scheme: http
+      interval: {{ $.Values.oxia.coordinator.podMonitor.interval }}
+      scrapeTimeout: {{ $.Values.oxia.coordinator.podMonitor.scrapeTimeout }}
+      relabelings:
+        - action: labelmap
+          regex: __meta_kubernetes_pod_label_(.+)
+        - sourceLabels: [__meta_kubernetes_namespace]
+          action: replace
+          targetLabel: kubernetes_namespace
+        - sourceLabels: [__meta_kubernetes_pod_label_component]
+          action: replace
+          targetLabel: job
+        - sourceLabels: [__meta_kubernetes_pod_name]
+          action: replace
+          targetLabel: kubernetes_pod_name
+    {{- if $.Values.oxia.coordinator.podMonitor.metricRelabelings }}
+      metricRelabelings: {{ toYaml $.Values.oxia.coordinator.podMonitor.metricRelabelings | nindent 8 }}
+    {{- end }}
+  selector:
+    matchLabels:
+      {{- include "pulsar.matchLabels" . | nindent 6 }}
+      app.kubernetes.io/component: oxia-coordinator
+{{- end }}

charts/pulsar/templates/oxia-coordinator-role.yaml

@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+{{- if .Values.components.oxia }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.oxia.component }}-coordinator
+rules:
+  - apiGroups: [ "" ]
+    resources: [ "configmaps" ]
+    verbs: [ "*" ]
+{{- end }}

charts/pulsar/templates/oxia-coordinator-rolebinding.yaml

@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+{{- if .Values.components.oxia }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.oxia.component }}-coordinator
+subjects:
+  - kind: ServiceAccount
+    name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator
+    namespace: {{ template "pulsar.namespace" . }}
+roleRef:
+  apiGroup: ""
+  kind: Role
+  name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator
+{{- end }}

charts/pulsar/templates/oxia-coordinator-service.yaml

@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+{{- if .Values.components.oxia }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.oxia.component }}-coordinator
+{{- with .Values.oxia.coordinator.service.annotations }}
+  annotations:
+{{ toYaml . | indent 4 }}
+{{- end }}
+spec:
+  ports:
+    {{- range $key, $value := .Values.oxia.coordinator.ports }}
+    - name: {{ $key }}
+      port: {{ $value }}
+      targetPort: {{ $key }}
+    {{- end}}
+  selector:
+    {{- include "pulsar.matchLabels" . | nindent 4 }}
+    component: {{ .Values.oxia.component }}-coordinator
+{{- end }}

charts/pulsar/templates/oxia-coordinator-serviceaccount.yaml

@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+{{- if .Values.components.oxia }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.oxia.component }}-coordinator
+{{- with .Values.oxia.coordinator.service_account.annotations }}
+  annotations:
+{{ toYaml . | indent 4 }}
+{{- end }}
+{{- if .Values.images.imagePullSecrets }}
+imagePullSecrets:
+  - name: {{ .Values.images.imagePullSecrets.secretName }}
+{{- end}}
+{{- end}}

charts/pulsar/templates/oxia-server-podmonitor.yaml

@@ -0,0 +1,58 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# deploy oxia-server PodMonitor only when `$.Values.oxia.podMonitor.enabled` is true
+{{- if and $.Values.components.oxia $.Values.oxia.server.podMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: {{ template "pulsar.fullname" . }}-oxia-server
+  labels:
+    app: {{ template "pulsar.name" . }}
+    chart: {{ template "pulsar.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  jobLabel: oxia-server
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics
+      scheme: http
+      interval: {{ $.Values.oxia.server.podMonitor.interval }}
+      scrapeTimeout: {{ $.Values.oxia.server.podMonitor.scrapeTimeout }}
+      relabelings:
+        - action: labelmap
+          regex: __meta_kubernetes_pod_label_(.+)
+        - sourceLabels: [__meta_kubernetes_namespace]
+          action: replace
+          targetLabel: kubernetes_namespace
+        - sourceLabels: [__meta_kubernetes_pod_label_component]
+          action: replace
+          targetLabel: job
+        - sourceLabels: [__meta_kubernetes_pod_name]
+          action: replace
+          targetLabel: kubernetes_pod_name
+    {{- if $.Values.oxia.server.podMonitor.metricRelabelings }}
+      metricRelabelings: {{ toYaml $.Values.oxia.server.podMonitor.metricRelabelings | nindent 8 }}
+    {{- end }}
+  selector:
+    matchLabels:
+      {{- include "pulsar.matchLabels" . | nindent 6 }}
+      app.kubernetes.io/component: oxia-server
+{{- end }}

charts/pulsar/templates/oxia-server-service-public.yaml

@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+{{- if .Values.components.oxia }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.oxia.component }}-server
+{{- with .Values.oxia.server.service.public.annotations }}
+  annotations:
+{{ toYaml . | indent 4 }}
+{{- end }}
+spec:
+  ports:
+    {{- range $key, $value := .Values.oxia.server.ports }}
+    - name: {{ $key }}
+      port: {{ $value }}
+      targetPort: {{ $key }}
+    {{- end}}
+  selector:
+    {{- include "pulsar.matchLabels" . | nindent 4 }}
+    component: {{ .Values.oxia.component }}-server
+{{- end}}

charts/pulsar/templates/oxia-server-service.yaml

@@ -0,0 +1,45 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+{{- if .Values.components.oxia }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-svc
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.oxia.component }}-server
+{{- with .Values.oxia.server.service.internal.annotations }}
+  annotations:
+{{ toYaml . | indent 4 }}
+{{- end }}
+spec:
+  clusterIP: None
+  publishNotReadyAddresses: true
+  ports:
+    {{- range $key, $value := .Values.oxia.server.ports }}
+    - name: {{ $key }}
+      port: {{ $value }}
+      targetPort: {{ $key }}
+    {{- end}}
+  selector:
+    {{- include "pulsar.matchLabels" . | nindent 4 }}
+    component: {{ .Values.oxia.component }}-server
+{{- end}}

charts/pulsar/templates/oxia-server-serviceaccount.yaml

@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+{{- if .Values.components.oxia }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.oxia.component }}-server
+{{- with .Values.oxia.server.service_account.annotations }}
+  annotations:
+{{ toYaml . | indent 4 }}
+{{- end }}
+{{- if .Values.images.imagePullSecrets }}
+imagePullSecrets:
+  - name: {{ .Values.images.imagePullSecrets.secretName }}
+{{- end}}
+{{- end}}

charts/pulsar/templates/oxia-server-statefulset.yaml

@@ -0,0 +1,147 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+{{- if .Values.components.oxia }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-server
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.oxia.component }}-server
+spec:
+  replicas: {{ .Values.oxia.server.replicas }}
+  selector:
+    matchLabels:
+      {{- include "pulsar.matchLabels" . | nindent 6 }}
+      component: {{ .Values.oxia.component }}-server
+  serviceName: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-svc
+  podManagementPolicy: Parallel
+  template:
+    metadata:
+      labels:
+        {{- include "pulsar.template.labels" . | nindent 8 }}
+        component: {{ .Values.oxia.component }}-server
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "{{ .Values.oxia.server.ports.metrics }}"
+    spec:
+    {{- if .Values.oxia.server.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.oxia.server.nodeSelector | indent 8 }}
+    {{- end }}
+    {{- if .Values.oxia.server.tolerations }}
+      tolerations:
+{{ toYaml .Values.oxia.server.tolerations | indent 8 }}
+    {{- end }}
+    {{- if .Values.oxia.server.topologySpreadConstraints }}
+      topologySpreadConstraints:
+        {{- toYaml .Values.oxia.server.topologySpreadConstraints | nindent 8 }}
+    {{- end }}
+      affinity:
+        {{- if and .Values.affinity.anti_affinity .Values.oxia.server.affinity.anti_affinity}}
+        podAntiAffinity:
+          {{ if eq .Values.oxia.server.affinity.type "requiredDuringSchedulingIgnoredDuringExecution"}}
+          {{ .Values.oxia.server.affinity.type }}:
+          - labelSelector:
+              matchExpressions:
+              - key: "app"
+                operator: In
+                values:
+                - "{{ template "pulsar.name" . }}"
+              - key: "release"
+                operator: In
+                values:
+                - {{ .Release.Name }}
+              - key: "component"
+                operator: In
+                values:
+                - {{ .Values.oxia.component }}-server
+            topologyKey: {{ .Values.oxia.server.affinity.anti_affinity_topology_key }}
+        {{ else }}
+          {{ .Values.oxia.server.affinity.type }}:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchExpressions:
+                    - key: "app"
+                      operator: In
+                      values:
+                      - "{{ template "pulsar.name" . }}"
+                    - key: "release"
+                      operator: In
+                      values:
+                      - {{ .Release.Name }}
+                    - key: "component" 
+                      operator: In
+                      values:
+                      - {{ .Values.oxia.component }}-server
+                topologyKey: {{ .Values.oxia.server.affinity.anti_affinity_topology_key }}
+        {{ end }}
+        {{- end }}    
+      serviceAccountName: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}
+      {{- if .Values.oxia.server.securityContext }}
+      securityContext:
+{{ toYaml .Values.oxia.server.securityContext | indent 8 }}
+      {{- end }}
+      containers:
+        - command:
+            - "oxia"
+            - "server"
+            - "--log-json"
+            - "--data-dir=/data/db"
+            - "--wal-dir=/data/wal"
+            - "--db-cache-size-mb={{ .Values.oxia.server.dbCacheSizeMb }}"
+            {{- if .Values.oxia.pprofEnabled }}
+            - "--profile"
+            {{- end}}
+          image: "{{ .Values.images.oxia.repository }}:{{ .Values.images.oxia.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.images.oxia.pullPolicy }}
+          name: server
+          ports:
+            {{- range $key, $value := .Values.oxia.server.ports }}
+            - containerPort: {{ $value | int }}
+              name: {{ $key }}
+            {{- end}}
+          resources:
+            limits:
+              cpu: {{ .Values.oxia.server.cpuLimit }}
+              memory: {{ .Values.oxia.server.memoryLimit }}
+          volumeMounts:
+            - name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-data
+              mountPath: /data
+          livenessProbe:
+            {{- include "oxia-cluster.probe" .Values.oxia.server.ports.internal | nindent 12 }}
+          readinessProbe:
+            {{- include "oxia-cluster.readiness-probe" .Values.oxia.server.ports.internal | nindent 12 }}
+          startupProbe:
+            {{- include "oxia-cluster.startup-probe" .Values.oxia.server.ports.internal | nindent 12 }}
+  volumeClaimTemplates:
+    - metadata:
+        name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-data
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        {{- if .Values.oxia.server.storageClassName }}
+        storageClassName: {{ .Values.oxia.server.storageClassName }}
+        {{- end}}
+        resources:
+          requests:
+            storage: {{ .Values.oxia.server.storageSize }}
+{{- end}}

charts/pulsar/templates/proxy-statefulset.yaml

@@ -109,7 +109,7 @@ spec:
       terminationGracePeriodSeconds: {{ .Values.proxy.gracePeriod }}
       serviceAccountName: "{{ template "pulsar.fullname" . }}-{{ .Values.proxy.component }}"
       initContainers:
-      {{- if and .Values.proxy.waitZookeeperTimeout (not (eq (.Values.proxy.waitZookeeperTimeout | toString) "0")) }}
+      {{- if and .Values.components.zookeeper .Values.proxy.waitZookeeperTimeout (gt (.Values.proxy.waitZookeeperTimeout | int) 0) }}
       # This init container will wait for zookeeper to be ready before
       # deploying the bookies
       - name: wait-zookeeper-ready
@@ -130,7 +130,19 @@ spec:
             done;
             {{- end}}
       {{- end}}
-      {{- if and .Values.proxy.waitBrokerTimeout (not (eq (.Values.proxy.waitBrokerTimeout | toString) "0")) }}
+      {{- if and .Values.components.oxia .Values.proxy.waitOxiaTimeout (gt (.Values.proxy.waitOxiaTimeout | int) 0) }}
+      - name: wait-oxia-ready
+        image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.proxy "root" .) }}"
+        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.proxy "root" .) }}"
+        resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
+        command: ["timeout", "{{ .Values.proxy.waitOxiaTimeout }}", "sh", "-c"]
+        args:
+          - >-
+            until nslookup {{ template "pulsar.oxia.server.service" . }}; do
+              sleep 3;
+            done;
+      {{- end }}      
+      {{- if and .Values.proxy.waitBrokerTimeout (gt (.Values.proxy.waitBrokerTimeout | int) 0) }}
       # This init container will wait for at least one broker to be ready before
       # deploying the proxy
       - name: wait-broker-ready

charts/pulsar/templates/pulsar-cluster-initialize.yaml

@@ -22,7 +22,7 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_metadata.component }}"
+  name: {{ template "pulsar.fullname" . }}-{{ .Values.pulsar_metadata.component }}
   namespace: {{ template "pulsar.namespace" . }}
   labels:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
@@ -41,9 +41,9 @@ spec:
 {{ toYaml .Values.pulsar_metadata.nodeSelector | indent 8 }}
     {{- end }}
       initContainers:
-      {{- if and .Values.pulsar_metadata.waitZookeeperTimeout (not (eq (.Values.pulsar_metadata.waitZookeeperTimeout | toString) "0")) }}
+      {{- if and .Values.components.zookeeper .Values.pulsar_metadata.waitZookeeperTimeout (gt (.Values.pulsar_metadata.waitZookeeperTimeout | int) 0) }}
       {{- if .Values.pulsar_metadata.configurationStore }}
-      - name: wait-cs-ready
+      - name: wait-zk-cs-ready
         image: "{{ template "pulsar.imageFullName" (dict "image" .Values.pulsar_metadata.image "root" .) }}"
         imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.pulsar_metadata.image "root" .) }}"
         resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
@@ -54,34 +54,46 @@ spec:
               sleep 3;
             done;
       {{- end }}
-      - name: wait-zookeeper-ready
+      - name: wait-zk-metastore-ready
         image: "{{ template "pulsar.imageFullName" (dict "image" .Values.pulsar_metadata.image "root" .) }}"
         imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.pulsar_metadata.image "root" .) }}"
         resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
         command: ["timeout", "{{ .Values.pulsar_metadata.waitZookeeperTimeout }}", "sh", "-c"]
         args:
           - >-
-            {{- if $zk:=.Values.pulsar_metadata.userProvidedZookeepers }}
+            {{- if $zk := .Values.pulsar_metadata.userProvidedZookeepers }}
             export PULSAR_MEM="-Xmx128M";
             until timeout 15 bin/pulsar zookeeper-shell -server {{ $zk }} ls {{ or .Values.metadataPrefix "/" }}; do
               echo "user provided zookeepers {{ $zk }} are unreachable... check in 3 seconds ..." && sleep 3;
             done;
-            {{ else }}
+            {{ else if .Values.components.zookeeper }}
             until nslookup {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-{{ add (.Values.zookeeper.replicaCount | int) -1 }}.{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}.{{ template "pulsar.namespace" . }}; do
               sleep 3;
             done;
-            {{- end}}
+            {{- end }}
       {{- end }}
-      {{- if and .Values.pulsar_metadata.waitBookkeeperTimeout (not (eq (.Values.pulsar_metadata.waitBookkeeperTimeout | toString) "0")) }}
+      {{- if and .Values.components.oxia .Values.pulsar_metadata.waitOxiaTimeout (gt (.Values.pulsar_metadata.waitOxiaTimeout | int) 0) }}
-      # This initContainer will wait for bookkeeper initnewcluster to complete
+      - name: wait-oxia-ready
-      # before initializing pulsar metadata
-      - name: pulsar-bookkeeper-verify-clusterid
         image: "{{ template "pulsar.imageFullName" (dict "image" .Values.pulsar_metadata.image "root" .) }}"
         imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.pulsar_metadata.image "root" .) }}"
         resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
+        command: ["timeout", "{{ .Values.pulsar_metadata.waitOxiaTimeout }}", "sh", "-c"]
+        args:
+          - >-
+            until nslookup {{ template "pulsar.oxia.server.service" . }}; do
+              sleep 3;
+            done;
+      {{- end }}
+      {{- if and .Values.pulsar_metadata.waitBookkeeperTimeout (gt (.Values.pulsar_metadata.waitBookkeeperTimeout | int) 0) }}
+      # This initContainer will wait for bookkeeper initnewcluster to complete
+      # before initializing pulsar metadata
+      - name: pulsar-bookkeeper-verify-clusterid
+        image: {{ template "pulsar.imageFullName" (dict "image" .Values.pulsar_metadata.image "root" .) }}
+        imagePullPolicy: {{ template "pulsar.imagePullPolicy" (dict "image" .Values.pulsar_metadata.image "root" .) }}
+        resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
         command: ["timeout", "{{ .Values.pulsar_metadata.waitBookkeeperTimeout }}", "sh", "-c"]
         args:
-        - >
+        - >-
           bin/apply-config-from-env.py conf/bookkeeper.conf;
           echo Default BOOKIE_MEM settings are set very high, which can cause the init container to fail.;
           echo Setting the memory to a lower value to avoid OOM as operations below are not memory intensive.;
@@ -92,32 +104,32 @@ spec:
           done;
         envFrom:
         - configMapRef:
-            name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
+            name: {{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}
         volumeMounts:
         {{- include "pulsar.toolset.certs.volumeMounts" . | nindent 8 }}
       {{- end }}
       containers:
-      - name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_metadata.component }}"
+      - name: {{ template "pulsar.fullname" . }}-{{ .Values.pulsar_metadata.component }}
-        image: "{{ template "pulsar.imageFullName" (dict "image" .Values.pulsar_metadata.image "root" .) }}"
+        image: {{ template "pulsar.imageFullName" (dict "image" .Values.pulsar_metadata.image "root" .) }}
-        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.pulsar_metadata.image "root" .) }}"
+        imagePullPolicy: {{ template "pulsar.imagePullPolicy" (dict "image" .Values.pulsar_metadata.image "root" .) }}
       {{- if .Values.pulsar_metadata.resources }}
         resources:
 {{ toYaml .Values.pulsar_metadata.resources | indent 10 }}
       {{- end }}
         command: ["timeout", "{{ .Values.pulsar_metadata.initTimeout | default 60 }}", "sh", "-c"]
+        {{- if .Values.components.zookeeper }}
         args:
-          - |
+          - >-
             {{- include "pulsar.toolset.zookeeper.tls.settings" . | nindent 12 }}
             export PULSAR_MEM="-Xmx128M";
             bin/pulsar initialize-cluster-metadata \
               --cluster {{ template "pulsar.cluster.name" . }} \
               --zookeeper {{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }} \
-              {{- if .Values.pulsar_metadata.configurationStore }}
+            {{- if .Values.pulsar_metadata.configurationStore }}
               --configuration-store {{ template "pulsar.configurationStore.connect" . }}{{ .Values.pulsar_metadata.configurationStoreMetadataPrefix }} \
-              {{- end }}
+            {{- else }}
-              {{- if not .Values.pulsar_metadata.configurationStore }}
               --configuration-store {{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }} \
-              {{- end }}
+            {{- end }}
               --web-service-url http://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}.{{ template "pulsar.namespace" . }}.svc.{{ .Values.clusterDomain }}:{{ .Values.broker.ports.http }}/ \
               --web-service-url-tls https://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}.{{ template "pulsar.namespace" . }}.svc.{{ .Values.clusterDomain }}:{{ .Values.broker.ports.https }}/ \
               --broker-service-url pulsar://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}.{{ template "pulsar.namespace" . }}.svc.{{ .Values.clusterDomain }}:{{ .Values.broker.ports.pulsar }}/ \
@@ -125,10 +137,26 @@ spec:
             {{- if .Values.extraInitCommand }}
               {{ .Values.extraInitCommand }}
             {{- end }}
+        {{- else if .Values.components.oxia }}
+        args:
+          - >-
+            export PULSAR_MEM="-Xmx128M";
+            bin/pulsar initialize-cluster-metadata \
+              --cluster {{ template "pulsar.cluster.name" . }} \
+              --metadata-store "{{ template "pulsar.oxia.metadata.url.broker" . }}" \
+              --configuration-store "{{ template "pulsar.oxia.metadata.url.broker" . }}" \
+              --web-service-url http://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}.{{ template "pulsar.namespace" . }}.svc.{{ .Values.clusterDomain }}:{{ .Values.broker.ports.http }}/ \
+              --web-service-url-tls https://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}.{{ template "pulsar.namespace" . }}.svc.{{ .Values.clusterDomain }}:{{ .Values.broker.ports.https }}/ \
+              --broker-service-url pulsar://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}.{{ template "pulsar.namespace" . }}.svc.{{ .Values.clusterDomain }}:{{ .Values.broker.ports.pulsar }}/ \
+              --broker-service-url-tls pulsar+ssl://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}.{{ template "pulsar.namespace" . }}.svc.{{ .Values.clusterDomain }}:{{ .Values.broker.ports.pulsarssl }}/ ;
+            {{- if .Values.extraInitCommand }}
+              {{ .Values.extraInitCommand }}
+            {{- end }}
+        {{- end }}
         volumeMounts:
-        {{- include "pulsar.toolset.certs.volumeMounts" . | nindent 8 }}
+        {{- include "pulsar.toolset.certs.volumeMounts" . | nindent 10 }}
       volumes:
-      {{- include "pulsar.toolset.certs.volumes" . | nindent 6 }}
+      {{- include "pulsar.toolset.certs.volumes" . | nindent 8 }}
       restartPolicy: OnFailure
       {{- if .Values.pulsar_metadata.nodeSelector }}
       nodeSelector:

charts/pulsar/templates/toolset-service-account.yaml

@@ -26,8 +26,8 @@ metadata:
   labels:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
     component: {{ .Values.toolset.component }}
-  annotations:
 {{- with .Values.toolset.service_account.annotations }}
+  annotations:
 {{ toYaml . | indent 4 }}
 {{- end }}
 {{- end }}

charts/pulsar/templates/zookeeper-podmonitor.yaml

@@ -18,6 +18,7 @@
 #
 
 # deploy zookeeper PodMonitor only when `$.Values.zookeeper.podMonitor.enabled` is true
+{{- if .Values.components.zookeeper }}
 {{- if $.Values.zookeeper.podMonitor.enabled }}
 apiVersion: monitoring.coreos.com/v1
 kind: PodMonitor
@@ -56,3 +57,4 @@ spec:
       {{- include "pulsar.matchLabels" . | nindent 6 }}
       component: zookeeper
 {{- end }}
+{{- end }}

charts/pulsar/values.yaml

@@ -113,6 +113,8 @@ affinity:
 components:
   # zookeeper
   zookeeper: true
+  # oxia
+  oxia: false
   # bookkeeper
   bookkeeper: true
   # bookkeeper - autorecovery
@@ -202,6 +204,10 @@ images:
     # uses defaultPullPolicy when unspecified
     pullPolicy:
     hasCommand: false
+  oxia:
+    repository: streamnative/oxia
+    tag: 0.11.9
+    pullPolicy: Always
 
 ## TLS
 ## templates/tls-certs.yaml
@@ -249,6 +255,8 @@ tls:
   function_instance:
     # controls the use of TLS for function runtime connections towards brokers
     enabled: false
+  oxia:
+    enabled: false
 
 # Enable or disable broker authentication and authorization.
 auth:
@@ -468,7 +476,81 @@ zookeeper:
     usePolicy: true
     maxUnavailable: 1
 
-
+## Pulsar: Oxia cluster
+oxia:
+  component: oxia
+  initialShardCount: 3
+  replicationFactor: 3
+## templates/coordinator-deployment.yaml
+  coordinator:
+    # This is how prometheus discovers this component
+    podMonitor:
+      enabled: true
+      interval: 60s
+      scrapeTimeout: 60s
+      metricRelabelings:
+        # - action: labeldrop
+        #   regex: cluster
+    cpuLimit: 100m
+    memoryLimit: 128Mi
+    ports:
+      internal: 6649
+      metrics: 8080
+    service:
+      annotations: {}
+    service_account:
+      annotations: {}
+    tolerations: []
+    # nodeSelector:
+      # cloud.google.com/gke-nodepool: default-pool
+## templates/server-statefulset.yaml
+  server:
+    # This is how prometheus discovers this component
+    podMonitor:
+      enabled: true
+      interval: 60s
+      scrapeTimeout: 60s
+      metricRelabelings:
+        # - action: labeldrop
+        #   regex: cluster
+    replicas: 3
+    # CPU limit for the server pod
+    cpuLimit: 1
+    # Memory limit for the server pod
+    memoryLimit: 1Gi
+    # Oxia database cache size in MB
+    dbCacheSizeMb: 512
+    # Storage size for the PVC of the server pod
+    storageSize: 8Gi
+    # Storage class name for the PVC of the server pod
+    # storageClassName: existent-storage-class
+    ports:
+      public: 6648
+      internal: 6649
+      metrics: 8080
+    service:
+      public:
+        annotations: {}
+      internal:
+        annotations: {}
+    service_account:
+      annotations: {}
+    securityContext:
+      fsGroup: 0
+      fsGroupChangePolicy: "OnRootMismatch"
+    affinity:
+      anti_affinity: true
+      anti_affinity_topology_key: kubernetes.io/hostname
+      # Set the anti affinity type. Valid values:
+      # requiredDuringSchedulingIgnoredDuringExecution - rules must be met for pod to be scheduled (hard) requires at least one node per replica
+      # preferredDuringSchedulingIgnoredDuringExecution - scheduler will try to enforce but not guranentee
+      type: requiredDuringSchedulingIgnoredDuringExecution
+    # set topologySpreadConstraint to deploy pods across different zones
+    topologySpreadConstraints: []
+    tolerations: []
+    # nodeSelector:
+      # cloud.google.com/gke-nodepool: default-pool
+  pprofEnabled: false
 ## Pulsar: Bookkeeper cluster
 ## templates/bookkeeper-statefulset.yaml
 ##
@@ -483,6 +565,8 @@ bookkeeper:
     waitZookeeperTimeout: 600
     ## Timeout for running metadata initialization
     initTimeout: 60
+    ## Timeout for waiting for oxia to be available before running metadata initialization. This setting applies only when oxia is enabled.
+    waitOxiaTimeout: 600
     ## Set the resources used for running `bin/bookkeeper shell initnewcluster`
     ##
     resources:
@@ -765,12 +849,16 @@ pulsar_metadata:
     tag:
     # uses defaultPullPolicy when unspecified
     pullPolicy:
-  ## set an existing configuration store
+  ## set an existing configuration store. This setting applies only when zookeeper is enabled.
   # configurationStore:
+  # the prefix for the configuration store metadata. This setting applies only when zookeeper is enabled.
   configurationStoreMetadataPrefix: ""
+  # the configuration store port. This setting applies only when zookeeper is enabled.
   configurationStorePort: 2181
-  ## Timeout for waiting for zookeeper to become available before running metadata initialization
+  # the zookeeper timeout. This setting applies only when zookeeper is enabled.
   waitZookeeperTimeout: 600
+  ## Timeout for waiting for oxia to be available before running metadata initialization. This setting applies only when oxia is enabled.
+  waitOxiaTimeout: 600
   ## Timeout for waiting for bookkeeper to be initialized before running metadata initialization
   waitBookkeeperTimeout: 120
   ## Timeout for running metadata initialization
@@ -866,6 +954,8 @@ broker:
   gracePeriod: 30
   ## Timeout for waiting for zookeeper to become available before starting a broker
   waitZookeeperTimeout: 600
+  ## Timeout for waiting for oxia to be available before starting a broker. This setting applies only when oxia is enabled.
+  waitOxiaTimeout: 600
   ## Timeout for waiting for bookkeeper to become available before starting a broker
   waitBookkeeperTimeout: 120
   resources:
@@ -1106,6 +1196,8 @@ proxy:
   gracePeriod: 30
   ## Timeout for waiting for zookeeper to become available before starting a proxy
   waitZookeeperTimeout: 600
+  ## Timeout for waiting for oxia to be available before starting a proxy. This setting applies only when oxia is enabled.
+  waitOxiaTimeout: 600
   ## Timeout for waiting for brokers to become available before starting a proxy
   waitBrokerTimeout: 120
   resources:

examples/values-oxia.yaml

@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+components:
+  zookeeper: false
+  oxia: true