6c2edba8b1
### Changes - using "exec" to run a command replaces the shell process with the executed process - this is required so that the process running in the container is able to receive OS signals - explained in https://docs.docker.com/develop/develop-images/dockerfile_best-practices/ and https://docs.docker.com/engine/reference/builder/#entrypoint - receiving SIGTERM signal is required for graceful shutdown. This is explained in https://pracucci.com/graceful-shutdown-of-kubernetes-pods.html This change might fix issues such as https://github.com/apache/pulsar/issues/6603 . One expectation of this fix is that graceful shutdown would allow Pulsar components such as a bookies to deregistered from Zookeeper properly before shutdown. ### Motivation Dockerfile best practices mention that "exec" should be used so that the process running in a container can receive OS signals. This is explained in https://docs.docker.com/develop/develop-images/dockerfile_best-practices/ and https://docs.docker.com/engine/reference/builder/#entrypoint . Kubernetes documention explains pod termination in https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-termination : "Typically, the container runtime sends a TERM signal to the main process in each container. Once the grace period has expired, the KILL signal is sent to any remaining processes, and the Pod is then deleted from the API Server ." Currently some issues while running Pulsar are caused by the lack of graceful shutdown. Graceful shutdown isn't happening at all since the Pulsar processes never receive the TERM signal that would allow graceful shutdown. This PR fixes that. This PR was inspired by https://github.com/kafkaesque-io/pulsar-helm-chart/pull/31
261 lines
10 KiB
YAML
261 lines
10 KiB
YAML
#
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
#
|
|
|
|
{{- if or .Values.components.proxy .Values.extra.proxy }}
|
|
apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
metadata:
|
|
name: "{{ template "pulsar.fullname" . }}-{{ .Values.proxy.component }}"
|
|
namespace: {{ .Values.namespace }}
|
|
labels:
|
|
{{- include "pulsar.standardLabels" . | nindent 4 }}
|
|
component: {{ .Values.proxy.component }}
|
|
spec:
|
|
serviceName: "{{ template "pulsar.fullname" . }}-{{ .Values.proxy.component }}"
|
|
replicas: {{ .Values.proxy.replicaCount }}
|
|
selector:
|
|
matchLabels:
|
|
{{- include "pulsar.matchLabels" . | nindent 6 }}
|
|
component: {{ .Values.proxy.component }}
|
|
updateStrategy:
|
|
type: RollingUpdate
|
|
podManagementPolicy: Parallel
|
|
template:
|
|
metadata:
|
|
labels:
|
|
{{- include "pulsar.template.labels" . | nindent 8 }}
|
|
component: {{ .Values.proxy.component }}
|
|
annotations:
|
|
prometheus.io/scrape: "true"
|
|
prometheus.io/port: "{{ .Values.proxy.ports.http }}"
|
|
{{- with .Values.proxy.annotations }}
|
|
{{ toYaml . | indent 8 }}
|
|
{{- end }}
|
|
spec:
|
|
{{- if .Values.proxy.nodeSelector }}
|
|
nodeSelector:
|
|
{{ toYaml .Values.proxy.nodeSelector | indent 8 }}
|
|
{{- end }}
|
|
{{- if .Values.proxy.tolerations }}
|
|
tolerations:
|
|
{{ toYaml .Values.proxy.tolerations | indent 8 }}
|
|
{{- end }}
|
|
affinity:
|
|
{{- if and .Values.affinity.anti_affinity .Values.proxy.affinity.anti_affinity}}
|
|
podAntiAffinity:
|
|
{{ if eq .Values.proxy.affinity.type "requiredDuringSchedulingIgnoredDuringExecution"}}
|
|
{{ .Values.proxy.affinity.type }}:
|
|
- labelSelector:
|
|
matchExpressions:
|
|
- key: "app"
|
|
operator: In
|
|
values:
|
|
- "{{ template "pulsar.name" . }}"
|
|
- key: "release"
|
|
operator: In
|
|
values:
|
|
- {{ .Release.Name }}
|
|
- key: "component"
|
|
operator: In
|
|
values:
|
|
- {{ .Values.proxy.component }}
|
|
topologyKey: "kubernetes.io/hostname"
|
|
{{ else }}
|
|
{{ .Values.proxy.affinity.type }}:
|
|
- weight: 100
|
|
podAffinityTerm:
|
|
labelSelector:
|
|
matchExpressions:
|
|
- key: "app"
|
|
operator: In
|
|
values:
|
|
- "{{ template "pulsar.name" . }}"
|
|
- key: "release"
|
|
operator: In
|
|
values:
|
|
- {{ .Release.Name }}
|
|
- key: "component"
|
|
operator: In
|
|
values:
|
|
- {{ .Values.proxy.component }}
|
|
topologyKey: "kubernetes.io/hostname"
|
|
{{ end }}
|
|
{{- end }}
|
|
terminationGracePeriodSeconds: {{ .Values.proxy.gracePeriod }}
|
|
initContainers:
|
|
# This init container will wait for zookeeper to be ready before
|
|
# deploying the bookies
|
|
- name: wait-zookeeper-ready
|
|
image: "{{ .Values.images.proxy.repository }}:{{ .Values.images.proxy.tag }}"
|
|
imagePullPolicy: {{ .Values.images.proxy.pullPolicy }}
|
|
command: ["sh", "-c"]
|
|
args:
|
|
- >-
|
|
{{- if $zk:=.Values.pulsar_metadata.userProvidedZookeepers }}
|
|
until bin/pulsar zookeeper-shell -server {{ $zk }} ls {{ or .Values.metadataPrefix "/" }}; do
|
|
echo "user provided zookeepers {{ $zk }} are unreachable... check in 3 seconds ..." && sleep 3;
|
|
done;
|
|
{{ else }}
|
|
until bin/pulsar zookeeper-shell -server {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }} get {{ .Values.metadataPrefix }}/admin/clusters/{{ template "pulsar.fullname" . }}; do
|
|
sleep 3;
|
|
done;
|
|
{{- end}}
|
|
# This init container will wait for at least one broker to be ready before
|
|
# deploying the proxy
|
|
- name: wait-broker-ready
|
|
image: "{{ .Values.images.proxy.repository }}:{{ .Values.images.proxy.tag }}"
|
|
imagePullPolicy: {{ .Values.images.proxy.pullPolicy }}
|
|
command: ["sh", "-c"]
|
|
args:
|
|
- >-
|
|
set -e;
|
|
brokerServiceNumber="$(nslookup -timeout=10 {{ template "pulsar.fullname" . }}-{{ .Values.broker.component }} | grep Name | wc -l)";
|
|
until [ ${brokerServiceNumber} -ge 1 ]; do
|
|
echo "pulsar cluster {{ template "pulsar.fullname" . }} isn't initialized yet ... check in 10 seconds ...";
|
|
sleep 10;
|
|
brokerServiceNumber="$(nslookup -timeout=10 {{ template "pulsar.fullname" . }}-{{ .Values.broker.component }} | grep Name | wc -l)";
|
|
done;
|
|
containers:
|
|
- name: "{{ template "pulsar.fullname" . }}-{{ .Values.proxy.component }}"
|
|
image: "{{ .Values.images.proxy.repository }}:{{ .Values.images.proxy.tag }}"
|
|
imagePullPolicy: {{ .Values.images.proxy.pullPolicy }}
|
|
{{- if .Values.proxy.probe.liveness.enabled }}
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /status.html
|
|
port: {{ .Values.proxy.ports.http }}
|
|
initialDelaySeconds: {{ .Values.proxy.probe.liveness.initialDelaySeconds }}
|
|
periodSeconds: {{ .Values.proxy.probe.liveness.periodSeconds }}
|
|
failureThreshold: {{ .Values.proxy.probe.liveness.failureThreshold }}
|
|
{{- end }}
|
|
{{- if .Values.proxy.probe.readiness.enabled }}
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /status.html
|
|
port: {{ .Values.proxy.ports.http }}
|
|
initialDelaySeconds: {{ .Values.proxy.probe.readiness.initialDelaySeconds }}
|
|
periodSeconds: {{ .Values.proxy.probe.readiness.periodSeconds }}
|
|
failureThreshold: {{ .Values.proxy.probe.readiness.failureThreshold }}
|
|
{{- end }}
|
|
{{- if .Values.proxy.probe.startup.enabled }}
|
|
startupProbe:
|
|
httpGet:
|
|
path: /status.html
|
|
port: {{ .Values.proxy.ports.http }}
|
|
initialDelaySeconds: {{ .Values.proxy.probe.startup.initialDelaySeconds }}
|
|
periodSeconds: {{ .Values.proxy.probe.startup.periodSeconds }}
|
|
failureThreshold: {{ .Values.proxy.probe.startup.failureThreshold }}
|
|
{{- end }}
|
|
{{- if .Values.proxy.resources }}
|
|
resources:
|
|
{{ toYaml .Values.proxy.resources | indent 10 }}
|
|
{{- end }}
|
|
command: ["sh", "-c"]
|
|
args:
|
|
- >
|
|
bin/apply-config-from-env.py conf/proxy.conf &&
|
|
echo "OK" > status &&
|
|
exec bin/pulsar proxy
|
|
ports:
|
|
# prometheus needs to access /metrics endpoint
|
|
- name: http
|
|
containerPort: {{ .Values.proxy.ports.http }}
|
|
{{- if or (not .Values.tls.enabled) (not .Values.tls.proxy.enabled) }}
|
|
- name: pulsar
|
|
containerPort: {{ .Values.proxy.ports.pulsar }}
|
|
{{- end }}
|
|
{{- if and (.Values.tls.enabled) (.Values.tls.proxy.enabled) }}
|
|
- name: https
|
|
containerPort: {{ .Values.proxy.ports.https }}
|
|
- name: pulsarssl
|
|
containerPort: {{ .Values.proxy.ports.pulsarssl }}
|
|
{{- end }}
|
|
envFrom:
|
|
- configMapRef:
|
|
name: "{{ template "pulsar.fullname" . }}-{{ .Values.proxy.component }}"
|
|
{{- if or .Values.auth.authentication.enabled (and .Values.tls.enabled (or .Values.tls.proxy.enabled .Values.tls.broker.enabled)) }}
|
|
volumeMounts:
|
|
{{- if .Values.auth.authentication.enabled }}
|
|
{{- if eq .Values.auth.authentication.provider "jwt" }}
|
|
- mountPath: "/pulsar/keys"
|
|
name: token-keys
|
|
readOnly: true
|
|
- mountPath: "/pulsar/tokens"
|
|
name: proxy-token
|
|
readOnly: true
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if .Values.tls.proxy.enabled }}
|
|
- mountPath: "/pulsar/certs/proxy"
|
|
name: proxy-certs
|
|
readOnly: true
|
|
{{- end}}
|
|
{{- if .Values.tls.enabled }}
|
|
- mountPath: "/pulsar/certs/ca"
|
|
name: ca
|
|
readOnly: true
|
|
{{- end}}
|
|
{{- end}}
|
|
{{- if or .Values.auth.authentication.enabled (and .Values.tls.enabled .Values.tls.proxy.enabled) }}
|
|
volumes:
|
|
{{- if .Values.auth.authentication.enabled }}
|
|
{{- if eq .Values.auth.authentication.provider "jwt" }}
|
|
- name: token-keys
|
|
secret:
|
|
{{- if not .Values.auth.authentication.jwt.usingSecretKey }}
|
|
secretName: "{{ .Release.Name }}-token-asymmetric-key"
|
|
{{- end}}
|
|
{{- if .Values.auth.authentication.jwt.usingSecretKey }}
|
|
secretName: "{{ .Release.Name }}-token-symmetric-key"
|
|
{{- end}}
|
|
items:
|
|
{{- if .Values.auth.authentication.jwt.usingSecretKey }}
|
|
- key: SECRETKEY
|
|
path: token/secret.key
|
|
{{- else }}
|
|
- key: PUBLICKEY
|
|
path: token/public.key
|
|
{{- end}}
|
|
- name: proxy-token
|
|
secret:
|
|
secretName: "{{ .Release.Name }}-token-{{ .Values.auth.superUsers.proxy }}"
|
|
items:
|
|
- key: TOKEN
|
|
path: proxy/token
|
|
{{- end}}
|
|
{{- end}}
|
|
{{- if .Values.tls.proxy.enabled }}
|
|
- name: ca
|
|
secret:
|
|
secretName: "{{ .Release.Name }}-ca-tls"
|
|
items:
|
|
- key: ca.crt
|
|
path: ca.crt
|
|
- name: proxy-certs
|
|
secret:
|
|
secretName: "{{ .Release.Name }}-{{ .Values.tls.proxy.cert_name }}"
|
|
items:
|
|
- key: tls.crt
|
|
path: tls.crt
|
|
- key: tls.key
|
|
path: tls.key
|
|
{{- end}}
|
|
{{- end}}
|
|
{{- end }}
|