From 6c2edba8b1c3e56b8b2bdc2689004fe88637db28 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 31 Aug 2020 08:05:49 +0300 Subject: [PATCH] Get OS signals passed to container process by using shell built-in "exec" (#59) ### Changes - using "exec" to run a command replaces the shell process with the executed process - this is required so that the process running in the container is able to receive OS signals - explained in https://docs.docker.com/develop/develop-images/dockerfile_best-practices/ and https://docs.docker.com/engine/reference/builder/#entrypoint - receiving SIGTERM signal is required for graceful shutdown. This is explained in https://pracucci.com/graceful-shutdown-of-kubernetes-pods.html This change might fix issues such as https://github.com/apache/pulsar/issues/6603 . One expectation of this fix is that graceful shutdown would allow Pulsar components such as a bookies to deregistered from Zookeeper properly before shutdown. ### Motivation Dockerfile best practices mention that "exec" should be used so that the process running in a container can receive OS signals. This is explained in https://docs.docker.com/develop/develop-images/dockerfile_best-practices/ and https://docs.docker.com/engine/reference/builder/#entrypoint . Kubernetes documention explains pod termination in https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-termination : "Typically, the container runtime sends a TERM signal to the main process in each container. Once the grace period has expired, the KILL signal is sent to any remaining processes, and the Pod is then deleted from the API Server ." Currently some issues while running Pulsar are caused by the lack of graceful shutdown. Graceful shutdown isn't happening at all since the Pulsar processes never receive the TERM signal that would allow graceful shutdown. This PR fixes that. This PR was inspired by https://github.com/kafkaesque-io/pulsar-helm-chart/pull/31 --- charts/pulsar/templates/autorecovery-statefulset.yaml | 2 +- charts/pulsar/templates/bookkeeper-statefulset.yaml | 2 +- charts/pulsar/templates/broker-statefulset.yaml | 2 +- charts/pulsar/templates/proxy-statefulset.yaml | 2 +- charts/pulsar/templates/zookeeper-statefulset.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/charts/pulsar/templates/autorecovery-statefulset.yaml b/charts/pulsar/templates/autorecovery-statefulset.yaml index 3fb22ce..0ededb7 100644 --- a/charts/pulsar/templates/autorecovery-statefulset.yaml +++ b/charts/pulsar/templates/autorecovery-statefulset.yaml @@ -129,7 +129,7 @@ spec: - > bin/apply-config-from-env.py conf/bookkeeper.conf; {{- include "pulsar.autorecovery.zookeeper.tls.settings" . | nindent 10 }} - bin/bookkeeper autorecovery + exec bin/bookkeeper autorecovery ports: - name: http containerPort: {{ .Values.autorecovery.ports.http }} diff --git a/charts/pulsar/templates/bookkeeper-statefulset.yaml b/charts/pulsar/templates/bookkeeper-statefulset.yaml index 93db78d..f5fc81a 100644 --- a/charts/pulsar/templates/bookkeeper-statefulset.yaml +++ b/charts/pulsar/templates/bookkeeper-statefulset.yaml @@ -154,7 +154,7 @@ spec: - > bin/apply-config-from-env.py conf/bookkeeper.conf; {{- include "pulsar.bookkeeper.zookeeper.tls.settings" . | nindent 10 }} - bin/pulsar bookie; + exec bin/pulsar bookie; ports: - name: bookie containerPort: {{ .Values.bookkeeper.ports.bookie }} diff --git a/charts/pulsar/templates/broker-statefulset.yaml b/charts/pulsar/templates/broker-statefulset.yaml index 63c9596..a5ce2b1 100644 --- a/charts/pulsar/templates/broker-statefulset.yaml +++ b/charts/pulsar/templates/broker-statefulset.yaml @@ -195,7 +195,7 @@ spec: bin/pulsar zookeeper-shell -server {{ template "pulsar.zookeeper.connect" . }} get {{ template "pulsar.broker.znode" . }}; done; cat conf/pulsar_env.sh; - bin/pulsar broker; + exec bin/pulsar broker; ports: # prometheus needs to access /metrics endpoint - name: http diff --git a/charts/pulsar/templates/proxy-statefulset.yaml b/charts/pulsar/templates/proxy-statefulset.yaml index 1044f52..e10722b 100644 --- a/charts/pulsar/templates/proxy-statefulset.yaml +++ b/charts/pulsar/templates/proxy-statefulset.yaml @@ -171,7 +171,7 @@ spec: - > bin/apply-config-from-env.py conf/proxy.conf && echo "OK" > status && - bin/pulsar proxy + exec bin/pulsar proxy ports: # prometheus needs to access /metrics endpoint - name: http diff --git a/charts/pulsar/templates/zookeeper-statefulset.yaml b/charts/pulsar/templates/zookeeper-statefulset.yaml index 205d540..d3effe8 100644 --- a/charts/pulsar/templates/zookeeper-statefulset.yaml +++ b/charts/pulsar/templates/zookeeper-statefulset.yaml @@ -109,7 +109,7 @@ spec: bin/apply-config-from-env.py conf/zookeeper.conf; {{- include "pulsar.zookeeper.tls.settings" . | nindent 10 }} bin/generate-zookeeper-config.sh conf/zookeeper.conf; - bin/pulsar zookeeper; + exec bin/pulsar zookeeper; ports: # prometheus needs to access /metrics endpoint - name: http