diff --git a/README.md b/README.md index bdc48ce..7e51a9c 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ It includes support for: - [ ] Mutal TLS - [ ] Kerberos - [x] Authorization + - [x] Non-root broker, bookkeeper, proxy, and zookeeper containers (version 2.10.0 and above) - [x] Storage - [x] Non-persistence storage - [x] Persistence Volume @@ -178,6 +179,56 @@ helm upgrade -f pulsar.yaml \ For more detailed information, see our [Upgrading](http://pulsar.apache.org/docs/en/helm-upgrade/) guide. +## Upgrading to 2.10.0 and above + +The 2.10.0+ Apache Pulsar docker image is a non-root container, by default. That complicates an upgrade to 2.10.0 +because the existing files are owned by the root user but are not writable by the root group. In order to leverage this +new security feature, the Bookkeeper and Zookeeper StatefulSet [securityContexts](https://kubernetes.io/docs/tasks/configure-pod-container/security-context) +are configurable in the `values.yaml`. They default to: + +```yaml + securityContext: + fsGroup: 0 + fsGroupChangePolicy: "OnRootMismatch" +``` + +This configuration is ideal for regular Kubernetes clusters where the UID is stable across restarts. If the process +UID is subject to change (like it is in OpenShift), you'll need to set `fsGroupChangePolicy: "Always"`. + +The official docker image assumes that it is run as a member of the root group. + +If you upgrade to the latest version of the helm chart before upgrading to Pulsar 2.10.0, then when you perform your +first upgrade to version >= 2.10.0, you will need to set `fsGroupChangePolicy: "Always"` on the first upgrade and then +set it back to `fsGroupChangePolicy: "OnRootMismatch"` on subsequent upgrades. This is because the root file won't +mismatch permissions, but the RocksDB lock file will. If you have direct access to the persistent volumes, you can +alternatively run `chgrp -R g+w /pulsar/data` before upgrading. + +Here is a sample error you can expect if the RocksDB lock file is not correctly owned by the root group: + +```text +2022-05-14T03:45:06,903+0000 ERROR org.apache.bookkeeper.server.Main - Failed to build bookie server +java.io.IOException: Error open RocksDB database + at org.apache.bookkeeper.bookie.storage.ldb.KeyValueStorageRocksDB.(KeyValueStorageRocksDB.java:199) ~[org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4] + at org.apache.bookkeeper.bookie.storage.ldb.KeyValueStorageRocksDB.(KeyValueStorageRocksDB.java:88) ~[org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4] + at org.apache.bookkeeper.bookie.storage.ldb.KeyValueStorageRocksDB.lambda$static$0(KeyValueStorageRocksDB.java:62) ~[org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4] + at org.apache.bookkeeper.bookie.storage.ldb.LedgerMetadataIndex.(LedgerMetadataIndex.java:68) ~[org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4] + at org.apache.bookkeeper.bookie.storage.ldb.SingleDirectoryDbLedgerStorage.(SingleDirectoryDbLedgerStorage.java:169) ~[org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4] + at org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage.newSingleDirectoryDbLedgerStorage(DbLedgerStorage.java:150) ~[org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4] + at org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage.initialize(DbLedgerStorage.java:129) ~[org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4] + at org.apache.bookkeeper.bookie.Bookie.(Bookie.java:818) ~[org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4] + at org.apache.bookkeeper.proto.BookieServer.newBookie(BookieServer.java:152) ~[org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4] + at org.apache.bookkeeper.proto.BookieServer.(BookieServer.java:120) ~[org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4] + at org.apache.bookkeeper.server.service.BookieService.(BookieService.java:52) ~[org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4] + at org.apache.bookkeeper.server.Main.buildBookieServer(Main.java:304) ~[org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4] + at org.apache.bookkeeper.server.Main.doMain(Main.java:226) [org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4] + at org.apache.bookkeeper.server.Main.main(Main.java:208) [org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4] +Caused by: org.rocksdb.RocksDBException: while open a file for lock: /pulsar/data/bookkeeper/ledgers/current/ledgers/LOCK: Permission denied + at org.rocksdb.RocksDB.open(Native Method) ~[org.rocksdb-rocksdbjni-6.10.2.jar:?] + at org.rocksdb.RocksDB.open(RocksDB.java:239) ~[org.rocksdb-rocksdbjni-6.10.2.jar:?] + at org.apache.bookkeeper.bookie.storage.ldb.KeyValueStorageRocksDB.(KeyValueStorageRocksDB.java:196) ~[org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4] + ... 13 more +``` + ## Uninstall To uninstall the Pulsar Chart, run the following command: diff --git a/charts/pulsar/templates/bookkeeper-statefulset.yaml b/charts/pulsar/templates/bookkeeper-statefulset.yaml index db63c82..43c4ba0 100644 --- a/charts/pulsar/templates/bookkeeper-statefulset.yaml +++ b/charts/pulsar/templates/bookkeeper-statefulset.yaml @@ -104,6 +104,10 @@ spec: {{- if and .Values.rbac.enabled .Values.rbac.psp }} serviceAccountName: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}" {{- end}} + {{- if .Values.bookkeeper.securityContext }} + securityContext: +{{ toYaml .Values.bookkeeper.securityContext | indent 8 }} + {{- end }} initContainers: # This initContainer will wait for bookkeeper initnewcluster to complete # before deploying the bookies diff --git a/charts/pulsar/templates/zookeeper-statefulset.yaml b/charts/pulsar/templates/zookeeper-statefulset.yaml index 4313f7f..12640df 100644 --- a/charts/pulsar/templates/zookeeper-statefulset.yaml +++ b/charts/pulsar/templates/zookeeper-statefulset.yaml @@ -101,6 +101,10 @@ spec: {{- if and .Values.rbac.enabled .Values.rbac.psp }} serviceAccountName: "{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}" {{- end }} + {{- if .Values.zookeeper.securityContext }} + securityContext: +{{ toYaml .Values.zookeeper.securityContext | indent 8 }} + {{- end }} containers: - name: "{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}" image: "{{ .Values.images.zookeeper.repository }}:{{ .Values.images.zookeeper.tag }}" diff --git a/charts/pulsar/values.yaml b/charts/pulsar/values.yaml index 2193169..4c6b5c4 100644 --- a/charts/pulsar/values.yaml +++ b/charts/pulsar/values.yaml @@ -361,6 +361,10 @@ zookeeper: # readOnly: true extraVolumes: [] extraVolumeMounts: [] + # Ensures 2.10.0 non-root docker image works correctly. + securityContext: + fsGroup: 0 + fsGroupChangePolicy: "OnRootMismatch" volumes: # use a persistent volume or emptyDir persistence: true @@ -489,6 +493,10 @@ bookkeeper: # readOnly: true extraVolumes: [] extraVolumeMounts: [] + # Ensures 2.10.0 non-root docker image works correctly. + securityContext: + fsGroup: 0 + fsGroupChangePolicy: "OnRootMismatch" volumes: # use a persistent volume or emptyDir persistence: true @@ -572,7 +580,6 @@ bookkeeper: -Xlog:safepoint -Xlog:gc+heap=trace -verbosegc - -Xlog:gc:/var/log/bookie-gc.log # configure the memory settings based on jvm memory settings dbStorage_writeCacheMaxSizeMb: "32" dbStorage_readAheadCacheMaxSizeMb: "32"