apache/pulsar-helm-chart
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
pulsar-helm-chart/charts/pulsar/templates/broker-configmap.yaml
Bruno Domenici a9f2ba76ae
OpenID: introducing support for OpenID configuration (#509) 
* feat!(openid): introducing support for openid configuration

BREAKING CHANGE: provider configuration changed from auth.authentication.provider to auth.authentication.jwt.enabled

* add upgrading to 4.1.0

* add validation for deprecated values

* add openid CI with keycloak

* fix chart-testing lint new-line-at-end-of-file

* fix keycloak dependency repository

* fix keycloak repository

* fix yaml to json convert error

* disable keycloak to validate github actions before re-enable it

* disable openid test scenario

* disable keycloak in values

* enable keycloak without authentication and authorization

* add openid test scenario

* disable test scenario other than openid

* enable all test scenario

* disable functions component

* create openid resources

* test truncate command

* test truncate command

* change client_secret generator

* change client_secret generator

* test python

* fix script

* fix script

* print python result

* test python

* test python

* fix client_secret generation

* fix create openid resources

* fix secret name

* fix mount keycloak config

* fix keycloak service

* exclude keycloak from chart

* add license

* add license

* wait keycloak is alive

* fix keycloak chart install namespace

* add test pulsar real openid config

* fix keycloak issuer url

* fix pod name

* remove check keycloak alive

* check realm pulsar openid configuration

* change keycloak service

* remove test keyclock service

* remove selector to get all pod log

* wait keycloak is alive

* check keycloak realm pulsar urls

* wait until keycloak is ready

* add wait timeout

* fix realm pulsar name

* add log to debug

* add openid for toolset

* set authorization

* set authorization

* fix client template filename

* fix install keycloak

* disable authorization

* debug sub claim value

* fix sub claim value

* cleanup

* enable all build

---------

Co-authored-by: glecroc <guillaume.lecroc@cnp.fr>
2025-05-20 14:09:12 +03:00

276 lines
17 KiB
YAML
Raw Blame History

#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.components.broker }}
apiVersion: v1
kind: ConfigMap
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}"
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.broker.component }}
data:
# Metadata settings
{{- if .Values.components.zookeeper }}
metadataStoreUrl: "zk:{{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }}"
{{- $configMetadataStoreUrl := "" }}
{{- if .Values.pulsar_metadata.configurationStore }}
{{- $configMetadataStoreUrl = printf "zk:%s%s" (include "pulsar.configurationStore.connect" .) .Values.pulsar_metadata.configurationStoreMetadataPrefix }}
{{- else }}
{{- $configMetadataStoreUrl = printf "zk:%s%s" (include "pulsar.zookeeper.connect" .) .Values.metadataPrefix }}
{{- end }}
configurationMetadataStoreUrl: "{{ $configMetadataStoreUrl }}"
{{- if .Values.pulsar_metadata.bookkeeper.usePulsarMetadataClientDriver }}
bookkeeperMetadataServiceUri: "metadata-store:{{ $configMetadataStoreUrl }}/ledgers"
{{- else }}
bookkeeperMetadataServiceUri: "zk+hierarchical://{{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }}/ledgers"
{{- end }}
{{- end }}
{{- if .Values.components.oxia }}
metadataStoreUrl: "{{ template "pulsar.oxia.metadata.url.broker" . }}"
configurationMetadataStoreUrl: "{{ template "pulsar.oxia.metadata.url.broker" . }}"
bookkeeperMetadataServiceUri: "{{ template "pulsar.oxia.metadata.url.bookkeeper" . }}"
{{- end }}
{{- if hasKey .Values.pulsar_metadata "metadataStoreAllowReadOnlyOperations" }}
PULSAR_PREFIX_metadataStoreAllowReadOnlyOperations: "{{ .Values.pulsar_metadata.metadataStoreAllowReadOnlyOperations }}"
{{- end }}
{{- if hasKey .Values.pulsar_metadata "metadataStoreSessionTimeoutMillis" }}
metadataStoreSessionTimeoutMillis: "{{ .Values.pulsar_metadata.metadataStoreSessionTimeoutMillis }}"
{{- end }}
{{- if hasKey .Values.pulsar_metadata "metadataStoreOperationTimeoutSeconds" }}
metadataStoreOperationTimeoutSeconds: "{{ .Values.pulsar_metadata.metadataStoreOperationTimeoutSeconds }}"
{{- end }}
{{- if hasKey .Values.pulsar_metadata "metadataStoreCacheExpirySeconds" }}
metadataStoreCacheExpirySeconds: "{{ .Values.pulsar_metadata.metadataStoreCacheExpirySeconds }}"
{{- end }}
{{- if hasKey .Values.pulsar_metadata "metadataStoreBatchingEnabled" }}
metadataStoreBatchingEnabled: "{{ .Values.pulsar_metadata.metadataStoreBatchingEnabled }}"
{{- end }}
{{- if hasKey .Values.pulsar_metadata "metadataStoreBatchingMaxDelayMillis" }}
metadataStoreBatchingMaxDelayMillis: "{{ .Values.pulsar_metadata.metadataStoreBatchingMaxDelayMillis }}"
{{- end }}
{{- if hasKey .Values.pulsar_metadata "metadataStoreBatchingMaxOperations" }}
metadataStoreBatchingMaxOperations: "{{ .Values.pulsar_metadata.metadataStoreBatchingMaxOperations }}"
{{- end }}
{{- if hasKey .Values.pulsar_metadata "metadataStoreBatchingMaxSizeKb" }}
metadataStoreBatchingMaxSizeKb: "{{ .Values.pulsar_metadata.metadataStoreBatchingMaxSizeKb }}"
{{- end }}
# Broker settings
clusterName: {{ template "pulsar.cluster.name" . }}
# Enable all metrics by default
exposeTopicLevelMetricsInPrometheus: "true"
exposeConsumerLevelMetricsInPrometheus: "true"
exposeProducerLevelMetricsInPrometheus: "true"
exposeManagedLedgerMetricsInPrometheus: "true"
exposeManagedCursorMetricsInPrometheus: "true"
exposeBundlesMetricsInPrometheus: "true"
exposePublisherStats: "true"
exposePreciseBacklogInPrometheus: "true"
replicationMetricsEnabled: "true"
splitTopicAndPartitionLabelInPrometheus: "true"
aggregatePublisherStatsByProducerName: "true"
bookkeeperClientExposeStatsToPrometheus: "true"
numHttpServerThreads: "8"
statusFilePath: "{{ template "pulsar.home" . }}/logs/status"
# Tiered storage settings
{{- if .Values.broker.storageOffload.driver }}
{{- if eq .Values.broker.storageOffload.driver "aws-s3" }}
managedLedgerOffloadDriver: "{{ .Values.broker.storageOffload.driver }}"
s3ManagedLedgerOffloadBucket: "{{ .Values.broker.storageOffload.bucket }}"
s3ManagedLedgerOffloadRegion: "{{ .Values.broker.storageOffload.region }}"
{{- if .Values.broker.storageOffload.managedLedgerOffloadAutoTriggerSizeThresholdBytes }}
PULSAR_PREFIX_managedLedgerOffloadThresholdInBytes: "{{ .Values.broker.storageOffload.managedLedgerOffloadAutoTriggerSizeThresholdBytes }}"
{{- end }}
{{- if .Values.broker.storageOffload.managedLedgerOffloadDeletionLagMs }}
PULSAR_PREFIX_managedLedgerOffloadDeletionLagInMillis: "{{ .Values.broker.storageOffload.managedLedgerOffloadDeletionLagMs }}"
{{- end }}
{{- if .Values.broker.storageOffload.maxBlockSizeInBytes }}
s3ManagedLedgerOffloadMaxBlockSizeInBytes: "{{ .Values.broker.storageOffload.maxBlockSizeInBytes }}"
{{- end }}
{{- if .Values.broker.storageOffload.readBufferSizeInBytes }}
s3ManagedLedgerOffloadReadBufferSizeInBytes: "{{ .Values.broker.storageOffload.readBufferSizeInBytes }}"
{{- end }}
{{- end }}
{{- if eq .Values.broker.storageOffload.driver "google-cloud-storage" }}
managedLedgerOffloadDriver: "{{ .Values.broker.storageOffload.driver }}"
gcsManagedLedgerOffloadBucket: "{{ .Values.broker.storageOffload.bucket }}"
gcsManagedLedgerOffloadRegion: "{{ .Values.broker.storageOffload.region }}"
gcsManagedLedgerOffloadServiceAccountKeyFile: "/pulsar/gcp-service-account/{{ .Values.broker.storageOffload.gcsServiceAccountJsonFile }}"
{{- if .Values.broker.storageOffload.managedLedgerOffloadAutoTriggerSizeThresholdBytes }}
PULSAR_PREFIX_managedLedgerOffloadThresholdInBytes: "{{ .Values.broker.storageOffload.managedLedgerOffloadAutoTriggerSizeThresholdBytes }}"
{{- end }}
{{- if .Values.broker.storageOffload.managedLedgerOffloadDeletionLagMs }}
PULSAR_PREFIX_managedLedgerOffloadDeletionLagInMillis: "{{ .Values.broker.storageOffload.managedLedgerOffloadDeletionLagMs }}"
{{- end }}
{{- if .Values.broker.storageOffload.maxBlockSizeInBytes }}
gcsManagedLedgerOffloadMaxBlockSizeInBytes: "{{ .Values.broker.storageOffload.maxBlockSizeInBytes }}"
{{- end }}
{{- if .Values.broker.storageOffload.readBufferSizeInBytes }}
gcsManagedLedgerOffloadReadBufferSizeInBytes: "{{ .Values.broker.storageOffload.readBufferSizeInBytes }}"
{{- end }}
{{- end }}
{{- if eq .Values.broker.storageOffload.driver "azureblob" }}
managedLedgerOffloadDriver: "{{ .Values.broker.storageOffload.driver }}"
managedLedgerOffloadBucket: "{{ .Values.broker.storageOffload.bucket }}"
{{- if .Values.broker.storageOffload.managedLedgerOffloadAutoTriggerSizeThresholdBytes }}
PULSAR_PREFIX_managedLedgerOffloadThresholdInBytes: "{{ .Values.broker.storageOffload.managedLedgerOffloadAutoTriggerSizeThresholdBytes }}"
{{- end }}
{{- if .Values.broker.storageOffload.managedLedgerOffloadDeletionLagMs }}
PULSAR_PREFIX_managedLedgerOffloadDeletionLagInMillis: "{{ .Values.broker.storageOffload.managedLedgerOffloadDeletionLagMs }}"
{{- end }}
{{- if .Values.broker.storageOffload.maxBlockSizeInBytes }}
managedLedgerOffloadMaxBlockSizeInBytes: "{{ .Values.broker.storageOffload.maxBlockSizeInBytes }}"
{{- end }}
{{- end }}
{{- end }}
# Function Worker Settings
# function worker configuration
{{- if not .Values.components.functions }}
functionsWorkerEnabled: "false"
{{- end }}
{{- if .Values.components.functions }}
functionsWorkerEnabled: "true"
{{- if .Values.functions.useBookieAsStateStore }}
PF_stateStorageServiceUrl: "bk://{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}:{{ .Values.bookkeeper.ports.statestore }}"
{{- end }}
PF_functionRuntimeFactoryClassName: "org.apache.pulsar.functions.runtime.kubernetes.KubernetesRuntimeFactory"
PF_pulsarFunctionsCluster: {{ template "pulsar.cluster.name" . }}
PF_connectorsDirectory: ./connectors
PF_containerFactory: k8s
PF_numFunctionPackageReplicas: "{{ .Values.broker.configData.managedLedgerDefaultEnsembleSize }}"
# support version >= 2.5.0
PF_functionRuntimeFactoryConfigs_pulsarRootDir: {{ template "pulsar.home" . }}
PF_kubernetesContainerFactory_pulsarRootDir: {{ template "pulsar.home" . }}
PF_functionRuntimeFactoryConfigs_pulsarDockerImageName: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.functions "root" .) }}"
PF_functionRuntimeFactoryConfigs_imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.functions "root" .) }}"
PF_functionRuntimeFactoryConfigs_submittingInsidePod: "true"
PF_functionRuntimeFactoryConfigs_installUserCodeDependencies: "true"
PF_functionRuntimeFactoryConfigs_jobNamespace: {{ template "pulsar.namespace" . }}
PF_functionRuntimeFactoryConfigs_expectedMetricsCollectionInterval: "30"
{{- if not (and .Values.tls.enabled .Values.tls.broker.enabled .Values.tls.function_instance.enabled) }}
PF_functionRuntimeFactoryConfigs_pulsarAdminUrl: "http://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.http }}/"
PF_functionRuntimeFactoryConfigs_pulsarServiceUrl: "pulsar://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.pulsar }}/"
{{- else }}
PF_functionRuntimeFactoryConfigs_pulsarAdminUrl: "https://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.https }}/"
PF_functionRuntimeFactoryConfigs_pulsarServiceUrl: "pulsar+ssl://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.pulsarssl }}/"
{{- end }}
# support version < 2.5.0
PF_kubernetesContainerFactory_pulsarDockerImageName: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.functions "root" .) }}"
PF_kubernetesContainerFactory_imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.functions "root" .) }}"
PF_kubernetesContainerFactory_submittingInsidePod: "true"
PF_kubernetesContainerFactory_installUserCodeDependencies: "true"
PF_kubernetesContainerFactory_jobNamespace: {{ template "pulsar.namespace" . }}
PF_kubernetesContainerFactory_expectedMetricsCollectionInterval: "30"
{{- if not (and .Values.tls.enabled .Values.tls.broker.enabled .Values.tls.function_instance.enabled) }}
PF_kubernetesContainerFactory_pulsarAdminUrl: "http://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.http }}/"
PF_kubernetesContainerFactory_pulsarServiceUrl: "pulsar://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.pulsar }}/"
{{- else }}
PF_kubernetesContainerFactory_pulsarAdminUrl: "https://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.https }}/"
PF_kubernetesContainerFactory_pulsarServiceUrl: "pulsar+ssl://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.pulsarssl }}/"
{{- end }}
{{- end }}
# prometheus needs to access /metrics endpoint
webServicePort: "{{ .Values.broker.ports.http }}"
{{- if or (not .Values.tls.enabled) (not .Values.tls.broker.enabled) }}
brokerServicePort: "{{ .Values.broker.ports.pulsar }}"
{{- end }}
{{- if and .Values.tls.enabled .Values.tls.broker.enabled }}
brokerServicePortTls: "{{ .Values.broker.ports.pulsarssl }}"
webServicePortTls: "{{ .Values.broker.ports.https }}"
# TLS Settings
tlsCertificateFilePath: "/pulsar/certs/broker/tls.crt"
tlsKeyFilePath: "/pulsar/certs/broker/tls.key"
tlsTrustCertsFilePath: "/pulsar/certs/ca/ca.crt"
{{- end }}
# Authentication Settings
{{- if .Values.auth.authentication.enabled }}
authenticationEnabled: "true"
{{- if .Values.auth.authorization.enabled }}
authorizationEnabled: "true"
superUserRoles: {{ .Values.auth.superUsers | values | compact | sortAlpha | join "," }}
{{- if .Values.auth.useProxyRoles }}
proxyRoles: {{ .Values.auth.superUsers.proxy }}
{{- end }}
{{- end }}
{{- if and .Values.auth.authentication.enabled .Values.auth.authentication.jwt.enabled }}
# token authentication configuration
{{- if and .Values.auth.authentication.enabled .Values.auth.authentication.jwt.enabled .Values.auth.authentication.openid.enabled }}
authenticationProviders: "org.apache.pulsar.broker.authentication.AuthenticationProviderToken,org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID"
{{- end }}
{{- if and .Values.auth.authentication.enabled .Values.auth.authentication.jwt.enabled ( not .Values.auth.authentication.openid.enabled ) }}
authenticationProviders: "org.apache.pulsar.broker.authentication.AuthenticationProviderToken"
{{- end }}
brokerClientAuthenticationParameters: "file:///pulsar/tokens/broker/token"
brokerClientAuthenticationPlugin: "org.apache.pulsar.client.impl.auth.AuthenticationToken"
{{- if .Values.auth.authentication.jwt.usingSecretKey }}
tokenSecretKey: "file:///pulsar/keys/token/secret.key"
{{- else }}
tokenPublicKey: "file:///pulsar/keys/token/public.key"
{{- end }}
{{- end }}
{{- if and .Values.auth.authentication.enabled .Values.auth.authentication.openid.enabled }}
# openid authentication configuration
{{- if and .Values.auth.authentication.enabled .Values.auth.authentication.openid.enabled ( not .Values.auth.authentication.jwt.enabled ) }}
authenticationProviders: "org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID"
{{- end }}
PULSAR_PREFIX_openIDAllowedTokenIssuers: {{ .Values.auth.authentication.openid.openIDAllowedTokenIssuers | uniq | compact | sortAlpha | join "," | quote }}
PULSAR_PREFIX_openIDAllowedAudiences: {{ .Values.auth.authentication.openid.openIDAllowedAudiences | uniq | compact | sortAlpha | join "," | quote }}
PULSAR_PREFIX_openIDTokenIssuerTrustCertsFilePath: {{ .Values.auth.authentication.openid.openIDTokenIssuerTrustCertsFilePath | quote }}
PULSAR_PREFIX_openIDRoleClaim: {{ .Values.auth.authentication.openid.openIDRoleClaim | quote }}
PULSAR_PREFIX_openIDAcceptedTimeLeewaySeconds: {{ .Values.auth.authentication.openid.openIDAcceptedTimeLeewaySeconds | quote }}
PULSAR_PREFIX_openIDCacheSize: {{ .Values.auth.authentication.openid.openIDCacheSize | quote }}
PULSAR_PREFIX_openIDCacheRefreshAfterWriteSeconds: {{ .Values.auth.authentication.openid.openIDCacheRefreshAfterWriteSeconds | quote }}
PULSAR_PREFIX_openIDCacheExpirationSeconds: {{ .Values.auth.authentication.openid.openIDCacheExpirationSeconds | quote }}
PULSAR_PREFIX_openIDHttpConnectionTimeoutMillis: {{ .Values.auth.authentication.openid.openIDHttpConnectionTimeoutMillis | quote }}
PULSAR_PREFIX_openIDHttpReadTimeoutMillis: {{ .Values.auth.authentication.openid.openIDHttpReadTimeoutMillis | quote }}
PULSAR_PREFIX_openIDKeyIdCacheMissRefreshSeconds: {{ .Values.auth.authentication.openid.openIDKeyIdCacheMissRefreshSeconds | quote }}
PULSAR_PREFIX_openIDRequireIssuersUseHttps: {{ .Values.auth.authentication.openid.openIDRequireIssuersUseHttps | quote }}
PULSAR_PREFIX_openIDFallbackDiscoveryMode: {{ .Values.auth.authentication.openid.openIDFallbackDiscoveryMode | quote }}
{{- end }}
{{- end }}
{{- if and .Values.tls.enabled .Values.tls.bookie.enabled }}
# bookkeeper tls settings
bookkeeperTLSClientAuthentication: "true"
bookkeeperTLSKeyFileType: "PEM"
bookkeeperTLSKeyFilePath: "/pulsar/certs/broker/tls.key"
bookkeeperTLSCertificateFilePath: "/pulsar/certs/broker/tls.crt"
bookkeeperTLSTrustCertsFilePath: "/pulsar/certs/ca/ca.crt"
bookkeeperTLSTrustCertTypes: "PEM"
PULSAR_PREFIX_bookkeeperTLSClientAuthentication: "true"
PULSAR_PREFIX_bookkeeperTLSKeyFileType: "PEM"
PULSAR_PREFIX_bookkeeperTLSKeyFilePath: "/pulsar/certs/broker/tls.key"
PULSAR_PREFIX_bookkeeperTLSCertificateFilePath: "/pulsar/certs/broker/tls.crt"
PULSAR_PREFIX_bookkeeperTLSTrustCertsFilePath: "/pulsar/certs/ca/ca.crt"
PULSAR_PREFIX_bookkeeperTLSTrustCertTypes: "PEM"
# https://github.com/apache/bookkeeper/pull/2300
bookkeeperUseV2WireProtocol: "false"
{{- end }}
{{ toYaml .Values.broker.configData | indent 2 }}
{{- end }}
Reference in New Issue View Git Blame Copy Permalink