Compare commits

...

No commits in common. "master" and "gh-pages" have entirely different histories.

161 changed files with 20 additions and 17387 deletions

View File

@ -1,47 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
github:
description: "Official Apache Pulsar Helm Chart"
homepage: https://pulsar.apache.org/
labels:
- pulsar
- pubsub
- messaging
- streaming
- queuing
- event-streaming
- kubernetes
- helm
- helm-chart
features:
wiki: true
issues: true
projects: true
enabled_merge_buttons:
squash: true
merge: false
rebase: false
notifications:
commits: commits@pulsar.apache.org
issues: commits@pulsar.apache.org
pullrequests: commits@pulsar.apache.org
discussions: dev@pulsar.apache.org
jira_options: link label

File diff suppressed because it is too large Load Diff

View File

@ -1,73 +0,0 @@
{
"clientId": $ARGS.named.CLIENT_ID,
"enabled": true,
"clientAuthenticatorType": "client-secret",
"secret": $ARGS.named.CLIENT_SECRET,
"standardFlowEnabled" : false,
"implicitFlowEnabled" : false,
"serviceAccountsEnabled": true,
"protocol": "openid-connect",
"attributes": {
"realm_client": "false",
"oidc.ciba.grant.enabled": "false",
"client.secret.creation.time": "1735689600",
"backchannel.logout.session.required": "true",
"standard.token.exchange.enabled": "false",
"frontchannel.logout.session.required": "true",
"oauth2.device.authorization.grant.enabled": "false",
"display.on.consent.screen": "false",
"backchannel.logout.revoke.offline.tokens": "false"
},
"protocolMappers": [
{
"name": "sub",
"protocol": "openid-connect",
"protocolMapper": "oidc-hardcoded-claim-mapper",
"consentRequired": false,
"config": {
"introspection.token.claim": "true",
"claim.value": $ARGS.named.SUB_CLAIM_VALUE,
"userinfo.token.claim": "true",
"id.token.claim": "true",
"lightweight.claim": "false",
"access.token.claim": "true",
"claim.name": "sub",
"jsonType.label": "String",
"access.tokenResponse.claim": "false"
}
},
{
"name": "nbf",
"protocol": "openid-connect",
"protocolMapper": "oidc-hardcoded-claim-mapper",
"consentRequired": false,
"config": {
"introspection.token.claim": "true",
"claim.value": "1735689600",
"userinfo.token.claim": "true",
"id.token.claim": "true",
"lightweight.claim": "false",
"access.token.claim": "true",
"claim.name": "nbf",
"jsonType.label": "long",
"access.tokenResponse.claim": "false"
}
}
],
"defaultClientScopes": [
"web-origins",
"service_account",
"acr",
"profile",
"roles",
"basic",
"email"
],
"optionalClientScopes": [
"address",
"phone",
"organization",
"offline_access",
"microprofile-jwt"
]
}

View File

@ -1,26 +0,0 @@
# Keycloak
Keycloak is used to validate OIDC configuration.
To create the pulsar realm configuration, we use :
* `0-realm-pulsar-partial-export.json` : after creating pulsar realm in Keycloack UI, this file is the result of the partial export in Keycloak UI without options.
* `1-client-template.json` : this is the template to create pulsar clients.
To create the final `realm-pulsar.json`, merge files with `jq` command :
* create a client with `CLIENT_ID`, `CLIENT_SECRET` and `SUB_CLAIM_VALUE` :
```
CLIENT_ID=xx
CLIENT_SECRET=yy
SUB_CLAIM_VALUE=zz
jq -n --arg CLIENT_ID "$CLIENT_ID" --arg CLIENT_SECRET "$CLIENT_SECRET" --arg SUB_CLAIM_VALUE "$SUB_CLAIM_VALUE" 1-client-template.json > client.json
```
* then merge the realm and the client :
```
jq '.clients += [input]' 0-realm-pulsar-partial-export.json client.json > realm-pulsar.json
```

View File

@ -1,34 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
tls:
enabled: false
# This block sets up an example Pulsar Realm
# https://www.keycloak.org/server/importExport#_importing_a_realm_from_a_directory
extraEnvVars:
- name: KEYCLOAK_EXTRA_ARGS
value: "--import-realm"
extraVolumes:
- name: realm-config
secret:
secretName: keycloak-ci-realm-config
extraVolumeMounts:
- name: realm-config
mountPath: "/opt/bitnami/keycloak/data/import"
readOnly: true

View File

@ -1,5 +0,0 @@
{
"type": "client_credentials",
"client_id": $ARGS.named.CLIENT_ID,
"client_secret": $ARGS.named.CLIENT_SECRET
}

View File

@ -1,116 +0,0 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
set -e
BINDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
PULSAR_HOME="$(cd "${BINDIR}/.." && pwd)"
VALUES_FILE=$1
TLS=${TLS:-"false"}
SYMMETRIC=${SYMMETRIC:-"false"}
FUNCTION=${FUNCTION:-"false"}
MANAGER=${MANAGER:-"false"}
ALLOW_LOADBALANCERS=${ALLOW_LOADBALANCERS:-"false"}
source ${PULSAR_HOME}/.ci/helm.sh
# create cluster
ci::create_cluster
ci::helm_repo_add
extra_opts=()
# Add any arguments after $1 to extra_opts
shift # Remove $1 from the argument list
while [[ $# -gt 0 ]]; do
extra_opts+=("$1")
shift
done
if [[ "x${SYMMETRIC}" == "xtrue" ]]; then
extra_opts+=("-s")
fi
if [[ "x${EXTRA_SUPERUSERS}" != "x" ]]; then
extra_opts+=("--pulsar-superusers" "proxy-admin,broker-admin,admin,${EXTRA_SUPERUSERS}")
fi
install_type="install"
test_action="produce-consume"
if [[ "$UPGRADE_FROM_VERSION" != "" ]]; then
ALLOW_LOADBALANCERS="true"
# install older version of pulsar chart
PULSAR_CHART_VERSION="$UPGRADE_FROM_VERSION"
# Install Prometheus Operator CRDs using the upgrade script since kube-prometheus-stack is now disabled before the upgrade
${PULSAR_HOME}/scripts/kube-prometheus-stack/upgrade_prometheus_operator_crds.sh
ci::install_pulsar_chart install ${PULSAR_HOME}/.ci/values-common.yaml ${PULSAR_HOME}/${VALUES_FILE} --set kube-prometheus-stack.enabled=false "${extra_opts[@]}"
install_type="upgrade"
echo "Wait 10 seconds"
sleep 10
# check pulsar environment
ci::check_pulsar_environment
# test that we can access the admin api
ci::test_pulsar_admin_api_access
# produce messages with old version of pulsar and consume with new version
ci::test_pulsar_producer_consumer "produce"
test_action="consume"
if [[ "$(ci::helm_values_for_deployment | yq .victoria-metrics-k8s-stack.enabled)" == "true" ]]; then
echo "Upgrade Victoria Metrics Operator CRDs before upgrading the deployment"
${PULSAR_HOME}/scripts/victoria-metrics-k8s-stack/upgrade_vm_operator_crds.sh
fi
fi
PULSAR_CHART_VERSION="local"
# install (or upgrade) pulsar chart
ci::install_pulsar_chart ${install_type} ${PULSAR_HOME}/.ci/values-common.yaml ${PULSAR_HOME}/${VALUES_FILE} "${extra_opts[@]}"
echo "Wait 10 seconds"
sleep 10
# check that there aren't any loadbalancers if ALLOW_LOADBALANCERS is false
if [[ "${ALLOW_LOADBALANCERS}" == "false" ]]; then
ci::check_loadbalancers
fi
# check pulsar environment
ci::check_pulsar_environment
# test that we can access the admin api
ci::test_pulsar_admin_api_access
# test producer/consumer
ci::test_pulsar_producer_consumer "${test_action}"
if [[ "$(ci::helm_values_for_deployment | yq .components.functions)" == "true" ]]; then
# test functions
ci::test_pulsar_function
fi
if [[ "$(ci::helm_values_for_deployment | yq .components.pulsar_manager)" == "true" ]]; then
# test manager
ci::test_pulsar_manager
fi
# delete the cluster
ci::delete_cluster

View File

@ -1,30 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# enable TLS
tls:
enabled: true
bookie:
enabled: true
# enable cert-manager
certs:
internal_issuer:
enabled: true
type: selfsigning

View File

@ -1,32 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# enable TLS
tls:
enabled: true
proxy:
enabled: true
broker:
enabled: true
# enable cert-manager
certs:
internal_issuer:
enabled: true
type: selfsigning

View File

@ -1,105 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# enable TLS with cacerts
tls:
enabled: true
proxy:
enabled: true
cacerts:
enabled: true
certs:
- name: common-cacert
existingSecret: "pulsar-ci-common-cacert"
secretKeys:
- ca.crt
broker:
enabled: true
cacerts:
enabled: true
certs:
- name: common-cacert
existingSecret: "pulsar-ci-common-cacert"
secretKeys:
- ca.crt
bookie:
enabled: true
cacerts:
enabled: true
certs:
- name: common-cacert
existingSecret: "pulsar-ci-common-cacert"
secretKeys:
- ca.crt
zookeeper:
enabled: true
cacerts:
enabled: true
certs:
- name: common-cacert
existingSecret: "pulsar-ci-common-cacert"
secretKeys:
- ca.crt
toolset:
cacerts:
enabled: true
certs:
- name: common-cacert
existingSecret: "pulsar-ci-common-cacert"
secretKeys:
- ca.crt
autorecovery:
cacerts:
enabled: true
certs:
- name: common-cacert
existingSecret: "pulsar-ci-common-cacert"
secretKeys:
- ca.crt
# enable cert-manager
certs:
internal_issuer:
enabled: true
type: selfsigning
# deploy cacerts
extraDeploy:
- |
apiVersion: "{{ .Values.certs.internal_issuer.apiVersion }}"
kind: Certificate
metadata:
name: "{{ template "pulsar.fullname" . }}-common-cacert"
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
spec:
secretName: "{{ template "pulsar.fullname" . }}-common-cacert"
commonName: "common-cacert"
duration: "{{ .Values.certs.internal_issuer.duration }}"
renewBefore: "{{ .Values.certs.internal_issuer.renewBefore }}"
usages:
- server auth
- client auth
isCA: true
issuerRef:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.certs.internal_issuer.component }}"
kind: Issuer
group: cert-manager.io

View File

@ -1,44 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
auth:
authentication:
enabled: true
jwt:
# Enable JWT authentication
enabled: true
# If the token is generated by a secret key, set the usingSecretKey as true.
# If the token is generated by a private key, set the usingSecretKey as false.
usingSecretKey: false
authorization:
enabled: true
superUsers:
# broker to broker communication
broker: "broker-admin"
# proxy to broker communication
proxy: "proxy-admin"
# pulsar-admin client to broker/proxy communication
client: "admin"
# pulsar-manager to broker communication
manager: "manager-admin"
components:
pulsar_manager: true

View File

@ -1,43 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
auth:
authentication:
enabled: true
jwt:
# Enable JWT authentication
enabled: true
# If the token is generated by a secret key, set the usingSecretKey as true.
# If the token is generated by a private key, set the usingSecretKey as false.
usingSecretKey: true
authorization:
enabled: true
superUsers:
# broker to broker communication
broker: "broker-admin"
# proxy to broker communication
proxy: "proxy-admin"
# pulsar-admin client to broker/proxy communication
client: "admin"
# pulsar manager to broker
manager: "manager-admin"
components:
pulsar_manager: true

View File

@ -1,94 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Mount crendentials to each component
proxy:
configData:
# Authentication settings of the broker itself. Used when the broker connects to other brokers, or when the proxy connects to brokers, either in same or other clusters
brokerClientAuthenticationPlugin: "org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2"
brokerClientAuthenticationParameters: '{"privateKey":"file:///pulsar/auth/proxy/credentials_file.json","audience":"account","issuerUrl":"http://keycloak-ci-headless:8080/realms/pulsar"}'
extraVolumes:
- name: pulsar-proxy-credentials
secret:
secretName: pulsar-proxy-credentials
extraVolumeMounts:
- name: pulsar-proxy-credentials
mountPath: "/pulsar/auth/proxy"
readOnly: true
broker:
configData:
# Authentication settings of the broker itself. Used when the broker connects to other brokers, or when the proxy connects to brokers, either in same or other clusters
brokerClientAuthenticationPlugin: "org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2"
brokerClientAuthenticationParameters: '{"privateKey":"file:///pulsar/auth/broker/credentials_file.json","audience":"account","issuerUrl":"http://keycloak-ci-headless:8080/realms/pulsar"}'
extraVolumes:
- name: pulsar-broker-credentials
secret:
secretName: pulsar-broker-credentials
extraVolumeMounts:
- name: pulsar-broker-credentials
mountPath: "/pulsar/auth/broker"
readOnly: true
toolset:
configData:
authPlugin: "org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2"
authParams: '{"privateKey":"file:///pulsar/auth/admin/credentials_file.json","audience":"account","issuerUrl":"http://keycloak-ci-headless:8080/realms/pulsar"}'
extraVolumes:
- name: pulsar-admin-credentials
secret:
secretName: pulsar-admin-credentials
extraVolumeMounts:
- name: pulsar-admin-credentials
mountPath: "/pulsar/auth/admin"
readOnly: true
auth:
authentication:
enabled: true
openid:
# Enable openid authentication
enabled: true
# https://pulsar.apache.org/docs/next/security-openid-connect/#enable-openid-connect-authentication-in-the-broker-and-proxy
openIDAllowedTokenIssuers:
- http://keycloak-ci-headless:8080/realms/pulsar
openIDAllowedAudiences:
- account
#openIDTokenIssuerTrustCertsFilePath:
openIDRoleClaim: "sub"
openIDAcceptedTimeLeewaySeconds: "0"
openIDCacheSize: "5"
openIDCacheRefreshAfterWriteSeconds: "64800"
openIDCacheExpirationSeconds: "86400"
openIDHttpConnectionTimeoutMillis: "10000"
openIDHttpReadTimeoutMillis: "10000"
openIDKeyIdCacheMissRefreshSeconds: "300"
openIDRequireIssuersUseHttps: "false"
openIDFallbackDiscoveryMode: "DISABLED"
authorization:
enabled: true
superUsers:
# broker to broker communication
broker: "broker-admin"
# proxy to broker communication
proxy: "proxy-admin"
# pulsar-admin client to broker/proxy communication
client: "admin"
# pulsar manager to broker
manager: "manager-admin"

View File

@ -1,35 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
components:
zookeeper: false
oxia: true
# disable functions for oxia tests since there's no support for Oxia in
# BookKeeperPackagesStorage which requires Zookeeper
functions: false
oxia:
initialShardCount: 3
replicationFactor: 3
server:
replicas: 3
cpuLimit: 333m
memoryLimit: 200Mi
dbCacheSizeMb: 100
storageSize: 1Gi

View File

@ -1,21 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
components:
pulsar_manager: true

View File

@ -1,20 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
defaultPulsarImageTag: 3.0.12

View File

@ -1,37 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# enable TLS
tls:
enabled: true
proxy:
enabled: true
broker:
enabled: true
bookie:
enabled: true
zookeeper:
enabled: true
# enable cert-manager
certs:
internal_issuer:
enabled: true
type: selfsigning

View File

@ -1,19 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#

View File

@ -1,60 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
victoria-metrics-k8s-stack:
enabled: true
victoria-metrics-operator:
enabled: true
vmsingle:
enabled: true
vmagent:
enabled: true
grafana:
enabled: true
adminPassword: pulsar-ci-admin
prometheus-node-exporter:
enabled: true
zookeeper:
podMonitor:
enabled: true
bookkeeper:
podMonitor:
enabled: true
broker:
podMonitor:
enabled: true
autorecovery:
podMonitor:
enabled: true
proxy:
podMonitor:
enabled: true
oxia:
coordinator:
podMonitor:
enabled: true
server:
podMonitor:
enabled: true

View File

@ -1,30 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# enable TLS
tls:
enabled: true
zookeeper:
enabled: true
# enable cert-manager
certs:
internal_issuer:
enabled: true
type: selfsigning

View File

@ -1,32 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# enable TLS
tls:
enabled: true
zookeeper:
enabled: true
bookie:
enabled: true
# enable cert-manager
certs:
internal_issuer:
enabled: true
type: selfsigning

View File

@ -1,41 +0,0 @@
#!/bin/bash
# this script is used to install tools for the GitHub Actions CI runner while debugging with ssh
if [[ -z "${GITHUB_ACTIONS}" ]]; then
echo "Error: This script is intended to run only in GitHub Actions environment"
exit 1
fi
cat >> $HOME/.bashrc <<'EOF'
function use_kind_kubeconfig() {
export KUBECONFIG=$(ls $HOME/kind/pulsar-ci-*/kubeconfig.yaml)
}
function kubectl() {
# use kind environment's kubeconfig
if [ -z "$KUBECONFIG" ]; then
use_kind_kubeconfig
fi
command kubectl "$@"
}
function k9s() {
# use kind environment's kubeconfig
if [ -z "$KUBECONFIG" ]; then
use_kind_kubeconfig
fi
# install k9s on the fly
if [ ! -x /usr/local/bin/k9s ]; then
echo "Installing k9s..."
curl -L -s https://github.com/derailed/k9s/releases/download/v0.40.5/k9s_Linux_amd64.tar.gz | sudo tar xz -C /usr/local/bin k9s
fi
command k9s "$@"
}
alias k=kubectl
EOF
cat >> $HOME/.bash_profile <<'EOF'
if [ -f ~/.bashrc ]; then
source ~/.bashrc
fi
EOF

View File

@ -1,587 +0,0 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
BINDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
PULSAR_HOME="$(cd "${BINDIR}/.." && pwd)"
CHARTS_HOME=${PULSAR_HOME}
PULSAR_CHART_LOCAL=${CHARTS_HOME}/charts/pulsar
PULSAR_CHART_VERSION=${PULSAR_CHART_VERSION:-"local"}
OUTPUT_BIN=${CHARTS_HOME}/output/bin
KIND_BIN=$OUTPUT_BIN/kind
HELM=${OUTPUT_BIN}/helm
KUBECTL=${OUTPUT_BIN}/kubectl
NAMESPACE=pulsar
CLUSTER=pulsar-ci
: ${CLUSTER_ID:=$(uuidgen)}
K8S_LOGS_DIR="${K8S_LOGS_DIR:-/tmp/k8s-logs}"
export PATH="$OUTPUT_BIN:$PATH"
# brew package 'coreutils' is required on MacOSX
# coreutils includes the 'timeout' command
if [[ "$OSTYPE" == "darwin"* ]]; then
brew_gnubin_packages=(coreutils)
if ! type -P brew &>/dev/null; then
echo "On MacOSX, you must install required binaries with the following command:"
echo "brew install" "${brew_gnubin_packages[@]}"
exit 1
fi
for dep in "${brew_gnubin_packages[@]}"; do
path_element="$(brew --prefix)/opt/${dep}/libexec/gnubin"
if [ ! -d "${path_element}" ]; then
echo "'${path_element}' is missing. Quick fix: 'brew install ${dep}'."
echo "On MacOSX, you must install required binaries with the following command:"
echo "brew install" "${brew_gnubin_packages[@]}"
exit 1
fi
PATH="${path_element}:$PATH"
done
export PATH
fi
function ci::create_cluster() {
echo "Creating a kind cluster ..."
${CHARTS_HOME}/hack/kind-cluster-build.sh --name pulsar-ci-${CLUSTER_ID} -c 1 -v 10
echo "Successfully created a kind cluster."
}
function ci::delete_cluster() {
echo "Deleting a kind cluster ..."
kind delete cluster --name=pulsar-ci-${CLUSTER_ID}
echo "Successfully delete a kind cluster."
}
function ci::install_cert_manager() {
echo "Installing the cert-manager ..."
${KUBECTL} create namespace cert-manager
${CHARTS_HOME}/scripts/cert-manager/install-cert-manager.sh
WC=$(${KUBECTL} get pods -n cert-manager --field-selector=status.phase=Running | wc -l)
while [[ ${WC} -lt 3 ]]; do
echo ${WC};
sleep 15
${KUBECTL} get pods -n cert-manager
${KUBECTL} get events --sort-by=.lastTimestamp -A | tail -n 30 || true
WC=$(${KUBECTL} get pods -n cert-manager --field-selector=status.phase=Running | wc -l)
done
echo "Successfully installed the cert manager."
}
function ci::helm_repo_add() {
echo "Adding the helm repo ..."
${HELM} repo add prometheus-community https://prometheus-community.github.io/helm-charts
${HELM} repo add vm https://victoriametrics.github.io/helm-charts/
${HELM} repo update
echo "Successfully added the helm repo."
}
function ci::print_pod_logs() {
echo "Logs for all containers:"
for k8sobject in $(${KUBECTL} get pods,jobs -n ${NAMESPACE} -o=name); do
${KUBECTL} logs -n ${NAMESPACE} "$k8sobject" --all-containers=true --ignore-errors=true --prefix=true --tail=100 || true
done;
}
function ci::collect_k8s_logs() {
mkdir -p "${K8S_LOGS_DIR}" && cd "${K8S_LOGS_DIR}"
echo "Collecting k8s logs to ${K8S_LOGS_DIR}"
for k8sobject in $(${KUBECTL} get pods,jobs -n ${NAMESPACE} -o=name); do
filebase="${k8sobject//\//_}"
${KUBECTL} logs -n ${NAMESPACE} "$k8sobject" --all-containers=true --ignore-errors=true --prefix=true > "${filebase}.$$.log.txt" || true
${KUBECTL} logs -n ${NAMESPACE} "$k8sobject" --all-containers=true --ignore-errors=true --prefix=true --previous=true > "${filebase}.previous.$$.log.txt" || true
done;
${KUBECTL} get events --sort-by=.lastTimestamp -A > events.$$.log.txt || true
${KUBECTL} get events --sort-by=.lastTimestamp -A -o yaml > events.$$.log.yaml || true
${KUBECTL} get -n ${NAMESPACE} all -o yaml > k8s_resources.$$.yaml || true
}
function ci::install_pulsar_chart() {
local install_type=$1
local common_value_file=$2
local value_file=$3
shift 3
local extra_values=()
local extra_opts=()
local values_next=false
for arg in "$@"; do
if [[ "$arg" == "--values" || "$arg" == "--set" ]]; then
extra_values+=("$arg")
values_next=true
elif [[ "$values_next" == true ]]; then
extra_values+=("$arg")
values_next=false
else
extra_opts+=("$arg")
fi
done
local install_args
if [[ "${install_type}" == "install" ]]; then
echo "Installing the pulsar chart"
${KUBECTL} create namespace ${NAMESPACE}
ci::install_cert_manager
echo ${CHARTS_HOME}/scripts/pulsar/prepare_helm_release.sh -k ${CLUSTER} -n ${NAMESPACE} "${extra_opts[@]}"
${CHARTS_HOME}/scripts/pulsar/prepare_helm_release.sh -k ${CLUSTER} -n ${NAMESPACE} "${extra_opts[@]}"
sleep 10
# install metallb for loadbalancer support
# following instructions from https://kind.sigs.k8s.io/docs/user/loadbalancer/
${KUBECTL} apply -f https://raw.githubusercontent.com/metallb/metallb/v0.13.12/config/manifests/metallb-native.yaml
# wait until metallb is ready
${KUBECTL} wait --namespace metallb-system \
--for=condition=ready pod \
--selector=app=metallb \
--timeout=120s
# configure metallb
${KUBECTL} apply -f ${BINDIR}/metallb/metallb-config.yaml
install_args=""
# create auth resources
if [[ "x${AUTHENTICATION_PROVIDER}" == "xopenid" ]]; then
ci::create_openid_resources
fi
else
install_args="--wait --wait-for-jobs --timeout 360s --debug"
fi
CHART_ARGS=""
if [[ "${PULSAR_CHART_VERSION}" == "local" ]]; then
set -x
${HELM} dependency update ${PULSAR_CHART_LOCAL}
set +x
CHART_ARGS="${PULSAR_CHART_LOCAL}"
else
set -x
${HELM} repo add apache https://pulsar.apache.org/charts
set +x
CHART_ARGS="apache/pulsar --dependency-update"
if [[ "${PULSAR_CHART_VERSION}" != "latest" ]]; then
CHART_ARGS="${CHART_ARGS} --version ${PULSAR_CHART_VERSION}"
fi
fi
set -x
${HELM} template --values ${common_value_file} --values ${value_file} "${extra_values[@]}" ${CLUSTER} ${CHART_ARGS}
${HELM} ${install_type} --values ${common_value_file} --values ${value_file} "${extra_values[@]}" --namespace=${NAMESPACE} ${CLUSTER} ${CHART_ARGS} ${install_args}
set +x
if [[ "${install_type}" == "install" ]]; then
echo "wait until broker is alive"
WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep ${CLUSTER}-broker | wc -l)
counter=1
while [[ ${WC} -lt 1 ]]; do
((counter++))
echo ${WC};
sleep 15
${KUBECTL} get pods,jobs -n ${NAMESPACE}
${KUBECTL} get events --sort-by=.lastTimestamp -A | tail -n 30 || true
if [[ $((counter % 20)) -eq 0 ]]; then
ci::print_pod_logs
if [[ $counter -gt 100 ]]; then
echo >&2 "Timeout waiting..."
exit 1
fi
fi
WC=$(${KUBECTL} get pods -n ${NAMESPACE} | grep ${CLUSTER}-broker | wc -l)
if [[ ${WC} -gt 1 ]]; then
${KUBECTL} describe pod -n ${NAMESPACE} pulsar-ci-broker-0
${KUBECTL} logs -n ${NAMESPACE} pulsar-ci-broker-0
fi
WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep ${CLUSTER}-broker | wc -l)
done
timeout 300s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-broker; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
timeout 120s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until [ "$(curl -s -L http://pulsar-ci-broker:8080/status.html)" == "OK" ]; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep ${CLUSTER}-proxy | wc -l)
counter=1
while [[ ${WC} -lt 1 ]]; do
((counter++))
echo ${WC};
sleep 15
${KUBECTL} get pods,jobs -n ${NAMESPACE}
${KUBECTL} get events --sort-by=.lastTimestamp -A | tail -n 30 || true
if [[ $((counter % 8)) -eq 0 ]]; then
ci::print_pod_logs
if [[ $counter -gt 16 ]]; then
echo >&2 "Timeout waiting..."
exit 1
fi
fi
WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep ${CLUSTER}-proxy | wc -l)
done
timeout 300s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-proxy; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
echo "Install complete"
else
echo "wait until broker is alive"
timeout 300s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-broker; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
timeout 120s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until [ "$(curl -s -L http://pulsar-ci-broker:8080/status.html)" == "OK" ]; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
echo "wait until proxy is alive"
timeout 300s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-proxy; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
echo "Upgrade complete"
fi
}
helm_values_cached=""
function ci::helm_values_for_deployment() {
if [[ -z "${helm_values_cached}" ]]; then
helm_values_cached=$(helm get values -n ${NAMESPACE} ${CLUSTER} -a -o yaml)
fi
printf "%s" "${helm_values_cached}"
}
function ci::check_pulsar_environment() {
echo "Wait until pulsar-ci-broker is ready"
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-broker; do sleep 3; done'
echo "Wait until pulsar-ci-proxy is ready"
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-proxy; do sleep 3; done'
echo "bookie-0 disk usage"
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-bookie-0 -- df -h
echo "bookie-0 bookkeeper.conf"
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-bookie-0 -- cat conf/bookkeeper.conf
echo "bookie-0 bookies list (rw)"
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/bookkeeper shell listbookies -rw | grep ListBookiesCommand
echo "bookie-0 bookies list (ro)"
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/bookkeeper shell listbookies -ro | grep ListBookiesCommand
}
# function to retry a given commend 3 times with a backoff of 10 seconds in between
function ci::retry() {
local n=1
local max=3
local delay=10
while true; do
"$@" && break || {
if [[ $n -lt $max ]]; then
((n++))
echo "::warning::Command failed. Attempt $n/$max:"
sleep $delay
else
fail "::error::The command has failed after $n attempts."
fi
}
done
}
function ci::test_pulsar_admin_api_access() {
echo "Test pulsar admin api access"
ci::retry ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin tenants list
}
function ci::test_create_test_namespace() {
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin tenants create pulsar-ci
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin namespaces create pulsar-ci/test
}
function ci::test_pulsar_producer_consumer() {
action="${1:-"produce-consume"}"
echo "Testing with ${action}"
if [[ "$(ci::helm_values_for_deployment | yq .tls.proxy.enabled)" == "true" ]]; then
PROXY_URL="pulsar+ssl://pulsar-ci-proxy:6651"
else
PROXY_URL="pulsar://pulsar-ci-proxy:6650"
fi
set -x
if [[ "${action}" == "produce" || "${action}" == "produce-consume" ]]; then
ci::test_create_test_namespace
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin topics create pulsar-ci/test/test-topic
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin topics create-subscription -s test pulsar-ci/test/test-topic
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client produce -m "test-message" pulsar-ci/test/test-topic
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin topics create-subscription -s test2 pulsar-ci/test/test-topic
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client --url "${PROXY_URL}" produce -m "test-message2" pulsar-ci/test/test-topic
fi
if [[ "${action}" == "consume" || "${action}" == "produce-consume" ]]; then
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client consume -s test pulsar-ci/test/test-topic
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client --url "${PROXY_URL}" consume -s test2 pulsar-ci/test/test-topic
fi
set +x
}
function ci::wait_function_running() {
num_running=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'bin/pulsar-admin functions status --tenant pulsar-ci --namespace test --name test-function' | jq .numRunning)
counter=1
while [[ ${num_running} -lt 1 ]]; do
((counter++))
if [[ $counter -gt 6 ]]; then
echo >&2 "Timeout waiting..."
return 1
fi
echo "Waiting 15 seconds for function to be running"
sleep 15
${KUBECTL} get pods -n ${NAMESPACE} -l component=function || true
${KUBECTL} get events --sort-by=.lastTimestamp -A | tail -n 30 || true
podname=$(${KUBECTL} get pods -l component=function -n ${NAMESPACE} --no-headers -o custom-columns=":metadata.name") || true
if [[ -n "$podname" ]]; then
echo "Function pod is $podname"
${KUBECTL} describe pod -n ${NAMESPACE} $podname
echo "Function pod logs"
${KUBECTL} logs -n ${NAMESPACE} $podname
fi
num_running=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'bin/pulsar-admin functions status --tenant pulsar-ci --namespace test --name test-function' | jq .numRunning)
done
}
function ci::wait_message_processed() {
num_processed=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'bin/pulsar-admin functions stats --tenant pulsar-ci --namespace test --name test-function' | jq .processedSuccessfullyTotal)
podname=$(${KUBECTL} get pods -l component=function -n ${NAMESPACE} --no-headers -o custom-columns=":metadata.name")
counter=1
while [[ ${num_processed} -lt 1 ]]; do
((counter++))
if [[ $counter -gt 6 ]]; then
echo >&2 "Timeout waiting..."
return 1
fi
echo "Waiting 15 seconds for message to be processed"
sleep 15
echo "Function pod is $podname"
${KUBECTL} describe pod -n ${NAMESPACE} $podname
echo "Function pod logs"
${KUBECTL} logs -n ${NAMESPACE} $podname
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin functions stats --tenant pulsar-ci --namespace test --name test-function
num_processed=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'bin/pulsar-admin functions stats --tenant pulsar-ci --namespace test --name test-function' | jq .processedSuccessfullyTotal)
done
}
function ci::test_pulsar_function() {
echo "Testing functions"
echo "Creating function"
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin functions create --tenant pulsar-ci --namespace test --name test-function --inputs "pulsar-ci/test/test_input" --output "pulsar-ci/test/test_output" --parallelism 1 --classname org.apache.pulsar.functions.api.examples.ExclamationFunction --jar /pulsar/examples/api-examples.jar
echo "Creating subscription for output topic"
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin topics create-subscription -s test pulsar-ci/test/test_output
echo "Waiting for function to be ready"
# wait until the function is running
ci::wait_function_running
echo "Sending input message"
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client produce -m 'hello pulsar function!' pulsar-ci/test/test_input
echo "Waiting for message to be processed"
ci::wait_message_processed
echo "Consuming output message"
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client consume -s test pulsar-ci/test/test_output
}
function ci::test_pulsar_manager() {
echo "Testing pulsar manager"
until ${KUBECTL} get jobs -n ${NAMESPACE} ${CLUSTER}-pulsar-manager-init -o json | jq -r '.status.conditions[] | select (.type | test("Complete")).status' | grep True; do sleep 3; done
${KUBECTL} describe job -n ${NAMESPACE} ${CLUSTER}-pulsar-manager-init
${KUBECTL} logs -n ${NAMESPACE} job.batch/${CLUSTER}-pulsar-manager-init
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-manager-0 -- cat /pulsar-manager/pulsar-manager.log
echo "Checking Podname"
podname=$(${KUBECTL} get pods -n ${NAMESPACE} -l component=pulsar-manager --no-headers -o custom-columns=":metadata.name")
echo "Getting pulsar manager UI password"
PASSWORD=$(${KUBECTL} get secret -n ${NAMESPACE} -l component=pulsar-manager -o=jsonpath="{.items[0].data.UI_PASSWORD}" | base64 --decode)
echo "Getting CSRF_TOKEN"
CSRF_TOKEN=$(${KUBECTL} exec -n ${NAMESPACE} ${podname} -- curl http://127.0.0.1:7750/pulsar-manager/csrf-token)
echo "Performing login"
${KUBECTL} exec -n ${NAMESPACE} ${podname} -- curl -X POST http://127.0.0.1:9527/pulsar-manager/login \
-H 'Accept: application/json, text/plain, */*' \
-H 'Content-Type: application/json' \
-H "X-XSRF-TOKEN: $CSRF_TOKEN" \
-H "Cookie: XSRF-TOKEN=$CSRF_TOKEN" \
-sS -D headers.txt \
-d '{"username": "pulsar", "password": "'${PASSWORD}'"}'
LOGIN_TOKEN=$(${KUBECTL} exec -n ${NAMESPACE} ${podname} -- grep "token:" headers.txt | sed 's/^.*: //')
LOGIN_JSESSIONID=$(${KUBECTL} exec -n ${NAMESPACE} ${podname} -- grep -o "JSESSIONID=[a-zA-Z0-9_]*" headers.txt | sed 's/^.*=//')
echo "Checking environment"
envs=$(${KUBECTL} exec -n ${NAMESPACE} ${podname} -- curl -X GET http://127.0.0.1:9527/pulsar-manager/environments \
-H 'Content-Type: application/json' \
-H "token: $LOGIN_TOKEN" \
-H "X-XSRF-TOKEN: $CSRF_TOKEN" \
-H "username: pulsar" \
-H "Cookie: XSRF-TOKEN=$CSRF_TOKEN; JSESSIONID=$LOGIN_JSESSIONID;")
echo "$envs"
number_of_envs=$(echo $envs | jq '.total')
if [ "$number_of_envs" -ne 1 ]; then
echo "Error: Did not find expected environment"
exit 1
fi
# Force manager to query broker for tenant info. This will require use of the manager's JWT, if JWT authentication is enabled.
echo "Checking tenants"
pulsar_env=$(echo $envs | jq -r '.data[0].name')
tenants=$(${KUBECTL} exec -n ${NAMESPACE} ${podname} -- curl -X GET http://127.0.0.1:9527/pulsar-manager/admin/v2/tenants \
-H 'Content-Type: application/json' \
-H "token: $LOGIN_TOKEN" \
-H "X-XSRF-TOKEN: $CSRF_TOKEN" \
-H "username: pulsar" \
-H "tenant: pulsar" \
-H "environment: ${pulsar_env}" \
-H "Cookie: XSRF-TOKEN=$CSRF_TOKEN; JSESSIONID=$LOGIN_JSESSIONID;")
echo "$tenants"
number_of_tenants=$(echo $tenants | jq '.total')
if [ "$number_of_tenants" -lt 1 ]; then
echo "Error: Found no tenants!"
exit 1
fi
}
function ci::check_loadbalancers() {
(
set +e
${KUBECTL} get services -n ${NAMESPACE} | grep LoadBalancer
if [ $? -eq 0 ]; then
echo "Error: Found service with type LoadBalancer. This is not allowed because of security reasons."
exit 1
fi
exit 0
)
}
function ci::validate_kustomize_yaml() {
# if kustomize is not installed, install kustomize to a temp directory
if ! command -v kustomize &> /dev/null; then
KUSTOMIZE_VERSION=5.6.0
KUSTOMIZE_DIR=$(mktemp -d)
echo "Installing kustomize ${KUSTOMIZE_VERSION} to ${KUSTOMIZE_DIR}"
curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash -s ${KUSTOMIZE_VERSION} ${KUSTOMIZE_DIR}
export PATH=${KUSTOMIZE_DIR}:$PATH
fi
# prevent regression of https://github.com/apache/pulsar-helm-chart/issues/569
local kustomize_yaml_dir=$(mktemp -d)
cp ${PULSAR_HOME}/.ci/kustomization.yaml ${kustomize_yaml_dir}
PULSAR_HOME=${PULSAR_HOME} yq -i '.helmGlobals.chartHome = env(PULSAR_HOME) + "/charts"' ${kustomize_yaml_dir}/kustomization.yaml
failures=0
# validate zookeeper init
echo "Validating kustomize yaml output with zookeeper init"
_ci::validate_kustomize_yaml ${kustomize_yaml_dir} || ((failures++))
# validate oxia init
yq -i '.helmCharts[0].valuesInline.components += {"zookeeper": false, "oxia": true}' ${kustomize_yaml_dir}/kustomization.yaml
echo "Validating kustomize yaml output with oxia init"
_ci::validate_kustomize_yaml ${kustomize_yaml_dir} || ((failures++))
if [ $failures -gt 0 ]; then
exit 1
fi
}
function _ci::validate_kustomize_yaml() {
local kustomize_yaml_dir=$1
kustomize build --enable-helm --helm-kube-version 1.23.0 --load-restrictor=LoadRestrictionsNone ${kustomize_yaml_dir} | yq 'select(.spec.template.spec.containers[0].args != null) | .spec.template.spec.containers[0].args' | \
awk '{
if (prev_line ~ /\\$/ && $0 ~ /^$/) {
print "Found issue: backslash at end of line followed by empty line. Must use pipe character for multiline strings to support kustomize due to kubernetes-sigs/kustomize#4201.";
print "Line: " prev_line;
has_issue = 1;
}
prev_line = $0;
}
END {
if (!has_issue) {
print "No issues found: no backslash followed by empty line";
exit 0;
}
exit 1;
}'
}
# Create all resources needed for openid authentication
function ci::create_openid_resources() {
echo "Creating openid resources"
cp ${PULSAR_HOME}/.ci/auth/keycloak/0-realm-pulsar-partial-export.json /tmp/realm-pulsar.json
for component in broker proxy admin manager; do
echo "Creating openid resources for ${component}"
local client_id=pulsar-${component}
# Github action hang up when read string from /dev/urandom, so use python to generate a random string
local client_secret=$(python -c "import secrets; import string; length = 32; random_string = ''.join(secrets.choice(string.ascii_letters + string.digits) for _ in range(length)); print(random_string);")
if [[ "${component}" == "admin" ]]; then
local sub_claim_value="admin"
else
local sub_claim_value="${component}-admin"
fi
# Create the client credentials file
jq -n --arg CLIENT_ID $client_id --arg CLIENT_SECRET "$client_secret" -f ${PULSAR_HOME}/.ci/auth/oauth2/credentials_file.json > /tmp/${component}-credentials_file.json
# Create the secret for the client credentials
local secret_name="pulsar-${component}-credentials"
${KUBECTL} create secret generic ${secret_name} --from-file=credentials_file.json=/tmp/${component}-credentials_file.json -n ${NAMESPACE}
# Create the keycloak client file
jq -n --arg CLIENT_ID $client_id --arg CLIENT_SECRET "$client_secret" --arg SUB_CLAIM_VALUE "$sub_claim_value" -f ${PULSAR_HOME}/.ci/auth/keycloak/1-client-template.json > /tmp/${component}-keycloak-client.json
# Merge the keycloak client file with the realm
jq '.clients += [input]' /tmp/realm-pulsar.json /tmp/${component}-keycloak-client.json > /tmp/realm-pulsar.json.tmp
mv /tmp/realm-pulsar.json.tmp /tmp/realm-pulsar.json
done
echo "Create keycloak realm configuration"
${KUBECTL} create secret generic keycloak-ci-realm-config --from-file=realm-pulsar.json=/tmp/realm-pulsar.json -n ${NAMESPACE}
echo "Installing keycloak helm chart"
${HELM} install keycloak-ci oci://registry-1.docker.io/bitnamicharts/keycloak --version 24.6.4 --values ${PULSAR_HOME}/.ci/auth/keycloak/values.yaml -n ${NAMESPACE}
echo "Wait until keycloak is running"
WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep keycloak-ci-0 | wc -l)
counter=1
while [[ ${WC} -lt 1 ]]; do
((counter++))
echo ${WC};
sleep 15
${KUBECTL} get pods,jobs -n ${NAMESPACE}
${KUBECTL} get events --sort-by=.lastTimestamp -A | tail -n 30 || true
if [[ $((counter % 20)) -eq 0 ]]; then
ci::print_pod_logs
if [[ $counter -gt 100 ]]; then
echo >&2 "Timeout waiting..."
exit 1
fi
fi
WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep keycloak-ci-0 | wc -l)
done
echo "Wait until keycloak is ready"
${KUBECTL} wait --for=condition=Ready pod/keycloak-ci-0 -n ${NAMESPACE} --timeout 180s
echo "Check keycloack realm pulsar issuer url"
${KUBECTL} exec -n ${NAMESPACE} keycloak-ci-0 -c keycloak -- bash -c 'curl -sSL http://keycloak-ci-headless:8080/realms/pulsar'
}
# lists all available functions in this tool
function ci::list_functions() {
declare -F | awk '{print $NF}' | sort | grep -E '^ci::' | sed 's/^ci:://'
}
# Only run this section if the script is being executed directly (not sourced)
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
if [ -z "$1" ]; then
echo "usage: $0 [function_name]"
echo "Available functions:"
ci::list_functions
exit 1
fi
ci_function_name="ci::$1"
shift
if [[ "$(LC_ALL=C type -t "${ci_function_name}")" == "function" ]]; then
eval "$ci_function_name" "$@"
exit $?
else
echo "Invalid ci function"
echo "Available functions:"
ci::list_functions
exit 1
fi
fi

View File

@ -1,32 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
helmGlobals:
chartHome: ../charts
helmCharts:
- name: pulsar
releaseName: pulsar
valuesInline:
victoria-metrics-k8s-stack:
enabled: false
components:
pulsar_manager: true
zookeeper: true

View File

@ -1,33 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
name: example
namespace: metallb-system
spec:
addresses:
- 172.19.255.200-172.19.255.250
---
apiVersion: metallb.io/v1beta1
kind: L2Advertisement
metadata:
name: empty
namespace: metallb-system

View File

@ -1,122 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
victoria-metrics-k8s-stack:
enabled: false
victoria-metrics-operator:
enabled: false
vmsingle:
enabled: false
vmagent:
enabled: false
vmalert:
enabled: false
alertmanager:
enabled: false
grafana:
enabled: false
prometheus-node-exporter:
enabled: false
kube-state-metrics:
enabled: false
kubelet:
enabled: false
kubeApiServer:
enabled: false
kubeControllerManager:
enabled: false
coreDns:
enabled: false
kubeEtcd:
enabled: false
kubeScheduler:
enabled: false
# disabled AntiAffinity
affinity:
anti_affinity: false
# disable auto recovery and pulsar manager
components:
autorecovery: false
pulsar_manager: false
# enable functions by default in CI
functions: true
zookeeper:
replicaCount: 1
# Disable pod monitor since we're disabling CRD installation
podMonitor:
enabled: false
bookkeeper:
replicaCount: 3
# Disable pod monitor since we're disabling CRD installation
podMonitor:
enabled: false
configData:
diskUsageThreshold: "0.999"
diskUsageWarnThreshold: "0.999"
PULSAR_PREFIX_diskUsageThreshold: "0.999"
PULSAR_PREFIX_diskUsageWarnThreshold: "0.999"
# minimal memory use for bookkeeper
# https://bookkeeper.apache.org/docs/reference/config#db-ledger-storage-settings
dbStorage_writeCacheMaxSizeMb: "32"
dbStorage_readAheadCacheMaxSizeMb: "32"
dbStorage_rocksDB_writeBufferSizeMB: "8"
dbStorage_rocksDB_blockCacheSize: "8388608"
broker:
replicaCount: 1
# Disable pod monitor since we're disabling CRD installation
podMonitor:
enabled: false
configData:
## Enable `autoSkipNonRecoverableData` since bookkeeper is running
## without persistence
autoSkipNonRecoverableData: "true"
# storage settings
managedLedgerDefaultEnsembleSize: "1"
managedLedgerDefaultWriteQuorum: "1"
managedLedgerDefaultAckQuorum: "1"
PF_functionInstanceMinResources_cpu: "0.2"
PF_functionInstanceMinResources_ram: "268435456"
PF_functionInstanceMinResources_disk: "268435456"
autorecovery:
# Disable pod monitor since we're disabling CRD installation
podMonitor:
enabled: false
proxy:
replicaCount: 1
# Disable pod monitor since we're disabling CRD installation
podMonitor:
enabled: false
toolset:
useProxy: false
oxia:
coordinator:
podMonitor:
enabled: false
server:
podMonitor:
enabled: false

View File

@ -1,14 +0,0 @@
#### Expected behavior
Tell us what should happen
#### Actual behavior
Tell us what happens instead
#### Steps to reproduce
How can we reproduce the issue
#### System configuration
**Pulsar version**: x.y

View File

@ -1,30 +0,0 @@
---
name: Bug report
about: Create a report to help us improve
title: ''
labels: type/bug
assignees: ''
---
**Describe the bug**
A clear and concise description of what the bug is.
**To Reproduce**
Steps to reproduce the behavior:
1. Go to '...'
2. Click on '....'
3. Scroll down to '....'
4. See error
**Expected behavior**
A clear and concise description of what you expected to happen.
**Screenshots**
If applicable, add screenshots to help explain your problem.
**Desktop (please complete the following information):**
- OS: [e.g. iOS]
**Additional context**
Add any other context about the problem here.

View File

@ -1,20 +0,0 @@
---
name: Feature request
about: Suggest an idea for this project
title: ''
labels: type/feature
assignees: ''
---
**Is your feature request related to a problem? Please describe.**
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
**Describe the solution you'd like**
A clear and concise description of what you want to happen.
**Describe alternatives you've considered**
A clear and concise description of any alternative solutions or features you've considered.
**Additional context**
Add any other context or screenshots about the feature request here.

View File

@ -1,13 +0,0 @@
Fixes #<xyz>
### Motivation
*Explain here the context, and why you're making that change. What is the problem you're trying to solve.*
### Modifications
*Describe the modifications you've done.*
### Verifying this change
- [ ] Make sure that the change passes the CI checks.

View File

@ -1,3 +0,0 @@
# chart-testing Action
This action is an identical fork of [helm/chart-testing-action@v3.7.1](https://github.com/helm/chart-testing-action).

View File

@ -1,60 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Copyright The Helm Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: "Helm Chart Testing"
description: "Install the Helm chart-testing tool"
author: "The Helm authors"
branding:
color: blue
icon: anchor
inputs:
version:
description: "The chart-testing version to install"
required: false
default: v3.12.0
yamllint_version:
description: "The yamllint version to install"
required: false
default: '1.35.1'
yamale_version:
description: "The yamale version to install"
required: false
default: '6.0.0'
runs:
using: composite
steps:
- run: |
cd $GITHUB_ACTION_PATH \
&& ./ct.sh \
--version ${{ inputs.version }} \
--yamllint-version ${{ inputs.yamllint_version }} \
--yamale-version ${{ inputs.yamale_version }}
shell: bash

View File

@ -1,168 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Copyright The Helm Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
set -o pipefail
DEFAULT_CHART_TESTING_VERSION=v3.12.0
DEFAULT_YAMLLINT_VERSION=1.35.1
DEFAULT_YAMALE_VERSION=6.0.0
ARCH=$(uname -m)
case $ARCH in
x86) ARCH="386";;
x86_64) ARCH="amd64";;
i686) ARCH="386";;
i386) ARCH="386";;
arm64) ARCH="arm64";;
aarch64) ARCH="arm64";;
esac
OS=$(uname|tr '[:upper:]' '[:lower:]')
show_help() {
cat << EOF
Usage: $(basename "$0") <options>
-h, --help Display help
-v, --version The chart-testing version to use (default: $DEFAULT_CHART_TESTING_VERSION)"
EOF
}
main() {
local version="$DEFAULT_CHART_TESTING_VERSION"
local yamllint_version="$DEFAULT_YAMLLINT_VERSION"
local yamale_version="$DEFAULT_YAMALE_VERSION"
parse_command_line "$@"
install_chart_testing
}
parse_command_line() {
while :; do
case "${1:-}" in
-h|--help)
show_help
exit
;;
-v|--version)
if [[ -n "${2:-}" ]]; then
version="$2"
shift
else
echo "ERROR: '-v|--version' cannot be empty." >&2
show_help
exit 1
fi
;;
--yamllint-version)
if [[ -n "${2:-}" ]]; then
yamllint_version="$2"
shift
else
echo "ERROR: '--yamllint-version' cannot be empty." >&2
show_help
exit 1
fi
;;
--yamale-version)
if [[ -n "${2:-}" ]]; then
yamale_version="$2"
shift
else
echo "ERROR: '--yamale-version' cannot be empty." >&2
show_help
exit 1
fi
;;
*)
break
;;
esac
shift
done
}
install_chart_testing() {
if [[ ! -d "$RUNNER_TOOL_CACHE" ]]; then
echo "Cache directory '$RUNNER_TOOL_CACHE' does not exist" >&2
exit 1
fi
local cache_dir="$RUNNER_TOOL_CACHE/ct/$version/${ARCH}"
local venv_dir="$cache_dir/venv"
if [[ ! -d "$cache_dir" ]]; then
mkdir -p "$cache_dir"
echo "Installing chart-testing..."
curl -sSLo ct.tar.gz "https://github.com/helm/chart-testing/releases/download/$version/chart-testing_${version#v}_${OS}_${ARCH}.tar.gz"
tar -xzf ct.tar.gz -C "$cache_dir"
rm -f ct.tar.gz
# if uv (https://docs.astral.sh/uv/) is not installed, install it
if ! command -v uv &> /dev/null; then
echo 'Installing uv...'
curl -LsSf https://astral.sh/uv/install.sh | sh
fi
echo 'Creating virtual Python environment...'
uv venv "$venv_dir"
echo 'Activating virtual environment...'
# shellcheck disable=SC1090
source "$venv_dir/bin/activate"
echo 'Installing yamllint...'
uv pip install "yamllint==${yamllint_version}"
echo 'Installing Yamale...'
uv pip install "yamale==${yamale_version}"
fi
# https://github.com/helm/chart-testing-action/issues/62
echo 'Adding ct directory to PATH...'
echo "$cache_dir" >> "$GITHUB_PATH"
echo 'Setting CT_CONFIG_DIR...'
echo "CT_CONFIG_DIR=$cache_dir/etc" >> "$GITHUB_ENV"
echo 'Configuring environment variables for virtual environment for subsequent workflow steps...'
echo "VIRTUAL_ENV=$venv_dir" >> "$GITHUB_ENV"
echo "$venv_dir/bin" >> "$GITHUB_PATH"
"$cache_dir/ct" version
}
main "$@"

View File

@ -1,161 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
name: ssh access
description: Sets up SSH access to build VM with upterm
inputs:
action:
description: |
Action to perform: options are "start" and "wait"
"start" will install, configure and start upterm.
"wait" will wait until a connection is established to upterm and will continue to wait until the session is closed.
required: false
default: 'start'
limit-access-to-actor:
description: 'If only the public SSH keys of the user triggering the workflow should be authorized'
required: false
default: 'false'
limit-access-to-users:
description: 'If only the public SSH keys of the listed GitHub users should be authorized. Comma separate list of GitHub user names.'
required: false
default: ''
secure-access:
description: |
Set to false for allowing public access when limit-access-to-actor and limit-access-to-users are unset.
required: false
default: 'true'
timeout:
description: 'When action=wait, the timeout in seconds to wait for the user to connect'
required: false
default: '300'
runs:
using: composite
steps:
- run: |
if [[ "${{ inputs.action }}" == "start" ]]; then
echo "::group::Installing upterm & tmux"
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
# install upterm
curl -sL https://github.com/owenthereal/upterm/releases/download/v0.7.6/upterm_linux_amd64.tar.gz | tar zxvf - -C /tmp upterm && sudo install /tmp/upterm /usr/local/bin/ && rm -rf /tmp/upterm
# install tmux if it's not present
if ! command -v tmux &>/dev/null; then
sudo apt-get -y install tmux
fi
elif [[ "$OSTYPE" == "darwin"* ]]; then
brew install owenthereal/upterm/upterm
# install tmux if it's not present
if ! command -v tmux &>/dev/null; then
brew install tmux
fi
else
echo "Unsupported $OSTYPE"
exit 0
fi
echo '::endgroup::'
echo "::group::Configuring ssh and ssh keys"
# generate ssh key
mkdir -p ~/.ssh
chmod 0700 ~/.ssh
if [ ! -f ~/.ssh/id_rsa ]; then
ssh-keygen -q -t rsa -N "" -f ~/.ssh/id_rsa
fi
if [ ! -f ~/.ssh/id_ed25519 ]; then
ssh-keygen -q -t ed25519 -N "" -f ~/.ssh/id_ed25519
fi
# configure ssh
echo -e "Host *\nStrictHostKeyChecking no\nCheckHostIP no\nTCPKeepAlive yes\nServerAliveInterval 30\nServerAliveCountMax 180\nVerifyHostKeyDNS yes\nUpdateHostKeys yes\n" > ~/.ssh/config
# Auto-generate ~/.ssh/known_hosts by attempting connection to uptermd.upterm.dev
ssh -i ~/.ssh/id_ed25519 uptermd.upterm.dev || true
# @cert-authority entry is a mandatory entry when connecting to upterm. generate the entry based on the known_hosts entry key
cat <(cat ~/.ssh/known_hosts | awk '{ print "@cert-authority * " $2 " " $3 }') >> ~/.ssh/known_hosts
authorizedKeysParameter=""
authorizedKeysFile=${HOME}/.ssh/authorized_keys
if [[ "${{ inputs.secure-access }}" != "false" ]]; then
ssh-keygen -q -t ed25519 -N "$(echo $RANDOM | md5sum | awk '{ print $1 }')" -C "Prevent public access" -f /tmp/dummykey$$
cat /tmp/dummykey$$.pub >> $authorizedKeysFile
rm /tmp/dummykey$$ /tmp/dummykey$$.pub
fi
limit_access_to_actor="${{ inputs.limit-access-to-actor }}"
if [[ "${limit_access_to_actor}" == "true" ]]; then
echo "Adding ${GITHUB_ACTOR} to allowed users (identified by ssh key registered in GitHub)"
curl -s https://github.com/${GITHUB_ACTOR}.keys >> $authorizedKeysFile
fi
limit_access_to_users="${{ inputs.limit-access-to-users }}"
for github_user in ${limit_access_to_users//,/ }; do
if [[ -n "${github_user}" ]]; then
echo "Adding ${github_user} to allowed users (identified by ssh key registered in GitHub)"
curl -s https://github.com/${github_user}.keys >> $authorizedKeysFile
fi
done
if [ -f $authorizedKeysFile ]; then
chmod 0600 $authorizedKeysFile
authorizedKeysParameter="-a $authorizedKeysFile"
echo -e "Using $authorizedKeysFile\nContent:\n---------------------------"
cat $authorizedKeysFile
echo "---------------------------"
fi
echo '::endgroup::'
echo "::group::Starting terminal session and connecting to server"
tmux new -d -s upterm-wrapper -x 132 -y 43 "upterm host ${authorizedKeysParameter} --force-command 'tmux attach -t upterm' -- tmux new -s upterm -x 132 -y 43"
sleep 2
tmux send-keys -t upterm-wrapper q C-m
sleep 1
tmux set -t upterm-wrapper window-size largest
tmux set -t upterm window-size largest
echo '::endgroup::'
echo -e "\nSSH connection information"
# wait up to 10 seconds for upterm admin socket to appear
for i in {1..10}; do
ADMIN_SOCKET=$(find $HOME/.upterm -name "*.sock")
if [ ! -S "$ADMIN_SOCKET" ]; then
echo "Waiting for upterm admin socket to appear in ~/.upterm/*.sock ..."
sleep 1
else
echo "upterm admin socket available in $ADMIN_SOCKET"
break
fi
done
shopt -s nullglob
upterm session current --admin-socket ~/.upterm/*.sock || {
echo "Starting upterm failed."
exit 0
}
elif [[ "${{ inputs.action }}" == "wait" ]]; then
# only wait if upterm was installed
if command -v upterm &>/dev/null; then
shopt -s nullglob
echo "SSH connection information"
upterm session current --admin-socket ~/.upterm/*.sock || {
echo "upterm isn't running. Not waiting any longer."
exit 0
}
timeout=${{ inputs.timeout }}
echo "Waiting $timeout seconds..."
sleep $timeout
echo "Keep waiting as long as there's a connected session"
while upterm session current --admin-socket ~/.upterm/*.sock|grep Connected &>/dev/null; do
sleep 30
done
echo "No session is connected. Not waiting any longer."
else
echo "upterm isn't installed"
fi
fi
shell: bash

View File

@ -1,96 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
name: Tune Runner VM performance
description: tunes the GitHub Runner VM operation system
runs:
using: composite
steps:
- run: |
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
echo "::group::Configure and tune OS"
# Ensure that reverse lookups for current hostname are handled properly
# Add the current IP address, long hostname and short hostname record to /etc/hosts file
echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
# The default vm.swappiness setting is 60 which has a tendency to start swapping when memory
# consumption is high.
# Set vm.swappiness=1 to avoid swapping and allow high RAM usage
echo 1 | sudo tee /proc/sys/vm/swappiness
(
shopt -s nullglob
# Set swappiness to 1 for all cgroups and sub-groups
for swappiness_file in /sys/fs/cgroup/memory/*/memory.swappiness /sys/fs/cgroup/memory/*/*/memory.swappiness; do
echo 1 | sudo tee $swappiness_file > /dev/null
done
) || true
# use "madvise" Linux Transparent HugePages (THP) setting
# https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html
# "madvise" is generally a better option than the default "always" setting
# Based on Azul instructions from https://docs.azul.com/prime/Enable-Huge-Pages#transparent-huge-pages-thp
echo madvise | sudo tee /sys/kernel/mm/transparent_hugepage/enabled
echo advise | sudo tee /sys/kernel/mm/transparent_hugepage/shmem_enabled
echo defer+madvise | sudo tee /sys/kernel/mm/transparent_hugepage/defrag
echo 1 | sudo tee /sys/kernel/mm/transparent_hugepage/khugepaged/defrag
# tune filesystem mount options, https://www.kernel.org/doc/Documentation/filesystems/ext4.txt
# commit=999999, effectively disables automatic syncing to disk (default is every 5 seconds)
# nobarrier/barrier=0, loosen data consistency on system crash (no negative impact to empheral CI nodes)
sudo mount -o remount,nodiscard,commit=999999,barrier=0 / || true
sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt || true
# disable discard/trim at device level since remount with nodiscard doesn't seem to be effective
# https://www.spinics.net/lists/linux-ide/msg52562.html
for i in /sys/block/sd*/queue/discard_max_bytes; do
echo 0 | sudo tee $i
done
# disable any background jobs that run SSD discard/trim
sudo systemctl disable fstrim.timer || true
sudo systemctl stop fstrim.timer || true
sudo systemctl disable fstrim.service || true
sudo systemctl stop fstrim.service || true
# stop php-fpm
sudo systemctl stop php8.0-fpm.service || true
sudo systemctl stop php7.4-fpm.service || true
# stop mono-xsp4
sudo systemctl disable mono-xsp4.service || true
sudo systemctl stop mono-xsp4.service || true
sudo killall mono || true
# stop Azure Linux agent to save RAM
sudo systemctl stop walinuxagent.service || true
echo '::endgroup::'
# show memory
echo "::group::Available Memory"
free -m
echo '::endgroup::'
# show disk
echo "::group::Available diskspace"
df -BM
echo "::endgroup::"
# show cggroup
echo "::group::Cgroup settings for current cgroup $CURRENT_CGGROUP"
CURRENT_CGGROUP=$(cat /proc/self/cgroup | grep '0::' | awk -F: '{ print $3 }')
sudo cgget -a $CURRENT_CGGROUP || true
echo '::endgroup::'
fi
shell: bash

View File

@ -1,28 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# contains pattern definitions used in workflows "changes" step
# pattern syntax: https://github.com/micromatch/picomatch
all:
- '**'
docs:
- 'examples/**'
- '.asf.yaml'
- '*.md'
- '**/*.md'

View File

@ -1,357 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
name: Pulsar Helm Chart CI
on:
push:
branches:
- master
pull_request:
branches:
- master
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
preconditions:
name: Preconditions
runs-on: ubuntu-24.04
if: (github.event_name != 'schedule') || (github.repository == 'apache/pulsar-helm-chart')
outputs:
docs_only: ${{ steps.check_changes.outputs.docs_only }}
steps:
- name: checkout
uses: actions/checkout@v4
- name: Detect changed files
id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
filters: .github/changes-filter.yaml
list-files: csv
- name: Check changed files
id: check_changes
run: |
if [[ "${GITHUB_EVENT_NAME}" != "schedule" && "${GITHUB_EVENT_NAME}" != "workflow_dispatch" ]]; then
echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT
else
echo docs_only=false >> $GITHUB_OUTPUT
fi
license-check:
needs: preconditions
name: License Check
runs-on: ubuntu-24.04
timeout-minutes: 10
if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
steps:
- name: Set up Go 1.12
uses: actions/setup-go@v5
with:
go-version: 1.12
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v4
- name: Check license
run: |
go test license_test.go
# run "ct lint" https://github.com/helm/chart-testing/blob/main/doc/ct_lint.md
ct-lint:
needs: ['preconditions', 'license-check']
name: chart-testing lint
runs-on: ubuntu-24.04
timeout-minutes: 45
if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
steps:
- name: checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
if: ${{ github.repository != 'apache/pulsar-helm-chart' && github.event_name == 'pull_request' }}
uses: ./.github/actions/ssh-access
continue-on-error: true
with:
limit-access-to-actor: true
- name: Set up Helm
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
uses: azure/setup-helm@v4
with:
version: v3.16.4
- name: Set up Python
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install uv, a fast modern package manager for Python
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
run: curl -LsSf https://astral.sh/uv/install.sh | sh
- name: Set up chart-testing
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
uses: ./.github/actions/chart-testing-action
- name: Run chart-testing (lint)
id: ct-lint
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
run: |
ct lint --check-version-increment=false \
--validate-maintainers=false \
--target-branch ${{ github.event.repository.default_branch }}
- name: Run kubeconform check for helm template with every major k8s version 1.25.0-1.32.0
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
run: |
PULSAR_CHART_HOME=$(pwd)
source ${PULSAR_CHART_HOME}/hack/common.sh
source ${PULSAR_CHART_HOME}/.ci/helm.sh
hack::ensure_kubectl
hack::ensure_helm
hack::ensure_kubeconform
ci::helm_repo_add
helm dependency build charts/pulsar
validate_helm_template_with_k8s_version() {
local kube_version=$1
shift
echo -n "Validating helm template with kubeconform for k8s version $kube_version"
if [ $# -gt 0 ]; then
echo " Extra args: $*"
else
echo ""
fi
helm template charts/pulsar --set victoria-metrics-k8s-stack.enabled=false --set components.pulsar_manager=true --kube-version $kube_version "$@" | \
kubeconform -schema-location default -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' -strict -kubernetes-version $kube_version -summary
}
set -o pipefail
for k8s_version_part in {25..32}; do
k8s_version="1.${k8s_version_part}.0"
echo "Validating default values with k8s version $k8s_version"
validate_helm_template_with_k8s_version $k8s_version
for config in .ci/clusters/*.yaml; do
echo "Validating $config with k8s version $k8s_version"
validate_helm_template_with_k8s_version $k8s_version --values .ci/values-common.yaml --values $config
done
done
- name: Validate kustomize yaml for extra new lines in pulsar-init commands
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
run: |
./.ci/helm.sh validate_kustomize_yaml
- name: Wait for ssh connection when build fails
# ssh access is enabled for builds in own forks
uses: ./.github/actions/ssh-access
if: ${{ failure() && github.repository != 'apache/pulsar-helm-chart' && github.event_name == 'pull_request' }}
continue-on-error: true
with:
action: wait
install-chart-tests:
name: ${{ matrix.testScenario.name }} - k8s ${{ matrix.k8sVersion.version }} - ${{ matrix.testScenario.type || 'install' }}
runs-on: ubuntu-24.04
timeout-minutes: ${{ matrix.testScenario.timeout || 45 }}
needs: ['preconditions', 'ct-lint']
if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
strategy:
fail-fast: false
matrix:
# see https://github.com/kubernetes-sigs/kind/releases/tag/v0.27.0 for the list of supported k8s versions for kind 0.27.0
# docker images are available at https://hub.docker.com/r/kindest/node/tags
k8sVersion:
- version: "1.25.16"
kind_image_tag: v1.25.16@sha256:6110314339b3b44d10da7d27881849a87e092124afab5956f2e10ecdb463b025
- version: "1.32.2"
kind_image_tag: v1.32.2@sha256:f226345927d7e348497136874b6d207e0b32cc52154ad8323129352923a3142f
testScenario:
- name: Upgrade latest released version
values_file: .ci/clusters/values-upgrade.yaml
shortname: upgrade
type: upgrade
- name: Use previous LTS Pulsar Image
values_file: .ci/clusters/values-pulsar-previous-lts.yaml
shortname: pulsar-previous-lts
- name: JWT Asymmetric Keys
values_file: .ci/clusters/values-jwt-asymmetric.yaml
shortname: jwt-asymmetric
- name: JWT Symmetric Key
values_file: .ci/clusters/values-jwt-symmetric.yaml
shortname: jwt-symmetric
- name: TLS
values_file: .ci/clusters/values-tls.yaml
shortname: tls
- name: Broker & Proxy TLS
values_file: .ci/clusters/values-broker-tls.yaml
shortname: broker-tls
- name: BK TLS Only
values_file: .ci/clusters/values-bk-tls.yaml
shortname: bk-tls
- name: ZK TLS Only
values_file: .ci/clusters/values-zk-tls.yaml
shortname: zk-tls
- name: ZK & BK TLS Only
values_file: .ci/clusters/values-zkbk-tls.yaml
shortname: zkbk-tls
- name: Pulsar Manager
values_file: .ci/clusters/values-pulsar-manager.yaml
shortname: pulsar-manager
- name: Oxia
values_file: .ci/clusters/values-oxia.yaml
shortname: oxia
- name: OpenID
values_file: .ci/clusters/values-openid.yaml
shortname: openid
- name: CA certificates
values_file: .ci/clusters/values-cacerts.yaml
shortname: cacerts
include:
- k8sVersion:
version: "1.25.16"
kind_image_tag: v1.25.16@sha256:6110314339b3b44d10da7d27881849a87e092124afab5956f2e10ecdb463b025
testScenario:
name: "Upgrade TLS"
values_file: .ci/clusters/values-tls.yaml
shortname: tls
type: upgrade
- k8sVersion:
version: "1.25.16"
kind_image_tag: v1.25.16@sha256:6110314339b3b44d10da7d27881849a87e092124afab5956f2e10ecdb463b025
testScenario:
name: "Upgrade victoria-metrics-k8s-stack for previous LTS"
values_file: .ci/clusters/values-victoria-metrics-grafana.yaml --values .ci/clusters/values-pulsar-previous-lts.yaml
shortname: victoria-metrics-grafana
type: upgrade
upgradeFromVersion: 3.2.0
- k8sVersion:
version: "1.25.16"
kind_image_tag: v1.25.16@sha256:6110314339b3b44d10da7d27881849a87e092124afab5956f2e10ecdb463b025
testScenario:
name: "TLS with helm 3.12.0"
values_file: .ci/clusters/values-tls.yaml
shortname: tls
type: install
helmVersion: 3.12.0
env:
k8sVersion: ${{ matrix.k8sVersion.kind_image_tag }}
KUBECTL_VERSION: ${{ matrix.k8sVersion.version }}
HELM_VERSION: ${{ matrix.helmVersion || '3.14.4' }}
steps:
- name: checkout
uses: actions/checkout@v4
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Setup debugging tools for ssh access
if: ${{ github.repository != 'apache/pulsar-helm-chart' && github.event_name == 'pull_request' }}
run: .ci/configure_ci_runner_for_debugging.sh
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
if: ${{ github.repository != 'apache/pulsar-helm-chart' && github.event_name == 'pull_request' }}
uses: ./.github/actions/ssh-access
continue-on-error: true
with:
limit-access-to-actor: true
- name: Run chart-testing (${{ matrix.testScenario.type || 'install' }}) with helm ${{ env.HELM_VERSION }}
run: |
case "${{ matrix.testScenario.shortname }}" in
"jwt-symmetric")
export SYMMETRIC=true
export EXTRA_SUPERUSERS=manager-admin
;;
"jwt-asymmetric")
export EXTRA_SUPERUSERS=manager-admin
;;
"openid")
export AUTHENTICATION_PROVIDER=openid
;;
esac
if [[ "${{ matrix.testScenario.type || 'install' }}" == "upgrade" ]]; then
export UPGRADE_FROM_VERSION="${{ matrix.testScenario.upgradeFromVersion || 'latest' }}"
fi
.ci/chart_test.sh ${{ matrix.testScenario.values_file }}
- name: Collect k8s logs on failure
if: ${{ cancelled() || failure() }}
continue-on-error: true
shell: bash
run: |
source .ci/helm.sh
set +e
ci::collect_k8s_logs
- name: Upload k8s logs on failure
uses: actions/upload-artifact@v4
if: ${{ cancelled() || failure() }}
continue-on-error: true
with:
name: k8s-logs-${{ matrix.testScenario.shortname }}
path: /tmp/k8s-logs
retention-days: 7
if-no-files-found: ignore
- name: Wait for ssh connection when build fails
# ssh access is enabled for builds in own forks
uses: ./.github/actions/ssh-access
if: ${{ failure() && github.repository != 'apache/pulsar-helm-chart' && github.event_name == 'pull_request' }}
continue-on-error: true
with:
action: wait
# This job is required for pulls to be merged.
# It depends on all other jobs in this workflow.
pulsar-helm-chart-ci-checks-completed:
name: "CI checks completed"
if: ${{ always() && ((github.event_name != 'schedule') || (github.repository == 'apache/pulsar-helm-chart')) }}
runs-on: ubuntu-24.04
timeout-minutes: 10
needs: [
'preconditions',
'license-check',
'install-chart-tests'
]
steps:
- name: Check that all required jobs were completed successfully
if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
run: |
if [[ ! ( \
"${{ needs.license-check.result }}" == "success" \
&& "${{ needs.install-chart-tests.result }}" == "success" \
) ]]; then
echo "Required jobs haven't been completed successfully."
exit 1
fi

19
.gitignore vendored
View File

@ -1,19 +0,0 @@
# Intellij
.idea
# Mac
**/.DS_Store
**/output
**/~
# charts
.chart-packages/**
.chart-index/**
.vs/
charts/**/*.tgz
charts/**/*.lock
PRIVATEKEY
PUBLICKEY
.vagrant/

View File

@ -1,20 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
.gitignore
# Generated Helm file
Chart.lock

239
LICENSE
View File

@ -1,239 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
----------------------------------------------------------------------------------------------------
pulsar-common/src/main/java/org/apache/pulsar/common/util/protobuf/ByteBufCoded{Input,Output}Stream.java
Copyright 2014, Google Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Code generated by the Protocol Buffer compiler is owned by the owner
of the input file used when generating it. This code is not
standalone and requires a support library to be linked with it. This
support library is itself covered by the above license.

5
NOTICE
View File

@ -1,5 +0,0 @@
Apache Pulsar
Copyright 2017-2022 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).

576
README.md
View File

@ -1,576 +0,0 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Apache Pulsar Helm Chart
This project provides Helm Charts for installing Apache Pulsar on Kubernetes.
Read [Deploying Pulsar on Kubernetes](http://pulsar.apache.org/docs/deploy-kubernetes/) for more details.
> :warning: This helm chart is updated outside of the regular Pulsar release cycle and might lag behind a bit. It only supports basic Kubernetes features now. Currently, it can be used as no more than a template and starting point for a Kubernetes deployment. In many cases, it would require some customizations.
## Important Security Advisory for Helm Chart Usage
### Notice of Default Configuration
This Helm chart's default configuration DOES NOT meet production security requirements.
Users MUST review and customize security settings for their specific environment.
IMPORTANT: This Helm chart provides a starting point for Pulsar deployments but requires
significant security customization before use in production environments. We strongly
recommend implementing:
1. Authentication and authorization for all components
2. TLS encryption for all communication channels
3. Proper network isolation and access controls
4. Regular security updates and vulnerability assessments
As an open source project, we welcome contributions to improve security features.
Please consider submitting pull requests to address security gaps or enhance
existing security implementations.
### Pulsar Proxy Security Considerations
As per the [Pulsar Proxy documentation](https://pulsar.apache.org/docs/3.1.x/administration-proxy/), it is explicitly stated that the Pulsar proxy is not designed for exposure to the public internet. The design assumes that deployments will be protected by network perimeter security measures. It is crucial to understand that relying solely on the default configuration can expose your deployment to significant security vulnerabilities.
### Upgrading
#### To 4.1.0
This version introduces `OpenID` authentication. Setting `auth.authentication.provider` is no longer supported, you need to enable the provider with `auth.authentication.<provider>.enabled`.
#### To 4.0.0
The default service type for the Pulsar proxy has changed from `LoadBalancer` to `ClusterIP` for security reasons. This limits access to within the Kubernetes environment by default.
### External Access Recommendations
If you need to expose the Pulsar Proxy outside the cluster:
1. **USE INTERNAL LOAD BALANCERS ONLY**
- Set type to LoadBalancer only in secured environments with proper network controls
- Add cloud provider-specific annotations for internal load balancers:
- Kubernetes documentation about internal load balancers:
- [Internal load balancer](https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer)
- See cloud provider documentation:
- AWS / EKS: [AWS Load Balancer Controller / Service Annotations](https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/guide/service/annotations/)
- Azure / AKS: [Use an internal load balancer with Azure Kubernetes Service (AKS)](https://learn.microsoft.com/en-us/azure/aks/internal-lb)
- GCP / GKE: [LoadBalancer service parameters](https://cloud.google.com/kubernetes-engine/docs/concepts/service-load-balancer-parameters)
- Examples (verify correctness for your environment):
- AWS / EKS: `service.beta.kubernetes.io/aws-load-balancer-internal: "true"`
- Azure / AKS: `service.beta.kubernetes.io/azure-load-balancer-internal: "true"`
- GCP / GKE: `networking.gke.io/load-balancer-type: "Internal"`
2. **IMPLEMENT AUTHENTICATION AND AUTHORIZATION**
- Configure all clients to authenticate properly
- Set up appropriate authorization policies
3. **USE TLS FOR ALL CONNECTIONS**
- Enable TLS for client-to-proxy connections
- Enable TLS for proxy-to-broker connections
- Enable TLS for all internal cluster communications
- Note: TLS alone is NOT sufficient as a security solution. Even with TLS enabled, clusters exposed to untrusted networks remain vulnerable to denial-of-service attacks, authentication bypass attempts, and protocol-level exploits.
4. **NETWORK SECURITY**
- Use private networks (VPCs)
- Configure firewalls, security groups, and IP restrictions
5. **CLIENT IP ADDRESS BASED ACCESS RESTRICTIONS**
- When using a LoadBalancer service type, restrict access to specific IP ranges by configuring `proxy.service.loadBalancerSourceRanges` in your values.yaml:
```yaml
proxy:
service:
loadBalancerSourceRanges:
- 10.0.0.0/8 # Private network range
- 172.16.0.0/12 # Private network range
- 192.168.0.0/16 # Private network range
```
- This feature:
- Provides an additional defense layer by filtering traffic at the load balancer level
- Only allows connections from specified CIDR blocks
- Works only with LoadBalancer service type and when your cloud provider supports the `loadBalancerSourceRanges` parameter
- Important: This should be implemented alongside other security measures (internal load balancer, authentication, TLS, network policies) as part of a defense-in-depth strategy,
not as a standalone security solution
### Alternative for External Access
As an alternative method for external access, Pulsar has support for [SNI proxy routing](https://pulsar.apache.org/docs/next/concepts-proxy-sni-routing/). SNI Proxy routing is supported with proxy servers such as Apache Traffic Server, HAProxy and Nginx.
Note: This option isn't currently implemented in the Apache Pulsar Helm chart.
**IMPORTANT**: Pulsar binary protocol cannot be exposed outside of the Kubernetes cluster using Kubernetes Ingress. Kubernetes Ingress works for the Admin REST API and topic lookups, but clients would be connecting to the advertised listener addresses returned by the brokers and it would only work when clients can connect directly to brokers. This is not a supported secure option for exposing Pulsar to untrusted networks.
### General Recommendations
- **Network Perimeter Security:** It is imperative to implement robust network perimeter security to safeguard your deployment. The absence of such security measures can lead to unauthorized access and potential data breaches.
- **Restricted Access:** For environments where security is less critical, such as certain development or testing scenarios, the use of `loadBalancerSourceRanges` may be employed to restrict access to specified IP addresses or ranges. This, however, should not be considered a substitute for comprehensive security measures in production environments.
### User Responsibility
The user assumes full responsibility for the security and integrity of their deployment. This includes, but is not limited to, the proper configuration of security features and adherence to best practices for securing network access. The providers of this Helm chart disclaim all warranties, whether express or implied, including any warranties of merchantability, fitness for a particular purpose, and non-infringement of third-party rights.
### No Security Guarantees
The providers of this Helm chart make no guarantees regarding the security of the chart under any circumstances. It is the user's responsibility to ensure that their deployment is secure and complies with all relevant security standards and regulations.
By using this Helm chart, the user acknowledges the risks associated with its default configuration and the necessity for proper security customization. The user further agrees that the providers of the Helm chart shall not be liable for any security breaches or incidents resulting from the use of the chart.
## Features
This Helm Chart includes all the components of Apache Pulsar for a complete experience.
- [x] Pulsar core components:
- [x] ZooKeeper
- [x] Bookies
- [x] Brokers
- [x] Functions
- [x] Proxies
- [x] Management & monitoring components:
- [x] Pulsar Manager
- [x] Optional PodMonitors for each component (enabled by default)
- [x] [victoria-metrics-k8s-stack](hhttps://github.com/VictoriaMetrics/helm-charts/tree/master/charts/victoria-metrics-k8s-stack) (as of 4.0.0)
It includes support for:
- [x] Security
- [x] Automatically provisioned TLS certs, using [Jetstack](https://www.jetstack.io/)'s [cert-manager](https://cert-manager.io/docs/)
- [x] self-signed
- [x] [Let's Encrypt](https://letsencrypt.org/)
- [x] TLS Encryption
- [x] Proxy
- [x] Broker
- [x] Toolset
- [x] Bookie
- [x] ZooKeeper (requires the `AdditionalCertificateOutputFormats=true` feature gate to be enabled in the cert-manager deployment when using cert-manager versions below 1.15.0)
- [x] Authentication
- [x] JWT
- [x] OpenID
- [ ] Mutal TLS
- [ ] Kerberos
- [x] Authorization
- [x] Non-root broker, bookkeeper, proxy, and zookeeper containers (version 2.10.0 and above)
- [x] Storage
- [x] Non-persistence storage
- [x] Persistence Volume
- [x] Local Persistent Volumes
- [x] Tiered Storage
- [x] Functions
- [x] Kubernetes Runtime
- [x] Process Runtime
- [x] Thread Runtime
- [x] Operations
- [x] Independent Image Versions for all components, enabling controlled upgrades
## Requirements
In order to use this chart to deploy Apache Pulsar on Kubernetes, the followings are required.
1. kubectl 1.25 or higher, compatible with your cluster ([+/- 1 minor release from your cluster](https://kubernetes.io/docs/tasks/tools/install-kubectl/#before-you-begin))
2. Helm v3 (3.12.0 or higher)
3. A Kubernetes cluster, version 1.25 or higher.
## Environment setup
Before proceeding to deploying Pulsar, you need to prepare your environment.
### Tools
`helm` and `kubectl` need to be [installed on your computer](https://pulsar.apache.org/docs/helm-tools/).
## Add to local Helm repository
To add this chart to your local Helm repository:
```bash
helm repo add apachepulsar https://pulsar.apache.org/charts
helm repo update
```
## Kubernetes cluster preparation
You need a Kubernetes cluster whose version is 1.25 or higher in order to use this chart, due to the usage of certain Kubernetes features.
We provide some instructions to guide you through the preparation: http://pulsar.apache.org/docs/helm-prepare/
## Deploy Pulsar to Kubernetes
1. Configure your values file. The best way to know which values are available is to read the [values.yaml](./charts/pulsar/values.yaml).
A best practice is to start with an empty values file and only set the keys that differ from the default configuration.
Anti-affinity rules for Zookeeper and Bookie components require at least one node per replica. For Kubernetes clusters with less than 3 nodes,
you must disable this feature by adding this to your initial values.yaml file:
```yaml
affinity:
anti_affinity: false
```
2. Install the chart:
```bash
helm install -n <namespace> --create-namespace <release-name> -f your-values.yaml apachepulsar/pulsar
```
3. Observe the deployment progress
Watching events to view progress of deployment:
```shell
kubectl get -n <namespace> events -o wide --watch
```
Watching state of deployed Kubernetes objects, updated every 2 seconds:
```shell
watch kubectl get -n <namespace> all
```
Waiting until Pulsar Proxy is available:
```shell
kubectl wait --timeout=600s --for=condition=ready pod -n <namespace> -l component=proxy
```
Watching state with k9s (https://k9scli.io/topics/install/):
```shell
k9s -n <namespace>
```
4. Access the Pulsar cluster
The default values will create a `ClusterIP` for the proxy you can use to interact with the cluster. To find the IP address of proxy use:
```bash
kubectl get service -n <k8s-namespace>
```
For more information, please follow our detailed
[quick start guide](https://pulsar.apache.org/docs/getting-started-helm/).
## Customize the deployment
We provide a [detailed guideline](https://pulsar.apache.org/docs/helm-deploy/) for you to customize
the Helm Chart for a production-ready deployment.
You can also checkout out the example values file for different deployments.
- [Deploy ZooKeeper only](examples/values-cs.yaml)
- [Deploy a Pulsar cluster with an external configuration store](examples/values-cs.yaml)
- [Deploy a Pulsar cluster with local persistent volume](examples/values-local-pv.yaml)
- [Deploy a Pulsar cluster to Minikube](examples/values-minikube.yaml)
- [Deploy a Pulsar cluster with no persistence](examples/values-no-persistence.yaml)
- [Deploy a Pulsar cluster with TLS encryption](examples/values-tls.yaml)
- [Deploy a Pulsar cluster with JWT authentication using symmetric key](examples/values-jwt-symmetric.yaml)
- [Deploy a Pulsar cluster with JWT authentication using asymmetric key](examples/values-jwt-asymmetric.yaml)
## Disabling victoria-metrics-k8s-stack components
In order to disable the victoria-metrics-k8s-stack, you can add the following to your `values.yaml`.
Victoria Metrics components can also be disabled and enabled individually if you only need specific monitoring features.
```yaml
# disable VictoriaMetrics and related components
victoria-metrics-k8s-stack:
enabled: false
victoria-metrics-operator:
enabled: false
vmsingle:
enabled: false
vmagent:
enabled: false
kube-state-metrics:
enabled: false
prometheus-node-exporter:
enabled: false
grafana:
enabled: false
Additionally, you'll need to set each component's `podMonitor` property to `false`.
```yaml
# disable pod monitors
autorecovery:
podMonitor:
enabled: false
bookkeeper:
podMonitor:
enabled: false
oxia:
server:
podMonitor:
enabled: false
coordinator:
podMonitor:
enabled: false
broker:
podMonitor:
enabled: false
proxy:
podMonitor:
enabled: false
zookeeper:
podMonitor:
enabled: false
```
This is shown in some [examples/values-disable-monitoring.yaml](examples/values-disable-monitoring.yaml).
## Pulsar Manager
The Pulsar Manager can be deployed alongside the pulsar cluster instance.
Depending on the given settings it uses an existing Secret within the given namespace or creates a new one, with random
passwords for both, the UI and the internal database.
To forward the UI use (assumes you did not change the namespace):
```
kubectl port-forward $(kubectl get pods -l component=pulsar-manager -o jsonpath='{.items[0].metadata.name}') 9527:9527
```
And then opening the browser to http://localhost:9527
The default user is `pulsar` and you can find out the password with this command
```
kubectl get secret -l component=pulsar-manager -o=jsonpath="{.items[0].data.UI_PASSWORD}" | base64 --decode
```
## Grafana Dashboards
The Apache Pulsar Helm Chart uses the `victoria-metrics-k8s-stack` Helm Chart to deploy Grafana.
There are several ways to configure Grafana dashboards. The default [`values.yaml`](charts/pulsar/values.yaml) comes with examples of Pulsar dashboards which get downloaded from the Apache-2.0 licensed [lhotari/pulsar-grafana-dashboards OSS project](https://github.com/lhotari/pulsar-grafana-dashboards) by URL.
Dashboards can be configured in [`values.yaml`](charts/pulsar/values.yaml) or by adding `ConfigMap` items with the label `grafana_dashboard: "1"`.
In [`values.yaml`](charts/pulsar/values.yaml), it's possible to include dashboards by URL or by grafana.com dashboard id (`gnetId` and `revision`).
Please see the [Grafana Helm chart documentation for importing dashboards](https://github.com/grafana/helm-charts/blob/main/charts/grafana/README.md#import-dashboards).
You can connect to Grafana by forwarding port 3000
```
kubectl port-forward $(kubectl get pods -l app.kubernetes.io/name=grafana -o jsonpath='{.items[0].metadata.name}') 3000:3000
```
And then opening the browser to http://localhost:3000 . The default user is `admin`.
You can find out the password with this command
```
kubectl get secret -l app.kubernetes.io/name=grafana -o=jsonpath="{.items[0].data.admin-password}" | base64 --decode
```
### Pulsar Grafana Dashboards
* The `apache/pulsar` GitHub repo contains some Grafana dashboards [here](https://github.com/apache/pulsar/tree/master/grafana).
* StreamNative provides Grafana Dashboards for Apache Pulsar in this [GitHub repository](https://github.com/streamnative/apache-pulsar-grafana-dashboard).
* DataStax provides Grafana Dashboards for Apache Pulsar in this [GitHub repository](https://github.com/datastax/pulsar-helm-chart/tree/master/helm-chart-sources/pulsar/grafana-dashboards).
Note: if you have third party dashboards that you would like included in this list, please open a pull request.
## Upgrading
Once your Pulsar Chart is installed, configuration changes and chart
updates should be done using `helm upgrade`.
```bash
helm repo add apachepulsar https://pulsar.apache.org/charts
helm repo update
# If you are using the provided victoria-metrics-k8s-stack for monitoring, this installs or upgrades the required CRDs
./scripts/victoria-metrics-k8s-stack/upgrade_vm_operator_crds.sh
# get the existing values.yaml used for the most recent deployment
helm get values -n <namespace> <pulsar-release-name> > values.yaml
# upgrade the deployment
helm upgrade -n <namespace> -f values.yaml <pulsar-release-name> apachepulsar/pulsar
```
For more detailed information, see our [Upgrading](http://pulsar.apache.org/docs/helm-upgrade/) guide.
## Upgrading to Helm chart version 4.2.0 (not released yet)
### TLS configuration for ZooKeeper has changed
The TLS configuration for ZooKeeper has been changed to fix certificate and private key expiration issues.
This change impacts configurations that have `tls.enabled` and `tls.zookeeper.enabled` set in `values.yaml`.
The revised solution requires the `AdditionalCertificateOutputFormats=true` feature gate to be enabled in the `cert-manager` deployment when using cert-manager versions below 1.15.0.
If you installed `cert-manager` using `./scripts/cert-manager/install-cert-manager.sh`, you can re-run the updated script to set the feature gate. The script currently installs or upgrades cert-manager LTS version 1.12.17, where the feature gate must be explicitly enabled.
## Upgrading from Helm Chart versions before 4.0.0 to 4.0.0 version and above
### Pulsar Proxy service's default type has been changed from `LoadBalancer` to `ClusterIP`
Please check the section "External Access Recommendations" for guidance and also check the security advisory section.
You will need to configure keys under `proxy.service` in your `values.yaml` to preserve existing functionality since the default has been changed.
### kube-prometheus-stack replaced with victoria-metrics-k8s-stack
The `kube-prometheus-stack` was replaced with `victoria-metrics-k8s-stack` in Pulsar Helm chart version 4.0.0. The trigger for the change was incompatibilities discovered in testing with most recent `kube-prometheus-stack` and Prometheus 3.2.1 which failed to scrape Pulsar metrics in certain cases without providing proper error messages or debug information at debug level logging.
[Victoria Metrics](https://docs.victoriametrics.com/) is Apache 2.0 Licensed OSS and it's a fully compatible drop-in replacement for Prometheus which is fast and efficient.
Before upgrading to Pulsar Helm Chart version 4.0.0, it is recommended to disable kube-prometheus-stack in the original Helm chart version that
is used:
```shell
# get the existing values.yaml used for the most recent deployment
helm get values -n <namespace> <pulsar-release-name> > values.yaml
# disable kube-prometheus-stack in the currently used version before upgrading to Pulsar Helm chart 4.0.0
helm upgrade -n <namespace> -f values.yaml --version <your-current-chart-version> --set kube-prometheus-stack.enabled=false <pulsar-release-name> apachepulsar/pulsar
```
After, this you can proceed with `helm upgrade`.
## Upgrading to Apache Pulsar 2.10.0 and above (or Helm Chart version 3.0.0 and above)
The 2.10.0+ Apache Pulsar docker image is a non-root container, by default. That complicates an upgrade to 2.10.0
because the existing files are owned by the root user but are not writable by the root group. In order to leverage this
new security feature, the Bookkeeper and Zookeeper StatefulSet [securityContexts](https://kubernetes.io/docs/tasks/configure-pod-container/security-context)
are configurable in the [`values.yaml`](charts/pulsar/values.yaml). They default to:
```yaml
securityContext:
fsGroup: 0
fsGroupChangePolicy: "OnRootMismatch"
```
This configuration is ideal for regular Kubernetes clusters where the UID is stable across restarts. If the process
UID is subject to change (like it is in OpenShift), you'll need to set `fsGroupChangePolicy: "Always"`.
The official docker image assumes that it is run as a member of the root group.
If you upgrade to the latest version of the helm chart before upgrading to Pulsar 2.10.0, then when you perform your
first upgrade to version >= 2.10.0, you will need to set `fsGroupChangePolicy: "Always"` on the first upgrade and then
set it back to `fsGroupChangePolicy: "OnRootMismatch"` on subsequent upgrades. This is because the root file won't
mismatch permissions, but the RocksDB lock file will. If you have direct access to the persistent volumes, you can
alternatively run `chgrp -R g+w /pulsar/data` before upgrading.
Here is a sample error you can expect if the RocksDB lock file is not correctly owned by the root group:
```text
2022-05-14T03:45:06,903+0000 ERROR org.apache.bookkeeper.server.Main - Failed to build bookie server
java.io.IOException: Error open RocksDB database
at org.apache.bookkeeper.bookie.storage.ldb.KeyValueStorageRocksDB.<init>(KeyValueStorageRocksDB.java:199) ~[org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4]
at org.apache.bookkeeper.bookie.storage.ldb.KeyValueStorageRocksDB.<init>(KeyValueStorageRocksDB.java:88) ~[org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4]
at org.apache.bookkeeper.bookie.storage.ldb.KeyValueStorageRocksDB.lambda$static$0(KeyValueStorageRocksDB.java:62) ~[org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4]
at org.apache.bookkeeper.bookie.storage.ldb.LedgerMetadataIndex.<init>(LedgerMetadataIndex.java:68) ~[org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4]
at org.apache.bookkeeper.bookie.storage.ldb.SingleDirectoryDbLedgerStorage.<init>(SingleDirectoryDbLedgerStorage.java:169) ~[org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4]
at org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage.newSingleDirectoryDbLedgerStorage(DbLedgerStorage.java:150) ~[org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4]
at org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage.initialize(DbLedgerStorage.java:129) ~[org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4]
at org.apache.bookkeeper.bookie.Bookie.<init>(Bookie.java:818) ~[org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4]
at org.apache.bookkeeper.proto.BookieServer.newBookie(BookieServer.java:152) ~[org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4]
at org.apache.bookkeeper.proto.BookieServer.<init>(BookieServer.java:120) ~[org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4]
at org.apache.bookkeeper.server.service.BookieService.<init>(BookieService.java:52) ~[org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4]
at org.apache.bookkeeper.server.Main.buildBookieServer(Main.java:304) ~[org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4]
at org.apache.bookkeeper.server.Main.doMain(Main.java:226) [org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4]
at org.apache.bookkeeper.server.Main.main(Main.java:208) [org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4]
Caused by: org.rocksdb.RocksDBException: while open a file for lock: /pulsar/data/bookkeeper/ledgers/current/ledgers/LOCK: Permission denied
at org.rocksdb.RocksDB.open(Native Method) ~[org.rocksdb-rocksdbjni-6.10.2.jar:?]
at org.rocksdb.RocksDB.open(RocksDB.java:239) ~[org.rocksdb-rocksdbjni-6.10.2.jar:?]
at org.apache.bookkeeper.bookie.storage.ldb.KeyValueStorageRocksDB.<init>(KeyValueStorageRocksDB.java:196) ~[org.apache.bookkeeper-bookkeeper-server-4.14.4.jar:4.14.4]
... 13 more
```
### Recovering from `helm upgrade` error "unable to build kubernetes objects from current release manifest"
Example of the error message:
```bash
Error: UPGRADE FAILED: unable to build kubernetes objects from current release manifest:
[resource mapping not found for name: "pulsar-bookie" namespace: "pulsar" from "":
no matches for kind "PodDisruptionBudget" in version "policy/v1beta1" ensure CRDs are installed first,
resource mapping not found for name: "pulsar-broker" namespace: "pulsar" from "":
no matches for kind "PodDisruptionBudget" in version "policy/v1beta1" ensure CRDs are installed first,
resource mapping not found for name: "pulsar-zookeeper" namespace: "pulsar" from "":
no matches for kind "PodDisruptionBudget" in version "policy/v1beta1" ensure CRDs are installed first]
```
Helm documentation [explains issues with managing releases deployed using outdated APIs](https://helm.sh/docs/topics/kubernetes_apis/#helm-users) when the Kubernetes cluster has been upgraded
to a version where these APIs are removed. This happens regardless of whether the chart in the upgrade includes supported API versions.
In this case, you can use the following workaround:
1. Install the [Helm mapkubeapis plugin](https://github.com/helm/helm-mapkubeapis):
```bash
helm plugin install https://github.com/helm/helm-mapkubeapis
```
2. Run the `helm mapkubeapis` command with the appropriate namespace and release name. In this example, we use the namespace "pulsar" and release name "pulsar":
```bash
helm mapkubeapis --namespace pulsar pulsar
```
This workaround addresses the issue by updating in-place Helm release metadata that contains deprecated or removed Kubernetes APIs to a new instance with supported Kubernetes APIs and should allow for a successful Helm upgrade.
## Uninstall
To uninstall the Pulsar Chart, run the following command:
```bash
helm uninstall <pulsar-release-name>
```
For the purposes of continuity, these charts have some Kubernetes objects that are not removed when performing `helm uninstall`.
These items we require you to *conciously* remove them, as they affect re-deployment should you choose to.
* PVCs for stateful data, which you must *consciously* remove
- ZooKeeper: This is your metadata.
- BookKeeper: This is your data.
- Prometheus: This is your metrics data, which can be safely removed.
* Secrets, if generated by our [prepare release script](https://github.com/apache/pulsar-helm-chart/blob/master/scripts/pulsar/prepare_helm_release.sh). They contain secret keys, tokens, etc. You can use [cleanup release script](https://github.com/apache/pulsar-helm-chart/blob/master/scripts/pulsar/cleanup_helm_release.sh) to remove these secrets and tokens as needed.
## Troubleshooting
We've done our best to make these charts as seamless as possible,
occasionally troubles do surface outside of our control. We've collected
tips and tricks for troubleshooting common issues. Please examine these first before raising an [issue](https://github.com/apache/pulsar-helm-chart/issues/new/choose), and feel free to add to them by raising a [Pull Request](https://github.com/apache/pulsar-helm-chart/compare)!
### VictoriaMetrics Troubleshooting
In example commands, k8s is namespace `pulsar` replace with your deployment namespace.
#### VictoriaMetrics Web UI
Connecting to `vmsingle` pod for web UI.
```shell
kubectl port-forward -n pulsar $(kubectl get pods -n pulsar -l app.kubernetes.io/name=vmsingle -o jsonpath='{.items[0].metadata.name}') 8429:8429
```
Now you can access the UI at http://localhost:8429 and http://localhost:8429/vmui (for similar UI as in Prometheus)
#### VictoriaMetrics Scraping debugging UI - Active Targets
Connection to `vmagent` pod for debugging targets.
```shell
kubectl port-forward -n pulsar $(kubectl get pods -n pulsar -l app.kubernetes.io/name=vmagent -o jsonpath='{.items[0].metadata.name}') 8429:8429
```
Now you can access the UI at http://localhost:8429
Active Targets UI
- http://localhost:8429/targets
Scraping Configuration
- http://localhost:8429/config
## Release Process
See [RELEASE.md](RELEASE.md)

View File

@ -1,652 +0,0 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
This document details the steps for releasing the Apache Pulsar Helm Chart.
# Prepare the Apache Pulsar Helm Chart Release Candidate
## Prerequisites
- Helm version >= 3.12.0
- Helm gpg plugin (one option: https://github.com/technosophos/helm-gpg)
## Build Release Notes
Before creating the RC, you need to build and commit the release notes for the release.
## Build RC artifacts
The Release Candidate artifacts we vote upon should be the exact ones we vote against,
without any modification than renaming i.e. the contents of the files must be
the same between voted release candidate and final release.
Because of this the version in the built artifacts that will become the
official Apache releases must not include the rcN suffix.
- Set environment variables
```shell
# Set Version
export VERSION_RC=3.0.0-candidate-1
export VERSION_WITHOUT_RC=${VERSION_RC%-candidate-*}
# set your ASF user id
export APACHE_USER=<your ASF userid>
```
- Clone clean repository and set PULSAR_REPO_ROOT
```shell
git clone https://github.com/apache/pulsar-helm-chart.git
cd pulsar-helm-chart
export PULSAR_REPO_ROOT=$(pwd)
```
- Alternatively (not recommended), go to your already checked out pulsar-helm-chart directory and ensure that it's clean
```shell
git checkout master
git fetch origin
git reset --hard origin/master
# clean the checkout
git clean -fdX .
export PULSAR_REPO_ROOT=$(pwd)
```
- Update Helm Chart version in `Chart.yaml`, example: `version: 1.0.0` (without
the RC tag). Verify that the `appVersion` matches the `values.yaml` versions for Pulsar components.
```shell
yq -i '.version=strenv(VERSION_WITHOUT_RC)' charts/pulsar/Chart.yaml
```
- Add and commit the version change.
```shell
git add charts/pulsar/Chart.yaml
git commit -m "Chart: Bump version to $VERSION_WITHOUT_RC"
git push origin master
```
Note: You will tag this commit, you do not need to open a PR for it.
- Tag your release
```shell
git tag -u $APACHE_USER@apache.org -s pulsar-${VERSION_RC} -m "Apache Pulsar Helm Chart $VERSION_RC"
```
- Tarball the repo
NOTE: Make sure your checkout is clean at this stage - any untracked or changed files will otherwise be included
in the file produced.
```shell
git archive --format=tar.gz pulsar-${VERSION_RC} --prefix=pulsar-chart-${VERSION_WITHOUT_RC}/ \
-o pulsar-chart-${VERSION_WITHOUT_RC}-source.tar.gz .
```
- Generate chart binary
```shell
helm package charts/pulsar --dependency-update
```
- Sign the chart binary
In the following command, replace the email address with your email address or your KEY ID
so GPG uses the right key to sign the chart.
(If you have not generated a key yet, generate it by following instructions on
http://www.apache.org/dev/openpgp.html#key-gen-generate-key)
```shell
helm gpg sign -u $APACHE_USER@apache.org pulsar-${VERSION_WITHOUT_RC}.tgz
```
Warning: you need the `helm gpg` plugin to sign the chart. It can be found at: https://github.com/technosophos/helm-gpg
This should also generate a provenance file (Example: `pulsar-1.0.0.tgz.prov`) as described in
https://helm.sh/docs/topics/provenance/, which can be used to verify integrity of the Helm chart.
Verify the signed chart:
```shell
helm gpg verify pulsar-${VERSION_WITHOUT_RC}.tgz
```
Example output:
```
gpg: Signature made Thu Oct 20 16:36:24 2022 CDT
gpg: using RSA key BD4291E509D771B79E7BD1F5C5724B3F5588C4EB
gpg: issuer "mmarshall@apache.org"
gpg: Good signature from "Michael Marshall <mmarshall@apache.org>" [ultimate]
plugin: Chart SHA verified. sha256:deb035dcb765b1989ed726eabe3d7d89529df05658c8eec6cdd4dc213fa0513e
```
- Generate SHA512/ASC
```shell
${PULSAR_REPO_ROOT}/scripts/sign.sh pulsar-chart-${VERSION_WITHOUT_RC}-source.tar.gz
${PULSAR_REPO_ROOT}/scripts/sign.sh pulsar-${VERSION_WITHOUT_RC}.tgz
```
- Move the artifacts to ASF dev dist repo, generate convenience `index.yaml` & publish them
```shell
# Create new folder for the release
svn mkdir --username $APACHE_USER -m "Add directory for pulsar-helm-chart $VERSION_RC release" https://dist.apache.org/repos/dist/dev/pulsar/helm-chart/$VERSION_RC
# checkout the directory
svn co --username $APACHE_USER https://dist.apache.org/repos/dist/dev/pulsar/helm-chart/$VERSION_RC helm-chart-$VERSION_RC
# Move the artifacts to svn folder
mv ${PULSAR_REPO_ROOT}/pulsar-${VERSION_WITHOUT_RC}.tgz* helm-chart-${VERSION_RC}/
mv ${PULSAR_REPO_ROOT}/pulsar-chart-${VERSION_WITHOUT_RC}-source.tar.gz* helm-chart-${VERSION_RC}/
cd helm-chart-${VERSION_RC}/
###### Generate index.yaml file - Start
# Download the latest index.yaml on Pulsar Website
curl https://pulsar.apache.org/charts/index.yaml --output index.yaml
# Replace the URLs from "https://downloads.apache.org" to "https://archive.apache.org"
# as the downloads.apache.org only contains latest releases.
sed -i 's|https://downloads.apache.org/pulsar/helm-chart/|https://archive.apache.org/dist/pulsar/helm-chart/|' index.yaml
# Generate / Merge the new version with existing index.yaml
helm repo index --merge ./index.yaml . --url "https://dist.apache.org/repos/dist/dev/pulsar/helm-chart/${VERSION_RC}"
###### Generate index.yaml file - End
# Commit the artifacts
svn add *
svn commit -m "Add artifacts for Helm Chart ${VERSION_RC}"
```
- Remove old Helm Chart versions from the dev repo
First check if this is required by viewing the versions available at https://dist.apache.org/repos/dist/dev/pulsar/helm-chart
```shell
export PREVIOUS_VERSION_RC=3.0.0-candidate-1
svn rm --username $APACHE_USER -m "Remove old Helm Chart release: ${PREVIOUS_VERSION_RC}" https://dist.apache.org/repos/dist/dev/pulsar/helm-chart/${PREVIOUS_VERSION_RC}
```
- Push Tag for the release candidate
```shell
cd ${PULSAR_REPO_ROOT}
git push origin tag pulsar-${VERSION_RC}
```
## Create release notes for the release candidate in GitHub UI
```shell
# open this URL and create release notes by clicking "Create release from tag"
echo https://github.com/apache/pulsar-helm-chart/releases/tag/pulsar-${VERSION_RC}
```
1. Open the above URL in a browser and create release notes by clicking "Create release from tag".
2. Find "Previous tag: auto" in the UI above the text box and choose the previous release there.
3. Click "Generate release notes".
4. Review the generated release notes.
5. Select "Set as a pre-release"
6. Click "Publish release".
## Prepare Vote email on the Apache Pulsar release candidate
- Send out a vote to the dev@pulsar.apache.org mailing list:
> [!TIP]
> The template output will get copied to the clipboard using pbpaste. On Linux, you can install xsel and add `alias pbcopy='xsel --clipboard --input'` to the shell.
Subject:
```shell
tee >(pbcopy) <<EOF
[VOTE] Release Apache Pulsar Helm Chart ${VERSION_WITHOUT_RC} based on ${VERSION_RC}
EOF
```
Body:
```shell
tee >(pbcopy) <<EOF
Hello Apache Pulsar Community,
This is a call for the vote to release the Apache Pulsar Helm Chart version ${VERSION_WITHOUT_RC}.
Release notes for $VERSION_RC:
https://github.com/apache/pulsar-helm-chart/releases/tag/pulsar-$VERSION_RC
The release candidate is available at:
https://dist.apache.org/repos/dist/dev/pulsar/helm-chart/$VERSION_RC/
pulsar-chart-${VERSION_WITHOUT_RC}-source.tar.gz - is the "main source release".
pulsar-${VERSION_WITHOUT_RC}.tgz - is the binary Helm Chart release.
Public keys are available at: https://www.apache.org/dist/pulsar/KEYS
For convenience "index.yaml" has been uploaded (though excluded from voting), so you can also run the below commands.
helm repo add --force-update apache-pulsar-dist-dev \\
https://dist.apache.org/repos/dist/dev/pulsar/helm-chart/$VERSION_RC/
helm repo update
helm install pulsar apache-pulsar-dist-dev/pulsar \\
--version ${VERSION_WITHOUT_RC} --set affinity.anti_affinity=false \\
--wait --timeout 10m --debug
For observing the deployment progress, you can use the k9s tool to view the cluster state changes in a different terminal window.
The k9s tool is available at https://k9scli.io/topics/install/.
pulsar-${VERSION_WITHOUT_RC}.tgz.prov - is also uploaded for verifying Chart Integrity, though it is not strictly required for releasing the artifact based on ASF Guidelines.
You can optionally verify this file using this helm plugin https://github.com/technosophos/helm-gpg, or by using helm --verify (https://helm.sh/docs/helm/helm_verify/).
helm fetch --prov apache-pulsar-dist-dev/pulsar
helm plugin install https://github.com/technosophos/helm-gpg
helm gpg verify pulsar-${VERSION_WITHOUT_RC}.tgz
The vote will be open for at least 72 hours.
Only votes from PMC members are binding, but members of the community are
encouraged to test the release and vote with "(non-binding)".
For license checks, the .rat-excludes files is included, so you can run the following to verify licenses (just update $PATH_TO_RAT):
tar -xvf pulsar-chart-${VERSION_WITHOUT_RC}-source.tar.gz
cd pulsar-chart-${VERSION_WITHOUT_RC}
java -jar $PATH_TO_RAT/apache-rat-0.15/apache-rat-0.15.jar . -E .rat-excludes
Please note that the version number excludes the \`-candidate-X\` string, so it's now
simply ${VERSION_WITHOUT_RC}. This will allow us to rename the artifact without modifying
the artifact checksums when we actually release it.
Thanks,
<your name>
EOF
```
Note, you need to update the `helm gpg verify` output and verify the end of the voting period in the body.
## Note about `helm gpg` vs `helm --verify`
Helm ships with a gpg verification tool, but it appears not to work with the currently used format for our KEYS file.
# Verify the release candidate by the PMC
The PMC should verify the releases in order to make sure the release is following the
[Apache Legal Release Policy](http://www.apache.org/legal/release-policy.html).
At least 3 (+1) votes from PMC members should be recorded in accordance to
[Votes on Package Releases](https://www.apache.org/foundation/voting.html#ReleaseVotes)
The legal checks include:
* checking if the packages are present in the right dist folder on svn
* verifying if all the sources have correct licences
* verifying if release manager signed the releases with the right key
* verifying if all the checksums are valid for the release
## SVN check
The files should be present in the sub-folder of
[Pulsar dist](https://dist.apache.org/repos/dist/dev/pulsar/helm-chart)
The following files should be present (7 files):
* `pulsar-chart-${VERSION_WITHOUT_RC}-source.tar.gz` + .asc + .sha512
* `pulsar-${VERSION_WITHOUT_RC}.tgz` + .asc + .sha512
* `pulsar-${VERSION_WITHOUT_RC}.tgz.prov`
As a PMC member you should be able to clone the SVN repository:
```shell
svn co https://dist.apache.org/repos/dist/dev/pulsar/helm-chart
```
Or update it if you already checked it out:
```shell
svn update .
```
## Licence check
This can be done with the Apache RAT tool.
* Download the latest jar from https://creadur.apache.org/rat/download_rat.cgi (unpack the binary,
the jar is inside)
* Unpack the release source archive (the `<package + version>-source.tar.gz` file) to a folder
* Enter the sources folder run the check
```shell
java -jar $PATH_TO_RAT/apache-rat-0.15/apache-rat-0.15.jar pulsar-chart-${VERSION_WITHOUT_RC} -E .rat-excludes
```
where `.rat-excludes` is the file in the root of git repo.
## Signature check
Make sure you have imported into your GPG the PGP key of the person signing the release. You can find the valid keys in
[KEYS](https://dist.apache.org/repos/dist/release/pulsar/KEYS).
You can import the whole KEYS file:
```shell script
gpg --import KEYS
```
You can also import the keys individually from a keyserver. The below one uses a key and
retrieves it from the default GPG keyserver
[OpenPGP.org](https://keys.openpgp.org):
```shell script
gpg --keyserver keys.openpgp.org --receive-keys <some_key>
```
You should choose to import the key when asked.
Note that by being default, the OpenPGP server tends to be overloaded often and might respond with
errors or timeouts. Many of the release managers also uploaded their keys to the
[GNUPG.net](https://keys.gnupg.net) keyserver, and you can retrieve it from there.
```shell script
gpg --keyserver keys.gnupg.net --receive-keys <some_key>
```
Once you have the keys, the signatures can be verified by running this:
```shell script
for i in *.asc
do
echo -e "Checking $i\n"; gpg --verify $i
done
```
This should produce results similar to the below. The "Good signature from ..." is indication
that the signatures are correct. Do not worry about the "not certified with a trusted signature"
warning. Most of the certificates used by release managers are self-signed, and that's why you get this
warning. By importing the key either from the server in the previous step or from the
[KEYS](https://dist.apache.org/repos/dist/release/pulsar/KEYS) page, you know that
this is a valid key already.
## SHA512 sum check
Run this:
```shell
for i in *.sha512
do
echo "Checking $i"; shasum -a 512 `basename $i .sha512 ` | diff - $i
done
```
You should get output similar to:
```
Checking pulsar-1.0.0.tgz.sha512
Checking pulsar-chart-1.0.0-source.tar.gz.sha512
```
# Verify release candidates by Contributors
Contributors can run below commands to test the Helm Chart
```shell
export VERSION_RC=3.0.0-candidate-1
export VERSION_WITHOUT_RC=${VERSION_RC%-candidate-*}
```
```shell
helm repo add --force-update \
apache-pulsar-dist-dev https://dist.apache.org/repos/dist/dev/pulsar/helm-chart/$VERSION_RC/
helm repo update
helm install pulsar apache-pulsar-dist-dev/pulsar \
--version ${VERSION_WITHOUT_RC} --set affinity.anti_affinity=false
```
You can then perform any other verifications to check that it works as you expected by
upgrading the Chart or installing by overriding default of `values.yaml`.
# Publish the final release
## Summarize the voting for the release
Once the vote has been passed, you will need to send a result vote to [dev@pulsar.apache.org](mailto:dev@pulsar.apache.org):
Subject:
```shell
tee >(pbcopy) <<EOF
[RESULT][VOTE] Release Apache Pulsar Helm Chart ${VERSION_WITHOUT_RC} based on ${VERSION_RC}
EOF
```
Message:
```shell
tee >(pbcopy) <<EOF
Hello all,
The vote to release Apache Pulsar Helm Chart version ${VERSION_WITHOUT_RC} based on ${VERSION_RC} is now closed.
The vote PASSED with X binding "+1", Y non-binding "+1" and 0 "-1" votes:
"+1" Binding votes:
- <name>
"+1" Non-Binding votes:
- <name>
I'll continue with the release process and the release announcement will follow shortly.
Thanks,
<your name>
EOF
```
## Publish release to SVN
Set environment variables
```shell
export VERSION_RC=3.0.0-candidate-1
export VERSION_WITHOUT_RC=${VERSION_RC%-candidate-*}
export APACHE_USER=<your ASF userid>
```
Migrating the approved RC artifacts to the release directory:
https://dist.apache.org/repos/dist/release/pulsar/helm-chart/
svn commands for handling this:
```shell
svn rm --username $APACHE_USER -m "Remove temporary index.yaml file" https://dist.apache.org/repos/dist/dev/pulsar/helm-chart/${VERSION_RC}/index.yaml
svn move --username $APACHE_USER -m "Release Pulsar Helm Chart ${VERSION_WITHOUT_RC} from ${VERSION_RC}" \
https://dist.apache.org/repos/dist/dev/pulsar/helm-chart/${VERSION_RC} \
https://dist.apache.org/repos/dist/release/pulsar/helm-chart/${VERSION_WITHOUT_RC}
```
Verify that the packages appear in [Pulsar Helm Chart](https://dist.apache.org/repos/dist/release/pulsar/helm-chart/).
## Publish release tag
Create and push the release tag:
```shell
git tag -u $APACHE_USER@apache.org pulsar-$VERSION_WITHOUT_RC $(git rev-parse pulsar-$VERSION_RC^{}) -m "Apache Pulsar Helm Chart ${VERSION_WITHOUT_RC}"
git push origin pulsar-${VERSION_WITHOUT_RC}
```
## Update index.yaml
The `index.yaml` file is the way helm users discover the binaries for the helm distribution. We currently host the
file at `pulsar.apache.org/charts/index.yaml`.
Then, run the following command from within `github.com/apache/pulsar-site` in the git repo.
```shell
# checkout pulsar-site
git clone https://github.com/apache/pulsar-site
cd pulsar-site
```
```shell
# Run on a branch based on main branch
cd static/charts
# need the chart file temporarily to update the index
wget https://dist.apache.org/repos/dist/release/pulsar/helm-chart/${VERSION_WITHOUT_RC}/pulsar-${VERSION_WITHOUT_RC}.tgz
# store the license header temporarily
head -n 17 index.yaml > license_header.txt
# update the index
helm repo index --merge ./index.yaml . --url "https://downloads.apache.org/pulsar/helm-chart/${VERSION_WITHOUT_RC}"
# restore the license header
mv index.yaml index.yaml.new
cat license_header.txt index.yaml.new > index.yaml
rm license_header.txt index.yaml.new
# remove the temp file
rm pulsar-${VERSION_WITHOUT_RC}.tgz
```
Verify that the updated `index.yaml` file has the most recent version.
Wait until the file is available:
```shell
while ! curl -fIL https://downloads.apache.org/pulsar/helm-chart/${VERSION_WITHOUT_RC}/pulsar-${VERSION_WITHOUT_RC}.tgz; do
echo "Waiting for pulsar-${VERSION_WITHOUT_RC}.tgz to become available..."
sleep 10
done
```
Then run:
```shell
git add index.yaml
git commit -m "Adding Pulsar Helm Chart ${VERSION_WITHOUT_RC} to index.yaml"
```
Then commit the change.
```
git push origin main
```
## Create release notes for the tag in GitHub UI
```shell
# open this URL and create release notes by clicking "Create release from tag"
echo https://github.com/apache/pulsar-helm-chart/releases/tag/pulsar-${VERSION_WITHOUT_RC}
```
1. Open the above URL in a browser and create release notes by clicking "Create release from tag".
2. Find "Previous tag: auto" in the UI above the text box and choose the previous release there.
3. Click "Generate release notes".
4. Review the generated release notes.
5. Click "Publish release".
## Notify developers of release
Once the `index.yaml` is live on the website, it is time to announce the release.
- Notify users@pulsar.apache.org (cc'ing dev@pulsar.apache.org) that
the artifacts have been published:
Subject:
```shell
tee >(pbcopy) <<EOF
[ANNOUNCE] Apache Pulsar Helm Chart version ${VERSION_WITHOUT_RC} Released
EOF
```
Body:
```shell
tee >(pbcopy) <<EOF
Dear community,
The Apache Pulsar team is pleased to announce the release of the Apache
Pulsar Helm Chart $VERSION_WITHOUT_RC.
The official source release, as well as the binary Helm Chart release,
are available at
https://downloads.apache.org/pulsar/helm-chart/$VERSION_WITHOUT_RC/.
The helm chart index at https://pulsar.apache.org/charts/ has been
updated and the release is also available directly via helm.
Release Notes:
https://github.com/apache/pulsar-helm-chart/releases/tag/pulsar-$VERSION_WITHOUT_RC
Docs: https://github.com/apache/pulsar-helm-chart#readme and https://pulsar.apache.org/docs/helm-overview
ArtifactHub: https://artifacthub.io/packages/helm/apache/pulsar/$VERSION_WITHOUT_RC
Thanks to all the contributors who made this possible.
Regards,
The Apache Pulsar Team
EOF
```
Send the same email to announce@apache.org.
It is more reliable to send it via the web ui at https://lists.apache.org/list.html?announce@apache.org
(press "c" to compose a new thread).
## Create release on GitHub
Create a new release on GitHub with the release notes and assets from the release svn.
## Close the milestone
Close the milestone on GitHub. Create the next one if it hasn't been already.
## Announce the release on the community slack
Post this in the #announce channel:
```shell
tee >(pbcopy) <<EOF
We've just released Apache Pulsar Helm Chart ${VERSION_WITHOUT_RC} 🎉
The official source release, as well as the binary Helm Chart release,
are available at
https://downloads.apache.org/pulsar/helm-chart/$VERSION_WITHOUT_RC/.
The helm chart index at https://pulsar.apache.org/charts/ has been
updated and the release is also available directly via helm.
Release Notes:
https://github.com/apache/pulsar-helm-chart/releases/tag/pulsar-$VERSION_WITHOUT_RC
Docs: https://github.com/apache/pulsar-helm-chart#readme and https://pulsar.apache.org/docs/helm-overview
ArtifactHub: https://artifacthub.io/packages/helm/apache/pulsar/$VERSION_WITHOUT_RC
Thanks to all the contributors who made this possible.
EOF
```
## Maintaining svn https://dist.apache.org/repos/dist/release/pulsar/helm-chart/ content
The chart references the files in https://downloads.apache.org/pulsar/helm-chart/ which are maintained
by SVN directory https://dist.apache.org/repos/dist/release/pulsar/helm-chart/.
If you remove releases from this directory, the URLs in index.yaml should be updated point to the
https://archive.apache.org/dist/pulsar/helm-chart/ URL base instead of https://downloads.apache.org/pulsar/helm-chart/.

53
Vagrantfile vendored
View File

@ -1,53 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# -*- mode: ruby -*-
# vi: set ft=ruby :
# vagrant configuration file for setting up local environment for Pulsar Helm Chart
# CI script development.
#
# usage:
# Starting vagrant box:
# vagrant up
# Connecting to vagrant box and running a ci script:
# vagrant ssh
# byobu
# cd /vagrant
# .ci/chart_test.sh .ci/clusters/values-local-pv.yaml
# Shutting down vagrant box:
# vagrant halt
# Destroying vagrant box:
# vagrant destroy
Vagrant.configure("2") do |config|
config.vm.box = "ubuntu/focal64"
config.vm.provider "virtualbox" do |vb|
vb.memory = "7168"
vb.cpus = 2
end
config.vm.provision "shell", inline: <<-SHELL
export DEBIAN_FRONTEND=noninteractive
sudo apt-get update
sudo apt-get -y install docker.io
sudo adduser vagrant docker
echo 'PATH="/vagrant/output/bin:$PATH"' >> /home/vagrant/.profile
SHELL
end

View File

@ -1,38 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*~
# Various IDEs
.project
.idea/
*.tmproj

View File

@ -1,38 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
apiVersion: v2
appVersion: "4.0.5"
description: Apache Pulsar Helm chart for Kubernetes
name: pulsar
version: 4.1.0
kubeVersion: ">=1.25.0-0"
home: https://pulsar.apache.org
sources:
- https://github.com/apache/pulsar
- https://github.com/apache/pulsar-helm-chart
icon: https://pulsar.apache.org/img/pulsar.svg
maintainers:
- name: The Apache Pulsar Team
email: dev@pulsar.apache.org
dependencies:
- name: victoria-metrics-k8s-stack
version: 0.38.x
repository: https://victoriametrics.github.io/helm-charts/
condition: victoria-metrics-k8s-stack.enabled

View File

@ -1,239 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
----------------------------------------------------------------------------------------------------
pulsar-common/src/main/java/org/apache/pulsar/common/util/protobuf/ByteBufCoded{Input,Output}Stream.java
Copyright 2014, Google Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Code generated by the Protocol Buffer compiler is owned by the owner
of the input file used when generating it. This code is not
standalone and requires a support library to be linked with it. This
support library is itself covered by the above license.

View File

@ -1,5 +0,0 @@
Apache Pulsar
Copyright 2017-2022 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).

View File

@ -1,185 +0,0 @@
======================================================================================
APACHE PULSAR HELM CHART
======================================================================================
======================================================================================
SECURITY ADVISORY
======================================================================================
This Helm chart's default configuration DOES NOT meet production security requirements.
Users MUST review and customize security settings for their specific environment.
IMPORTANT: This Helm chart provides a starting point for Pulsar deployments but requires
significant security customization before use in production environments. We strongly
recommend implementing:
1. Proper network isolation and access controls
2. Authentication and authorization for all components
3. TLS encryption for all communication channels
4. Regular security updates and vulnerability assessments
As an open source project, we welcome contributions to improve security features.
Please consider submitting pull requests to address security gaps or enhance
existing security implementations.
---------------------------------------------------------------------------------------
SECURITY NOTICE: The Pulsar proxy is not designed for direct public internet exposure.
It lacks security features required for untrusted networks and should only be deployed
within secured environments with proper network controls.
IMPORTANT CHANGE IN v4.0.0: Default service type changed from LoadBalancer to ClusterIP
for security reasons. This limits access to within the Kubernetes environment by default.
---------------------------------------------------------------------------------------
IF YOU NEED EXTERNAL ACCESS FOR YOUR PULSAR CLUSTER:
---------------------------------------------------------------------------------------
Note: This information might be outdated. Please go to https://github.com/apache/pulsar-helm-chart for updated information.
If you need to expose the Pulsar Proxy outside the cluster using a LoadBalancer service type:
1. USE INTERNAL LOAD BALANCERS ONLY
- Set type to LoadBalancer only in secured environments with proper network controls
- Add cloud provider-specific annotations for internal load balancers
- See cloud provider documentation:
* AWS / EKS: https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/guide/service/annotations/
* Azure / AKS: https://learn.microsoft.com/en-us/azure/aks/internal-lb
* GCP / GKE: https://cloud.google.com/kubernetes-engine/docs/concepts/service-load-balancer-parameters
- Examples (verify correctness for your environment):
* AWS / EKS: service.beta.kubernetes.io/aws-load-balancer-internal: "true"
* Azure / AKS: service.beta.kubernetes.io/azure-load-balancer-internal: "true"
* GCP / GKE: networking.gke.io/load-balancer-type: "Internal"
2. IMPLEMENT AUTHENTICATION AND AUTHORIZATION
- Configure all clients to authenticate properly
- Set up appropriate authorization policies
3. USE TLS FOR ALL CONNECTIONS
- Enable TLS for client-to-proxy connections
- Enable TLS for proxy-to-broker connections
- Enable TLS for all internal cluster communications (brokers, zookeepers, bookies)
- Note: TLS alone is NOT sufficient as a security solution in Pulsar. Even with TLS enabled,
clusters exposed to untrusted networks remain vulnerable to denial-of-service attacks,
authentication bypass attempts, and protocol-level exploits. Always implement defense-in-depth
security measures and limit exposure to trusted networks only.
4. NETWORK SECURITY
- Use private networks (VPCs)
- Configure firewalls, security groups, and IP restrictions appropriately
- In addition, consider using loadBalancerSourceRanges to limit access to specific IP ranges
5. CLIENT IP ADDRESS BASED ACCESS RESTRICTIONS
- When using a LoadBalancer service type, restrict access to specific IP ranges by configuring
`proxy.service.loadBalancerSourceRanges` in your values.yaml
- Important: This should be implemented alongside other security measures (internal load balancer,
authentication, TLS, network policies) as part of a defense-in-depth strategy,
not as a standalone security solution
---------------------------------------------------------------------------------------
ALTERNATIVE FOR EXTERNAL ACCESS
---------------------------------------------------------------------------------------
As an alternative method for external access, Pulsar has support for SNI proxy routing:
https://pulsar.apache.org/docs/next/concepts-proxy-sni-routing/
SNI Proxy routing is supported with proxy servers such as Apache Traffic Server, HAProxy and Nginx.
Note: This option isn't currently implemented in the Apache Pulsar Helm chart.
IMPORTANT: Pulsar binary protocol cannot be exposed outside of the Kubernetes cluster
using Kubernetes Ingress. Kubernetes Ingress works for the Admin REST API and topic lookups,
but clients would be connecting to the advertised listener addresses returned by the brokers and it
would only work when clients can connect directly to brokers. This is not a supported secure option
for exposing Pulsar to untrusted networks.
{{- if .Values.useReleaseStatus }}
======================================================================================
🚀 QUICK START 🚀
======================================================================================
Watching events to view progress of deployment:
kubectl get -n {{ .Values.namespace | default .Release.Namespace }} events -o wide --watch
Watching state of deployed Kubernetes objects, updated every 2 seconds:
watch kubectl get -n {{ .Values.namespace | default .Release.Namespace }} all
{{- if .Values.components.proxy }}
Waiting until Pulsar Proxy is available:
kubectl wait --timeout=600s --for=condition=ready pod -n {{ .Values.namespace | default .Release.Namespace }} -l component=proxy
{{- end }}
Watching state with k9s (https://k9scli.io/topics/install/):
k9s -n {{ .Values.namespace | default .Release.Namespace }}
{{- if and .Values.affinity.anti_affinity (or (gt (int .Values.bookkeeper.replicaCount) 1) (gt (int .Values.zookeeper.replicaCount) 1)) }}
======================================================================================
⚠️ NOTICE FOR DEV K8S CLUSTER USERS ⚠️
======================================================================================
Please note that anti-affinity rules for Zookeeper and Bookie components require at least
one node per replica. There are currently {{ .Values.bookkeeper.replicaCount }} bookies and {{ .Values.zookeeper.replicaCount }} zookeepers configured.
For Kubernetes clusters with fewer than 3 nodes, such as single-node Kubernetes clusters in
development environments like minikube, Docker Desktop, Rancher Desktop (k3s), or Podman
Desktop, you must disable the anti-affinity feature by either:
Adding to your values.yaml:
affinity:
anti_affinity: false
Or adding "--set affinity.anti_affinity=false" to the helm command line.
After making the changes to your values yaml file, redeploy with "helm upgrade":
helm upgrade -n {{ .Release.Namespace }} -f your_values_file.yaml {{ .Release.Name }} apachepulsar/pulsar
These configuration instructions can be omitted for Kubernetes clusters with 3 or more nodes.
{{- end }}
{{- end }}
{{- if and (eq .Values.proxy.service.type "LoadBalancer") (not .Values.proxy.service.annotations) }}
======================================================================================
⚠️ 🚨 INSECURE CONFIGURATION DETECTED 🚨 ⚠️
======================================================================================
WARNING: You are using a LoadBalancer service type without internal load balancer
annotations. This is potentially an insecure configuration. Please carefully review
the security recommendations above and visit https://github.com/apache/pulsar-helm-chart
for more information.
======================================================================================
{{- end }}
======================================================================================
DISCLAIMER
======================================================================================
The providers of this Helm chart make no guarantees regarding the security of the chart under
any circumstances. It is the user's responsibility to ensure that their deployment is secure
and complies with all relevant security standards and regulations.
By using this Helm chart, the user acknowledges the risks associated with its default
configuration and the necessity for proper security customization. The user further
agrees that the providers of the Helm chart shall not be liable for any security breaches
or incidents resulting from the use of the chart.
The user assumes full responsibility for the security and integrity of their deployment.
This includes, but is not limited to, the proper configuration of security features and
adherence to best practices for securing network access. The providers of this Helm chart
disclaim all warranties, whether express or implied, including any warranties of
merchantability, fitness for a particular purpose, and non-infringement of third-party rights.
======================================================================================
RESOURCES
======================================================================================
- 🖥️ Install k9s terminal interface for viewing and managing k8s clusters: https://k9scli.io/topics/install/
- ❓ Usage Questions: https://github.com/apache/pulsar/discussions/categories/q-a
- 🐛 Report Issues: https://github.com/apache/pulsar-helm-chart/issues
- 🔒 Security Issues: https://pulsar.apache.org/security/
- 📚 Documentation: https://github.com/apache/pulsar-helm-chart
🌟 Please contribute to improve the Apache Pulsar Helm chart and its documentation:
- 🤝 Contribute: https://github.com/apache/pulsar-helm-chart
Thank you for installing Apache Pulsar Helm chart version {{ .Chart.Version }}.

View File

@ -1,124 +0,0 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
{{/*
Define the pulsar autorecovery service
*/}}
{{- define "pulsar.autorecovery.service" -}}
{{ template "pulsar.fullname" . }}-{{ .Values.autorecovery.component }}
{{- end }}
{{/*
Define the autorecovery hostname
*/}}
{{- define "pulsar.autorecovery.hostname" -}}
${HOSTNAME}.{{ template "pulsar.autorecovery.service" . }}.{{ template "pulsar.namespace" . }}.svc.{{ .Values.clusterDomain }}
{{- end -}}
{{/*
Define autorecovery zookeeper client tls settings
*/}}
{{- define "pulsar.autorecovery.zookeeper.tls.settings" -}}
{{- if and .Values.tls.enabled .Values.tls.zookeeper.enabled }}
{{- include "pulsar.component.zookeeper.tls.settings" (dict "component" "autorecovery" "isClient" true "isCacerts" .Values.tls.autorecovery.cacerts.enabled) -}}
{{- end }}
{{- end }}
{{/*
Define autorecovery tls certs mounts
*/}}
{{- define "pulsar.autorecovery.certs.volumeMounts" -}}
{{- if and .Values.tls.enabled .Values.tls.zookeeper.enabled }}
- name: autorecovery-certs
mountPath: "/pulsar/certs/autorecovery"
readOnly: true
- name: ca
mountPath: "/pulsar/certs/ca"
readOnly: true
{{- end }}
{{- if .Values.tls.autorecovery.cacerts.enabled }}
- mountPath: "/pulsar/certs/cacerts"
name: autorecovery-cacerts
{{- range $cert := .Values.tls.autorecovery.cacerts.certs }}
- name: {{ $cert.name }}
mountPath: "/pulsar/certs/{{ $cert.name }}"
readOnly: true
{{- end }}
- name: certs-scripts
mountPath: "/pulsar/bin/certs-combine-pem.sh"
subPath: certs-combine-pem.sh
- name: certs-scripts
mountPath: "/pulsar/bin/certs-combine-pem-infinity.sh"
subPath: certs-combine-pem-infinity.sh
{{- end }}
{{- end }}
{{/*
Define autorecovery tls certs volumes
*/}}
{{- define "pulsar.autorecovery.certs.volumes" -}}
{{- if and .Values.tls.enabled .Values.tls.zookeeper.enabled }}
- name: autorecovery-certs
secret:
secretName: "{{ .Release.Name }}-{{ .Values.tls.autorecovery.cert_name }}"
items:
- key: tls.crt
path: tls.crt
- key: tls.key
path: tls.key
- key: tls-combined.pem
path: tls-combined.pem
- name: ca
secret:
secretName: "{{ template "pulsar.certs.issuers.ca.secretName" . }}"
items:
- key: ca.crt
path: ca.crt
{{- end }}
{{- if .Values.tls.autorecovery.cacerts.enabled }}
- name: autorecovery-cacerts
emptyDir: {}
{{- range $cert := .Values.tls.autorecovery.cacerts.certs }}
- name: {{ $cert.name }}
secret:
secretName: "{{ $cert.existingSecret }}"
items:
{{- range $key := $cert.secretKeys }}
- key: {{ $key }}
path: {{ $key }}
{{- end }}
{{- end }}
- name: certs-scripts
configMap:
name: "{{ template "pulsar.fullname" . }}-certs-scripts"
defaultMode: 0755
{{- end }}
{{- end }}
{{/*
Define autorecovery init container : verify cluster id
*/}}
{{- define "pulsar.autorecovery.init.verify_cluster_id" -}}
bin/apply-config-from-env.py conf/bookkeeper.conf;
export BOOKIE_MEM="-Xmx128M";
{{- include "pulsar.autorecovery.zookeeper.tls.settings" . }}
until timeout 15 bin/bookkeeper shell whatisinstanceid; do
sleep 3;
done;
{{- end }}

View File

@ -1,195 +0,0 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
{{/*
Define the pulsar bookkeeper service
*/}}
{{- define "pulsar.bookkeeper.service" -}}
{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}
{{- end }}
{{/*
Define the bookkeeper hostname
*/}}
{{- define "pulsar.bookkeeper.hostname" -}}
${HOSTNAME}.{{ template "pulsar.bookkeeper.service" . }}.{{ template "pulsar.namespace" . }}.svc.{{ .Values.clusterDomain }}
{{- end -}}
{{/*
Define bookie zookeeper client tls settings
*/}}
{{- define "pulsar.bookkeeper.zookeeper.tls.settings" -}}
{{- if and .Values.tls.enabled .Values.tls.zookeeper.enabled }}
{{- include "pulsar.component.zookeeper.tls.settings" (dict "component" "bookie" "isClient" true "isCacerts" .Values.tls.bookie.cacerts.enabled) -}}
{{- end }}
{{- end }}
{{/*
Define bookie tls certs mounts
*/}}
{{- define "pulsar.bookkeeper.certs.volumeMounts" -}}
{{- if .Values.tls.enabled }}
{{- if or .Values.tls.bookie.enabled .Values.tls.zookeeper.enabled }}
- name: bookie-certs
mountPath: "/pulsar/certs/bookie"
readOnly: true
{{- end }}
- name: ca
mountPath: "/pulsar/certs/ca"
readOnly: true
{{- end }}
{{- if .Values.tls.bookie.cacerts.enabled }}
- mountPath: "/pulsar/certs/cacerts"
name: bookie-cacerts
{{- range $cert := .Values.tls.bookie.cacerts.certs }}
- name: {{ $cert.name }}
mountPath: "/pulsar/certs/{{ $cert.name }}"
readOnly: true
{{- end }}
- name: certs-scripts
mountPath: "/pulsar/bin/certs-combine-pem.sh"
subPath: certs-combine-pem.sh
- name: certs-scripts
mountPath: "/pulsar/bin/certs-combine-pem-infinity.sh"
subPath: certs-combine-pem-infinity.sh
{{- end }}
{{- end }}
{{/*
Define bookie tls certs volumes
*/}}
{{- define "pulsar.bookkeeper.certs.volumes" -}}
{{- if .Values.tls.enabled }}
{{- if or .Values.tls.bookie.enabled .Values.tls.zookeeper.enabled }}
- name: bookie-certs
secret:
secretName: "{{ .Release.Name }}-{{ .Values.tls.bookie.cert_name }}"
items:
- key: tls.crt
path: tls.crt
- key: tls.key
path: tls.key
{{- if .Values.tls.zookeeper.enabled }}
- key: tls-combined.pem
path: tls-combined.pem
{{- end }}
{{- end }}
- name: ca
secret:
secretName: "{{ template "pulsar.certs.issuers.ca.secretName" . }}"
items:
- key: ca.crt
path: ca.crt
{{- end }}
{{- if .Values.tls.bookie.cacerts.enabled }}
- name: bookie-cacerts
emptyDir: {}
{{- range $cert := .Values.tls.bookie.cacerts.certs }}
- name: {{ $cert.name }}
secret:
secretName: "{{ $cert.existingSecret }}"
items:
{{- range $key := $cert.secretKeys }}
- key: {{ $key }}
path: {{ $key }}
{{- end }}
{{- end }}
- name: certs-scripts
configMap:
name: "{{ template "pulsar.fullname" . }}-certs-scripts"
defaultMode: 0755
{{- end }}
{{- end }}
{{/*
Define bookie common config
*/}}
{{- define "pulsar.bookkeeper.config.common" -}}
{{/*
Configure BookKeeper's metadata store (available since BookKeeper 4.7.0 / BP-29)
https://bookkeeper.apache.org/bps/BP-29-metadata-store-api-module/
https://bookkeeper.apache.org/docs/deployment/manual#cluster-metadata-setup
*/}}
# Set empty values for zkServers and zkLedgersRootPath since we're using the metadataServiceUri to configure BookKeeper's metadata store
zkServers: ""
zkLedgersRootPath: ""
{{- if .Values.components.zookeeper }}
{{- if (and (hasKey .Values.pulsar_metadata "bookkeeper") .Values.pulsar_metadata.bookkeeper.usePulsarMetadataBookieDriver) }}
# there's a bug when using PulsarMetadataBookieDriver since it always appends /ledgers to the metadataServiceUri
# Possibly a bug in org.apache.pulsar.metadata.bookkeeper.AbstractMetadataDriver#resolveLedgersRootPath in Pulsar code base
metadataServiceUri: "metadata-store:zk:{{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }}"
{{- else }}
# use zk+hierarchical:// when using BookKeeper's built-in metadata driver
metadataServiceUri: "zk+hierarchical://{{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }}/ledgers"
{{- end }}
{{- else if .Values.components.oxia }}
metadataServiceUri: "{{ template "pulsar.oxia.metadata.url.bookkeeper" . }}"
{{- end }}
{{- /* metadataStoreSessionTimeoutMillis maps to zkTimeout in bookkeeper.conf for both zookeeper and oxia metadata stores */}}
{{- if (and (hasKey .Values.pulsar_metadata "bookkeeper") (hasKey .Values.pulsar_metadata.bookkeeper "metadataStoreSessionTimeoutMillis")) }}
zkTimeout: "{{ .Values.pulsar_metadata.bookkeeper.metadataStoreSessionTimeoutMillis }}"
{{- end }}
# enable bookkeeper http server
httpServerEnabled: "true"
httpServerPort: "{{ .Values.bookkeeper.ports.http }}"
# config the stats provider
statsProviderClass: org.apache.bookkeeper.stats.prometheus.PrometheusMetricsProvider
# use hostname as the bookie id
useHostNameAsBookieID: "true"
{{- end }}
{{/*
Define bookie tls config
*/}}
{{- define "pulsar.bookkeeper.config.tls" -}}
{{- if and .Values.tls.enabled .Values.tls.bookie.enabled }}
PULSAR_PREFIX_tlsProviderFactoryClass: org.apache.bookkeeper.tls.TLSContextFactory
PULSAR_PREFIX_tlsCertificatePath: /pulsar/certs/bookie/tls.crt
PULSAR_PREFIX_tlsKeyStoreType: PEM
PULSAR_PREFIX_tlsKeyStore: /pulsar/certs/bookie/tls.key
PULSAR_PREFIX_tlsTrustStoreType: PEM
PULSAR_PREFIX_tlsTrustStore: {{ ternary "/pulsar/certs/cacerts/ca-combined.pem" "/pulsar/certs/ca/ca.crt" .Values.tls.bookie.cacerts.enabled | quote }}
{{- end }}
{{- end }}
{{/*
Define bookie init container : verify cluster id
*/}}
{{- define "pulsar.bookkeeper.init.verify_cluster_id" -}}
{{- if not (and .Values.volumes.persistence .Values.bookkeeper.volumes.persistence) }}
bin/apply-config-from-env.py conf/bookkeeper.conf;
export BOOKIE_MEM="-Xmx128M";
{{- include "pulsar.bookkeeper.zookeeper.tls.settings" . }}
until timeout 15 bin/bookkeeper shell whatisinstanceid; do
sleep 3;
done;
bin/bookkeeper shell bookieformat -nonInteractive -force -deleteCookie || true
{{- end }}
{{- if and .Values.volumes.persistence .Values.bookkeeper.volumes.persistence }}
set -e;
bin/apply-config-from-env.py conf/bookkeeper.conf;
export BOOKIE_MEM="-Xmx128M";
{{- include "pulsar.bookkeeper.zookeeper.tls.settings" . }}
until timeout 15 bin/bookkeeper shell whatisinstanceid; do
sleep 3;
done;
{{- end }}
{{- end }}

View File

@ -1,125 +0,0 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
{{/*
Define the pulsar brroker service
*/}}
{{- define "pulsar.broker.service" -}}
{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}
{{- end }}
{{/*
Define the hostname
*/}}
{{- define "pulsar.broker.hostname" -}}
${HOSTNAME}.{{ template "pulsar.broker.service" . }}.{{ template "pulsar.namespace" . }}.svc.{{ .Values.clusterDomain }}
{{- end -}}
{{/*
Define the broker znode
*/}}
{{- define "pulsar.broker.znode" -}}
{{ .Values.metadataPrefix }}/loadbalance/brokers/{{ template "pulsar.broker.hostname" . }}:{{ .Values.broker.ports.http }}
{{- end }}
{{/*
Define broker zookeeper client tls settings
*/}}
{{- define "pulsar.broker.zookeeper.tls.settings" -}}
{{- if and .Values.tls.enabled .Values.tls.zookeeper.enabled }}
{{- include "pulsar.component.zookeeper.tls.settings" (dict "component" "broker" "isClient" true "isCacerts" .Values.tls.broker.cacerts.enabled) -}}
{{- end }}
{{- end }}
{{/*
Define broker tls certs mounts
*/}}
{{- define "pulsar.broker.certs.volumeMounts" -}}
{{- if .Values.tls.enabled }}
{{- if or .Values.tls.broker.enabled (or .Values.tls.bookie.enabled .Values.tls.zookeeper.enabled) }}
- name: broker-certs
mountPath: "/pulsar/certs/broker"
readOnly: true
{{- end }}
- name: ca
mountPath: "/pulsar/certs/ca"
readOnly: true
{{- end }}
{{- if .Values.tls.broker.cacerts.enabled }}
- mountPath: "/pulsar/certs/cacerts"
name: broker-cacerts
{{- range $cert := .Values.tls.broker.cacerts.certs }}
- name: {{ $cert.name }}
mountPath: "/pulsar/certs/{{ $cert.name }}"
readOnly: true
{{- end }}
- name: certs-scripts
mountPath: "/pulsar/bin/certs-combine-pem.sh"
subPath: certs-combine-pem.sh
- name: certs-scripts
mountPath: "/pulsar/bin/certs-combine-pem-infinity.sh"
subPath: certs-combine-pem-infinity.sh
{{- end }}
{{- end }}
{{/*
Define broker tls certs volumes
*/}}
{{- define "pulsar.broker.certs.volumes" -}}
{{- if .Values.tls.enabled }}
{{- if or .Values.tls.broker.enabled (or .Values.tls.bookie.enabled .Values.tls.zookeeper.enabled) }}
- name: broker-certs
secret:
secretName: "{{ .Release.Name }}-{{ .Values.tls.broker.cert_name }}"
items:
- key: tls.crt
path: tls.crt
- key: tls.key
path: tls.key
{{- if .Values.tls.zookeeper.enabled }}
- key: tls-combined.pem
path: tls-combined.pem
{{- end }}
{{- end }}
- name: ca
secret:
secretName: "{{ template "pulsar.certs.issuers.ca.secretName" . }}"
items:
- key: ca.crt
path: ca.crt
{{- end }}
{{- if .Values.tls.broker.cacerts.enabled }}
- name: broker-cacerts
emptyDir: {}
{{- range $cert := .Values.tls.broker.cacerts.certs }}
- name: {{ $cert.name }}
secret:
secretName: "{{ $cert.existingSecret }}"
items:
{{- range $key := $cert.secretKeys }}
- key: {{ $key }}
path: {{ $key }}
{{- end }}
{{- end }}
- name: certs-scripts
configMap:
name: "{{ template "pulsar.fullname" . }}-certs-scripts"
defaultMode: 0755
{{- end }}
{{- end }}

View File

@ -1,132 +0,0 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
{{/*
Define the pulsar certs ca issuer name
*/}}
{{- define "pulsar.certs.issuers.ca.name" -}}
{{- if .Values.certs.internal_issuer.enabled -}}
{{- if and (eq .Values.certs.internal_issuer.type "selfsigning") .Values.certs.issuers.selfsigning.name -}}
{{- .Values.certs.issuers.selfsigning.name -}}
{{- else if and (eq .Values.certs.internal_issuer.type "ca") .Values.certs.issuers.ca.name -}}
{{- .Values.certs.issuers.ca.name -}}
{{- else -}}
{{- template "pulsar.fullname" . }}-{{ .Values.certs.internal_issuer.component }}-ca-issuer
{{- end -}}
{{- else -}}
{{- if .Values.certs.issuers.ca.name -}}
{{- .Values.certs.issuers.ca.name -}}
{{- else -}}
{{- fail "certs.issuers.ca.name is required when TLS is enabled and certs.internal_issuer.enabled is false" -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{/*
Define the pulsar certs ca issuer secret name
*/}}
{{- define "pulsar.certs.issuers.ca.secretName" -}}
{{- if .Values.certs.internal_issuer.enabled -}}
{{- if and (eq .Values.certs.internal_issuer.type "selfsigning") .Values.certs.issuers.selfsigning.secretName -}}
{{- .Values.certs.issuers.selfsigning.secretName -}}
{{- else if and (eq .Values.certs.internal_issuer.type "ca") .Values.certs.issuers.ca.secretName -}}
{{- .Values.certs.issuers.ca.secretName -}}
{{- else -}}
{{- printf "%s-%s" .Release.Name .Values.tls.ca_suffix -}}
{{- end -}}
{{- else -}}
{{- if .Values.certs.issuers.ca.secretName -}}
{{- .Values.certs.issuers.ca.secretName -}}
{{- else -}}
{{- fail "certs.issuers.ca.secretName is required when TLS is enabled and certs.internal_issuer.enabled is false" -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{/*
Common certificate template
Usage: {{- include "pulsar.cert.template" (dict "root" . "componentConfig" .Values.proxy "tlsConfig" .Values.tls.proxy) -}}
*/}}
{{- define "pulsar.cert.template" -}}
{{- if eq .root.Values.certs.internal_issuer.apiVersion "cert-manager.io/v1beta1" -}}
{{- fail "cert-manager.io/v1beta1 is no longer supported. Please set certs.internal_issuer.apiVersion to cert-manager.io/v1" -}}
{{- end -}}
apiVersion: "{{ .root.Values.certs.internal_issuer.apiVersion }}"
kind: Certificate
metadata:
name: "{{ template "pulsar.fullname" .root }}-{{ .tlsConfig.cert_name }}"
namespace: {{ template "pulsar.namespace" .root }}
labels:
{{- include "pulsar.standardLabels" .root | nindent 4 }}
spec:
# Secret names are always required.
secretName: "{{ .root.Release.Name }}-{{ .tlsConfig.cert_name }}"
{{- if .root.Values.tls.zookeeper.enabled }}
additionalOutputFormats:
- type: CombinedPEM
{{- end }}
duration: "{{ .root.Values.tls.common.duration }}"
renewBefore: "{{ .root.Values.tls.common.renewBefore }}"
subject:
organizations:
{{ toYaml .root.Values.tls.common.organization | indent 4 }}
# The use of the common name field has been deprecated since 2000 and is
# discouraged from being used.
commonName: "{{ template "pulsar.fullname" .root }}-{{ .componentConfig.component }}"
isCA: false
privateKey:
size: {{ .root.Values.tls.common.keySize }}
algorithm: {{ .root.Values.tls.common.keyAlgorithm }}
encoding: {{ .root.Values.tls.common.keyEncoding }}
usages:
- server auth
- client auth
# At least one of a DNS Name, USI SAN, or IP address is required.
dnsNames:
{{- if .tlsConfig.dnsNames }}
{{ toYaml .tlsConfig.dnsNames | indent 4 }}
{{- end }}
- {{ printf "*.%s-%s.%s.svc.%s" (include "pulsar.fullname" .root) .componentConfig.component (include "pulsar.namespace" .root) .root.Values.clusterDomain | quote }}
- {{ printf "%s-%s" (include "pulsar.fullname" .root) .componentConfig.component | quote }}
# Issuer references are always required.
issuerRef:
name: "{{ template "pulsar.certs.issuers.ca.name" .root }}"
# We can reference ClusterIssuers by changing the kind here.
# The default value is Issuer (i.e. a locally namespaced Issuer)
kind: Issuer
# This is optional since cert-manager will default to this value however
# if you are using an external issuer, change this to that issuer group.
group: cert-manager.io
{{- end -}}
{{/*
CA certificates template
Usage: {{ include "pulsar.certs.cacerts" (dict "certs" .Values.tls.<component>.cacerts.certs) }}
*/}}
{{- define "pulsar.certs.cacerts" -}}
{{- $certs := .certs -}}
{{- $cacerts := list -}}
{{- $cacerts = print "/pulsar/certs/ca/ca.crt" | append $cacerts -}}
{{- range $cert := $certs -}}
{{- range $key := $cert.secretKeys -}}
{{- $cacerts = print "/pulsar/certs/" $cert.name "/" $key | append $cacerts -}}
{{- end -}}
{{- end -}}
{{ join " " $cacerts }}
{{- end -}}

View File

@ -1,39 +0,0 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
{{/*
Define configuration store endpoint
*/}}
{{- define "pulsar.configurationStore.service" -}}
{{- if .Values.pulsar_metadata.configurationStore }}
{{- .Values.pulsar_metadata.configurationStore }}
{{- else -}}
{{ template "pulsar.zookeeper.service" . }}
{{- end -}}
{{- end -}}
{{/*
Define configuration store connection string
*/}}
{{- define "pulsar.configurationStore.connect" -}}
{{- if .Values.pulsar_metadata.configurationStore }}
{{- template "pulsar.configurationStore.service" . }}:{{ .Values.pulsar_metadata.configurationStorePort }}
{{- end -}}
{{- end -}}

View File

@ -1,138 +0,0 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
{{/* vim: set filetype=mustache: */}}
{{/*
pulsar home
*/}}
{{- define "pulsar.home" -}}
{{- print "/pulsar" -}}
{{- end -}}
{{/*
Expand the name of the chart.
*/}}
{{- define "pulsar.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{/*
Expand the namespace of the chart.
*/}}
{{- define "pulsar.namespace" -}}
{{- default .Release.Namespace .Values.namespace -}}
{{- end -}}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "pulsar.fullname" -}}
{{- if .Values.fullnameOverride -}}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- if contains $name .Release.Name -}}
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{/*
Define cluster's name
*/}}
{{- define "pulsar.cluster.name" -}}
{{- if .Values.clusterName }}
{{- .Values.clusterName }}
{{- else -}}
{{- template "pulsar.fullname" .}}
{{- end -}}
{{- end -}}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "pulsar.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{/*
Create the common labels.
*/}}
{{- define "pulsar.standardLabels" -}}
app: {{ template "pulsar.name" . }}
chart: {{ template "pulsar.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
cluster: {{ template "pulsar.cluster.name" . }}
{{- if .Values.labels }}
{{ .Values.labels | toYaml | trim }}
{{- end }}
{{- end }}
{{/*
Create the template labels.
*/}}
{{- define "pulsar.template.labels" -}}
app: {{ template "pulsar.name" . }}
release: {{ .Release.Name }}
cluster: {{ template "pulsar.cluster.name" . }}
{{- if .Values.labels }}
{{ .Values.labels | toYaml | trim }}
{{- end }}
{{- end }}
{{/*
Create the match labels.
*/}}
{{- define "pulsar.matchLabels" -}}
app: {{ template "pulsar.name" . }}
release: {{ .Release.Name }}
{{- end }}
{{/*
Create ImagePullSecrets
*/}}
{{- define "pulsar.imagePullSecrets" -}}
{{- if .Values.images.imagePullSecrets -}}
imagePullSecrets:
{{- range .Values.images.imagePullSecrets }}
- name: {{ . }}
{{- end }}
{{- end -}}
{{- end }}
{{/*
Create full image name
*/}}
{{- define "pulsar.imageFullName" -}}
{{- printf "%s:%s" (.image.repository | default .root.Values.defaultPulsarImageRepository) (.image.tag | default .root.Values.defaultPulsarImageTag | default .root.Chart.AppVersion) -}}
{{- end -}}
{{/*
Lookup pull policy, default to defaultPullPolicy
*/}}
{{- define "pulsar.imagePullPolicy" -}}
{{- printf "%s" (.image.pullPolicy | default .root.Values.defaultPullPolicy) -}}
{{- end -}}

View File

@ -1,97 +0,0 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
{{- define "pulsar.podMonitor" -}}
{{- $root := index . 0 }}
{{- $component := index . 1 }}
{{- $matchLabel := index . 2 }}
{{- $portName := "http" }}
{{- if gt (len .) 3 }}
{{- $portName = index . 3 }}
{{- end }}
{{/* Extract component parts for nested values */}}
{{- $componentParts := splitList "." $component }}
{{- $valuesPath := $root.Values }}
{{- range $componentParts }}
{{- $valuesPath = index $valuesPath . }}
{{- end }}
{{- if index $root.Values "victoria-metrics-k8s-stack" "enabled" }}
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMPodScrape
{{- else }}
apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
{{- end }}
metadata:
name: {{ template "pulsar.fullname" $root }}-{{ replace "." "-" $component }}
labels:
{{- include "pulsar.standardLabels" $root | nindent 4 }}
spec:
jobLabel: {{ replace "." "-" $component }}
podMetricsEndpoints:
- port: {{ $portName }}
path: /metrics
scheme: http
interval: {{ $valuesPath.podMonitor.interval }}
scrapeTimeout: {{ $valuesPath.podMonitor.scrapeTimeout }}
# Set honor labels to true to allow overriding namespace label with Pulsar's namespace label
honorLabels: true
{{- if index $root.Values "victoria-metrics-k8s-stack" "enabled" }}
relabelConfigs:
{{- else }}
relabelings:
{{- end }}
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- sourceLabels: [__meta_kubernetes_namespace]
action: replace
targetLabel: kubernetes_namespace
- sourceLabels: [__meta_kubernetes_pod_label_component]
action: replace
targetLabel: job
- sourceLabels: [__meta_kubernetes_pod_name]
action: replace
targetLabel: kubernetes_pod_name
{{- if or $valuesPath.podMonitor.metricRelabelings (and $valuesPath.podMonitor.dropUnderscoreCreatedMetrics (index $valuesPath.podMonitor.dropUnderscoreCreatedMetrics "enabled")) }}
{{- if index $root.Values "victoria-metrics-k8s-stack" "enabled" }}
metricRelabelConfigs:
{{- else }}
metricRelabelings:
{{- end }}
{{- if and $valuesPath.podMonitor.dropUnderscoreCreatedMetrics (index $valuesPath.podMonitor.dropUnderscoreCreatedMetrics "enabled") }}
# Drop metrics that end with _created, auto-created by metrics library to match OpenMetrics format
- sourceLabels: [__name__]
{{- if and (hasKey $valuesPath.podMonitor.dropUnderscoreCreatedMetrics "excludePatterns") $valuesPath.podMonitor.dropUnderscoreCreatedMetrics.excludePatterns }}
regex: "(?!{{ $valuesPath.podMonitor.dropUnderscoreCreatedMetrics.excludePatterns | join "|" }}).*_created$"
{{- else }}
regex: ".*_created$"
{{- end }}
action: drop
{{- end }}
{{- with $valuesPath.podMonitor.metricRelabelings }}
{{ toYaml . | indent 8 }}
{{- end }}
{{- end }}
selector:
matchLabels:
{{- include "pulsar.matchLabels" $root | nindent 6 }}
{{ $matchLabel }}
{{- end -}}

View File

@ -1,122 +0,0 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
{{/*
Probe
*/}}
{{- define "oxia-cluster.probe" -}}
exec:
command: ["oxia", "health", "--port={{ . }}"]
initialDelaySeconds: 10
timeoutSeconds: 10
{{- end }}
{{/*
Probe
*/}}
{{- define "oxia-cluster.readiness-probe" -}}
exec:
command: ["oxia", "health", "--port={{ . }}", "--service=oxia-readiness"]
initialDelaySeconds: 10
timeoutSeconds: 10
{{- end }}
{{/*
Probe
*/}}
{{- define "oxia-cluster.startup-probe" -}}
exec:
command: ["oxia", "health", "--port={{ . }}"]
initialDelaySeconds: 60
timeoutSeconds: 10
{{- end }}
{{/*
Define the pulsar oxia
*/}}
{{- define "pulsar.oxia.server.service" -}}
{{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-svc
{{- end }}
{{/*
oxia url for broker metadata
*/}}
{{- define "pulsar.oxia.metadata.url.broker" -}}
{{- if .Values.components.oxia -}}
oxia://{{ template "pulsar.oxia.server.service" . }}:{{ .Values.oxia.server.ports.public }}/broker
{{- end -}}
{{- end -}}
{{/*
oxia url for bookkeeper metadata
*/}}
{{- define "pulsar.oxia.metadata.url.bookkeeper" -}}
{{- if .Values.components.oxia -}}
metadata-store:oxia://{{ template "pulsar.oxia.server.service" . }}:{{ .Values.oxia.server.ports.public }}/bookkeeper
{{- end -}}
{{- end -}}
{{/*
Define coordinator configmap
*/}}
{{- define "oxia.coordinator.config.yaml" -}}
namespaces:
- name: default
initialShardCount: {{ .Values.oxia.initialShardCount }}
replicationFactor: {{ .Values.oxia.replicationFactor }}
- name: broker
initialShardCount: {{ .Values.oxia.initialShardCount }}
replicationFactor: {{ .Values.oxia.replicationFactor }}
- name: bookkeeper
initialShardCount: {{ .Values.oxia.initialShardCount }}
replicationFactor: {{ .Values.oxia.replicationFactor }}
servers:
{{- $servicename := printf "%s-%s-svc" (include "pulsar.fullname" .) .Values.oxia.component }}
{{- $fqdnSuffix := printf "%s.svc.cluster.local" (include "pulsar.namespace" .) }}
{{- $podnamePrefix := printf "%s-%s-server-" (include "pulsar.fullname" .) .Values.oxia.component }}
{{- range until (int .Values.oxia.server.replicas) }}
{{- $podnameIndex := . }}
{{- $podname := printf "%s%d.%s" $podnamePrefix $podnameIndex $servicename }}
{{- $podnameFQDN := printf "%s.%s" $podname $fqdnSuffix }}
- public: {{ $podnameFQDN }}:{{ $.Values.oxia.server.ports.public }}
internal: {{ $podname }}:{{ $.Values.oxia.server.ports.internal }}
{{- end }}
{{- end }}
{{/*
Define coordinator entrypoint
*/}}
{{- define "oxia.coordinator.entrypoint" -}}
- "oxia"
- "coordinator"
{{- if .Values.oxia.coordinator.customConfigMapName }}
- "--conf=configmap:{{ template "pulsar.namespace" . }}/{{ .Values.oxia.coordinator.customConfigMapName }}"
{{- else }}
- "--conf=configmap:{{ template "pulsar.namespace" . }}/{{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator"
{{- end }}
- "--log-json"
- "--metadata=configmap"
- "--k8s-namespace={{ template "pulsar.namespace" . }}"
- "--k8s-configmap-name={{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator-status"
{{- if .Values.oxia.pprofEnabled }}
- "--profile"
{{- end}}
{{- end}}

View File

@ -1,95 +0,0 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
{{/*
Define proxy tls certs mounts
*/}}
{{- define "pulsar.proxy.certs.volumeMounts" -}}
{{- if .Values.tls.enabled }}
{{- if .Values.tls.proxy.enabled }}
- mountPath: "/pulsar/certs/proxy"
name: proxy-certs
readOnly: true
{{- end }}
- mountPath: "/pulsar/certs/ca"
name: ca
readOnly: true
{{- end }}
{{- if .Values.tls.proxy.cacerts.enabled }}
- mountPath: "/pulsar/certs/cacerts"
name: proxy-cacerts
{{- range $cert := .Values.tls.proxy.cacerts.certs }}
- name: {{ $cert.name }}
mountPath: "/pulsar/certs/{{ $cert.name }}"
readOnly: true
{{- end }}
- name: certs-scripts
mountPath: "/pulsar/bin/certs-combine-pem.sh"
subPath: certs-combine-pem.sh
- name: certs-scripts
mountPath: "/pulsar/bin/certs-combine-pem-infinity.sh"
subPath: certs-combine-pem-infinity.sh
{{- end }}
{{- end }}
{{/*
Define proxy tls certs volumes
*/}}
{{- define "pulsar.proxy.certs.volumes" -}}
{{- if .Values.tls.enabled }}
{{- if .Values.tls.proxy.enabled }}
- name: proxy-certs
secret:
secretName: "{{ .Release.Name }}-{{ .Values.tls.proxy.cert_name }}"
items:
- key: tls.crt
path: tls.crt
- key: tls.key
path: tls.key
{{- if .Values.tls.zookeeper.enabled }}
- key: tls-combined.pem
path: tls-combined.pem
{{- end }}
{{- end }}
- name: ca
secret:
secretName: "{{ template "pulsar.certs.issuers.ca.secretName" . }}"
items:
- key: ca.crt
path: ca.crt
{{- end }}
{{- if .Values.tls.proxy.cacerts.enabled }}
- name: proxy-cacerts
emptyDir: {}
{{- range $cert := .Values.tls.proxy.cacerts.certs }}
- name: {{ $cert.name }}
secret:
secretName: "{{ $cert.existingSecret }}"
items:
{{- range $key := $cert.secretKeys }}
- key: {{ $key }}
path: {{ $key }}
{{- end }}
{{- end }}
- name: certs-scripts
configMap:
name: "{{ template "pulsar.fullname" . }}-certs-scripts"
defaultMode: 0755
{{- end }}
{{- end }}

View File

@ -1,116 +0,0 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
{{/*
Define the pulsar toolset service
*/}}
{{- define "pulsar.toolset.service" -}}
{{ template "pulsar.fullname" . }}-{{ .Values.toolset.component }}
{{- end }}
{{/*
Define the toolset hostname
*/}}
{{- define "pulsar.toolset.hostname" -}}
${HOSTNAME}.{{ template "pulsar.toolset.service" . }}.{{ template "pulsar.namespace" . }}.svc.{{ .Values.clusterDomain }}
{{- end -}}
{{/*
Define toolset zookeeper client tls settings
*/}}
{{- define "pulsar.toolset.zookeeper.tls.settings" -}}
{{- if and .Values.tls.enabled .Values.tls.zookeeper.enabled -}}
{{- include "pulsar.component.zookeeper.tls.settings" (dict "component" "toolset" "isClient" true "isCacerts" .Values.tls.toolset.cacerts.enabled) -}}
{{- end -}}
{{- end }}
{{/*
Define toolset tls certs mounts
*/}}
{{- define "pulsar.toolset.certs.volumeMounts" -}}
{{- if .Values.tls.enabled }}
{{- if .Values.tls.zookeeper.enabled }}
- name: toolset-certs
mountPath: "/pulsar/certs/toolset"
readOnly: true
{{- end }}
- name: ca
mountPath: "/pulsar/certs/ca"
readOnly: true
{{- end }}
{{- if .Values.tls.toolset.cacerts.enabled }}
- mountPath: "/pulsar/certs/cacerts"
name: toolset-cacerts
{{- range $cert := .Values.tls.toolset.cacerts.certs }}
- name: {{ $cert.name }}
mountPath: "/pulsar/certs/{{ $cert.name }}"
readOnly: true
{{- end }}
- name: certs-scripts
mountPath: "/pulsar/bin/certs-combine-pem.sh"
subPath: certs-combine-pem.sh
- name: certs-scripts
mountPath: "/pulsar/bin/certs-combine-pem-infinity.sh"
subPath: certs-combine-pem-infinity.sh
{{- end }}
{{- end }}
{{/*
Define toolset tls certs volumes
*/}}
{{- define "pulsar.toolset.certs.volumes" -}}
{{- if .Values.tls.enabled }}
{{- if .Values.tls.zookeeper.enabled }}
- name: toolset-certs
secret:
secretName: "{{ .Release.Name }}-{{ .Values.tls.toolset.cert_name }}"
items:
- key: tls.crt
path: tls.crt
- key: tls.key
path: tls.key
- key: tls-combined.pem
path: tls-combined.pem
{{- end }}
- name: ca
secret:
secretName: "{{ template "pulsar.certs.issuers.ca.secretName" . }}"
items:
- key: ca.crt
path: ca.crt
{{- end }}
{{- if .Values.tls.toolset.cacerts.enabled }}
- name: toolset-cacerts
emptyDir: {}
{{- range $cert := .Values.tls.toolset.cacerts.certs }}
- name: {{ $cert.name }}
secret:
secretName: "{{ $cert.existingSecret }}"
items:
{{- range $key := $cert.secretKeys }}
- key: {{ $key }}
path: {{ $key }}
{{- end }}
{{- end }}
- name: certs-scripts
configMap:
name: "{{ template "pulsar.fullname" . }}-certs-scripts"
defaultMode: 0755
{{- end }}
{{- end }}

View File

@ -1,37 +0,0 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
{{/*
Renders a value that contains template perhaps with scope if the scope is present.
Usage:
{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ ) }}
{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ "scope" $app ) }}
*/}}
{{- define "common.tplvalues.render" -}}
{{- $value := typeIs "string" .value | ternary .value (.value | toYaml) }}
{{- if contains "{{" (toJson .value) }}
{{- if .scope }}
{{- tpl (cat "{{- with $.RelativeScope -}}" $value "{{- end }}") (merge (dict "RelativeScope" .scope) .context) }}
{{- else }}
{{- tpl $value .context }}
{{- end }}
{{- else }}
{{- $value }}
{{- end }}
{{- end -}}

View File

@ -1,25 +0,0 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
{{/*
Check deprecated setting auth.authentication.provider since 4.1.0
*/}}
{{- if (and .Values.auth.authentication.enabled (not (empty .Values.auth.authentication.provider))) }}
{{- fail "ERROR: Setting auth.authentication.provider is no longer supported. For details, see the migration guide in README.md." }}
{{- end }}

View File

@ -1,145 +0,0 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
{{/*
Define the pulsar zookeeper
*/}}
{{- define "pulsar.zookeeper.service" -}}
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}
{{- end }}
{{/*
Define the pulsar zookeeper
*/}}
{{- define "pulsar.zookeeper.connect" -}}
{{$zk:=.Values.pulsar_metadata.userProvidedZookeepers}}
{{- if and (not .Values.components.zookeeper) $zk }}
{{- $zk -}}
{{ else }}
{{- if not (and .Values.tls.enabled .Values.tls.zookeeper.enabled) -}}
{{ template "pulsar.zookeeper.service" . }}:{{ .Values.zookeeper.ports.client }}
{{- end -}}
{{- if and .Values.tls.enabled .Values.tls.zookeeper.enabled -}}
{{ template "pulsar.zookeeper.service" . }}:{{ .Values.zookeeper.ports.clientTls }}
{{- end -}}
{{- end -}}
{{- end -}}
{{/*
Define the zookeeper hostname
*/}}
{{- define "pulsar.zookeeper.hostname" -}}
${HOSTNAME}.{{ template "pulsar.zookeeper.service" . }}.{{ template "pulsar.namespace" . }}.svc.{{ .Values.clusterDomain }}
{{- end -}}
{{/*
Define zookeeper tls settings
*/}}
{{- define "pulsar.zookeeper.tls.settings" -}}
{{- if and .Values.tls.enabled .Values.tls.zookeeper.enabled }}
{{- include "pulsar.component.zookeeper.tls.settings" (dict "component" "zookeeper" "isClient" false "isCacerts" .Values.tls.zookeeper.cacerts.enabled) -}}
{{- end }}
{{- end }}
{{- define "pulsar.component.zookeeper.tls.settings" }}
{{- $component := .component -}}
{{- $isClient := .isClient -}}
{{- $keyFile := printf "/pulsar/certs/%s/tls-combined.pem" $component -}}
{{- $caFile := ternary "/pulsar/certs/cacerts/ca-combined.pem" "/pulsar/certs/ca/ca.crt" .isCacerts -}}
{{- if $isClient }}
echo $'\n' >> conf/pulsar_env.sh
echo "PULSAR_EXTRA_OPTS=\"\${PULSAR_EXTRA_OPTS} -Dzookeeper.clientCnxnSocket=org.apache.zookeeper.ClientCnxnSocketNetty -Dzookeeper.client.secure=true -Dzookeeper.client.certReload=true -Dzookeeper.ssl.keyStore.location={{- $keyFile }} -Dzookeeper.ssl.keyStore.type=PEM -Dzookeeper.ssl.trustStore.location={{- $caFile }} -Dzookeeper.ssl.trustStore.type=PEM\"" >> conf/pulsar_env.sh
echo $'\n' >> conf/bkenv.sh
echo "BOOKIE_EXTRA_OPTS=\"\${BOOKIE_EXTRA_OPTS} -Dzookeeper.clientCnxnSocket=org.apache.zookeeper.ClientCnxnSocketNetty -Dzookeeper.client.secure=true -Dzookeeper.client.certReload=true -Dzookeeper.ssl.keyStore.location={{- $keyFile }} -Dzookeeper.ssl.keyStore.type=PEM -Dzookeeper.ssl.trustStore.location={{- $caFile }} -Dzookeeper.ssl.trustStore.type=PEM\"" >> conf/bkenv.sh
{{- else }}
echo $'\n' >> conf/pulsar_env.sh
echo "PULSAR_EXTRA_OPTS=\"\${PULSAR_EXTRA_OPTS} -Dzookeeper.ssl.keyStore.location={{- $keyFile }} -Dzookeeper.ssl.keyStore.type=PEM -Dzookeeper.ssl.trustStore.location={{- $caFile }} -Dzookeeper.ssl.trustStore.type=PEM\"" >> conf/pulsar_env.sh
{{- end }}
{{- end }}
{{/*
Define zookeeper tls certs mounts
*/}}
{{- define "pulsar.zookeeper.certs.volumeMounts" -}}
{{- if and .Values.tls.enabled .Values.tls.zookeeper.enabled }}
- mountPath: "/pulsar/certs/zookeeper"
name: zookeeper-certs
readOnly: true
- mountPath: "/pulsar/certs/ca"
name: ca
readOnly: true
{{- end }}
{{- if .Values.tls.zookeeper.cacerts.enabled }}
- mountPath: "/pulsar/certs/cacerts"
name: zookeeper-cacerts
{{- range $cert := .Values.tls.zookeeper.cacerts.certs }}
- name: {{ $cert.name }}
mountPath: "/pulsar/certs/{{ $cert.name }}"
readOnly: true
{{- end }}
- name: certs-scripts
mountPath: "/pulsar/bin/certs-combine-pem.sh"
subPath: certs-combine-pem.sh
- name: certs-scripts
mountPath: "/pulsar/bin/certs-combine-pem-infinity.sh"
subPath: certs-combine-pem-infinity.sh
{{- end }}
{{- end }}
{{/*
Define zookeeper tls certs volumes
*/}}
{{- define "pulsar.zookeeper.certs.volumes" -}}
{{- if and .Values.tls.enabled .Values.tls.zookeeper.enabled }}
- name: zookeeper-certs
secret:
secretName: "{{ .Release.Name }}-{{ .Values.tls.zookeeper.cert_name }}"
items:
- key: tls.crt
path: tls.crt
- key: tls.key
path: tls.key
- key: tls-combined.pem
path: tls-combined.pem
- name: ca
secret:
secretName: "{{ template "pulsar.certs.issuers.ca.secretName" . }}"
items:
- key: ca.crt
path: ca.crt
{{- end }}
{{- if .Values.tls.zookeeper.cacerts.enabled }}
- name: zookeeper-cacerts
emptyDir: {}
{{- range $cert := .Values.tls.zookeeper.cacerts.certs }}
- name: {{ $cert.name }}
secret:
secretName: "{{ $cert.existingSecret }}"
items:
{{- range $key := $cert.secretKeys }}
- key: {{ $key }}
path: {{ $key }}
{{- end }}
{{- end }}
- name: certs-scripts
configMap:
name: "{{ template "pulsar.fullname" . }}-certs-scripts"
defaultMode: 0755
{{- end }}
{{- end }}

View File

@ -1,33 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.components.autorecovery }}
apiVersion: v1
kind: ConfigMap
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.autorecovery.component }}"
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.autorecovery.component }}
data:
# common config
{{- include "pulsar.bookkeeper.config.common" . | nindent 2 }}
{{ toYaml .Values.autorecovery.configData | indent 2 }}
{{- end }}

View File

@ -1,23 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# deploy autorecovery PodMonitor only when `$.Values.autorecovery.podMonitor.enabled` is true
{{- if $.Values.autorecovery.podMonitor.enabled }}
{{- include "pulsar.podMonitor" (list . "autorecovery" (printf "component: %s" .Values.autorecovery.component)) }}
{{- end }}

View File

@ -1,33 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.components.autorecovery }}
apiVersion: v1
kind: ServiceAccount
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.autorecovery.component }}"
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.autorecovery.component }}
{{- with .Values.autorecovery.service_account.annotations }}
annotations:
{{ toYaml . | indent 4 }}
{{- end }}
{{- end }}

View File

@ -1,42 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.components.autorecovery }}
apiVersion: v1
kind: Service
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.autorecovery.component }}"
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.autorecovery.component }}
{{- with .Values.autorecovery.service.annotations }}
annotations:
{{ toYaml . | indent 4 }}
{{- end }}
spec:
ports:
- name: http
port: {{ .Values.autorecovery.ports.http }}
clusterIP: None
selector:
{{- include "pulsar.matchLabels" . | nindent 4 }}
component: {{ .Values.autorecovery.component }}
{{- end }}

View File

@ -1,185 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.components.autorecovery }}
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.autorecovery.component }}"
namespace: {{ template "pulsar.namespace" . }}
annotations: {{ .Values.autorecovery.appAnnotations | toYaml | nindent 4 }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.autorecovery.component }}
spec:
serviceName: "{{ template "pulsar.fullname" . }}-{{ .Values.autorecovery.component }}"
replicas: {{ .Values.autorecovery.replicaCount }}
updateStrategy:
type: RollingUpdate
podManagementPolicy: Parallel
# nodeSelector:
selector:
matchLabels:
{{- include "pulsar.matchLabels" . | nindent 6 }}
component: {{ .Values.autorecovery.component }}
template:
metadata:
labels:
{{- include "pulsar.template.labels" . | nindent 8 }}
component: {{ .Values.autorecovery.component }}
annotations:
{{- if not .Values.autorecovery.podMonitor.enabled }}
prometheus.io/scrape: "true"
prometheus.io/port: "{{ .Values.autorecovery.ports.http }}"
{{- end }}
{{- if .Values.autorecovery.restartPodsOnConfigMapChange }}
checksum/config: {{ include (print $.Template.BasePath "/autorecovery-configmap.yaml") . | sha256sum }}
{{- end }}
{{- with .Values.autorecovery.annotations }}
{{ toYaml . | indent 8 }}
{{- end }}
spec:
{{- if .Values.autorecovery.nodeSelector }}
nodeSelector:
{{ toYaml .Values.autorecovery.nodeSelector | indent 8 }}
{{- end }}
{{- if .Values.autorecovery.tolerations }}
tolerations:
{{- with .Values.autorecovery.tolerations }}
{{ toYaml . | indent 8 }}
{{- end }}
{{- end }}
{{- if .Values.autorecovery.topologySpreadConstraints }}
topologySpreadConstraints:
{{- toYaml .Values.autorecovery.topologySpreadConstraints | nindent 8 }}
{{- end }}
affinity:
{{- if and .Values.affinity.anti_affinity .Values.autorecovery.affinity.anti_affinity}}
podAntiAffinity:
{{ if eq .Values.autorecovery.affinity.type "requiredDuringSchedulingIgnoredDuringExecution"}}
{{ .Values.autorecovery.affinity.type }}:
- labelSelector:
matchExpressions:
- key: "app"
operator: In
values:
- "{{ template "pulsar.name" . }}"
- key: "release"
operator: In
values:
- {{ .Release.Name }}
- key: "component"
operator: In
values:
- {{ .Values.autorecovery.component }}
topologyKey: {{ .Values.autorecovery.affinity.anti_affinity_topology_key }}
{{ else }}
{{ .Values.autorecovery.affinity.type }}:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: "app"
operator: In
values:
- "{{ template "pulsar.name" . }}"
- key: "release"
operator: In
values:
- {{ .Release.Name }}
- key: "component"
operator: In
values:
- {{ .Values.autorecovery.component }}
topologyKey: {{ .Values.autorecovery.affinity.anti_affinity_topology_key }}
{{ end }}
{{- end }}
terminationGracePeriodSeconds: {{ .Values.autorecovery.gracePeriod }}
serviceAccountName: "{{ template "pulsar.fullname" . }}-{{ .Values.autorecovery.component }}"
initContainers:
{{- if .Values.tls.autorecovery.cacerts.enabled }}
- name: cacerts
image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.autorecovery "root" .) }}"
imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.autorecovery "root" .) }}"
resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
command: ["sh", "-c"]
args:
- |
bin/certs-combine-pem.sh /pulsar/certs/cacerts/ca-combined.pem {{ template "pulsar.certs.cacerts" (dict "certs" .Values.tls.autorecovery.cacerts.certs) }}
volumeMounts:
{{- include "pulsar.autorecovery.certs.volumeMounts" . | nindent 8 }}
{{- end }}
{{- if and .Values.autorecovery.waitBookkeeperTimeout (gt (.Values.autorecovery.waitBookkeeperTimeout | int) 0) }}
# This initContainer will wait for bookkeeper initnewcluster to complete
# before deploying the bookies
- name: pulsar-bookkeeper-verify-clusterid
image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.autorecovery "root" .) }}"
imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.autorecovery "root" .) }}"
resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
command: ["timeout", "{{ .Values.autorecovery.waitBookkeeperTimeout }}", "sh", "-c"]
args:
- |
{{- include "pulsar.autorecovery.init.verify_cluster_id" . | nindent 10 }}
envFrom:
- configMapRef:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.autorecovery.component }}"
volumeMounts:
{{- if .Values.autorecovery.extraVolumeMounts }}
{{ toYaml .Values.autorecovery.extraVolumeMounts | indent 8 }}
{{- end }}
{{- include "pulsar.autorecovery.certs.volumeMounts" . | nindent 8 }}
{{- end }}
{{- if .Values.autorecovery.initContainers }}
{{- toYaml .Values.autorecovery.initContainers | nindent 6 }}
{{- end }}
containers:
- name: "{{ template "pulsar.fullname" . }}-{{ .Values.autorecovery.component }}"
image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.autorecovery "root" .) }}"
imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.autorecovery "root" .) }}"
{{- if .Values.autorecovery.resources }}
resources:
{{ toYaml .Values.autorecovery.resources | indent 10 }}
{{- end }}
command: ["sh", "-c"]
args:
- |
{{- if .Values.tls.autorecovery.cacerts.enabled }}
cd /pulsar/certs/cacerts;
nohup /pulsar/bin/certs-combine-pem-infinity.sh /pulsar/certs/cacerts/ca-combined.pem {{ template "pulsar.certs.cacerts" (dict "certs" .Values.tls.autorecovery.cacerts.certs) }} > /pulsar/certs/cacerts/certs-combine-pem-infinity.log 2>&1 &
cd /pulsar;
{{- end }}
bin/apply-config-from-env.py conf/bookkeeper.conf;
{{- include "pulsar.autorecovery.zookeeper.tls.settings" . | nindent 10 }}
OPTS="${OPTS} -Dlog4j2.formatMsgNoLookups=true" exec bin/bookkeeper autorecovery
ports:
- name: http
containerPort: {{ .Values.autorecovery.ports.http }}
envFrom:
- configMapRef:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.autorecovery.component }}"
volumeMounts:
{{- include "pulsar.autorecovery.certs.volumeMounts" . | nindent 8 }}
volumes:
{{- include "pulsar.autorecovery.certs.volumes" . | nindent 6 }}
{{- if .Values.autorecovery.extraVolumes }}
{{ toYaml .Values.autorecovery.extraVolumes | indent 6 }}
{{- end }}
{{- include "pulsar.imagePullSecrets" . | nindent 6}}
{{- end }}

View File

@ -1,129 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if or (and .Values.useReleaseStatus .Release.IsInstall) .Values.initialize }}
{{- if .Values.components.bookkeeper }}
apiVersion: batch/v1
kind: Job
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-init"
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: "{{ .Values.bookkeeper.component }}-init"
spec:
# This feature was previously behind a feature gate for several Kubernetes versions and will default to true in 1.23 and beyond
# https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/
{{- if and .Values.job.ttl.enabled (semverCompare ">=1.23-0" .Capabilities.KubeVersion.Version) }}
ttlSecondsAfterFinished: {{ .Values.job.ttl.secondsAfterFinished | default 600 }}
{{- end }}
template:
metadata:
labels:
{{- include "pulsar.template.labels" . | nindent 8 }}
component: {{ .Values.bookkeeper.component }}-init
spec:
{{- include "pulsar.imagePullSecrets" . | nindent 6 }}
serviceAccountName: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
{{- if .Values.pulsar_metadata.nodeSelector }}
nodeSelector:
{{ toYaml .Values.pulsar_metadata.nodeSelector | indent 8 }}
{{- end }}
{{- with .Values.pulsar_metadata.tolerations }}
tolerations:
{{ toYaml .Values.pulsar_metadata.tolerations | indent 8 }}
{{- end }}
initContainers:
{{- if .Values.tls.bookie.cacerts.enabled }}
- name: cacerts
image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.bookie "root" .) }}"
imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.bookie "root" .) }}"
resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
command: ["sh", "-c"]
args:
- |
bin/certs-combine-pem.sh /pulsar/certs/cacerts/ca-combined.pem {{ template "pulsar.certs.cacerts" (dict "certs" .Values.tls.bookie.cacerts.certs) }}
volumeMounts:
{{- include "pulsar.toolset.certs.volumeMounts" . | nindent 8 }}
{{- end }}
{{- if and .Values.components.zookeeper .Values.bookkeeper.metadata.waitZookeeperTimeout (gt (.Values.bookkeeper.metadata.waitZookeeperTimeout | int) 0) }}
- name: wait-zookeeper-ready
image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.bookie "root" .) }}"
imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.bookie "root" .) }}"
resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
command: ["timeout", "{{ .Values.bookkeeper.metadata.waitZookeeperTimeout }}", "sh", "-c"]
args:
- |
{{- if $zk:=.Values.pulsar_metadata.userProvidedZookeepers }}
export PULSAR_MEM="-Xmx128M";
until timeout 15 bin/pulsar zookeeper-shell -server {{ $zk }} ls {{ or .Values.metadataPrefix "/" }}; do
echo "user provided zookeepers {{ $zk }} are unreachable... check in 3 seconds ..." && sleep 3;
done;
{{ else }}
until nslookup {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-{{ add (.Values.zookeeper.replicaCount | int) -1 }}.{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}.{{ template "pulsar.namespace" . }}; do
sleep 3;
done;
{{- end}}
{{- end}}
{{- if and .Values.components.oxia .Values.bookkeeper.metadata.waitOxiaTimeout (gt (.Values.bookkeeper.metadata.waitOxiaTimeout | int) 0) }}
- name: wait-oxia-ready
image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.bookie "root" .) }}"
imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.bookie "root" .) }}"
resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
command: ["timeout", "{{ .Values.bookkeeper.metadata.waitOxiaTimeout }}", "sh", "-c"]
args:
- |
until nslookup {{ template "pulsar.oxia.server.service" . }}; do
sleep 3;
done;
{{- end }}
containers:
- name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-init"
image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.bookie "root" .) }}"
imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.bookie "root" .) }}"
{{- if .Values.bookkeeper.metadata.resources }}
resources:
{{ toYaml .Values.bookkeeper.metadata.resources | indent 10 }}
{{- end }}
command: ["timeout", "{{ .Values.bookkeeper.metadata.initTimeout | default 60 }}", "sh", "-c"]
args:
- |
bin/apply-config-from-env.py conf/bookkeeper.conf;
{{- include "pulsar.toolset.zookeeper.tls.settings" . | nindent 12 }}
export BOOKIE_MEM="-Xmx128M";
if timeout 15 bin/bookkeeper shell whatisinstanceid; then
echo "bookkeeper cluster already initialized";
else
{{- if and .Values.components.zookeeper (not (eq .Values.metadataPrefix "")) }}
bin/pulsar zookeeper-shell -server {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }} create {{ .Values.metadataPrefix }} && echo 'created for pulsar cluster "{{ template "pulsar.cluster.name" . }}"' &&
{{- end }}
bin/bookkeeper shell initnewcluster;
fi
{{- if .Values.extraInitCommand }}
{{ .Values.extraInitCommand }}
{{- end }}
envFrom:
- configMapRef:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
volumeMounts:
{{- include "pulsar.toolset.certs.volumeMounts" . | nindent 8 }}
volumes:
{{- include "pulsar.toolset.certs.volumes" . | nindent 6 }}
restartPolicy: OnFailure
{{- end }}
{{- end }}

View File

@ -1,65 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.components.bookkeeper }}
apiVersion: v1
kind: ConfigMap
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.bookkeeper.component }}
data:
# common config
{{- include "pulsar.bookkeeper.config.common" . | nindent 2 }}
{{- if .Values.components.autorecovery }}
# disable auto recovery on bookies since we will start AutoRecovery in separated pods
autoRecoveryDaemonEnabled: "false"
{{- end }}
# Do not retain journal files as it increase the disk utilization
journalMaxBackups: "0"
{{- if .Values.bookkeeper.volumes.journal.useMultiVolumes }}
{{- $journalDirs := list -}}
{{ range .Values.bookkeeper.volumes.journal.multiVolumes }}
{{- $journalDirs = append $journalDirs .mountPath -}}
{{- end }}
journalDirectories: {{ $journalDirs | join "," | quote }}
PULSAR_PREFIX_journalDirectories: {{ $journalDirs | join "," | quote }}
{{- else }}
journalDirectories: "/pulsar/data/bookkeeper/journal"
PULSAR_PREFIX_journalDirectories: "/pulsar/data/bookkeeper/journal"
{{- end }}
{{- if .Values.bookkeeper.volumes.ledgers.useMultiVolumes }}
{{- $ledgerDirs := list -}}
{{ range .Values.bookkeeper.volumes.ledgers.multiVolumes }}
{{- $ledgerDirs = append $ledgerDirs .mountPath -}}
{{- end }}
ledgerDirectories: {{ $ledgerDirs | join "," | quote }}
{{- else }}
ledgerDirectories: "/pulsar/data/bookkeeper/ledgers"
{{- end }}
{{- if .Values.functions.useBookieAsStateStore }}
# Stateful function config
extraServerComponents: "org.apache.bookkeeper.stream.server.StreamStorageLifecycleComponent"
{{- end }}
# TLS config
{{- include "pulsar.bookkeeper.config.tls" . | nindent 2 }}
{{ toYaml .Values.bookkeeper.configData | indent 2 }}
{{- end }}

View File

@ -1,42 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.components.bookkeeper }}
{{- if .Values.bookkeeper.pdb.usePolicy }}
# pdb version detection
{{- if semverCompare "<1.21-0" .Capabilities.KubeVersion.Version }}
apiVersion: policy/v1beta1
{{- else }}
apiVersion: policy/v1
{{- end }}
kind: PodDisruptionBudget
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.bookkeeper.component }}
spec:
selector:
matchLabels:
{{- include "pulsar.matchLabels" . | nindent 6 }}
component: {{ .Values.bookkeeper.component }}
maxUnavailable: {{ .Values.bookkeeper.pdb.maxUnavailable }}
{{- end }}
{{- end }}

View File

@ -1,23 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# deploy bookkeeper PodMonitor only when `$.Values.bookkeeper.podMonitor.enabled` is true
{{- if $.Values.bookkeeper.podMonitor.enabled }}
{{- include "pulsar.podMonitor" (list . "bookkeeper" (printf "component: %s" .Values.bookkeeper.component)) }}
{{- end }}

View File

@ -1,33 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.components.bookkeeper }}
apiVersion: v1
kind: ServiceAccount
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.bookkeeper.component }}
{{- with .Values.bookkeeper.service_account.annotations }}
annotations:
{{ toYaml . | indent 4 }}
{{- end }}
{{- end }}

View File

@ -1,46 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.components.bookkeeper }}
apiVersion: v1
kind: Service
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.bookkeeper.component }}
{{- with .Values.bookkeeper.service.annotations }}
annotations:
{{ toYaml . | indent 4 }}
{{- end }}
spec:
ports:
- name: "{{ .Values.tcpPrefix }}bookie"
port: {{ .Values.bookkeeper.ports.bookie }}
- name: http
port: {{ .Values.bookkeeper.ports.http }}
clusterIP: None
selector:
{{- include "pulsar.matchLabels" . | nindent 4 }}
component: {{ .Values.bookkeeper.component }}
{{- if .Values.bookkeeper.service.spec }}
{{- toYaml .Values.bookkeeper.service.spec | trim | nindent 2 }}
{{- end }}
{{- end }}

View File

@ -1,372 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.components.bookkeeper }}
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
namespace: {{ template "pulsar.namespace" . }}
annotations: {{ .Values.bookkeeper.appAnnotations | toYaml | nindent 4 }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.bookkeeper.component }}
spec:
serviceName: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
replicas: {{ .Values.bookkeeper.replicaCount }}
selector:
matchLabels:
{{- include "pulsar.matchLabels" . | nindent 6 }}
component: {{ .Values.bookkeeper.component }}
updateStrategy:
{{ toYaml .Values.bookkeeper.updateStrategy | indent 4 }}
podManagementPolicy: {{ .Values.bookkeeper.podManagementPolicy }}
template:
metadata:
labels:
{{- include "pulsar.template.labels" . | nindent 8 }}
component: {{ .Values.bookkeeper.component }}
annotations:
{{- if not .Values.bookkeeper.podMonitor.enabled }}
prometheus.io/scrape: "true"
prometheus.io/port: "{{ .Values.bookkeeper.ports.http }}"
{{- end }}
{{- if .Values.bookkeeper.restartPodsOnConfigMapChange }}
checksum/config: {{ include (print $.Template.BasePath "/bookkeeper-configmap.yaml") . | sha256sum }}
{{- end }}
{{- with .Values.bookkeeper.annotations }}
{{ toYaml . | indent 8 }}
{{- end }}
spec:
{{- if .Values.bookkeeper.nodeSelector }}
nodeSelector:
{{ toYaml .Values.bookkeeper.nodeSelector | indent 8 }}
{{- end }}
{{- if .Values.bookkeeper.tolerations }}
tolerations:
{{ toYaml .Values.bookkeeper.tolerations | indent 8 }}
{{- end }}
{{- if .Values.bookkeeper.topologySpreadConstraints }}
topologySpreadConstraints:
{{- toYaml .Values.bookkeeper.topologySpreadConstraints | nindent 8 }}
{{- end }}
affinity:
{{- if and .Values.affinity.anti_affinity .Values.bookkeeper.affinity.anti_affinity}}
podAntiAffinity:
{{- if eq .Values.bookkeeper.affinity.type "requiredDuringSchedulingIgnoredDuringExecution"}}
{{ .Values.bookkeeper.affinity.type }}:
- labelSelector:
matchExpressions:
- key: "app"
operator: In
values:
- "{{ template "pulsar.name" . }}"
- key: "release"
operator: In
values:
- {{ .Release.Name }}
- key: "component"
operator: In
values:
- {{ .Values.bookkeeper.component }}
topologyKey: {{ .Values.bookkeeper.affinity.anti_affinity_topology_key }}
{{- else }}
{{ .Values.bookkeeper.affinity.type }}:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: "app"
operator: In
values:
- "{{ template "pulsar.name" . }}"
- key: "release"
operator: In
values:
- {{ .Release.Name }}
- key: "component"
operator: In
values:
- {{ .Values.bookkeeper.component }}
topologyKey: {{ .Values.bookkeeper.affinity.anti_affinity_topology_key }}
{{ end }}
{{- end }}
terminationGracePeriodSeconds: {{ .Values.bookkeeper.gracePeriod }}
serviceAccountName: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
{{- if .Values.bookkeeper.securityContext }}
securityContext:
{{ toYaml .Values.bookkeeper.securityContext | indent 8 }}
{{- end }}
{{- if and .Values.bookkeeper.waitMetadataTimeout (gt (.Values.bookkeeper.waitMetadataTimeout | int) 0) }}
initContainers:
{{- if .Values.tls.bookie.cacerts.enabled }}
- name: cacerts
image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.bookie "root" .) }}"
imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.bookie "root" .) }}"
resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
command: ["sh", "-c"]
args:
- |
bin/certs-combine-pem.sh /pulsar/certs/cacerts/ca-combined.pem {{ template "pulsar.certs.cacerts" (dict "certs" .Values.tls.bookie.cacerts.certs) }}
volumeMounts:
{{- include "pulsar.bookkeeper.certs.volumeMounts" . | nindent 8 }}
{{- end }}
# This initContainer will wait for bookkeeper initnewcluster to complete
# before deploying the bookies
- name: pulsar-bookkeeper-verify-clusterid
image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.bookie "root" .) }}"
imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.bookie "root" .) }}"
resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
command: ["timeout", "{{ .Values.bookkeeper.waitMetadataTimeout }}", "sh", "-c"]
args:
# only reformat bookie if bookkeeper is running without persistence
- |
{{- include "pulsar.bookkeeper.init.verify_cluster_id" . | nindent 10 }}
envFrom:
- configMapRef:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
volumeMounts:
{{- include "pulsar.bookkeeper.certs.volumeMounts" . | nindent 8 }}
{{- end }}
{{- if .Values.bookkeeper.initContainers }}
{{- toYaml .Values.bookkeeper.initContainers | nindent 6 }}
{{- end }}
containers:
- name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.bookie "root" .) }}"
imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.bookie "root" .) }}"
{{- if .Values.bookkeeper.probe.liveness.enabled }}
livenessProbe:
httpGet:
path: /api/v1/bookie/state
port: {{ .Values.bookkeeper.ports.http }}
initialDelaySeconds: {{ .Values.bookkeeper.probe.liveness.initialDelaySeconds }}
periodSeconds: {{ .Values.bookkeeper.probe.liveness.periodSeconds }}
timeoutSeconds: {{ .Values.bookkeeper.probe.liveness.timeoutSeconds }}
failureThreshold: {{ .Values.bookkeeper.probe.liveness.failureThreshold }}
{{- end }}
{{- if .Values.bookkeeper.probe.readiness.enabled }}
readinessProbe:
httpGet:
path: /api/v1/bookie/is_ready
port: {{ .Values.bookkeeper.ports.http }}
initialDelaySeconds: {{ .Values.bookkeeper.probe.readiness.initialDelaySeconds }}
periodSeconds: {{ .Values.bookkeeper.probe.readiness.periodSeconds }}
timeoutSeconds: {{ .Values.bookkeeper.probe.readiness.timeoutSeconds }}
failureThreshold: {{ .Values.bookkeeper.probe.readiness.failureThreshold }}
{{- end }}
{{- if .Values.bookkeeper.probe.startup.enabled }}
startupProbe:
httpGet:
path: /api/v1/bookie/is_ready
port: {{ .Values.bookkeeper.ports.http }}
initialDelaySeconds: {{ .Values.bookkeeper.probe.startup.initialDelaySeconds }}
periodSeconds: {{ .Values.bookkeeper.probe.startup.periodSeconds }}
timeoutSeconds: {{ .Values.bookkeeper.probe.startup.timeoutSeconds }}
failureThreshold: {{ .Values.bookkeeper.probe.startup.failureThreshold }}
{{- end }}
{{- if .Values.bookkeeper.resources }}
resources:
{{ toYaml .Values.bookkeeper.resources | indent 10 }}
{{- end }}
command: ["sh", "-c"]
args:
- |
# set required environment variables to use rocksdb config files provided in the Pulsar image
export PULSAR_PREFIX_defaultRocksdbConf=${PULSAR_PREFIX_defaultRocksdbConf:-conf/default_rocksdb.conf}
export PULSAR_PREFIX_entryLocationRocksdbConf=${PULSAR_PREFIX_entryLocationRocksdbConf:-conf/entry_location_rocksdb.conf}
export PULSAR_PREFIX_ledgerMetadataRocksdbConf=${PULSAR_PREFIX_ledgerMetadataRocksdbConf:-conf/ledger_metadata_rocksdb.conf}
if [ -x bin/update-rocksdb-conf-from-env.py ] && [ -f "${PULSAR_PREFIX_entryLocationRocksdbConf}" ]; then
echo "Updating ${PULSAR_PREFIX_entryLocationRocksdbConf} from environment variables starting with dbStorage_rocksDB_*"
bin/update-rocksdb-conf-from-env.py "${PULSAR_PREFIX_entryLocationRocksdbConf}"
else
# Ensure that Bookkeeper will not load RocksDB config from existing files and fallback to use default RocksDB config
# See https://github.com/apache/bookkeeper/pull/3523 as reference
export PULSAR_PREFIX_defaultRocksdbConf=conf/non_existing_default_rocksdb.conf
export PULSAR_PREFIX_entryLocationRocksdbConf=conf/non_existing_entry_location_rocksdb.conf
export PULSAR_PREFIX_ledgerMetadataRocksdbConf=conf/non_existing_ledger_metadata_rocksdb.conf
# Ensure that Bookkeeper will use RocksDB format_version 5 (this currently applies only to the entry location rocksdb due to a bug in Bookkeeper)
export PULSAR_PREFIX_dbStorage_rocksDB_format_version=${PULSAR_PREFIX_dbStorage_rocksDB_format_version:-5}
fi
{{- if .Values.bookkeeper.additionalCommand }}
{{ .Values.bookkeeper.additionalCommand }}
{{- end }}
{{- if .Values.tls.bookie.cacerts.enabled }}
cd /pulsar/certs/cacerts;
nohup /pulsar/bin/certs-combine-pem-infinity.sh /pulsar/certs/cacerts/ca-combined.pem {{ template "pulsar.certs.cacerts" (dict "certs" .Values.tls.bookie.cacerts.certs) }} > /pulsar/certs/cacerts/certs-combine-pem-infinity.log 2>&1 &
cd /pulsar;
{{- end }}
bin/apply-config-from-env.py conf/bookkeeper.conf;
{{- include "pulsar.bookkeeper.zookeeper.tls.settings" . | nindent 10 }}
OPTS="${OPTS} -Dlog4j2.formatMsgNoLookups=true" exec bin/pulsar bookie;
ports:
- name: "{{ .Values.tcpPrefix }}bookie"
containerPort: {{ .Values.bookkeeper.ports.bookie }}
- name: http
containerPort: {{ .Values.bookkeeper.ports.http }}
envFrom:
- configMapRef:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
volumeMounts:
{{- if .Values.bookkeeper.volumes.useSingleCommonVolume }}
- name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.common.name }}"
mountPath: /pulsar/data/bookkeeper
{{- else }}
{{- if .Values.bookkeeper.volumes.journal.useMultiVolumes }}
{{- $fullname := include "pulsar.fullname" . -}}
{{- $bkComponent := .Values.bookkeeper.component -}}
{{ range .Values.bookkeeper.volumes.journal.multiVolumes }}
- name: "{{ $fullname }}-{{ $bkComponent }}-{{ .name }}"
mountPath: {{ .mountPath }}
{{- end }}
{{- else }}
- name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.journal.name }}"
mountPath: /pulsar/data/bookkeeper/journal
{{- end}}
{{- if .Values.bookkeeper.volumes.ledgers.useMultiVolumes }}
{{- $fullname := include "pulsar.fullname" . -}}
{{- $bkComponent := .Values.bookkeeper.component -}}
{{ range .Values.bookkeeper.volumes.ledgers.multiVolumes }}
- name: "{{ $fullname }}-{{ $bkComponent }}-{{ .name }}"
mountPath: {{ .mountPath }}
{{- end }}
{{- else }}
- name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.ledgers.name }}"
mountPath: /pulsar/data/bookkeeper/ledgers
{{- end }}
{{- end }}
{{- if .Values.bookkeeper.extraVolumeMounts }}
{{ toYaml .Values.bookkeeper.extraVolumeMounts | indent 8 }}
{{- end }}
{{- include "pulsar.bookkeeper.certs.volumeMounts" . | nindent 8 }}
volumes:
{{- if not (and (and .Values.persistence .Values.volumes.persistence) .Values.bookkeeper.volumes.persistence) }}
- name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.journal.name }}"
emptyDir: {}
- name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.ledgers.name }}"
emptyDir: {}
{{- end }}
{{- include "pulsar.bookkeeper.certs.volumes" . | nindent 6 }}
{{- if .Values.bookkeeper.extraVolumes }}
{{ toYaml .Values.bookkeeper.extraVolumes | indent 6 }}
{{- end }}
{{- include "pulsar.imagePullSecrets" . | nindent 6}}
{{- if and (and .Values.persistence .Values.volumes.persistence) .Values.bookkeeper.volumes.persistence}}
volumeClaimTemplates:
{{- if .Values.bookkeeper.volumes.useSingleCommonVolume }}
- metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.common.name }}"
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: {{ .Values.bookkeeper.volumes.common.size }}
{{- if .Values.bookkeeper.volumes.common.storageClassName }}
storageClassName: "{{ .Values.bookkeeper.volumes.common.storageClassName }}"
{{- else if and (not (and .Values.volumes.local_storage .Values.bookkeeper.volumes.common.local_storage)) .Values.bookkeeper.volumes.common.storageClass }}
storageClassName: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.common.name }}"
{{- else if and .Values.volumes.local_storage .Values.bookkeeper.volumes.common.local_storage }}
storageClassName: "local-storage"
{{- end }}
{{- with .Values.bookkeeper.volumes.common.selector }}
selector:
{{- toYaml . | nindent 10 }}
{{- end }}
{{- else }}
{{- if .Values.bookkeeper.volumes.journal.useMultiVolumes }}
{{- $fullname := include "pulsar.fullname" . -}}
{{- $bkComponent := .Values.bookkeeper.component -}}
{{ range .Values.bookkeeper.volumes.journal.multiVolumes }}
- metadata:
name: "{{ $fullname }}-{{ $bkComponent }}-{{ .name }}"
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: {{ .size }}
{{- if .storageClassName }}
storageClassName: "{{ .storageClassName }}"
{{- end }}
{{- with $.Values.bookkeeper.volumes.journal.selector }}
selector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}
{{- else }}
- metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.journal.name }}"
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: {{ .Values.bookkeeper.volumes.journal.size }}
{{- if .Values.bookkeeper.volumes.journal.storageClassName }}
storageClassName: "{{ .Values.bookkeeper.volumes.journal.storageClassName }}"
{{- else if and (not (and .Values.volumes.local_storage .Values.bookkeeper.volumes.journal.local_storage)) .Values.bookkeeper.volumes.journal.storageClass }}
storageClassName: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.journal.name }}"
{{- else if and .Values.volumes.local_storage .Values.bookkeeper.volumes.journal.local_storage }}
storageClassName: "local-storage"
{{- end }}
{{- with .Values.bookkeeper.volumes.journal.selector }}
selector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}
{{- if .Values.bookkeeper.volumes.ledgers.useMultiVolumes }}
{{- $fullname := include "pulsar.fullname" . -}}
{{- $bkComponent := .Values.bookkeeper.component -}}
{{ range .Values.bookkeeper.volumes.ledgers.multiVolumes }}
- metadata:
name: "{{ $fullname }}-{{ $bkComponent }}-{{ .name }}"
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: {{ .size }}
{{- if .storageClassName }}
storageClassName: "{{ .storageClassName }}"
{{- end }}
{{- with $.Values.bookkeeper.volumes.ledgers.selector }}
selector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}
{{- else }}
- metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.ledgers.name }}"
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: {{ .Values.bookkeeper.volumes.ledgers.size }}
{{- if .Values.bookkeeper.volumes.ledgers.storageClassName }}
storageClassName: "{{ .Values.bookkeeper.volumes.ledgers.storageClassName }}"
{{- else if and (not (and .Values.volumes.local_storage .Values.bookkeeper.volumes.ledgers.local_storage)) .Values.bookkeeper.volumes.ledgers.storageClass }}
storageClassName: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.ledgers.name }}"
{{- else if and .Values.volumes.local_storage .Values.bookkeeper.volumes.ledgers.local_storage }}
storageClassName: "local-storage"
{{- end }}
{{- with .Values.bookkeeper.volumes.ledgers.selector }}
selector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}

View File

@ -1,74 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.components.bookkeeper }}
{{- if and (and .Values.persistence .Values.volumes.persistence) .Values.bookkeeper.volumes.persistence }}
{{- if not .Values.volumes.local_storage }}
{{- if .Values.bookkeeper.volumes.useSingleCommonVolume}}
{{- if and (not .Values.bookkeeper.volumes.common.local_storage) .Values.bookkeeper.volumes.common.storageClass }}
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.common.name }}"
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.bookkeeper.component }}
provisioner: {{ .Values.bookkeeper.volumes.common.storageClass.provisioner }}
parameters:
type: {{ .Values.bookkeeper.volumes.common.storageClass.type }}
fsType: {{ .Values.bookkeeper.volumes.common.storageClass.fsType }}
{{- end }}
{{- else }}
{{- if and (not .Values.bookkeeper.volumes.journal.local_storage) .Values.bookkeeper.volumes.journal.storageClass }}
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.journal.name }}"
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.bookkeeper.component }}
provisioner: {{ .Values.bookkeeper.volumes.journal.storageClass.provisioner }}
parameters:
type: {{ .Values.bookkeeper.volumes.journal.storageClass.type }}
fsType: {{ .Values.bookkeeper.volumes.journal.storageClass.fsType }}
{{- end }}
---
{{- if and (not .Values.bookkeeper.volumes.ledgers.local_storage) .Values.bookkeeper.volumes.ledgers.storageClass }}
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.ledgers.name }}"
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.bookkeeper.component }}
provisioner: {{ .Values.bookkeeper.volumes.ledgers.storageClass.provisioner }}
parameters:
type: {{ .Values.bookkeeper.volumes.ledgers.storageClass.type }}
fsType: {{ .Values.bookkeeper.volumes.ledgers.storageClass.fsType }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}

View File

@ -1,93 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.components.broker }}
## TODO create our own cluster role with less privledges than admin
apiVersion: rbac.authorization.k8s.io/v1
{{- if .Values.rbac.limit_to_namespace }}
kind: RoleBinding
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}-rolebinding"
{{- else}}
kind: ClusterRoleBinding
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}-clusterrolebinding"
{{- end}}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
roleRef:
apiGroup: rbac.authorization.k8s.io
{{- if .Values.rbac.limit_to_namespace }}
kind: Role
name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}-role"
{{- else}}
kind: ClusterRole
name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}-clusterrole"
{{- end}}
subjects:
- kind: ServiceAccount
name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}-acct"
namespace: {{ template "pulsar.namespace" . }}
---
apiVersion: rbac.authorization.k8s.io/v1
{{- if .Values.rbac.limit_to_namespace }}
kind: Role
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}-role"
{{- else}}
kind: ClusterRole
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}-clusterrole"
{{- end}}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
rules:
- apiGroups: [""]
resources:
- configmaps
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources:
- pods
- services
- secrets
verbs:
- list
- watch
- get
- update
- create
- delete
- patch
- apiGroups: ["apps"]
resources:
- deployments
- statefulsets
verbs:
- list
- watch
- get
- update
- create
- delete
- patch
---
{{- end }}

View File

@ -1,275 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.components.broker }}
apiVersion: v1
kind: ConfigMap
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}"
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.broker.component }}
data:
# Metadata settings
{{- if .Values.components.zookeeper }}
metadataStoreUrl: "zk:{{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }}"
{{- $configMetadataStoreUrl := "" }}
{{- if .Values.pulsar_metadata.configurationStore }}
{{- $configMetadataStoreUrl = printf "zk:%s%s" (include "pulsar.configurationStore.connect" .) .Values.pulsar_metadata.configurationStoreMetadataPrefix }}
{{- else }}
{{- $configMetadataStoreUrl = printf "zk:%s%s" (include "pulsar.zookeeper.connect" .) .Values.metadataPrefix }}
{{- end }}
configurationMetadataStoreUrl: "{{ $configMetadataStoreUrl }}"
{{- if .Values.pulsar_metadata.bookkeeper.usePulsarMetadataClientDriver }}
bookkeeperMetadataServiceUri: "metadata-store:{{ $configMetadataStoreUrl }}/ledgers"
{{- else }}
bookkeeperMetadataServiceUri: "zk+hierarchical://{{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }}/ledgers"
{{- end }}
{{- end }}
{{- if .Values.components.oxia }}
metadataStoreUrl: "{{ template "pulsar.oxia.metadata.url.broker" . }}"
configurationMetadataStoreUrl: "{{ template "pulsar.oxia.metadata.url.broker" . }}"
bookkeeperMetadataServiceUri: "{{ template "pulsar.oxia.metadata.url.bookkeeper" . }}"
{{- end }}
{{- if hasKey .Values.pulsar_metadata "metadataStoreAllowReadOnlyOperations" }}
PULSAR_PREFIX_metadataStoreAllowReadOnlyOperations: "{{ .Values.pulsar_metadata.metadataStoreAllowReadOnlyOperations }}"
{{- end }}
{{- if hasKey .Values.pulsar_metadata "metadataStoreSessionTimeoutMillis" }}
metadataStoreSessionTimeoutMillis: "{{ .Values.pulsar_metadata.metadataStoreSessionTimeoutMillis }}"
{{- end }}
{{- if hasKey .Values.pulsar_metadata "metadataStoreOperationTimeoutSeconds" }}
metadataStoreOperationTimeoutSeconds: "{{ .Values.pulsar_metadata.metadataStoreOperationTimeoutSeconds }}"
{{- end }}
{{- if hasKey .Values.pulsar_metadata "metadataStoreCacheExpirySeconds" }}
metadataStoreCacheExpirySeconds: "{{ .Values.pulsar_metadata.metadataStoreCacheExpirySeconds }}"
{{- end }}
{{- if hasKey .Values.pulsar_metadata "metadataStoreBatchingEnabled" }}
metadataStoreBatchingEnabled: "{{ .Values.pulsar_metadata.metadataStoreBatchingEnabled }}"
{{- end }}
{{- if hasKey .Values.pulsar_metadata "metadataStoreBatchingMaxDelayMillis" }}
metadataStoreBatchingMaxDelayMillis: "{{ .Values.pulsar_metadata.metadataStoreBatchingMaxDelayMillis }}"
{{- end }}
{{- if hasKey .Values.pulsar_metadata "metadataStoreBatchingMaxOperations" }}
metadataStoreBatchingMaxOperations: "{{ .Values.pulsar_metadata.metadataStoreBatchingMaxOperations }}"
{{- end }}
{{- if hasKey .Values.pulsar_metadata "metadataStoreBatchingMaxSizeKb" }}
metadataStoreBatchingMaxSizeKb: "{{ .Values.pulsar_metadata.metadataStoreBatchingMaxSizeKb }}"
{{- end }}
# Broker settings
clusterName: {{ template "pulsar.cluster.name" . }}
# Enable all metrics by default
exposeTopicLevelMetricsInPrometheus: "true"
exposeConsumerLevelMetricsInPrometheus: "true"
exposeProducerLevelMetricsInPrometheus: "true"
exposeManagedLedgerMetricsInPrometheus: "true"
exposeManagedCursorMetricsInPrometheus: "true"
exposeBundlesMetricsInPrometheus: "true"
exposePublisherStats: "true"
exposePreciseBacklogInPrometheus: "true"
replicationMetricsEnabled: "true"
splitTopicAndPartitionLabelInPrometheus: "true"
aggregatePublisherStatsByProducerName: "true"
bookkeeperClientExposeStatsToPrometheus: "true"
numHttpServerThreads: "8"
statusFilePath: "{{ template "pulsar.home" . }}/logs/status"
# Tiered storage settings
{{- if .Values.broker.storageOffload.driver }}
{{- if eq .Values.broker.storageOffload.driver "aws-s3" }}
managedLedgerOffloadDriver: "{{ .Values.broker.storageOffload.driver }}"
s3ManagedLedgerOffloadBucket: "{{ .Values.broker.storageOffload.bucket }}"
s3ManagedLedgerOffloadRegion: "{{ .Values.broker.storageOffload.region }}"
{{- if .Values.broker.storageOffload.managedLedgerOffloadAutoTriggerSizeThresholdBytes }}
PULSAR_PREFIX_managedLedgerOffloadThresholdInBytes: "{{ .Values.broker.storageOffload.managedLedgerOffloadAutoTriggerSizeThresholdBytes }}"
{{- end }}
{{- if .Values.broker.storageOffload.managedLedgerOffloadDeletionLagMs }}
PULSAR_PREFIX_managedLedgerOffloadDeletionLagInMillis: "{{ .Values.broker.storageOffload.managedLedgerOffloadDeletionLagMs }}"
{{- end }}
{{- if .Values.broker.storageOffload.maxBlockSizeInBytes }}
s3ManagedLedgerOffloadMaxBlockSizeInBytes: "{{ .Values.broker.storageOffload.maxBlockSizeInBytes }}"
{{- end }}
{{- if .Values.broker.storageOffload.readBufferSizeInBytes }}
s3ManagedLedgerOffloadReadBufferSizeInBytes: "{{ .Values.broker.storageOffload.readBufferSizeInBytes }}"
{{- end }}
{{- end }}
{{- if eq .Values.broker.storageOffload.driver "google-cloud-storage" }}
managedLedgerOffloadDriver: "{{ .Values.broker.storageOffload.driver }}"
gcsManagedLedgerOffloadBucket: "{{ .Values.broker.storageOffload.bucket }}"
gcsManagedLedgerOffloadRegion: "{{ .Values.broker.storageOffload.region }}"
gcsManagedLedgerOffloadServiceAccountKeyFile: "/pulsar/gcp-service-account/{{ .Values.broker.storageOffload.gcsServiceAccountJsonFile }}"
{{- if .Values.broker.storageOffload.managedLedgerOffloadAutoTriggerSizeThresholdBytes }}
PULSAR_PREFIX_managedLedgerOffloadThresholdInBytes: "{{ .Values.broker.storageOffload.managedLedgerOffloadAutoTriggerSizeThresholdBytes }}"
{{- end }}
{{- if .Values.broker.storageOffload.managedLedgerOffloadDeletionLagMs }}
PULSAR_PREFIX_managedLedgerOffloadDeletionLagInMillis: "{{ .Values.broker.storageOffload.managedLedgerOffloadDeletionLagMs }}"
{{- end }}
{{- if .Values.broker.storageOffload.maxBlockSizeInBytes }}
gcsManagedLedgerOffloadMaxBlockSizeInBytes: "{{ .Values.broker.storageOffload.maxBlockSizeInBytes }}"
{{- end }}
{{- if .Values.broker.storageOffload.readBufferSizeInBytes }}
gcsManagedLedgerOffloadReadBufferSizeInBytes: "{{ .Values.broker.storageOffload.readBufferSizeInBytes }}"
{{- end }}
{{- end }}
{{- if eq .Values.broker.storageOffload.driver "azureblob" }}
managedLedgerOffloadDriver: "{{ .Values.broker.storageOffload.driver }}"
managedLedgerOffloadBucket: "{{ .Values.broker.storageOffload.bucket }}"
{{- if .Values.broker.storageOffload.managedLedgerOffloadAutoTriggerSizeThresholdBytes }}
PULSAR_PREFIX_managedLedgerOffloadThresholdInBytes: "{{ .Values.broker.storageOffload.managedLedgerOffloadAutoTriggerSizeThresholdBytes }}"
{{- end }}
{{- if .Values.broker.storageOffload.managedLedgerOffloadDeletionLagMs }}
PULSAR_PREFIX_managedLedgerOffloadDeletionLagInMillis: "{{ .Values.broker.storageOffload.managedLedgerOffloadDeletionLagMs }}"
{{- end }}
{{- if .Values.broker.storageOffload.maxBlockSizeInBytes }}
managedLedgerOffloadMaxBlockSizeInBytes: "{{ .Values.broker.storageOffload.maxBlockSizeInBytes }}"
{{- end }}
{{- end }}
{{- end }}
# Function Worker Settings
# function worker configuration
{{- if not .Values.components.functions }}
functionsWorkerEnabled: "false"
{{- end }}
{{- if .Values.components.functions }}
functionsWorkerEnabled: "true"
{{- if .Values.functions.useBookieAsStateStore }}
PF_stateStorageServiceUrl: "bk://{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}:{{ .Values.bookkeeper.ports.statestore }}"
{{- end }}
PF_functionRuntimeFactoryClassName: "org.apache.pulsar.functions.runtime.kubernetes.KubernetesRuntimeFactory"
PF_pulsarFunctionsCluster: {{ template "pulsar.cluster.name" . }}
PF_connectorsDirectory: ./connectors
PF_containerFactory: k8s
PF_numFunctionPackageReplicas: "{{ .Values.broker.configData.managedLedgerDefaultEnsembleSize }}"
# support version >= 2.5.0
PF_functionRuntimeFactoryConfigs_pulsarRootDir: {{ template "pulsar.home" . }}
PF_kubernetesContainerFactory_pulsarRootDir: {{ template "pulsar.home" . }}
PF_functionRuntimeFactoryConfigs_pulsarDockerImageName: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.functions "root" .) }}"
PF_functionRuntimeFactoryConfigs_imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.functions "root" .) }}"
PF_functionRuntimeFactoryConfigs_submittingInsidePod: "true"
PF_functionRuntimeFactoryConfigs_installUserCodeDependencies: "true"
PF_functionRuntimeFactoryConfigs_jobNamespace: {{ template "pulsar.namespace" . }}
PF_functionRuntimeFactoryConfigs_expectedMetricsCollectionInterval: "30"
{{- if not (and .Values.tls.enabled .Values.tls.broker.enabled .Values.tls.function_instance.enabled) }}
PF_functionRuntimeFactoryConfigs_pulsarAdminUrl: "http://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.http }}/"
PF_functionRuntimeFactoryConfigs_pulsarServiceUrl: "pulsar://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.pulsar }}/"
{{- else }}
PF_functionRuntimeFactoryConfigs_pulsarAdminUrl: "https://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.https }}/"
PF_functionRuntimeFactoryConfigs_pulsarServiceUrl: "pulsar+ssl://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.pulsarssl }}/"
{{- end }}
# support version < 2.5.0
PF_kubernetesContainerFactory_pulsarDockerImageName: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.functions "root" .) }}"
PF_kubernetesContainerFactory_imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.functions "root" .) }}"
PF_kubernetesContainerFactory_submittingInsidePod: "true"
PF_kubernetesContainerFactory_installUserCodeDependencies: "true"
PF_kubernetesContainerFactory_jobNamespace: {{ template "pulsar.namespace" . }}
PF_kubernetesContainerFactory_expectedMetricsCollectionInterval: "30"
{{- if not (and .Values.tls.enabled .Values.tls.broker.enabled .Values.tls.function_instance.enabled) }}
PF_kubernetesContainerFactory_pulsarAdminUrl: "http://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.http }}/"
PF_kubernetesContainerFactory_pulsarServiceUrl: "pulsar://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.pulsar }}/"
{{- else }}
PF_kubernetesContainerFactory_pulsarAdminUrl: "https://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.https }}/"
PF_kubernetesContainerFactory_pulsarServiceUrl: "pulsar+ssl://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.pulsarssl }}/"
{{- end }}
{{- end }}
# prometheus needs to access /metrics endpoint
webServicePort: "{{ .Values.broker.ports.http }}"
{{- if or (not .Values.tls.enabled) (not .Values.tls.broker.enabled) }}
brokerServicePort: "{{ .Values.broker.ports.pulsar }}"
{{- end }}
{{- if and .Values.tls.enabled .Values.tls.broker.enabled }}
brokerServicePortTls: "{{ .Values.broker.ports.pulsarssl }}"
webServicePortTls: "{{ .Values.broker.ports.https }}"
# TLS Settings
tlsCertificateFilePath: "/pulsar/certs/broker/tls.crt"
tlsKeyFilePath: "/pulsar/certs/broker/tls.key"
tlsTrustCertsFilePath: {{ ternary "/pulsar/certs/cacerts/ca-combined.pem" "/pulsar/certs/ca/ca.crt" .Values.tls.broker.cacerts.enabled | quote }}
{{- end }}
# Authentication Settings
{{- if .Values.auth.authentication.enabled }}
authenticationEnabled: "true"
{{- if .Values.auth.authorization.enabled }}
authorizationEnabled: "true"
superUserRoles: {{ .Values.auth.superUsers | values | compact | sortAlpha | join "," }}
{{- if .Values.auth.useProxyRoles }}
proxyRoles: {{ .Values.auth.superUsers.proxy }}
{{- end }}
{{- end }}
{{- if and .Values.auth.authentication.enabled .Values.auth.authentication.jwt.enabled }}
# token authentication configuration
{{- if and .Values.auth.authentication.enabled .Values.auth.authentication.jwt.enabled .Values.auth.authentication.openid.enabled }}
authenticationProviders: "org.apache.pulsar.broker.authentication.AuthenticationProviderToken,org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID"
{{- end }}
{{- if and .Values.auth.authentication.enabled .Values.auth.authentication.jwt.enabled ( not .Values.auth.authentication.openid.enabled ) }}
authenticationProviders: "org.apache.pulsar.broker.authentication.AuthenticationProviderToken"
{{- end }}
brokerClientAuthenticationParameters: "file:///pulsar/tokens/broker/token"
brokerClientAuthenticationPlugin: "org.apache.pulsar.client.impl.auth.AuthenticationToken"
{{- if .Values.auth.authentication.jwt.usingSecretKey }}
tokenSecretKey: "file:///pulsar/keys/token/secret.key"
{{- else }}
tokenPublicKey: "file:///pulsar/keys/token/public.key"
{{- end }}
{{- end }}
{{- if and .Values.auth.authentication.enabled .Values.auth.authentication.openid.enabled }}
# openid authentication configuration
{{- if and .Values.auth.authentication.enabled .Values.auth.authentication.openid.enabled ( not .Values.auth.authentication.jwt.enabled ) }}
authenticationProviders: "org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID"
{{- end }}
PULSAR_PREFIX_openIDAllowedTokenIssuers: {{ .Values.auth.authentication.openid.openIDAllowedTokenIssuers | uniq | compact | sortAlpha | join "," | quote }}
PULSAR_PREFIX_openIDAllowedAudiences: {{ .Values.auth.authentication.openid.openIDAllowedAudiences | uniq | compact | sortAlpha | join "," | quote }}
PULSAR_PREFIX_openIDTokenIssuerTrustCertsFilePath: {{ .Values.auth.authentication.openid.openIDTokenIssuerTrustCertsFilePath | quote }}
PULSAR_PREFIX_openIDRoleClaim: {{ .Values.auth.authentication.openid.openIDRoleClaim | quote }}
PULSAR_PREFIX_openIDAcceptedTimeLeewaySeconds: {{ .Values.auth.authentication.openid.openIDAcceptedTimeLeewaySeconds | quote }}
PULSAR_PREFIX_openIDCacheSize: {{ .Values.auth.authentication.openid.openIDCacheSize | quote }}
PULSAR_PREFIX_openIDCacheRefreshAfterWriteSeconds: {{ .Values.auth.authentication.openid.openIDCacheRefreshAfterWriteSeconds | quote }}
PULSAR_PREFIX_openIDCacheExpirationSeconds: {{ .Values.auth.authentication.openid.openIDCacheExpirationSeconds | quote }}
PULSAR_PREFIX_openIDHttpConnectionTimeoutMillis: {{ .Values.auth.authentication.openid.openIDHttpConnectionTimeoutMillis | quote }}
PULSAR_PREFIX_openIDHttpReadTimeoutMillis: {{ .Values.auth.authentication.openid.openIDHttpReadTimeoutMillis | quote }}
PULSAR_PREFIX_openIDKeyIdCacheMissRefreshSeconds: {{ .Values.auth.authentication.openid.openIDKeyIdCacheMissRefreshSeconds | quote }}
PULSAR_PREFIX_openIDRequireIssuersUseHttps: {{ .Values.auth.authentication.openid.openIDRequireIssuersUseHttps | quote }}
PULSAR_PREFIX_openIDFallbackDiscoveryMode: {{ .Values.auth.authentication.openid.openIDFallbackDiscoveryMode | quote }}
{{- end }}
{{- end }}
{{- if and .Values.tls.enabled .Values.tls.bookie.enabled }}
# bookkeeper tls settings
bookkeeperTLSClientAuthentication: "true"
bookkeeperTLSKeyFileType: "PEM"
bookkeeperTLSKeyFilePath: "/pulsar/certs/broker/tls.key"
bookkeeperTLSCertificateFilePath: "/pulsar/certs/broker/tls.crt"
bookkeeperTLSTrustCertsFilePath: {{ ternary "/pulsar/certs/cacerts/ca-combined.pem" "/pulsar/certs/ca/ca.crt" .Values.tls.broker.cacerts.enabled | quote }}
bookkeeperTLSTrustCertTypes: "PEM"
PULSAR_PREFIX_bookkeeperTLSClientAuthentication: "true"
PULSAR_PREFIX_bookkeeperTLSKeyFileType: "PEM"
PULSAR_PREFIX_bookkeeperTLSKeyFilePath: "/pulsar/certs/broker/tls.key"
PULSAR_PREFIX_bookkeeperTLSCertificateFilePath: "/pulsar/certs/broker/tls.crt"
PULSAR_PREFIX_bookkeeperTLSTrustCertsFilePath: {{ ternary "/pulsar/certs/cacerts/ca-combined.pem" "/pulsar/certs/ca/ca.crt" .Values.tls.broker.cacerts.enabled | quote }}
PULSAR_PREFIX_bookkeeperTLSTrustCertTypes: "PEM"
# https://github.com/apache/bookkeeper/pull/2300
bookkeeperUseV2WireProtocol: "false"
{{- end }}
{{ toYaml .Values.broker.configData | indent 2 }}
{{- end }}

View File

@ -1,45 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.broker.autoscaling.enabled }}
{{- if (semverCompare "<1.23-0" .Capabilities.KubeVersion.Version) }}
apiVersion: autoscaling/v2beta2
{{- else }}
apiVersion: autoscaling/v2
{{- end }}
kind: HorizontalPodAutoscaler
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}"
namespace: {{ template "pulsar.namespace" . }}
spec:
maxReplicas: {{ .Values.broker.autoscaling.maxReplicas }}
{{- with .Values.broker.autoscaling.metrics }}
metrics:
{{- toYaml . | nindent 4 }}
{{- end }}
minReplicas: {{ .Values.broker.autoscaling.minReplicas }}
{{- with .Values.broker.autoscaling.behavior }}
behavior:
{{- toYaml . | nindent 4 }}
{{- end }}
scaleTargetRef:
apiVersion: apps/v1
kind: StatefulSet
name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}"
{{- end }}

View File

@ -1,42 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.components.broker }}
{{- if .Values.broker.pdb.usePolicy }}
# pdb version detection
{{- if semverCompare "<1.21-0" .Capabilities.KubeVersion.Version }}
apiVersion: policy/v1beta1
{{- else }}
apiVersion: policy/v1
{{- end }}
kind: PodDisruptionBudget
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}"
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.broker.component }}
spec:
selector:
matchLabels:
{{- include "pulsar.matchLabels" . | nindent 6 }}
component: {{ .Values.broker.component }}
maxUnavailable: {{ .Values.broker.pdb.maxUnavailable }}
{{- end }}
{{- end }}

View File

@ -1,23 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# deploy broker PodMonitor only when `$.Values.broker.podMonitor.enabled` is true
{{- if $.Values.broker.podMonitor.enabled }}
{{- include "pulsar.podMonitor" (list . "broker" (printf "component: %s" .Values.broker.component)) }}
{{- end }}

View File

@ -1,70 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.components.functions }}
apiVersion: rbac.authorization.k8s.io/v1
{{- if .Values.functions.rbac.limit_to_namespace }}
kind: Role
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.functions.component }}-role"
{{- else}}
kind: ClusterRole
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.functions.component }}"
{{- end}}
rules:
- apiGroups: [""]
resources:
- services
- configmaps
- pods
verbs:
- '*'
- apiGroups:
- apps
resources:
- statefulsets
verbs:
- '*'
---
apiVersion: rbac.authorization.k8s.io/v1
{{- if .Values.functions.rbac.limit_to_namespace }}
kind: RoleBinding
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.functions.component }}-rolebinding"
{{- else}}
kind: ClusterRoleBinding
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.functions.component }}"
{{- end}}
roleRef:
apiGroup: rbac.authorization.k8s.io
{{- if .Values.functions.rbac.limit_to_namespace }}
kind: Role
name: "{{ template "pulsar.fullname" . }}-{{ .Values.functions.component }}-role"
{{- else}}
kind: ClusterRole
name: "{{ template "pulsar.fullname" . }}-{{ .Values.functions.component }}"
{{- end}}
subjects:
- kind: ServiceAccount
name: "{{ template "pulsar.fullname" . }}-{{ .Values.functions.component }}"
namespace: {{ template "pulsar.namespace" . }}
{{- end }}

View File

@ -1,50 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.components.broker }}
apiVersion: v1
kind: ServiceAccount
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}-acct"
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.broker.component }}
{{- with .Values.broker.service_account.annotations }}
annotations:
{{ toYaml . | indent 4 }}
{{- end }}
---
{{- end }}
{{- if .Values.components.functions }}
apiVersion: v1
kind: ServiceAccount
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.functions.component }}"
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.functions.component }}
{{- with .Values.functions.service_account.annotations }}
annotations:
{{ toYaml . | indent 4 }}
{{- end }}
---
{{- end }}

View File

@ -1,53 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.components.broker }}
apiVersion: v1
kind: Service
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}"
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.broker.component }}
{{- with .Values.broker.service.annotations }}
annotations:
{{ toYaml . | indent 4 }}
{{- end }}
spec:
type: ClusterIP
ports:
# prometheus needs to access /metrics endpoint
- name: http
port: {{ .Values.broker.ports.http }}
{{- if or (not .Values.tls.enabled) (not .Values.tls.broker.enabled) }}
- name: "{{ .Values.tcpPrefix }}pulsar"
port: {{ .Values.broker.ports.pulsar }}
{{- end }}
{{- if and .Values.tls.enabled .Values.tls.broker.enabled }}
- name: https
port: {{ .Values.broker.ports.https }}
- name: "{{ .Values.tlsPrefix }}pulsarssl"
port: {{ .Values.broker.ports.pulsarssl }}
{{- end }}
clusterIP: "{{ .Values.broker.service.clusterIP }}"
selector:
{{- include "pulsar.matchLabels" . | nindent 4 }}
component: {{ .Values.broker.component }}
{{- end }}

View File

@ -1,383 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.components.broker }}
apiVersion: apps/v1
kind: StatefulSet
metadata:
{{- $stsName := printf "%s-%s" (include "pulsar.fullname" .) .Values.broker.component }}
name: {{ $stsName | quote }}
{{- $namespace := include "pulsar.namespace" . }}
namespace: {{ $namespace | quote }}
annotations: {{ .Values.broker.appAnnotations | toYaml | nindent 4 }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.broker.component }}
spec:
serviceName: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}"
{{- if not .Values.broker.autoscaling.enabled }}
replicas: {{ .Values.broker.replicaCount }}
{{- end }}
selector:
matchLabels:
{{- include "pulsar.matchLabels" . | nindent 6 }}
component: {{ .Values.broker.component }}
updateStrategy:
type: RollingUpdate
{{- /*
When functions are enabled, podManagementPolicy must be OrderedReady to ensure that other started brokers are available via DNS
for the function worker to connect to.
Since podManagementPolicy is immutable, this rule is only applied when the broker is first installed.
*/}}
{{- $stsObj := lookup "apps/v1" "StatefulSet" $namespace $stsName }}
{{- if $stsObj }}
podManagementPolicy: {{ $stsObj.spec.podManagementPolicy }}
{{- else }}
{{- if .Values.broker.podManagementPolicy }}
podManagementPolicy: {{ .Values.broker.podManagementPolicy }}
{{- else if not .Values.components.functions }}
podManagementPolicy: Parallel
{{- else }}
podManagementPolicy: OrderedReady
{{- end }}
{{- end }}
template:
metadata:
labels:
{{- include "pulsar.template.labels" . | nindent 8 }}
component: {{ .Values.broker.component }}
annotations:
{{- if not .Values.broker.podMonitor.enabled }}
prometheus.io/scrape: "true"
prometheus.io/port: "{{ .Values.broker.ports.http }}"
{{- end }}
{{- if .Values.broker.restartPodsOnConfigMapChange }}
checksum/config: {{ include (print $.Template.BasePath "/broker-configmap.yaml") . | sha256sum }}
{{- end }}
{{- with .Values.broker.annotations }}
{{ toYaml . | indent 8 }}
{{- end }}
spec:
serviceAccountName: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}-acct"
{{- if .Values.broker.nodeSelector }}
nodeSelector:
{{ toYaml .Values.broker.nodeSelector | indent 8 }}
{{- end }}
{{- if .Values.broker.tolerations }}
tolerations:
{{ toYaml .Values.broker.tolerations | indent 8 }}
{{- end }}
{{- if .Values.broker.topologySpreadConstraints }}
topologySpreadConstraints:
{{- toYaml .Values.broker.topologySpreadConstraints | nindent 8 }}
{{- end }}
affinity:
{{- if and .Values.affinity.anti_affinity .Values.broker.affinity.anti_affinity}}
podAntiAffinity:
{{- if eq .Values.broker.affinity.type "requiredDuringSchedulingIgnoredDuringExecution"}}
{{ .Values.broker.affinity.type }}:
- labelSelector:
matchExpressions:
- key: "app"
operator: In
values:
- "{{ template "pulsar.name" . }}"
- key: "release"
operator: In
values:
- {{ .Release.Name }}
- key: "component"
operator: In
values:
- {{ .Values.broker.component }}
topologyKey: {{ .Values.broker.affinity.anti_affinity_topology_key }}
{{- else }}
{{ .Values.broker.affinity.type }}:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: "app"
operator: In
values:
- "{{ template "pulsar.name" . }}"
- key: "release"
operator: In
values:
- {{ .Release.Name }}
- key: "component"
operator: In
values:
- {{ .Values.broker.component }}
topologyKey: {{ .Values.broker.affinity.anti_affinity_topology_key }}
{{- end }}
{{- end }}
terminationGracePeriodSeconds: {{ .Values.broker.gracePeriod }}
initContainers:
{{- if .Values.tls.broker.cacerts.enabled }}
- name: cacerts
image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.broker "root" .) }}"
imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.broker "root" .) }}"
resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
command: ["sh", "-c"]
args:
- |
bin/certs-combine-pem.sh /pulsar/certs/cacerts/ca-combined.pem {{ template "pulsar.certs.cacerts" (dict "certs" .Values.tls.broker.cacerts.certs) }}
volumeMounts:
{{- include "pulsar.broker.certs.volumeMounts" . | nindent 8 }}
{{- end }}
{{- if and .Values.components.zookeeper .Values.broker.waitZookeeperTimeout (gt (.Values.broker.waitZookeeperTimeout | int) 0) }}
# This init container will wait for zookeeper to be ready before
# deploying the bookies
- name: wait-zookeeper-ready
image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.broker "root" .) }}"
imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.broker "root" .) }}"
resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
command: ["timeout", "{{ .Values.broker.waitZookeeperTimeout }}", "sh", "-c"]
args:
- |
{{- include "pulsar.broker.zookeeper.tls.settings" . | nindent 12 }}
export PULSAR_MEM="-Xmx128M";
{{- if .Values.pulsar_metadata.configurationStore }}
until timeout 15 bin/pulsar zookeeper-shell -server {{ template "pulsar.configurationStore.connect" . }} get {{ .Values.pulsar_metadata.configurationStoreMetadataPrefix }}/admin/clusters/{{ template "pulsar.cluster.name" . }}; do
{{- end }}
{{- if not .Values.pulsar_metadata.configurationStore }}
until timeout 15 bin/pulsar zookeeper-shell -server {{ template "pulsar.zookeeper.connect" . }} get {{ .Values.metadataPrefix }}/admin/clusters/{{ template "pulsar.cluster.name" . }}; do
{{- end }}
echo "pulsar cluster {{ template "pulsar.cluster.name" . }} isn't initialized yet ... check in 3 seconds ..." && sleep 3;
done;
volumeMounts:
{{- include "pulsar.broker.certs.volumeMounts" . | nindent 8 }}
{{- end }}
{{- if and .Values.components.oxia .Values.broker.waitOxiaTimeout (gt (.Values.broker.waitOxiaTimeout | int) 0) }}
- name: wait-oxia-ready
image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.broker "root" .) }}"
imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.broker "root" .) }}"
resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
command: ["timeout", "{{ .Values.broker.waitOxiaTimeout }}", "sh", "-c"]
args:
- |
until nslookup {{ template "pulsar.oxia.server.service" . }}; do
sleep 3;
done;
{{- end }}
{{- if and .Values.broker.waitBookkeeperTimeout (gt (.Values.broker.waitBookkeeperTimeout | int) 0) }}
# This init container will wait for bookkeeper to be ready before
# deploying the broker
- name: wait-bookkeeper-ready
image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.broker "root" .) }}"
imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.broker "root" .) }}"
resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
command: ["timeout", "{{ .Values.broker.waitBookkeeperTimeout }}", "sh", "-c"]
args:
- |
{{- include "pulsar.broker.zookeeper.tls.settings" . | nindent 12 }}
bin/apply-config-from-env.py conf/bookkeeper.conf;
export BOOKIE_MEM="-Xmx128M";
until timeout 15 bin/bookkeeper shell whatisinstanceid; do
echo "bookkeeper cluster is not initialized yet. backoff for 3 seconds ...";
sleep 3;
done;
echo "bookkeeper cluster is already initialized";
bookieServiceNumber="$(nslookup -timeout=10 {{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }} | grep Name | wc -l)";
until [ ${bookieServiceNumber} -ge {{ .Values.broker.configData.managedLedgerDefaultEnsembleSize }} ]; do
echo "bookkeeper cluster {{ template "pulsar.cluster.name" . }} isn't ready yet ... check in 10 seconds ...";
sleep 10;
bookieServiceNumber="$(nslookup -timeout=10 {{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }} | grep Name | wc -l)";
done;
echo "bookkeeper cluster is ready";
envFrom:
- configMapRef:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
volumeMounts:
{{- include "pulsar.broker.certs.volumeMounts" . | nindent 10 }}
{{- end }}
{{- if .Values.broker.initContainers }}
{{- toYaml .Values.broker.initContainers | nindent 6 }}
{{- end }}
containers:
- name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}"
image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.broker "root" .) }}"
imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.broker "root" .) }}"
{{- if .Values.broker.probe.liveness.enabled }}
livenessProbe:
httpGet:
path: /status.html
port: {{ .Values.broker.ports.http }}
initialDelaySeconds: {{ .Values.broker.probe.liveness.initialDelaySeconds }}
periodSeconds: {{ .Values.broker.probe.liveness.periodSeconds }}
timeoutSeconds: {{ .Values.broker.probe.liveness.timeoutSeconds }}
failureThreshold: {{ .Values.broker.probe.liveness.failureThreshold }}
{{- end }}
{{- if .Values.broker.probe.readiness.enabled }}
readinessProbe:
httpGet:
path: /status.html
port: {{ .Values.broker.ports.http }}
initialDelaySeconds: {{ .Values.broker.probe.readiness.initialDelaySeconds }}
periodSeconds: {{ .Values.broker.probe.readiness.periodSeconds }}
timeoutSeconds: {{ .Values.broker.probe.readiness.timeoutSeconds }}
failureThreshold: {{ .Values.broker.probe.readiness.failureThreshold }}
{{- end }}
{{- if .Values.broker.probe.startup.enabled }}
startupProbe:
httpGet:
path: /status.html
port: {{ .Values.broker.ports.http }}
initialDelaySeconds: {{ .Values.broker.probe.startup.initialDelaySeconds }}
periodSeconds: {{ .Values.broker.probe.startup.periodSeconds }}
timeoutSeconds: {{ .Values.broker.probe.startup.timeoutSeconds }}
failureThreshold: {{ .Values.broker.probe.startup.failureThreshold }}
{{- end }}
{{- if .Values.broker.resources }}
resources:
{{ toYaml .Values.broker.resources | indent 10 }}
{{- end }}
command: ["sh", "-c"]
args:
- |
{{- if .Values.broker.additionalCommand }}
{{ .Values.broker.additionalCommand }}
{{- end }}
{{- if .Values.tls.broker.cacerts.enabled }}
cd /pulsar/certs/cacerts;
nohup /pulsar/bin/certs-combine-pem-infinity.sh /pulsar/certs/cacerts/ca-combined.pem {{ template "pulsar.certs.cacerts" (dict "certs" .Values.tls.broker.cacerts.certs) }} > /pulsar/certs/cacerts/certs-combine-pem-infinity.log 2>&1 &
cd /pulsar;
{{- end }}
bin/apply-config-from-env.py conf/broker.conf;
bin/gen-yml-from-env.py conf/functions_worker.yml;
echo "OK" > "${statusFilePath:-status}";
{{- if .Values.components.zookeeper }}
{{- include "pulsar.broker.zookeeper.tls.settings" . | nindent 10 }}
timeout 15 bin/pulsar zookeeper-shell -server {{ template "pulsar.zookeeper.connect" . }} get {{ template "pulsar.broker.znode" . }};
while [ $? -eq 0 ]; do
echo "broker {{ template "pulsar.broker.hostname" . }} znode still exists ... check in 10 seconds ...";
sleep 10;
timeout 15 bin/pulsar zookeeper-shell -server {{ template "pulsar.zookeeper.connect" . }} get {{ template "pulsar.broker.znode" . }};
done;
{{- end }}
cat conf/pulsar_env.sh;
OPTS="${OPTS} -Dlog4j2.formatMsgNoLookups=true" exec bin/pulsar broker;
ports:
# prometheus needs to access /metrics endpoint
- name: http
containerPort: {{ .Values.broker.ports.http }}
{{- if or (not .Values.tls.enabled) (not .Values.tls.broker.enabled) }}
- name: "{{ .Values.tcpPrefix }}pulsar"
containerPort: {{ .Values.broker.ports.pulsar }}
{{- end }}
{{- if and .Values.tls.enabled .Values.tls.broker.enabled }}
- name: https
containerPort: {{ .Values.broker.ports.https }}
- name: "{{ .Values.tlsPrefix }}pulsarssl"
containerPort: {{ .Values.broker.ports.pulsarssl }}
{{- end }}
envFrom:
- configMapRef:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}"
volumeMounts:
{{- if .Values.auth.authentication.enabled }}
{{- if .Values.auth.authentication.jwt.enabled }}
- mountPath: "/pulsar/keys"
name: token-keys
readOnly: true
- mountPath: "/pulsar/tokens"
name: broker-token
readOnly: true
{{- end }}
{{- end }}
{{- if .Values.broker.storageOffload.driver }}
{{- if eq .Values.broker.storageOffload.driver "google-cloud-storage" }}
- name: gcp-service-account
readOnly: true
mountPath: /pulsar/gcp-service-account
{{- end }}
{{- end }}
{{- if .Values.broker.extraVolumeMounts }}
{{ toYaml .Values.broker.extraVolumeMounts | indent 10 }}
{{- end }}
{{- include "pulsar.broker.certs.volumeMounts" . | nindent 10 }}
env:
{{- if and (and .Values.broker.storageOffload (eq .Values.broker.storageOffload.driver "aws-s3")) .Values.broker.storageOffload.secret }}
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: {{ .Values.broker.storageOffload.secret }}
key: AWS_ACCESS_KEY_ID
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: {{ .Values.broker.storageOffload.secret }}
key: AWS_SECRET_ACCESS_KEY
{{- end }}
{{- if and .Values.broker.storageOffload (eq .Values.broker.storageOffload.driver "azureblob") }}
- name: AZURE_STORAGE_ACCOUNT
valueFrom:
secretKeyRef:
name: {{ .Values.broker.storageOffload.secret }}
key: AZURE_STORAGE_ACCOUNT
- name: AZURE_STORAGE_ACCESS_KEY
valueFrom:
secretKeyRef:
name: {{ .Values.broker.storageOffload.secret }}
key: AZURE_STORAGE_ACCESS_KEY
{{- end }}
{{- if .Values.broker.extraEnvs }}
{{- toYaml .Values.broker.extraEnvs | nindent 10 }}
{{- end }}
volumes:
{{- if .Values.broker.extraVolumes }}
{{ toYaml .Values.broker.extraVolumes | indent 6 }}
{{- end }}
{{- if .Values.auth.authentication.enabled }}
{{- if .Values.auth.authentication.jwt.enabled }}
- name: token-keys
secret:
{{- if not .Values.auth.authentication.jwt.usingSecretKey }}
secretName: "{{ .Release.Name }}-token-asymmetric-key"
{{- end}}
{{- if .Values.auth.authentication.jwt.usingSecretKey }}
secretName: "{{ .Release.Name }}-token-symmetric-key"
{{- end}}
items:
{{- if .Values.auth.authentication.jwt.usingSecretKey }}
- key: SECRETKEY
path: token/secret.key
{{- else }}
- key: PUBLICKEY
path: token/public.key
{{- end}}
- name: broker-token
secret:
secretName: "{{ .Release.Name }}-token-{{ .Values.auth.superUsers.broker }}"
items:
- key: TOKEN
path: broker/token
{{- end}}
{{- end}}
{{- if .Values.broker.storageOffload.driver }}
{{- if eq .Values.broker.storageOffload.driver "google-cloud-storage" }}
- name: gcp-service-account
secret:
secretName: {{ .Values.broker.storageOffload.gcsServiceAccountSecret }}
{{- end }}
{{- end }}
{{- include "pulsar.broker.certs.volumes" . | nindent 6 }}
{{- include "pulsar.imagePullSecrets" . | nindent 6}}
{{- end }}

View File

@ -1,82 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
apiVersion: v1
kind: ConfigMap
metadata:
name: "{{ template "pulsar.fullname" . }}-certs-scripts"
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: certs-scripts
data:
certs-combine-pem.sh: |
#!/bin/bash
# This script combines all certificates into a single file.
# Usage: certs-combine-pem.sh <output_file> <cert1> <cert2> ...
set -eu -o pipefail
if [ "$#" -lt 2 ]; then
echo "Usage: $0 <output_file> <cert1> <cert2> ..."
exit 1
fi
OUTPUT_FILE="$1"
shift
OUTPUT_FILE_TMP="${OUTPUT_FILE}.tmp"
rm -f "$OUTPUT_FILE_TMP"
for CERT in "$@"; do
if [ -f "$CERT" ]; then
echo "# $CERT" >> "$OUTPUT_FILE_TMP"
cat "$CERT" >> "$OUTPUT_FILE_TMP"
else
echo "Certificate file '$CERT' does not exist, skipping"
fi
done
if [ ! -f "$OUTPUT_FILE" ]; then
touch "$OUTPUT_FILE"
fi
if diff -q "$OUTPUT_FILE" "$OUTPUT_FILE_TMP" > /dev/null; then
# No changes detected, skipping update
rm -f "$OUTPUT_FILE_TMP"
else
# Update $OUTPUT_FILE with new certificates
mv "$OUTPUT_FILE_TMP" "$OUTPUT_FILE"
fi
certs-combine-pem-infinity.sh: |
#!/bin/bash
# This script combines all certificates into a single file, every minutes.
# Usage: certs-combine-pem-infinity.sh <output_file> <cert1> <cert2> ...
set -eu -o pipefail
if [ "$#" -lt 2 ]; then
echo "Usage: $0 <output_file> <cert1> <cert2> ..."
exit 1
fi
while true; do
/pulsar/bin/certs-combine-pem.sh "$@"
sleep 60
done

View File

@ -1,22 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if semverCompare "<3.12.0-0" .Capabilities.HelmVersion.Version -}}
{{- fail "Your Helm version is not supported. Please upgrade to Helm 3.12.0 or later. The recommended version is currently 3.14.4 or newer. You can find more about Helm releases and installation at https://github.com/helm/helm/releases. " -}}
{{- end -}}

View File

@ -1,23 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- range .Values.extraDeploy }}
---
{{ include "common.tplvalues.render" (dict "value" . "context" $) }}
{{- end }}

View File

@ -1,25 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.namespaceCreate }}
apiVersion: v1
kind: Namespace
metadata:
name: {{ template "pulsar.namespace" . }}
{{- end }}

View File

@ -1,32 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if and .Values.components.oxia (not .Values.oxia.coordinator.customConfigMapName) }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.oxia.component }}-coordinator
data:
config.yaml: |
{{- include "oxia.coordinator.config.yaml" . | nindent 4 }}
{{- end }}

View File

@ -1,95 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.components.oxia }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.oxia.component }}-coordinator
annotations: {{ .Values.oxia.coordinator.appAnnotations | toYaml | nindent 4 }}
spec:
replicas: 1
selector:
matchLabels:
{{- include "pulsar.matchLabels" . | nindent 6 }}
component: {{ .Values.oxia.component }}-coordinator
strategy:
type: Recreate
template:
metadata:
labels:
{{- include "pulsar.template.labels" . | nindent 8 }}
component: {{ .Values.oxia.component }}-coordinator
annotations:
{{- if not .Values.oxia.coordinator.podMonitor.enabled }}
prometheus.io/scrape: "true"
prometheus.io/port: "{{ .Values.oxia.coordinator.ports.metrics }}"
{{- end }}
{{- with .Values.oxia.coordinator.annotations }}
{{ toYaml . | indent 8 }}
{{- end }}
spec:
{{- if .Values.oxia.coordinator.nodeSelector }}
nodeSelector:
{{ toYaml .Values.oxia.coordinator.nodeSelector | indent 8 }}
{{- end }}
{{- if .Values.oxia.coordinator.tolerations }}
tolerations:
{{ toYaml .Values.oxia.coordinator.tolerations | indent 8 }}
{{- end }}
serviceAccountName: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator
containers:
- command:
{{- if .Values.oxia.coordinator.entrypoint }}
{{ toYaml .Values.oxia.coordinator.entrypoint | indent 12 }}
{{- else }}
{{- include "oxia.coordinator.entrypoint" . | nindent 12 }}
{{- end }}
image: "{{ .Values.images.oxia.repository }}:{{ .Values.images.oxia.tag }}"
imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.oxia "root" .) }}"
name: coordinator
ports:
{{- range $key, $value := .Values.oxia.coordinator.ports }}
- containerPort: {{ $value | int }}
name: {{ $key }}
{{- end}}
resources:
limits:
cpu: {{ .Values.oxia.coordinator.cpuLimit }}
memory: {{ .Values.oxia.coordinator.memoryLimit }}
{{- if .Values.oxia.coordinator.extraVolumeMounts }}
volumeMounts:
{{- toYaml .Values.oxia.coordinator.extraVolumeMounts | nindent 12 }}
{{- end }}
livenessProbe:
{{- include "oxia-cluster.probe" .Values.oxia.coordinator.ports.internal | nindent 12 }}
readinessProbe:
{{- include "oxia-cluster.probe" .Values.oxia.coordinator.ports.internal | nindent 12 }}
{{- if .Values.oxia.coordinator.extraContainers }}
{{- toYaml .Values.oxia.coordinator.extraContainers | nindent 8 }}
{{- end }}
{{- if .Values.oxia.coordinator.extraVolumes }}
volumes:
{{- toYaml .Values.oxia.coordinator.extraVolumes | nindent 8 }}
{{- end }}
{{- end }}

View File

@ -1,23 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# deploy oxia-coordinator PodMonitor only when `$.Values.oxia.coordinator.podMonitor.enabled` is true
{{- if and $.Values.components.oxia $.Values.oxia.coordinator.podMonitor.enabled }}
{{- include "pulsar.podMonitor" (list . "oxia.coordinator" (printf "component: %s-coordinator" .Values.oxia.component) "metrics") }}
{{- end }}

View File

@ -1,33 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.components.oxia }}
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.oxia.component }}-coordinator
rules:
- apiGroups: [ "" ]
resources: [ "configmaps" ]
verbs: [ "*" ]
{{- end }}

View File

@ -1,37 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.components.oxia }}
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.oxia.component }}-coordinator
subjects:
- kind: ServiceAccount
name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator
namespace: {{ template "pulsar.namespace" . }}
roleRef:
apiGroup: ""
kind: Role
name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator
{{- end }}

View File

@ -1,43 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.components.oxia }}
apiVersion: v1
kind: Service
metadata:
name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.oxia.component }}-coordinator
{{- with .Values.oxia.coordinator.service.annotations }}
annotations:
{{ toYaml . | indent 4 }}
{{- end }}
spec:
ports:
{{- range $key, $value := .Values.oxia.coordinator.ports }}
- name: {{ $key }}
port: {{ $value }}
targetPort: {{ $key }}
{{- end}}
selector:
{{- include "pulsar.matchLabels" . | nindent 4 }}
component: {{ .Values.oxia.component }}-coordinator
{{- end }}

View File

@ -1,36 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.components.oxia }}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.oxia.component }}-coordinator
{{- with .Values.oxia.coordinator.service_account.annotations }}
annotations:
{{ toYaml . | indent 4 }}
{{- end }}
{{- if .Values.images.imagePullSecrets }}
imagePullSecrets:
- name: {{ .Values.images.imagePullSecrets.secretName }}
{{- end}}
{{- end}}

View File

@ -1,23 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# deploy oxia-server PodMonitor only when `$.Values.oxia.server.podMonitor.enabled` is true
{{- if and $.Values.components.oxia $.Values.oxia.server.podMonitor.enabled }}
{{- include "pulsar.podMonitor" (list . "oxia.server" (printf "component: %s-server" .Values.oxia.component) "metrics") }}
{{- end }}

View File

@ -1,43 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.components.oxia }}
apiVersion: v1
kind: Service
metadata:
name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.oxia.component }}-server
{{- with .Values.oxia.server.service.public.annotations }}
annotations:
{{ toYaml . | indent 4 }}
{{- end }}
spec:
ports:
{{- range $key, $value := .Values.oxia.server.ports }}
- name: {{ $key }}
port: {{ $value }}
targetPort: {{ $key }}
{{- end}}
selector:
{{- include "pulsar.matchLabels" . | nindent 4 }}
component: {{ .Values.oxia.component }}-server
{{- end}}

View File

@ -1,45 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.components.oxia }}
apiVersion: v1
kind: Service
metadata:
name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-svc
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.oxia.component }}-server
{{- with .Values.oxia.server.service.internal.annotations }}
annotations:
{{ toYaml . | indent 4 }}
{{- end }}
spec:
clusterIP: None
publishNotReadyAddresses: true
ports:
{{- range $key, $value := .Values.oxia.server.ports }}
- name: {{ $key }}
port: {{ $value }}
targetPort: {{ $key }}
{{- end}}
selector:
{{- include "pulsar.matchLabels" . | nindent 4 }}
component: {{ .Values.oxia.component }}-server
{{- end}}

View File

@ -1,36 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.components.oxia }}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.oxia.component }}-server
{{- with .Values.oxia.server.service_account.annotations }}
annotations:
{{ toYaml . | indent 4 }}
{{- end }}
{{- if .Values.images.imagePullSecrets }}
imagePullSecrets:
- name: {{ .Values.images.imagePullSecrets.secretName }}
{{- end}}
{{- end}}

View File

@ -1,153 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
{{- if .Values.components.oxia }}
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-server
namespace: {{ template "pulsar.namespace" . }}
labels:
{{- include "pulsar.standardLabels" . | nindent 4 }}
component: {{ .Values.oxia.component }}-server
annotations: {{ .Values.oxia.server.appAnnotations | toYaml | nindent 4 }}
spec:
replicas: {{ .Values.oxia.server.replicas }}
selector:
matchLabels:
{{- include "pulsar.matchLabels" . | nindent 6 }}
component: {{ .Values.oxia.component }}-server
serviceName: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-svc
podManagementPolicy: Parallel
template:
metadata:
labels:
{{- include "pulsar.template.labels" . | nindent 8 }}
component: {{ .Values.oxia.component }}-server
annotations:
{{- if not .Values.oxia.server.podMonitor.enabled }}
prometheus.io/scrape: "true"
prometheus.io/port: "{{ .Values.oxia.server.ports.metrics }}"
{{- end }}
{{- with .Values.oxia.server.annotations }}
{{ toYaml . | indent 8 }}
{{- end }}
spec:
{{- if .Values.oxia.server.nodeSelector }}
nodeSelector:
{{ toYaml .Values.oxia.server.nodeSelector | indent 8 }}
{{- end }}
{{- if .Values.oxia.server.tolerations }}
tolerations:
{{ toYaml .Values.oxia.server.tolerations | indent 8 }}
{{- end }}
{{- if .Values.oxia.server.topologySpreadConstraints }}
topologySpreadConstraints:
{{- toYaml .Values.oxia.server.topologySpreadConstraints | nindent 8 }}
{{- end }}
affinity:
{{- if and .Values.affinity.anti_affinity .Values.oxia.server.affinity.anti_affinity}}
podAntiAffinity:
{{ if eq .Values.oxia.server.affinity.type "requiredDuringSchedulingIgnoredDuringExecution"}}
{{ .Values.oxia.server.affinity.type }}:
- labelSelector:
matchExpressions:
- key: "app"
operator: In
values:
- "{{ template "pulsar.name" . }}"
- key: "release"
operator: In
values:
- {{ .Release.Name }}
- key: "component"
operator: In
values:
- {{ .Values.oxia.component }}-server
topologyKey: {{ .Values.oxia.server.affinity.anti_affinity_topology_key }}
{{ else }}
{{ .Values.oxia.server.affinity.type }}:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: "app"
operator: In
values:
- "{{ template "pulsar.name" . }}"
- key: "release"
operator: In
values:
- {{ .Release.Name }}
- key: "component"
operator: In
values:
- {{ .Values.oxia.component }}-server
topologyKey: {{ .Values.oxia.server.affinity.anti_affinity_topology_key }}
{{ end }}
{{- end }}
serviceAccountName: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}
{{- if .Values.oxia.server.securityContext }}
securityContext:
{{ toYaml .Values.oxia.server.securityContext | indent 8 }}
{{- end }}
containers:
- command:
- "oxia"
- "server"
- "--log-json"
- "--data-dir=/data/db"
- "--wal-dir=/data/wal"
- "--db-cache-size-mb={{ .Values.oxia.server.dbCacheSizeMb }}"
{{- if .Values.oxia.pprofEnabled }}
- "--profile"
{{- end}}
image: "{{ .Values.images.oxia.repository }}:{{ .Values.images.oxia.tag }}"
imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.oxia "root" .) }}"
name: server
ports:
{{- range $key, $value := .Values.oxia.server.ports }}
- containerPort: {{ $value | int }}
name: {{ $key }}
{{- end}}
resources:
limits:
cpu: {{ .Values.oxia.server.cpuLimit }}
memory: {{ .Values.oxia.server.memoryLimit }}
volumeMounts:
- name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-data
mountPath: /data
livenessProbe:
{{- include "oxia-cluster.probe" .Values.oxia.server.ports.internal | nindent 12 }}
readinessProbe:
{{- include "oxia-cluster.readiness-probe" .Values.oxia.server.ports.internal | nindent 12 }}
startupProbe:
{{- include "oxia-cluster.startup-probe" .Values.oxia.server.ports.internal | nindent 12 }}
volumeClaimTemplates:
- metadata:
name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-data
spec:
accessModes: [ "ReadWriteOnce" ]
{{- if .Values.oxia.server.storageClassName }}
storageClassName: {{ .Values.oxia.server.storageClassName }}
{{- end}}
resources:
requests:
storage: {{ .Values.oxia.server.storageSize }}
{{- end}}

Some files were not shown because too many files have changed in this diff Show More