apache/pulsar-helm-chart
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor GitHub Actions CI to a single workflow (#371)

Browse Source 
* Refactor GitHub Actions CI to a single workflow

* Handle case where "ct lint" fails because of no chart changes

* Re-order scenarios

* Remove excessive default GC logging

* Bump cert-manager version to v1.12.2

* Use compatible cert-manager version

* Install debugging tools (k9s) for ssh access

* Only apply for interactive shells

* Fix JWT symmetric test

* Fix part that was missing from #356

* Install k9s on the fly when k9s is used

- set KUBECONFIG on the fly for kubectl too
This commit is contained in:
Lari Hotari 2023-07-11 15:55:35 +03:00 committed by GitHub
parent 2d646f4efe
commit 49f4acdf5a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 487 additions and 860 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

161
.github/actions/ssh-access/action.yml vendored Normal file
View File

@ -0,0 +1,161 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
name: ssh access
description: Sets up SSH access to build VM with upterm
inputs:
action:
description: |
Action to perform: options are "start" and "wait"
"start" will install, configure and start upterm.
"wait" will wait until a connection is established to upterm and will continue to wait until the session is closed.
required: false
default: 'start'
limit-access-to-actor:
description: 'If only the public SSH keys of the user triggering the workflow should be authorized'
required: false
default: 'false'
limit-access-to-users:
description: 'If only the public SSH keys of the listed GitHub users should be authorized. Comma separate list of GitHub user names.'
required: false
default: ''
secure-access:
description: |
Set to false for allowing public access when limit-access-to-actor and limit-access-to-users are unset.
required: false
default: 'true'
timeout:
description: 'When action=wait, the timeout in seconds to wait for the user to connect'
required: false
default: '300'
runs:
using: composite
steps:
- run: |
if [[ "${{ inputs.action }}" == "start" ]]; then
echo "::group::Installing upterm & tmux"
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
# install upterm
curl -sL https://github.com/owenthereal/upterm/releases/download/v0.7.6/upterm_linux_amd64.tar.gz | tar zxvf - -C /tmp upterm && sudo install /tmp/upterm /usr/local/bin/ && rm -rf /tmp/upterm
# install tmux if it's not present
if ! command -v tmux &>/dev/null; then
sudo apt-get -y install tmux
fi
elif [[ "$OSTYPE" == "darwin"* ]]; then
brew install owenthereal/upterm/upterm
# install tmux if it's not present
if ! command -v tmux &>/dev/null; then
brew install tmux
fi
else
echo "Unsupported $OSTYPE"
exit 0
fi
echo '::endgroup::'
echo "::group::Configuring ssh and ssh keys"
# generate ssh key
mkdir -p ~/.ssh
chmod 0700 ~/.ssh
if [ ! -f ~/.ssh/id_rsa ]; then
ssh-keygen -q -t rsa -N "" -f ~/.ssh/id_rsa
fi
if [ ! -f ~/.ssh/id_ed25519 ]; then
ssh-keygen -q -t ed25519 -N "" -f ~/.ssh/id_ed25519
fi
# configure ssh
echo -e "Host *\nStrictHostKeyChecking no\nCheckHostIP no\nTCPKeepAlive yes\nServerAliveInterval 30\nServerAliveCountMax 180\nVerifyHostKeyDNS yes\nUpdateHostKeys yes\n" > ~/.ssh/config
# Auto-generate ~/.ssh/known_hosts by attempting connection to uptermd.upterm.dev
ssh -i ~/.ssh/id_ed25519 uptermd.upterm.dev || true
# @cert-authority entry is a mandatory entry when connecting to upterm. generate the entry based on the known_hosts entry key
cat <(cat ~/.ssh/known_hosts | awk '{ print "@cert-authority * " $2 " " $3 }') >> ~/.ssh/known_hosts
authorizedKeysParameter=""
authorizedKeysFile=${HOME}/.ssh/authorized_keys
if [[ "${{ inputs.secure-access }}" != "false" ]]; then
ssh-keygen -q -t ed25519 -N "$(echo $RANDOM | md5sum | awk '{ print $1 }')" -C "Prevent public access" -f /tmp/dummykey$$
cat /tmp/dummykey$$.pub >> $authorizedKeysFile
rm /tmp/dummykey$$ /tmp/dummykey$$.pub
fi
limit_access_to_actor="${{ inputs.limit-access-to-actor }}"
if [[ "${limit_access_to_actor}" == "true" ]]; then
echo "Adding ${GITHUB_ACTOR} to allowed users (identified by ssh key registered in GitHub)"
curl -s https://github.com/${GITHUB_ACTOR}.keys >> $authorizedKeysFile
fi
limit_access_to_users="${{ inputs.limit-access-to-users }}"
for github_user in ${limit_access_to_users//,/ }; do
if [[ -n "${github_user}" ]]; then
echo "Adding ${github_user} to allowed users (identified by ssh key registered in GitHub)"
curl -s https://github.com/${github_user}.keys >> $authorizedKeysFile
fi
done
if [ -f $authorizedKeysFile ]; then
chmod 0600 $authorizedKeysFile
authorizedKeysParameter="-a $authorizedKeysFile"
echo -e "Using $authorizedKeysFile\nContent:\n---------------------------"
cat $authorizedKeysFile
echo "---------------------------"
fi
echo '::endgroup::'
echo "::group::Starting terminal session and connecting to server"
tmux new -d -s upterm-wrapper -x 132 -y 43 "upterm host ${authorizedKeysParameter} --force-command 'tmux attach -t upterm' -- tmux new -s upterm -x 132 -y 43"
sleep 2
tmux send-keys -t upterm-wrapper q C-m
sleep 1
tmux set -t upterm-wrapper window-size largest
tmux set -t upterm window-size largest
echo '::endgroup::'
echo -e "\nSSH connection information"
# wait up to 10 seconds for upterm admin socket to appear
for i in {1..10}; do
ADMIN_SOCKET=$(find $HOME/.upterm -name "*.sock")
if [ ! -S "$ADMIN_SOCKET" ]; then
echo "Waiting for upterm admin socket to appear in ~/.upterm/*.sock ..."
sleep 1
else
echo "upterm admin socket available in $ADMIN_SOCKET"
break
fi
done
shopt -s nullglob
upterm session current --admin-socket ~/.upterm/*.sock || {
echo "Starting upterm failed."
exit 0
}
elif [[ "${{ inputs.action }}" == "wait" ]]; then
# only wait if upterm was installed
if command -v upterm &>/dev/null; then
shopt -s nullglob
echo "SSH connection information"
upterm session current --admin-socket ~/.upterm/*.sock || {
echo "upterm isn't running. Not waiting any longer."
exit 0
}
timeout=${{ inputs.timeout }}
echo "Waiting $timeout seconds..."
sleep $timeout
echo "Keep waiting as long as there's a connected session"
while upterm session current --admin-socket ~/.upterm/*.sock|grep Connected &>/dev/null; do
sleep 30
done
echo "No session is connected. Not waiting any longer."
else
echo "upterm isn't installed"
fi
fi
shell: bash

34
.github/actions/tune-runner-vm/action.yml vendored
View File

@ -24,6 +24,7 @@ runs:
steps:
- run: |
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
echo "::group::Configure and tune OS"
# Ensure that reverse lookups for current hostname are handled properly
# Add the current IP address, long hostname and short hostname record to /etc/hosts file
echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
@ -32,17 +33,22 @@ runs:
# consumption is high.
# Set vm.swappiness=1 to avoid swapping and allow high RAM usage
echo 1 | sudo tee /proc/sys/vm/swappiness
(
shopt -s nullglob
# Set swappiness to 1 for all cgroups and sub-groups
for swappiness_dir in /sys/fs/cgroup/memory/*/ /sys/fs/cgroup/memory/*/*/; do
if [ -d "swappiness_dir" ]; then
echo 1 | sudo tee $(swappiness_dir)memory.swappiness > /dev/null
fi
for swappiness_file in /sys/fs/cgroup/memory/*/memory.swappiness /sys/fs/cgroup/memory/*/*/memory.swappiness; do
echo 1 | sudo tee $swappiness_file > /dev/null
done
) || true
# use "madvise" Linux Transparent HugePages (THP) setting
# https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html
# "madvise" is generally a better option than the default "always" setting
# Based on Azul instructions from https://docs.azul.com/prime/Enable-Huge-Pages#transparent-huge-pages-thp
echo madvise | sudo tee /sys/kernel/mm/transparent_hugepage/enabled
echo advise | sudo tee /sys/kernel/mm/transparent_hugepage/shmem_enabled
echo defer+madvise | sudo tee /sys/kernel/mm/transparent_hugepage/defrag
echo 1 | sudo tee /sys/kernel/mm/transparent_hugepage/khugepaged/defrag
# tune filesystem mount options, https://www.kernel.org/doc/Documentation/filesystems/ext4.txt
# commit=999999, effectively disables automatic syncing to disk (default is every 5 seconds)
@ -71,12 +77,26 @@ runs:
# stop Azure Linux agent to save RAM
sudo systemctl stop walinuxagent.service || true
# enable docker experimental mode which is
# required for using "docker build --squash" / "-Ddocker.squash=true"
daemon_json="$(sudo cat /etc/docker/daemon.json | jq '.experimental = true')"
echo "$daemon_json" | sudo tee /etc/docker/daemon.json
# restart docker daemon
sudo systemctl restart docker
echo '::endgroup::'
# show memory
echo "::group::Available Memory"
free -m
echo '::endgroup::'
# show disk
df -h
echo "::group::Available diskspace"
df -BM
echo "::endgroup::"
# show cggroup
echo "/actions_job cgroup settings:"
sudo cgget actions_job
echo "::group::Cgroup settings for current cgroup $CURRENT_CGGROUP"
CURRENT_CGGROUP=$(cat /proc/self/cgroup | grep '0::' | awk -F: '{ print $3 }')
sudo cgget -a $CURRENT_CGGROUP || true
echo '::endgroup::'
fi
shell: bash

68
.github/workflows/lint.yml vendored
View File

@ -1,68 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
name: Precommit - Helm Chart Lint
on:
pull_request:
branches:
- '*'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
lint-test:
runs-on: ubuntu-22.04
timeout-minutes: 45
steps:
- name: checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Detect changed files
id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
filters: .github/changes-filter.yaml
- name: Check changed files
id: check_changes
run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
- name: Set up Helm
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
uses: azure/setup-helm@v3
with:
version: v3.10.0
- name: Set up Python
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: Set up chart-testing
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
uses: ./.github/actions/chart-testing-action
- name: Run chart-testing (lint)
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
run: ct lint --check-version-increment=false --validate-maintainers=false --target-branch ${{ github.event.repository.default_branch }}

292
.github/workflows/pulsar-helm-chart-ci.yaml vendored Normal file
View File

@ -0,0 +1,292 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
name: Pulsar Helm Chart CI
on:
pull_request:
branches:
- master
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
preconditions:
name: Preconditions
runs-on: ubuntu-22.04
if: (github.event_name != 'schedule') || (github.repository == 'apache/pulsar-helm-chart')
outputs:
docs_only: ${{ steps.check_changes.outputs.docs_only }}
steps:
- name: checkout
uses: actions/checkout@v3
- name: Detect changed files
id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
filters: .github/changes-filter.yaml
list-files: csv
- name: Check changed files
id: check_changes
run: |
if [[ "${GITHUB_EVENT_NAME}" != "schedule" && "${GITHUB_EVENT_NAME}" != "workflow_dispatch" ]]; then
echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT
else
echo docs_only=false >> $GITHUB_OUTPUT
fi
license-check:
needs: preconditions
name: License Check
runs-on: ubuntu-22.04
timeout-minutes: 10
if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
steps:
- name: Set up Go 1.12
uses: actions/setup-go@v1
with:
go-version: 1.12
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v3
- name: Check license
run: |
go test license_test.go
# run "ct lint" https://github.com/helm/chart-testing/blob/main/doc/ct_lint.md
ct-lint:
needs: ['preconditions', 'license-check']
name: chart-testing lint
runs-on: ubuntu-22.04
timeout-minutes: 45
if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
outputs:
no_chart_changes: ${{ steps.ct-lint.outputs.no_chart_changes }}
steps:
- name: checkout
uses: actions/checkout@v3
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
if: ${{ github.repository != 'apache/pulsar-helm-chart' && github.event_name == 'pull_request' }}
uses: ./.github/actions/ssh-access
continue-on-error: true
with:
limit-access-to-actor: true
- name: Set up Helm
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
uses: azure/setup-helm@v3
with:
version: v3.10.0
- name: Set up Python
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: Set up chart-testing
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
uses: ./.github/actions/chart-testing-action
- name: Run chart-testing (lint)
id: ct-lint
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
run: |
ct lint --check-version-increment=false \
--validate-maintainers=false \
--target-branch ${{ github.event.repository.default_branch }} \
| tee /tmp/ct-lint.log || {
if grep -q "No chart changes detected." /tmp/ct-lint.log; then
echo no_chart_changes=true >> $GITHUB_OUTPUT
exit 0
else
echo no_chart_changes=false >> $GITHUB_OUTPUT
exit 1
fi
}
- name: Wait for ssh connection when build fails
# ssh access is enabled for builds in own forks
uses: ./.github/actions/ssh-access
if: ${{ failure() && github.repository != 'apache/pulsar-helm-chart' && github.event_name == 'pull_request' }}
continue-on-error: true
with:
action: wait
install-chart-tests:
name: ${{ matrix.name }} - Install
runs-on: ubuntu-22.04
timeout-minutes: ${{ matrix.timeout || 45 }}
needs: ['preconditions', 'ct-lint']
if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
strategy:
fail-fast: false
matrix:
include:
- name: Basic
values_file: .ci/clusters/values-local-pv.yaml
shortname: basic
- name: Pulsar Function
values_file: .ci/clusters/values-function.yaml
shortname: function
- name: Use Pulsar Image
values_file: .ci/clusters/values-pulsar-image.yaml
shortname: pulsar-image
- name: JWT Asymmetric Keys
values_file: .ci/clusters/values-jwt-asymmetric.yaml
shortname: jwt-asymmetric
- name: JWT Symmetric Key
values_file: .ci/clusters/values-jwt-symmetric.yaml
shortname: jwt-symmetric
- name: TLS
values_file: .ci/clusters/values-tls.yaml
shortname: tls
- name: Broker & Proxy TLS
values_file: .ci/clusters/values-broker-tls.yaml
shortname: broker-tls
- name: BK TLS Only
values_file: .ci/clusters/values-bk-tls.yaml
shortname: bk-tls
- name: ZK TLS Only
values_file: .ci/clusters/values-zk-tls.yaml
shortname: zk-tls
- name: ZK & BK TLS Only
values_file: .ci/clusters/values-zkbk-tls.yaml
shortname: zkbk-tls
steps:
- name: checkout
uses: actions/checkout@v3
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Setup debugging tools for ssh access
if: ${{ github.repository != 'apache/pulsar-helm-chart' && github.event_name == 'pull_request' }}
run: |
cat >> $HOME/.bashrc <<'EOF'
function use_kind_kubeconfig() {
export KUBECONFIG=$(ls $HOME/kind/pulsar-ci-*/kubeconfig.yaml)
}
function kubectl() {
# use kind environment's kubeconfig
if [ -z "$KUBECONFIG" ]; then
use_kind_kubeconfig
fi
command kubectl "$@"
}
function k9s() {
# use kind environment's kubeconfig
if [ -z "$KUBECONFIG" ]; then
use_kind_kubeconfig
fi
# install k9s on the fly
if [ ! -x /usr/local/bin/k9s ]; then
echo "Installing k9s..."
curl -L -s https://github.com/derailed/k9s/releases/download/v0.27.4/k9s_Linux_amd64.tar.gz | sudo tar xz -C /usr/local/bin k9s
fi
command k9s "$@"
}
EOF
cat >> $HOME/.bash_profile <<'EOF'
if [ -f ~/.bashrc ]; then
source ~/.bashrc
fi
EOF
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
if: ${{ github.repository != 'apache/pulsar-helm-chart' && github.event_name == 'pull_request' }}
uses: ./.github/actions/ssh-access
continue-on-error: true
with:
limit-access-to-actor: true
- name: Run chart-testing (install)
run: |
case "${{ matrix.shortname }}" in
"jwt-symmetric")
export SYMMETRIC=true
;;
esac
.ci/chart_test.sh ${{ matrix.values_file }}
- name: Collect k8s logs on failure
if: ${{ cancelled() || failure() }}
continue-on-error: true
shell: bash
run: |
source .ci/helm.sh
set +e
ci::collect_k8s_logs
- name: Upload k8s logs on failure
uses: actions/upload-artifact@v2
if: ${{ cancelled() || failure() }}
continue-on-error: true
with:
name: k8s-logs-${{ matrix.shortname }}
path: /tmp/k8s-logs
retention-days: 7
if-no-files-found: ignore
- name: Wait for ssh connection when build fails
# ssh access is enabled for builds in own forks
uses: ./.github/actions/ssh-access
if: ${{ failure() && github.repository != 'apache/pulsar-helm-chart' && github.event_name == 'pull_request' }}
continue-on-error: true
with:
action: wait
# This job is required for pulls to be merged.
# It depends on all other jobs in this workflow.
pulsar-helm-chart-ci-checks-completed:
name: "CI checks completed"
if: ${{ always() && ((github.event_name != 'schedule') || (github.repository == 'apache/pulsar-helm-chart')) }}
runs-on: ubuntu-22.04
timeout-minutes: 10
needs: [
'preconditions',
'license-check',
'install-chart-tests'
]
steps:
- name: Check that all required jobs were completed successfully
if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
run: |
if [[ ! ( \
"${{ needs.license-check.result }}" == "success" \
&& "${{ needs.install-chart-tests.result }}" == "success" \
) ]]; then
echo "Required jobs haven't been completed successfully."
exit 1
fi

72
.github/workflows/pulsar.yml vendored
View File

@ -1,72 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
name: Precommit - Pulsar Helm Chart (Basic Installation)
on:
pull_request:
branches:
- '*'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
lint-test:
runs-on: ubuntu-22.04
timeout-minutes: 45
steps:
- name: checkout
uses: actions/checkout@v2
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Detect changed files
id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
filters: .github/changes-filter.yaml
- name: Check changed files
id: check_changes
run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
- name: Install chart
run: |
.ci/chart_test.sh .ci/clusters/values-local-pv.yaml
# Only build a kind cluster if there are chart changes to test.
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- name: Collect k8s logs on failure
if: ${{ cancelled() || failure() }}
continue-on-error: true
shell: bash
run: |
source .ci/helm.sh
set +e
ci::collect_k8s_logs
- name: Upload k8s logs on failure
uses: actions/upload-artifact@v2
if: ${{ cancelled() || failure() }}
continue-on-error: true
with:
name: k8s-logs
path: /tmp/k8s-logs

72
.github/workflows/pulsar_bk_tls.yml vendored
View File

@ -1,72 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
name: Precommit - Pulsar Helm Chart (BK TLS Only)
on:
pull_request:
branches:
- '*'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
lint-test:
runs-on: ubuntu-22.04
timeout-minutes: 45
steps:
- name: checkout
uses: actions/checkout@v2
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Detect changed files
id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
filters: .github/changes-filter.yaml
- name: Check changed files
id: check_changes
run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
- name: Run chart-testing (install)
run: |
.ci/chart_test.sh .ci/clusters/values-bk-tls.yaml
# Only build a kind cluster if there are chart changes to test.
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- name: Collect k8s logs on failure
if: ${{ cancelled() || failure() }}
continue-on-error: true
shell: bash
run: |
source .ci/helm.sh
set +e
ci::collect_k8s_logs
- name: Upload k8s logs on failure
uses: actions/upload-artifact@v2
if: ${{ cancelled() || failure() }}
continue-on-error: true
with:
name: k8s-logs
path: /tmp/k8s-logs

72
.github/workflows/pulsar_broker_tls.yml vendored
View File

@ -1,72 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
name: Precommit - Pulsar Helm Chart (Broker & Proxy TLS Installation)
on:
pull_request:
branches:
- '*'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
lint-test:
runs-on: ubuntu-22.04
timeout-minutes: 45
steps:
- name: checkout
uses: actions/checkout@v2
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Detect changed files
id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
filters: .github/changes-filter.yaml
- name: Check changed files
id: check_changes
run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
- name: Run chart-testing (install)
run: |
.ci/chart_test.sh .ci/clusters/values-broker-tls.yaml
# Only build a kind cluster if there are chart changes to test.
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- name: Collect k8s logs on failure
if: ${{ cancelled() || failure() }}
continue-on-error: true
shell: bash
run: |
source .ci/helm.sh
set +e
ci::collect_k8s_logs
- name: Upload k8s logs on failure
uses: actions/upload-artifact@v2
if: ${{ cancelled() || failure() }}
continue-on-error: true
with:
name: k8s-logs
path: /tmp/k8s-logs

74
.github/workflows/pulsar_function.yml vendored
View File

@ -1,74 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
name: Precommit - Pulsar Helm Chart (Pulsar Function)
on:
pull_request:
branches:
- '*'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
lint-test:
runs-on: ubuntu-22.04
timeout-minutes: 45
steps:
- name: checkout
uses: actions/checkout@v2
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Detect changed files
id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
filters: .github/changes-filter.yaml
- name: Check changed files
id: check_changes
run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
- name: Install chart
run: |
.ci/chart_test.sh .ci/clusters/values-function.yaml
env:
FUNCTION: "true"
# Only build a kind cluster if there are chart changes to test.
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- name: Collect k8s logs on failure
if: ${{ cancelled() || failure() }}
continue-on-error: true
shell: bash
run: |
source .ci/helm.sh
set +e
ci::collect_k8s_logs
- name: Upload k8s logs on failure
uses: actions/upload-artifact@v2
if: ${{ cancelled() || failure() }}
continue-on-error: true
with:
name: k8s-logs
path: /tmp/k8s-logs

72
.github/workflows/pulsar_image.yml vendored
View File

@ -1,72 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
name: Precommit - Pulsar Helm Chart (Use Pulsar Image)
on:
pull_request:
branches:
- '*'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
lint-test:
runs-on: ubuntu-22.04
timeout-minutes: 45
steps:
- name: checkout
uses: actions/checkout@v2
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Detect changed files
id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
filters: .github/changes-filter.yaml
- name: Check changed files
id: check_changes
run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
- name: Install chart
run: |
.ci/chart_test.sh .ci/clusters/values-pulsar-image.yaml
# Only build a kind cluster if there are chart changes to test.
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- name: Collect k8s logs on failure
if: ${{ cancelled() || failure() }}
continue-on-error: true
shell: bash
run: |
source .ci/helm.sh
set +e
ci::collect_k8s_logs
- name: Upload k8s logs on failure
uses: actions/upload-artifact@v2
if: ${{ cancelled() || failure() }}
continue-on-error: true
with:
name: k8s-logs
path: /tmp/k8s-logs

74
.github/workflows/pulsar_jwt_asymmetric.yml vendored
View File

@ -1,74 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
name: Precommit - Pulsar Helm Chart (JWT Secret Key Installation)
on:
pull_request:
branches:
- '*'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
lint-test:
runs-on: ubuntu-22.04
timeout-minutes: 45
steps:
- name: checkout
uses: actions/checkout@v2
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Detect changed files
id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
filters: .github/changes-filter.yaml
- name: Check changed files
id: check_changes
run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
- name: Run chart-testing (install)
run: |
.ci/chart_test.sh .ci/clusters/values-jwt-asymmetric.yaml
env:
SYMMETRIC: "false"
# Only build a kind cluster if there are chart changes to test.
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- name: Collect k8s logs on failure
if: ${{ cancelled() || failure() }}
continue-on-error: true
shell: bash
run: |
source .ci/helm.sh
set +e
ci::collect_k8s_logs
- name: Upload k8s logs on failure
uses: actions/upload-artifact@v2
if: ${{ cancelled() || failure() }}
continue-on-error: true
with:
name: k8s-logs
path: /tmp/k8s-logs

74
.github/workflows/pulsar_jwt_symmetric.yml vendored
View File

@ -1,74 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
name: Precommit - Pulsar Helm Chart (JWT Public/Private Key Installation)
on:
pull_request:
branches:
- '*'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
lint-test:
runs-on: ubuntu-22.04
timeout-minutes: 45
steps:
- name: checkout
uses: actions/checkout@v2
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Detect changed files
id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
filters: .github/changes-filter.yaml
- name: Check changed files
id: check_changes
run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
- name: Run chart-testing (install)
run: |
.ci/chart_test.sh .ci/clusters/values-jwt-symmetric.yaml
env:
SYMMETRIC: "true"
# Only build a kind cluster if there are chart changes to test.
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- name: Collect k8s logs on failure
if: ${{ cancelled() || failure() }}
continue-on-error: true
shell: bash
run: |
source .ci/helm.sh
set +e
ci::collect_k8s_logs
- name: Upload k8s logs on failure
uses: actions/upload-artifact@v2
if: ${{ cancelled() || failure() }}
continue-on-error: true
with:
name: k8s-logs
path: /tmp/k8s-logs

72
.github/workflows/pulsar_tls.yml vendored
View File

@ -1,72 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
name: Precommit - Pulsar Helm Chart (TLS Installation)
on:
pull_request:
branches:
- '*'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
lint-test:
runs-on: ubuntu-22.04
timeout-minutes: 45
steps:
- name: checkout
uses: actions/checkout@v2
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Detect changed files
id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
filters: .github/changes-filter.yaml
- name: Check changed files
id: check_changes
run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
- name: Install chart
run: |
.ci/chart_test.sh .ci/clusters/values-tls.yaml
# Only build a kind cluster if there are chart changes to test.
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- name: Collect k8s logs on failure
if: ${{ cancelled() || failure() }}
continue-on-error: true
shell: bash
run: |
source .ci/helm.sh
set +e
ci::collect_k8s_logs
- name: Upload k8s logs on failure
uses: actions/upload-artifact@v2
if: ${{ cancelled() || failure() }}
continue-on-error: true
with:
name: k8s-logs
path: /tmp/k8s-logs

72
.github/workflows/pulsar_zk_tls.yml vendored
View File

@ -1,72 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
name: Precommit - Pulsar Helm Chart (ZK TLS Only)
on:
pull_request:
branches:
- '*'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
lint-test:
runs-on: ubuntu-22.04
timeout-minutes: 45
steps:
- name: checkout
uses: actions/checkout@v2
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Detect changed files
id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
filters: .github/changes-filter.yaml
- name: Check changed files
id: check_changes
run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
- name: Install chart
run: |
.ci/chart_test.sh .ci/clusters/values-zk-tls.yaml
# Only build a kind cluster if there are chart changes to test.
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- name: Collect k8s logs on failure
if: ${{ cancelled() || failure() }}
continue-on-error: true
shell: bash
run: |
source .ci/helm.sh
set +e
ci::collect_k8s_logs
- name: Upload k8s logs on failure
uses: actions/upload-artifact@v2
if: ${{ cancelled() || failure() }}
continue-on-error: true
with:
name: k8s-logs
path: /tmp/k8s-logs

72
.github/workflows/pulsar_zkbk_tls.yml vendored
View File

@ -1,72 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
name: Precommit - Pulsar Helm Chart (ZK & BK TLS Only)
on:
pull_request:
branches:
- '*'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
lint-test:
runs-on: ubuntu-22.04
timeout-minutes: 45
steps:
- name: checkout
uses: actions/checkout@v2
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Detect changed files
id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
filters: .github/changes-filter.yaml
- name: Check changed files
id: check_changes
run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
- name: Install chart
run: |
.ci/chart_test.sh .ci/clusters/values-zkbk-tls.yaml
# Only build a kind cluster if there are chart changes to test.
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- name: Collect k8s logs on failure
if: ${{ cancelled() || failure() }}
continue-on-error: true
shell: bash
run: |
source .ci/helm.sh
set +e
ci::collect_k8s_logs
- name: Upload k8s logs on failure
uses: actions/upload-artifact@v2
if: ${{ cancelled() || failure() }}
continue-on-error: true
with:
name: k8s-logs
path: /tmp/k8s-logs

48
.github/workflows/style.yml vendored
View File

@ -1,48 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
name: Precommit Style Check
on:
pull_request:
branches:
- '*'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
build:
name: Build
runs-on: ubuntu-22.04
timeout-minutes: 45
steps:
- name: Set up Go 1.12
uses: actions/setup-go@v1
with:
go-version: 1.12
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v2
- name: Check license
run: |
go test license_test.go

4
charts/pulsar/templates/proxy-configmap.yaml
View File

@ -30,7 +30,7 @@ data:
clusterName: {{ template "pulsar.cluster.name" . }}
statusFilePath: "{{ template "pulsar.home" . }}/status"
# prometheus needs to access /metrics endpoint
webServicePort: "{{ .Values.proxy.ports.http }}"
webServicePort: "{{ .Values.proxy.ports.containerPorts.http }}"
{{- if or (not .Values.tls.enabled) (not .Values.tls.proxy.enabled) }}
servicePort: "{{ .Values.proxy.ports.pulsar }}"
brokerServiceURL: pulsar://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:{{ .Values.broker.ports.pulsar }}
@ -39,7 +39,7 @@ data:
{{- if and .Values.tls.enabled .Values.tls.proxy.enabled }}
tlsEnabledInProxy: "true"
servicePortTls: "{{ .Values.proxy.ports.pulsarssl }}"
webServicePortTls: "{{ .Values.proxy.ports.https }}"
webServicePortTls: "{{ .Values.proxy.ports.containerPorts.https }}"
tlsCertificateFilePath: "/pulsar/certs/proxy/tls.crt"
tlsKeyFilePath: "/pulsar/certs/proxy/tls.key"
tlsTrustCertsFilePath: "/pulsar/certs/ca/ca.crt"

5
charts/pulsar/values.yaml
View File

@ -574,11 +574,6 @@ bookkeeper:
-XX:-ResizePLAB
-XX:+ExitOnOutOfMemoryError
-XX:+PerfDisableSharedMem
-Xlog:gc*
-Xlog:gc::utctime
-Xlog:safepoint
-Xlog:gc+heap=trace
-verbosegc
# configure the memory settings based on jvm memory settings
dbStorage_writeCacheMaxSizeMb: "32"
dbStorage_readAheadCacheMaxSizeMb: "32"

3
scripts/cert-manager/install-cert-manager.sh
View File

@ -24,7 +24,8 @@ set -e
NAMESPACE=cert-manager
NAME=cert-manager
VERSION=v1.7.3
# check compatibility with k8s versions from https://cert-manager.io/docs/installation/supported-releases/
VERSION=v1.11.4
# Install cert-manager CustomResourceDefinition resources
echo "Installing cert-manager CRD resources ..."