[CI] Improve change detection by reusing solution from apache/pulsar (#194)

* [CI] Improve change detection by reusing solution from apache/pulsar

* Fix verify release
This commit is contained in:
Lari Hotari 2022-01-04 11:40:40 +02:00 committed by GitHub
parent 83bb8bd60f
commit 9b672379dd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 296 additions and 150 deletions

View File

@ -81,7 +81,7 @@ function release::publish_charts() {
if [[ "x${PUBLISH_CHARTS}" == "xtrue" ]]; then
git push --set-upstream origin asf-site
else
git push --dry-run --set-upstream origin asf-site
echo "Skipping publishing charts"
fi
}

View File

@ -0,0 +1,82 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
name: Tune Runner VM performance
description: tunes the GitHub Runner VM operation system
runs:
using: composite
steps:
- run: |
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
# Ensure that reverse lookups for current hostname are handled properly
# Add the current IP address, long hostname and short hostname record to /etc/hosts file
echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
# The default vm.swappiness setting is 60 which has a tendency to start swapping when memory
# consumption is high.
# Set vm.swappiness=1 to avoid swapping and allow high RAM usage
echo 1 | sudo tee /proc/sys/vm/swappiness
# Set swappiness to 1 for all cgroups and sub-groups
for swappiness_dir in /sys/fs/cgroup/memory/*/ /sys/fs/cgroup/memory/*/*/; do
if [ -d "swappiness_dir" ]; then
echo 1 | sudo tee $(swappiness_dir)memory.swappiness > /dev/null
fi
done
# use "madvise" Linux Transparent HugePages (THP) setting
# https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html
# "madvise" is generally a better option than the default "always" setting
echo madvise | sudo tee /sys/kernel/mm/transparent_hugepage/enabled
# tune filesystem mount options, https://www.kernel.org/doc/Documentation/filesystems/ext4.txt
# commit=999999, effectively disables automatic syncing to disk (default is every 5 seconds)
# nobarrier/barrier=0, loosen data consistency on system crash (no negative impact to empheral CI nodes)
sudo mount -o remount,nodiscard,commit=999999,barrier=0 /
sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt
# disable discard/trim at device level since remount with nodiscard doesn't seem to be effective
# https://www.spinics.net/lists/linux-ide/msg52562.html
for i in /sys/block/sd*/queue/discard_max_bytes; do
echo 0 | sudo tee $i
done
# disable any background jobs that run SSD discard/trim
sudo systemctl disable fstrim.timer || true
sudo systemctl stop fstrim.timer || true
sudo systemctl disable fstrim.service || true
sudo systemctl stop fstrim.service || true
# stop php-fpm
sudo systemctl stop php8.0-fpm.service || true
sudo systemctl stop php7.4-fpm.service || true
# stop mono-xsp4
sudo systemctl disable mono-xsp4.service || true
sudo systemctl stop mono-xsp4.service || true
sudo killall mono || true
# stop Azure Linux agent to save RAM
sudo systemctl stop walinuxagent.service || true
# show memory
free -m
# show disk
df -h
# show cggroup
echo "/actions_job cgroup settings:"
sudo cgget actions_job
fi
shell: bash

28
.github/changes-filter.yaml vendored Normal file
View File

@ -0,0 +1,28 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# contains pattern definitions used in workflows "changes" step
# pattern syntax: https://github.com/micromatch/picomatch
all:
- '**'
docs:
- 'examples/**'
- '.asf.yaml'
- '*.md'
- '**/*.md'

View File

@ -22,28 +22,29 @@ on:
pull_request:
branches:
- '*'
paths:
- 'charts/pulsar/**'
- '.ci/**'
- 'hack/**'
jobs:
lint-test:
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- name: Checkout
- name: checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- name: Check if this pull request only changes documentation
id: docs
uses: apache/pulsar-test-infra/diff-only@master
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Detect changed files
id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
args: site2 .asf.yaml ct.yaml
filters: .github/changes-filter.yaml
- name: Check changed files
id: check_changes
run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
- name: Lint chart
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
id: lint
uses: helm/chart-testing-action@v2.0.0
with:

View File

@ -22,29 +22,30 @@ on:
pull_request:
branches:
- '*'
paths:
- 'charts/pulsar/**'
- '.ci/**'
- 'hack/**'
jobs:
lint-test:
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- name: Checkout
- name: checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- name: Check if this pull request only changes documentation
id: docs
uses: apache/pulsar-test-infra/diff-only@master
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Detect changed files
id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
args: site2 .asf.yaml ct.yaml
filters: .github/changes-filter.yaml
- name: Check changed files
id: check_changes
run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
- name: Install chart
run: |
.ci/chart_test.sh .ci/clusters/values-local-pv.yaml
# Only build a kind cluster if there are chart changes to test.
if: steps.docs.outputs.changed_only == 'no'
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}

View File

@ -22,29 +22,29 @@ on:
pull_request:
branches:
- '*'
paths:
- 'charts/pulsar/**'
- '.ci/**'
- 'hack/**'
jobs:
lint-test:
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- name: Checkout
- name: checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- name: Check if this pull request only changes documentation
id: docs
uses: apache/pulsar-test-infra/diff-only@master
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Detect changed files
id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
args: site2 .asf.yaml ct.yaml
filters: .github/changes-filter.yaml
- name: Check changed files
id: check_changes
run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
- name: Run chart-testing (install)
run: |
.ci/chart_test.sh .ci/clusters/values-bk-tls.yaml
# Only build a kind cluster if there are chart changes to test.
if: steps.docs.outputs.changed_only == 'no'
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}

View File

@ -22,29 +22,30 @@ on:
pull_request:
branches:
- '*'
paths:
- 'charts/pulsar/**'
- '.ci/**'
- 'hack/**'
jobs:
lint-test:
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- name: Checkout
- name: checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- name: Check if this pull request only changes documentation
id: docs
uses: apache/pulsar-test-infra/diff-only@master
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Detect changed files
id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
args: site2 .asf.yaml ct.yaml
filters: .github/changes-filter.yaml
- name: Check changed files
id: check_changes
run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
- name: Run chart-testing (install)
run: |
.ci/chart_test.sh .ci/clusters/values-broker-tls.yaml
# Only build a kind cluster if there are chart changes to test.
if: steps.docs.outputs.changed_only == 'no'
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}

View File

@ -22,26 +22,27 @@ on:
pull_request:
branches:
- '*'
paths:
- 'charts/pulsar/**'
- '.ci/**'
- 'hack/**'
jobs:
lint-test:
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- name: Checkout
- name: checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- name: Check if this pull request only changes documentation
id: docs
uses: apache/pulsar-test-infra/diff-only@master
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Detect changed files
id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
args: site2 .asf.yaml ct.yaml
filters: .github/changes-filter.yaml
- name: Check changed files
id: check_changes
run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
- name: Install chart
run: |
@ -49,4 +50,4 @@ jobs:
env:
FUNCTION: "true"
# Only build a kind cluster if there are chart changes to test.
if: steps.docs.outputs.changed_only == 'no'
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}

View File

@ -22,29 +22,30 @@ on:
pull_request:
branches:
- '*'
paths:
- 'charts/pulsar/**'
- '.ci/**'
- 'hack/**'
jobs:
lint-test:
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- name: Checkout
- name: checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- name: Check if this pull request only changes documentation
id: docs
uses: apache/pulsar-test-infra/diff-only@master
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Detect changed files
id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
args: site2 .asf.yaml ct.yaml
filters: .github/changes-filter.yaml
- name: Check changed files
id: check_changes
run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
- name: Install chart
run: |
.ci/chart_test.sh .ci/clusters/values-pulsar-image.yaml
# Only build a kind cluster if there are chart changes to test.
if: steps.docs.outputs.changed_only == 'no'
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}

View File

@ -22,26 +22,27 @@ on:
pull_request:
branches:
- '*'
paths:
- 'charts/pulsar/**'
- '.ci/**'
- 'hack/**'
jobs:
lint-test:
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- name: Checkout
- name: checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- name: Check if this pull request only changes documentation
id: docs
uses: apache/pulsar-test-infra/diff-only@master
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Detect changed files
id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
args: site2 .asf.yaml ct.yaml
filters: .github/changes-filter.yaml
- name: Check changed files
id: check_changes
run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
- name: Run chart-testing (install)
run: |
@ -49,4 +50,4 @@ jobs:
env:
SYMMETRIC: "false"
# Only build a kind cluster if there are chart changes to test.
if: steps.docs.outputs.changed_only == 'no'
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}

View File

@ -22,26 +22,27 @@ on:
pull_request:
branches:
- '*'
paths:
- 'charts/pulsar/**'
- '.ci/**'
- 'hack/**'
jobs:
lint-test:
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- name: Checkout
- name: checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- name: Check if this pull request only changes documentation
id: docs
uses: apache/pulsar-test-infra/diff-only@master
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Detect changed files
id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
args: site2 .asf.yaml ct.yaml
filters: .github/changes-filter.yaml
- name: Check changed files
id: check_changes
run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
- name: Run chart-testing (install)
run: |
@ -49,4 +50,4 @@ jobs:
env:
SYMMETRIC: "true"
# Only build a kind cluster if there are chart changes to test.
if: steps.docs.outputs.changed_only == 'no'
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}

View File

@ -22,29 +22,30 @@ on:
pull_request:
branches:
- '*'
paths:
- 'charts/pulsar/**'
- '.ci/**'
- 'hack/**'
jobs:
lint-test:
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- name: Checkout
- name: checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- name: Check if this pull request only changes documentation
id: docs
uses: apache/pulsar-test-infra/diff-only@master
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Detect changed files
id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
args: site2 .asf.yaml ct.yaml
filters: .github/changes-filter.yaml
- name: Check changed files
id: check_changes
run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
- name: Install chart
run: |
.ci/chart_test.sh .ci/clusters/values-tls.yaml
# Only build a kind cluster if there are chart changes to test.
if: steps.docs.outputs.changed_only == 'no'
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}

View File

@ -22,29 +22,30 @@ on:
pull_request:
branches:
- '*'
paths:
- 'charts/pulsar/**'
- '.ci/**'
- 'hack/**'
jobs:
lint-test:
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- name: Checkout
- name: checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- name: Check if this pull request only changes documentation
id: docs
uses: apache/pulsar-test-infra/diff-only@master
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Detect changed files
id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
args: site2 .asf.yaml ct.yaml
filters: .github/changes-filter.yaml
- name: Check changed files
id: check_changes
run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
- name: Install chart
run: |
.ci/chart_test.sh .ci/clusters/values-zk-tls.yaml
# Only build a kind cluster if there are chart changes to test.
if: steps.docs.outputs.changed_only == 'no'
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}

View File

@ -22,29 +22,30 @@ on:
pull_request:
branches:
- '*'
paths:
- 'charts/pulsar/**'
- '.ci/**'
- 'hack/**'
jobs:
lint-test:
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- name: Checkout
- name: checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- name: Check if this pull request only changes documentation
id: docs
uses: apache/pulsar-test-infra/diff-only@master
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Detect changed files
id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
args: site2 .asf.yaml ct.yaml
filters: .github/changes-filter.yaml
- name: Check changed files
id: check_changes
run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
- name: Install chart
run: |
.ci/chart_test.sh .ci/clusters/values-zkbk-tls.yaml
# Only build a kind cluster if there are chart changes to test.
if: steps.docs.outputs.changed_only == 'no'
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}

View File

@ -27,13 +27,27 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- name: Checkout
- name: checkout
uses: actions/checkout@v2
with:
token: ${{ secrets.PULSARBOT_TOKEN }}
fetch-depth: 0
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Detect changed files
id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
filters: .github/changes-filter.yaml
- name: Check changed files
id: check_changes
run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
- name: Install chart
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
env:
GITHUB_TOKEN: ${{ secrets.PULSARBOT_TOKEN }}
PUBLISH_CHARTS: true

View File

@ -29,7 +29,6 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- name: Set up Go 1.12
uses: actions/setup-go@v1
with:

View File

@ -22,19 +22,32 @@ on:
pull_request:
branches:
- '*'
paths:
- 'charts/pulsar/**'
- '.ci/**'
- 'hack/**'
jobs:
release:
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- name: Checkout
- name: checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Detect changed files
id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
filters: .github/changes-filter.yaml
- name: Check changed files
id: check_changes
run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
- name: Install chart
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PUBLISH_CHARTS: false