From 0348b1b2e5b51398475496737a1aabf1dad7af5c Mon Sep 17 00:00:00 2001 From: Paul Spooren Date: Thu, 1 Dec 2022 21:02:10 +0100 Subject: [PATCH 01/12] ci: only comment AUTORELEASE deprecation if exists If it doesn't exists, don't confuse the contributors. Signed-off-by: Paul Spooren (cherry picked from commit b3b01af314ec675b0b2ee0497a7b9c90d2e9e631) --- .../check-autorelease-deprecation.yml | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 .github/workflows/check-autorelease-deprecation.yml diff --git a/.github/workflows/check-autorelease-deprecation.yml b/.github/workflows/check-autorelease-deprecation.yml new file mode 100644 index 0000000..b85b324 --- /dev/null +++ b/.github/workflows/check-autorelease-deprecation.yml @@ -0,0 +1,91 @@ +name: Check autorelease deprecation + +on: + pull_request_target: + types: [opened, synchronize, converted_to_draft, ready_for_review, edited] + +jobs: + build: + name: Check autorelease deprecation + runs-on: ubuntu-latest + strategy: + fail-fast: false + + permissions: + pull-requests: write + + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + + - name: Determine branch name + run: | + BRANCH="${GITHUB_BASE_REF#refs/heads/}" + echo "Building for $BRANCH" + echo "BRANCH=$BRANCH" >> $GITHUB_ENV + + - name: Determine changed packages + run: | + RET=0 + + # only detect packages with changes + PKG_ROOTS=$(find . -name Makefile | \ + grep -v ".*/src/Makefile" | \ + sed -e 's@./\(.*\)/Makefile@\1/@') + CHANGES=$(git diff --diff-filter=d --name-only origin/$BRANCH...) + + for ROOT in $PKG_ROOTS; do + for CHANGE in $CHANGES; do + if [[ "$CHANGE" == "$ROOT"* ]]; then + if grep -q '$(AUTORELEASE)' "$ROOT/Makefile"; then + CONTAINS_AUTORELEASE+="$ROOT" + fi + break + fi + done + done + + if [ -n "$CONTAINS_AUTORELEASE" ]; then + RET=1 + cat > "$GITHUB_WORKSPACE/pr_comment.md" << EOF + Please do no longer set *PKG_RELEASE* to *AUTORELEASE* as the + feature is deprecated. Please use an integer instead. Below is a + list of affected packages including correct *PKG_RELEASE*: + + EOF + fi + + for ROOT in $CONTAINS_AUTORELEASE; do + echo -n " - ${ROOT}Makefile: PKG_RELEASE:=" >> "$GITHUB_WORKSPACE/pr_comment.md" + last_bump="$(git log --pretty=format:'%h %s' "$ROOT" | + grep --max-count=1 -e ': [uU]pdate to ' -e ': [bB]ump to ' | + cut -f 1 -d ' ')" + + if [ -n "$last_bump" ]; then + echo -n $(($(git rev-list --count "$last_bump..HEAD" "$ROOT") + 2)) >> "$GITHUB_WORKSPACE/pr_comment.md" + else + echo -n $(($(git rev-list --count HEAD "$ROOT") + 2)) >> "$GITHUB_WORKSPACE/pr_comment.md" + fi + echo >> "$GITHUB_WORKSPACE/pr_comment.md" + done + + exit $RET + + - name: Find Comment + uses: peter-evans/find-comment@v2 + if: ${{ failure() }} + id: fc + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + + - name: Create or update comment + uses: peter-evans/create-or-update-comment@v2 + if: ${{ failure() }} + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body-file: 'pr_comment.md' + edit-mode: replace From fa213a15a70279248b86079d93b7f2f1834a543a Mon Sep 17 00:00:00 2001 From: Tianling Shen Date: Thu, 9 Mar 2023 14:17:35 +0800 Subject: [PATCH 02/12] CI: update build architectures Removed arc_archs - archs38 was marked as source-only [1]. Renamed powerpc_8540 to powerpc_8548 [2]. 1. https://git.openwrt.org/?p=openwrt/openwrt.git;a=commit;h=c01641bcc7236d2e2de3ea65444b0cf2898df351 2. https://git.openwrt.org/?p=openwrt/openwrt.git;a=commit;h=2cad88b99fdae9766de84e6c1cb56f111eb53748 Signed-off-by: Tianling Shen (cherry picked from commit 7928f7d3990c7bae097224d90c28590a45405686) --- .github/workflows/multi-arch-test-build.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/multi-arch-test-build.yml b/.github/workflows/multi-arch-test-build.yml index c6ae0c0..7ce98f0 100644 --- a/.github/workflows/multi-arch-test-build.yml +++ b/.github/workflows/multi-arch-test-build.yml @@ -11,10 +11,6 @@ jobs: fail-fast: false matrix: include: - - arch: arc_archs - target: archs38-generic - runtime_test: false - - arch: arm_cortex-a9_vfpv3-d16 target: mvebu-cortexa9 runtime_test: false @@ -27,7 +23,7 @@ jobs: target: apm821xx-nand runtime_test: false - - arch: powerpc_8540 + - arch: powerpc_8548 target: mpc85xx-p1010 runtime_test: false From 9934454586faa749af79f45a9cab1807ff963128 Mon Sep 17 00:00:00 2001 From: Jeffery To Date: Tue, 21 Mar 2023 14:41:48 +0800 Subject: [PATCH 03/12] CI: Add --autoremove, ignore removal errors 836b4e1c734f9705bfd8db0da0c04214c2647932 added --force-removal-of-dependent-packages but it does not do what the commit message says it does. When package A depends on package B (package B is a dependency of package A; package A is a dependent of package B), trying to remove package B while package A is installed will result in an error. Adding --force-removal-of-dependent-packages in this case will cause the removal of package B and package A (package B's dependent). But in the case of the CI testing step, it is package A that is being installed and removed. Removing package A with --force-removal-of-dependent-packages will not cause package B (package A's dependency) to be removed. This adds --autoremove to actually remove the package's dependencies. This also ignores any errors returned by opkg remove as --autoremove can sometimes falsely return an error[1]. [1]: https://github.com/openwrt/openwrt/issues/12241 Fixes: 836b4e1c734f ("github-ci: add --force-removal-of-dependent-packages") Signed-off-by: Jeffery To (cherry picked from commit 53a53b8ee38d79083ff59e656798948f9b0a7e68) --- .github/workflows/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/entrypoint.sh b/.github/workflows/entrypoint.sh index b88a6c1..28b1934 100755 --- a/.github/workflows/entrypoint.sh +++ b/.github/workflows/entrypoint.sh @@ -39,5 +39,5 @@ for PKG in /ci/*.ipk; do echo "No test.sh script available" fi - opkg remove "$PKG_NAME" --force-removal-of-dependent-packages --force-remove + opkg remove "$PKG_NAME" --force-removal-of-dependent-packages --force-remove --autoremove || true done From 2ae05f5841af1b5117ad3327033df38c0db5e510 Mon Sep 17 00:00:00 2001 From: Jeffery To Date: Wed, 22 Mar 2023 15:46:13 +0800 Subject: [PATCH 04/12] CI: Add local feed for CI-built packages To test each package, the CI-built target package (ipk) file is installed, but currently the target package's dependencies are installed from the standard opkg feeds. There are cases when the CI-built target packages should be installed/tested together: * If a pull request contains several new packages that depend on each other, the test step will fail as the new dependencies cannot be found in the current packages feed. * If a pull request upgrades a source package that builds several target packages that depend on each other, the test step may fail due to the version/ABI mismatch between a newer target package and the older dependencies installed from the packages feed. This sets up a local feed for the CI-built packages so that dependencies are also installed from the same set of packages. Signed-off-by: Jeffery To (cherry picked from commit f818fe716ea126dd8240c9600f92e5b412d59060) --- .github/workflows/entrypoint.sh | 9 +++++++-- .github/workflows/multi-arch-test-build.yml | 21 ++++++++++++++++++--- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/.github/workflows/entrypoint.sh b/.github/workflows/entrypoint.sh index 28b1934..6332011 100755 --- a/.github/workflows/entrypoint.sh +++ b/.github/workflows/entrypoint.sh @@ -2,7 +2,12 @@ # not enabling `errtrace` and `pipefail` since those are bash specific set -o errexit # failing commands causes script to fail -set -o nounset # undefined variables causes script to fail +set -o nounset # undefined variables causes script to fail + +echo "src/gz packages_ci file:///ci" >> /etc/opkg/distfeeds.conf + +FINGERPRINT="$(usign -F -p /ci/packages_ci.pub)" +cp /ci/packages_ci.pub "/etc/opkg/keys/$FINGERPRINT" mkdir -p /var/lock/ @@ -11,7 +16,7 @@ opkg update [ -n "${CI_HELPER:=''}" ] || CI_HELPER="/ci/.github/workflows/ci_helpers.sh" for PKG in /ci/*.ipk; do - tar -xzOf "$PKG" ./control.tar.gz | tar xzf - ./control + tar -xzOf "$PKG" ./control.tar.gz | tar xzf - ./control # package name including variant PKG_NAME=$(sed -ne 's#^Package: \(.*\)$#\1#p' ./control) # package version without release diff --git a/.github/workflows/multi-arch-test-build.yml b/.github/workflows/multi-arch-test-build.yml index 7ce98f0..9214af7 100644 --- a/.github/workflows/multi-arch-test-build.yml +++ b/.github/workflows/multi-arch-test-build.yml @@ -78,20 +78,35 @@ jobs: echo "Building $PACKAGES" echo "PACKAGES=$PACKAGES" >> $GITHUB_ENV + - name: Generate build keys + run: | + sudo apt-get install -y signify-openbsd + signify-openbsd -G -n -c 'DO NOT USE - OpenWrt packages feed CI' -p packages_ci.pub -s packages_ci.sec + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + echo "KEY_BUILD<<$EOF" >> $GITHUB_ENV + cat packages_ci.sec >> $GITHUB_ENV + echo "$EOF" >> $GITHUB_ENV + - name: Build uses: openwrt/gh-action-sdk@v5 env: ARCH: ${{ matrix.arch }}-${{ env.BRANCH }} FEEDNAME: packages_ci + INDEX: 1 + KEY_BUILD: ${{ env.KEY_BUILD }} - name: Move created packages to project dir - run: cp bin/packages/${{ matrix.arch }}/packages_ci/*.ipk . || true + run: cp bin/packages/${{ matrix.arch }}/packages_ci/* . || true - name: Store packages uses: actions/upload-artifact@v3 with: - name: ${{ matrix.arch}}-packages - path: "*.ipk" + name: ${{env.ARCHIVE_NAME}}-packages + path: | + Packages + Packages.* + *.ipk + PKG-INFO - name: Store logs uses: actions/upload-artifact@v3 From c960329d0e5d2cc43c5a8082c919c8e3bf5b61c9 Mon Sep 17 00:00:00 2001 From: Gerard Ryan Date: Sat, 29 Apr 2023 16:25:49 +1000 Subject: [PATCH 05/12] cache-domains: added pre-test.sh CI step Some packages variants have conflicting dependencies with the base packages and the CI test will fail to install before anything can be done by the packages to setup the system for install. This change adds a pre-test.sh that runs before the install so things like the default libustream variant can be swapped out as shown in the updated cache-domains. Signed-off-by: Gerard Ryan (cherry picked from commit 61997c86b01c83320a2cbb1b490941a8af741253) --- .github/workflows/entrypoint.sh | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/entrypoint.sh b/.github/workflows/entrypoint.sh index 6332011..a0fdae3 100755 --- a/.github/workflows/entrypoint.sh +++ b/.github/workflows/entrypoint.sh @@ -26,10 +26,24 @@ for PKG in /ci/*.ipk; do echo "Testing package $PKG_NAME in version $PKG_VERSION from $PKG_SOURCE" - opkg install "$PKG" - export PKG_NAME PKG_VERSION CI_HELPER + PRE_TEST_SCRIPT=$(find /ci/ -name "$PKG_SOURCE" -type d)/pre-test.sh + + if [ -f "$PRE_TEST_SCRIPT" ]; then + echo "Use package specific pre-test.sh" + if sh "$PRE_TEST_SCRIPT" "$PKG_NAME" "$PKG_VERSION"; then + echo "Pre-test successful" + else + echo "Pre-test failed" + exit 1 + fi + else + echo "No pre-test.sh script available" + fi + + opkg install "$PKG" + TEST_SCRIPT=$(find /ci/ -name "$PKG_SOURCE" -type d)/test.sh if [ -f "$TEST_SCRIPT" ]; then From 68987a112f07d6e64b45966806febe2a4dfa6d50 Mon Sep 17 00:00:00 2001 From: Van Waholtz Date: Mon, 1 May 2023 19:04:24 +0800 Subject: [PATCH 06/12] CI: Enable runtime_test for mips_24kc Signed-off-by: Van Waholtz (cherry picked from commit 28d204148a7cad8f1ee903bf6af4dae234b28119) --- .github/workflows/multi-arch-test-build.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/multi-arch-test-build.yml b/.github/workflows/multi-arch-test-build.yml index 9214af7..b0da82c 100644 --- a/.github/workflows/multi-arch-test-build.yml +++ b/.github/workflows/multi-arch-test-build.yml @@ -17,7 +17,7 @@ jobs: - arch: mips_24kc target: ath79-generic - runtime_test: false + runtime_test: true - arch: powerpc_464fp target: apm821xx-nand @@ -120,7 +120,9 @@ jobs: - name: Register QEMU if: ${{ matrix.runtime_test }} run: | - sudo docker run --rm --privileged aptman/qus -s -- -p + sudo apt-get update + sudo apt-get install -y qemu-user-static binfmt-support + sudo update-binfmts --import - name: Build Docker container if: ${{ matrix.runtime_test }} From 6ed96f5be10da3476b5641353cfb661be6061a2d Mon Sep 17 00:00:00 2001 From: Jeffery To Date: Wed, 10 May 2023 13:51:59 +0800 Subject: [PATCH 07/12] CI: Run "apt-get update" before installing signify-openbsd Signed-off-by: Jeffery To (cherry picked from commit 89cf0d561977c028b45e5ab6db1b34dab27b9a11) --- .github/workflows/multi-arch-test-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/multi-arch-test-build.yml b/.github/workflows/multi-arch-test-build.yml index b0da82c..25af766 100644 --- a/.github/workflows/multi-arch-test-build.yml +++ b/.github/workflows/multi-arch-test-build.yml @@ -80,6 +80,7 @@ jobs: - name: Generate build keys run: | + sudo apt-get update sudo apt-get install -y signify-openbsd signify-openbsd -G -n -c 'DO NOT USE - OpenWrt packages feed CI' -p packages_ci.pub -s packages_ci.sec EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) From 30b11070ade2c7d0273b50f85253c7a662da0371 Mon Sep 17 00:00:00 2001 From: Jeffery To Date: Wed, 10 May 2023 14:10:37 +0800 Subject: [PATCH 08/12] CI: Do package run-tests only if target packages were built Currently, the package run-test phase will fail for PRs that only add/update host-only packages, as no target packages (*.ipk) are built. This checks if any target packages are built before attempting the run-tests. Signed-off-by: Jeffery To (cherry picked from commit 299ee2b1cfd20a7c9d31e8024b4333e8da035af3) --- .github/workflows/multi-arch-test-build.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/multi-arch-test-build.yml b/.github/workflows/multi-arch-test-build.yml index 25af766..3aabab4 100644 --- a/.github/workflows/multi-arch-test-build.yml +++ b/.github/workflows/multi-arch-test-build.yml @@ -118,21 +118,32 @@ jobs: - name: Remove logs run: sudo rm -rf logs/ || true + - name: Check if any packages were built + run: | + if [ -n "$(find . -maxdepth 1 -type f -name '*.ipk' -print -quit)" ]; then + echo "Found *.ipk files" + HAVE_IPKS=true + else + echo "No *.ipk files found" + HAVE_IPKS=false + fi + echo "HAVE_IPKS=$HAVE_IPKS" >> $GITHUB_ENV + - name: Register QEMU - if: ${{ matrix.runtime_test }} + if: ${{ matrix.runtime_test && fromJSON(env.HAVE_IPKS) }} run: | sudo apt-get update sudo apt-get install -y qemu-user-static binfmt-support sudo update-binfmts --import - name: Build Docker container - if: ${{ matrix.runtime_test }} + if: ${{ matrix.runtime_test && fromJSON(env.HAVE_IPKS) }} run: | docker build -t test-container --build-arg ARCH .github/workflows/ env: ARCH: ${{ matrix.arch }}-${{ env.BRANCH }} - name: Test via Docker container - if: ${{ matrix.runtime_test }} + if: ${{ matrix.runtime_test && fromJSON(env.HAVE_IPKS) }} run: | docker run --rm -v $GITHUB_WORKSPACE:/ci test-container From ad46bbb4a663721e89061d99f41ed7228101dc8f Mon Sep 17 00:00:00 2001 From: Christian Marangi Date: Fri, 26 May 2023 14:43:12 +0200 Subject: [PATCH 09/12] CI: add concurrency rules to skip redundant build Add concurrency rules to skip redundant build to skip extra build test on force push on pull request. Signed-off-by: Christian Marangi (cherry picked from commit 5aa683be13c9b1f4a2bdb09ec0481b7cd546dd77) --- .github/workflows/multi-arch-test-build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/multi-arch-test-build.yml b/.github/workflows/multi-arch-test-build.yml index 3aabab4..c40bf01 100644 --- a/.github/workflows/multi-arch-test-build.yml +++ b/.github/workflows/multi-arch-test-build.yml @@ -3,6 +3,10 @@ name: Test Build on: pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: build: name: Test ${{ matrix.arch }} From 0de5679741a84a60a1ef44e02ebb1a27ecc730a9 Mon Sep 17 00:00:00 2001 From: Jeffery To Date: Fri, 2 Jun 2023 22:04:38 +0800 Subject: [PATCH 10/12] CI: Sort build architectures in alphabetical order Signed-off-by: Jeffery To (cherry picked from commit 06b7d9b665d6299571a305b49e1bc159290d16cd) --- .github/workflows/multi-arch-test-build.yml | 24 ++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/multi-arch-test-build.yml b/.github/workflows/multi-arch-test-build.yml index c40bf01..ceb843d 100644 --- a/.github/workflows/multi-arch-test-build.yml +++ b/.github/workflows/multi-arch-test-build.yml @@ -15,10 +15,22 @@ jobs: fail-fast: false matrix: include: + - arch: aarch64_cortex-a53 + target: mvebu-cortexa53 + runtime_test: true + + - arch: arm_cortex-a15_neon-vfpv4 + target: armvirt-32 + runtime_test: true + - arch: arm_cortex-a9_vfpv3-d16 target: mvebu-cortexa9 runtime_test: false + - arch: i386_pentium-mmx + target: x86-geode + runtime_test: true + - arch: mips_24kc target: ath79-generic runtime_test: true @@ -31,18 +43,6 @@ jobs: target: mpc85xx-p1010 runtime_test: false - - arch: aarch64_cortex-a53 - target: mvebu-cortexa53 - runtime_test: true - - - arch: arm_cortex-a15_neon-vfpv4 - target: armvirt-32 - runtime_test: true - - - arch: i386_pentium-mmx - target: x86-geode - runtime_test: true - - arch: x86_64 target: x86-64 runtime_test: true From 565e1cbdd99cac3a0f6c4ab46067c6a088618680 Mon Sep 17 00:00:00 2001 From: Jeffery To Date: Fri, 2 Jun 2023 22:09:46 +0800 Subject: [PATCH 11/12] CI: Add build test for riscv64_riscv64 Signed-off-by: Jeffery To (cherry picked from commit f165789cedad0b17fcfed4422a82d769159a46f1) --- .github/workflows/multi-arch-test-build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/multi-arch-test-build.yml b/.github/workflows/multi-arch-test-build.yml index ceb843d..a51b32c 100644 --- a/.github/workflows/multi-arch-test-build.yml +++ b/.github/workflows/multi-arch-test-build.yml @@ -43,6 +43,10 @@ jobs: target: mpc85xx-p1010 runtime_test: false + - arch: riscv64_riscv64 + target: sifiveu-generic + runtime_test: false + - arch: x86_64 target: x86-64 runtime_test: true From ea8de65fb7095d77e40863fb1f75f667b98b2018 Mon Sep 17 00:00:00 2001 From: Tianling Shen Date: Sun, 11 Jun 2023 13:28:03 +0800 Subject: [PATCH 12/12] CI: update build architectures - armvirt target has been renamed to armsr (Arm SystemReady) [1]. - armsr-armv8 has been switched to aarch64_generic architecture [2]. 1. https://github.com/openwrt/openwrt/commit/40b02a230167626def69389452f19b7109aaeac1 2. https://github.com/openwrt/openwrt/commit/e0f06ddc23b2503a1791ae7e97b02e2647e8a70d Signed-off-by: Tianling Shen (cherry picked from commit 04a287eea05b066fc198836d7ada431887497436) --- .github/workflows/multi-arch-test-build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/multi-arch-test-build.yml b/.github/workflows/multi-arch-test-build.yml index a51b32c..0fb6c60 100644 --- a/.github/workflows/multi-arch-test-build.yml +++ b/.github/workflows/multi-arch-test-build.yml @@ -15,12 +15,12 @@ jobs: fail-fast: false matrix: include: - - arch: aarch64_cortex-a53 - target: mvebu-cortexa53 + - arch: aarch64_generic + target: armsr-armv8 runtime_test: true - arch: arm_cortex-a15_neon-vfpv4 - target: armvirt-32 + target: armsr-armv7 runtime_test: true - arch: arm_cortex-a9_vfpv3-d16