build: Add CircleCI

This adds CircleCI as Continous Integration. It will provide compile test against ath79 target and together with it, it checks if the commit starts with the package name and it checks if the Signed-off-by matches the author of the commit. For example, it can avoid mistakes with broken patches, mismatched changes. This comes from packages feed and it is being used there and also in LuCI repository. Signed-off-by: Josef Schlehofer <pepe.schlehofer@gmail.com>
2020-03-08 00:01:54 +01:00 · 2020-03-08 00:01:54 +01:00 · 019232528e
commit 019232528e
parent 2d17fc0295
3 changed files with 251 additions and 0 deletions
--- a/.circleci/Dockerfile
+++ b/.circleci/Dockerfile
@ -0,0 +1,63 @@
+FROM debian:9
+
+
+# Configuration version history
+# v1.0   - Initial version by Etienne Champetier
+# v1.0.1 - Run as non-root, add unzip, xz-utils
+# v1.0.2 - Add bzr
+# v1.0.3 - Verify usign signatures
+# v1.0.4 - Add support for Python3
+# v1.0.5 - Add 19.07 public keys, verify keys
+
+RUN apt update && apt install -y \
+build-essential \
+bzr \
+curl \
+jq \
+gawk \
+gettext \
+git \
+libncurses5-dev \
+libssl-dev \
+python \
+python3 \
+signify-openbsd \
+subversion \
+time \
+unzip \
+wget \
+xz-utils \
+zlib1g-dev \
+&& rm -rf /var/lib/apt/lists/*
+
+RUN useradd -c "OpenWrt Builder" -m -d /home/build -s /bin/bash build
+USER build
+ENV HOME /home/build
+
+# OpenWrt Build System (PGP key for unattended snapshot builds)
+RUN curl 'https://git.openwrt.org/?p=keyring.git;a=blob_plain;f=gpg/626471F1.asc' | gpg --import \
+ && gpg --fingerprint --with-colons '<pgpsign-snapshots@openwrt.org>' | grep '^fpr:::::::::54CC74307A2C6DC9CE618269CD84BCED626471F1:$' \
+ && echo '54CC74307A2C6DC9CE618269CD84BCED626471F1:6:' | gpg --import-ownertrust
+
+# OpenWrt Build System (PGP key for 17.01 "Reboot" release builds)
+RUN curl 'https://git.openwrt.org/?p=keyring.git;a=blob_plain;f=gpg/D52BBB6B.asc' | gpg --import \
+ && gpg --fingerprint --with-colons '<pgpsign-17.01@openwrt.org>' | grep '^fpr:::::::::B09BE781AE8A0CD4702FDCD3833C6010D52BBB6B:$' \
+ && echo 'B09BE781AE8A0CD4702FDCD3833C6010D52BBB6B:6:' | gpg --import-ownertrust
+
+# OpenWrt Release Builder (18.06 Signing Key)
+RUN curl 'https://git.openwrt.org/?p=keyring.git;a=blob_plain;f=gpg/17E1CE16.asc' | gpg --import \
+ && gpg --fingerprint --with-colons '<openwrt-devel@lists.openwrt.org>' | grep '^fpr:::::::::6768C55E79B032D77A28DA5F0F20257417E1CE16:$' \
+ && echo '6768C55E79B032D77A28DA5F0F20257417E1CE16:6:' | gpg --import-ownertrust
+
+# OpenWrt Build System (PGP key for 19.07 release builds)
+RUN curl 'https://git.openwrt.org/?p=keyring.git;a=blob_plain;f=gpg/2074BE7A.asc' | gpg --import \
+ && gpg --fingerprint --with-colons '<pgpsign-19.07@openwrt.org>' | grep '^fpr:::::::::D9C6901F45C9B86858687DFF28A39BC32074BE7A:$' \
+ && echo 'D9C6901F45C9B86858687DFF28A39BC32074BE7A:6:' | gpg --import-ownertrust
+
+# untrusted comment: Public usign key for unattended snapshot builds
+RUN curl 'https://git.openwrt.org/?p=keyring.git;a=blob_plain;f=usign/b5043e70f9a75cde' --create-dirs -o /home/build/usign/b5043e70f9a75cde \
+ && echo 'd7ac10f9ed1b38033855f3d27c9327d558444fca804c685b17d9dcfb0648228f */home/build/usign/b5043e70f9a75cde' | sha256sum --check
+
+# untrusted comment: Public usign key for 19.07 release builds
+RUN curl 'https://git.openwrt.org/?p=keyring.git;a=blob_plain;f=usign/f94b9dd6febac963' --create-dirs -o /home/build/usign/f94b9dd6febac963 \
+ && echo 'b1d09457cfbc36fccfe18382d65c54a2ade3e7fd3902da490a53aa517b512755 */home/build/usign/f94b9dd6febac963' | sha256sum --check
--- a/.circleci/README
+++ b/.circleci/README
@ -0,0 +1,6 @@
+# Build/update the docker image
+
+docker pull debian:9
+docker build --rm -t docker.io/openwrtorg/packages-cci:latest .
+docker tag <IMAGE ID> docker.io/openwrtorg/packages-cci:<VERSION-TAG>
+docker push docker.io/openwrtorg/packages-cci
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -0,0 +1,182 @@
+version: 2.0
+jobs:
+  build:
+    docker:
+      - image: docker.io/openwrtorg/packages-cci:v1.0.5
+    environment:
+      - SDK_HOST: "downloads.openwrt.org"
+      - SDK_PATH: "snapshots/targets/ath79/generic"
+      - SDK_FILE: "openwrt-sdk-ath79-generic_*.Linux-x86_64.tar.xz"
+      - BRANCH: "master"
+    steps:
+      - checkout:
+          path: ~/openwrt_telephony
+
+      - run:
+          name: Check changes / verify commits
+          working_directory: ~/openwrt_telephony
+          command: |
+             cat >> $BASH_ENV <<EOF
+             echo_red()   { printf "\033[1;31m\$*\033[m\n"; }
+             echo_green() { printf "\033[1;32m\$*\033[m\n"; }
+             echo_blue()  { printf "\033[1;34m\$*\033[m\n"; }
+             EOF
+             source $BASH_ENV
+
+             RET=0
+             for commit in $(git rev-list HEAD ^origin/$BRANCH); do
+               echo_blue "=== Checking commit '$commit'"
+               if git show --format='%P' -s $commit | grep -qF ' '; then
+                 echo_red "Pull request should not include merge commits"
+                 RET=1
+               fi
+
+               author="$(git show -s --format=%aN $commit)"
+               if echo $author | grep -q '\S\+\s\+\S\+'; then
+                 echo_green "Author name ($author) seems ok"
+               else
+                 echo_red "Author name ($author) need to be your real name 'firstname lastname'"
+                 RET=1
+               fi
+
+               subject="$(git show -s --format=%s $commit)"
+               if echo "$subject" | grep -q -e '^[0-9A-Za-z,+/_-]\+: ' -e '^Revert '; then
+                 echo_green "Commit subject line seems ok ($subject)"
+               else
+                 echo_red "Commit subject line MUST start with '<package name>: ' ($subject)"
+                 RET=1
+               fi
+
+               body="$(git show -s --format=%b $commit)"
+               sob="$(git show -s --format='Signed-off-by: %aN <%aE>' $commit)"
+               if echo "$body" | grep -qF "$sob"; then
+                 echo_green "Signed-off-by match author"
+               else
+                 echo_red "Signed-off-by is missing or doesn't match author (should be '$sob')"
+                 RET=1
+               fi
+             done
+
+             exit $RET
+
+      - run:
+          name: Download the SDK
+          working_directory: ~/sdk
+          command: |
+             curl "https://$SDK_HOST/$SDK_PATH/sha256sums" -sS -o sha256sums
+             curl "https://$SDK_HOST/$SDK_PATH/sha256sums.asc" -fs -o sha256sums.asc || true
+             curl "https://$SDK_HOST/$SDK_PATH/sha256sums.sig" -fs -o sha256sums.sig || true
+             if [ ! -f sha256sums.asc ] && [ ! -f sha256sums.sig ]; then
+                 echo_red "Missing sha256sums signature files"
+                 exit 1
+             fi
+             [ ! -f sha256sums.asc ] || gpg --with-fingerprint --verify sha256sums.asc sha256sums
+             if [ -f sha256sums.sig ]; then
+                 VERIFIED=
+                 for KEY in ~/usign/*; do
+                     echo "Trying $KEY..."
+                     if signify-openbsd -V -q -p "$KEY" -x sha256sums.sig -m sha256sums; then
+                         echo "...verified"
+                         VERIFIED=1
+                         break
+                     fi
+                 done
+                 if [ -z "$VERIFIED" ]; then
+                     echo_red "Could not verify usign signature"
+                     exit 1
+                 fi
+             fi
+             rsync -av "$SDK_HOST::downloads/$SDK_PATH/$SDK_FILE" .
+             sha256sum -c --ignore-missing sha256sums
+
+      - run:
+          name: Prepare build_dir
+          working_directory: ~/build_dir
+          command: |
+             tar Jxf ~/sdk/$SDK_FILE --strip=1
+             touch .config
+             make prepare-tmpinfo scripts/config/conf
+             ./scripts/config/conf --defconfig=.config Config.in
+             make prereq
+             rm .config
+             cat > feeds.conf <<EOF
+             src-git base https://github.com/openwrt/openwrt.git;$BRANCH
+             src-link telephony $HOME/openwrt_telephony
+             src-git luci https://github.com/openwrt/luci.git;$BRANCH
+             EOF
+             cat feeds.conf
+             ./scripts/feeds update -a > /dev/null
+             make defconfig > /dev/null
+             # enable BUILD_LOG
+             sed -i 's/# CONFIG_BUILD_LOG is not set/CONFIG_BUILD_LOG=y/' .config
+
+      - run:
+          name: Install & download source, check package, compile
+          working_directory: ~/build_dir
+          command: |
+             set +o pipefail
+             PKGS=$(cd ~/openwrt_telephony; git diff --diff-filter=d --name-only "origin/$BRANCH..." | grep 'Makefile$' | grep -Ev '/files/|/src/' | awk -F/ '{ print $(NF-1) }')
+             if [ -z "$PKGS" ] ; then
+                 echo_blue "WARNING: No new or modified packages found!"
+                 exit 0
+             fi
+
+             echo_blue "=== Found new/modified packages: $PKGS"
+             for PKG in $PKGS ; do
+                 echo_blue "===+ Install: $PKG"
+                 ./scripts/feeds install "$PKG"
+
+                 echo_blue "===+ Download: $PKG"
+                 make "package/$PKG/download" V=s
+
+                 echo_blue "===+ Check package: $PKG"
+                 make "package/$PKG/check" V=s 2>&1 | tee logtmp
+                 RET=${PIPESTATUS[0]}
+
+                 if [ $RET -ne 0 ]; then
+                     echo_red   "=> Package check failed: $RET)"
+                     exit $RET
+                 fi
+
+                 badhash_msg="HASH does not match "
+                 badhash_msg+="|HASH uses deprecated hash,"
+                 badhash_msg+="|HASH is missing,"
+                 if grep -qE "$badhash_msg" logtmp; then
+                     echo_red   "=> Package HASH check failed"
+                     exit 1
+                 fi
+                 echo_green "=> Package check OK"
+             done
+
+             make \
+                 -f .config \
+                 -f tmp/.packagedeps \
+                 -f <(echo '$(info $(sort $(package-y) $(package-m)))'; echo -en 'a:\n\t@:') \
+              | tr ' ' '\n' >enabled-package-subdirs.txt
+             for PKG in $PKGS ; do
+                 if ! grep -m1 -qE "(^|/)$PKG$" enabled-package-subdirs.txt; then
+                        echo_red "===+ Building: $PKG skipped. It cannot be enabled with $SDK_FILE"
+                        continue
+                 fi
+                 echo_blue "===+ Building: $PKG"
+                 make "package/$PKG/compile" -j3 V=s || {
+                        RET=$?
+                        echo_red "===+ Building: $PKG failed, rebuilding with -j1 for human readable error log"
+                        make "package/$PKG/compile" -j1 V=s; exit $RET
+                 }
+             done
+
+      - store_artifacts:
+          path: ~/build_dir/logs
+
+      - store_artifacts:
+          path: ~/build_dir/bin
+
+workflows:
+  version: 2
+  buildpr:
+    jobs:
+      - build:
+          filters:
+            branches:
+              ignore: master