diff --git a/.github/workflows/Dockerfile b/.github/workflows/Dockerfile
new file mode 100644
index 000000000..aa269b778
--- /dev/null
+++ b/.github/workflows/Dockerfile
@@ -0,0 +1,6 @@
+ARG ARCH=x86-64
+FROM openwrtorg/rootfs:$ARCH
+
+ADD entrypoint.sh /entrypoint.sh
+
+CMD ["/entrypoint.sh"]
diff --git a/.github/workflows/entrypoint.sh b/.github/workflows/entrypoint.sh
new file mode 100755
index 000000000..21e9035d0
--- /dev/null
+++ b/.github/workflows/entrypoint.sh
@@ -0,0 +1,30 @@
+#!/bin/sh
+
+mkdir -p /var/lock/
+
+opkg update
+
+for PKG in /ci/*.ipk; do
+	tar -xzOf "$PKG" ./control.tar.gz | tar xzf - ./control 
+	PKG_NAME=$(sed -ne 's#^Package: \(.*\)$#\1#p' ./control)
+	PKG_VERSION=$(sed -ne 's#^Version: \(.*\)$#\1#p' ./control)
+
+	echo "Testing package $PKG_NAME ($PKG_VERSION)"
+
+	opkg install "$PKG"
+
+	TEST_SCRIPT=$(find /ci/ -name "$PKG_NAME" -type d)/test.sh
+	if [ -f "$TEST_SCRIPT" ]; then
+		echo "Use package specific test.sh"
+		if sh "$TEST_SCRIPT" "$PKG_NAME" "$PKG_VERSION"; then
+			echo "Test successfull"
+		else
+			echo "Test failed"
+			exit 1
+		fi
+	else
+		echo "No test.sh script available"
+	fi
+
+	opkg remove "$PKG_NAME"
+done
diff --git a/.github/workflows/multi-arch-test-build.yml b/.github/workflows/multi-arch-test-build.yml
index b026b10b2..97fa8176b 100644
--- a/.github/workflows/multi-arch-test-build.yml
+++ b/.github/workflows/multi-arch-test-build.yml
@@ -7,23 +7,29 @@ on:
 
 jobs:
   build:
-    name: ${{ matrix.arch }} build
+    name: Test ${{ matrix.arch }}
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         arch:
-          - aarch64_generic
           - arc_arc700
           - arc_archs
-          - arm_cortex-a15_neon-vfpv4
           - arm_cortex-a9_neon
           - arm_cortex-a9_vfpv3-d16
-          - i386_pentium4
           - mips_24kc
           - powerpc_464fp
           - powerpc_8540
-          - x86_64
+        runtime_test: [false]
+        include:
+          - arch: aarch64_generic
+            runtime_test: true
+          - arch: arm_cortex-a15_neon-vfpv4
+            runtime_test: true
+          - arch: i386_pentium4
+            runtime_test: true
+          - arch: x86_64
+            runtime_test: true
 
     steps:
       - uses: actions/checkout@v2
@@ -50,11 +56,14 @@ jobs:
           ARCH: ${{ matrix.arch }}
           FEEDNAME: packages_ci
 
+      - name: Move created packages to project dir
+        run: cp bin/packages/${{ matrix.arch }}/packages_ci/*.ipk . || true
+
       - name: Store packages
         uses: actions/upload-artifact@v2
         with:
           name: ${{ matrix.arch}}-packages
-          path: bin/packages/${{ matrix.arch }}/packages_ci/*.ipk
+          path: "*.ipk"
 
       - name: Store logs
         uses: actions/upload-artifact@v2
@@ -62,3 +71,22 @@ jobs:
           name: ${{ matrix.arch}}-logs
           path: logs/
 
+      - name: Remove logs
+        run: sudo rm -rf logs/ || true
+
+      - name: Register QEMU
+        if: ${{ matrix.runtime_test }}
+        run: |
+          sudo docker run --rm --privileged aptman/qus -s -- -p
+
+      - name: Build Docker container
+        if: ${{ matrix.runtime_test }}
+        run: |
+          docker build -t test-container --build-arg ARCH .github/workflows/
+        env:
+          ARCH: ${{ matrix.arch }}
+
+      - name: Test via Docker container
+        if: ${{ matrix.runtime_test }}
+        run: |
+          docker run --rm -v $GITHUB_WORKSPACE:/ci test-container