From fc7cf379e381997f0c31e69f808911d756d613e3 Mon Sep 17 00:00:00 2001 From: Jeffery To Date: Tue, 18 Aug 2020 05:45:14 +0800 Subject: [PATCH 01/15] golang: Update to 1.15.2 1.15.1 includes a fix for CVE-2020-24553: net/http/cgi,net/http/fcgi: Cross-Site Scripting (XSS) when Content-Type is not specified Signed-off-by: Jeffery To --- lang/golang/golang-values.mk | 13 ++++++++----- lang/golang/golang/Makefile | 12 ++++++------ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/lang/golang/golang-values.mk b/lang/golang/golang-values.mk index d52ad481f..9ea34be82 100644 --- a/lang/golang/golang-values.mk +++ b/lang/golang/golang-values.mk @@ -21,6 +21,7 @@ unexport \ GOARCH \ GOBIN \ GOCACHE \ + GOMODCACHE \ GODEBUG \ GOENV \ GOFLAGS \ @@ -110,14 +111,14 @@ unexport \ BOOT_GO_GCFLAGS \ BOOT_GO_LDFLAGS +# From https://golang.org/src/cmd/dist/buildruntime.go +unexport \ + GOEXPERIMENT + # From https://golang.org/src/cmd/dist/buildtool.go unexport \ GOBOOTSTRAP_TOOLEXEC -# From https://golang.org/src/cmd/internal/objabi/util.go -unexport \ - GOEXPERIMENT - # GOOS / GOARCH @@ -198,6 +199,8 @@ GO_PIE_SUPPORTED_OS_ARCH:= \ android_386 android_amd64 android_arm android_arm64 \ linux_386 linux_amd64 linux_arm linux_arm64 \ \ + windows_386 windows_amd64 windows_arm \ + \ darwin_amd64 \ freebsd_amd64 \ \ @@ -205,7 +208,7 @@ GO_PIE_SUPPORTED_OS_ARCH:= \ \ linux_ppc64le linux_s390x -go_pie_install_suffix=$(if $(filter $(1),aix_ppc64),,shared) +go_pie_install_suffix=$(if $(filter $(1),aix_ppc64 windows_386 windows_amd64 windows_arm),,shared) ifneq ($(filter $(GO_HOST_OS_ARCH),$(GO_PIE_SUPPORTED_OS_ARCH)),) GO_HOST_PIE_SUPPORTED:=1 diff --git a/lang/golang/golang/Makefile b/lang/golang/golang/Makefile index 320ed7812..45d79bc32 100644 --- a/lang/golang/golang/Makefile +++ b/lang/golang/golang/Makefile @@ -7,8 +7,8 @@ include $(TOPDIR)/rules.mk -GO_VERSION_MAJOR_MINOR:=1.14 -GO_VERSION_PATCH:=7 +GO_VERSION_MAJOR_MINOR:=1.15 +GO_VERSION_PATCH:=2 PKG_NAME:=golang PKG_VERSION:=$(GO_VERSION_MAJOR_MINOR)$(if $(GO_VERSION_PATCH),.$(GO_VERSION_PATCH)) @@ -20,7 +20,7 @@ GO_SOURCE_URLS:=https://dl.google.com/go/ \ PKG_SOURCE:=go$(PKG_VERSION).src.tar.gz PKG_SOURCE_URL:=$(GO_SOURCE_URLS) -PKG_HASH:=064392433563660c73186991c0a315787688e7c38a561e26647686f89b6c30e3 +PKG_HASH:=28bf9d0bcde251011caae230a4a05d917b172ea203f2a62f2c2f9533589d4b4d PKG_MAINTAINER:=Jeffery To PKG_LICENSE:=BSD-3-Clause @@ -52,15 +52,15 @@ HOST_GO_ROOT:=$(HOST_GO_PREFIX)/lib/go-$(HOST_GO_VERSION_ID) HOST_GO_VALID_OS_ARCH:= \ android_386 android_amd64 android_arm android_arm64 \ - darwin_386 darwin_amd64 darwin_arm darwin_arm64 \ freebsd_386 freebsd_amd64 freebsd_arm freebsd_arm64 \ linux_386 linux_amd64 linux_arm linux_arm64 \ + openbsd_386 openbsd_amd64 openbsd_arm openbsd_arm64 \ \ - openbsd_386 openbsd_amd64 openbsd_arm \ netbsd_386 netbsd_amd64 netbsd_arm \ plan9_386 plan9_amd64 plan9_arm \ + windows_386 windows_amd64 windows_arm \ \ - windows_386 windows_amd64 \ + darwin_amd64 darwin_arm64 \ \ dragonfly_amd64 \ illumos_amd64 \ From 4674564e42e0b30f477af6fd7dbb3a92c1656998 Mon Sep 17 00:00:00 2001 From: Jeffery To Date: Wed, 2 Sep 2020 16:11:45 +0800 Subject: [PATCH 02/15] golang: Move config option into separate file Signed-off-by: Jeffery To --- lang/golang/golang/Config.in | 15 +++++++++++++++ lang/golang/golang/Makefile | 15 +-------------- 2 files changed, 16 insertions(+), 14 deletions(-) create mode 100644 lang/golang/golang/Config.in diff --git a/lang/golang/golang/Config.in b/lang/golang/golang/Config.in new file mode 100644 index 000000000..6bbbf728c --- /dev/null +++ b/lang/golang/golang/Config.in @@ -0,0 +1,15 @@ +menu "Configuration" + +config GOLANG_EXTERNAL_BOOTSTRAP_ROOT + string "External bootstrap Go root directory" + default "" + help + Path to a working Go tree (>= Go 1.4), with bin, pkg, and src + subdirectories and the Go compiler at bin/go. + + If specified, the existing Go installation will be used to + compile host (buildroot) Go. + + Leave blank to compile the default bootstrap Go. + +endmenu diff --git a/lang/golang/golang/Makefile b/lang/golang/golang/Makefile index 45d79bc32..c08028d09 100644 --- a/lang/golang/golang/Makefile +++ b/lang/golang/golang/Makefile @@ -155,20 +155,7 @@ libraries for the Go programming language. endef define Package/golang/config -menu "Configuration" - -config GOLANG_EXTERNAL_BOOTSTRAP_ROOT - string "External bootstrap Go root directory" - help - Path to a working Go tree (>= Go 1.4), with bin, pkg, and src - subdirectories and the Go compiler at bin/go. - - If specified, the existing Go installation will be used to - compile host (buildroot) Go. - - Leave blank to compile the default bootstrap Go. - -endmenu + source "$(SOURCE)/Config.in" endef define Package/golang-doc From 3dd55b504cb4d7d4ca0251f4ce7b45ec8effeaf8 Mon Sep 17 00:00:00 2001 From: Jeffery To Date: Wed, 19 Aug 2020 05:39:41 +0800 Subject: [PATCH 03/15] golang: Move module cache into DL_DIR This also adds a config option GOLANG_MOD_CACHE_WORLD_READABLE; if enabled, chmod is run after a Go package build to make all files/directories in the module cache world-readable. Signed-off-by: Jeffery To --- lang/golang/golang-build.sh | 48 +++++++++++++++++++++++++++++++++++ lang/golang/golang-package.mk | 17 ++++++++----- lang/golang/golang-values.mk | 17 +++++++++++++ lang/golang/golang/Config.in | 4 +++ 4 files changed, 80 insertions(+), 6 deletions(-) create mode 100644 lang/golang/golang-build.sh diff --git a/lang/golang/golang-build.sh b/lang/golang/golang-build.sh new file mode 100644 index 000000000..d39949dca --- /dev/null +++ b/lang/golang/golang-build.sh @@ -0,0 +1,48 @@ +#!/bin/sh + +log() { + # shellcheck disable=SC2039 + local IFS=" " + printf '%s\n' "$*" +} + +log_error() { + # shellcheck disable=SC2039 + local IFS=" " + printf 'Error: %s\n' "$*" >&2 +} + +cache_cleanup() { + if ! [ -d "$GO_MOD_CACHE_DIR" ]; then + return 0 + fi + + # in case go is called without -modcacherw + find "$GO_MOD_CACHE_DIR" -type d -not -perm -u+w -exec chmod u+w '{}' + + + if [ -n "$CONFIG_GOLANG_MOD_CACHE_WORLD_READABLE" ]; then + find "$GO_MOD_CACHE_DIR" -type d -not -perm -go+rx -exec chmod go+rx '{}' + + find "$GO_MOD_CACHE_DIR" -not -type d -not -perm -go+r -exec chmod go+r '{}' + + fi + + return 0 +} + + +if [ "$#" -lt 1 ]; then + log_error "Missing command" + exit 1 +fi + +command="$1" +shift 1 + +case "$command" in + cache_cleanup) + cache_cleanup + ;; + *) + log_error "Invalid command \"$command\"" + exit 1 + ;; +esac diff --git a/lang/golang/golang-package.mk b/lang/golang/golang-package.mk index 1e279797b..266bfeda4 100644 --- a/lang/golang/golang-package.mk +++ b/lang/golang/golang-package.mk @@ -186,6 +186,7 @@ GO_PKG_TARGET_VARS= \ GO_PKG_BUILD_VARS= \ GOPATH=$(GO_PKG_BUILD_DIR) \ GOCACHE=$(GO_PKG_CACHE_DIR) \ + GOMODCACHE=$(GO_MOD_CACHE_DIR) \ GOENV=off GO_PKG_DEFAULT_VARS= \ @@ -226,7 +227,7 @@ GoPackage/has_binaries=$(call GoPackage/is_dir_not_empty,$(GO_PKG_BUILD_BIN_DIR) define GoPackage/Build/Configure ( \ cd $(PKG_BUILD_DIR) ; \ - mkdir -p $(GO_PKG_BUILD_DIR)/bin $(GO_PKG_BUILD_DIR)/src $(GO_PKG_CACHE_DIR) ; \ + mkdir -p $(GO_PKG_BUILD_DIR)/bin $(GO_PKG_BUILD_DIR)/src $(GO_PKG_CACHE_DIR) $(GO_MOD_CACHE_DIR) ; \ \ files=$$$$($(FIND) ./ \ -type d -a \( -path './.git' -o -path './$(GO_PKG_WORK_DIR_NAME)' \) -prune -o \ @@ -303,9 +304,12 @@ define GoPackage/Build/Compile ( \ cd $(GO_PKG_BUILD_DIR) ; \ export $(GO_PKG_VARS) ; \ + if [ -f "$(PKG_BUILD_DIR)/go.mod" ] ; then \ + modargs="$(GO_MOD_ARGS)" ; \ + fi ; \ \ echo "Finding targets" ; \ - targets=$$$$(go list $(GO_PKG_BUILD_PKG)) ; \ + targets=$$$$(go list $$$$modargs $(GO_PKG_BUILD_PKG)) ; \ for pattern in $(GO_PKG_EXCLUDES); do \ targets=$$$$(echo "$$$$targets" | grep -v "$$$$pattern") ; \ done ; \ @@ -319,7 +323,7 @@ define GoPackage/Build/Compile \ if [ "$(strip $(GO_PKG_SOURCE_ONLY))" != 1 ]; then \ echo "Building targets" ; \ - go install $(GO_PKG_INSTALL_ARGS) $(1) $$$$targets ; \ + go install $(GO_PKG_INSTALL_ARGS) $$$$modargs $(1) $$$$targets ; \ retval=$$$$? ; \ echo ; \ \ @@ -328,9 +332,9 @@ define GoPackage/Build/Compile echo ; \ fi ; \ \ - echo "Cleaning module download cache (golang/go#27455)" ; \ - go clean -modcache ; \ - echo ; \ + if [ "$$$$retval" -ne 0 ]; then \ + $(call Go/CacheCleanup) ; \ + fi ; \ fi ; \ exit $$$$retval ; \ ) @@ -362,6 +366,7 @@ endef ifneq ($(strip $(GO_PKG)),) Build/Configure=$(call GoPackage/Build/Configure) Build/Compile=$(call GoPackage/Build/Compile) + Hooks/Compile/Post+=Go/CacheCleanup Build/InstallDev=$(call GoPackage/Build/InstallDev,$(1)) endif diff --git a/lang/golang/golang-values.mk b/lang/golang/golang-values.mk index 9ea34be82..84b6b5892 100644 --- a/lang/golang/golang-values.mk +++ b/lang/golang/golang-values.mk @@ -219,3 +219,20 @@ ifneq ($(filter $(GO_OS_ARCH),$(GO_PIE_SUPPORTED_OS_ARCH)),) GO_TARGET_PIE_SUPPORTED:=1 GO_TARGET_PIE_INSTALL_SUFFIX:=$(call go_pie_install_suffix,$(GO_OS_ARCH)) endif + + +# General build info + +GO_MOD_CACHE_DIR:=$(DL_DIR)/go-mod-cache + +GO_MOD_ARGS= \ + -modcacherw + +GO_GENERAL_BUILD_CONFIG_VARS= \ + CONFIG_GOLANG_MOD_CACHE_WORLD_READABLE="$(CONFIG_GOLANG_MOD_CACHE_WORLD_READABLE)" \ + GO_MOD_CACHE_DIR="$(GO_MOD_CACHE_DIR)" + +define Go/CacheCleanup + $(GENERAL_BUILD_CONFIG_VARS) \ + $(SHELL) $(GO_INCLUDE_DIR)/golang-build.sh cache_cleanup +endef diff --git a/lang/golang/golang/Config.in b/lang/golang/golang/Config.in index 6bbbf728c..cb5fec6d5 100644 --- a/lang/golang/golang/Config.in +++ b/lang/golang/golang/Config.in @@ -12,4 +12,8 @@ config GOLANG_EXTERNAL_BOOTSTRAP_ROOT Leave blank to compile the default bootstrap Go. +config GOLANG_MOD_CACHE_WORLD_READABLE + bool "Ensure Go module cache is world-readable" + default n + endmenu From 9074dbd91770d9913378a3c20e31ea5bde2c2be7 Mon Sep 17 00:00:00 2001 From: Jeffery To Date: Mon, 24 Aug 2020 04:59:05 +0800 Subject: [PATCH 04/15] golang: Share build cache across package builds This also adds a config option GOLANG_BUILD_CACHE_DIR to customize the location of the build cache directory. Signed-off-by: Jeffery To --- lang/golang/golang-package.mk | 22 ++++++++++++---------- lang/golang/golang-values.mk | 1 + lang/golang/golang/Config.in | 7 +++++++ lang/golang/golang/Makefile | 26 ++++++-------------------- utils/ctop/Makefile | 5 ++--- utils/docker-ce/Makefile | 3 +-- 6 files changed, 29 insertions(+), 35 deletions(-) diff --git a/lang/golang/golang-package.mk b/lang/golang/golang-package.mk index 266bfeda4..46b484b4d 100644 --- a/lang/golang/golang-package.mk +++ b/lang/golang/golang-package.mk @@ -123,11 +123,7 @@ GO_PKG_PATH:=/usr/share/gocode GO_PKG_BUILD_PKG?=$(strip $(GO_PKG))/... GO_PKG_WORK_DIR_NAME:=.go_work -GO_PKG_WORK_DIR=$(PKG_BUILD_DIR)/$(GO_PKG_WORK_DIR_NAME) - -GO_PKG_BUILD_DIR=$(GO_PKG_WORK_DIR)/build -GO_PKG_CACHE_DIR=$(GO_PKG_WORK_DIR)/cache - +GO_PKG_BUILD_DIR=$(PKG_BUILD_DIR)/$(GO_PKG_WORK_DIR_NAME)/build GO_PKG_BUILD_BIN_DIR=$(GO_PKG_BUILD_DIR)/bin$(if $(GO_HOST_TARGET_DIFFERENT),/$(GO_OS_ARCH)) GO_PKG_BUILD_DEPENDS_SRC=$(STAGING_DIR)$(GO_PKG_PATH)/src @@ -185,7 +181,7 @@ GO_PKG_TARGET_VARS= \ GO_PKG_BUILD_VARS= \ GOPATH=$(GO_PKG_BUILD_DIR) \ - GOCACHE=$(GO_PKG_CACHE_DIR) \ + GOCACHE=$(GO_BUILD_CACHE_DIR) \ GOMODCACHE=$(GO_MOD_CACHE_DIR) \ GOENV=off @@ -195,9 +191,6 @@ GO_PKG_DEFAULT_VARS= \ GO_PKG_VARS=$(GO_PKG_DEFAULT_VARS) -# do not use for new code; this will be removed after the next OpenWrt release -GoPackage/Environment=$(GO_PKG_VARS) - GO_PKG_DEFAULT_LDFLAGS= \ -buildid '$(SOURCE_DATE_EPOCH)' \ -linkmode external \ @@ -227,7 +220,9 @@ GoPackage/has_binaries=$(call GoPackage/is_dir_not_empty,$(GO_PKG_BUILD_BIN_DIR) define GoPackage/Build/Configure ( \ cd $(PKG_BUILD_DIR) ; \ - mkdir -p $(GO_PKG_BUILD_DIR)/bin $(GO_PKG_BUILD_DIR)/src $(GO_PKG_CACHE_DIR) $(GO_MOD_CACHE_DIR) ; \ + mkdir -p \ + $(GO_PKG_BUILD_DIR)/bin $(GO_PKG_BUILD_DIR)/src \ + $(GO_BUILD_CACHE_DIR) $(GO_MOD_CACHE_DIR) ; \ \ files=$$$$($(FIND) ./ \ -type d -a \( -path './.git' -o -path './$(GO_PKG_WORK_DIR_NAME)' \) -prune -o \ @@ -387,3 +382,10 @@ define GoSrcPackage Package/$(1)/install=$$(call GoPackage/Package/Install/Src,$$(1)) endif endef + + +# Deprecated variables - these will be removed after the next OpenWrt release +GO_PKG_WORK_DIR=$(PKG_BUILD_DIR)/$(GO_PKG_WORK_DIR_NAME) +GO_PKG_CACHE_DIR=$(GO_BUILD_CACHE_DIR) +GoPackage/Environment=$(GO_PKG_VARS) +# End of deprecated variables diff --git a/lang/golang/golang-values.mk b/lang/golang/golang-values.mk index 84b6b5892..596270caf 100644 --- a/lang/golang/golang-values.mk +++ b/lang/golang/golang-values.mk @@ -223,6 +223,7 @@ endif # General build info +GO_BUILD_CACHE_DIR:=$(or $(call qstrip,$(CONFIG_GOLANG_BUILD_CACHE_DIR)),$(TOPDIR)/.go-build) GO_MOD_CACHE_DIR:=$(DL_DIR)/go-mod-cache GO_MOD_ARGS= \ diff --git a/lang/golang/golang/Config.in b/lang/golang/golang/Config.in index cb5fec6d5..2e7159173 100644 --- a/lang/golang/golang/Config.in +++ b/lang/golang/golang/Config.in @@ -12,6 +12,13 @@ config GOLANG_EXTERNAL_BOOTSTRAP_ROOT Leave blank to compile the default bootstrap Go. +config GOLANG_BUILD_CACHE_DIR + string "Go build cache directory" + default "" + help + Store the Go build cache in this directory. + If not set, uses './.go-build'. + config GOLANG_MOD_CACHE_WORLD_READABLE bool "Ensure Go module cache is world-readable" default n diff --git a/lang/golang/golang/Makefile b/lang/golang/golang/Makefile index c08028d09..8557179b0 100644 --- a/lang/golang/golang/Makefile +++ b/lang/golang/golang/Makefile @@ -32,10 +32,6 @@ PKG_BUILD_DIR:=$(BUILD_DIR)/go-$(PKG_VERSION) PKG_BUILD_PARALLEL:=1 PKG_USE_MIPS16:=0 -PKG_GO_WORK_DIR:=$(PKG_BUILD_DIR)/.go_work -PKG_GO_HOST_CACHE_DIR:=$(PKG_GO_WORK_DIR)/host_cache -PKG_GO_TARGET_CACHE_DIR:=$(PKG_GO_WORK_DIR)/target_cache - PKG_GO_PREFIX:=/usr PKG_GO_VERSION_ID:=$(GO_VERSION_MAJOR_MINOR) PKG_GO_ROOT:=$(PKG_GO_PREFIX)/lib/go-$(PKG_GO_VERSION_ID) @@ -43,9 +39,6 @@ PKG_GO_ROOT:=$(PKG_GO_PREFIX)/lib/go-$(PKG_GO_VERSION_ID) HOST_BUILD_DIR:=$(BUILD_DIR_HOST)/go-$(PKG_VERSION) HOST_BUILD_PARALLEL:=1 -HOST_GO_WORK_DIR:=$(HOST_BUILD_DIR)/.go_work -HOST_GO_CACHE_DIR:=$(HOST_GO_WORK_DIR)/cache - HOST_GO_PREFIX:=$(STAGING_DIR_HOSTPKG) HOST_GO_VERSION_ID:=cross HOST_GO_ROOT:=$(HOST_GO_PREFIX)/lib/go-$(HOST_GO_VERSION_ID) @@ -78,8 +71,6 @@ BOOTSTRAP_SOURCE_URL:=$(GO_SOURCE_URLS) BOOTSTRAP_HASH:=f4ff5b5eb3a3cae1c993723f3eab519c5bae18866b5e5f96fe1102f0cb5c3e52 BOOTSTRAP_BUILD_DIR:=$(HOST_BUILD_DIR)/.go_bootstrap -BOOTSTRAP_WORK_DIR:=$(BOOTSTRAP_BUILD_DIR)/.go_work -BOOTSTRAP_CACHE_DIR:=$(BOOTSTRAP_WORK_DIR)/cache BOOTSTRAP_GO_VALID_OS_ARCH:= \ darwin_386 darwin_amd64 \ @@ -236,19 +227,16 @@ define Host/Compile $(call GoCompiler/Bootstrap/CheckHost,$(BOOTSTRAP_GO_VALID_OS_ARCH)) $(call GoCompiler/Host/CheckHost,$(HOST_GO_VALID_OS_ARCH)) - mkdir -p \ - $(BOOTSTRAP_CACHE_DIR) \ - $(HOST_GO_CACHE_DIR) + mkdir -p $(GO_BUILD_CACHE_DIR) $(call GoCompiler/Bootstrap/Make, \ - GOCACHE=$(BOOTSTRAP_CACHE_DIR) \ CC=$(HOSTCC_NOCACHE) \ CXX=$(HOSTCXX_NOCACHE) \ ) $(call GoCompiler/Host/Make, \ GOROOT_BOOTSTRAP=$(if $(USE_DEFAULT_BOOTSTRAP),$(BOOTSTRAP_BUILD_DIR),$(EXTERNAL_BOOTSTRAP_DIR)) \ - GOCACHE=$(HOST_GO_CACHE_DIR) \ + GOCACHE=$(GO_BUILD_CACHE_DIR) \ CC=$(HOSTCC_NOCACHE) \ CXX=$(HOSTCXX_NOCACHE) \ ) @@ -259,7 +247,7 @@ define Host/Compile ( \ cd $(HOST_BUILD_DIR)/bin ; \ $(CP) go go-nopie ; \ - GOCACHE=$(HOST_GO_CACHE_DIR) \ + GOCACHE=$(GO_BUILD_CACHE_DIR) \ GOENV=off \ CC=$(HOSTCC_NOCACHE) \ CXX=$(HOSTCXX_NOCACHE) \ @@ -299,15 +287,13 @@ define Host/Uninstall endef define Build/Compile - mkdir -p \ - $(PKG_GO_HOST_CACHE_DIR) \ - $(PKG_GO_TARGET_CACHE_DIR) + mkdir -p $(GO_BUILD_CACHE_DIR) @echo "Building target Go first stage" $(call GoCompiler/Package/Make, \ GOROOT_BOOTSTRAP=$(HOST_GO_ROOT) \ - GOCACHE=$(PKG_GO_HOST_CACHE_DIR) \ + GOCACHE=$(GO_BUILD_CACHE_DIR) \ GO_GCC_HELPER_CC="$(HOSTCC)" \ GO_GCC_HELPER_CXX="$(HOSTCXX)" \ GO_GCC_HELPER_PATH=$$$$PATH \ @@ -327,7 +313,7 @@ define Build/Compile export $(GO_PKG_TARGET_VARS) ; \ $(CP) go go-host ; \ GOROOT_FINAL=$(PKG_GO_ROOT) \ - GOCACHE=$(PKG_GO_TARGET_CACHE_DIR) \ + GOCACHE=$(GO_BUILD_CACHE_DIR) \ GOENV=off \ GO_GCC_HELPER_CC="$(TARGET_CC)" \ GO_GCC_HELPER_CXX="$(TARGET_CXX)" \ diff --git a/utils/ctop/Makefile b/utils/ctop/Makefile index aae82f010..85242dbe5 100644 --- a/utils/ctop/Makefile +++ b/utils/ctop/Makefile @@ -35,12 +35,11 @@ endef MAKE_PATH:=$(GO_PKG_WORK_DIR_NAME)/build/src/$(GO_PKG) MAKE_VARS += \ GOPATH=$(GO_PKG_BUILD_DIR) \ - GOCACHE=$(GO_PKG_CACHE_DIR) \ - GOTMPDIR=$(GO_PKG_TMP_DIR) \ + GOCACHE=$(GO_BUILD_CACHE_DIR) \ GOROOT_FINAL=$(GO_TARGET_ROOT) \ CC=$(TARGET_CC) \ CXX=$(TARGET_CXX) \ - $(call GoPackage/Environment) + $(GO_PKG_VARS) MAKE_FLAGS += \ COMMIT=$(PKG_SOURCE_VERSION) diff --git a/utils/docker-ce/Makefile b/utils/docker-ce/Makefile index 1f6c1ee79..fe072902e 100644 --- a/utils/docker-ce/Makefile +++ b/utils/docker-ce/Makefile @@ -77,8 +77,7 @@ define Build/Configure mkdir -p $(GO_PKG_BUILD_DIR)/bin \ $(GO_PKG_BUILD_DIR)/src \ $(GO_PKG_BUILD_DIR)/src/$(GO_PKG)/ \ - $(GO_PKG_CACHE_DIR) \ - $(GO_PKG_TMP_DIR) + $(GO_BUILD_CACHE_DIR) $(LN) $(PKG_BUILD_DIR)/cli $(GO_PKG_BUILD_DIR)/src/$(GO_PKG)/cli $(LN) $(PKG_BUILD_DIR)/engine $(GO_PKG_BUILD_DIR)/src/$(GO_PKG)/docker endef From 40bff64ad38a72cc6acb2849ea4004cade470cb7 Mon Sep 17 00:00:00 2001 From: Jeffery To Date: Wed, 2 Sep 2020 20:43:25 +0800 Subject: [PATCH 05/15] golang: Group variables by stage (bootstrap/host/target) Signed-off-by: Jeffery To --- lang/golang/golang/Makefile | 93 +++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 41 deletions(-) diff --git a/lang/golang/golang/Makefile b/lang/golang/golang/Makefile index 8557179b0..6588b77bc 100644 --- a/lang/golang/golang/Makefile +++ b/lang/golang/golang/Makefile @@ -96,20 +96,6 @@ BOOTSTRAP_UNPACK:=$(HOST_TAR) -C $(BOOTSTRAP_BUILD_DIR) --strip-components=1 -xz RSTRIP:=: STRIP:=: -ifdef CONFIG_PKG_ASLR_PIE_ALL - ifeq ($(GO_TARGET_PIE_SUPPORTED),1) - PKG_GO_ENABLE_PIE:=1 - PKG_GO_INSTALL_SUFFIX:=$(GO_TARGET_PIE_INSTALL_SUFFIX) - else - $(warning PIE buildmode is not supported for $(GO_OS)/$(GO_ARCH)) - endif -endif - -ifeq ($(GO_HOST_PIE_SUPPORTED),1) - HOST_GO_ENABLE_PIE:=1 - HOST_GO_INSTALL_SUFFIX:=$(GO_HOST_PIE_INSTALL_SUFFIX) -endif - define Package/golang/Default $(call GoPackage/GoSubMenu) TITLE:=Go programming language @@ -172,42 +158,18 @@ This package provides the Go programming language source files needed for cross-compilation. endef + +# Bootstrap + EXTERNAL_BOOTSTRAP_DIR:=$(call qstrip,$(CONFIG_GOLANG_EXTERNAL_BOOTSTRAP_ROOT)) USE_DEFAULT_BOOTSTRAP:=$(if $(EXTERNAL_BOOTSTRAP_DIR),,1) -PKG_GO_ZBOOTSTRAP_MODS:= \ - s/defaultGO386 = `[^`]*`/defaultGO386 = `$(if $(GO_386),$(GO_386),387)`/; \ - s/defaultGOARM = `[^`]*`/defaultGOARM = `$(if $(GO_ARM),$(GO_ARM),5)`/; \ - s/defaultGOMIPS = `[^`]*`/defaultGOMIPS = `$(if $(GO_MIPS),$(GO_MIPS),hardfloat)`/; \ - s/defaultGOMIPS64 = `[^`]*`/defaultGOMIPS64 = `$(if $(GO_MIPS64),$(GO_MIPS64),hardfloat)`/; \ - s/defaultGOPPC64 = `[^`]*`/defaultGOPPC64 = `power8`/; - -PKG_GO_LDFLAGS= \ - -buildid '$(SOURCE_DATE_EPOCH)' - -ifeq ($(CONFIG_NO_STRIP)$(CONFIG_DEBUG),) - PKG_GO_LDFLAGS+= -s -w -endif - -# setting -trimpath is not necessary here because the paths inside the -# compiler binary are relative to GOROOT_FINAL (PKG_GO_ROOT), which is -# static / not dependent on the build environment -PKG_GO_INSTALL_ARGS= \ - -ldflags "all=$(PKG_GO_LDFLAGS)" - -ifeq ($(PKG_GO_ENABLE_PIE),1) - PKG_GO_INSTALL_ARGS+= -buildmode pie -endif - define Download/golang-bootstrap FILE:=$(BOOTSTRAP_SOURCE) URL:=$(BOOTSTRAP_SOURCE_URL) HASH:=$(BOOTSTRAP_HASH) endef -$(eval $(call GoCompiler/AddProfile,Host,$(HOST_BUILD_DIR),$(HOST_GO_PREFIX),$(HOST_GO_VERSION_ID),$(GO_HOST_OS_ARCH),$(HOST_GO_INSTALL_SUFFIX))) -$(eval $(call GoCompiler/AddProfile,Package,$(PKG_BUILD_DIR),$(PKG_GO_PREFIX),$(PKG_GO_VERSION_ID),$(GO_OS_ARCH),$(PKG_GO_INSTALL_SUFFIX))) - ifeq ($(USE_DEFAULT_BOOTSTRAP),1) $(eval $(call GoCompiler/AddProfile,Bootstrap,$(BOOTSTRAP_BUILD_DIR),,bootstrap,$(GO_HOST_OS_ARCH))) @@ -220,6 +182,16 @@ ifeq ($(USE_DEFAULT_BOOTSTRAP),1) endef endif + +# Host + +ifeq ($(GO_HOST_PIE_SUPPORTED),1) + HOST_GO_ENABLE_PIE:=1 + HOST_GO_INSTALL_SUFFIX:=$(GO_HOST_PIE_INSTALL_SUFFIX) +endif + +$(eval $(call GoCompiler/AddProfile,Host,$(HOST_BUILD_DIR),$(HOST_GO_PREFIX),$(HOST_GO_VERSION_ID),$(GO_HOST_OS_ARCH),$(HOST_GO_INSTALL_SUFFIX))) + # when https://github.com/golang/go/issues/31544 is fixed, # we should be able to set GO_LDFLAGS=-buildmode=pie for host make # instead of doing a rebuild for pie @@ -286,6 +258,44 @@ define Host/Uninstall $(call GoCompiler/Host/Uninstall,) endef + +# Target + +ifdef CONFIG_PKG_ASLR_PIE_ALL + ifeq ($(GO_TARGET_PIE_SUPPORTED),1) + PKG_GO_ENABLE_PIE:=1 + PKG_GO_INSTALL_SUFFIX:=$(GO_TARGET_PIE_INSTALL_SUFFIX) + else + $(warning PIE buildmode is not supported for $(GO_OS)/$(GO_ARCH)) + endif +endif + +$(eval $(call GoCompiler/AddProfile,Package,$(PKG_BUILD_DIR),$(PKG_GO_PREFIX),$(PKG_GO_VERSION_ID),$(GO_OS_ARCH),$(PKG_GO_INSTALL_SUFFIX))) + +PKG_GO_ZBOOTSTRAP_MODS:= \ + s/defaultGO386 = `[^`]*`/defaultGO386 = `$(if $(GO_386),$(GO_386),387)`/; \ + s/defaultGOARM = `[^`]*`/defaultGOARM = `$(if $(GO_ARM),$(GO_ARM),5)`/; \ + s/defaultGOMIPS = `[^`]*`/defaultGOMIPS = `$(if $(GO_MIPS),$(GO_MIPS),hardfloat)`/; \ + s/defaultGOMIPS64 = `[^`]*`/defaultGOMIPS64 = `$(if $(GO_MIPS64),$(GO_MIPS64),hardfloat)`/; \ + s/defaultGOPPC64 = `[^`]*`/defaultGOPPC64 = `power8`/; + +PKG_GO_LDFLAGS= \ + -buildid '$(SOURCE_DATE_EPOCH)' + +ifeq ($(CONFIG_NO_STRIP)$(CONFIG_DEBUG),) + PKG_GO_LDFLAGS+= -s -w +endif + +# setting -trimpath is not necessary here because the paths inside the +# compiler binary are relative to GOROOT_FINAL (PKG_GO_ROOT), which is +# static / not dependent on the build environment +PKG_GO_INSTALL_ARGS= \ + -ldflags "all=$(PKG_GO_LDFLAGS)" + +ifeq ($(PKG_GO_ENABLE_PIE),1) + PKG_GO_INSTALL_ARGS+= -buildmode pie +endif + define Build/Compile mkdir -p $(GO_BUILD_CACHE_DIR) @@ -350,6 +360,7 @@ define Package/golang-src/extra_provides echo 'libc.so.6' endef + $(eval $(call HostBuild)) $(eval $(call BuildPackage,golang)) $(eval $(call BuildPackage,golang-doc)) From 4fb4ec963f6d8e830f41dbcf91eb1f2ae54f2f6c Mon Sep 17 00:00:00 2001 From: Jeffery To Date: Wed, 2 Sep 2020 20:59:47 +0800 Subject: [PATCH 06/15] golang: Simplify bootstrap stage Signed-off-by: Jeffery To --- lang/golang/golang/Makefile | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/lang/golang/golang/Makefile b/lang/golang/golang/Makefile index 6588b77bc..d28a37428 100644 --- a/lang/golang/golang/Makefile +++ b/lang/golang/golang/Makefile @@ -161,25 +161,25 @@ endef # Bootstrap -EXTERNAL_BOOTSTRAP_DIR:=$(call qstrip,$(CONFIG_GOLANG_EXTERNAL_BOOTSTRAP_ROOT)) -USE_DEFAULT_BOOTSTRAP:=$(if $(EXTERNAL_BOOTSTRAP_DIR),,1) +BOOTSTRAP_ROOT_DIR:=$(call qstrip,$(CONFIG_GOLANG_EXTERNAL_BOOTSTRAP_ROOT)) -define Download/golang-bootstrap - FILE:=$(BOOTSTRAP_SOURCE) - URL:=$(BOOTSTRAP_SOURCE_URL) - HASH:=$(BOOTSTRAP_HASH) -endef - -ifeq ($(USE_DEFAULT_BOOTSTRAP),1) - $(eval $(call GoCompiler/AddProfile,Bootstrap,$(BOOTSTRAP_BUILD_DIR),,bootstrap,$(GO_HOST_OS_ARCH))) +ifeq ($(BOOTSTRAP_ROOT_DIR),) + BOOTSTRAP_ROOT_DIR:=$(BOOTSTRAP_BUILD_DIR) + define Download/golang-bootstrap + FILE:=$(BOOTSTRAP_SOURCE) + URL:=$(BOOTSTRAP_SOURCE_URL) + HASH:=$(BOOTSTRAP_HASH) + endef $(eval $(call Download,golang-bootstrap)) - define Host/Prepare - $(call Host/Prepare/Default) + define Bootstrap/Prepare mkdir -p $(BOOTSTRAP_BUILD_DIR) $(BOOTSTRAP_UNPACK) endef + Hooks/HostPrepare/Post+=Bootstrap/Prepare + + $(eval $(call GoCompiler/AddProfile,Bootstrap,$(BOOTSTRAP_BUILD_DIR),,bootstrap,$(GO_HOST_OS_ARCH))) endif @@ -207,7 +207,7 @@ define Host/Compile ) $(call GoCompiler/Host/Make, \ - GOROOT_BOOTSTRAP=$(if $(USE_DEFAULT_BOOTSTRAP),$(BOOTSTRAP_BUILD_DIR),$(EXTERNAL_BOOTSTRAP_DIR)) \ + GOROOT_BOOTSTRAP=$(BOOTSTRAP_ROOT_DIR) \ GOCACHE=$(GO_BUILD_CACHE_DIR) \ CC=$(HOSTCC_NOCACHE) \ CXX=$(HOSTCXX_NOCACHE) \ From 66eabf18f53468442563cc4dc4510649050cc629 Mon Sep 17 00:00:00 2001 From: Jeffery To Date: Wed, 2 Sep 2020 21:07:17 +0800 Subject: [PATCH 07/15] golang: Refactor vars common to each host compile stage into variable Signed-off-by: Jeffery To --- lang/golang/golang/Makefile | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/lang/golang/golang/Makefile b/lang/golang/golang/Makefile index d28a37428..162b5ad5e 100644 --- a/lang/golang/golang/Makefile +++ b/lang/golang/golang/Makefile @@ -187,11 +187,20 @@ endif ifeq ($(GO_HOST_PIE_SUPPORTED),1) HOST_GO_ENABLE_PIE:=1 +endif + +ifeq ($(HOST_GO_ENABLE_PIE),1) HOST_GO_INSTALL_SUFFIX:=$(GO_HOST_PIE_INSTALL_SUFFIX) endif $(eval $(call GoCompiler/AddProfile,Host,$(HOST_BUILD_DIR),$(HOST_GO_PREFIX),$(HOST_GO_VERSION_ID),$(GO_HOST_OS_ARCH),$(HOST_GO_INSTALL_SUFFIX))) +HOST_GO_VARS= \ + GOCACHE=$(GO_BUILD_CACHE_DIR) \ + GOENV=off \ + CC=$(HOSTCC_NOCACHE) \ + CXX=$(HOSTCXX_NOCACHE) + # when https://github.com/golang/go/issues/31544 is fixed, # we should be able to set GO_LDFLAGS=-buildmode=pie for host make # instead of doing a rebuild for pie @@ -202,15 +211,12 @@ define Host/Compile mkdir -p $(GO_BUILD_CACHE_DIR) $(call GoCompiler/Bootstrap/Make, \ - CC=$(HOSTCC_NOCACHE) \ - CXX=$(HOSTCXX_NOCACHE) \ + $(HOST_GO_VARS) \ ) $(call GoCompiler/Host/Make, \ GOROOT_BOOTSTRAP=$(BOOTSTRAP_ROOT_DIR) \ - GOCACHE=$(GO_BUILD_CACHE_DIR) \ - CC=$(HOSTCC_NOCACHE) \ - CXX=$(HOSTCXX_NOCACHE) \ + $(HOST_GO_VARS) \ ) ifneq ($(HOST_GO_ENABLE_PIE),) @@ -219,11 +225,8 @@ define Host/Compile ( \ cd $(HOST_BUILD_DIR)/bin ; \ $(CP) go go-nopie ; \ - GOCACHE=$(GO_BUILD_CACHE_DIR) \ - GOENV=off \ - CC=$(HOSTCC_NOCACHE) \ - CXX=$(HOSTCXX_NOCACHE) \ - ./go-nopie install -a -buildmode=pie std cmd ; \ + $(HOST_GO_VARS) \ + ./go-nopie install -a -buildmode pie std cmd ; \ retval=$$$$? ; \ rm -f go-nopie ; \ exit $$$$retval ; \ From 03a1f875496d11e7f3fbba8c148ef33969f6ec03 Mon Sep 17 00:00:00 2001 From: Jeffery To Date: Tue, 15 Sep 2020 23:32:53 +0800 Subject: [PATCH 08/15] golang: Use GO_LDFLAGS to set buildmode=pie for host Go Signed-off-by: Jeffery To --- lang/golang/golang/Makefile | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/lang/golang/golang/Makefile b/lang/golang/golang/Makefile index 162b5ad5e..5610ccafb 100644 --- a/lang/golang/golang/Makefile +++ b/lang/golang/golang/Makefile @@ -189,9 +189,8 @@ ifeq ($(GO_HOST_PIE_SUPPORTED),1) HOST_GO_ENABLE_PIE:=1 endif -ifeq ($(HOST_GO_ENABLE_PIE),1) - HOST_GO_INSTALL_SUFFIX:=$(GO_HOST_PIE_INSTALL_SUFFIX) -endif +# when using GO_LDFLAGS to set buildmode=pie, the PIE install suffix +# does not apply (we also delete the std lib during Host/Install) $(eval $(call GoCompiler/AddProfile,Host,$(HOST_BUILD_DIR),$(HOST_GO_PREFIX),$(HOST_GO_VERSION_ID),$(GO_HOST_OS_ARCH),$(HOST_GO_INSTALL_SUFFIX))) @@ -201,9 +200,6 @@ HOST_GO_VARS= \ CC=$(HOSTCC_NOCACHE) \ CXX=$(HOSTCXX_NOCACHE) -# when https://github.com/golang/go/issues/31544 is fixed, -# we should be able to set GO_LDFLAGS=-buildmode=pie for host make -# instead of doing a rebuild for pie define Host/Compile $(call GoCompiler/Bootstrap/CheckHost,$(BOOTSTRAP_GO_VALID_OS_ARCH)) $(call GoCompiler/Host/CheckHost,$(HOST_GO_VALID_OS_ARCH)) @@ -216,22 +212,9 @@ define Host/Compile $(call GoCompiler/Host/Make, \ GOROOT_BOOTSTRAP=$(BOOTSTRAP_ROOT_DIR) \ + $(if $(HOST_GO_ENABLE_PIE),GO_LDFLAGS="-buildmode pie") \ $(HOST_GO_VARS) \ ) - - ifneq ($(HOST_GO_ENABLE_PIE),) - @echo "Rebuilding host Go with PIE" - - ( \ - cd $(HOST_BUILD_DIR)/bin ; \ - $(CP) go go-nopie ; \ - $(HOST_GO_VARS) \ - ./go-nopie install -a -buildmode pie std cmd ; \ - retval=$$$$? ; \ - rm -f go-nopie ; \ - exit $$$$retval ; \ - ) - endif endef # if host and target os/arch are the same, From 93ee27edb8785514142633d41a973a8c5ca79e4c Mon Sep 17 00:00:00 2001 From: Jeffery To Date: Wed, 2 Sep 2020 21:22:15 +0800 Subject: [PATCH 09/15] golang: Simplify target compile * Reuse GO_PKG_ENABLE_PIE from golang-package.mk * Replace $(if ...) with $(or ...) to get default values * Refactor vars common to each compile stage into variable Signed-off-by: Jeffery To --- lang/golang/golang/Makefile | 55 ++++++++++++++----------------------- 1 file changed, 21 insertions(+), 34 deletions(-) diff --git a/lang/golang/golang/Makefile b/lang/golang/golang/Makefile index 5610ccafb..241ba1e37 100644 --- a/lang/golang/golang/Makefile +++ b/lang/golang/golang/Makefile @@ -247,40 +247,38 @@ endef # Target -ifdef CONFIG_PKG_ASLR_PIE_ALL - ifeq ($(GO_TARGET_PIE_SUPPORTED),1) - PKG_GO_ENABLE_PIE:=1 - PKG_GO_INSTALL_SUFFIX:=$(GO_TARGET_PIE_INSTALL_SUFFIX) - else - $(warning PIE buildmode is not supported for $(GO_OS)/$(GO_ARCH)) - endif +ifeq ($(GO_PKG_ENABLE_PIE),1) + PKG_GO_INSTALL_SUFFIX:=$(GO_TARGET_PIE_INSTALL_SUFFIX) endif $(eval $(call GoCompiler/AddProfile,Package,$(PKG_BUILD_DIR),$(PKG_GO_PREFIX),$(PKG_GO_VERSION_ID),$(GO_OS_ARCH),$(PKG_GO_INSTALL_SUFFIX))) PKG_GO_ZBOOTSTRAP_MODS:= \ - s/defaultGO386 = `[^`]*`/defaultGO386 = `$(if $(GO_386),$(GO_386),387)`/; \ - s/defaultGOARM = `[^`]*`/defaultGOARM = `$(if $(GO_ARM),$(GO_ARM),5)`/; \ - s/defaultGOMIPS = `[^`]*`/defaultGOMIPS = `$(if $(GO_MIPS),$(GO_MIPS),hardfloat)`/; \ - s/defaultGOMIPS64 = `[^`]*`/defaultGOMIPS64 = `$(if $(GO_MIPS64),$(GO_MIPS64),hardfloat)`/; \ + s/defaultGO386 = `[^`]*`/defaultGO386 = `$(or $(GO_386),387)`/; \ + s/defaultGOARM = `[^`]*`/defaultGOARM = `$(or $(GO_ARM),5)`/; \ + s/defaultGOMIPS = `[^`]*`/defaultGOMIPS = `$(or $(GO_MIPS),hardfloat)`/; \ + s/defaultGOMIPS64 = `[^`]*`/defaultGOMIPS64 = `$(or $(GO_MIPS64),hardfloat)`/; \ s/defaultGOPPC64 = `[^`]*`/defaultGOPPC64 = `power8`/; -PKG_GO_LDFLAGS= \ - -buildid '$(SOURCE_DATE_EPOCH)' +PKG_GO_VARS= \ + GOCACHE=$(GO_BUILD_CACHE_DIR) \ + GOENV=off \ + GO_GCC_HELPER_PATH=$$$$PATH \ + CC=gcc \ + CXX=g++ \ + PKG_CONFIG=pkg-config \ + PATH=$(HOST_GO_ROOT)/openwrt:$$$$PATH -ifeq ($(CONFIG_NO_STRIP)$(CONFIG_DEBUG),) - PKG_GO_LDFLAGS+= -s -w -endif +PKG_GO_LDFLAGS= \ + -buildid '$(SOURCE_DATE_EPOCH)' \ + $(if $(CONFIG_NO_STRIP)$(CONFIG_DEBUG),,-s -w) # setting -trimpath is not necessary here because the paths inside the # compiler binary are relative to GOROOT_FINAL (PKG_GO_ROOT), which is # static / not dependent on the build environment PKG_GO_INSTALL_ARGS= \ - -ldflags "all=$(PKG_GO_LDFLAGS)" - -ifeq ($(PKG_GO_ENABLE_PIE),1) - PKG_GO_INSTALL_ARGS+= -buildmode pie -endif + -ldflags "all=$(PKG_GO_LDFLAGS)" \ + $(if $(filter $(GO_PKG_ENABLE_PIE),1),-buildmode pie) define Build/Compile mkdir -p $(GO_BUILD_CACHE_DIR) @@ -289,14 +287,9 @@ define Build/Compile $(call GoCompiler/Package/Make, \ GOROOT_BOOTSTRAP=$(HOST_GO_ROOT) \ - GOCACHE=$(GO_BUILD_CACHE_DIR) \ GO_GCC_HELPER_CC="$(HOSTCC)" \ GO_GCC_HELPER_CXX="$(HOSTCXX)" \ - GO_GCC_HELPER_PATH=$$$$PATH \ - CC=gcc \ - CXX=g++ \ - PKG_CONFIG=pkg-config \ - PATH=$(HOST_GO_ROOT)/openwrt:$$$$PATH \ + $(PKG_GO_VARS) \ ) $(SED) '$(PKG_GO_ZBOOTSTRAP_MODS)' \ @@ -309,15 +302,9 @@ define Build/Compile export $(GO_PKG_TARGET_VARS) ; \ $(CP) go go-host ; \ GOROOT_FINAL=$(PKG_GO_ROOT) \ - GOCACHE=$(GO_BUILD_CACHE_DIR) \ - GOENV=off \ GO_GCC_HELPER_CC="$(TARGET_CC)" \ GO_GCC_HELPER_CXX="$(TARGET_CXX)" \ - GO_GCC_HELPER_PATH=$$$$PATH \ - CC=gcc \ - CXX=g++ \ - PKG_CONFIG=pkg-config \ - PATH=$(HOST_GO_ROOT)/openwrt:$$$$PATH \ + $(PKG_GO_VARS) \ ./go-host install -a $(PKG_GO_INSTALL_ARGS) std cmd ; \ retval=$$$$? ; \ rm -f go-host ; \ From a95afd67df44efa803e4d3d134e4f8d4f70e7863 Mon Sep 17 00:00:00 2001 From: Jeffery To Date: Sat, 3 Oct 2020 19:12:11 +0800 Subject: [PATCH 10/15] golang: Use external linking for target Go In Go 1.15, the linker now defaults to internal linking mode for -buildmode=pie on amd64 and arm64[1], however this results in go tool binaries with the wrong dynamic linker/interpreter. External linking is still used when PIE is enabled for other platforms, whereas internal linking is used when PIE is not enabled. This changes target Go to always use external linking, to fix PIE binaries for amd64/arm64 and for consistency. [1]: https://golang.org/doc/go1.15#linker Signed-off-by: Jeffery To --- lang/golang/golang/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lang/golang/golang/Makefile b/lang/golang/golang/Makefile index 241ba1e37..6990c249e 100644 --- a/lang/golang/golang/Makefile +++ b/lang/golang/golang/Makefile @@ -271,6 +271,8 @@ PKG_GO_VARS= \ PKG_GO_LDFLAGS= \ -buildid '$(SOURCE_DATE_EPOCH)' \ + -linkmode external \ + -extldflags '$(patsubst -z%,-Wl$(comma)-z$(comma)%,$(TARGET_LDFLAGS))' \ $(if $(CONFIG_NO_STRIP)$(CONFIG_DEBUG),,-s -w) # setting -trimpath is not necessary here because the paths inside the From 7aaaa8ae64ecebd2c16b2c1ce2174fe4b750fb56 Mon Sep 17 00:00:00 2001 From: Jeffery To Date: Fri, 28 Aug 2020 16:45:02 +0800 Subject: [PATCH 11/15] golang: Add more error handling for go-gcc-helper Signed-off-by: Jeffery To --- lang/golang/golang/files/go-gcc-helper | 32 ++++++++++++++++++++------ 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/lang/golang/golang/files/go-gcc-helper b/lang/golang/golang/files/go-gcc-helper index 3d3f8935d..b396e302c 100644 --- a/lang/golang/golang/files/go-gcc-helper +++ b/lang/golang/golang/files/go-gcc-helper @@ -1,23 +1,41 @@ #!/bin/sh me=go-gcc-helper -name=$(basename $0) +name="${0##*/}" -case $name in +log() { + # shellcheck disable=SC2039 + local IFS=" " + printf '%s\n' "$me: $*" +} + +case "$name" in gcc) - cmd=$GO_GCC_HELPER_CC + if [ -z "$GO_GCC_HELPER_CC" ]; then + log "missing GO_GCC_HELPER_CC" + exit 1 + fi + cmd="$GO_GCC_HELPER_CC" ;; g++) - cmd=$GO_GCC_HELPER_CXX + if [ -z "$GO_GCC_HELPER_CXX" ]; then + log "missing GO_GCC_HELPER_CXX" + exit 1 + fi + cmd="$GO_GCC_HELPER_CXX" ;; *) - echo "$me: unknown command \"$name\"" + log "unknown command \"$name\"" exit 1 ;; esac -export PATH="$GO_GCC_HELPER_PATH" +if [ -n "$GO_GCC_HELPER_PATH" ]; then + export PATH="$GO_GCC_HELPER_PATH" +else + log "missing GO_GCC_HELPER_PATH" +fi -echo "$me: running $cmd $@" +log "running $cmd $*" $cmd "$@" From 4773c288cd961acef429ff4b93f45147426a91bb Mon Sep 17 00:00:00 2001 From: Jeffery To Date: Sat, 29 Aug 2020 02:07:34 +0800 Subject: [PATCH 12/15] golang: Quote variables passed to the shell Signed-off-by: Jeffery To --- lang/golang/golang-compiler.mk | 60 ++++++++++----------- lang/golang/golang-package.mk | 98 +++++++++++++++++----------------- lang/golang/golang/Makefile | 50 ++++++++--------- 3 files changed, 104 insertions(+), 104 deletions(-) diff --git a/lang/golang/golang-compiler.mk b/lang/golang/golang-compiler.mk index cf350d82c..e8b4b9156 100644 --- a/lang/golang/golang-compiler.mk +++ b/lang/golang/golang-compiler.mk @@ -24,8 +24,8 @@ endef # $(4) additional environment variables (optional) define GoCompiler/Default/Make ( \ - cd $(1)/src ; \ - $(if $(2),GOROOT_FINAL=$(2)/lib/go-$(3)) \ + cd "$(1)/src" ; \ + $(if $(2),GOROOT_FINAL="$(2)/lib/go-$(3)") \ $(4) \ $(BASH) make.bash --no-banner ; \ ) @@ -34,8 +34,8 @@ endef # $(1) destination prefix # $(2) go version id define GoCompiler/Default/Install/make-dirs - $(INSTALL_DIR) $(1)/lib/go-$(2) - $(INSTALL_DIR) $(1)/share/go-$(2) + $(INSTALL_DIR) "$(1)/lib/go-$(2)" + $(INSTALL_DIR) "$(1)/share/go-$(2)" endef # $(1) source go root @@ -43,8 +43,8 @@ endef # $(3) go version id # $(4) file/directory name define GoCompiler/Default/Install/install-share-data - $(CP) $(1)/$(4) $(2)/share/go-$(3)/ - $(LN) ../../share/go-$(3)/$(4) $(2)/lib/go-$(3)/ + $(CP) "$(1)/$(4)" "$(2)/share/go-$(3)/" + $(LN) "../../share/go-$(3)/$(4)" "$(2)/lib/go-$(3)/" endef # $(1) source go root @@ -57,35 +57,35 @@ define GoCompiler/Default/Install/Bin $(call GoCompiler/Default/Install/install-share-data,$(1),$(2),$(3),api) - $(INSTALL_DATA) -p $(1)/VERSION $(2)/lib/go-$(3)/ + $(INSTALL_DATA) -p "$(1)/VERSION" "$(2)/lib/go-$(3)/" for file in AUTHORS CONTRIBUTING.md CONTRIBUTORS LICENSE PATENTS README.md SECURITY.md; do \ - if [ -f $(1)/$$$$file ]; then \ - $(INSTALL_DATA) -p $(1)/$$$$file $(2)/share/go-$(3)/ ; \ + if [ -f "$(1)/$$$$file" ]; then \ + $(INSTALL_DATA) -p "$(1)/$$$$file" "$(2)/share/go-$(3)/" ; \ fi ; \ done - $(INSTALL_DIR) $(2)/lib/go-$(3)/bin + $(INSTALL_DIR) "$(2)/lib/go-$(3)/bin" ifeq ($(4),$(GO_HOST_OS_ARCH)) - $(INSTALL_BIN) -p $(1)/bin/* $(2)/lib/go-$(3)/bin/ + $(INSTALL_BIN) -p "$(1)/bin"/* "$(2)/lib/go-$(3)/bin/" else - $(INSTALL_BIN) -p $(1)/bin/$(4)/* $(2)/lib/go-$(3)/bin/ + $(INSTALL_BIN) -p "$(1)/bin/$(4)"/* "$(2)/lib/go-$(3)/bin/" endif - $(INSTALL_DIR) $(2)/lib/go-$(3)/pkg - $(CP) $(1)/pkg/$(4)$(if $(5),_$(5)) $(2)/lib/go-$(3)/pkg/ + $(INSTALL_DIR) "$(2)/lib/go-$(3)/pkg" + $(CP) "$(1)/pkg/$(4)$(if $(5),_$(5))" "$(2)/lib/go-$(3)/pkg/" - $(INSTALL_DIR) $(2)/lib/go-$(3)/pkg/tool/$(4) - $(INSTALL_BIN) -p $(1)/pkg/tool/$(4)/* $(2)/lib/go-$(3)/pkg/tool/$(4)/ + $(INSTALL_DIR) "$(2)/lib/go-$(3)/pkg/tool/$(4)" + $(INSTALL_BIN) -p "$(1)/pkg/tool/$(4)"/* "$(2)/lib/go-$(3)/pkg/tool/$(4)/" endef # $(1) destination prefix # $(2) go version id define GoCompiler/Default/Install/BinLinks - $(INSTALL_DIR) $(1)/bin - $(LN) ../lib/go-$(2)/bin/go $(1)/bin/go - $(LN) ../lib/go-$(2)/bin/gofmt $(1)/bin/gofmt + $(INSTALL_DIR) "$(1)/bin" + $(LN) "../lib/go-$(2)/bin/go" "$(1)/bin/go" + $(LN) "../lib/go-$(2)/bin/gofmt" "$(1)/bin/gofmt" endef # $(1) source go root @@ -111,29 +111,29 @@ define GoCompiler/Default/Install/Src $(call GoCompiler/Default/Install/install-share-data,$(1),$(2),$(3),test) $(FIND) \ - $(2)/share/go-$(3)/src/ \ - \! -type d -a \( -name '*.bat' -o -name '*.rc' \) \ + "$(2)/share/go-$(3)/src/" \ + \! -type d -a \( -name "*.bat" -o -name "*.rc" \) \ -delete - if [ -d $(1)/pkg/include ]; then \ - $(INSTALL_DIR) $(2)/lib/go-$(3)/pkg ; \ - $(INSTALL_DIR) $(2)/share/go-$(3)/pkg ; \ - $(CP) $(1)/pkg/include $(2)/share/go-$(3)/pkg/ ; \ - $(LN) ../../../share/go-$(3)/pkg/include $(2)/lib/go-$(3)/pkg/ ; \ + if [ -d "$(1)/pkg/include" ]; then \ + $(INSTALL_DIR) "$(2)/lib/go-$(3)/pkg" ; \ + $(INSTALL_DIR) "$(2)/share/go-$(3)/pkg" ; \ + $(CP) "$(1)/pkg/include" "$(2)/share/go-$(3)/pkg/" ; \ + $(LN) "../../../share/go-$(3)/pkg/include" "$(2)/lib/go-$(3)/pkg/" ; \ fi endef # $(1) destination prefix # $(2) go version id define GoCompiler/Default/Uninstall - rm -rf $(1)/lib/go-$(2) - rm -rf $(1)/share/go-$(2) + rm -rf "$(1)/lib/go-$(2)" + rm -rf "$(1)/share/go-$(2)" endef # $(1) destination prefix define GoCompiler/Default/Uninstall/BinLinks - rm -f $(1)/bin/go - rm -f $(1)/bin/gofmt + rm -f "$(1)/bin/go" + rm -f "$(1)/bin/gofmt" endef diff --git a/lang/golang/golang-package.mk b/lang/golang/golang-package.mk index 46b484b4d..b3d01857f 100644 --- a/lang/golang/golang-package.mk +++ b/lang/golang/golang-package.mk @@ -165,24 +165,24 @@ define GoPackage/GoSubMenu endef GO_PKG_TARGET_VARS= \ - GOOS=$(GO_OS) \ - GOARCH=$(GO_ARCH) \ - GO386=$(GO_386) \ - GOARM=$(GO_ARM) \ - GOMIPS=$(GO_MIPS) \ - GOMIPS64=$(GO_MIPS64) \ + GOOS="$(GO_OS)" \ + GOARCH="$(GO_ARCH)" \ + GO386="$(GO_386)" \ + GOARM="$(GO_ARM)" \ + GOMIPS="$(GO_MIPS)" \ + GOMIPS64="$(GO_MIPS64)" \ CGO_ENABLED=1 \ - CC=$(TARGET_CC) \ - CXX=$(TARGET_CXX) \ + CC="$(TARGET_CC)" \ + CXX="$(TARGET_CXX)" \ CGO_CFLAGS="$(filter-out $(GO_CFLAGS_TO_REMOVE),$(TARGET_CFLAGS))" \ CGO_CPPFLAGS="$(TARGET_CPPFLAGS)" \ CGO_CXXFLAGS="$(filter-out $(GO_CFLAGS_TO_REMOVE),$(TARGET_CXXFLAGS))" \ CGO_LDFLAGS="$(TARGET_LDFLAGS)" GO_PKG_BUILD_VARS= \ - GOPATH=$(GO_PKG_BUILD_DIR) \ - GOCACHE=$(GO_BUILD_CACHE_DIR) \ - GOMODCACHE=$(GO_MOD_CACHE_DIR) \ + GOPATH="$(GO_PKG_BUILD_DIR)" \ + GOCACHE="$(GO_BUILD_CACHE_DIR)" \ + GOMODCACHE="$(GO_MOD_CACHE_DIR)" \ GOENV=off GO_PKG_DEFAULT_VARS= \ @@ -213,77 +213,77 @@ GO_PKG_INSTALL_ARGS= \ $(if $(GO_PKG_TAGS),-tags "$(GO_PKG_TAGS)") # false if directory does not exist -GoPackage/is_dir_not_empty=$$$$($(FIND) $(1) -maxdepth 0 -type d \! -empty 2>/dev/null) +GoPackage/is_dir_not_empty=$$$$($(FIND) "$(1)" -maxdepth 0 -type d \! -empty 2>/dev/null) GoPackage/has_binaries=$(call GoPackage/is_dir_not_empty,$(GO_PKG_BUILD_BIN_DIR)) define GoPackage/Build/Configure ( \ - cd $(PKG_BUILD_DIR) ; \ + cd "$(PKG_BUILD_DIR)" ; \ mkdir -p \ - $(GO_PKG_BUILD_DIR)/bin $(GO_PKG_BUILD_DIR)/src \ - $(GO_BUILD_CACHE_DIR) $(GO_MOD_CACHE_DIR) ; \ + "$(GO_PKG_BUILD_DIR)/bin" "$(GO_PKG_BUILD_DIR)/src" \ + "$(GO_BUILD_CACHE_DIR)" "$(GO_MOD_CACHE_DIR)" ; \ \ - files=$$$$($(FIND) ./ \ - -type d -a \( -path './.git' -o -path './$(GO_PKG_WORK_DIR_NAME)' \) -prune -o \ + files="$$$$($(FIND) ./ \ + -type d -a \( -path "./.git" -o -path "./$(GO_PKG_WORK_DIR_NAME)" \) -prune -o \ \! -type d -print | \ - sed 's|^\./||') ; \ + sed 's|^\./||')" ; \ \ if [ "$(strip $(GO_PKG_INSTALL_ALL))" != 1 ]; then \ - code=$$$$(echo "$$$$files" | grep '\.\(c\|cc\|cpp\|go\|h\|hh\|hpp\|proto\|s\)$$$$') ; \ - testdata=$$$$(echo "$$$$files" | grep '\(^\|/\)testdata/') ; \ - gomod=$$$$(echo "$$$$files" | grep '\(^\|/\)go\.\(mod\|sum\)$$$$') ; \ + code="$$$$(echo "$$$$files" | grep '\.\(c\|cc\|cpp\|go\|h\|hh\|hpp\|proto\|s\)$$$$')" ; \ + testdata="$$$$(echo "$$$$files" | grep '\(^\|/\)testdata/')" ; \ + gomod="$$$$(echo "$$$$files" | grep '\(^\|/\)go\.\(mod\|sum\)$$$$')" ; \ \ for pattern in $(GO_PKG_INSTALL_EXTRA); do \ - extra=$$$$(echo "$$$$extra"; echo "$$$$files" | grep "$$$$pattern") ; \ + extra="$$$$(echo "$$$$extra"; echo "$$$$files" | grep "$$$$pattern")" ; \ done ; \ \ - files=$$$$(echo "$$$$code"; echo "$$$$testdata"; echo "$$$$gomod"; echo "$$$$extra") ; \ - files=$$$$(echo "$$$$files" | grep -v '^[[:space:]]*$$$$' | sort -u) ; \ + files="$$$$(echo "$$$$code"; echo "$$$$testdata"; echo "$$$$gomod"; echo "$$$$extra")" ; \ + files="$$$$(echo "$$$$files" | grep -v '^[[:space:]]*$$$$' | sort -u)" ; \ fi ; \ \ IFS=$$$$'\n' ; \ \ echo "Copying files from $(PKG_BUILD_DIR) into $(GO_PKG_BUILD_DIR)/src/$(strip $(GO_PKG))" ; \ for file in $$$$files; do \ - echo $$$$file ; \ - dest=$(GO_PKG_BUILD_DIR)/src/$(strip $(GO_PKG))/$$$$file ; \ - mkdir -p $$$$(dirname $$$$dest) ; \ - $(CP) $$$$file $$$$dest ; \ + echo "$$$$file" ; \ + dest="$(GO_PKG_BUILD_DIR)/src/$(strip $(GO_PKG))/$$$$file" ; \ + mkdir -p "$$$$(dirname "$$$$dest")" ; \ + $(CP) "$$$$file" "$$$$dest" ; \ done ; \ echo ; \ \ link_contents() { \ - local src=$$$$1 ; \ - local dest=$$$$2 ; \ + local src="$$$$1" ; \ + local dest="$$$$2" ; \ local dirs dir base ; \ \ - if [ -n "$$$$($(FIND) $$$$src -mindepth 1 -maxdepth 1 -name '*.go' \! -type d)" ]; then \ + if [ -n "$$$$($(FIND) "$$$$src" -mindepth 1 -maxdepth 1 -name "*.go" \! -type d)" ]; then \ echo "$$$$src is already a Go library" ; \ return 1 ; \ fi ; \ \ - dirs=$$$$($(FIND) $$$$src -mindepth 1 -maxdepth 1 -type d) ; \ + dirs="$$$$($(FIND) "$$$$src" -mindepth 1 -maxdepth 1 -type d)" ; \ for dir in $$$$dirs; do \ - base=$$$$(basename $$$$dir) ; \ - if [ -d $$$$dest/$$$$base ]; then \ - case $$$$dir in \ + base="$$$$(basename "$$$$dir")" ; \ + if [ -d "$$$$dest/$$$$base" ]; then \ + case "$$$$dir" in \ *$(GO_PKG_PATH)/src/$(strip $(GO_PKG))) \ echo "$(strip $(GO_PKG)) is already installed. Please check for circular dependencies." ;; \ *) \ - link_contents $$$$src/$$$$base $$$$dest/$$$$base ;; \ + link_contents "$$$$src/$$$$base" "$$$$dest/$$$$base" ;; \ esac ; \ else \ echo "...$$$${src#$(GO_PKG_BUILD_DEPENDS_SRC)}/$$$$base" ; \ - $(LN) $$$$src/$$$$base $$$$dest/$$$$base ; \ + $(LN) "$$$$src/$$$$base" "$$$$dest/$$$$base" ; \ fi ; \ done ; \ } ; \ \ if [ "$(strip $(GO_PKG_SOURCE_ONLY))" != 1 ]; then \ - if [ -d $(GO_PKG_BUILD_DEPENDS_SRC) ]; then \ + if [ -d "$(GO_PKG_BUILD_DEPENDS_SRC)" ]; then \ echo "Symlinking directories from $(GO_PKG_BUILD_DEPENDS_SRC) into $(GO_PKG_BUILD_DIR)/src" ; \ - link_contents $(GO_PKG_BUILD_DEPENDS_SRC) $(GO_PKG_BUILD_DIR)/src ; \ + link_contents "$(GO_PKG_BUILD_DEPENDS_SRC)" "$(GO_PKG_BUILD_DIR)/src" ; \ else \ echo "$(GO_PKG_BUILD_DEPENDS_SRC) does not exist, skipping symlinks" ; \ fi ; \ @@ -297,16 +297,16 @@ endef # $(1) additional arguments for go command line (optional) define GoPackage/Build/Compile ( \ - cd $(GO_PKG_BUILD_DIR) ; \ + cd "$(GO_PKG_BUILD_DIR)" ; \ export $(GO_PKG_VARS) ; \ if [ -f "$(PKG_BUILD_DIR)/go.mod" ] ; then \ modargs="$(GO_MOD_ARGS)" ; \ fi ; \ \ echo "Finding targets" ; \ - targets=$$$$(go list $$$$modargs $(GO_PKG_BUILD_PKG)) ; \ + targets="$$$$(go list $$$$modargs $(GO_PKG_BUILD_PKG))" ; \ for pattern in $(GO_PKG_EXCLUDES); do \ - targets=$$$$(echo "$$$$targets" | grep -v "$$$$pattern") ; \ + targets="$$$$(echo "$$$$targets" | grep -v "$$$$pattern")" ; \ done ; \ echo ; \ \ @@ -319,7 +319,7 @@ define GoPackage/Build/Compile if [ "$(strip $(GO_PKG_SOURCE_ONLY))" != 1 ]; then \ echo "Building targets" ; \ go install $(GO_PKG_INSTALL_ARGS) $$$$modargs $(1) $$$$targets ; \ - retval=$$$$? ; \ + retval="$$$$?" ; \ echo ; \ \ if [ "$$$$retval" -eq 0 ] && [ -z "$(call GoPackage/has_binaries)" ]; then \ @@ -331,7 +331,7 @@ define GoPackage/Build/Compile $(call Go/CacheCleanup) ; \ fi ; \ fi ; \ - exit $$$$retval ; \ + exit "$$$$retval" ; \ ) endef @@ -341,15 +341,15 @@ endef define GoPackage/Package/Install/Bin if [ -n "$(call GoPackage/has_binaries)" ]; then \ - $(INSTALL_DIR) $(1)/usr/bin ; \ - $(INSTALL_BIN) $(GO_PKG_BUILD_BIN_DIR)/* $(1)/usr/bin/ ; \ + $(INSTALL_DIR) "$(1)/usr/bin" ; \ + $(INSTALL_BIN) "$(GO_PKG_BUILD_BIN_DIR)"/* "$(1)/usr/bin/" ; \ fi endef define GoPackage/Package/Install/Src - dir=$$$$(dirname $(GO_PKG)) ; \ - $(INSTALL_DIR) $(1)$(GO_PKG_PATH)/src/$$$$dir ; \ - $(CP) $(GO_PKG_BUILD_DIR)/src/$(strip $(GO_PKG)) $(1)$(GO_PKG_PATH)/src/$$$$dir/ + dir="$$$$(dirname "$(GO_PKG)")" ; \ + $(INSTALL_DIR) "$(1)$(GO_PKG_PATH)/src/$$$$dir" ; \ + $(CP) "$(GO_PKG_BUILD_DIR)/src/$(strip $(GO_PKG))" "$(1)$(GO_PKG_PATH)/src/$$$$dir/" endef define GoPackage/Package/Install diff --git a/lang/golang/golang/Makefile b/lang/golang/golang/Makefile index 6990c249e..a5474753b 100644 --- a/lang/golang/golang/Makefile +++ b/lang/golang/golang/Makefile @@ -88,9 +88,9 @@ include $(INCLUDE_DIR)/package.mk include ../golang-compiler.mk include ../golang-package.mk -PKG_UNPACK:=$(HOST_TAR) -C $(PKG_BUILD_DIR) --strip-components=1 -xzf $(DL_DIR)/$(PKG_SOURCE) -HOST_UNPACK:=$(HOST_TAR) -C $(HOST_BUILD_DIR) --strip-components=1 -xzf $(DL_DIR)/$(PKG_SOURCE) -BOOTSTRAP_UNPACK:=$(HOST_TAR) -C $(BOOTSTRAP_BUILD_DIR) --strip-components=1 -xzf $(DL_DIR)/$(BOOTSTRAP_SOURCE) +PKG_UNPACK:=$(HOST_TAR) -C "$(PKG_BUILD_DIR)" --strip-components=1 -xzf "$(DL_DIR)/$(PKG_SOURCE)" +HOST_UNPACK:=$(HOST_TAR) -C "$(HOST_BUILD_DIR)" --strip-components=1 -xzf "$(DL_DIR)/$(PKG_SOURCE)" +BOOTSTRAP_UNPACK:=$(HOST_TAR) -C "$(BOOTSTRAP_BUILD_DIR)" --strip-components=1 -xzf "$(DL_DIR)/$(BOOTSTRAP_SOURCE)" # don't strip ELF executables in test data RSTRIP:=: @@ -174,7 +174,7 @@ ifeq ($(BOOTSTRAP_ROOT_DIR),) $(eval $(call Download,golang-bootstrap)) define Bootstrap/Prepare - mkdir -p $(BOOTSTRAP_BUILD_DIR) + mkdir -p "$(BOOTSTRAP_BUILD_DIR)" $(BOOTSTRAP_UNPACK) endef Hooks/HostPrepare/Post+=Bootstrap/Prepare @@ -195,23 +195,23 @@ endif $(eval $(call GoCompiler/AddProfile,Host,$(HOST_BUILD_DIR),$(HOST_GO_PREFIX),$(HOST_GO_VERSION_ID),$(GO_HOST_OS_ARCH),$(HOST_GO_INSTALL_SUFFIX))) HOST_GO_VARS= \ - GOCACHE=$(GO_BUILD_CACHE_DIR) \ + GOCACHE="$(GO_BUILD_CACHE_DIR)" \ GOENV=off \ - CC=$(HOSTCC_NOCACHE) \ - CXX=$(HOSTCXX_NOCACHE) + CC="$(HOSTCC_NOCACHE)" \ + CXX="$(HOSTCXX_NOCACHE)" define Host/Compile $(call GoCompiler/Bootstrap/CheckHost,$(BOOTSTRAP_GO_VALID_OS_ARCH)) $(call GoCompiler/Host/CheckHost,$(HOST_GO_VALID_OS_ARCH)) - mkdir -p $(GO_BUILD_CACHE_DIR) + mkdir -p "$(GO_BUILD_CACHE_DIR)" $(call GoCompiler/Bootstrap/Make, \ $(HOST_GO_VARS) \ ) $(call GoCompiler/Host/Make, \ - GOROOT_BOOTSTRAP=$(BOOTSTRAP_ROOT_DIR) \ + GOROOT_BOOTSTRAP="$(BOOTSTRAP_ROOT_DIR)" \ $(if $(HOST_GO_ENABLE_PIE),GO_LDFLAGS="-buildmode pie") \ $(HOST_GO_VARS) \ ) @@ -228,16 +228,16 @@ define Host/Install $(call GoCompiler/Host/Install/BinLinks,) - rm -rf $(HOST_GO_ROOT)/pkg/$(GO_HOST_OS_ARCH)$(if $(HOST_GO_INSTALL_SUFFIX),_$(HOST_GO_INSTALL_SUFFIX)) + rm -rf "$(HOST_GO_ROOT)/pkg/$(GO_HOST_OS_ARCH)$(if $(HOST_GO_INSTALL_SUFFIX),_$(HOST_GO_INSTALL_SUFFIX))" - $(INSTALL_DIR) $(HOST_GO_ROOT)/openwrt - $(INSTALL_BIN) ./files/go-gcc-helper $(HOST_GO_ROOT)/openwrt/ - $(LN) go-gcc-helper $(HOST_GO_ROOT)/openwrt/gcc - $(LN) go-gcc-helper $(HOST_GO_ROOT)/openwrt/g++ + $(INSTALL_DIR) "$(HOST_GO_ROOT)/openwrt" + $(INSTALL_BIN) ./files/go-gcc-helper "$(HOST_GO_ROOT)/openwrt/" + $(LN) go-gcc-helper "$(HOST_GO_ROOT)/openwrt/gcc" + $(LN) go-gcc-helper "$(HOST_GO_ROOT)/openwrt/g++" endef define Host/Uninstall - rm -rf $(HOST_GO_ROOT)/openwrt + rm -rf "$(HOST_GO_ROOT)/openwrt" $(call GoCompiler/Host/Uninstall/BinLinks,) @@ -261,13 +261,13 @@ PKG_GO_ZBOOTSTRAP_MODS:= \ s/defaultGOPPC64 = `[^`]*`/defaultGOPPC64 = `power8`/; PKG_GO_VARS= \ - GOCACHE=$(GO_BUILD_CACHE_DIR) \ + GOCACHE="$(GO_BUILD_CACHE_DIR)" \ GOENV=off \ - GO_GCC_HELPER_PATH=$$$$PATH \ + GO_GCC_HELPER_PATH="$$$$PATH" \ CC=gcc \ CXX=g++ \ PKG_CONFIG=pkg-config \ - PATH=$(HOST_GO_ROOT)/openwrt:$$$$PATH + PATH="$(HOST_GO_ROOT)/openwrt:$$$$PATH" PKG_GO_LDFLAGS= \ -buildid '$(SOURCE_DATE_EPOCH)' \ @@ -283,34 +283,34 @@ PKG_GO_INSTALL_ARGS= \ $(if $(filter $(GO_PKG_ENABLE_PIE),1),-buildmode pie) define Build/Compile - mkdir -p $(GO_BUILD_CACHE_DIR) + mkdir -p "$(GO_BUILD_CACHE_DIR)" @echo "Building target Go first stage" $(call GoCompiler/Package/Make, \ - GOROOT_BOOTSTRAP=$(HOST_GO_ROOT) \ + GOROOT_BOOTSTRAP="$(HOST_GO_ROOT)" \ GO_GCC_HELPER_CC="$(HOSTCC)" \ GO_GCC_HELPER_CXX="$(HOSTCXX)" \ $(PKG_GO_VARS) \ ) $(SED) '$(PKG_GO_ZBOOTSTRAP_MODS)' \ - $(PKG_BUILD_DIR)/src/cmd/internal/objabi/zbootstrap.go + "$(PKG_BUILD_DIR)/src/cmd/internal/objabi/zbootstrap.go" @echo "Building target Go second stage" ( \ - cd $(PKG_BUILD_DIR)/bin ; \ + cd "$(PKG_BUILD_DIR)/bin" ; \ export $(GO_PKG_TARGET_VARS) ; \ $(CP) go go-host ; \ - GOROOT_FINAL=$(PKG_GO_ROOT) \ + GOROOT_FINAL="$(PKG_GO_ROOT)" \ GO_GCC_HELPER_CC="$(TARGET_CC)" \ GO_GCC_HELPER_CXX="$(TARGET_CXX)" \ $(PKG_GO_VARS) \ ./go-host install -a $(PKG_GO_INSTALL_ARGS) std cmd ; \ - retval=$$$$? ; \ + retval="$$$$?" ; \ rm -f go-host ; \ - exit $$$$retval ; \ + exit "$$$$retval" ; \ ) endef From 2d22a629f5494b8c032e0b56c639dd380f6202fb Mon Sep 17 00:00:00 2001 From: Jeffery To Date: Wed, 26 Aug 2020 16:17:05 +0800 Subject: [PATCH 13/15] golang: Move package scripts into golang-build.sh This also adds GO_PKG_INSTALL_BIN_PATH which determines the path where binaries are installed by GoPackage/Package/Install/Bin (this was previously hard-coded to /usr/bin). Signed-off-by: Jeffery To --- lang/golang/golang-build.sh | 156 +++++++++++++++++++++++++++++ lang/golang/golang-package.mk | 178 +++++++++------------------------- lang/golang/golang-values.mk | 4 +- 3 files changed, 206 insertions(+), 132 deletions(-) diff --git a/lang/golang/golang-build.sh b/lang/golang/golang-build.sh index d39949dca..5e40436bb 100644 --- a/lang/golang/golang-build.sh +++ b/lang/golang/golang-build.sh @@ -1,5 +1,8 @@ #!/bin/sh +nl=" +" + log() { # shellcheck disable=SC2039 local IFS=" " @@ -12,6 +15,147 @@ log_error() { printf 'Error: %s\n' "$*" >&2 } +link_contents() { + # shellcheck disable=SC2039 + local src="$1" dest="$2" IFS="$nl" dirs dir base + + if [ -n "$(find "$src" -mindepth 1 -maxdepth 1 -name "*.go" -not -type d)" ]; then + log_error "$src is already a Go library" + return 1 + fi + + dirs="$(find "$src" -mindepth 1 -maxdepth 1 -type d)" + for dir in $dirs; do + base="${dir##*/}" + if [ -d "$dest/$base" ]; then + case "$dir" in + *$GO_BUILD_DEPENDS_SRC/$GO_PKG) + log "$GO_PKG is already installed. Please check for circular dependencies." + ;; + *) + link_contents "$src/$base" "$dest/$base" + ;; + esac + else + log "...${src#$GO_BUILD_DEPENDS_SRC}/$base" + ln -sf "$src/$base" "$dest/$base" + fi + done + + return 0 +} + +configure() { + # shellcheck disable=SC2039 + local files code testdata gomod pattern extra IFS file dest + + cd "$BUILD_DIR" || return 1 + + files="$(find ./ -path "*/.*" -prune -o -not -type d -print)" + + if [ "$GO_INSTALL_ALL" != 1 ]; then + code="$(printf '%s\n' "$files" | grep '\.\(c\|cc\|cpp\|go\|h\|hh\|hpp\|proto\|s\)$')" + testdata="$(printf '%s\n' "$files" | grep '/testdata/')" + gomod="$(printf '%s\n' "$files" | grep '/go\.\(mod\|sum\)$')" + + for pattern in $GO_INSTALL_EXTRA; do + extra="$(printf '%s\n' "$extra"; printf '%s\n' "$files" | grep -e "$pattern")" + done + + files="$(printf '%s\n%s\n%s\n%s\n' "$code" "$testdata" "$gomod" "$extra" | grep -v '^[[:space:]]*$' | sort -u)" + fi + + IFS="$nl" + + log "Copying files from $BUILD_DIR into $GO_BUILD_DIR/src/$GO_PKG" + mkdir -p "$GO_BUILD_DIR/src" + for file in $files; do + log "${file#./}" + dest="$GO_BUILD_DIR/src/$GO_PKG/${file#./}" + mkdir -p "${dest%/*}" + cp -fpR "$file" "$dest" + done + log + + if [ "$GO_SOURCE_ONLY" != 1 ]; then + if [ -d "$GO_BUILD_DEPENDS_SRC" ]; then + log "Symlinking directories from $GO_BUILD_DEPENDS_SRC into $GO_BUILD_DIR/src" + link_contents "$GO_BUILD_DEPENDS_SRC" "$GO_BUILD_DIR/src" + else + log "$GO_BUILD_DEPENDS_SRC does not exist, skipping symlinks" + fi + else + log "Not building binaries, skipping symlinks" + fi + log + + return 0 +} + +build() { + # shellcheck disable=SC2039 + local modargs pattern targets retval + + cd "$GO_BUILD_DIR" || return 1 + + if [ -f "$BUILD_DIR/go.mod" ] ; then + mkdir -p "$GO_MOD_CACHE_DIR" + modargs="$GO_MOD_ARGS" + fi + + log "Finding targets" + # shellcheck disable=SC2086 + targets="$(go list $modargs $GO_BUILD_PKG)" + for pattern in $GO_EXCLUDES; do + targets="$(printf '%s\n' "$targets" | grep -v "$pattern")" + done + log + + if [ "$GO_GO_GENERATE" = 1 ]; then + log "Calling go generate" + # shellcheck disable=SC2086 + GOOS='' GOARCH='' GO386='' GOARM='' GOMIPS='' GOMIPS64='' \ + go generate -v $targets + log + fi + + if [ "$GO_SOURCE_ONLY" = 1 ]; then + return 0 + fi + + log "Building targets" + mkdir -p "$GO_BUILD_DIR/bin" "$GO_BUILD_CACHE_DIR" + # shellcheck disable=SC2086 + go install $modargs "$@" $targets + retval="$?" + log + + if [ "$retval" -eq 0 ] && [ -z "$(find "$GO_BUILD_BIN_DIR" -maxdepth 0 -type d -not -empty 2>/dev/null)" ]; then + log_error "No binaries were built" + retval=1 + fi + + if [ "$retval" -ne 0 ]; then + cache_cleanup + fi + + return "$retval" +} + +install_bin() { + # shellcheck disable=SC2039 + local dest="$1" + install -d -m0755 "$dest/$GO_INSTALL_BIN_PATH" + install -m0755 "$GO_BUILD_BIN_DIR"/* "$dest/$GO_INSTALL_BIN_PATH/" +} + +install_src() { + # shellcheck disable=SC2039 + local dest="$1" dir="${GO_PKG%/*}" + install -d -m0755 "$dest/$GO_BUILD_DEPENDS_PATH/src/$dir" + cp -fpR "$GO_BUILD_DIR/src/$GO_PKG" "$dest/$GO_BUILD_DEPENDS_PATH/src/$dir/" +} + cache_cleanup() { if ! [ -d "$GO_MOD_CACHE_DIR" ]; then return 0 @@ -38,6 +182,18 @@ command="$1" shift 1 case "$command" in + configure) + configure + ;; + build) + build "$@" + ;; + install_bin) + install_bin "$@" + ;; + install_src) + install_src "$@" + ;; cache_cleanup) cache_cleanup ;; diff --git a/lang/golang/golang-package.mk b/lang/golang/golang-package.mk index b3d01857f..8c62e6399 100644 --- a/lang/golang/golang-package.mk +++ b/lang/golang/golang-package.mk @@ -111,22 +111,29 @@ include $(GO_INCLUDE_DIR)/golang-values.mk # parameter to the -tags option for 'go install'. # # e.g. GO_PKG_TAGS:=release,noupgrade +# +# +# GO_PKG_INSTALL_BIN_PATH - target directory path, default /usr/bin +# +# Directory path under "dest_dir" where binaries will be installed by +# '$(call GoPackage/Package/Install/Bin,dest_dir)'. +# +# e.g. GO_PKG_INSTALL_BIN_PATH:=/sbin # Credit for this package build process (GoPackage/Build/Configure and # GoPackage/Build/Compile) belong to Debian's dh-golang completely. # https://salsa.debian.org/go-team/packages/dh-golang -# for building packages, not user code -GO_PKG_PATH:=/usr/share/gocode - GO_PKG_BUILD_PKG?=$(strip $(GO_PKG))/... +GO_PKG_INSTALL_BIN_PATH?=/usr/bin GO_PKG_WORK_DIR_NAME:=.go_work GO_PKG_BUILD_DIR=$(PKG_BUILD_DIR)/$(GO_PKG_WORK_DIR_NAME)/build GO_PKG_BUILD_BIN_DIR=$(GO_PKG_BUILD_DIR)/bin$(if $(GO_HOST_TARGET_DIFFERENT),/$(GO_OS_ARCH)) -GO_PKG_BUILD_DEPENDS_SRC=$(STAGING_DIR)$(GO_PKG_PATH)/src +GO_PKG_BUILD_DEPENDS_PATH:=/usr/share/gocode +GO_PKG_BUILD_DEPENDS_SRC=$(STAGING_DIR)$(GO_PKG_BUILD_DEPENDS_PATH)/src ifdef CONFIG_PKG_ASLR_PIE_ALL ifeq ($(strip $(PKG_ASLR_PIE)),1) @@ -164,6 +171,21 @@ define GoPackage/GoSubMenu CATEGORY:=Languages endef +GO_PKG_BUILD_CONFIG_VARS= \ + GO_PKG="$(strip $(GO_PKG))" \ + GO_INSTALL_EXTRA="$(strip $(GO_PKG_INSTALL_EXTRA))" \ + GO_INSTALL_ALL="$(strip $(GO_PKG_INSTALL_ALL))" \ + GO_SOURCE_ONLY="$(strip $(GO_PKG_SOURCE_ONLY))" \ + GO_BUILD_PKG="$(strip $(GO_PKG_BUILD_PKG))" \ + GO_EXCLUDES="$(strip $(GO_PKG_EXCLUDES))" \ + GO_GO_GENERATE="$(strip $(GO_PKG_GO_GENERATE))" \ + GO_INSTALL_BIN_PATH="$(strip $(GO_PKG_INSTALL_BIN_PATH))" \ + BUILD_DIR="$(PKG_BUILD_DIR)" \ + GO_BUILD_DIR="$(GO_PKG_BUILD_DIR)" \ + GO_BUILD_BIN_DIR="$(GO_PKG_BUILD_BIN_DIR)" \ + GO_BUILD_DEPENDS_PATH="$(GO_PKG_BUILD_DEPENDS_PATH)" \ + GO_BUILD_DEPENDS_SRC="$(GO_PKG_BUILD_DEPENDS_SRC)" + GO_PKG_TARGET_VARS= \ GOOS="$(GO_OS)" \ GOARCH="$(GO_ARCH)" \ @@ -185,12 +207,10 @@ GO_PKG_BUILD_VARS= \ GOMODCACHE="$(GO_MOD_CACHE_DIR)" \ GOENV=off -GO_PKG_DEFAULT_VARS= \ +GO_PKG_VARS= \ $(GO_PKG_TARGET_VARS) \ $(GO_PKG_BUILD_VARS) -GO_PKG_VARS=$(GO_PKG_DEFAULT_VARS) - GO_PKG_DEFAULT_LDFLAGS= \ -buildid '$(SOURCE_DATE_EPOCH)' \ -linkmode external \ @@ -212,127 +232,18 @@ GO_PKG_INSTALL_ARGS= \ $(if $(GO_PKG_CUSTOM_LDFLAGS),-ldflags "$(GO_PKG_CUSTOM_LDFLAGS) $(GO_PKG_DEFAULT_LDFLAGS)") \ $(if $(GO_PKG_TAGS),-tags "$(GO_PKG_TAGS)") -# false if directory does not exist -GoPackage/is_dir_not_empty=$$$$($(FIND) "$(1)" -maxdepth 0 -type d \! -empty 2>/dev/null) - -GoPackage/has_binaries=$(call GoPackage/is_dir_not_empty,$(GO_PKG_BUILD_BIN_DIR)) - define GoPackage/Build/Configure - ( \ - cd "$(PKG_BUILD_DIR)" ; \ - mkdir -p \ - "$(GO_PKG_BUILD_DIR)/bin" "$(GO_PKG_BUILD_DIR)/src" \ - "$(GO_BUILD_CACHE_DIR)" "$(GO_MOD_CACHE_DIR)" ; \ - \ - files="$$$$($(FIND) ./ \ - -type d -a \( -path "./.git" -o -path "./$(GO_PKG_WORK_DIR_NAME)" \) -prune -o \ - \! -type d -print | \ - sed 's|^\./||')" ; \ - \ - if [ "$(strip $(GO_PKG_INSTALL_ALL))" != 1 ]; then \ - code="$$$$(echo "$$$$files" | grep '\.\(c\|cc\|cpp\|go\|h\|hh\|hpp\|proto\|s\)$$$$')" ; \ - testdata="$$$$(echo "$$$$files" | grep '\(^\|/\)testdata/')" ; \ - gomod="$$$$(echo "$$$$files" | grep '\(^\|/\)go\.\(mod\|sum\)$$$$')" ; \ - \ - for pattern in $(GO_PKG_INSTALL_EXTRA); do \ - extra="$$$$(echo "$$$$extra"; echo "$$$$files" | grep "$$$$pattern")" ; \ - done ; \ - \ - files="$$$$(echo "$$$$code"; echo "$$$$testdata"; echo "$$$$gomod"; echo "$$$$extra")" ; \ - files="$$$$(echo "$$$$files" | grep -v '^[[:space:]]*$$$$' | sort -u)" ; \ - fi ; \ - \ - IFS=$$$$'\n' ; \ - \ - echo "Copying files from $(PKG_BUILD_DIR) into $(GO_PKG_BUILD_DIR)/src/$(strip $(GO_PKG))" ; \ - for file in $$$$files; do \ - echo "$$$$file" ; \ - dest="$(GO_PKG_BUILD_DIR)/src/$(strip $(GO_PKG))/$$$$file" ; \ - mkdir -p "$$$$(dirname "$$$$dest")" ; \ - $(CP) "$$$$file" "$$$$dest" ; \ - done ; \ - echo ; \ - \ - link_contents() { \ - local src="$$$$1" ; \ - local dest="$$$$2" ; \ - local dirs dir base ; \ - \ - if [ -n "$$$$($(FIND) "$$$$src" -mindepth 1 -maxdepth 1 -name "*.go" \! -type d)" ]; then \ - echo "$$$$src is already a Go library" ; \ - return 1 ; \ - fi ; \ - \ - dirs="$$$$($(FIND) "$$$$src" -mindepth 1 -maxdepth 1 -type d)" ; \ - for dir in $$$$dirs; do \ - base="$$$$(basename "$$$$dir")" ; \ - if [ -d "$$$$dest/$$$$base" ]; then \ - case "$$$$dir" in \ - *$(GO_PKG_PATH)/src/$(strip $(GO_PKG))) \ - echo "$(strip $(GO_PKG)) is already installed. Please check for circular dependencies." ;; \ - *) \ - link_contents "$$$$src/$$$$base" "$$$$dest/$$$$base" ;; \ - esac ; \ - else \ - echo "...$$$${src#$(GO_PKG_BUILD_DEPENDS_SRC)}/$$$$base" ; \ - $(LN) "$$$$src/$$$$base" "$$$$dest/$$$$base" ; \ - fi ; \ - done ; \ - } ; \ - \ - if [ "$(strip $(GO_PKG_SOURCE_ONLY))" != 1 ]; then \ - if [ -d "$(GO_PKG_BUILD_DEPENDS_SRC)" ]; then \ - echo "Symlinking directories from $(GO_PKG_BUILD_DEPENDS_SRC) into $(GO_PKG_BUILD_DIR)/src" ; \ - link_contents "$(GO_PKG_BUILD_DEPENDS_SRC)" "$(GO_PKG_BUILD_DIR)/src" ; \ - else \ - echo "$(GO_PKG_BUILD_DEPENDS_SRC) does not exist, skipping symlinks" ; \ - fi ; \ - else \ - echo "Not building binaries, skipping symlinks" ; \ - fi ; \ - echo ; \ - ) + $(GO_GENERAL_BUILD_CONFIG_VARS) \ + $(GO_PKG_BUILD_CONFIG_VARS) \ + $(SHELL) $(GO_INCLUDE_DIR)/golang-build.sh configure endef # $(1) additional arguments for go command line (optional) define GoPackage/Build/Compile - ( \ - cd "$(GO_PKG_BUILD_DIR)" ; \ - export $(GO_PKG_VARS) ; \ - if [ -f "$(PKG_BUILD_DIR)/go.mod" ] ; then \ - modargs="$(GO_MOD_ARGS)" ; \ - fi ; \ - \ - echo "Finding targets" ; \ - targets="$$$$(go list $$$$modargs $(GO_PKG_BUILD_PKG))" ; \ - for pattern in $(GO_PKG_EXCLUDES); do \ - targets="$$$$(echo "$$$$targets" | grep -v "$$$$pattern")" ; \ - done ; \ - echo ; \ - \ - if [ "$(strip $(GO_PKG_GO_GENERATE))" = 1 ]; then \ - echo "Calling go generate" ; \ - go generate -v $(1) $$$$targets ; \ - echo ; \ - fi ; \ - \ - if [ "$(strip $(GO_PKG_SOURCE_ONLY))" != 1 ]; then \ - echo "Building targets" ; \ - go install $(GO_PKG_INSTALL_ARGS) $$$$modargs $(1) $$$$targets ; \ - retval="$$$$?" ; \ - echo ; \ - \ - if [ "$$$$retval" -eq 0 ] && [ -z "$(call GoPackage/has_binaries)" ]; then \ - echo "No binaries were generated, consider adding GO_PKG_SOURCE_ONLY:=1 to Makefile" ; \ - echo ; \ - fi ; \ - \ - if [ "$$$$retval" -ne 0 ]; then \ - $(call Go/CacheCleanup) ; \ - fi ; \ - fi ; \ - exit "$$$$retval" ; \ - ) + $(GO_GENERAL_BUILD_CONFIG_VARS) \ + $(GO_PKG_BUILD_CONFIG_VARS) \ + $(GO_PKG_VARS) \ + $(SHELL) $(GO_INCLUDE_DIR)/golang-build.sh build $(GO_PKG_INSTALL_ARGS) $(1) endef define GoPackage/Build/InstallDev @@ -340,20 +251,21 @@ define GoPackage/Build/InstallDev endef define GoPackage/Package/Install/Bin - if [ -n "$(call GoPackage/has_binaries)" ]; then \ - $(INSTALL_DIR) "$(1)/usr/bin" ; \ - $(INSTALL_BIN) "$(GO_PKG_BUILD_BIN_DIR)"/* "$(1)/usr/bin/" ; \ - fi + $(GO_GENERAL_BUILD_CONFIG_VARS) \ + $(GO_PKG_BUILD_CONFIG_VARS) \ + $(SHELL) $(GO_INCLUDE_DIR)/golang-build.sh install_bin "$(1)" endef define GoPackage/Package/Install/Src - dir="$$$$(dirname "$(GO_PKG)")" ; \ - $(INSTALL_DIR) "$(1)$(GO_PKG_PATH)/src/$$$$dir" ; \ - $(CP) "$(GO_PKG_BUILD_DIR)/src/$(strip $(GO_PKG))" "$(1)$(GO_PKG_PATH)/src/$$$$dir/" + $(GO_GENERAL_BUILD_CONFIG_VARS) \ + $(GO_PKG_BUILD_CONFIG_VARS) \ + $(SHELL) $(GO_INCLUDE_DIR)/golang-build.sh install_src "$(1)" endef define GoPackage/Package/Install - $(call GoPackage/Package/Install/Bin,$(1)) + $(if $(filter $(GO_PKG_SOURCE_ONLY),1),, \ + $(call GoPackage/Package/Install/Bin,$(1)) \ + ) $(call GoPackage/Package/Install/Src,$(1)) endef @@ -385,7 +297,11 @@ endef # Deprecated variables - these will be removed after the next OpenWrt release +GO_PKG_PATH=$(GO_PKG_BUILD_DEPENDS_PATH) GO_PKG_WORK_DIR=$(PKG_BUILD_DIR)/$(GO_PKG_WORK_DIR_NAME) GO_PKG_CACHE_DIR=$(GO_BUILD_CACHE_DIR) +GO_PKG_DEFAULT_VARS=$(GO_PKG_VARS) GoPackage/Environment=$(GO_PKG_VARS) +GoPackage/is_dir_not_empty=$$$$($(FIND) "$(1)" -maxdepth 0 -type d \! -empty 2>/dev/null) +GoPackage/has_binaries=$(call GoPackage/is_dir_not_empty,$(GO_PKG_BUILD_BIN_DIR)) # End of deprecated variables diff --git a/lang/golang/golang-values.mk b/lang/golang/golang-values.mk index 596270caf..7429ae0de 100644 --- a/lang/golang/golang-values.mk +++ b/lang/golang/golang-values.mk @@ -231,7 +231,9 @@ GO_MOD_ARGS= \ GO_GENERAL_BUILD_CONFIG_VARS= \ CONFIG_GOLANG_MOD_CACHE_WORLD_READABLE="$(CONFIG_GOLANG_MOD_CACHE_WORLD_READABLE)" \ - GO_MOD_CACHE_DIR="$(GO_MOD_CACHE_DIR)" + GO_BUILD_CACHE_DIR="$(GO_BUILD_CACHE_DIR)" \ + GO_MOD_CACHE_DIR="$(GO_MOD_CACHE_DIR)" \ + GO_MOD_ARGS="$(GO_MOD_ARGS)" define Go/CacheCleanup $(GENERAL_BUILD_CONFIG_VARS) \ From 3d6df5d7d99df4180f95523f4cd74a90657a44fe Mon Sep 17 00:00:00 2001 From: Jeffery To Date: Fri, 2 Oct 2020 06:13:28 +0800 Subject: [PATCH 14/15] golang: Add golang-host-build.mk This adds a Makefile for host builds of Go programs. Signed-off-by: Jeffery To --- lang/golang/golang-host-build.mk | 220 +++++++++++++++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 lang/golang/golang-host-build.mk diff --git a/lang/golang/golang-host-build.mk b/lang/golang/golang-host-build.mk new file mode 100644 index 000000000..ee4f1ea31 --- /dev/null +++ b/lang/golang/golang-host-build.mk @@ -0,0 +1,220 @@ +# +# Copyright (C) 2020 Jeffery To +# +# This is free software, licensed under the GNU General Public License v2. +# See /LICENSE for more information. +# + +ifeq ($(origin GO_INCLUDE_DIR),undefined) + GO_INCLUDE_DIR:=$(dir $(lastword $(MAKEFILE_LIST))) +endif + +include $(GO_INCLUDE_DIR)/golang-values.mk + + +# these variables have the same meanings as in golang-package.mk +GO_HOST_INSTALL_EXTRA?=$(GO_PKG_INSTALL_EXTRA) +GO_HOST_INSTALL_ALL?=$(GO_PKG_INSTALL_ALL) +GO_HOST_SOURCE_ONLY?=$(GO_PKG_SOURCE_ONLY) +GO_HOST_BUILD_PKG?=$(GO_PKG_BUILD_PKG) +GO_HOST_EXCLUDES?=$(GO_PKG_EXCLUDES) +GO_HOST_GO_GENERATE?=$(GO_PKG_GO_GENERATE) +GO_HOST_GCFLAGS?=$(GO_PKG_GCFLAGS) +GO_HOST_LDFLAGS?=$(GO_PKG_LDFLAGS) +GO_HOST_LDFLAGS_X?=$(GO_PKG_LDFLAGS_X) +GO_HOST_TAGS?=$(GO_PKG_TAGS) +GO_HOST_INSTALL_BIN_PATH?=/bin + + +# need to repeat this here in case golang-package.mk is not included +GO_PKG_BUILD_PKG?=$(strip $(GO_PKG))/... + +GO_HOST_WORK_DIR_NAME:=.go_work +GO_HOST_BUILD_DIR=$(HOST_BUILD_DIR)/$(GO_HOST_WORK_DIR_NAME)/build +GO_HOST_BUILD_BIN_DIR=$(GO_HOST_BUILD_DIR)/bin + +GO_HOST_BUILD_DEPENDS_PATH:=/share/gocode +GO_HOST_BUILD_DEPENDS_SRC=$(STAGING_DIR_HOSTPKG)$(GO_HOST_BUILD_DEPENDS_PATH)/src + +GO_HOST_DIR_NAME:=$(lastword $(subst /,$(space),$(CURDIR))) +GO_HOST_STAGING_DIR:=$(TMP_DIR)/host-stage-$(GO_HOST_DIR_NAME) +GO_HOST_STAGING_FILES_LIST_DIR:=$(HOST_BUILD_PREFIX)/stamp +GO_HOST_BIN_STAGING_FILES_LIST:=$(GO_HOST_STAGING_FILES_LIST_DIR)/$(GO_HOST_DIR_NAME)-bin.list +GO_HOST_SRC_STAGING_FILES_LIST:=$(GO_HOST_STAGING_FILES_LIST_DIR)/$(GO_HOST_DIR_NAME)-src.list + +ifeq ($(GO_HOST_PIE_SUPPORTED),1) + GO_HOST_ENABLE_PIE:=1 +endif + +GO_HOST_BUILD_CONFIG_VARS= \ + GO_PKG="$(strip $(GO_PKG))" \ + GO_INSTALL_EXTRA="$(strip $(GO_HOST_INSTALL_EXTRA))" \ + GO_INSTALL_ALL="$(strip $(GO_HOST_INSTALL_ALL))" \ + GO_SOURCE_ONLY="$(strip $(GO_HOST_SOURCE_ONLY))" \ + GO_BUILD_PKG="$(strip $(GO_HOST_BUILD_PKG))" \ + GO_EXCLUDES="$(strip $(GO_HOST_EXCLUDES))" \ + GO_GO_GENERATE="$(strip $(GO_HOST_GO_GENERATE))" \ + GO_INSTALL_BIN_PATH="$(strip $(GO_HOST_INSTALL_BIN_PATH))" \ + BUILD_DIR="$(HOST_BUILD_DIR)" \ + GO_BUILD_DIR="$(GO_HOST_BUILD_DIR)" \ + GO_BUILD_BIN_DIR="$(GO_HOST_BUILD_BIN_DIR)" \ + GO_BUILD_DEPENDS_PATH="$(GO_HOST_BUILD_DEPENDS_PATH)" \ + GO_BUILD_DEPENDS_SRC="$(GO_HOST_BUILD_DEPENDS_SRC)" + +GO_HOST_MORE_CFLAGS?= \ + -Wformat -Werror=format-security \ + -fstack-protector-strong \ + -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 \ + -Wl,-z,now -Wl,-z,relro \ + $(if $(GO_HOST_ENABLE_PIE),$(FPIC)) + +GO_HOST_MORE_LDFLAGS?= \ + -znow -zrelro \ + $(if $(GO_HOST_ENABLE_PIE),$(FPIC) -specs=$(INCLUDE_DIR)/hardened-ld-pie.specs) + +GO_HOST_TARGET_VARS= \ + CGO_ENABLED=1 \ + CC=gcc \ + CXX=g++ \ + PKG_CONFIG=pkg-config \ + CGO_CFLAGS="$(HOST_CFLAGS) $(GO_HOST_MORE_CFLAGS)" \ + CGO_CPPFLAGS="$(HOST_CPPFLAGS) $(GO_HOST_MORE_CPPFLAGS)" \ + CGO_CXXFLAGS="$(HOST_CFLAGS) $(GO_HOST_MORE_CFLAGS)" \ + CGO_LDFLAGS="$(HOST_LDFLAGS) $(GO_HOST_MORE_LDFLAGS)" \ + GO_GCC_HELPER_CC="$(HOSTCC)" \ + GO_GCC_HELPER_CXX="$(HOSTCXX)" \ + GO_GCC_HELPER_PATH="$$$$PATH" \ + PATH="$(STAGING_DIR_HOSTPKG)/lib/go-cross/openwrt:$$$$PATH" + +GO_HOST_BUILD_VARS= \ + GOPATH="$(GO_HOST_BUILD_DIR)" \ + GOCACHE="$(GO_BUILD_CACHE_DIR)" \ + GOMODCACHE="$(GO_MOD_CACHE_DIR)" \ + GOENV=off + +GO_HOST_VARS= \ + $(GO_HOST_TARGET_VARS) \ + $(GO_HOST_BUILD_VARS) + +GO_HOST_DEFAULT_LDFLAGS= \ + -linkmode external \ + -extldflags '$(patsubst -z%,-Wl$(comma)-z$(comma)%,$(HOST_LDFLAGS) $(GO_HOST_MORE_LDFLAGS))' + +GO_HOST_CUSTOM_LDFLAGS= \ + $(GO_HOST_LDFLAGS) \ + $(patsubst %,-X %,$(GO_HOST_LDFLAGS_X)) + +GO_HOST_INSTALL_ARGS= \ + -v \ + -ldflags "all=$(GO_HOST_DEFAULT_LDFLAGS)" \ + $(if $(filter $(GO_HOST_ENABLE_PIE),1),-buildmode pie) \ + $(if $(GO_HOST_GCFLAGS),-gcflags "$(GO_HOST_GCFLAGS)") \ + $(if $(GO_HOST_CUSTOM_LDFLAGS),-ldflags "$(GO_HOST_CUSTOM_LDFLAGS) $(GO_HOST_DEFAULT_LDFLAGS)") \ + $(if $(GO_HOST_TAGS),-tags "$(GO_HOST_TAGS)") + +define GoHost/Host/Configure + $(GO_GENERAL_BUILD_CONFIG_VARS) \ + $(GO_HOST_BUILD_CONFIG_VARS) \ + $(SHELL) $(GO_INCLUDE_DIR)/golang-build.sh configure +endef + +# $(1) additional arguments for go command line (optional) +define GoHost/Host/Compile + $(GO_GENERAL_BUILD_CONFIG_VARS) \ + $(GO_HOST_BUILD_CONFIG_VARS) \ + $(GO_HOST_VARS) \ + $(SHELL) $(GO_INCLUDE_DIR)/golang-build.sh build $(GO_HOST_INSTALL_ARGS) $(1) +endef + +define GoHost/Host/Install/Bin + rm -rf "$(GO_HOST_STAGING_DIR)" + mkdir -p "$(GO_HOST_STAGING_DIR)" "$(GO_HOST_STAGING_FILES_LIST_DIR)" + + $(GO_GENERAL_BUILD_CONFIG_VARS) \ + $(GO_HOST_BUILD_CONFIG_VARS) \ + $(SHELL) $(GO_INCLUDE_DIR)/golang-build.sh install_bin "$(GO_HOST_STAGING_DIR)" + + if [ -f "$(GO_HOST_BIN_STAGING_FILES_LIST)" ]; then \ + "$(SCRIPT_DIR)/clean-package.sh" \ + "$(GO_HOST_BIN_STAGING_FILES_LIST)" \ + "$(1)" ; \ + fi + + cd "$(GO_HOST_STAGING_DIR)" && find ./ > "$(GO_HOST_STAGING_DIR).files" + + $(call locked, \ + mv "$(GO_HOST_STAGING_DIR).files" "$(GO_HOST_BIN_STAGING_FILES_LIST)" && \ + $(CP) "$(GO_HOST_STAGING_DIR)"/* "$(1)/", \ + host-staging-dir \ + ) + + rm -rf "$(GO_HOST_STAGING_DIR)" +endef + +define GoHost/Host/Install/Src + rm -rf "$(GO_HOST_STAGING_DIR)" + mkdir -p "$(GO_HOST_STAGING_DIR)" "$(GO_HOST_STAGING_FILES_LIST_DIR)" + + $(GO_GENERAL_BUILD_CONFIG_VARS) \ + $(GO_HOST_BUILD_CONFIG_VARS) \ + $(SHELL) $(GO_INCLUDE_DIR)/golang-build.sh install_src "$(GO_HOST_STAGING_DIR)" + + if [ -f "$(GO_HOST_SRC_STAGING_FILES_LIST)" ]; then \ + "$(SCRIPT_DIR)/clean-package.sh" \ + "$(GO_HOST_SRC_STAGING_FILES_LIST)" \ + "$(1)" ; \ + fi + + cd "$(GO_HOST_STAGING_DIR)" && find ./ > "$(GO_HOST_STAGING_DIR).files" + + $(call locked, \ + mv "$(GO_HOST_STAGING_DIR).files" "$(GO_HOST_SRC_STAGING_FILES_LIST)" && \ + $(CP) "$(GO_HOST_STAGING_DIR)"/* "$(1)/", \ + host-staging-dir \ + ) + + rm -rf "$(GO_HOST_STAGING_DIR)" +endef + +define GoHost/Host/Install + $(if $(filter $(GO_HOST_SOURCE_ONLY),1),, \ + $(call GoHost/Host/Install/Bin,$(1)) \ + ) + $(call GoHost/Host/Install/Src,$(1)) +endef + +define GoHost/Host/Uninstall + if [ -f "$(GO_HOST_BIN_STAGING_FILES_LIST)" ]; then \ + "$(SCRIPT_DIR)/clean-package.sh" \ + "$(GO_HOST_BIN_STAGING_FILES_LIST)" \ + "$(HOST_BUILD_PREFIX)" ; \ + rm -f "$(GO_HOST_BIN_STAGING_FILES_LIST)" ; \ + fi + + if [ -f "$(GO_HOST_SRC_STAGING_FILES_LIST)" ]; then \ + "$(SCRIPT_DIR)/clean-package.sh" \ + "$(GO_HOST_SRC_STAGING_FILES_LIST)" \ + "$(HOST_BUILD_PREFIX)" ; \ + rm -f "$(GO_HOST_SRC_STAGING_FILES_LIST)" ; \ + fi +endef + + +ifneq ($(strip $(GO_PKG)),) + Host/Configure=$(call GoHost/Host/Configure) + Host/Compile=$(call GoHost/Host/Compile) + Hooks/HostCompile/Post+=Go/CacheCleanup + Host/Uninstall=$(call GoHost/Host/Uninstall,$(1)) +endif + +define GoHostBuild + Host/Install=$$(call GoHost/Host/Install,$$(1)) +endef + +define GoBinHostBuild + Host/Install=$$(call GoHost/Host/Install/Bin,$$(1)) +endef + +define GoSrcHostBuild + Host/Install=$$(call GoHost/Host/Install/Src,$$(1)) +endef From d5dc30fe8039e2c969443fc4024f4871b7a9c2f3 Mon Sep 17 00:00:00 2001 From: Jeffery To Date: Tue, 25 Aug 2020 01:01:52 +0800 Subject: [PATCH 15/15] golang: Add option to enable Spectre mitigations Signed-off-by: Jeffery To --- lang/golang/golang-package.mk | 22 +++++++++++++++++++++- lang/golang/golang-values.mk | 13 +++++++++++++ lang/golang/golang/Config.in | 7 +++++++ lang/golang/golang/Makefile | 12 ++++++++++++ 4 files changed, 53 insertions(+), 1 deletion(-) diff --git a/lang/golang/golang-package.mk b/lang/golang/golang-package.mk index 8c62e6399..7144a4524 100644 --- a/lang/golang/golang-package.mk +++ b/lang/golang/golang-package.mk @@ -155,6 +155,14 @@ ifdef CONFIG_PKG_ASLR_PIE_REGULAR endif endif +ifdef CONFIG_GOLANG_SPECTRE + ifeq ($(GO_TARGET_SPECTRE_SUPPORTED),1) + GO_PKG_ENABLE_SPECTRE:=1 + else + $(warning Spectre mitigations are not supported for $(GO_ARCH)) + endif +endif + # sstrip causes corrupted section header size ifneq ($(CONFIG_USE_SSTRIP),) ifneq ($(CONFIG_DEBUG),) @@ -211,6 +219,12 @@ GO_PKG_VARS= \ $(GO_PKG_TARGET_VARS) \ $(GO_PKG_BUILD_VARS) +GO_PKG_DEFAULT_GCFLAGS= \ + $(if $(GO_PKG_ENABLE_SPECTRE),-spectre all) + +GO_PKG_DEFAULT_ASMFLAGS= \ + $(if $(GO_PKG_ENABLE_SPECTRE),-spectre all) + GO_PKG_DEFAULT_LDFLAGS= \ -buildid '$(SOURCE_DATE_EPOCH)' \ -linkmode external \ @@ -224,11 +238,13 @@ GO_PKG_INSTALL_ARGS= \ -v \ -trimpath \ -ldflags "all=$(GO_PKG_DEFAULT_LDFLAGS)" \ + $(if $(GO_PKG_DEFAULT_GCFLAGS),-gcflags "all=$(GO_PKG_DEFAULT_GCFLAGS)") \ + $(if $(GO_PKG_DEFAULT_ASMFLAGS),-asmflags "all=$(GO_PKG_DEFAULT_ASMFLAGS)") \ $(if $(filter $(GO_PKG_ENABLE_PIE),1),-buildmode pie) \ $(if $(filter $(GO_ARCH),arm),-installsuffix "v$(GO_ARM)") \ $(if $(filter $(GO_ARCH),mips mipsle),-installsuffix "$(GO_MIPS)") \ $(if $(filter $(GO_ARCH),mips64 mips64le),-installsuffix "$(GO_MIPS64)") \ - $(if $(GO_PKG_GCFLAGS),-gcflags "$(GO_PKG_GCFLAGS)") \ + $(if $(GO_PKG_GCFLAGS),-gcflags "$(GO_PKG_GCFLAGS) $(GO_PKG_DEFAULT_GCFLAGS)") \ $(if $(GO_PKG_CUSTOM_LDFLAGS),-ldflags "$(GO_PKG_CUSTOM_LDFLAGS) $(GO_PKG_DEFAULT_LDFLAGS)") \ $(if $(GO_PKG_TAGS),-tags "$(GO_PKG_TAGS)") @@ -271,6 +287,10 @@ endef ifneq ($(strip $(GO_PKG)),) + ifeq ($(GO_TARGET_SPECTRE_SUPPORTED),1) + PKG_CONFIG_DEPENDS+=CONFIG_GOLANG_SPECTRE + endif + Build/Configure=$(call GoPackage/Build/Configure) Build/Compile=$(call GoPackage/Build/Compile) Hooks/Compile/Post+=Go/CacheCleanup diff --git a/lang/golang/golang-values.mk b/lang/golang/golang-values.mk index 7429ae0de..69126a1ed 100644 --- a/lang/golang/golang-values.mk +++ b/lang/golang/golang-values.mk @@ -221,6 +221,19 @@ ifneq ($(filter $(GO_OS_ARCH),$(GO_PIE_SUPPORTED_OS_ARCH)),) endif +# Spectre mitigations + +GO_SPECTRE_SUPPORTED_ARCH:=amd64 + +ifneq ($(filter $(GO_HOST_ARCH),$(GO_SPECTRE_SUPPORTED_ARCH)),) + GO_HOST_SPECTRE_SUPPORTED:=1 +endif + +ifneq ($(filter $(GO_ARCH),$(GO_SPECTRE_SUPPORTED_ARCH)),) + GO_TARGET_SPECTRE_SUPPORTED:=1 +endif + + # General build info GO_BUILD_CACHE_DIR:=$(or $(call qstrip,$(CONFIG_GOLANG_BUILD_CACHE_DIR)),$(TOPDIR)/.go-build) diff --git a/lang/golang/golang/Config.in b/lang/golang/golang/Config.in index 2e7159173..714bf01fd 100644 --- a/lang/golang/golang/Config.in +++ b/lang/golang/golang/Config.in @@ -23,4 +23,11 @@ config GOLANG_MOD_CACHE_WORLD_READABLE bool "Ensure Go module cache is world-readable" default n +config GOLANG_SPECTRE + bool "Enable Spectre mitigations" + default n + depends on x86_64 + help + Currently only available for x86-64 (amd64). + endmenu diff --git a/lang/golang/golang/Makefile b/lang/golang/golang/Makefile index a5474753b..2016dfb8c 100644 --- a/lang/golang/golang/Makefile +++ b/lang/golang/golang/Makefile @@ -96,6 +96,10 @@ BOOTSTRAP_UNPACK:=$(HOST_TAR) -C "$(BOOTSTRAP_BUILD_DIR)" --strip-components=1 - RSTRIP:=: STRIP:=: +ifeq ($(GO_TARGET_SPECTRE_SUPPORTED),1) + PKG_CONFIG_DEPENDS+=CONFIG_GOLANG_SPECTRE +endif + define Package/golang/Default $(call GoPackage/GoSubMenu) TITLE:=Go programming language @@ -269,6 +273,12 @@ PKG_GO_VARS= \ PKG_CONFIG=pkg-config \ PATH="$(HOST_GO_ROOT)/openwrt:$$$$PATH" +PKG_GO_GCFLAGS= \ + $(if $(GO_PKG_ENABLE_SPECTRE),-spectre all) + +PKG_GO_ASMFLAGS= \ + $(if $(GO_PKG_ENABLE_SPECTRE),-spectre all) + PKG_GO_LDFLAGS= \ -buildid '$(SOURCE_DATE_EPOCH)' \ -linkmode external \ @@ -280,6 +290,8 @@ PKG_GO_LDFLAGS= \ # static / not dependent on the build environment PKG_GO_INSTALL_ARGS= \ -ldflags "all=$(PKG_GO_LDFLAGS)" \ + $(if $(PKG_GO_GCFLAGS),-gcflags "all=$(PKG_GO_GCFLAGS)") \ + $(if $(PKG_GO_ASMFLAGS),-asmflags "all=$(PKG_GO_ASMFLAGS)") \ $(if $(filter $(GO_PKG_ENABLE_PIE),1),-buildmode pie) define Build/Compile