Difuse/packages
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #22339 from jefferyto/rust-build-performance

Browse source 
rust: Improve build performance
This commit is contained in:
Tianling Shen 2023-10-11 21:03:33 +08:00 committed by GitHub
commit 175271940e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 212 additions and 76 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
admin/bottom/Makefile
View file

@ -17,6 +17,7 @@ PKG_LICENSE:=MIT
PKG_LICENSE_FILES:=LICENSE
PKG_BUILD_DEPENDS:=rust/host
PKG_BUILD_PARALLEL:=1
include $(INCLUDE_DIR)/package.mk
include ../../lang/rust/rust-package.mk

1
lang/maturin/Makefile
View file

@ -17,6 +17,7 @@ PKG_LICENSE:=Apache-2.0 MIT
PKG_LICENSE_FILES:=license-apache license-mit
HOST_BUILD_DEPENDS:=rust/host
HOST_BUILD_PARALLEL:=1
PKG_HOST_ONLY:=1
include $(INCLUDE_DIR)/host-build.mk

2
lang/python/python-setuptools-rust/Makefile
View file

@ -9,7 +9,7 @@ include $(TOPDIR)/rules.mk
PKG_NAME:=python-setuptools-rust
PKG_VERSION:=1.7.0
PKG_RELEASE:=1
PKG_RELEASE:=2
PYPI_NAME:=setuptools-rust
PKG_HASH:=c7100999948235a38ae7e555fe199aa66c253dc384b125f5d85473bf81eae3a3

48
lang/python/python-setuptools-rust/patches/0001-Allow-profile-to-be-set-by-SETUPTOOLS_RUST_CARGO_PROFILE-env-variable.patch Normal file
View file

@ -0,0 +1,48 @@
From b10cab4efeb80abb5a236d651c9ff9355e470527 Mon Sep 17 00:00:00 2001
From: Jeffery To <jeffery.to@gmail.com>
Date: Mon, 2 Oct 2023 16:13:51 +0800
Subject: [PATCH] Allow profile to be set by SETUPTOOLS_RUST_CARGO_PROFILE env
variable
This allows the profile to be set dynamically, without having to edit
pyproject.toml/setup.py.
---
setuptools_rust/build.py | 20 ++++++++++++++++----
1 file changed, 16 insertions(+), 4 deletions(-)
--- a/setuptools_rust/build.py
+++ b/setuptools_rust/build.py
@@ -528,10 +528,10 @@ class build_rust(RustCommand):
if target_triple is not None:
args.extend(["--target", target_triple])
- if release:
- profile = ext.get_cargo_profile()
- if not profile:
- args.append("--release")
+ ext_profile = ext.get_cargo_profile()
+ env_profile = os.getenv("SETUPTOOLS_RUST_CARGO_PROFILE")
+ if release and not ext_profile and not env_profile:
+ args.append("--release")
if quiet:
args.append("-q")
@@ -552,6 +552,18 @@ class build_rust(RustCommand):
if ext.args is not None:
args.extend(ext.args)
+ if env_profile:
+ if ext_profile:
+ args = [p for p in args if not p.startswith("--profile=")]
+ while True:
+ try:
+ index = args.index("--profile")
+ del args[index:index + 2]
+ except ValueError:
+ break
+
+ args.extend(["--profile", env_profile])
+
if ext.cargo_manifest_args is not None:
args.extend(ext.cargo_manifest_args)

4
lang/python/python3-host.mk
View file

@ -78,8 +78,8 @@ HOST_PYTHON3_VARS = \
CFLAGS="$(HOST_CFLAGS)" \
CPPFLAGS="$(HOST_CPPFLAGS) -I$(HOST_PYTHON3_INC_DIR)" \
LDFLAGS="$(HOST_LDFLAGS) -lpython$(PYTHON3_VERSION) -Wl$(comma)-rpath$(comma)$(STAGING_DIR_HOSTPKG)/lib" \
CARGO_HOME="$(CARGO_HOME)" \
PATH="$(CARGO_HOME)/bin:$(PATH)"
$(CARGO_HOST_CONFIG_VARS) \
SETUPTOOLS_RUST_CARGO_PROFILE="$(CARGO_HOST_PROFILE)"
# $(1) => directory of python script
# $(2) => python script and its arguments

6
lang/python/python3-package.mk
View file

@ -45,11 +45,9 @@ PYTHON3_VARS = \
_python_sysroot="$(STAGING_DIR)" \
_python_prefix="/usr" \
_python_exec_prefix="/usr" \
CARGO_BUILD_TARGET="$(RUSTC_TARGET_ARCH)" \
CARGO_HOME="$(CARGO_HOME)" \
PATH="$(CARGO_HOME)/bin:$(PATH)" \
$(CARGO_PKG_CONFIG_VARS) \
PYO3_CROSS_LIB_DIR="$(PYTHON3_LIB_DIR)" \
RUSTFLAGS="$(CARGO_RUSTFLAGS)"
SETUPTOOLS_RUST_CARGO_PROFILE="$(CARGO_PKG_PROFILE)"
# $(1) => directory of python script
# $(2) => python script and its arguments

15
lang/rust/Config.in Normal file
View file

@ -0,0 +1,15 @@
menu "Configuration options (for developers)"
config RUST_SCCACHE
bool "Use sccache"
help
Shared compilation cache; see https://github.com/mozilla/sccache
config RUST_SCCACHE_DIR
string "Set sccache directory" if RUST_SCCACHE
default ""
help
Store sccache in this directory.
If not set, uses './.sccache'
endmenu

49
lang/rust/Makefile
View file

@ -6,7 +6,7 @@ include $(TOPDIR)/rules.mk
PKG_NAME:=rust
PKG_VERSION:=1.72.0
PKG_RELEASE:=2
PKG_RELEASE:=3
PKG_SOURCE:=rustc-$(PKG_VERSION)-src.tar.gz
PKG_SOURCE_URL:=https://static.rust-lang.org/dist/
@ -17,7 +17,6 @@ PKG_MAINTAINER:=Luca Barbato <lu_zero@luminem.org>
PKG_LICENSE:=Apache-2.0 MIT
PKG_LICENSE_FILES:=LICENSE-APACHE LICENSE-MIT
HOST_BUILD_DEPENDS:=python3/host
PKG_HOST_ONLY:=1
include $(INCLUDE_DIR)/host-build.mk
@ -31,7 +30,6 @@ define Package/rust
TITLE:=Rust Programming Language Compiler
URL:=https://www.rust-lang.org/
DEPENDS:=$(RUST_ARCH_DEPENDS)
BUILDONLY:=1
endef
define Package/rust/description
@ -40,8 +38,12 @@ define Package/rust/description
guarantee memory safety by using a borrow checker to validate references.
endef
define Package/rust/config
source "$(SOURCE)/Config.in"
endef
# Rust-lang has an uninstall script
RUST_UNINSTALL:=$(CARGO_HOME)/lib/rustlib/uninstall.sh
RUST_UNINSTALL:=$(STAGING_DIR)/host/lib/rustlib/uninstall.sh
# Target Flags
TARGET_CONFIGURE_ARGS = \
@ -53,20 +55,20 @@ TARGET_CONFIGURE_ARGS = \
$(if $(CONFIG_USE_MUSL),--set=target.$(RUSTC_TARGET_ARCH).musl-root=$(TOOLCHAIN_DIR))
# CARGO_HOME is an environmental
HOST_CONFIGURE_OPTS += CARGO_HOME="$(CARGO_HOME)"
HOST_CONFIGURE_VARS += CARGO_HOME="$(CARGO_HOME)"
# Rust Configuration Arguments
HOST_CONFIGURE_ARGS = \
--build=$(RUSTC_HOST_ARCH) \
--target=$(RUSTC_TARGET_ARCH),$(RUSTC_HOST_ARCH) \
--host=$(RUSTC_HOST_ARCH) \
--prefix=$(CARGO_HOME) \
--bindir=$(CARGO_HOME)/bin \
--libdir=$(CARGO_HOME)/lib \
--sysconfdir=$(CARGO_HOME)/etc \
--datadir=$(CARGO_HOME)/share \
--mandir=$(CARGO_HOME)/man \
--dist-compression-formats=xz \
--prefix=$(STAGING_DIR)/host \
--bindir=$(STAGING_DIR)/host/bin \
--libdir=$(STAGING_DIR)/host/lib \
--sysconfdir=$(STAGING_DIR)/host/etc \
--datadir=$(STAGING_DIR)/host/share \
--mandir=$(STAGING_DIR)/host/man \
--dist-compression-formats=gz \
--enable-missing-tools \
--disable-sanitizers \
--release-channel=stable \
@ -81,22 +83,23 @@ define Host/Uninstall
endef
define Host/Compile
( \
cd $(HOST_BUILD_DIR) ; \
$(PYTHON) x.py --config ./config.toml dist build-manifest cargo llvm-tools \
rustc rust-std rust-src ; \
)
$(RUST_SCCACHE_VARS) \
CARGO_HOME=$(CARGO_HOME) \
OPENWRT_RUSTC_BOOTSTRAP_CACHE=$(DL_DIR)/rustc \
$(PYTHON) $(HOST_BUILD_DIR)/x.py \
--build-dir $(HOST_BUILD_DIR)/build \
--config $(HOST_BUILD_DIR)/config.toml \
dist build-manifest cargo llvm-tools rustc rust-std rust-src
endef
define Host/Install
( \
cd $(HOST_BUILD_DIR)/build/dist ; \
find -iname "*.xz" -exec tar -xJf {} \; ; \
find ./* -type f -name install.sh -execdir sh {} --prefix=$(CARGO_HOME) --disable-ldconfig \; ; \
\
sed -e 's|@RUSTC_TARGET_ARCH@|$(RUSTC_TARGET_ARCH)|g' \
-e 's|@TARGET_CC_NOCACHE@|$(TARGET_CC_NOCACHE)|g' \
$(CURDIR)/files/cargo-config > $(CARGO_HOME)/config.toml ; \
for targz in *.tar.gz; do \
$(STAGING_DIR_HOST)/bin/libdeflate-gzip -dc "$$$$targz" | tar -xf - ; \
done ; \
find . -mindepth 2 -maxdepth 2 -type f -name install.sh \
-execdir bash '{}' --prefix=$(STAGING_DIR)/host --disable-ldconfig \; ; \
)
endef

7
lang/rust/files/cargo-config
View file

@ -1,7 +0,0 @@
[target.@RUSTC_TARGET_ARCH@]
linker = "@TARGET_CC_NOCACHE@"
[profile.stripped]
inherits = "release"
opt-level = "s"
strip = true

37
lang/rust/patches/0002-rustc-bootstrap-cache.patch Normal file
View file

@ -0,0 +1,37 @@
--- a/src/bootstrap/bootstrap.py
+++ b/src/bootstrap/bootstrap.py
@@ -543,7 +543,7 @@ class RustBuild(object):
shutil.rmtree(bin_root)
key = self.stage0_compiler.date
- cache_dst = os.path.join(self.build_dir, "cache")
+ cache_dst = os.getenv('OPENWRT_RUSTC_BOOTSTRAP_CACHE', os.path.join(self.build_dir, "cache"))
rustc_cache = os.path.join(cache_dst, key)
if not os.path.exists(rustc_cache):
os.makedirs(rustc_cache)
--- a/src/bootstrap/download.rs
+++ b/src/bootstrap/download.rs
@@ -507,7 +507,10 @@ impl Config {
key: &str,
destination: &str,
) {
- let cache_dst = self.out.join("cache");
+ let cache_dst = match env::var_os("OPENWRT_RUSTC_BOOTSTRAP_CACHE") {
+ Some(v) => PathBuf::from(v),
+ None => self.out.join("cache"),
+ };
let cache_dir = cache_dst.join(key);
if !cache_dir.exists() {
t!(fs::create_dir_all(&cache_dir));
@@ -627,7 +630,10 @@ download-rustc = false
let llvm_assertions = self.llvm_assertions;
let cache_prefix = format!("llvm-{}-{}", llvm_sha, llvm_assertions);
- let cache_dst = self.out.join("cache");
+ let cache_dst = match env::var_os("OPENWRT_RUSTC_BOOTSTRAP_CACHE") {
+ Some(v) => PathBuf::from(v),
+ None => self.out.join("cache"),
+ };
let rustc_cache = cache_dst.join(cache_prefix);
if !rustc_cache.exists() {
t!(fs::create_dir_all(&rustc_cache));

44
lang/rust/rust-host-build.mk
View file

@ -2,36 +2,42 @@
#
# Copyright (C) 2023 Luca Barbato and Donald Hoskins
# Variables (all optional) to be set in package Makefiles:
#
# RUST_HOST_FEATURES - list of options, default empty
#
# Space or comma separated list of features to activate
#
# e.g. RUST_HOST_FEATURES:=enable-foo,with-bar
ifeq ($(origin RUST_INCLUDE_DIR),undefined)
RUST_INCLUDE_DIR:=$(dir $(lastword $(MAKEFILE_LIST)))
endif
include $(RUST_INCLUDE_DIR)/rust-values.mk
CARGO_HOST_VARS= \
$(CARGO_HOST_CONFIG_VARS) \
CC=$(HOSTCC_NOCACHE) \
MAKEFLAGS="$(HOST_JOBS)"
# $(1) path to the package (optional)
# $(2) additional arguments to cargo (optional)
define Host/Compile/Cargo
( \
cd $(HOST_BUILD_DIR) ; \
export PATH="$(CARGO_HOME)/bin:$(PATH)" ; \
CARGO_HOME=$(CARGO_HOME) \
CC=$(HOSTCC_NOCACHE) \
cargo install -v \
--profile stripped \
$(if $(RUST_PKG_FEATURES),--features "$(RUST_PKG_FEATURES)") \
--root $(HOST_INSTALL_DIR) \
--path "$(if $(strip $(1)),$(strip $(1)),.)" $(2) ; \
)
+$(CARGO_HOST_VARS) \
cargo install -v \
--profile $(CARGO_HOST_PROFILE) \
$(if $(RUST_HOST_FEATURES),--features "$(RUST_HOST_FEATURES)") \
--root $(HOST_INSTALL_DIR) \
--path "$(HOST_BUILD_DIR)/$(if $(strip $(1)),$(strip $(1)))" \
$(if $(filter --jobserver%,$(HOST_JOBS)),,-j1) \
$(2)
endef
define Host/Uninstall/Cargo
( \
cd $(HOST_BUILD_DIR) ; \
export PATH="$(CARGO_HOME)/bin:$(PATH)" ; \
CARGO_HOME=$(CARGO_HOME) \
CC=$(HOSTCC_NOCACHE) \
cargo uninstall -v \
--root $(HOST_INSTALL_DIR) || true ; \
)
+$(CARGO_HOST_VARS) \
cargo uninstall -v \
--root $(HOST_INSTALL_DIR) \
|| true
endef
define RustBinHostBuild

30
lang/rust/rust-package.mk
View file

@ -15,26 +15,22 @@ ifeq ($(origin RUST_INCLUDE_DIR),undefined)
endif
include $(RUST_INCLUDE_DIR)/rust-values.mk
CARGO_PKG_VARS= \
$(CARGO_PKG_CONFIG_VARS) \
CC=$(HOSTCC_NOCACHE) \
MAKEFLAGS="$(PKG_JOBS)"
# $(1) path to the package (optional)
# $(2) additional arguments to cargo (optional)
define Build/Compile/Cargo
( \
cd $(PKG_BUILD_DIR) ; \
export PATH="$(CARGO_HOME)/bin:$(PATH)" ; \
CARGO_HOME=$(CARGO_HOME) \
TARGET_CFLAGS="$(TARGET_CFLAGS) $(RUSTC_CFLAGS)" \
TARGET_CC=$(TARGET_CC_NOCACHE) \
CC=$(HOSTCC_NOCACHE) \
RUSTFLAGS="$(CARGO_RUSTFLAGS)" \
$(CARGO_VARS) \
cargo install -v \
--profile stripped \
--target $(RUSTC_TARGET_ARCH) \
$(if $(strip $(RUST_PKG_FEATURES)),--features "$(strip $(RUST_PKG_FEATURES))") \
--root $(PKG_INSTALL_DIR) \
--path "$(if $(strip $(1)),$(strip $(1)),.)" \
$(2) ; \
)
+$(CARGO_PKG_VARS) \
cargo install -v \
--profile $(CARGO_PKG_PROFILE) \
$(if $(strip $(RUST_PKG_FEATURES)),--features "$(strip $(RUST_PKG_FEATURES))") \
--root $(PKG_INSTALL_DIR) \
--path "$(PKG_BUILD_DIR)/$(if $(strip $(1)),$(strip $(1)))" \
$(if $(filter --jobserver%,$(PKG_JOBS)),,-j1) \
$(2)
endef
define RustBinPackage

37
lang/rust/rust-values.mk
View file

@ -5,8 +5,7 @@
# Rust Environmental Vars
RUSTC_HOST_SUFFIX:=$(word 4, $(subst -, ,$(GNU_HOST_NAME)))
RUSTC_HOST_ARCH:=$(HOST_ARCH)-unknown-linux-$(RUSTC_HOST_SUFFIX)
CARGO_HOME:=$(STAGING_DIR)/host/cargo
CARGO_VARS?=
CARGO_HOME:=$(DL_DIR)/cargo
ifeq ($(CONFIG_USE_MUSL),y)
# Force linking of the SSP library for musl
@ -62,3 +61,37 @@ endif
# Support only a subset for now.
RUST_ARCH_DEPENDS:=@(aarch64||arm||i386||i686||mips||mipsel||mips64||mips64el||mipsel||powerpc64||riscv64||x86_64)
ifneq ($(CONFIG_RUST_SCCACHE),)
RUST_SCCACHE_DIR:=$(if $(call qstrip,$(CONFIG_RUST_SCCACHE_DIR)),$(call qstrip,$(CONFIG_RUST_SCCACHE_DIR)),$(TOPDIR)/.sccache)
RUST_SCCACHE_VARS:= \
CARGO_INCREMENTAL=0 \
RUSTC_WRAPPER=sccache \
SCCACHE_DIR=$(RUST_SCCACHE_DIR)
endif
CARGO_HOST_CONFIG_VARS= \
$(RUST_SCCACHE_VARS) \
CARGO_HOME=$(CARGO_HOME)
CARGO_HOST_PROFILE:=release
CARGO_PKG_CONFIG_VARS= \
$(RUST_SCCACHE_VARS) \
CARGO_BUILD_TARGET=$(RUSTC_TARGET_ARCH) \
CARGO_HOME=$(CARGO_HOME) \
CARGO_PROFILE_RELEASE_CODEGEN_UNITS=1 \
CARGO_PROFILE_RELEASE_DEBUG=false \
CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS=false \
CARGO_PROFILE_RELEASE_LTO=true \
CARGO_PROFILE_RELEASE_OPT_LEVEL=z \
CARGO_PROFILE_RELEASE_OVERFLOW_CHECKS=true \
CARGO_PROFILE_RELEASE_PANIC=unwind \
CARGO_PROFILE_RELEASE_RPATH=false \
CARGO_TARGET_$(subst -,_,$(call toupper,$(RUSTC_TARGET_ARCH)))_LINKER=$(TARGET_CC_NOCACHE) \
RUSTFLAGS="$(CARGO_RUSTFLAGS)" \
TARGET_CC=$(TARGET_CC_NOCACHE) \
TARGET_CFLAGS="$(TARGET_CFLAGS) $(RUSTC_CFLAGS)"
CARGO_PKG_PROFILE:=$(if $(CONFIG_DEBUG),dev,release)

1
net/aardvark-dns/Makefile
View file

@ -13,6 +13,7 @@ PKG_LICENSE:=Apache-2.0
PKG_LICENSE_FILES:=LICENSE
PKG_BUILD_DEPENDS:=rust/host
PKG_BUILD_PARALLEL:=1
include $(INCLUDE_DIR)/package.mk
include ../../lang/rust/rust-package.mk

3
net/netavark/Makefile
View file

@ -15,6 +15,7 @@ PKG_LICENSE_FILES:=LICENSE
PKG_BUILD_DEPENDS:= \
rust/host \
protobuf/host
PKG_BUILD_PARALLEL:=1
include $(INCLUDE_DIR)/package.mk
include ../../lang/rust/rust-package.mk
@ -36,7 +37,7 @@ define Package/netavark/conffiles
/etc/config/netavark
endef
CARGO_VARS += \
CARGO_PKG_VARS += \
PROTOC=$(STAGING_DIR_HOSTPKG)/bin/protoc
define Package/netavark/install

1
utils/arp-whisper/Makefile
View file

@ -17,6 +17,7 @@ PKG_LICENSE:=GPL-3.0-or-later
PKG_LICENSE_FILES:=LICENCE
PKG_BUILD_DEPENDS:=rust/host
PKG_BUILD_PARALLEL:=1
include $(INCLUDE_DIR)/package.mk
include ../../lang/rust/rust-package.mk

1
utils/procs/Makefile
View file

@ -17,6 +17,7 @@ PKG_LICENSE:=MIT
PKG_LICENSE_FILES:=LICENCE
PKG_BUILD_DEPENDS:=rust/host
PKG_BUILD_PARALLEL:=1
include $(INCLUDE_DIR)/package.mk
include ../../lang/rust/rust-package.mk

1
utils/ripgrep/Makefile
View file

@ -17,6 +17,7 @@ PKG_LICENSE:=MIT Unlicense
PKG_LICENSE_FILES:=LICENSE-MIT UNLICENSE
PKG_BUILD_DEPENDS:=rust/host
PKG_BUILD_PARALLEL:=1
RUST_PKG_FEATURES:=pcre2