fe78c07a31
Using pip to install host packages with pyproject.toml-based (PEP 517) builds is problematic: * If build isolation is used, pip will create an isolated build environment, install any build dependencies for the requested package, then build the requested package. It does not appear currently possible to have pip install the build dependencies with hash-checking mode enabled[1]. * If build isolation is not used, any build dependencies must be installed in the build environment before invoking pip to build the requested package[2]. This would require creating a package dependency resolution system to install build dependencies, and any dependencies of dependencies, in the correct order. * It is very difficult to patch the packages installed by pip. This adds a new include file (python3-host-build.mk) with recipes to install host Python packages with pyproject.toml-based builds. This is backwards-compatible with packages that require running setup.py. Besides addressing the above issues (the OpenWrt build system already resolves dependencies between packages, checks all source downloads against known hashes, and supports patching packages), host packages also: * Capture package licensing and maintainer information * Enable uscan checking for package updates/CVEs * Are a known concept for OpenWrt packagers/developers The existing functionality of using host pip to install packages will remain for now, but should be considered deprecated and expected to be removed in the future. This also updates Py3Build/CheckHostPipVersionMatch for the case where the host-pip-requirements directory does not exist or is empty. [1]: https://pip.pypa.io/en/stable/user_guide/#changes-to-the-pip-dependency-resolver-in-20-3-2020 [2]: https://pip.pypa.io/en/stable/cli/pip_install/#cmdoption-no-build-isolation Signed-off-by: Jeffery To <jeffery.to@gmail.com>
120 lines
3.3 KiB
Makefile
120 lines
3.3 KiB
Makefile
#
|
|
# Copyright (C) 2017 OpenWrt.org
|
|
#
|
|
# This is free software, licensed under the GNU General Public License v2.
|
|
# See /LICENSE for more information.
|
|
#
|
|
|
|
# Note: include this file after `include $(TOPDIR)/rules.mk in your package Makefile
|
|
# if `python3-package.mk` is included, this will already be included
|
|
|
|
# For PYTHON3_VERSION
|
|
python3_mk_path:=$(dir $(lastword $(MAKEFILE_LIST)))
|
|
include $(python3_mk_path)python3-version.mk
|
|
|
|
# Unset environment variables
|
|
|
|
# https://docs.python.org/3/using/cmdline.html#environment-variables
|
|
unexport \
|
|
PYTHONHOME \
|
|
PYTHONPATH \
|
|
PYTHONSAFEPATH \
|
|
PYTHONPLATLIBDIR \
|
|
PYTHONSTARTUP \
|
|
PYTHONOPTIMIZE \
|
|
PYTHONBREAKPOINT \
|
|
PYTHONDEBUG \
|
|
PYTHONINSPECT \
|
|
PYTHONUNBUFFERED \
|
|
PYTHONVERBOSE \
|
|
PYTHONCASEOK \
|
|
PYTHONDONTWRITEBYTECODE \
|
|
PYTHONPYCACHEPREFIX \
|
|
PYTHONHASHSEED \
|
|
PYTHONINTMAXSTRDIGITS \
|
|
PYTHONIOENCODING \
|
|
PYTHONNOUSERSITE \
|
|
PYTHONUSERBASE \
|
|
PYTHONEXECUTABLE \
|
|
PYTHONWARNINGS \
|
|
PYTHONFAULTHANDLER \
|
|
PYTHONTRACEMALLOC \
|
|
PYTHONPROFILEIMPORTTIME \
|
|
PYTHONASYNCIODEBUG \
|
|
PYTHONMALLOC \
|
|
PYTHONMALLOCSTATS \
|
|
PYTHONLEGACYWINDOWSFSENCODING \
|
|
PYTHONLEGACYWINDOWSSTDIO \
|
|
PYTHONCOERCECLOCALE \
|
|
PYTHONDEVMODE \
|
|
PYTHONUTF8 \
|
|
PYTHONWARNDEFAULTENCODING \
|
|
PYTHONNODEBUGRANGES
|
|
|
|
# https://docs.python.org/3/using/cmdline.html#debug-mode-variables
|
|
unexport \
|
|
PYTHONTHREADDEBUG \
|
|
PYTHONDUMPREFS \
|
|
PYTHONDUMPREFSFILE
|
|
|
|
HOST_PYTHON3_DIR:=$(STAGING_DIR_HOSTPKG)
|
|
HOST_PYTHON3_INC_DIR:=$(HOST_PYTHON3_DIR)/include/python$(PYTHON3_VERSION)
|
|
HOST_PYTHON3_LIB_DIR:=$(HOST_PYTHON3_DIR)/lib/python$(PYTHON3_VERSION)
|
|
|
|
HOST_PYTHON3_PKG_DIR:=$(HOST_PYTHON3_DIR)/lib/python$(PYTHON3_VERSION)/site-packages
|
|
|
|
HOST_PYTHON3_BIN:=$(HOST_PYTHON3_DIR)/bin/python$(PYTHON3_VERSION)
|
|
|
|
HOST_PYTHON3PATH:=$(HOST_PYTHON3_LIB_DIR):$(HOST_PYTHON3_PKG_DIR)
|
|
|
|
HOST_PYTHON3_VARS = \
|
|
ARCH="$(HOST_ARCH)" \
|
|
CC="$(HOSTCC)" \
|
|
CCSHARED="$(HOSTCC) $(HOST_FPIC)" \
|
|
CXX="$(HOSTCXX)" \
|
|
LD="$(HOSTCC)" \
|
|
LDSHARED="$(HOSTCC) -shared" \
|
|
CFLAGS="$(HOST_CFLAGS)" \
|
|
CPPFLAGS="$(HOST_CPPFLAGS) -I$(HOST_PYTHON3_INC_DIR)" \
|
|
LDFLAGS="$(HOST_LDFLAGS) -lpython$(PYTHON3_VERSION) -Wl$(comma)-rpath$(comma)$(STAGING_DIR_HOSTPKG)/lib"
|
|
|
|
# $(1) => directory of python script
|
|
# $(2) => python script and its arguments
|
|
# $(3) => additional variables
|
|
define HostPython3/Run
|
|
cd "$(if $(strip $(1)),$(strip $(1)),.)" && \
|
|
$(HOST_PYTHON3_VARS) \
|
|
$(3) \
|
|
$(HOST_PYTHON3_BIN) $(2)
|
|
endef
|
|
|
|
# Note: I shamelessly copied this from Yousong's logic (from python-packages);
|
|
HOST_PYTHON3_PIP:=$(STAGING_DIR_HOSTPKG)/bin/pip$(PYTHON3_VERSION)
|
|
|
|
HOST_PYTHON3_PIP_CACHE_DIR:=$(DL_DIR)/pip-cache
|
|
|
|
define SetupPyShim
|
|
if [ -f $(1)/pyproject.toml ] && [ ! -f $(1)/setup.py ] ; then \
|
|
$(CP) $(python3_mk_path)setup.py.shim $(1)setup.py ; \
|
|
fi
|
|
endef
|
|
|
|
# Multiple concurrent pip processes can lead to errors or unexpected results: https://github.com/pypa/pip/issues/2361
|
|
# $(1) => packages to install
|
|
define HostPython3/PipInstall
|
|
$(call locked, \
|
|
$(HOST_PYTHON3_VARS) \
|
|
$(HOST_PYTHON3_PIP) \
|
|
--cache-dir "$(HOST_PYTHON3_PIP_CACHE_DIR)" \
|
|
--disable-pip-version-check \
|
|
install \
|
|
--no-binary :all: \
|
|
--require-hashes \
|
|
$(1) \
|
|
$(if $(CONFIG_PYTHON3_HOST_PIP_CACHE_WORLD_READABLE), \
|
|
&& $(FIND) $(HOST_PYTHON3_PIP_CACHE_DIR) -not -type d -exec chmod go+r '{}' \; \
|
|
&& $(FIND) $(HOST_PYTHON3_PIP_CACHE_DIR) -type d -exec chmod go+rx '{}' \; \
|
|
), \
|
|
pip \
|
|
)
|
|
endef
|