From fbba266e793941a89f9964f7171fdbdc82e9fd61 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Mon, 17 Oct 2016 18:05:50 +0800 Subject: [PATCH 1/3] python: introduce Build/Compile/HostPyRun{Host,Target} Signed-off-by: Yousong Zhou --- lang/python/files/python-host.mk | 21 ++++++++++++++++----- lang/python/files/python-package.mk | 22 ++++++++++++++++------ 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/lang/python/files/python-host.mk b/lang/python/files/python-host.mk index 8736d29ad..6f7977310 100644 --- a/lang/python/files/python-host.mk +++ b/lang/python/files/python-host.mk @@ -31,12 +31,12 @@ define HostPython $(HOST_PYTHON_BIN) $(2); endef -# $(1) => build subdir -# $(2) => additional arguments to setup.py +# $(1) => commands to execute before running pythons script +# $(2) => python script and its arguments # $(3) => additional variables -define Build/Compile/HostPyMod +define Build/Compile/HostPyRunHost $(call HostPython, \ - cd $(HOST_BUILD_DIR)/$(strip $(1)); \ + $(if $(1),$(1);) \ CC="$(HOSTCC)" \ CCSHARED="$(HOSTCC) $(HOST_FPIC)" \ CXX="$(HOSTCXX)" \ @@ -48,9 +48,20 @@ define Build/Compile/HostPyMod _PYTHON_HOST_PLATFORM=linux2 \ $(3) \ , \ - ./setup.py $(2) \ + $(2) \ , \ HOST \ ) endef + +# $(1) => build subdir +# $(2) => additional arguments to setup.py +# $(3) => additional variables +define Build/Compile/HostPyMod + $(call Build/Compile/HostPyRunHost, \ + cd $(HOST_BUILD_DIR)/$(strip $(1)), \ + ./setup.py $(2), \ + $(3)) +endef + diff --git a/lang/python/files/python-package.mk b/lang/python/files/python-package.mk index 1f824cba6..52b6cf37e 100644 --- a/lang/python/files/python-package.mk +++ b/lang/python/files/python-package.mk @@ -82,13 +82,12 @@ endef $(call include_mk, python-host.mk) -# $(1) => build subdir -# $(2) => additional arguments to setup.py +# $(1) => commands to execute before running pythons script +# $(2) => python script and its arguments # $(3) => additional variables -define Build/Compile/PyMod - $(INSTALL_DIR) $(PKG_INSTALL_DIR)/$(PYTHON_PKG_DIR) +define Build/Compile/HostPyRunTarget $(call HostPython, \ - cd $(PKG_BUILD_DIR)/$(strip $(1)); \ + $(if $(1),$(1);) \ CC="$(TARGET_CC)" \ CCSHARED="$(TARGET_CC) $(FPIC)" \ CXX="$(TARGET_CXX)" \ @@ -101,8 +100,19 @@ define Build/Compile/PyMod __PYVENV_LAUNCHER__="/usr/bin/$(PYTHON)" \ $(3) \ , \ - ./setup.py $(2) \ + $(2) \ ) +endef + +# $(1) => build subdir +# $(2) => additional arguments to setup.py +# $(3) => additional variables +define Build/Compile/PyMod + $(INSTALL_DIR) $(PKG_INSTALL_DIR)/$(PYTHON_PKG_DIR) + $(call Build/Compile/HostPyRunTarget, \ + cd $(PKG_BUILD_DIR)/$(strip $(1)), \ + ./setup.py $(2), \ + $(3)) find $(PKG_INSTALL_DIR) -name "*\.pyc" -o -name "*\.pyo" -o -name "*\.exe" | xargs rm -f endef From 3a710acf93e53a7f6a4605fb2e4399fb82aa163e Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Mon, 10 Oct 2016 11:11:14 +0800 Subject: [PATCH 2/3] python-pip: add host build Signed-off-by: Yousong Zhou --- lang/python-pip/Makefile | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/lang/python-pip/Makefile b/lang/python-pip/Makefile index 3be956516..db40f8f6a 100644 --- a/lang/python-pip/Makefile +++ b/lang/python-pip/Makefile @@ -17,10 +17,16 @@ PKG_MD5SUM:=87083c0b9867963b29f7aba3613e8f4a PKG_BUILD_DIR:=$(BUILD_DIR)/python-pip-$(PKG_VERSION) PKG_UNPACK=$(HOST_TAR) -C $(PKG_BUILD_DIR) --strip-components=1 -xzf $(DL_DIR)/$(PKG_SOURCE) +HOST_BUILD_DIR:=$(BUILD_DIR_HOST)/python-pip-$(PKG_VERSION) +HOST_UNPACK=$(HOST_TAR) -C $(HOST_BUILD_DIR) --strip-components=1 -xzf $(DL_DIR)/$(PKG_SOURCE) PKG_USE_MIPS16:=0 +HOST_BUILD_DEPENDS:=python python/host python-setuptools/host + +include $(INCLUDE_DIR)/host-build.mk include $(INCLUDE_DIR)/package.mk $(call include_mk, python-package.mk) +$(call include_mk, python-host.mk) define Package/python-pip SUBMENU:=Python @@ -54,6 +60,17 @@ define PyPackage/python-pip/install $(INSTALL_CONF) ./files/pip.conf $(1)/etc/ endef +define Host/Compile + $(call Build/Compile/HostPyMod,,\ + install --root="$(STAGING_DIR)/host" --prefix="" \ + --single-version-externally-managed \ + ) +endef + +define Host/Install +endef + +$(eval $(call HostBuild)) + $(eval $(call PyPackage,python-pip)) $(eval $(call BuildPackage,python-pip)) - From c6ddd3d7c0b9f4a86c6d39c03493e8f40aaa0dce Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 14 Oct 2016 10:43:42 +0800 Subject: [PATCH 3/3] python-packages: initial version 1.0 Signed-off-by: Yousong Zhou --- lang/python-packages/Makefile | 131 +++++++++++++++++++++++++++++++++ lang/python-packages/README.md | 72 ++++++++++++++++++ 2 files changed, 203 insertions(+) create mode 100644 lang/python-packages/Makefile create mode 100644 lang/python-packages/README.md diff --git a/lang/python-packages/Makefile b/lang/python-packages/Makefile new file mode 100644 index 000000000..f12b78e4d --- /dev/null +++ b/lang/python-packages/Makefile @@ -0,0 +1,131 @@ +# +# Copyright (C) 2016 Yousong Zhou +# +# This is free software, licensed under the GNU General Public License v2. +# See /LICENSE for more information. +# + +include $(TOPDIR)/rules.mk + +PKG_NAME:=python-packages +PKG_VERSION:=1.0 +PKG_RELEASE:=1 + +PKG_MAINTAINER:=Yousong Zhou + +# +# NOTE: move the host module installation to Host/Compile when +# HOST_CONFIG_DEPENDS is supported +# +# NOTE: PKG_CONFIG_DEPENDS cannot correctly track changes of string type config +# options, so you may want to do manual cleanup on config change. +# +PKG_CONFIG_DEPENDS:= \ + CONFIG_PACKAGE_python-packages-list-host \ + CONFIG_PACKAGE_python-packages-list \ + CONFIG_PACKAGE_python-packages-list-cleanup \ + CONFIG_PACKAGE_python-packages-envs \ + CONFIG_PACKAGE_python-packages-extra-deps \ + CONFIG_PACKAGE_python-packages-index-url \ + CONFIG_PACKAGE_python-packages-pip-opts \ + +PKG_BUILD_DEPENDS:=python python-pip/host + +include $(INCLUDE_DIR)/package.mk +$(call include_mk, python-host.mk) +$(call include_mk, python-package.mk) + +define Package/python-packages + SUBMENU:=Python + SECTION:=lang + CATEGORY:=Languages + TITLE:=A dummy package for packaging python modules with pip + DEPENDS:=@DEVEL +python +endef + +define Package/python-packages/config +if PACKAGE_python-packages +config PACKAGE_python-packages-list-host + string "List of python packages to install on host" +config PACKAGE_python-packages-list + string "List of python packages to install on target" +config PACKAGE_python-packages-list-cleanup + string "List of python packages to cleanup to avoid clash with existing packages" +config PACKAGE_python-packages-envs + string "Extra environment variables to pass on to pip and its children on target build" +config PACKAGE_python-packages-extra-deps + string "List of deps fulfilled but not tracked by the build system" +config PACKAGE_python-packages-index-url + string "Index URL passed to pip with --index-url" +config PACKAGE_python-packages-pip-opts + string "Additional arguments to pip command line" +endif +endef + +CONFIG_PACKAGE_python-packages-list-host:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-list-host)) +CONFIG_PACKAGE_python-packages-list:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-list)) +CONFIG_PACKAGE_python-packages-list-cleanup:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-list-cleanup)) +CONFIG_PACKAGE_python-packages-envs:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-envs)) +CONFIG_PACKAGE_python-packages-extra-deps:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-extra-deps)) +CONFIG_PACKAGE_python-packages-pip-opts:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-pip-opts)) + +HOST_PYTHON_PIP:=$(STAGING_DIR)/host/bin/pip$(PYTHON_VERSION) + +decr=$(word $(1),0 1 2 3 4 5 6 7 8 9 10) +recur=$(if $(subst 0,,$(2)),$(call recur,$(1),$(call decr,$(2)),$(call $(1)$(2),$(3))),$(3)) +_req2dir1=$(subst >,gt,$(1)) +_req2dir2=$(subst <,lt,$(1)) +_req2dir3=$(subst >=,geq,$(1)) +_req2dir4=$(subst <=,leq,$(1)) +_req2dir5=$(subst ://,:::,$(1)) +_req2dir6=$(subst *,_,$(1)) +_req2dir7=$(subst ?,_,$(1)) +req2dir=$(call recur,_req2dir,7,$(1)) + +# --ignore-installed, it may happen that host pip will ignore target install if +# it was already installed as host module, e.g. cffi deps of cryptograph +HOST_PYTHON_PIP_INSTALL=$(HOST_PYTHON_PIP) install \ + --root=$(1) \ + --prefix=$(2) \ + --ignore-installed \ + --no-compile \ + $(if $(CONFIG_PACKAGE_python-packages-index-url), --index-url $(CONFIG_PACKAGE_python-packages-index-url)) \ + $(if $(CONFIG_PACKAGE_python-packages-pip-opts), $(CONFIG_PACKAGE_python-packages-pip-opts)) \ + +HOST_PYTHON_PIP_INSTALL_HOST:=$(call HOST_PYTHON_PIP_INSTALL,$(STAGING_DIR)/host,"") +HOST_PYTHON_PIP_INSTALL_TARGET=$(call HOST_PYTHON_PIP_INSTALL,$(PKG_INSTALL_DIR)/$(call req2dir,$(pkg)),/usr) +HOST_PYTHON_PIP_INSTALL_CLEANUP:=$(call HOST_PYTHON_PIP_INSTALL,$(PKG_INSTALL_DIR)/_cleanup,/usr) + +define Build/Compile + $(foreach pkg,$(CONFIG_PACKAGE_python-packages-list-host), + $(call Build/Compile/HostPyRunHost,,$(HOST_PYTHON_PIP_INSTALL_HOST) $(pkg)) + ) + $(foreach pkg,$(CONFIG_PACKAGE_python-packages-list), + $(call Build/Compile/HostPyRunTarget,,$(call HOST_PYTHON_PIP_INSTALL_TARGET,$(pkg)) $(pkg),$(CONFIG_PACKAGE_python-packages-envs)) + ) + $(foreach pkg,$(CONFIG_PACKAGE_python-packages-list-cleanup), + $(call Build/Compile/HostPyRunTarget,,$(HOST_PYTHON_PIP_INSTALL_CLEANUP) $(pkg),$(CONFIG_PACKAGE_python-packages-envs)) + ) +endef + +define Package/python-packages/install + $(foreach pkg,$(CONFIG_PACKAGE_python-packages-list), + $(CP) "$(PKG_INSTALL_DIR)/$(call req2dir,$(pkg))"/* $(1) + ) + + find "$(PKG_INSTALL_DIR)/_cleanup" -mindepth 1 -depth | while read sf; do \ + tf="$$$${sf#$(PKG_INSTALL_DIR)/_cleanup/}"; \ + tf="$(1)/$$$$tf"; \ + if [ -f "$$$$tf" -o -L "$$$$tf" ]; then \ + rm -vf "$$$$tf"; \ + elif [ -d "$$$$tf" ]; then \ + rmdir -v -p "$$$$tf" || true; \ + fi \ + done +endef + +define Package/python-packages/extra_provides + echo $(CONFIG_PACKAGE_python-packages-extra-deps) | tr ' ' '\n' +endef + +$(eval $(call BuildPackage,python-packages)) diff --git a/lang/python-packages/README.md b/lang/python-packages/README.md new file mode 100644 index 000000000..3c09d8eb6 --- /dev/null +++ b/lang/python-packages/README.md @@ -0,0 +1,72 @@ +This package allows users to package python modules without creating package +Makefiles for each individual module and their dependencies. It provides a +way making packaging python packages faster and may also facilitate the process +of developing Makefiles for new python packages + +This is a raw DEVEL only package. Using it may entail a lot of implementation +details and you may need to resolve target dependencies and package details on +your own + +- Third party python packages may depend on features not included in e.g. + python-light +- Some python modules may require host install of another module to progress, + e.g. target cryptography requires host cffi +- Some python modules have external C library dependencies, e.g. pyOpenSSL + requires openssl libs +- Some packages may have an autoconf configure script whose arguments we + cannot control with pip and has to be passed on (hacked) by overriding some + environment variables + +## How it works + +1. Install host modules required for building target modules +2. Install each target module to separate directories +3. Install another copy of modules for cleanup purposes to make list of + installed files to be removed from target modules installed in step 2 + +Why should it be so + +1. Installing target cryptography requires host installation of cffi module +2. cryptography requires setuptools and pip will install its own copy with + --ignore-installed. When PACKAGE_python-setuptools is also selected, opkg + will complain of data file clashes if it was not removed here. + +Pip will handle dependency requirements of python modules, but external +dependencies like c libraries has to be prepared by the build system. The +issue is that there is currently no way to express such dependencies, thus may +cause build failure, e.g. pycrypto requires the presence of libgmp to build +successfully. + +## Tips + +If something goes wrong, we can add additional arguments to pip command +line to check the detailed build process. Some useful arguments may be + +- -v, for verbose output. Repeat this option if the current level of + verbosity is not enough +- --no-clean, for preserving pip build dir on build failure + +## Examples + +tornado (python-only module) + + CONFIG_PACKAGE_python-packages=y + CONFIG_PACKAGE_python-packages-list="tornado==4.4.2" + +cryptography (requires installation of host modules and cleanup on target modules) + + CONFIG_PACKAGE_python-packages=y + CONFIG_PACKAGE_python-packages-list-host="cffi" + CONFIG_PACKAGE_python-packages-list="cryptography" + CONFIG_PACKAGE_python-packages-list-cleanup="setuptools" + +pycrypto 2.7a1 (python module with autoconf configure script; depends on +libgmp; broken wmmintrin.h). 2.6.1 does not work because of a flaw in +the setup.py hardcoding host include directory + + CONFIG_PACKAGE_libgmp=y + CONFIG_PACKAGE_python-packages=y + CONFIG_PACKAGE_python-packages-list="https://github.com/dlitz/pycrypto/archive/v2.7a1.tar.gz" + CONFIG_PACKAGE_python-packages-envs="ac_cv_header_wmmintrin_h=no build_alias=$(GNU_HOST_NAME) host_alias=$(GNU_TARGET_NAME) target_alias=$(GNU_TARGET_NAME)" + CONFIG_PACKAGE_python-packages-extra-deps="libgmp.so.10" +