From b104c56e2ba23e972d8f14280b41dd54f22ba302 Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean Date: Mon, 9 Sep 2019 13:10:22 +0300 Subject: [PATCH 1/4] python: change condition check for legacy opt for Python3 As I remember this worked. But since `set -e` is set, I am a bit paranoid about it. In the sense that it may fail if `ver` != 3. Signed-off-by: Alexandru Ardelean --- lang/python/python-package-install.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lang/python/python-package-install.sh b/lang/python/python-package-install.sh index 2ca2abc16..f2646f848 100644 --- a/lang/python/python-package-install.sh +++ b/lang/python/python-package-install.sh @@ -77,8 +77,9 @@ if [ "$mode" == "sources" ] ; then exit 0 fi -legacy= -[ "$ver" == "3" ] && legacy="-b" +if [ "$ver" == "3" ] ; then + legacy="-b" +fi # default max recursion is 10 max_recursion_level=20 From ed0e77f3c385770065bf6ec6ed7c3bbbb6deb395 Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean Date: Tue, 10 Sep 2019 19:51:03 +0300 Subject: [PATCH 2/4] python,python3: split python[3]-pkg-resources from setuptools This package is required by other packages to run some binaries via `load_entry_point`. So, this splits this package away from setuptools. setuptools is pretty big, akd pkg-resources is also big, but not as big. Signed-off-by: Alexandru Ardelean --- lang/python/python/Makefile | 11 ++++++- .../files/python-package-pkg-resources.mk | 28 +++++++++++++++++ .../python/files/python-package-setuptools.mk | 3 +- lang/python/python3/Makefile | 11 ++++++- .../files/python3-package-pkg-resources.mk | 30 +++++++++++++++++++ .../files/python3-package-setuptools.mk | 3 +- 6 files changed, 80 insertions(+), 6 deletions(-) create mode 100644 lang/python/python/files/python-package-pkg-resources.mk create mode 100644 lang/python/python3/files/python3-package-pkg-resources.mk diff --git a/lang/python/python/Makefile b/lang/python/python/Makefile index b75e22966..a52e4a122 100644 --- a/lang/python/python/Makefile +++ b/lang/python/python/Makefile @@ -12,7 +12,7 @@ include ../python-version.mk PKG_NAME:=python PKG_VERSION:=$(PYTHON_VERSION).$(PYTHON_VERSION_MICRO) -PKG_RELEASE:=9 +PKG_RELEASE:=10 PKG_SOURCE:=Python-$(PKG_VERSION).tar.xz PKG_SOURCE_URL:=https://www.python.org/ftp/python/$(PKG_VERSION) @@ -39,6 +39,7 @@ PKG_BUILD_DIR:=$(BUILD_DIR)/Python-$(PKG_VERSION) HOST_BUILD_DIR:=$(BUILD_DIR_HOST)/Python-$(PKG_VERSION) PKG_CONFIG_DEPENDS:= \ + CONFIG_PACKAGE_python-pkg-resources \ CONFIG_PACKAGE_python-setuptools CONFIG_PACKAGE_python-pip \ CONFIG_PYTHON_BLUETOOTH_SUPPORT @@ -169,6 +170,14 @@ define Build/Prepare endef ifdef CONFIG_PACKAGE_python-setuptools +PYTHON_SETUPTOOLS_BUILD:=1 +endif + +ifdef CONFIG_PACKAGE_python-pkg-resources +PYTHON_SETUPTOOLS_BUILD:=1 +endif + +ifeq ($(PYTHON_SETUPTOOLS_BUILD),1) define Build/Compile/python-setuptools $(HOST_PYTHON_PIP) \ --disable-pip-version-check \ diff --git a/lang/python/python/files/python-package-pkg-resources.mk b/lang/python/python/files/python-package-pkg-resources.mk new file mode 100644 index 000000000..f0e819b54 --- /dev/null +++ b/lang/python/python/files/python-package-pkg-resources.mk @@ -0,0 +1,28 @@ +# +# Copyright (C) 2019 Alexandru Ardelean +# +# This is free software, licensed under the GNU General Public License v2. +# See /LICENSE for more information. +# + +define Package/python-pkg-resources +$(call Package/python/Default) + TITLE:=Python $(PYTHON_VERSION) pkg_resources module (part of etuptools) + VERSION:=$(PYTHON_SETUPTOOLS_VERSION)-$(PYTHON_SETUPTOOLS_PKG_RELEASE) + LICENSE:=MIT + LICENSE_FILES:=LICENSE +# CPE_ID:=cpe:/a:python:setuptools # not currently handled this way by uscan + DEPENDS:=+python +endef + +define PyPackage/python-pkg-resources/install + $(INSTALL_DIR) $(1)/usr/lib/python$(PYTHON_VERSION)/site-packages + $(CP) \ + $(PKG_BUILD_DIR)/install-setuptools/usr/lib/python$(PYTHON_VERSION)/site-packages/pkg_resources \ + $(1)/usr/lib/python$(PYTHON_VERSION)/site-packages +endef + +$(eval $(call PyBasePackage,python-pkg-resources, \ + , \ + DO_NOT_ADD_TO_PACKAGE_DEPENDS \ +)) diff --git a/lang/python/python/files/python-package-setuptools.mk b/lang/python/python/files/python-package-setuptools.mk index 95951ecec..305a515b7 100644 --- a/lang/python/python/files/python-package-setuptools.mk +++ b/lang/python/python/files/python-package-setuptools.mk @@ -12,14 +12,13 @@ $(call Package/python/Default) LICENSE:=MIT LICENSE_FILES:=LICENSE # CPE_ID:=cpe:/a:python:setuptools # not currently handled this way by uscan - DEPENDS:=+python + DEPENDS:=+python +python-pkg-resources endef define PyPackage/python-setuptools/install $(INSTALL_DIR) $(1)/usr/bin $(1)/usr/lib/python$(PYTHON_VERSION)/site-packages $(CP) $(PKG_BUILD_DIR)/install-setuptools/usr/bin/* $(1)/usr/bin $(CP) \ - $(PKG_BUILD_DIR)/install-setuptools/usr/lib/python$(PYTHON_VERSION)/site-packages/pkg_resources \ $(PKG_BUILD_DIR)/install-setuptools/usr/lib/python$(PYTHON_VERSION)/site-packages/setuptools \ $(PKG_BUILD_DIR)/install-setuptools/usr/lib/python$(PYTHON_VERSION)/site-packages/setuptools-$(PYTHON_SETUPTOOLS_VERSION).dist-info \ $(PKG_BUILD_DIR)/install-setuptools/usr/lib/python$(PYTHON_VERSION)/site-packages/easy_install.py \ diff --git a/lang/python/python3/Makefile b/lang/python/python3/Makefile index 79a71ccaf..d4bc11d51 100644 --- a/lang/python/python3/Makefile +++ b/lang/python/python3/Makefile @@ -14,7 +14,7 @@ PYTHON_VERSION:=$(PYTHON3_VERSION) PYTHON_VERSION_MICRO:=$(PYTHON3_VERSION_MICRO) PKG_NAME:=python3 -PKG_RELEASE:=2 +PKG_RELEASE:=3 PKG_VERSION:=$(PYTHON_VERSION).$(PYTHON_VERSION_MICRO) PKG_SOURCE:=Python-$(PKG_VERSION).tar.xz @@ -41,6 +41,7 @@ PKG_BUILD_DIR:=$(BUILD_DIR)/Python-$(PKG_VERSION) HOST_BUILD_DIR:=$(BUILD_DIR_HOST)/Python-$(PKG_VERSION) PKG_CONFIG_DEPENDS:= \ + CONFIG_PACKAGE_python3-pkg-resources \ CONFIG_PACKAGE_python3-setuptools CONFIG_PACKAGE_python3-pip \ CONFIG_PYTHON3_BLUETOOTH_SUPPORT @@ -172,6 +173,14 @@ define Build/Prepare endef ifdef CONFIG_PACKAGE_python3-setuptools +PYTHON3_SETUPTOOLS_BUILD:=1 +endif + +ifdef CONFIG_PACKAGE_python3-pkg-resources +PYTHON3_SETUPTOOLS_BUILD:=1 +endif + +ifeq ($(PYTHON3_SETUPTOOLS_BUILD),1) define Build/Compile/python3-setuptools $(HOST_PYTHON3_PIP) \ --disable-pip-version-check \ diff --git a/lang/python/python3/files/python3-package-pkg-resources.mk b/lang/python/python3/files/python3-package-pkg-resources.mk new file mode 100644 index 000000000..592fe2c8c --- /dev/null +++ b/lang/python/python3/files/python3-package-pkg-resources.mk @@ -0,0 +1,30 @@ +# +# Copyright (C) 2019 Alexandru Ardelean +# +# This is free software, licensed under the GNU General Public License v2. +# See /LICENSE for more information. +# + +define Package/python3-pkg-resources +$(call Package/python3/Default) + TITLE:=Python $(PYTHON3_VERSION) pkg_resources module (part of setuptools) + VERSION:=$(PYTHON3_SETUPTOOLS_VERSION)-$(PYTHON3_SETUPTOOLS_PKG_RELEASE) + LICENSE:=MIT + LICENSE_FILES:=LICENSE +# CPE_ID:=cpe:/a:python:setuptools # not currently handled this way by uscan + DEPENDS:=+python3 +endef + +define Py3Package/python3-pkg-resources/install + $(INSTALL_DIR) $(1)/usr/lib/python$(PYTHON3_VERSION)/site-packages + $(CP) \ + $(PKG_BUILD_DIR)/install-setuptools/usr/lib/python$(PYTHON3_VERSION)/site-packages/pkg_resources \ + $(1)/usr/lib/python$(PYTHON3_VERSION)/site-packages + find $(1)/usr/lib/python$(PYTHON3_VERSION)/site-packages/ -path '*/__pycache__/*' -delete + find $(1)/usr/lib/python$(PYTHON3_VERSION)/site-packages/ -type d -name __pycache__ -delete +endef + +$(eval $(call Py3BasePackage,python3-pkg-resources, \ + , \ + DO_NOT_ADD_TO_PACKAGE_DEPENDS \ +)) diff --git a/lang/python/python3/files/python3-package-setuptools.mk b/lang/python/python3/files/python3-package-setuptools.mk index b3ccce895..64a86b81d 100644 --- a/lang/python/python3/files/python3-package-setuptools.mk +++ b/lang/python/python3/files/python3-package-setuptools.mk @@ -12,7 +12,7 @@ $(call Package/python3/Default) LICENSE:=MIT LICENSE_FILES:=LICENSE # CPE_ID:=cpe:/a:python:setuptools # not currently handled this way by uscan - DEPENDS:=+python3 + DEPENDS:=+python3 +python3-pkg-resources endef define Py3Package/python3-setuptools/install @@ -20,7 +20,6 @@ define Py3Package/python3-setuptools/install $(CP) $(PKG_BUILD_DIR)/install-setuptools/usr/bin/easy_install-* $(1)/usr/bin $(LN) easy_install-$(PYTHON3_VERSION) $(1)/usr/bin/easy_install-3 $(CP) \ - $(PKG_BUILD_DIR)/install-setuptools/usr/lib/python$(PYTHON3_VERSION)/site-packages/pkg_resources \ $(PKG_BUILD_DIR)/install-setuptools/usr/lib/python$(PYTHON3_VERSION)/site-packages/setuptools \ $(PKG_BUILD_DIR)/install-setuptools/usr/lib/python$(PYTHON3_VERSION)/site-packages/setuptools-$(PYTHON3_SETUPTOOLS_VERSION).dist-info \ $(PKG_BUILD_DIR)/install-setuptools/usr/lib/python$(PYTHON3_VERSION)/site-packages/easy_install.py \ From 721642908c2bc2f16ff371bdd33e4df02813318c Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean Date: Thu, 12 Sep 2019 15:03:50 +0300 Subject: [PATCH 3/4] python,python3: add Py[3]Shebang functions & move outside of script Some packages just install some Python binaries, that may need their shebang fixed. This change adds some utilities to help with that and try to centralize the sed rules a bit. It also removes the logic from the `python-package-install.sh` into the `python-package[3].mk` files. This does 2 things: 1. It minimizes the need for the shell script to know the Python version 2/3 2. Makes the logic re-usable in packages; especially if the install rules differ a bit Signed-off-by: Alexandru Ardelean --- lang/python/python-package-install.sh | 8 -------- lang/python/python-package.mk | 10 ++++++++-- lang/python/python3-package.mk | 12 +++++++++--- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/lang/python/python-package-install.sh b/lang/python/python-package-install.sh index f2646f848..337727af1 100644 --- a/lang/python/python-package-install.sh +++ b/lang/python/python-package-install.sh @@ -54,8 +54,6 @@ python="$4" mode="$5" filespec="$6" -SED="${SED:-sed -e}" - find "$src_dir" -name "*.exe" -delete process_filespec "$src_dir" "$dst_dir" "$filespec" || { @@ -63,12 +61,6 @@ process_filespec "$src_dir" "$dst_dir" "$filespec" || { exit 1 } -usr_bin_dir="$dst_dir/usr/bin" - -if [ -d "$usr_bin_dir" ] ; then - $SED "1"'!'"b;s,^#"'!'".*python.*,#"'!'"/usr/bin/python${ver}," -i --follow-symlinks $usr_bin_dir/* -fi - if [ "$mode" == "sources" ] ; then # Copy only python source files find "$dst_dir" -not -type d -not -name "*.py" -delete diff --git a/lang/python/python-package.mk b/lang/python/python-package.mk index 9bd0b9aa4..019f4f028 100644 --- a/lang/python/python-package.mk +++ b/lang/python/python-package.mk @@ -35,6 +35,10 @@ ifdef CONFIG_USE_MIPS16 TARGET_CFLAGS += -mno-mips16 -mno-interlink-mips16 endif +define PyShebang +$(SED) "1"'!'"b;s,^#"'!'".*python.*,#"'!'"/usr/bin/python2," -i --follow-symlinks $(1) +endef + define PyPackage define Package/$(1)-src @@ -74,11 +78,13 @@ define PyPackage define Package/$(1)/install $$(call PyPackage/$(1)/install,$$(1)) - SED="$(SED)" \ $(SHELL) $(python_mk_path)python-package-install.sh "2" \ "$(PKG_INSTALL_DIR)" "$$(1)" \ "$(HOST_PYTHON_BIN)" "$$(2)" \ - "$$$$$$$$$$(call shvar,PyPackage/$(1)/filespec)" + "$$$$$$$$$$(call shvar,PyPackage/$(1)/filespec)" && \ + if [ -d "$$(1)/usr/bin" ]; then \ + $(call PyShebang,$$(1)/usr/bin/*) ; \ + fi endef define Package/$(1)-src/install diff --git a/lang/python/python3-package.mk b/lang/python/python3-package.mk index e14290081..d20d19801 100644 --- a/lang/python/python3-package.mk +++ b/lang/python/python3-package.mk @@ -34,6 +34,10 @@ ifdef CONFIG_USE_MIPS16 TARGET_CFLAGS += -mno-mips16 -mno-interlink-mips16 endif +define Py3Shebang +$(SED) "1"'!'"b;s,^#"'!'".*python.*,#"'!'"/usr/bin/python3," -i --follow-symlinks $(1) +endef + define Py3Package define Package/$(1)-src @@ -73,12 +77,14 @@ define Py3Package define Package/$(1)/install $$(call Py3Package/$(1)/install,$$(1)) - SED="$(SED)" \ $(SHELL) $(python3_mk_path)python-package-install.sh "3" \ "$(PKG_INSTALL_DIR)" "$$(1)" \ "$(HOST_PYTHON3_BIN)" "$$(2)" \ - "$$$$$$$$$$(call shvar,Py3Package/$(1)/filespec)" - endef + "$$$$$$$$$$(call shvar,Py3Package/$(1)/filespec)" && \ + if [ -d "$$(1)/usr/bin" ]; then \ + $(call Py3Shebang,$$(1)/usr/bin/*) ; \ + fi + endef define Package/$(1)-src/install $$(call Package/$(1)/install,$$(1),sources) From 5f9ea76e7a00bb2851204903521d053b76772f4d Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean Date: Mon, 12 Aug 2019 10:14:16 +0300 Subject: [PATCH 4/4] gunicorn: convert to Python packaging format + add Python3 variant And splitting the gunicorn[3] binary/executable away from the python[3]-gunicorn libraries. This was inspired from Debian packaging. The gunicorn[3] binaries require the new `python[3]-pkg-resources` libraries to run, which add ~1.1 MB on the [ram]disk when uncompressed. For the Python2 variant, the `_gaiohttp.py` is dropped as it fails to compile, so it would likely be unusable anyway: ``` File "/usr/lib/python2.7/site-packages/gunicorn/workers/_gaiohttp.py", line 84 yield from self.wsgi.close() ^ SyntaxError: invalid syntax ``` People around the web recommend this as well: https://stackoverflow.com/questions/25611140/syntax-error-installing-gunicorn https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=803170 https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=803202 Signed-off-by: Alexandru Ardelean --- lang/python/gunicorn/Makefile | 108 +++++++++++++++++++++++++++------- 1 file changed, 88 insertions(+), 20 deletions(-) diff --git a/lang/python/gunicorn/Makefile b/lang/python/gunicorn/Makefile index 2aa87f478..4ba9f2195 100644 --- a/lang/python/gunicorn/Makefile +++ b/lang/python/gunicorn/Makefile @@ -10,43 +10,111 @@ include $(TOPDIR)/rules.mk PKG_NAME:=gunicorn PKG_VERSION:=19.9.0 PKG_RELEASE=2 -PKG_LICENSE:=MIT PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz PKG_SOURCE_URL:=https://files.pythonhosted.org/packages/source/g/gunicorn PKG_HASH:=fa2662097c66f920f53f70621c6c58ca4a3c4d3434205e608e121b5b3b71f4f3 + PKG_MAINTAINER:=Alexandru Ardelean +PKG_LICENSE:=MIT +PKG_LICENSE_FILES:=LICENSE + +PKG_BUILD_DIR:=$(BUILD_DIR)/$(BUILD_VARIANT)-$(PKG_NAME)-$(PKG_VERSION) include $(INCLUDE_DIR)/package.mk include ../python-package.mk +include ../python3-package.mk + +PKG_UNPACK:=$(HOST_TAR) -C $(PKG_BUILD_DIR) --strip-components=1 -xzf $(DL_DIR)/$(PKG_SOURCE) + +define Package/gunicorn/Default + SUBMENU:=Python + SECTION:=lang + CATEGORY:=Languages + TITLE:=WSGI HTTP Server for UNIX + URL:=https://gunicorn.org +endef + +define Package/python-gunicorn +$(call Package/gunicorn/Default) + DEPENDS:=+python + VARIANT:=python +endef + +define Package/python-gunicorn/description + WSGI HTTP Server for UNIX (libraries) +endef + +define PyPackage/python-gunicorn/filespec ++|$(PYTHON_PKG_DIR) +-|$(PYTHON_PKG_DIR)/gunicorn/workers/_gaiohttp.py +endef + +define Package/python3-gunicorn +$(call Package/gunicorn/Default) + DEPENDS:=+python3 + VARIANT:=python3 +endef + +define Package/python3-gunicorn/description +$(call define Package/python-gunicorn/description) +. +(Variant for Python3) +endef + +# Make sure that the binaries are not installed with the libraries +# That means adding some empty Py[3]Package/gunicorn[3]/install rules +define PyPackage/python-gunicorn/install + : +endef + +define Py3Package/python3-gunicorn/install + : +endef define Package/gunicorn - SUBMENU:=Python - SECTION:=lang - CATEGORY:=Languages - TITLE:=WSGI HTTP Server for UNIX - URL:=https://gunicorn.org - DEPENDS:=+python +python-setuptools +$(call Package/gunicorn/Default) + DEPENDS:=+python +python-pkg-resources +python-gunicorn endef define Package/gunicorn/description - WSGI HTTP Server for UNIX -endef - -define Build/Compile - $(call Build/Compile/PyMod,,install --prefix=/usr --root=$(PKG_INSTALL_DIR)) + WSGI HTTP Server for UNIX (daemon) endef define Package/gunicorn/install - $(INSTALL_DIR) $(1)$(PYTHON_PKG_DIR) $(INSTALL_DIR) $(1)/usr/bin - $(CP) \ - $(PKG_INSTALL_DIR)$(PYTHON_PKG_DIR)/* \ - $(1)$(PYTHON_PKG_DIR) - $(INSTALL_BIN) $(PKG_INSTALL_DIR)/usr/bin/gunicorn* \ - $(1)/usr/bin - # fix python exec path in scripts - $(SED) 's/#!.*/#!\/usr\/bin\/python/g' $(1)/usr/bin/gunicorn* + $(INSTALL_BIN) $(PKG_INSTALL_DIR)/usr/bin/* $(1)/usr/bin/ + $(call PyShebang,$(1)/usr/bin/*) endef +define Package/gunicorn3 +$(call Package/gunicorn/Default) + DEPENDS:=+python3 +python3-pkg-resources +python3-gunicorn +endef + +define Package/gunicorn3/description +$(call define Package/gunicorn/description) +. +(Variant for Python3) +endef + +define Package/gunicorn3/install + $(INSTALL_DIR) $(1)/usr/bin + $(INSTALL_BIN) \ + $(PKG_INSTALL_DIR)/usr/bin/gunicorn \ + $(1)/usr/bin/gunicorn3 + $(INSTALL_BIN) \ + $(PKG_INSTALL_DIR)/usr/bin/gunicorn_paster \ + $(1)/usr/bin/gunicorn3_paster + $(call Py3Shebang,$(1)/usr/bin/*) +endef + +$(eval $(call PyPackage,python-gunicorn)) +$(eval $(call BuildPackage,python-gunicorn)) +$(eval $(call BuildPackage,python-gunicorn-src)) $(eval $(call BuildPackage,gunicorn)) + +$(eval $(call Py3Package,python3-gunicorn)) +$(eval $(call BuildPackage,python3-gunicorn)) +$(eval $(call BuildPackage,python3-gunicorn-src)) +$(eval $(call BuildPackage,gunicorn3))