From da35e6770db37106b4746e9b790012164729cba6 Mon Sep 17 00:00:00 2001 From: Jeffery To Date: Fri, 24 May 2019 13:18:09 +0800 Subject: [PATCH] python-setuptools: Add reproducibility patches from Debian This adds the ability to patch setuptools (and pip), and adds 3 reproducibility patches from Debian[1]. (003-PKG-INFO-output-reproducible.patch addresses the issue identified in #9039.) The patching is not perfect, in that the patches are applied to setuptools and pip after they have been installed, since they are installed from wheels which are already "precompiled". Also, patching for the host install cannot be updated in place, for example if a patch is added or removed. [1]: https://sources.debian.org/patches/python-setuptools/40.8.0-1/ Signed-off-by: Jeffery To --- lang/python/python-version.mk | 2 +- lang/python/python/Makefile | 10 ++++++++++ .../patches-setuptools/001-reproducible.patch | 16 ++++++++++++++++ .../patches-setuptools/002-sorted-requires.patch | 16 ++++++++++++++++ .../003-PKG-INFO-output-reproducible.patch | 14 ++++++++++++++ lang/python/python3-version.mk | 2 +- lang/python/python3/Makefile | 10 ++++++++++ .../patches-setuptools/001-reproducible.patch | 16 ++++++++++++++++ .../patches-setuptools/002-sorted-requires.patch | 16 ++++++++++++++++ .../003-PKG-INFO-output-reproducible.patch | 14 ++++++++++++++ 10 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 lang/python/python/patches-setuptools/001-reproducible.patch create mode 100644 lang/python/python/patches-setuptools/002-sorted-requires.patch create mode 100644 lang/python/python/patches-setuptools/003-PKG-INFO-output-reproducible.patch create mode 100644 lang/python/python3/patches-setuptools/001-reproducible.patch create mode 100644 lang/python/python3/patches-setuptools/002-sorted-requires.patch create mode 100644 lang/python/python3/patches-setuptools/003-PKG-INFO-output-reproducible.patch diff --git a/lang/python/python-version.mk b/lang/python/python-version.mk index 418217623..84b8e4f05 100644 --- a/lang/python/python-version.mk +++ b/lang/python/python-version.mk @@ -8,7 +8,7 @@ PYTHON_VERSION:=2.7 PYTHON_VERSION_MICRO:=16 -PYTHON_SETUPTOOLS_PKG_RELEASE:=2 +PYTHON_SETUPTOOLS_PKG_RELEASE:=3 PYTHON_PIP_PKG_RELEASE:=2 PYTHON_SETUPTOOLS_VERSION:=40.6.2 diff --git a/lang/python/python/Makefile b/lang/python/python/Makefile index df825bc0d..979ccc771 100644 --- a/lang/python/python/Makefile +++ b/lang/python/python/Makefile @@ -174,6 +174,7 @@ define Build/Compile/python-setuptools --ignore-installed \ --root=$(PKG_BUILD_DIR)/install-setuptools --prefix=. \ $(PKG_BUILD_DIR)/Lib/ensurepip/_bundled/setuptools-$(PYTHON_SETUPTOOLS_VERSION)-py2.py3-none-any.whl + $(call PatchDir,$(PKG_BUILD_DIR)/install-setuptools/lib/python$(PYTHON_VERSION)/site-packages,./patches-setuptools,) endef endif # CONFIG_PACKAGE_python-setuptools @@ -183,6 +184,7 @@ define Build/Compile/python-pip --ignore-installed \ --root=$(PKG_BUILD_DIR)/install-pip --prefix=. \ $(PKG_BUILD_DIR)/Lib/ensurepip/_bundled/pip-$(PYTHON_PIP_VERSION)-py2.py3-none-any.whl + $(call PatchDir,$(PKG_BUILD_DIR)/install-pip/lib/python$(PYTHON_VERSION)/site-packages,./patches-pip,) endef endif # CONFIG_PACKAGE_python-pip @@ -304,6 +306,14 @@ define Host/Install $(MAKE) -C $(HOST_BUILD_DIR) install $(INSTALL_DIR) $(HOST_PYTHON_DIR)/bin/ $(INSTALL_BIN) $(HOST_BUILD_DIR)/Parser/pgen $(HOST_PYTHON_DIR)/bin/pgen2 + ifeq ($(wildcard $(HOST_PYTHON_PKG_DIR)/.setuptools-patched),) + $(call HostPatchDir,$(HOST_PYTHON_PKG_DIR),./patches-setuptools,) + touch $(HOST_PYTHON_PKG_DIR)/.setuptools-patched + endif + ifeq ($(wildcard $(HOST_PYTHON_PKG_DIR)/.pip-patched),) + $(call HostPatchDir,$(HOST_PYTHON_PKG_DIR),./patches-pip,) + touch $(HOST_PYTHON_PKG_DIR)/.pip-patched + endif endef $(eval $(call HostBuild)) diff --git a/lang/python/python/patches-setuptools/001-reproducible.patch b/lang/python/python/patches-setuptools/001-reproducible.patch new file mode 100644 index 000000000..32edc568d --- /dev/null +++ b/lang/python/python/patches-setuptools/001-reproducible.patch @@ -0,0 +1,16 @@ +https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=848136 +https://sources.debian.org/patches/python-setuptools/40.8.0-1/reproducible.diff/ + +Index: b/setuptools/command/easy_install.py +=================================================================== +--- a/setuptools/command/easy_install.py ++++ b/setuptools/command/easy_install.py +@@ -436,7 +436,7 @@ consider to install to another location, + for spec in self.args: + self.easy_install(spec, not self.no_deps) + if self.record: +- outputs = self.outputs ++ outputs = list(sorted(self.outputs)) + if self.root: # strip any package prefix + root_len = len(self.root) + for counter in range(len(outputs)): diff --git a/lang/python/python/patches-setuptools/002-sorted-requires.patch b/lang/python/python/patches-setuptools/002-sorted-requires.patch new file mode 100644 index 000000000..2ad4795b3 --- /dev/null +++ b/lang/python/python/patches-setuptools/002-sorted-requires.patch @@ -0,0 +1,16 @@ +https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=804249 +https://sources.debian.org/patches/python-setuptools/40.8.0-1/sorted-requires.diff/ + +Index: b/setuptools/command/egg_info.py +=================================================================== +--- a/setuptools/command/egg_info.py ++++ b/setuptools/command/egg_info.py +@@ -621,7 +621,7 @@ def warn_depends_obsolete(cmd, basename, + def _write_requirements(stream, reqs): + lines = yield_lines(reqs or ()) + append_cr = lambda line: line + '\n' +- lines = map(append_cr, lines) ++ lines = map(append_cr, sorted(lines)) + stream.writelines(lines) + + diff --git a/lang/python/python/patches-setuptools/003-PKG-INFO-output-reproducible.patch b/lang/python/python/patches-setuptools/003-PKG-INFO-output-reproducible.patch new file mode 100644 index 000000000..15f34dcdb --- /dev/null +++ b/lang/python/python/patches-setuptools/003-PKG-INFO-output-reproducible.patch @@ -0,0 +1,14 @@ +https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=894215 +https://sources.debian.org/patches/python-setuptools/40.8.0-1/PKG-INFO-output-reproducible.diff/ + +--- a/setuptools/dist.py ++++ b/setuptools/dist.py +@@ -191,7 +191,7 @@ def write_pkg_file(self, file): + self.long_description_content_type + ) + if self.provides_extras: +- for extra in self.provides_extras: ++ for extra in sorted(self.provides_extras): + write_field('Provides-Extra', extra) + + diff --git a/lang/python/python3-version.mk b/lang/python/python3-version.mk index 9eebeb260..784ec1778 100644 --- a/lang/python/python3-version.mk +++ b/lang/python/python3-version.mk @@ -12,7 +12,7 @@ PYTHON3_VERSION_MICRO:=2 PYTHON3_VERSION:=$(PYTHON3_VERSION_MAJOR).$(PYTHON3_VERSION_MINOR) -PYTHON3_SETUPTOOLS_PKG_RELEASE:=2 +PYTHON3_SETUPTOOLS_PKG_RELEASE:=3 PYTHON3_PIP_PKG_RELEASE:=2 PYTHON3_SETUPTOOLS_VERSION:=40.6.2 diff --git a/lang/python/python3/Makefile b/lang/python/python3/Makefile index 4053ff0c9..d151459ed 100644 --- a/lang/python/python3/Makefile +++ b/lang/python/python3/Makefile @@ -178,6 +178,7 @@ define Build/Compile/python3-setuptools --ignore-installed \ --root=$(PKG_BUILD_DIR)/install-setuptools --prefix=. \ $(PKG_BUILD_DIR)/Lib/ensurepip/_bundled/setuptools-$(PYTHON3_SETUPTOOLS_VERSION)-py2.py3-none-any.whl + $(call PatchDir,$(PKG_BUILD_DIR)/install-setuptools/lib/python$(PYTHON3_VERSION)/site-packages,./patches-setuptools,) endef endif # CONFIG_PACKAGE_python3-setuptools @@ -187,6 +188,7 @@ define Build/Compile/python3-pip --ignore-installed \ --root=$(PKG_BUILD_DIR)/install-pip --prefix=. \ $(PKG_BUILD_DIR)/Lib/ensurepip/_bundled/pip-$(PYTHON3_PIP_VERSION)-py2.py3-none-any.whl + $(call PatchDir,$(PKG_BUILD_DIR)/install-pip/lib/python$(PYTHON3_VERSION)/site-packages,./patches-pip,) endef endif # CONFIG_PACKAGE_python3-pip @@ -299,6 +301,14 @@ define Host/Install $(MAKE) -C $(HOST_BUILD_DIR) install $(INSTALL_DIR) $(HOST_PYTHON3_DIR)/bin/ $(INSTALL_BIN) $(HOST_BUILD_DIR)/Parser/pgen $(HOST_PYTHON3_DIR)/bin/pgen3 + ifeq ($(wildcard $(HOST_PYTHON3_PKG_DIR)/.setuptools-patched),) + $(call HostPatchDir,$(HOST_PYTHON3_PKG_DIR),./patches-setuptools,) + touch $(HOST_PYTHON3_PKG_DIR)/.setuptools-patched + endif + ifeq ($(wildcard $(HOST_PYTHON3_PKG_DIR)/.pip-patched),) + $(call HostPatchDir,$(HOST_PYTHON3_PKG_DIR),./patches-pip,) + touch $(HOST_PYTHON3_PKG_DIR)/.pip-patched + endif endef $(eval $(call HostBuild)) diff --git a/lang/python/python3/patches-setuptools/001-reproducible.patch b/lang/python/python3/patches-setuptools/001-reproducible.patch new file mode 100644 index 000000000..32edc568d --- /dev/null +++ b/lang/python/python3/patches-setuptools/001-reproducible.patch @@ -0,0 +1,16 @@ +https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=848136 +https://sources.debian.org/patches/python-setuptools/40.8.0-1/reproducible.diff/ + +Index: b/setuptools/command/easy_install.py +=================================================================== +--- a/setuptools/command/easy_install.py ++++ b/setuptools/command/easy_install.py +@@ -436,7 +436,7 @@ consider to install to another location, + for spec in self.args: + self.easy_install(spec, not self.no_deps) + if self.record: +- outputs = self.outputs ++ outputs = list(sorted(self.outputs)) + if self.root: # strip any package prefix + root_len = len(self.root) + for counter in range(len(outputs)): diff --git a/lang/python/python3/patches-setuptools/002-sorted-requires.patch b/lang/python/python3/patches-setuptools/002-sorted-requires.patch new file mode 100644 index 000000000..2ad4795b3 --- /dev/null +++ b/lang/python/python3/patches-setuptools/002-sorted-requires.patch @@ -0,0 +1,16 @@ +https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=804249 +https://sources.debian.org/patches/python-setuptools/40.8.0-1/sorted-requires.diff/ + +Index: b/setuptools/command/egg_info.py +=================================================================== +--- a/setuptools/command/egg_info.py ++++ b/setuptools/command/egg_info.py +@@ -621,7 +621,7 @@ def warn_depends_obsolete(cmd, basename, + def _write_requirements(stream, reqs): + lines = yield_lines(reqs or ()) + append_cr = lambda line: line + '\n' +- lines = map(append_cr, lines) ++ lines = map(append_cr, sorted(lines)) + stream.writelines(lines) + + diff --git a/lang/python/python3/patches-setuptools/003-PKG-INFO-output-reproducible.patch b/lang/python/python3/patches-setuptools/003-PKG-INFO-output-reproducible.patch new file mode 100644 index 000000000..15f34dcdb --- /dev/null +++ b/lang/python/python3/patches-setuptools/003-PKG-INFO-output-reproducible.patch @@ -0,0 +1,14 @@ +https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=894215 +https://sources.debian.org/patches/python-setuptools/40.8.0-1/PKG-INFO-output-reproducible.diff/ + +--- a/setuptools/dist.py ++++ b/setuptools/dist.py +@@ -191,7 +191,7 @@ def write_pkg_file(self, file): + self.long_description_content_type + ) + if self.provides_extras: +- for extra in self.provides_extras: ++ for extra in sorted(self.provides_extras): + write_field('Provides-Extra', extra) + +