Merge pull request #8733 from jefferyto/openwrt-18.06-python-cve-2019-9636
[openwrt-18.06] python,python3: Fix CVE-2019-9636 - urlsplit missing NFKC normalization
This commit is contained in:
Rosen Penev
GitHub
commit
d42547cd3f
5 changed files with 353 additions and 2 deletions
|
|
@ -12,7 +12,7 @@ include ../python-version.mk
|
|||
|
||||
PKG_NAME:=python
|
||||
PKG_VERSION:=$(PYTHON_VERSION).$(PYTHON_VERSION_MICRO)
|
||||
PKG_RELEASE:=3
|
||||
PKG_RELEASE:=4
|
||||
|
||||
PKG_SOURCE:=Python-$(PKG_VERSION).tar.xz
|
||||
PKG_SOURCE_URL:=https://www.python.org/ftp/python/$(PKG_VERSION)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,155 @@
|
|||
From 3e3669c9c41a27e1466e2c28b3906e3dd0ce3e7e Mon Sep 17 00:00:00 2001
|
||||
From: Steve Dower <steve.dower@python.org>
|
||||
Date: Thu, 7 Mar 2019 08:25:22 -0800
|
||||
Subject: [PATCH] bpo-36216: Add check for characters in netloc that normalize
|
||||
to separators (GH-12201)
|
||||
|
||||
---
|
||||
Doc/library/urlparse.rst | 20 ++++++++++++++++
|
||||
Lib/test/test_urlparse.py | 24 +++++++++++++++++++
|
||||
Lib/urlparse.py | 17 +++++++++++++
|
||||
.../2019-03-06-09-38-40.bpo-36216.6q1m4a.rst | 3 +++
|
||||
4 files changed, 64 insertions(+)
|
||||
create mode 100644 Misc/NEWS.d/next/Security/2019-03-06-09-38-40.bpo-36216.6q1m4a.rst
|
||||
|
||||
diff --git a/Doc/library/urlparse.rst b/Doc/library/urlparse.rst
|
||||
index 22249da54fbb..0989c88c3022 100644
|
||||
--- a/Doc/library/urlparse.rst
|
||||
+++ b/Doc/library/urlparse.rst
|
||||
@@ -119,12 +119,22 @@ The :mod:`urlparse` module defines the following functions:
|
||||
See section :ref:`urlparse-result-object` for more information on the result
|
||||
object.
|
||||
|
||||
+ Characters in the :attr:`netloc` attribute that decompose under NFKC
|
||||
+ normalization (as used by the IDNA encoding) into any of ``/``, ``?``,
|
||||
+ ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is
|
||||
+ decomposed before parsing, or is not a Unicode string, no error will be
|
||||
+ raised.
|
||||
+
|
||||
.. versionchanged:: 2.5
|
||||
Added attributes to return value.
|
||||
|
||||
.. versionchanged:: 2.7
|
||||
Added IPv6 URL parsing capabilities.
|
||||
|
||||
+ .. versionchanged:: 2.7.17
|
||||
+ Characters that affect netloc parsing under NFKC normalization will
|
||||
+ now raise :exc:`ValueError`.
|
||||
+
|
||||
|
||||
.. function:: parse_qs(qs[, keep_blank_values[, strict_parsing[, max_num_fields]]])
|
||||
|
||||
@@ -232,11 +242,21 @@ The :mod:`urlparse` module defines the following functions:
|
||||
See section :ref:`urlparse-result-object` for more information on the result
|
||||
object.
|
||||
|
||||
+ Characters in the :attr:`netloc` attribute that decompose under NFKC
|
||||
+ normalization (as used by the IDNA encoding) into any of ``/``, ``?``,
|
||||
+ ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is
|
||||
+ decomposed before parsing, or is not a Unicode string, no error will be
|
||||
+ raised.
|
||||
+
|
||||
.. versionadded:: 2.2
|
||||
|
||||
.. versionchanged:: 2.5
|
||||
Added attributes to return value.
|
||||
|
||||
+ .. versionchanged:: 2.7.17
|
||||
+ Characters that affect netloc parsing under NFKC normalization will
|
||||
+ now raise :exc:`ValueError`.
|
||||
+
|
||||
|
||||
.. function:: urlunsplit(parts)
|
||||
|
||||
diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py
|
||||
index 4e1ded73c266..73b0228ea8e3 100644
|
||||
--- a/Lib/test/test_urlparse.py
|
||||
+++ b/Lib/test/test_urlparse.py
|
||||
@@ -1,4 +1,6 @@
|
||||
from test import test_support
|
||||
+import sys
|
||||
+import unicodedata
|
||||
import unittest
|
||||
import urlparse
|
||||
|
||||
@@ -624,6 +626,28 @@ def test_portseparator(self):
|
||||
self.assertEqual(urlparse.urlparse("http://www.python.org:80"),
|
||||
('http','www.python.org:80','','','',''))
|
||||
|
||||
+ def test_urlsplit_normalization(self):
|
||||
+ # Certain characters should never occur in the netloc,
|
||||
+ # including under normalization.
|
||||
+ # Ensure that ALL of them are detected and cause an error
|
||||
+ illegal_chars = u'/:#?@'
|
||||
+ hex_chars = {'{:04X}'.format(ord(c)) for c in illegal_chars}
|
||||
+ denorm_chars = [
|
||||
+ c for c in map(unichr, range(128, sys.maxunicode))
|
||||
+ if (hex_chars & set(unicodedata.decomposition(c).split()))
|
||||
+ and c not in illegal_chars
|
||||
+ ]
|
||||
+ # Sanity check that we found at least one such character
|
||||
+ self.assertIn(u'\u2100', denorm_chars)
|
||||
+ self.assertIn(u'\uFF03', denorm_chars)
|
||||
+
|
||||
+ for scheme in [u"http", u"https", u"ftp"]:
|
||||
+ for c in denorm_chars:
|
||||
+ url = u"{}://netloc{}false.netloc/path".format(scheme, c)
|
||||
+ print "Checking %r" % url
|
||||
+ with self.assertRaises(ValueError):
|
||||
+ urlparse.urlsplit(url)
|
||||
+
|
||||
def test_main():
|
||||
test_support.run_unittest(UrlParseTestCase)
|
||||
|
||||
diff --git a/Lib/urlparse.py b/Lib/urlparse.py
|
||||
index f7c2b032b097..54eda08651ab 100644
|
||||
--- a/Lib/urlparse.py
|
||||
+++ b/Lib/urlparse.py
|
||||
@@ -165,6 +165,21 @@ def _splitnetloc(url, start=0):
|
||||
delim = min(delim, wdelim) # use earliest delim position
|
||||
return url[start:delim], url[delim:] # return (domain, rest)
|
||||
|
||||
+def _checknetloc(netloc):
|
||||
+ if not netloc or not isinstance(netloc, unicode):
|
||||
+ return
|
||||
+ # looking for characters like \u2100 that expand to 'a/c'
|
||||
+ # IDNA uses NFKC equivalence, so normalize for this check
|
||||
+ import unicodedata
|
||||
+ netloc2 = unicodedata.normalize('NFKC', netloc)
|
||||
+ if netloc == netloc2:
|
||||
+ return
|
||||
+ _, _, netloc = netloc.rpartition('@') # anything to the left of '@' is okay
|
||||
+ for c in '/?#@:':
|
||||
+ if c in netloc2:
|
||||
+ raise ValueError("netloc '" + netloc2 + "' contains invalid " +
|
||||
+ "characters under NFKC normalization")
|
||||
+
|
||||
def urlsplit(url, scheme='', allow_fragments=True):
|
||||
"""Parse a URL into 5 components:
|
||||
<scheme>://<netloc>/<path>?<query>#<fragment>
|
||||
@@ -193,6 +208,7 @@ def urlsplit(url, scheme='', allow_fragments=True):
|
||||
url, fragment = url.split('#', 1)
|
||||
if '?' in url:
|
||||
url, query = url.split('?', 1)
|
||||
+ _checknetloc(netloc)
|
||||
v = SplitResult(scheme, netloc, url, query, fragment)
|
||||
_parse_cache[key] = v
|
||||
return v
|
||||
@@ -216,6 +232,7 @@ def urlsplit(url, scheme='', allow_fragments=True):
|
||||
url, fragment = url.split('#', 1)
|
||||
if '?' in url:
|
||||
url, query = url.split('?', 1)
|
||||
+ _checknetloc(netloc)
|
||||
v = SplitResult(scheme, netloc, url, query, fragment)
|
||||
_parse_cache[key] = v
|
||||
return v
|
||||
diff --git a/Misc/NEWS.d/next/Security/2019-03-06-09-38-40.bpo-36216.6q1m4a.rst b/Misc/NEWS.d/next/Security/2019-03-06-09-38-40.bpo-36216.6q1m4a.rst
|
||||
new file mode 100644
|
||||
index 000000000000..1e1ad92c6feb
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2019-03-06-09-38-40.bpo-36216.6q1m4a.rst
|
||||
@@ -0,0 +1,3 @@
|
||||
+Changes urlsplit() to raise ValueError when the URL contains characters that
|
||||
+decompose under IDNA encoding (NFKC-normalization) into characters that
|
||||
+affect how the URL is parsed.
|
||||
\ No newline at end of file
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
From 06b5ee585d6e76bdbb4002f642d864d860cbbd2b Mon Sep 17 00:00:00 2001
|
||||
From: Steve Dower <steve.dower@python.org>
|
||||
Date: Tue, 12 Mar 2019 08:23:33 -0700
|
||||
Subject: [PATCH] bpo-36216: Only print test messages when verbose
|
||||
|
||||
---
|
||||
Lib/test/test_urlparse.py | 3 ++-
|
||||
1 file changed, 2 insertions(+), 1 deletion(-)
|
||||
|
||||
diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py
|
||||
index 73b0228ea8e3..1830d0b28688 100644
|
||||
--- a/Lib/test/test_urlparse.py
|
||||
+++ b/Lib/test/test_urlparse.py
|
||||
@@ -644,7 +644,8 @@ def test_urlsplit_normalization(self):
|
||||
for scheme in [u"http", u"https", u"ftp"]:
|
||||
for c in denorm_chars:
|
||||
url = u"{}://netloc{}false.netloc/path".format(scheme, c)
|
||||
- print "Checking %r" % url
|
||||
+ if test_support.verbose:
|
||||
+ print "Checking %r" % url
|
||||
with self.assertRaises(ValueError):
|
||||
urlparse.urlsplit(url)
|
||||
|
||||
|
|
@ -14,7 +14,7 @@ PYTHON_VERSION:=$(PYTHON3_VERSION)
|
|||
PYTHON_VERSION_MICRO:=$(PYTHON3_VERSION_MICRO)
|
||||
|
||||
PKG_NAME:=python3
|
||||
PKG_RELEASE:=4
|
||||
PKG_RELEASE:=5
|
||||
PKG_VERSION:=$(PYTHON_VERSION).$(PYTHON_VERSION_MICRO)
|
||||
|
||||
PKG_SOURCE:=Python-$(PKG_VERSION).tar.xz
|
||||
|
|
|
|||
|
|
@ -0,0 +1,173 @@
|
|||
From 8f5285ac12bd185202fdaac56fd19dfb858ec947 Mon Sep 17 00:00:00 2001
|
||||
From: Steve Dower <steve.dower@python.org>
|
||||
Date: Thu, 7 Mar 2019 08:09:56 -0800
|
||||
Subject: [PATCH 1/2] bpo-36216: Add check for characters in netloc that
|
||||
normalize to separators (GH-12201)
|
||||
|
||||
---
|
||||
Doc/library/urllib.parse.rst | 18 +++++++++++++++
|
||||
Lib/test/test_urlparse.py | 23 +++++++++++++++++++
|
||||
Lib/urllib/parse.py | 17 ++++++++++++++
|
||||
.../2019-03-06-09-38-40.bpo-36216.6q1m4a.rst | 3 +++
|
||||
4 files changed, 61 insertions(+)
|
||||
create mode 100644 Misc/NEWS.d/next/Security/2019-03-06-09-38-40.bpo-36216.6q1m4a.rst
|
||||
|
||||
diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst
|
||||
index d991254d5ca1..647af613a315 100644
|
||||
--- a/Doc/library/urllib.parse.rst
|
||||
+++ b/Doc/library/urllib.parse.rst
|
||||
@@ -121,6 +121,11 @@ or on combining URL components into a URL string.
|
||||
Unmatched square brackets in the :attr:`netloc` attribute will raise a
|
||||
:exc:`ValueError`.
|
||||
|
||||
+ Characters in the :attr:`netloc` attribute that decompose under NFKC
|
||||
+ normalization (as used by the IDNA encoding) into any of ``/``, ``?``,
|
||||
+ ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is
|
||||
+ decomposed before parsing, no error will be raised.
|
||||
+
|
||||
.. versionchanged:: 3.2
|
||||
Added IPv6 URL parsing capabilities.
|
||||
|
||||
@@ -133,6 +138,10 @@ or on combining URL components into a URL string.
|
||||
Out-of-range port numbers now raise :exc:`ValueError`, instead of
|
||||
returning :const:`None`.
|
||||
|
||||
+ .. versionchanged:: 3.6.9
|
||||
+ Characters that affect netloc parsing under NFKC normalization will
|
||||
+ now raise :exc:`ValueError`.
|
||||
+
|
||||
|
||||
.. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None)
|
||||
|
||||
@@ -256,10 +265,19 @@ or on combining URL components into a URL string.
|
||||
Unmatched square brackets in the :attr:`netloc` attribute will raise a
|
||||
:exc:`ValueError`.
|
||||
|
||||
+ Characters in the :attr:`netloc` attribute that decompose under NFKC
|
||||
+ normalization (as used by the IDNA encoding) into any of ``/``, ``?``,
|
||||
+ ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is
|
||||
+ decomposed before parsing, no error will be raised.
|
||||
+
|
||||
.. versionchanged:: 3.6
|
||||
Out-of-range port numbers now raise :exc:`ValueError`, instead of
|
||||
returning :const:`None`.
|
||||
|
||||
+ .. versionchanged:: 3.6.9
|
||||
+ Characters that affect netloc parsing under NFKC normalization will
|
||||
+ now raise :exc:`ValueError`.
|
||||
+
|
||||
|
||||
.. function:: urlunsplit(parts)
|
||||
|
||||
diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py
|
||||
index be50b47603aa..e6638aee2244 100644
|
||||
--- a/Lib/test/test_urlparse.py
|
||||
+++ b/Lib/test/test_urlparse.py
|
||||
@@ -1,3 +1,5 @@
|
||||
+import sys
|
||||
+import unicodedata
|
||||
import unittest
|
||||
import urllib.parse
|
||||
|
||||
@@ -984,6 +986,27 @@ def test_all(self):
|
||||
expected.append(name)
|
||||
self.assertCountEqual(urllib.parse.__all__, expected)
|
||||
|
||||
+ def test_urlsplit_normalization(self):
|
||||
+ # Certain characters should never occur in the netloc,
|
||||
+ # including under normalization.
|
||||
+ # Ensure that ALL of them are detected and cause an error
|
||||
+ illegal_chars = '/:#?@'
|
||||
+ hex_chars = {'{:04X}'.format(ord(c)) for c in illegal_chars}
|
||||
+ denorm_chars = [
|
||||
+ c for c in map(chr, range(128, sys.maxunicode))
|
||||
+ if (hex_chars & set(unicodedata.decomposition(c).split()))
|
||||
+ and c not in illegal_chars
|
||||
+ ]
|
||||
+ # Sanity check that we found at least one such character
|
||||
+ self.assertIn('\u2100', denorm_chars)
|
||||
+ self.assertIn('\uFF03', denorm_chars)
|
||||
+
|
||||
+ for scheme in ["http", "https", "ftp"]:
|
||||
+ for c in denorm_chars:
|
||||
+ url = "{}://netloc{}false.netloc/path".format(scheme, c)
|
||||
+ with self.subTest(url=url, char='{:04X}'.format(ord(c))):
|
||||
+ with self.assertRaises(ValueError):
|
||||
+ urllib.parse.urlsplit(url)
|
||||
|
||||
class Utility_Tests(unittest.TestCase):
|
||||
"""Testcase to test the various utility functions in the urllib."""
|
||||
diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py
|
||||
index 85e68c8b42c7..24a628915e53 100644
|
||||
--- a/Lib/urllib/parse.py
|
||||
+++ b/Lib/urllib/parse.py
|
||||
@@ -391,6 +391,21 @@ def _splitnetloc(url, start=0):
|
||||
delim = min(delim, wdelim) # use earliest delim position
|
||||
return url[start:delim], url[delim:] # return (domain, rest)
|
||||
|
||||
+def _checknetloc(netloc):
|
||||
+ if not netloc or netloc.isascii():
|
||||
+ return
|
||||
+ # looking for characters like \u2100 that expand to 'a/c'
|
||||
+ # IDNA uses NFKC equivalence, so normalize for this check
|
||||
+ import unicodedata
|
||||
+ netloc2 = unicodedata.normalize('NFKC', netloc)
|
||||
+ if netloc == netloc2:
|
||||
+ return
|
||||
+ _, _, netloc = netloc.rpartition('@') # anything to the left of '@' is okay
|
||||
+ for c in '/?#@:':
|
||||
+ if c in netloc2:
|
||||
+ raise ValueError("netloc '" + netloc2 + "' contains invalid " +
|
||||
+ "characters under NFKC normalization")
|
||||
+
|
||||
def urlsplit(url, scheme='', allow_fragments=True):
|
||||
"""Parse a URL into 5 components:
|
||||
<scheme>://<netloc>/<path>?<query>#<fragment>
|
||||
@@ -420,6 +435,7 @@ def urlsplit(url, scheme='', allow_fragments=True):
|
||||
url, fragment = url.split('#', 1)
|
||||
if '?' in url:
|
||||
url, query = url.split('?', 1)
|
||||
+ _checknetloc(netloc)
|
||||
v = SplitResult(scheme, netloc, url, query, fragment)
|
||||
_parse_cache[key] = v
|
||||
return _coerce_result(v)
|
||||
@@ -443,6 +459,7 @@ def urlsplit(url, scheme='', allow_fragments=True):
|
||||
url, fragment = url.split('#', 1)
|
||||
if '?' in url:
|
||||
url, query = url.split('?', 1)
|
||||
+ _checknetloc(netloc)
|
||||
v = SplitResult(scheme, netloc, url, query, fragment)
|
||||
_parse_cache[key] = v
|
||||
return _coerce_result(v)
|
||||
diff --git a/Misc/NEWS.d/next/Security/2019-03-06-09-38-40.bpo-36216.6q1m4a.rst b/Misc/NEWS.d/next/Security/2019-03-06-09-38-40.bpo-36216.6q1m4a.rst
|
||||
new file mode 100644
|
||||
index 000000000000..5546394157f9
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2019-03-06-09-38-40.bpo-36216.6q1m4a.rst
|
||||
@@ -0,0 +1,3 @@
|
||||
+Changes urlsplit() to raise ValueError when the URL contains characters that
|
||||
+decompose under IDNA encoding (NFKC-normalization) into characters that
|
||||
+affect how the URL is parsed.
|
||||
|
||||
From 379c71551a3f005bbedccc3758568918612c1765 Mon Sep 17 00:00:00 2001
|
||||
From: Steve Dower <steve.dower@python.org>
|
||||
Date: Thu, 7 Mar 2019 08:28:39 -0800
|
||||
Subject: [PATCH 2/2] Replace isascii call with any() call
|
||||
|
||||
---
|
||||
Lib/urllib/parse.py | 2 +-
|
||||
1 file changed, 1 insertion(+), 1 deletion(-)
|
||||
|
||||
diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py
|
||||
index 24a628915e53..7b06f4d71d67 100644
|
||||
--- a/Lib/urllib/parse.py
|
||||
+++ b/Lib/urllib/parse.py
|
||||
@@ -392,7 +392,7 @@ def _splitnetloc(url, start=0):
|
||||
return url[start:delim], url[delim:] # return (domain, rest)
|
||||
|
||||
def _checknetloc(netloc):
|
||||
- if not netloc or netloc.isascii():
|
||||
+ if not netloc or not any(ord(c) > 127 for c in netloc):
|
||||
return
|
||||
# looking for characters like \u2100 that expand to 'a/c'
|
||||
# IDNA uses NFKC equivalence, so normalize for this check
|
||||
Loading…
Reference in a new issue