diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index e2b6f133e..f0d9d4d80 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -77,6 +77,9 @@ scheme_chars = ('abcdefghijklmnopqrstuvwxyz' '0123456789' '+-.') +# Unsafe bytes to be removed per WHATWG spec +_UNSAFE_URL_BYTES_TO_REMOVE = ['\t', '\r', '\n'] + # XXX: Consider replacing with functools.lru_cache MAX_CACHE_SIZE = 20 _parse_cache = {} @@ -414,6 +417,11 @@ def _checknetloc(netloc): raise ValueError("netloc '" + netloc + "' contains invalid " + "characters under NFKC normalization") +def _remove_unsafe_bytes_from_url(url): + for b in _UNSAFE_URL_BYTES_TO_REMOVE: + url = url.replace(b, "") + return url + def urlsplit(url, scheme='', allow_fragments=True): """Parse a URL into 5 components: :///?# @@ -421,6 +429,8 @@ def urlsplit(url, scheme='', allow_fragments=True): Note that we don't break the components up in smaller bits (e.g. netloc is a single string) and we don't expand % escapes.""" url, scheme, _coerce_result = _coerce_args(url, scheme) + url = _remove_unsafe_bytes_from_url(url) + scheme = _remove_unsafe_bytes_from_url(scheme) allow_fragments = bool(allow_fragments) key = url, scheme, allow_fragments, type(url), type(scheme) cached = _parse_cache.get(key, None) @@ -631,6 +641,8 @@ def unquote(string, encoding='utf-8', errors='replace'): unquote('abc%20def') -> 'abc def'. """ + if isinstance(string, bytes): + raise TypeError('Expected str, got bytes') if '%' not in string: string.split return string @@ -648,7 +660,7 @@ def unquote(string, encoding='utf-8', errors='replace'): def parse_qs(qs, keep_blank_values=False, strict_parsing=False, - encoding='utf-8', errors='replace', max_num_fields=None): + encoding='utf-8', errors='replace', max_num_fields=None, separator='&'): """Parse a query given as a string argument. Arguments: @@ -672,12 +684,15 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, max_num_fields: int. If set, then throws a ValueError if there are more than n fields read by parse_qsl(). + separator: str. The symbol to use for separating the query arguments. + Defaults to &. + Returns a dictionary. """ parsed_result = {} pairs = parse_qsl(qs, keep_blank_values, strict_parsing, encoding=encoding, errors=errors, - max_num_fields=max_num_fields) + max_num_fields=max_num_fields, separator=separator) for name, value in pairs: if name in parsed_result: parsed_result[name].append(value) @@ -687,7 +702,7 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, - encoding='utf-8', errors='replace', max_num_fields=None): + encoding='utf-8', errors='replace', max_num_fields=None, separator='&'): """Parse a query given as a string argument. Arguments: @@ -710,19 +725,26 @@ def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, max_num_fields: int. If set, then throws a ValueError if there are more than n fields read by parse_qsl(). + separator: str. The symbol to use for separating the query arguments. + Defaults to &. + Returns a list, as G-d intended. """ qs, _coerce_result = _coerce_args(qs) + separator, _ = _coerce_args(separator) + + if not separator or (not isinstance(separator, (str, bytes))): + raise ValueError("Separator must be of type string or bytes.") # If max_num_fields is defined then check that the number of fields # is less than max_num_fields. This prevents a memory exhaustion DOS # attack via post bodies with many fields. if max_num_fields is not None: - num_fields = 1 + qs.count('&') + qs.count(';') + num_fields = 1 + qs.count(separator) if max_num_fields < num_fields: raise ValueError('Max number of fields exceeded') - pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')] + pairs = [s1 for s1 in qs.split(separator)] r = [] for name_value in pairs: if not name_value and not strict_parsing: diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index e44073886..5f67077fb 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -779,7 +779,11 @@ def _parse_proxy(proxy): raise ValueError("proxy URL with no authority: %r" % proxy) # We have an authority, so for RFC 3986-compliant URLs (by ss 3. # and 3.3.), path is empty or starts with '/' - end = r_scheme.find("/", 2) + if '@' in r_scheme: + host_separator = r_scheme.find('@') + end = r_scheme.find("/", host_separator) + else: + end = r_scheme.find("/", 2) if end == -1: end = None authority = r_scheme[2:end] @@ -947,7 +951,7 @@ class AbstractBasicAuthHandler: # (single quotes are a violation of the RFC, but appear in the wild) rx = re.compile('(?:^|,)' # start of the string or ',' '[ \t]*' # optional whitespaces - '([^ \t]+)' # scheme like "Basic" + '([^ \t,]+)' # scheme like "Basic" '[ \t]+' # mandatory whitespaces # realm=xxx # realm='xxx' @@ -2604,6 +2608,11 @@ def _proxy_bypass_macosx_sysconf(host, proxy_settings): mask = 8 * (m.group(1).count('.') + 1) else: mask = int(mask[1:]) + + if mask < 0 or mask > 32: + # System libraries ignore invalid prefix lengths + continue + mask = 32 - mask if (hostIP >> mask) == (base >> mask):