diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index ffff57969..e2957dcac 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -9,7 +9,8 @@ import io import unittest from unittest.mock import patch from test import support -from test.support import os_helper, warnings_helper +from test.support import os_helper +from test.support import warnings_helper import os try: import ssl diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index f0d9d4d80..b35997bc0 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -29,6 +29,7 @@ test_urlparse.py provides a good indicator of parsing behavior. import re import sys +import types import collections import warnings @@ -179,6 +180,8 @@ class _NetlocResultMixinBase(object): raise ValueError("Port out of range 0-65535") return port + __class_getitem__ = classmethod(types.GenericAlias) + class _NetlocResultMixinStr(_NetlocResultMixinBase, _ResultMixinStr): __slots__ = () @@ -369,9 +372,23 @@ del _fix_result_transcoding def urlparse(url, scheme='', allow_fragments=True): """Parse a URL into 6 components: :///;?# - Return a 6-tuple: (scheme, netloc, path, params, query, fragment). - Note that we don't break the components up in smaller bits - (e.g. netloc is a single string) and we don't expand % escapes.""" + + The result is a named 6-tuple with fields corresponding to the + above. It is either a ParseResult or ParseResultBytes object, + depending on the type of the url parameter. + + The username, password, hostname, and port sub-components of netloc + can also be accessed as attributes of the returned object. + + The scheme argument provides the default value of the scheme + component when no scheme is found in url. + + If allow_fragments is False, no attempt is made to separate the + fragment component from the previous component, which can be either + path or query. + + Note that % escapes are not expanded. + """ url, scheme, _coerce_result = _coerce_args(url, scheme) splitresult = urlsplit(url, scheme, allow_fragments) scheme, netloc, url, query, fragment = splitresult @@ -417,20 +434,33 @@ def _checknetloc(netloc): raise ValueError("netloc '" + netloc + "' contains invalid " + "characters under NFKC normalization") -def _remove_unsafe_bytes_from_url(url): - for b in _UNSAFE_URL_BYTES_TO_REMOVE: - url = url.replace(b, "") - return url - def urlsplit(url, scheme='', allow_fragments=True): """Parse a URL into 5 components: :///?# - Return a 5-tuple: (scheme, netloc, path, query, fragment). - Note that we don't break the components up in smaller bits - (e.g. netloc is a single string) and we don't expand % escapes.""" + + The result is a named 5-tuple with fields corresponding to the + above. It is either a SplitResult or SplitResultBytes object, + depending on the type of the url parameter. + + The username, password, hostname, and port sub-components of netloc + can also be accessed as attributes of the returned object. + + The scheme argument provides the default value of the scheme + component when no scheme is found in url. + + If allow_fragments is False, no attempt is made to separate the + fragment component from the previous component, which can be either + path or query. + + Note that % escapes are not expanded. + """ + url, scheme, _coerce_result = _coerce_args(url, scheme) - url = _remove_unsafe_bytes_from_url(url) - scheme = _remove_unsafe_bytes_from_url(scheme) + + for b in _UNSAFE_URL_BYTES_TO_REMOVE: + url = url.replace(b, "") + scheme = scheme.replace(b, "") + allow_fragments = bool(allow_fragments) key = url, scheme, allow_fragments, type(url), type(scheme) cached = _parse_cache.get(key, None) @@ -441,31 +471,11 @@ def urlsplit(url, scheme='', allow_fragments=True): netloc = query = fragment = '' i = url.find(':') if i > 0: - if url[:i] == 'http': # optimize the common case - url = url[i+1:] - if url[:2] == '//': - netloc, url = _splitnetloc(url, 2) - if (('[' in netloc and ']' not in netloc) or - (']' in netloc and '[' not in netloc)): - raise ValueError("Invalid IPv6 URL") - if allow_fragments and '#' in url: - url, fragment = url.split('#', 1) - if '?' in url: - url, query = url.split('?', 1) - _checknetloc(netloc) - v = SplitResult('http', netloc, url, query, fragment) - _parse_cache[key] = v - return _coerce_result(v) for c in url[:i]: if c not in scheme_chars: break else: - # make sure "url" is not actually a port number (in which case - # "scheme" is really part of the path) - rest = url[i+1:] - if not rest or any(c not in '0123456789' for c in rest): - # not a port number - scheme, url = url[:i].lower(), rest + scheme, url = url[:i].lower(), url[i+1:] if url[:2] == '//': netloc, url = _splitnetloc(url, 2) @@ -642,7 +652,7 @@ def unquote(string, encoding='utf-8', errors='replace'): unquote('abc%20def') -> 'abc def'. """ if isinstance(string, bytes): - raise TypeError('Expected str, got bytes') + return unquote_to_bytes(string).decode(encoding, errors) if '%' not in string: string.split return string @@ -744,9 +754,8 @@ def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, if max_num_fields < num_fields: raise ValueError('Max number of fields exceeded') - pairs = [s1 for s1 in qs.split(separator)] r = [] - for name_value in pairs: + for name_value in qs.split(separator): if not name_value and not strict_parsing: continue nv = name_value.split('=', 1) diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 5f67077fb..a0ef60b30 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -64,7 +64,7 @@ opener = urllib.request.build_opener(proxy_support, authinfo, # install it urllib.request.install_opener(opener) -f = urllib.request.urlopen('http://www.python.org/') +f = urllib.request.urlopen('https://www.python.org/') """ # XXX issues: @@ -163,18 +163,10 @@ def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, The *cadefault* parameter is ignored. - This function always returns an object which can work as a context - manager and has methods such as - * geturl() - return the URL of the resource retrieved, commonly used to - determine if a redirect was followed - - * info() - return the meta-information of the page, such as headers, in the - form of an email.message_from_string() instance (see Quick Reference to - HTTP Headers) - - * getcode() - return the HTTP status code of the response. Raises URLError - on errors. + This function always returns an object which can work as a + context manager and has the properties url, headers, and status. + See urllib.response.addinfourl for more detail on these properties. For HTTP and HTTPS URLs, this function returns a http.client.HTTPResponse object slightly modified. In addition to the three new methods above, the @@ -210,6 +202,8 @@ def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=cafile, capath=capath) + # send ALPN extension to indicate HTTP/1.1 protocol + context.set_alpn_protocols(['http/1.1']) https_handler = HTTPSHandler(context=context) opener = build_opener(https_handler) elif context: @@ -895,10 +889,10 @@ class HTTPPasswordMgr: return True if base[0] != test[0]: return False - common = posixpath.commonprefix((base[1], test[1])) - if len(common) == len(base[1]): - return True - return False + prefix = base[1] + if prefix[-1:] != '/': + prefix += '/' + return test[1].startswith(prefix) class HTTPPasswordMgrWithDefaultRealm(HTTPPasswordMgr): @@ -1823,7 +1817,7 @@ class URLopener: hdrs = fp.info() fp.close() return url2pathname(_splithost(url1)[1]), hdrs - except OSError as msg: + except OSError: pass fp = self.open(url, data) try: @@ -2680,22 +2674,26 @@ elif os.name == 'nt': # Returned as Unicode but problems if not converted to ASCII proxyServer = str(winreg.QueryValueEx(internetSettings, 'ProxyServer')[0]) - if '=' in proxyServer: - # Per-protocol settings - for p in proxyServer.split(';'): - protocol, address = p.split('=', 1) - # See if address has a type:// prefix - if not re.match('(?:[^/:]+)://', address): - address = '%s://%s' % (protocol, address) - proxies[protocol] = address - else: - # Use one setting for all protocols - if proxyServer[:5] == 'http:': - proxies['http'] = proxyServer - else: - proxies['http'] = 'http://%s' % proxyServer - proxies['https'] = 'https://%s' % proxyServer - proxies['ftp'] = 'ftp://%s' % proxyServer + if '=' not in proxyServer and ';' not in proxyServer: + # Use one setting for all protocols. + proxyServer = 'http={0};https={0};ftp={0}'.format(proxyServer) + for p in proxyServer.split(';'): + protocol, address = p.split('=', 1) + # See if address has a type:// prefix + if not re.match('(?:[^/:]+)://', address): + # Add type:// prefix to address without specifying type + if protocol in ('http', 'https', 'ftp'): + # The default proxy type of Windows is HTTP + address = 'http://' + address + elif protocol == 'socks': + address = 'socks://' + address + proxies[protocol] = address + # Use SOCKS proxy for HTTP(S) protocols + if proxies.get('socks'): + # The default SOCKS proxy type of Windows is SOCKS4 + address = re.sub(r'^socks://', 'socks4://', proxies['socks']) + proxies['http'] = proxies.get('http') or address + proxies['https'] = proxies.get('https') or address internetSettings.Close() except (OSError, ValueError, TypeError): # Either registry key not found etc, or the value in an diff --git a/Lib/urllib/response.py b/Lib/urllib/response.py index 4778118db..5a2c3cc78 100644 --- a/Lib/urllib/response.py +++ b/Lib/urllib/response.py @@ -73,6 +73,10 @@ class addinfourl(addinfo): self.url = url self.code = code + @property + def status(self): + return self.code + def getcode(self): return self.code