From cfc7a7c7343506bebd7bb5908fbb2995c9b851c5 Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Sun, 19 Jul 2020 13:53:04 +1200 Subject: [PATCH] Copy updated urllib from CPython 3.8.3. --- Lib/urllib/error.py | 6 +- Lib/urllib/parse.py | 268 ++++++++++++++++++++++++++++++-------- Lib/urllib/request.py | 248 +++++++++++++++++++---------------- Lib/urllib/robotparser.py | 46 +++++-- 4 files changed, 386 insertions(+), 182 deletions(-) diff --git a/Lib/urllib/error.py b/Lib/urllib/error.py index c5b675d16..8cd901f13 100644 --- a/Lib/urllib/error.py +++ b/Lib/urllib/error.py @@ -16,14 +16,10 @@ import urllib.response __all__ = ['URLError', 'HTTPError', 'ContentTooShortError'] -# do these error classes make sense? -# make sure all of the OSError stuff is overridden. we just want to be -# subtypes. - class URLError(OSError): # URLError is a sub-type of OSError, but it doesn't share any of # the implementation. need to override __init__ and __str__. - # It sets self.args for compatibility with other EnvironmentError + # It sets self.args for compatibility with other OSError # subclasses, but args doesn't have the typical format with errno in # slot 0 and strerror in slot 1. This may be better than nothing. def __init__(self, reason, filename=None): diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 958767a08..e2b6f133e 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -30,6 +30,7 @@ test_urlparse.py provides a good indicator of parsing behavior. import re import sys import collections +import warnings __all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag", "urlsplit", "urlunsplit", "urlencode", "parse_qs", @@ -38,29 +39,37 @@ __all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag", "DefragResult", "ParseResult", "SplitResult", "DefragResultBytes", "ParseResultBytes", "SplitResultBytes"] -# A classification of schemes ('' means apply by default) -uses_relative = ['ftp', 'http', 'gopher', 'nntp', 'imap', +# A classification of schemes. +# The empty string classifies URLs with no scheme specified, +# being the default value returned by “urlsplit” and “urlparse”. + +uses_relative = ['', 'ftp', 'http', 'gopher', 'nntp', 'imap', 'wais', 'file', 'https', 'shttp', 'mms', - 'prospero', 'rtsp', 'rtspu', '', 'sftp', + 'prospero', 'rtsp', 'rtspu', 'sftp', 'svn', 'svn+ssh', 'ws', 'wss'] -uses_netloc = ['ftp', 'http', 'gopher', 'nntp', 'telnet', + +uses_netloc = ['', 'ftp', 'http', 'gopher', 'nntp', 'telnet', 'imap', 'wais', 'file', 'mms', 'https', 'shttp', - 'snews', 'prospero', 'rtsp', 'rtspu', 'rsync', '', + 'snews', 'prospero', 'rtsp', 'rtspu', 'rsync', 'svn', 'svn+ssh', 'sftp', 'nfs', 'git', 'git+ssh', 'ws', 'wss'] -uses_params = ['ftp', 'hdl', 'prospero', 'http', 'imap', + +uses_params = ['', 'ftp', 'hdl', 'prospero', 'http', 'imap', 'https', 'shttp', 'rtsp', 'rtspu', 'sip', 'sips', - 'mms', '', 'sftp', 'tel'] + 'mms', 'sftp', 'tel'] # These are not actually used anymore, but should stay for backwards # compatibility. (They are undocumented, but have a public-looking name.) + non_hierarchical = ['gopher', 'hdl', 'mailto', 'news', 'telnet', 'wais', 'imap', 'snews', 'sip', 'sips'] -uses_query = ['http', 'wais', 'imap', 'https', 'shttp', 'mms', - 'gopher', 'rtsp', 'rtspu', 'sip', 'sips', ''] -uses_fragment = ['ftp', 'hdl', 'http', 'gopher', 'news', + +uses_query = ['', 'http', 'wais', 'imap', 'https', 'shttp', 'mms', + 'gopher', 'rtsp', 'rtspu', 'sip', 'sips'] + +uses_fragment = ['', 'ftp', 'hdl', 'http', 'gopher', 'news', 'nntp', 'wais', 'https', 'shttp', 'snews', - 'file', 'prospero', ''] + 'file', 'prospero'] # Characters valid in scheme names scheme_chars = ('abcdefghijklmnopqrstuvwxyz' @@ -147,16 +156,22 @@ class _NetlocResultMixinBase(object): def hostname(self): hostname = self._hostinfo[0] if not hostname: - hostname = None - elif hostname is not None: - hostname = hostname.lower() - return hostname + return None + # Scoped IPv6 address may have zone info, which must not be lowercased + # like http://[fe80::822a:a8ff:fe49:470c%tESt]:1234/keys + separator = '%' if isinstance(hostname, str) else b'%' + hostname, percent, zone = hostname.partition(separator) + return hostname.lower() + percent + zone @property def port(self): port = self._hostinfo[1] if port is not None: - port = int(port, 10) + try: + port = int(port, 10) + except ValueError: + message = f'Port could not be cast to integer value as {port!r}' + raise ValueError(message) from None if not ( 0 <= port <= 65535): raise ValueError("Port out of range 0-65535") return port @@ -274,7 +289,7 @@ by reference to a primary resource and additional identifying information. """ _ParseResultBase.__doc__ = """ -ParseResult(scheme, netloc, path, params, query, fragment) +ParseResult(scheme, netloc, path, params, query, fragment) A 6-tuple that contains components of a parsed URL. """ @@ -381,6 +396,24 @@ def _splitnetloc(url, start=0): delim = min(delim, wdelim) # use earliest delim position return url[start:delim], url[delim:] # return (domain, rest) +def _checknetloc(netloc): + if not netloc or netloc.isascii(): + return + # looking for characters like \u2100 that expand to 'a/c' + # IDNA uses NFKC equivalence, so normalize for this check + import unicodedata + n = netloc.replace('@', '') # ignore characters already included + n = n.replace(':', '') # but not the surrounding text + n = n.replace('#', '') + n = n.replace('?', '') + netloc2 = unicodedata.normalize('NFKC', n) + if n == netloc2: + return + for c in '/?#@:': + if c in netloc2: + raise ValueError("netloc '" + netloc + "' contains invalid " + + "characters under NFKC normalization") + def urlsplit(url, scheme='', allow_fragments=True): """Parse a URL into 5 components: :///?# @@ -399,7 +432,6 @@ def urlsplit(url, scheme='', allow_fragments=True): i = url.find(':') if i > 0: if url[:i] == 'http': # optimize the common case - scheme = url[:i].lower() url = url[i+1:] if url[:2] == '//': netloc, url = _splitnetloc(url, 2) @@ -410,7 +442,8 @@ def urlsplit(url, scheme='', allow_fragments=True): url, fragment = url.split('#', 1) if '?' in url: url, query = url.split('?', 1) - v = SplitResult(scheme, netloc, url, query, fragment) + _checknetloc(netloc) + v = SplitResult('http', netloc, url, query, fragment) _parse_cache[key] = v return _coerce_result(v) for c in url[:i]: @@ -433,6 +466,7 @@ def urlsplit(url, scheme='', allow_fragments=True): url, fragment = url.split('#', 1) if '?' in url: url, query = url.split('?', 1) + _checknetloc(netloc) v = SplitResult(scheme, netloc, url, query, fragment) _parse_cache[key] = v return _coerce_result(v) @@ -574,7 +608,7 @@ def unquote_to_bytes(string): # if the function is never called global _hextobyte if _hextobyte is None: - _hextobyte = {(a + b).encode(): bytes([int(a + b, 16)]) + _hextobyte = {(a + b).encode(): bytes.fromhex(a + b) for a in _hexdig for b in _hexdig} for item in bits[1:]: try: @@ -612,8 +646,9 @@ def unquote(string, encoding='utf-8', errors='replace'): append(bits[i + 1]) return ''.join(res) + def parse_qs(qs, keep_blank_values=False, strict_parsing=False, - encoding='utf-8', errors='replace'): + encoding='utf-8', errors='replace', max_num_fields=None): """Parse a query given as a string argument. Arguments: @@ -633,10 +668,16 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding and errors: specify how to decode percent-encoded sequences into Unicode characters, as accepted by the bytes.decode() method. + + max_num_fields: int. If set, then throws a ValueError if there + are more than n fields read by parse_qsl(). + + Returns a dictionary. """ parsed_result = {} pairs = parse_qsl(qs, keep_blank_values, strict_parsing, - encoding=encoding, errors=errors) + encoding=encoding, errors=errors, + max_num_fields=max_num_fields) for name, value in pairs: if name in parsed_result: parsed_result[name].append(value) @@ -644,30 +685,43 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, parsed_result[name] = [value] return parsed_result + def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, - encoding='utf-8', errors='replace'): + encoding='utf-8', errors='replace', max_num_fields=None): """Parse a query given as a string argument. - Arguments: + Arguments: - qs: percent-encoded query string to be parsed + qs: percent-encoded query string to be parsed - keep_blank_values: flag indicating whether blank values in - percent-encoded queries should be treated as blank strings. A - true value indicates that blanks should be retained as blank - strings. The default false value indicates that blank values - are to be ignored and treated as if they were not included. + keep_blank_values: flag indicating whether blank values in + percent-encoded queries should be treated as blank strings. + A true value indicates that blanks should be retained as blank + strings. The default false value indicates that blank values + are to be ignored and treated as if they were not included. - strict_parsing: flag indicating what to do with parsing errors. If - false (the default), errors are silently ignored. If true, - errors raise a ValueError exception. + strict_parsing: flag indicating what to do with parsing errors. If + false (the default), errors are silently ignored. If true, + errors raise a ValueError exception. - encoding and errors: specify how to decode percent-encoded sequences - into Unicode characters, as accepted by the bytes.decode() method. + encoding and errors: specify how to decode percent-encoded sequences + into Unicode characters, as accepted by the bytes.decode() method. - Returns a list, as G-d intended. + max_num_fields: int. If set, then throws a ValueError + if there are more than n fields read by parse_qsl(). + + Returns a list, as G-d intended. """ qs, _coerce_result = _coerce_args(qs) + + # If max_num_fields is defined then check that the number of fields + # is less than max_num_fields. This prevents a memory exhaustion DOS + # attack via post bodies with many fields. + if max_num_fields is not None: + num_fields = 1 + qs.count('&') + qs.count(';') + if max_num_fields < num_fields: + raise ValueError('Max number of fields exceeded') + pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')] r = [] for name_value in pairs: @@ -704,7 +758,7 @@ def unquote_plus(string, encoding='utf-8', errors='replace'): _ALWAYS_SAFE = frozenset(b'ABCDEFGHIJKLMNOPQRSTUVWXYZ' b'abcdefghijklmnopqrstuvwxyz' b'0123456789' - b'_.-') + b'_.-~') _ALWAYS_SAFE_BYTES = bytes(_ALWAYS_SAFE) _safe_quoters = {} @@ -734,22 +788,32 @@ def quote(string, safe='/', encoding=None, errors=None): """quote('abc def') -> 'abc%20def' Each part of a URL, e.g. the path info, the query, etc., has a - different set of reserved characters that must be quoted. + different set of reserved characters that must be quoted. The + quote function offers a cautious (not minimal) way to quote a + string for most of these parts. - RFC 2396 Uniform Resource Identifiers (URI): Generic Syntax lists - the following reserved characters. + RFC 3986 Uniform Resource Identifier (URI): Generic Syntax lists + the following (un)reserved characters. - reserved = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" | - "$" | "," + unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" + reserved = gen-delims / sub-delims + gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@" + sub-delims = "!" / "$" / "&" / "'" / "(" / ")" + / "*" / "+" / "," / ";" / "=" - Each of these characters is reserved in some component of a URL, + Each of the reserved characters is reserved in some component of a URL, but not necessarily in all of them. - By default, the quote function is intended for quoting the path - section of a URL. Thus, it will not encode '/'. This character - is reserved, but in typical usage the quote function is being - called on a path where the existing slash characters are used as - reserved characters. + The quote function %-escapes all characters that are neither in the + unreserved chars ("always safe") nor the additional chars set via the + safe arg. + + The default for the safe arg is '/'. The character is reserved, but in + typical usage the quote function is being called on a path where the + existing slash characters are to be preserved. + + Python 3.7 updates from using RFC 2396 to RFC 3986 to quote URL strings. + Now, "~" is included in the set of unreserved characters. string and safe may be either str or bytes objects. encoding and errors must not be specified if string is a bytes object. @@ -893,7 +957,14 @@ def urlencode(query, doseq=False, safe='', encoding=None, errors=None, l.append(k + '=' + elt) return '&'.join(l) + def to_bytes(url): + warnings.warn("urllib.parse.to_bytes() is deprecated as of 3.8", + DeprecationWarning, stacklevel=2) + return _to_bytes(url) + + +def _to_bytes(url): """to_bytes(u"URL") --> 'URL'.""" # Most URL schemes require ASCII. If that changes, the conversion # can be relaxed. @@ -906,16 +977,29 @@ def to_bytes(url): " contains non-ASCII characters") return url + def unwrap(url): - """unwrap('') --> 'type://host/path'.""" + """Transform a string like '' into 'scheme://host/path'. + + The string is returned unchanged if it's not a wrapped URL. + """ url = str(url).strip() if url[:1] == '<' and url[-1:] == '>': url = url[1:-1].strip() - if url[:4] == 'URL:': url = url[4:].strip() + if url[:4] == 'URL:': + url = url[4:].strip() return url -_typeprog = None + def splittype(url): + warnings.warn("urllib.parse.splittype() is deprecated as of 3.8, " + "use urllib.parse.urlparse() instead", + DeprecationWarning, stacklevel=2) + return _splittype(url) + + +_typeprog = None +def _splittype(url): """splittype('type:opaquestring') --> 'type', 'opaquestring'.""" global _typeprog if _typeprog is None: @@ -927,12 +1011,20 @@ def splittype(url): return scheme.lower(), data return None, url -_hostprog = None + def splithost(url): + warnings.warn("urllib.parse.splithost() is deprecated as of 3.8, " + "use urllib.parse.urlparse() instead", + DeprecationWarning, stacklevel=2) + return _splithost(url) + + +_hostprog = None +def _splithost(url): """splithost('//host[:port]/path') --> 'host[:port]', '/path'.""" global _hostprog if _hostprog is None: - _hostprog = re.compile('//([^/?]*)(.*)', re.DOTALL) + _hostprog = re.compile('//([^/#?]*)(.*)', re.DOTALL) match = _hostprog.match(url) if match: @@ -942,32 +1034,64 @@ def splithost(url): return host_port, path return None, url + def splituser(host): + warnings.warn("urllib.parse.splituser() is deprecated as of 3.8, " + "use urllib.parse.urlparse() instead", + DeprecationWarning, stacklevel=2) + return _splituser(host) + + +def _splituser(host): """splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'.""" user, delim, host = host.rpartition('@') return (user if delim else None), host + def splitpasswd(user): + warnings.warn("urllib.parse.splitpasswd() is deprecated as of 3.8, " + "use urllib.parse.urlparse() instead", + DeprecationWarning, stacklevel=2) + return _splitpasswd(user) + + +def _splitpasswd(user): """splitpasswd('user:passwd') -> 'user', 'passwd'.""" user, delim, passwd = user.partition(':') return user, (passwd if delim else None) + +def splitport(host): + warnings.warn("urllib.parse.splitport() is deprecated as of 3.8, " + "use urllib.parse.urlparse() instead", + DeprecationWarning, stacklevel=2) + return _splitport(host) + + # splittag('/path#tag') --> '/path', 'tag' _portprog = None -def splitport(host): +def _splitport(host): """splitport('host:port') --> 'host', 'port'.""" global _portprog if _portprog is None: - _portprog = re.compile('(.*):([0-9]*)$', re.DOTALL) + _portprog = re.compile('(.*):([0-9]*)', re.DOTALL) - match = _portprog.match(host) + match = _portprog.fullmatch(host) if match: host, port = match.groups() if port: return host, port return host, None + def splitnport(host, defport=-1): + warnings.warn("urllib.parse.splitnport() is deprecated as of 3.8, " + "use urllib.parse.urlparse() instead", + DeprecationWarning, stacklevel=2) + return _splitnport(host, defport) + + +def _splitnport(host, defport=-1): """Split host and port, returning numeric port. Return given default port if no ':' found; defaults to -1. Return numerical port if a valid number are found after ':'. @@ -983,27 +1107,59 @@ def splitnport(host, defport=-1): return host, nport return host, defport + def splitquery(url): + warnings.warn("urllib.parse.splitquery() is deprecated as of 3.8, " + "use urllib.parse.urlparse() instead", + DeprecationWarning, stacklevel=2) + return _splitquery(url) + + +def _splitquery(url): """splitquery('/path?query') --> '/path', 'query'.""" path, delim, query = url.rpartition('?') if delim: return path, query return url, None + def splittag(url): + warnings.warn("urllib.parse.splittag() is deprecated as of 3.8, " + "use urllib.parse.urlparse() instead", + DeprecationWarning, stacklevel=2) + return _splittag(url) + + +def _splittag(url): """splittag('/path#tag') --> '/path', 'tag'.""" path, delim, tag = url.rpartition('#') if delim: return path, tag return url, None + def splitattr(url): + warnings.warn("urllib.parse.splitattr() is deprecated as of 3.8, " + "use urllib.parse.urlparse() instead", + DeprecationWarning, stacklevel=2) + return _splitattr(url) + + +def _splitattr(url): """splitattr('/path;attr1=value1;attr2=value2;...') -> '/path', ['attr1=value1', 'attr2=value2', ...].""" words = url.split(';') return words[0], words[1:] + def splitvalue(attr): + warnings.warn("urllib.parse.splitvalue() is deprecated as of 3.8, " + "use urllib.parse.parse_qsl() instead", + DeprecationWarning, stacklevel=2) + return _splitvalue(attr) + + +def _splitvalue(attr): """splitvalue('attr=value') --> 'attr', 'value'.""" attr, delim, value = attr.partition('=') return attr, (value if delim else None) diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 5f15b74f4..e44073886 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -94,7 +94,6 @@ import socket import string import sys import time -import collections import tempfile import contextlib import warnings @@ -103,8 +102,8 @@ import warnings from urllib.error import URLError, HTTPError, ContentTooShortError from urllib.parse import ( urlparse, urlsplit, urljoin, unwrap, quote, unquote, - splittype, splithost, splitport, splituser, splitpasswd, - splitattr, splitquery, splitvalue, splittag, to_bytes, + _splittype, _splithost, _splitport, _splituser, _splitpasswd, + _splitattr, _splitquery, _splitvalue, _splittag, _to_bytes, unquote_to_bytes, urlunparse) from urllib.response import addinfourl, addclosehook @@ -199,7 +198,7 @@ def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, global _opener if cafile or capath or cadefault: import warnings - warnings.warn("cafile, cpath and cadefault are deprecated, use a " + warnings.warn("cafile, capath and cadefault are deprecated, use a " "custom context instead.", DeprecationWarning, 2) if context is not None: raise ValueError( @@ -243,7 +242,7 @@ def urlretrieve(url, filename=None, reporthook=None, data=None): Returns a tuple containing the path to the newly created data file as well as the resulting HTTPMessage object. """ - url_type, path = splittype(url) + url_type, path = _splittype(url) with contextlib.closing(urlopen(url, data)) as fp: headers = fp.info() @@ -351,7 +350,7 @@ class Request: def full_url(self, url): # unwrap('') --> 'type://host/path' self._full_url = unwrap(url) - self._full_url, self.fragment = splittag(self._full_url) + self._full_url, self.fragment = _splittag(self._full_url) self._parse() @full_url.deleter @@ -379,10 +378,10 @@ class Request: self.data = None def _parse(self): - self.type, rest = splittype(self._full_url) + self.type, rest = _splittype(self._full_url) if self.type is None: raise ValueError("unknown url type: %r" % self.full_url) - self.host, self.selector = splithost(rest) + self.host, self.selector = _splithost(rest) if self.host: self.host = unquote(self.host) @@ -427,8 +426,7 @@ class Request: self.unredirected_hdrs.pop(header_name, None) def header_items(self): - hdrs = self.unredirected_hdrs.copy() - hdrs.update(self.headers) + hdrs = {**self.unredirected_hdrs, **self.headers} return list(hdrs.items()) class OpenerDirector: @@ -523,6 +521,7 @@ class OpenerDirector: meth = getattr(processor, meth_name) req = meth(req) + sys.audit('urllib.Request', req.full_url, req.data, req.headers, req.get_method()) response = self._open(req, data) # post-process response @@ -684,8 +683,8 @@ class HTTPRedirectHandler(BaseHandler): newurl = newurl.replace(' ', '%20') CONTENT_HEADERS = ("content-length", "content-type") - newheaders = dict((k, v) for k, v in req.headers.items() - if k.lower() not in CONTENT_HEADERS) + newheaders = {k: v for k, v in req.headers.items() + if k.lower() not in CONTENT_HEADERS} return Request(newurl, headers=newheaders, origin_req_host=req.origin_req_host, @@ -769,7 +768,7 @@ def _parse_proxy(proxy): According to RFC 3986, having an authority component means the URL must have two slashes after the scheme. """ - scheme, r_scheme = splittype(proxy) + scheme, r_scheme = _splittype(proxy) if not r_scheme.startswith("/"): # authority scheme = None @@ -784,9 +783,9 @@ def _parse_proxy(proxy): if end == -1: end = None authority = r_scheme[2:end] - userinfo, hostport = splituser(authority) + userinfo, hostport = _splituser(authority) if userinfo is not None: - user, password = splitpasswd(userinfo) + user, password = _splitpasswd(userinfo) else: user = password = None return scheme, user, password, hostport @@ -801,6 +800,7 @@ class ProxyHandler(BaseHandler): assert hasattr(proxies, 'keys'), "proxies must be a mapping" self.proxies = proxies for type, url in proxies.items(): + type = type.lower() setattr(self, '%s_open' % type, lambda r, proxy=url, type=type, meth=self.proxy_open: meth(r, proxy, type)) @@ -846,7 +846,7 @@ class HTTPPasswordMgr: self.passwd[realm] = {} for default_port in True, False: reduced_uri = tuple( - [self.reduce_uri(u, default_port) for u in uri]) + self.reduce_uri(u, default_port) for u in uri) self.passwd[realm][reduced_uri] = (user, passwd) def find_user_password(self, realm, authuri): @@ -873,7 +873,7 @@ class HTTPPasswordMgr: scheme = None authority = uri path = '/' - host, port = splitport(authority) + host, port = _splitport(authority) if default_port and port is None and scheme is not None: dport = {"http": 80, "https": 443, @@ -945,8 +945,15 @@ class AbstractBasicAuthHandler: # allow for double- and single-quoted realm values # (single quotes are a violation of the RFC, but appear in the wild) - rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+' - 'realm=(["\']?)([^"\']*)\\2', re.I) + rx = re.compile('(?:^|,)' # start of the string or ',' + '[ \t]*' # optional whitespaces + '([^ \t]+)' # scheme like "Basic" + '[ \t]+' # mandatory whitespaces + # realm=xxx + # realm='xxx' + # realm="xxx" + 'realm=(["\']?)([^"\']*)\\2', + re.I) # XXX could pre-emptively send auth info already accepted (RFC 2617, # end of section 2, and section 1.2 immediately after "credentials" @@ -958,27 +965,51 @@ class AbstractBasicAuthHandler: self.passwd = password_mgr self.add_password = self.passwd.add_password + def _parse_realm(self, header): + # parse WWW-Authenticate header: accept multiple challenges per header + found_challenge = False + for mo in AbstractBasicAuthHandler.rx.finditer(header): + scheme, quote, realm = mo.groups() + if quote not in ['"', "'"]: + warnings.warn("Basic Auth Realm was unquoted", + UserWarning, 3) + + yield (scheme, realm) + + found_challenge = True + + if not found_challenge: + if header: + scheme = header.split()[0] + else: + scheme = '' + yield (scheme, None) + def http_error_auth_reqed(self, authreq, host, req, headers): # host may be an authority (without userinfo) or a URL with an # authority - # XXX could be multiple headers - authreq = headers.get(authreq, None) + headers = headers.get_all(authreq) + if not headers: + # no header found + return - if authreq: - scheme = authreq.split()[0] - if scheme.lower() != 'basic': - raise ValueError("AbstractBasicAuthHandler does not" - " support the following scheme: '%s'" % - scheme) - else: - mo = AbstractBasicAuthHandler.rx.search(authreq) - if mo: - scheme, quote, realm = mo.groups() - if quote not in ['"',"'"]: - warnings.warn("Basic Auth Realm was unquoted", - UserWarning, 2) - if scheme.lower() == 'basic': - return self.retry_http_basic_auth(host, req, realm) + unsupported = None + for header in headers: + for scheme, realm in self._parse_realm(header): + if scheme.lower() != 'basic': + unsupported = scheme + continue + + if realm is not None: + # Use the first matching Basic challenge. + # Ignore following challenges even if they use the Basic + # scheme. + return self.retry_http_basic_auth(host, req, realm) + + if unsupported is not None: + raise ValueError("AbstractBasicAuthHandler does not " + "support the following scheme: %r" + % (scheme,)) def retry_http_basic_auth(self, host, req, realm): user, pw = self.passwd.find_user_password(realm, host) @@ -1144,7 +1175,11 @@ class AbstractDigestAuthHandler: A2 = "%s:%s" % (req.get_method(), # XXX selector: what about proxies and full urls req.selector) - if qop == 'auth': + # NOTE: As per RFC 2617, when server sends "auth,auth-int", the client could use either `auth` + # or `auth-int` to the response back. we use `auth` to send the response back. + if qop is None: + respdig = KD(H(A1), "%s:%s" % (nonce, H(A2))) + elif 'auth' in qop.split(','): if nonce == self.last_nonce: self.nonce_count += 1 else: @@ -1152,10 +1187,8 @@ class AbstractDigestAuthHandler: self.last_nonce = nonce ncvalue = '%08x' % self.nonce_count cnonce = self.get_cnonce(nonce) - noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2)) + noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, 'auth', H(A2)) respdig = KD(H(A1), noncebit) - elif qop is None: - respdig = KD(H(A1), "%s:%s" % (nonce, H(A2))) else: # XXX handle auth-int. raise URLError("qop '%s' is not supported." % qop) @@ -1262,8 +1295,8 @@ class AbstractHTTPHandler(BaseHandler): sel_host = host if request.has_proxy(): - scheme, sel = splittype(request.selector) - sel_host, sel_path = splithost(sel) + scheme, sel = _splittype(request.selector) + sel_host, sel_path = _splithost(sel) if not request.has_header('Host'): request.add_unredirected_header('Host', sel_host) for name, value in self.parent.addheaders: @@ -1287,8 +1320,8 @@ class AbstractHTTPHandler(BaseHandler): h.set_debuglevel(self._debuglevel) headers = dict(req.unredirected_hdrs) - headers.update(dict((k, v) for k, v in req.headers.items() - if k not in headers)) + headers.update({k: v for k, v in req.headers.items() + if k not in headers}) # TODO(jhylton): Should this be redesigned to handle # persistent connections? @@ -1300,7 +1333,7 @@ class AbstractHTTPHandler(BaseHandler): # So make sure the connection gets closed after the (only) # request. headers["Connection"] = "close" - headers = dict((name.title(), val) for name, val in headers.items()) + headers = {name.title(): val for name, val in headers.items()} if req._tunnel_host: tunnel_headers = {} @@ -1479,7 +1512,7 @@ class FileHandler(BaseHandler): 'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' % (mtype or 'text/plain', size, modified)) if host: - host, port = splitport(host) + host, port = _splitport(host) if not host or \ (not port and _safe_gethostbyname(host) in self.get_names()): if host: @@ -1488,7 +1521,6 @@ class FileHandler(BaseHandler): origurl = 'file://' + filename return addinfourl(open(localfile, 'rb'), headers, origurl) except OSError as exp: - # users shouldn't expect OSErrors coming from urlopen() raise URLError(exp) raise URLError('file not on local host') @@ -1505,16 +1537,16 @@ class FTPHandler(BaseHandler): host = req.host if not host: raise URLError('ftp error: no host given') - host, port = splitport(host) + host, port = _splitport(host) if port is None: port = ftplib.FTP_PORT else: port = int(port) # username/password handling - user, host = splituser(host) + user, host = _splituser(host) if user: - user, passwd = splitpasswd(user) + user, passwd = _splitpasswd(user) else: passwd = None host = unquote(host) @@ -1525,7 +1557,7 @@ class FTPHandler(BaseHandler): host = socket.gethostbyname(host) except OSError as msg: raise URLError(msg) - path, attrs = splitattr(req.selector) + path, attrs = _splitattr(req.selector) dirs = path.split('/') dirs = list(map(unquote, dirs)) dirs, file = dirs[:-1], dirs[-1] @@ -1535,7 +1567,7 @@ class FTPHandler(BaseHandler): fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout) type = file and 'I' or 'D' for attr in attrs: - attr, value = splitvalue(attr) + attr, value = _splitvalue(attr) if attr.lower() == 'type' and \ value in ('a', 'A', 'i', 'I', 'd', 'D'): type = value.upper() @@ -1658,14 +1690,10 @@ else: of the 'file' scheme; not recommended for general use.""" return quote(pathname) -# This really consists of two pieces: -# (1) a class which handles opening of all sorts of URLs -# (plus assorted utilities etc.) -# (2) a set of functions for parsing URLs -# XXX Should these be separated out into different modules? - ftpcache = {} + + class URLopener: """Class to open URLs. This is a class rather than just a subroutine because we may need @@ -1733,26 +1761,26 @@ class URLopener: # External interface def open(self, fullurl, data=None): """Use URLopener().open(file) instead of open(file, 'r').""" - fullurl = unwrap(to_bytes(fullurl)) + fullurl = unwrap(_to_bytes(fullurl)) fullurl = quote(fullurl, safe="%/:=&?~#+!$,;'@()*[]|") if self.tempcache and fullurl in self.tempcache: filename, headers = self.tempcache[fullurl] fp = open(filename, 'rb') return addinfourl(fp, headers, fullurl) - urltype, url = splittype(fullurl) + urltype, url = _splittype(fullurl) if not urltype: urltype = 'file' if urltype in self.proxies: proxy = self.proxies[urltype] - urltype, proxyhost = splittype(proxy) - host, selector = splithost(proxyhost) + urltype, proxyhost = _splittype(proxy) + host, selector = _splithost(proxyhost) url = (host, fullurl) # Signal special case to open_*() else: proxy = None name = 'open_' + urltype self.type = urltype name = name.replace('-', '_') - if not hasattr(self, name): + if not hasattr(self, name) or name == 'open_local_file': if proxy: return self.open_unknown_proxy(proxy, fullurl, data) else: @@ -1769,28 +1797,28 @@ class URLopener: def open_unknown(self, fullurl, data=None): """Overridable interface to open unknown URL type.""" - type, url = splittype(fullurl) + type, url = _splittype(fullurl) raise OSError('url error', 'unknown url type', type) def open_unknown_proxy(self, proxy, fullurl, data=None): """Overridable interface to open unknown URL type.""" - type, url = splittype(fullurl) + type, url = _splittype(fullurl) raise OSError('url error', 'invalid proxy for %s' % type, proxy) # External interface def retrieve(self, url, filename=None, reporthook=None, data=None): """retrieve(url) returns (filename, headers) for a local object or (tempfilename, headers) for a remote object.""" - url = unwrap(to_bytes(url)) + url = unwrap(_to_bytes(url)) if self.tempcache and url in self.tempcache: return self.tempcache[url] - type, url1 = splittype(url) + type, url1 = _splittype(url) if filename is None and (not type or type == 'file'): try: fp = self.open_local_file(url1) hdrs = fp.info() fp.close() - return url2pathname(splithost(url1)[1]), hdrs + return url2pathname(_splithost(url1)[1]), hdrs except OSError as msg: pass fp = self.open(url, data) @@ -1799,11 +1827,10 @@ class URLopener: if filename: tfp = open(filename, 'wb') else: - import tempfile - garbage, path = splittype(url) - garbage, path = splithost(path or "") - path, garbage = splitquery(path or "") - path, garbage = splitattr(path or "") + garbage, path = _splittype(url) + garbage, path = _splithost(path or "") + path, garbage = _splitquery(path or "") + path, garbage = _splitattr(path or "") suffix = os.path.splitext(path)[1] (fd, filename) = tempfile.mkstemp(suffix) self.__tempfiles.append(filename) @@ -1860,25 +1887,25 @@ class URLopener: user_passwd = None proxy_passwd= None if isinstance(url, str): - host, selector = splithost(url) + host, selector = _splithost(url) if host: - user_passwd, host = splituser(host) + user_passwd, host = _splituser(host) host = unquote(host) realhost = host else: host, selector = url # check whether the proxy contains authorization information - proxy_passwd, host = splituser(host) + proxy_passwd, host = _splituser(host) # now we proceed with the url we want to obtain - urltype, rest = splittype(selector) + urltype, rest = _splittype(selector) url = rest user_passwd = None if urltype.lower() != 'http': realhost = None else: - realhost, rest = splithost(rest) + realhost, rest = _splithost(rest) if realhost: - user_passwd, realhost = splituser(realhost) + user_passwd, realhost = _splituser(realhost) if user_passwd: selector = "%s://%s%s" % (urltype, realhost, rest) if proxy_bypass(realhost): @@ -1984,7 +2011,7 @@ class URLopener: """Use local file.""" import email.utils import mimetypes - host, file = splithost(url) + host, file = _splithost(url) localname = url2pathname(file) try: stats = os.stat(localname) @@ -2001,7 +2028,7 @@ class URLopener: if file[:1] == '/': urlfile = 'file://' + file return addinfourl(open(localname, 'rb'), headers, urlfile) - host, port = splitport(host) + host, port = _splitport(host) if (not port and socket.gethostbyname(host) in ((localhost(),) + thishost())): urlfile = file @@ -2017,11 +2044,11 @@ class URLopener: if not isinstance(url, str): raise URLError('ftp error: proxy support for ftp protocol currently not implemented') import mimetypes - host, path = splithost(url) + host, path = _splithost(url) if not host: raise URLError('ftp error: no host given') - host, port = splitport(host) - user, host = splituser(host) - if user: user, passwd = splitpasswd(user) + host, port = _splitport(host) + user, host = _splituser(host) + if user: user, passwd = _splitpasswd(user) else: passwd = None host = unquote(host) user = unquote(user or '') @@ -2032,7 +2059,7 @@ class URLopener: port = ftplib.FTP_PORT else: port = int(port) - path, attrs = splitattr(path) + path, attrs = _splitattr(path) path = unquote(path) dirs = path.split('/') dirs, file = dirs[:-1], dirs[-1] @@ -2054,7 +2081,7 @@ class URLopener: if not file: type = 'D' else: type = 'I' for attr in attrs: - attr, value = splitvalue(attr) + attr, value = _splitvalue(attr) if attr.lower() == 'type' and \ value in ('a', 'A', 'i', 'I', 'd', 'D'): type = value.upper() @@ -2237,11 +2264,11 @@ class FancyURLopener(URLopener): return getattr(self,name)(url, realm, data) def retry_proxy_http_basic_auth(self, url, realm, data=None): - host, selector = splithost(url) + host, selector = _splithost(url) newurl = 'http://' + host + selector proxy = self.proxies['http'] - urltype, proxyhost = splittype(proxy) - proxyhost, proxyselector = splithost(proxyhost) + urltype, proxyhost = _splittype(proxy) + proxyhost, proxyselector = _splithost(proxyhost) i = proxyhost.find('@') + 1 proxyhost = proxyhost[i:] user, passwd = self.get_user_passwd(proxyhost, realm, i) @@ -2255,11 +2282,11 @@ class FancyURLopener(URLopener): return self.open(newurl, data) def retry_proxy_https_basic_auth(self, url, realm, data=None): - host, selector = splithost(url) + host, selector = _splithost(url) newurl = 'https://' + host + selector proxy = self.proxies['https'] - urltype, proxyhost = splittype(proxy) - proxyhost, proxyselector = splithost(proxyhost) + urltype, proxyhost = _splittype(proxy) + proxyhost, proxyselector = _splithost(proxyhost) i = proxyhost.find('@') + 1 proxyhost = proxyhost[i:] user, passwd = self.get_user_passwd(proxyhost, realm, i) @@ -2273,7 +2300,7 @@ class FancyURLopener(URLopener): return self.open(newurl, data) def retry_http_basic_auth(self, url, realm, data=None): - host, selector = splithost(url) + host, selector = _splithost(url) i = host.find('@') + 1 host = host[i:] user, passwd = self.get_user_passwd(host, realm, i) @@ -2287,7 +2314,7 @@ class FancyURLopener(URLopener): return self.open(newurl, data) def retry_https_basic_auth(self, url, realm, data=None): - host, selector = splithost(url) + host, selector = _splithost(url) i = host.find('@') + 1 host = host[i:] user, passwd = self.get_user_passwd(host, realm, i) @@ -2504,23 +2531,26 @@ def proxy_bypass_environment(host, proxies=None): try: no_proxy = proxies['no'] except KeyError: - return 0 + return False # '*' is special case for always bypass if no_proxy == '*': - return 1 + return True + host = host.lower() # strip port off host - hostonly, port = splitport(host) + hostonly, port = _splitport(host) # check if the host ends with any of the DNS suffixes - no_proxy_list = [proxy.strip() for proxy in no_proxy.split(',')] - for name in no_proxy_list: + for name in no_proxy.split(','): + name = name.strip() if name: - name = re.escape(name) - pattern = r'(.+\.)?%s$' % name - if (re.match(pattern, hostonly, re.I) - or re.match(pattern, host, re.I)): - return 1 + name = name.lstrip('.') # ignore leading dots + name = name.lower() + if hostonly == name or host == name: + return True + name = '.' + name + if hostonly.endswith(name) or host.endswith(name): + return True # otherwise, don't bypass - return 0 + return False # This code tests an OSX specific data structure but is testable on all @@ -2539,7 +2569,7 @@ def _proxy_bypass_macosx_sysconf(host, proxy_settings): """ from fnmatch import fnmatch - hostonly, port = splitport(host) + hostonly, port = _splitport(host) def ip2num(ipAddr): parts = ipAddr.split('.') @@ -2646,7 +2676,7 @@ elif os.name == 'nt': for p in proxyServer.split(';'): protocol, address = p.split('=', 1) # See if address has a type:// prefix - if not re.match('^([^/:]+)://', address): + if not re.match('(?:[^/:]+)://', address): address = '%s://%s' % (protocol, address) proxies[protocol] = address else: @@ -2693,7 +2723,7 @@ elif os.name == 'nt': if not proxyEnable or not proxyOverride: return 0 # try to make a host list from name and IP address. - rawHost, port = splitport(host) + rawHost, port = _splitport(host) host = [rawHost] try: addr = socket.gethostbyname(rawHost) diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 9dab4c1c3..c58565e39 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -16,6 +16,9 @@ import urllib.request __all__ = ["RobotFileParser"] +RequestRate = collections.namedtuple("RequestRate", "requests seconds") + + class RobotFileParser: """ This class provides a set of methods to read, parse and answer questions about a single robots.txt file. @@ -24,6 +27,7 @@ class RobotFileParser: def __init__(self, url=''): self.entries = [] + self.sitemaps = [] self.default_entry = None self.disallow_all = False self.allow_all = False @@ -136,12 +140,14 @@ class RobotFileParser: # check if all values are sane if (len(numbers) == 2 and numbers[0].strip().isdigit() and numbers[1].strip().isdigit()): - req_rate = collections.namedtuple('req_rate', - 'requests seconds') - entry.req_rate = req_rate - entry.req_rate.requests = int(numbers[0]) - entry.req_rate.seconds = int(numbers[1]) + entry.req_rate = RequestRate(int(numbers[0]), int(numbers[1])) state = 2 + elif line[0] == "sitemap": + # According to http://www.sitemaps.org/protocol.html + # "This directive is independent of the user-agent line, + # so it doesn't matter where you place it in your file." + # Therefore we do not change the state of the parser. + self.sitemaps.append(line[1]) if state == 2: self._add_entry(entry) @@ -180,7 +186,9 @@ class RobotFileParser: for entry in self.entries: if entry.applies_to(useragent): return entry.delay - return self.default_entry.delay + if self.default_entry: + return self.default_entry.delay + return None def request_rate(self, useragent): if not self.mtime(): @@ -188,10 +196,20 @@ class RobotFileParser: for entry in self.entries: if entry.applies_to(useragent): return entry.req_rate - return self.default_entry.req_rate + if self.default_entry: + return self.default_entry.req_rate + return None + + def site_maps(self): + if not self.sitemaps: + return None + return self.sitemaps def __str__(self): - return ''.join([str(entry) + "\n" for entry in self.entries]) + entries = self.entries + if self.default_entry is not None: + entries = entries + [self.default_entry] + return '\n\n'.join(map(str, entries)) class RuleLine: @@ -223,10 +241,14 @@ class Entry: def __str__(self): ret = [] for agent in self.useragents: - ret.extend(["User-agent: ", agent, "\n"]) - for line in self.rulelines: - ret.extend([str(line), "\n"]) - return ''.join(ret) + ret.append(f"User-agent: {agent}") + if self.delay is not None: + ret.append(f"Crawl-delay: {self.delay}") + if self.req_rate is not None: + rate = self.req_rate + ret.append(f"Request-rate: {rate.requests}/{rate.seconds}") + ret.extend(map(str, self.rulelines)) + return '\n'.join(ret) def applies_to(self, useragent): """check if this entry applies to the specified agent"""