mirrors/RustPython
0
0 Fork 0
mirror of https://github.com/RustPython/RustPython.git synced 2026-10-28 10:44:36 +08:00
Code Activity

Update html* from 3.13.7 (#6133)

Browse Source
This commit is contained in:
Shahar Naveh authored and GitHub committed 2025-09-07 17:07:54 +09:00
1 parent 1d53e0c923
commit 9da2e04880
2 files changed
+435 -165
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Lib/html/parser.py
Vendored
+129 -89
View File
@@ -27,18 +27,48 @@ charref = re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]')
attr_charref = re.compile(r'&(#[0-9]+|#[xX][0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*)[;=]?')
starttagopen = re.compile('<[a-zA-Z]')
endtagopen = re.compile('</[a-zA-Z]')
piclose = re.compile('>')
commentclose = re.compile(r'--\s*>')
commentclose = re.compile(r'--!?>')
commentabruptclose = re.compile(r'-?>')
# Note:
# 1) if you change tagfind/attrfind remember to update locatestarttagend too;
# 2) if you change tagfind/attrfind and/or locatestarttagend the parser will
# 1) if you change tagfind/attrfind remember to update locatetagend too;
# 2) if you change tagfind/attrfind and/or locatetagend the parser will
# explode, so don't do it.
# see http://www.w3.org/TR/html5/tokenization.html#tag-open-state
# and http://www.w3.org/TR/html5/tokenization.html#tag-name-state
tagfind_tolerant = re.compile(r'([a-zA-Z][^\t\n\r\f />\x00]*)(?:\s|/(?!>))*')
attrfind_tolerant = re.compile(
r'((?<=[\'"\s/])[^\s/>][^\s/=>]*)(\s*=+\s*'
r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?(?:\s|/(?!>))*')
# see the HTML5 specs section "13.2.5.6 Tag open state",
# "13.2.5.8 Tag name state" and "13.2.5.33 Attribute name state".
# https://html.spec.whatwg.org/multipage/parsing.html#tag-open-state
# https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state
# https://html.spec.whatwg.org/multipage/parsing.html#attribute-name-state
tagfind_tolerant = re.compile(r'([a-zA-Z][^\t\n\r\f />]*)(?:[\t\n\r\f ]|/(?!>))*')
attrfind_tolerant = re.compile(r"""
(
(?<=['"\t\n\r\f /])[^\t\n\r\f />][^\t\n\r\f /=>]* # attribute name
)
([\t\n\r\f ]*=[\t\n\r\f ]* # value indicator
('[^']*' # LITA-enclosed value
|"[^"]*" # LIT-enclosed value
|(?!['"])[^>\t\n\r\f ]* # bare value
)
)?
(?:[\t\n\r\f ]|/(?!>))* # possibly followed by a space
""", re.VERBOSE)
locatetagend = re.compile(r"""
[a-zA-Z][^\t\n\r\f />]* # tag name
[\t\n\r\f /]* # optional whitespace before attribute name
(?:(?<=['"\t\n\r\f /])[^\t\n\r\f />][^\t\n\r\f /=>]* # attribute name
(?:[\t\n\r\f ]*=[\t\n\r\f ]* # value indicator
(?:'[^']*' # LITA-enclosed value
|"[^"]*" # LIT-enclosed value
|(?!['"])[^>\t\n\r\f ]* # bare value
)
)?
[\t\n\r\f /]* # possibly followed by a space
)*
>?
""", re.VERBOSE)
# The following variables are not used, but are temporarily left for
# backward compatibility.
locatestarttagend_tolerant = re.compile(r"""
<[a-zA-Z][^\t\n\r\f />\x00]* # tag name
(?:[\s/]* # optional whitespace before attribute name
@@ -55,8 +85,6 @@ locatestarttagend_tolerant = re.compile(r"""
\s* # trailing whitespace
""", re.VERBOSE)
endendtag = re.compile('>')
# the HTML 5 spec, section 8.1.2.2, doesn't allow spaces between
# </ and the tag name, so maybe this should be fixed
endtagfind = re.compile(r'</\s*([a-zA-Z][-.a-zA-Z0-9:_]*)\s*>')
# Character reference processing logic specific to attribute values
@@ -100,6 +128,7 @@ class HTMLParser(_markupbase.ParserBase):
"""
CDATA_CONTENT_ELEMENTS = ("script", "style")
RCDATA_CONTENT_ELEMENTS = ("textarea", "title")
def __init__(self, *, convert_charrefs=True):
"""Initialize and reset this instance.
@@ -117,6 +146,7 @@ class HTMLParser(_markupbase.ParserBase):
self.lasttag = '???'
self.interesting = interesting_normal
self.cdata_elem = None
self._escapable = True
super().reset()
def feed(self, data):
@@ -138,13 +168,20 @@ class HTMLParser(_markupbase.ParserBase):
"""Return full source of start tag: '<...>'."""
return self.__starttag_text
def set_cdata_mode(self, elem):
def set_cdata_mode(self, elem, *, escapable=False):
self.cdata_elem = elem.lower()
self.interesting = re.compile(r'</\s*%s\s*>' % self.cdata_elem, re.I)
self._escapable = escapable
if escapable and not self.convert_charrefs:
self.interesting = re.compile(r'&|</%s(?=[\t\n\r\f />])' % self.cdata_elem,
re.IGNORECASE|re.ASCII)
else:
self.interesting = re.compile(r'</%s(?=[\t\n\r\f />])' % self.cdata_elem,
re.IGNORECASE|re.ASCII)
def clear_cdata_mode(self):
self.interesting = interesting_normal
self.cdata_elem = None
self._escapable = True
# Internal -- handle data as far as reasonable. May leave state
# and data to be processed by a subsequent call. If 'end' is
@@ -165,7 +202,7 @@ class HTMLParser(_markupbase.ParserBase):
# & near the end and see if it's followed by a space or ;.
amppos = rawdata.rfind('&', max(i, n-34))
if (amppos >= 0 and
not re.compile(r'[\s;]').search(rawdata, amppos)):
not re.compile(r'[\t\n\r\f ;]').search(rawdata, amppos)):
break # wait till we get all the text
j = n
else:
@@ -177,7 +214,7 @@ class HTMLParser(_markupbase.ParserBase):
break
j = n
if i < j:
if self.convert_charrefs and not self.cdata_elem:
if self.convert_charrefs and self._escapable:
self.handle_data(unescape(rawdata[i:j]))
else:
self.handle_data(rawdata[i:j])
@@ -195,7 +232,7 @@ class HTMLParser(_markupbase.ParserBase):
k = self.parse_pi(i)
elif startswith("<!", i):
k = self.parse_html_declaration(i)
elif (i + 1) < n:
elif (i + 1) < n or end:
self.handle_data("<")
k = i + 1
else:
@@ -203,17 +240,35 @@ class HTMLParser(_markupbase.ParserBase):
if k < 0:
if not end:
break
k = rawdata.find('>', i + 1)
if k < 0:
k = rawdata.find('<', i + 1)
if k < 0:
k = i + 1
if starttagopen.match(rawdata, i): # < + letter
pass
elif startswith("</", i):
if i + 2 == n:
self.handle_data("</")
elif endtagopen.match(rawdata, i): # </ + letter
pass
else:
# bogus comment
self.handle_comment(rawdata[i+2:])
elif startswith("<!--", i):
j = n
for suffix in ("--!", "--", "-"):
if rawdata.endswith(suffix, i+4):
j -= len(suffix)
break
self.handle_comment(rawdata[i+4:j])
elif startswith("<![CDATA[", i):
self.unknown_decl(rawdata[i+3:])
elif rawdata[i:i+9].lower() == '<!doctype':
self.handle_decl(rawdata[i+2:])
elif startswith("<!", i):
# bogus comment
self.handle_comment(rawdata[i+2:])
elif startswith("<?", i):
self.handle_pi(rawdata[i+2:])
else:
k += 1
if self.convert_charrefs and not self.cdata_elem:
self.handle_data(unescape(rawdata[i:k]))
else:
self.handle_data(rawdata[i:k])
raise AssertionError("we should not get here!")
k = n
i = self.updatepos(i, k)
elif startswith("&#", i):
match = charref.match(rawdata, i)
@@ -261,7 +316,7 @@ class HTMLParser(_markupbase.ParserBase):
assert 0, "interesting.search() lied"
# end while
if end and i < n:
if self.convert_charrefs and not self.cdata_elem:
if self.convert_charrefs and self._escapable:
self.handle_data(unescape(rawdata[i:n]))
else:
self.handle_data(rawdata[i:n])
@@ -290,8 +345,23 @@ class HTMLParser(_markupbase.ParserBase):
else:
return self.parse_bogus_comment(i)
# Internal -- parse comment, return length or -1 if not terminated
# see https://html.spec.whatwg.org/multipage/parsing.html#comment-start-state
def parse_comment(self, i, report=True):
rawdata = self.rawdata
assert rawdata.startswith('<!--', i), 'unexpected call to parse_comment()'
match = commentclose.search(rawdata, i+4)
if not match:
match = commentabruptclose.match(rawdata, i+4)
if not match:
return -1
if report:
j = match.start()
self.handle_comment(rawdata[i+4: j])
return match.end()
# Internal -- parse bogus comment, return length or -1 if not terminated
# see http://www.w3.org/TR/html5/tokenization.html#bogus-comment-state
# see https://html.spec.whatwg.org/multipage/parsing.html#bogus-comment-state
def parse_bogus_comment(self, i, report=1):
rawdata = self.rawdata
assert rawdata[i:i+2] in ('<!', '</'), ('unexpected call to '
@@ -317,6 +387,8 @@ class HTMLParser(_markupbase.ParserBase):
# Internal -- handle starttag, return end or -1 if not terminated
def parse_starttag(self, i):
# See the HTML5 specs section "13.2.5.8 Tag name state"
# https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state
self.__starttag_text = None
endpos = self.check_for_whole_start_tag(i)
if endpos < 0:
@@ -356,82 +428,50 @@ class HTMLParser(_markupbase.ParserBase):
self.handle_starttag(tag, attrs)
if tag in self.CDATA_CONTENT_ELEMENTS:
self.set_cdata_mode(tag)
elif tag in self.RCDATA_CONTENT_ELEMENTS:
self.set_cdata_mode(tag, escapable=True)
return endpos
# Internal -- check to see if we have a complete starttag; return end
# or -1 if incomplete.
def check_for_whole_start_tag(self, i):
rawdata = self.rawdata
m = locatestarttagend_tolerant.match(rawdata, i)
if m:
j = m.end()
next = rawdata[j:j+1]
if next == ">":
return j + 1
if next == "/":
if rawdata.startswith("/>", j):
return j + 2
if rawdata.startswith("/", j):
# buffer boundary
return -1
# else bogus input
if j > i:
return j
else:
return i + 1
if next == "":
# end of input
return -1
if next in ("abcdefghijklmnopqrstuvwxyz=/"
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"):
# end of input in or before attribute value, or we have the
# '/' from a '/>' ending
return -1
if j > i:
return j
else:
return i + 1
raise AssertionError("we should not get here!")
match = locatetagend.match(rawdata, i+1)
assert match
j = match.end()
if rawdata[j-1] != ">":
return -1
return j
# Internal -- parse endtag, return end or -1 if incomplete
def parse_endtag(self, i):
# See the HTML5 specs section "13.2.5.7 End tag open state"
# https://html.spec.whatwg.org/multipage/parsing.html#end-tag-open-state
rawdata = self.rawdata
assert rawdata[i:i+2] == "</", "unexpected call to parse_endtag"
match = endendtag.search(rawdata, i+1) # >
if not match:
if rawdata.find('>', i+2) < 0: # fast check
return -1
gtpos = match.end()
match = endtagfind.match(rawdata, i) # </ + tag + >
if not match:
if self.cdata_elem is not None:
self.handle_data(rawdata[i:gtpos])
return gtpos
# find the name: w3.org/TR/html5/tokenization.html#tag-name-state
namematch = tagfind_tolerant.match(rawdata, i+2)
if not namematch:
# w3.org/TR/html5/tokenization.html#end-tag-open-state
if rawdata[i:i+3] == '</>':
return i+3
else:
return self.parse_bogus_comment(i)
tagname = namematch.group(1).lower()
# consume and ignore other stuff between the name and the >
# Note: this is not 100% correct, since we might have things like
# </tag attr=">">, but looking for > after the name should cover
# most of the cases and is much simpler
gtpos = rawdata.find('>', namematch.end())
self.handle_endtag(tagname)
return gtpos+1
if not endtagopen.match(rawdata, i): # </ + letter
if rawdata[i+2:i+3] == '>': # </> is ignored
# "missing-end-tag-name" parser error
return i+3
else:
return self.parse_bogus_comment(i)
elem = match.group(1).lower() # script or style
if self.cdata_elem is not None:
if elem != self.cdata_elem:
self.handle_data(rawdata[i:gtpos])
return gtpos
match = locatetagend.match(rawdata, i+2)
assert match
j = match.end()
if rawdata[j-1] != ">":
return -1
self.handle_endtag(elem)
# find the name: "13.2.5.8 Tag name state"
# https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state
match = tagfind_tolerant.match(rawdata, i+2)
assert match
tag = match.group(1).lower()
self.handle_endtag(tag)
self.clear_cdata_mode()
return gtpos
return j
# Overridable -- finish processing of start+end tag: <tag.../>
def handle_startendtag(self, tag, attrs):
Lib/test/test_htmlparser.py
Vendored
+306 -76
View File
@@ -5,6 +5,7 @@ import pprint
import unittest
from unittest.mock import patch
from test import support
class EventCollector(html.parser.HTMLParser):
@@ -80,6 +81,13 @@ class EventCollectorCharrefs(EventCollector):
self.fail('This should never be called with convert_charrefs=True')
# The normal event collector normalizes the events in get_events,
# so we override it to return the original list of events.
class EventCollectorNoNormalize(EventCollector):
def get_events(self):
return self.events
class TestCaseBase(unittest.TestCase):
def get_collector(self):
@@ -264,8 +272,7 @@ text
("starttag", "foo:bar", [("one", "1"), ("two", "2")]),
("starttag_text", s)])
def test_cdata_content(self):
contents = [
@support.subTests('content', [
'<!-- not a comment --> &not-an-entity-ref;',
"<not a='start tag'>",
'<a href="" /> <p> <span></span>',
@@ -278,70 +285,223 @@ text
'src="http://www.example.org/r=\'+new '
'Date().getTime()+\'"><\\/s\'+\'cript>\');\n//]]>'),
'\n<!-- //\nvar foo = 3.14;\n// -->\n',
'foo = "</sty" + "le>";',
'<!-- \u2603 -->',
# these two should be invalid according to the HTML 5 spec,
# section 8.1.2.2
#'foo = </\nscript>',
#'foo = </ script>',
]
elements = ['script', 'style', 'SCRIPT', 'STYLE', 'Script', 'Style']
for content in contents:
for element in elements:
element_lower = element.lower()
s = '<{element}>{content}</{element}>'.format(element=element,
content=content)
self._run_check(s, [("starttag", element_lower, []),
("data", content),
("endtag", element_lower)])
'foo = "</ script>"',
'foo = "</scripture>"',
'foo = "</script\v>"',
'foo = "</script\xa0>"',
'foo = "</ſcript>"',
'foo = "</scrıpt>"',
])
def test_script_content(self, content):
s = f'<script>{content}</script>'
self._run_check(s, [("starttag", "script", []),
("data", content),
("endtag", "script")])
def test_cdata_with_closing_tags(self):
@support.subTests('content', [
'a::before { content: "<!-- not a comment -->"; }',
'a::before { content: "&not-an-entity-ref;"; }',
'a::before { content: "<not a=\'start tag\'>"; }',
'a::before { content: "\u2603"; }',
'a::before { content: "< /style>"; }',
'a::before { content: "</ style>"; }',
'a::before { content: "</styled>"; }',
'a::before { content: "</style\v>"; }',
'a::before { content: "</style\xa0>"; }',
'a::before { content: "</ſtyle>"; }',
])
def test_style_content(self, content):
s = f'<style>{content}</style>'
self._run_check(s, [("starttag", "style", []),
("data", content),
("endtag", "style")])
@support.subTests('content', [
'<!-- not a comment -->',
"<not a='start tag'>",
'<![CDATA[not a cdata]]>',
'<!not a bogus comment>',
'</not a bogus comment>',
'\u2603',
'< /title>',
'</ title>',
'</titled>',
'</title\v>',
'</title\xa0>',
'</tıtle>',
])
def test_title_content(self, content):
source = f"<title>{content}</title>"
self._run_check(source, [
("starttag", "title", []),
("data", content),
("endtag", "title"),
])
@support.subTests('content', [
'<!-- not a comment -->',
"<not a='start tag'>",
'<![CDATA[not a cdata]]>',
'<!not a bogus comment>',
'</not a bogus comment>',
'\u2603',
'< /textarea>',
'</ textarea>',
'</textareable>',
'</textarea\v>',
'</textarea\xa0>',
])
def test_textarea_content(self, content):
source = f"<textarea>{content}</textarea>"
self._run_check(source, [
("starttag", "textarea", []),
("data", content),
("endtag", "textarea"),
])
@support.subTests('endtag', ['script', 'SCRIPT', 'script ', 'script\n',
'script/', 'script foo=bar', 'script foo=">"'])
def test_script_closing_tag(self, endtag):
# see issue #13358
# make sure that HTMLParser calls handle_data only once for each CDATA.
# The normal event collector normalizes the events in get_events,
# so we override it to return the original list of events.
class Collector(EventCollector):
def get_events(self):
return self.events
content = """<!-- not a comment --> &not-an-entity-ref;
<a href="" /> </p><p> <span></span></style>
'</script' + '>'"""
for element in [' script', 'script ', ' script ',
'\nscript', 'script\n', '\nscript\n']:
element_lower = element.lower().strip()
s = '<script>{content}</{element}>'.format(element=element,
content=content)
self._run_check(s, [("starttag", element_lower, []),
("data", content),
("endtag", element_lower)],
collector=Collector(convert_charrefs=False))
s = f'<ScrIPt>{content}</{endtag}>'
self._run_check(s, [("starttag", "script", []),
("data", content),
("endtag", "script")],
collector=EventCollectorNoNormalize(convert_charrefs=False))
def test_EOF_in_cdata(self):
content = """<!-- not a comment --> &not-an-entity-ref;
<a href="" /> </p><p> <span></span></style>
'</script' + '>'"""
s = f'<script>{content}'
self._run_check(s, [
("starttag", 'script', []),
("data", content)
])
@support.subTests('endtag', ['style', 'STYLE', 'style ', 'style\n',
'style/', 'style foo=bar', 'style foo=">"'])
def test_style_closing_tag(self, endtag):
content = """
b::before { content: "<!-- not a comment -->"; }
p::before { content: "&not-an-entity-ref;"; }
a::before { content: "<i>"; }
a::after { content: "</i>"; }
"""
s = f'<StyLE>{content}</{endtag}>'
self._run_check(s, [("starttag", "style", []),
("data", content),
("endtag", "style")],
collector=EventCollectorNoNormalize(convert_charrefs=False))
@support.subTests('endtag', ['title', 'TITLE', 'title ', 'title\n',
'title/', 'title foo=bar', 'title foo=">"'])
def test_title_closing_tag(self, endtag):
content = "<!-- not a comment --><i>Egg &amp; Spam</i>"
s = f'<TitLe>{content}</{endtag}>'
self._run_check(s, [("starttag", "title", []),
('data', '<!-- not a comment --><i>Egg & Spam</i>'),
("endtag", "title")],
collector=EventCollectorNoNormalize(convert_charrefs=True))
self._run_check(s, [("starttag", "title", []),
('data', '<!-- not a comment --><i>Egg '),
('entityref', 'amp'),
('data', ' Spam</i>'),
("endtag", "title")],
collector=EventCollectorNoNormalize(convert_charrefs=False))
@support.subTests('endtag', ['textarea', 'TEXTAREA', 'textarea ', 'textarea\n',
'textarea/', 'textarea foo=bar', 'textarea foo=">"'])
def test_textarea_closing_tag(self, endtag):
content = "<!-- not a comment --><i>Egg &amp; Spam</i>"
s = f'<TexTarEa>{content}</{endtag}>'
self._run_check(s, [("starttag", "textarea", []),
('data', '<!-- not a comment --><i>Egg & Spam</i>'),
("endtag", "textarea")],
collector=EventCollectorNoNormalize(convert_charrefs=True))
self._run_check(s, [("starttag", "textarea", []),
('data', '<!-- not a comment --><i>Egg '),
('entityref', 'amp'),
('data', ' Spam</i>'),
("endtag", "textarea")],
collector=EventCollectorNoNormalize(convert_charrefs=False))
@support.subTests('tail,end', [
('', False),
('<', False),
('</', False),
('</s', False),
('</script', False),
('</script ', True),
('</script foo=bar', True),
('</script foo=">', True),
])
def test_eof_in_script(self, tail, end):
content = "a = 123"
s = f'<ScrIPt>{content}{tail}'
self._run_check(s, [("starttag", "script", []),
("data", content if end else content + tail)],
collector=EventCollectorNoNormalize(convert_charrefs=False))
@support.subTests('tail,end', [
('', False),
('<', False),
('</', False),
('</t', False),
('</title', False),
('</title ', True),
('</title foo=bar', True),
('</title foo=">', True),
])
def test_eof_in_title(self, tail, end):
s = f'<TitLe>Egg &amp; Spam{tail}'
self._run_check(s, [("starttag", "title", []),
("data", "Egg & Spam" + ('' if end else tail))],
collector=EventCollectorNoNormalize(convert_charrefs=True))
self._run_check(s, [("starttag", "title", []),
('data', 'Egg '),
('entityref', 'amp'),
('data', ' Spam' + ('' if end else tail))],
collector=EventCollectorNoNormalize(convert_charrefs=False))
def test_comments(self):
html = ("<!-- I'm a valid comment -->"
'<!--me too!-->'
'<!------>'
'<!----->'
'<!---->'
# abrupt-closing-of-empty-comment
'<!--->'
'<!-->'
'<!----I have many hyphens---->'
'<!-- I have a > in the middle -->'
'<!-- and I have -- in the middle! -->')
'<!-- and I have -- in the middle! -->'
'<!--incorrectly-closed-comment--!>'
'<!----!>'
'<!----!-->'
'<!---- >-->'
'<!---!>-->'
'<!--!>-->'
# nested-comment
'<!-- <!-- nested --> -->'
'<!--<!-->'
'<!--<!--!>'
)
expected = [('comment', " I'm a valid comment "),
('comment', 'me too!'),
('comment', '--'),
('comment', '-'),
('comment', ''),
('comment', ''),
('comment', ''),
('comment', '--I have many hyphens--'),
('comment', ' I have a > in the middle '),
('comment', ' and I have -- in the middle! ')]
('comment', ' and I have -- in the middle! '),
('comment', 'incorrectly-closed-comment'),
('comment', ''),
('comment', '--!'),
('comment', '-- >'),
('comment', '-!>'),
('comment', '!>'),
('comment', ' <!-- nested '), ('data', ' -->'),
('comment', '<!'),
('comment', '<!'),
]
self._run_check(html, expected)
def test_condcoms(self):
@@ -430,28 +590,34 @@ text
('data', '<'),
('starttag', 'bc<', [('a', None)]),
('endtag', 'html'),
('data', '\n<img src="URL>'),
('comment', '/img'),
('endtag', 'html<')])
('data', '\n')])
def test_starttag_junk_chars(self):
self._run_check("<", [('data', '<')])
self._run_check("<>", [('data', '<>')])
self._run_check("< >", [('data', '< >')])
self._run_check("< ", [('data', '< ')])
self._run_check("</>", [])
self._run_check("<$>", [('data', '<$>')])
self._run_check("</$>", [('comment', '$')])
self._run_check("</", [('data', '</')])
self._run_check("</a", [('data', '</a')])
self._run_check("</a", [])
self._run_check("</ a>", [('comment', ' a')])
self._run_check("</ a", [('comment', ' a')])
self._run_check("<a<a>", [('starttag', 'a<a', [])])
self._run_check("</a<a>", [('endtag', 'a<a')])
self._run_check("<!", [('data', '<!')])
self._run_check("<a", [('data', '<a')])
self._run_check("<a foo='bar'", [('data', "<a foo='bar'")])
self._run_check("<a foo='bar", [('data', "<a foo='bar")])
self._run_check("<a foo='>'", [('data', "<a foo='>'")])
self._run_check("<a foo='>", [('data', "<a foo='>")])
self._run_check("<!", [('comment', '')])
self._run_check("<a", [])
self._run_check("<a foo='bar'", [])
self._run_check("<a foo='bar", [])
self._run_check("<a foo='>'", [])
self._run_check("<a foo='>", [])
self._run_check("<a$>", [('starttag', 'a$', [])])
self._run_check("<a$b>", [('starttag', 'a$b', [])])
self._run_check("<a$b/>", [('startendtag', 'a$b', [])])
self._run_check("<a$b >", [('starttag', 'a$b', [])])
self._run_check("<a$b />", [('startendtag', 'a$b', [])])
self._run_check("</a$b>", [('endtag', 'a$b')])
def test_slashes_in_starttag(self):
self._run_check('<a foo="var"/>', [('startendtag', 'a', [('foo', 'var')])])
@@ -484,6 +650,10 @@ text
]
self._run_check(html, expected)
def test_slashes_in_endtag(self):
self._run_check('</a/>', [('endtag', 'a')])
self._run_check('</a foo="var"/>', [('endtag', 'a')])
def test_declaration_junk_chars(self):
self._run_check("<!DOCTYPE foo $ >", [('decl', 'DOCTYPE foo $ ')])
@@ -518,15 +688,11 @@ text
self._run_check(html, expected)
def test_broken_invalid_end_tag(self):
# This is technically wrong (the "> shouldn't be included in the 'data')
# but is probably not worth fixing it (in addition to all the cases of
# the previous test, it would require a full attribute parsing).
# see #13993
html = '<b>This</b attr=">"> confuses the parser'
expected = [('starttag', 'b', []),
('data', 'This'),
('endtag', 'b'),
('data', '"> confuses the parser')]
('data', ' confuses the parser')]
self._run_check(html, expected)
def test_correct_detection_of_start_tags(self):
@@ -576,21 +742,50 @@ text
for html, expected in data:
self._run_check(html, expected)
def test_EOF_in_comments_or_decls(self):
def test_eof_in_comments(self):
data = [
('<!', [('data', '<!')]),
('<!-', [('data', '<!-')]),
('<!--', [('data', '<!--')]),
('<![', [('data', '<![')]),
('<![CDATA[', [('data', '<![CDATA[')]),
('<![CDATA[x', [('data', '<![CDATA[x')]),
('<!DOCTYPE', [('data', '<!DOCTYPE')]),
('<!DOCTYPE HTML', [('data', '<!DOCTYPE HTML')]),
('<!--', [('comment', '')]),
('<!---', [('comment', '')]),
('<!----', [('comment', '')]),
('<!-----', [('comment', '-')]),
('<!------', [('comment', '--')]),
('<!----!', [('comment', '')]),
('<!---!', [('comment', '-!')]),
('<!---!>', [('comment', '-!>')]),
('<!--foo', [('comment', 'foo')]),
('<!--foo-', [('comment', 'foo')]),
('<!--foo--', [('comment', 'foo')]),
('<!--foo--!', [('comment', 'foo')]),
('<!--<!--', [('comment', '<!')]),
('<!--<!--!', [('comment', '<!')]),
]
for html, expected in data:
self._run_check(html, expected)
def test_eof_in_declarations(self):
data = [
('<!', [('comment', '')]),
('<!-', [('comment', '-')]),
('<![', [('comment', '[')]),
('<![CDATA[', [('unknown decl', 'CDATA[')]),
('<![CDATA[x', [('unknown decl', 'CDATA[x')]),
('<![CDATA[x]', [('unknown decl', 'CDATA[x]')]),
('<![CDATA[x]]', [('unknown decl', 'CDATA[x]]')]),
('<!DOCTYPE', [('decl', 'DOCTYPE')]),
('<!DOCTYPE ', [('decl', 'DOCTYPE ')]),
('<!DOCTYPE html', [('decl', 'DOCTYPE html')]),
('<!DOCTYPE html ', [('decl', 'DOCTYPE html ')]),
('<!DOCTYPE html PUBLIC', [('decl', 'DOCTYPE html PUBLIC')]),
('<!DOCTYPE html PUBLIC "foo', [('decl', 'DOCTYPE html PUBLIC "foo')]),
('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "foo',
[('decl', 'DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "foo')]),
]
for html, expected in data:
self._run_check(html, expected)
def test_bogus_comments(self):
html = ('<! not really a comment >'
html = ('<!ELEMENT br EMPTY>'
'<! not really a comment >'
'<! not a comment either -->'
'<! -- close enough -->'
'<!><!<-- this was an empty comment>'
@@ -604,6 +799,7 @@ text
'<![CDATA]]>' # required '[' after CDATA
)
expected = [
('comment', 'ELEMENT br EMPTY'),
('comment', ' not really a comment '),
('comment', ' not a comment either --'),
('comment', ' -- close enough --'),
@@ -684,6 +880,26 @@ text
('endtag', 'a'), ('data', ' bar & baz')]
)
@support.requires_resource('cpu')
def test_eof_no_quadratic_complexity(self):
# Each of these examples used to take about an hour.
# Now they take a fraction of a second.
def check(source):
parser = html.parser.HTMLParser()
parser.feed(source)
parser.close()
n = 120_000
check("<a " * n)
check("<a a=" * n)
check("</a " * 14 * n)
check("</a a=" * 11 * n)
check("<!--" * 4 * n)
check("<!" * 60 * n)
check("<?" * 19 * n)
check("</$" * 15 * n)
check("<![CDATA[" * 9 * n)
check("<!doctype" * 35 * n)
class AttributesTestCase(TestCaseBase):
@@ -692,9 +908,15 @@ class AttributesTestCase(TestCaseBase):
("starttag", "a", [("b", "v"), ("c", "v"), ("d", "v"), ("e", None)])
]
self._run_check("""<a b='v' c="v" d=v e>""", output)
self._run_check("""<a b = 'v' c = "v" d = v e>""", output)
self._run_check("""<a\nb\n=\n'v'\nc\n=\n"v"\nd\n=\nv\ne>""", output)
self._run_check("""<a\tb\t=\t'v'\tc\t=\t"v"\td\t=\tv\te>""", output)
self._run_check("<a foo==bar>", [('starttag', 'a', [('foo', '=bar')])])
self._run_check("<a foo =bar>", [('starttag', 'a', [('foo', 'bar')])])
self._run_check("<a foo\t=bar>", [('starttag', 'a', [('foo', 'bar')])])
self._run_check("<a foo\v=bar>", [('starttag', 'a', [('foo\v', 'bar')])])
self._run_check("<a foo\xa0=bar>", [('starttag', 'a', [('foo\xa0', 'bar')])])
self._run_check("<a foo= bar>", [('starttag', 'a', [('foo', 'bar')])])
self._run_check("<a foo=\tbar>", [('starttag', 'a', [('foo', 'bar')])])
self._run_check("<a foo=\vbar>", [('starttag', 'a', [('foo', '\vbar')])])
self._run_check("<a foo=\xa0bar>", [('starttag', 'a', [('foo', '\xa0bar')])])
def test_attr_values(self):
self._run_check("""<a b='xxx\n\txxx' c="yyy\t\nyyy" d='\txyz\n'>""",
@@ -703,6 +925,10 @@ class AttributesTestCase(TestCaseBase):
("d", "\txyz\n")])])
self._run_check("""<a b='' c="">""",
[("starttag", "a", [("b", ""), ("c", "")])])
self._run_check("<a b=\tx c=\ny>",
[('starttag', 'a', [('b', 'x'), ('c', 'y')])])
self._run_check("<a b=\v c=\xa0>",
[("starttag", "a", [("b", "\v"), ("c", "\xa0")])])
# Regression test for SF patch #669683.
self._run_check("<e a=rgb(1,2,3)>",
[("starttag", "e", [("a", "rgb(1,2,3)")])])
@@ -769,13 +995,17 @@ class AttributesTestCase(TestCaseBase):
)
expected = [
('starttag', 'a', [('href', "test'style='color:red;bad1'")]),
('data', 'test - bad1'), ('endtag', 'a'),
('data', 'test - bad1'),
('endtag', 'a'),
('starttag', 'a', [('href', "test'+style='color:red;ba2'")]),
('data', 'test - bad2'), ('endtag', 'a'),
('data', 'test - bad2'),
('endtag', 'a'),
('starttag', 'a', [('href', "test'\xa0style='color:red;bad3'")]),
('data', 'test - bad3'), ('endtag', 'a'),
('data', 'test - bad3'),
('endtag', 'a'),
('starttag', 'a', [('href', "test'\xa0style='color:red;bad4'")]),
('data', 'test - bad4'), ('endtag', 'a')
('data', 'test - bad4'),
('endtag', 'a'),
]
self._run_check(html, expected)