diff --git a/Lib/html/parser.py b/Lib/html/parser.py
index 5d7050dad..80fb8c3f9 100644
--- a/Lib/html/parser.py
+++ b/Lib/html/parser.py
@@ -24,6 +24,7 @@ incomplete = re.compile('&[a-zA-Z#]')
entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]')
charref = re.compile('(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]')
+incomplete_charref = re.compile('(?:[0-9]|[xX][0-9a-fA-F])')
attr_charref = re.compile(r'&(#[0-9]+|#[xX][0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*)[;=]?')
starttagopen = re.compile('<[a-zA-Z]')
@@ -127,17 +128,25 @@ class HTMLParser(_markupbase.ParserBase):
argument.
"""
- CDATA_CONTENT_ELEMENTS = ("script", "style")
+ # See the HTML5 specs section "13.4 Parsing HTML fragments".
+ # https://html.spec.whatwg.org/multipage/parsing.html#parsing-html-fragments
+ # CDATA_CONTENT_ELEMENTS are parsed in RAWTEXT mode
+ CDATA_CONTENT_ELEMENTS = ("script", "style", "xmp", "iframe", "noembed", "noframes")
RCDATA_CONTENT_ELEMENTS = ("textarea", "title")
- def __init__(self, *, convert_charrefs=True):
+ def __init__(self, *, convert_charrefs=True, scripting=False):
"""Initialize and reset this instance.
- If convert_charrefs is True (the default), all character references
+ If convert_charrefs is true (the default), all character references
are automatically converted to the corresponding Unicode characters.
+
+ If *scripting* is false (the default), the content of the
+ ``noscript`` element is parsed normally; if it's true,
+ it's returned as is without being parsed.
"""
super().__init__()
self.convert_charrefs = convert_charrefs
+ self.scripting = scripting
self.reset()
def reset(self):
@@ -172,7 +181,9 @@ class HTMLParser(_markupbase.ParserBase):
def set_cdata_mode(self, elem, *, escapable=False):
self.cdata_elem = elem.lower()
self._escapable = escapable
- if escapable and not self.convert_charrefs:
+ if self.cdata_elem == 'plaintext':
+ self.interesting = re.compile(r'\z')
+ elif escapable and not self.convert_charrefs:
self.interesting = re.compile(r'&|%s(?=[\t\n\r\f />])' % self.cdata_elem,
re.IGNORECASE|re.ASCII)
else:
@@ -294,10 +305,20 @@ class HTMLParser(_markupbase.ParserBase):
k = k - 1
i = self.updatepos(i, k)
continue
+ match = incomplete_charref.match(rawdata, i)
+ if match:
+ if end:
+ self.handle_charref(rawdata[i+2:])
+ i = self.updatepos(i, n)
+ break
+ # incomplete
+ break
+ elif i + 3 < n: # larger than ""
+ # not the end of the buffer, and can't be confused
+ # with some other construct
+ self.handle_data("")
+ i = self.updatepos(i, i + 2)
else:
- if ";" in rawdata[i:]: # bail by consuming
- self.handle_data(rawdata[i:i+2])
- i = self.updatepos(i, i+2)
break
elif startswith('&', i):
match = entityref.match(rawdata, i)
@@ -311,15 +332,13 @@ class HTMLParser(_markupbase.ParserBase):
continue
match = incomplete.match(rawdata, i)
if match:
- # match.group() will contain at least 2 chars
- if end and match.group() == rawdata[i:]:
- k = match.end()
- if k <= i:
- k = n
- i = self.updatepos(i, i + 1)
+ if end:
+ self.handle_entityref(rawdata[i+1:])
+ i = self.updatepos(i, n)
+ break
# incomplete
break
- elif (i + 1) < n:
+ elif i + 1 < n:
# not the end of the buffer, and can't be confused
# with some other construct
self.handle_data("&")
@@ -444,8 +463,10 @@ class HTMLParser(_markupbase.ParserBase):
self.handle_startendtag(tag, attrs)
else:
self.handle_starttag(tag, attrs)
- if tag in self.CDATA_CONTENT_ELEMENTS:
- self.set_cdata_mode(tag)
+ if (tag in self.CDATA_CONTENT_ELEMENTS or
+ (self.scripting and tag == "noscript") or
+ tag == "plaintext"):
+ self.set_cdata_mode(tag, escapable=False)
elif tag in self.RCDATA_CONTENT_ELEMENTS:
self.set_cdata_mode(tag, escapable=True)
return endpos
diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py
index 6a1d69335..e4eff1ea1 100644
--- a/Lib/test/test_htmlparser.py
+++ b/Lib/test/test_htmlparser.py
@@ -8,6 +8,18 @@ from unittest.mock import patch
from test import support
+SAMPLE_RCDATA = (
+ ''
+ ""
+ ''
+ ''
+ ''
+ '\u2603'
+)
+
+SAMPLE_RAWTEXT = SAMPLE_RCDATA + '&☺'
+
+
class EventCollector(html.parser.HTMLParser):
def __init__(self, *args, autocdata=False, **kw):
@@ -97,12 +109,13 @@ class EventCollectorNoNormalize(EventCollector):
class TestCaseBase(unittest.TestCase):
- def get_collector(self):
- return EventCollector(convert_charrefs=False)
+ def get_collector(self, convert_charrefs=False):
+ return EventCollector(convert_charrefs=convert_charrefs)
- def _run_check(self, source, expected_events, collector=None):
+ def _run_check(self, source, expected_events,
+ *, collector=None, convert_charrefs=False):
if collector is None:
- collector = self.get_collector()
+ collector = self.get_collector(convert_charrefs=convert_charrefs)
parser = collector
for s in source:
parser.feed(s)
@@ -116,7 +129,7 @@ class TestCaseBase(unittest.TestCase):
def _run_check_extra(self, source, events):
self._run_check(source, events,
- EventCollectorExtra(convert_charrefs=False))
+ collector=EventCollectorExtra(convert_charrefs=False))
class HTMLParserTestCase(TestCaseBase):
@@ -175,10 +188,87 @@ text
])
def test_unclosed_entityref(self):
- self._run_check("&entityref foo", [
- ("entityref", "entityref"),
- ("data", " foo"),
- ])
+ self._run_check('> <', [('entityref', 'gt'), ('data', ' '), ('entityref', 'lt')],
+ convert_charrefs=False)
+ self._run_check('> <', [('data', '> <')], convert_charrefs=True)
+
+ self._run_check('&undefined <',
+ [('entityref', 'undefined'), ('data', ' '), ('entityref', 'lt')],
+ convert_charrefs=False)
+ self._run_check('&undefined <', [('data', '&undefined <')],
+ convert_charrefs=True)
+
+ self._run_check('>undefined <',
+ [('entityref', 'gtundefined'), ('data', ' '), ('entityref', 'lt')],
+ convert_charrefs=False)
+ self._run_check('>undefined <', [('data', '>undefined <')],
+ convert_charrefs=True)
+
+ self._run_check('& <', [('data', '& '), ('entityref', 'lt')],
+ convert_charrefs=False)
+ self._run_check('& <', [('data', '& <')], convert_charrefs=True)
+
+ def test_eof_in_entityref(self):
+ self._run_check('>', [('entityref', 'gt')], convert_charrefs=False)
+ self._run_check('>', [('data', '>')], convert_charrefs=True)
+
+ self._run_check('&g', [('entityref', 'g')], convert_charrefs=False)
+ self._run_check('&g', [('data', '&g')], convert_charrefs=True)
+
+ self._run_check('&undefined', [('entityref', 'undefined')],
+ convert_charrefs=False)
+ self._run_check('&undefined', [('data', '&undefined')],
+ convert_charrefs=True)
+
+ self._run_check('>undefined', [('entityref', 'gtundefined')],
+ convert_charrefs=False)
+ self._run_check('>undefined', [('data', '>undefined')],
+ convert_charrefs=True)
+
+ self._run_check('&', [('data', '&')], convert_charrefs=False)
+ self._run_check('&', [('data', '&')], convert_charrefs=True)
+
+ def test_unclosed_charref(self):
+ self._run_check('{ <', [('charref', '123'), ('data', ' '), ('entityref', 'lt')],
+ convert_charrefs=False)
+ self._run_check('{ <', [('data', '{ <')], convert_charrefs=True)
+ self._run_check('« <', [('charref', 'xab'), ('data', ' '), ('entityref', 'lt')],
+ convert_charrefs=False)
+ self._run_check('« <', [('data', '\xab <')], convert_charrefs=True)
+
+ self._run_check(' <',
+ [('charref', '123456789'), ('data', ' '), ('entityref', 'lt')],
+ convert_charrefs=False)
+ self._run_check(' <', [('data', '\ufffd <')],
+ convert_charrefs=True)
+ self._run_check(' <',
+ [('charref', 'x123456789'), ('data', ' '), ('entityref', 'lt')],
+ convert_charrefs=False)
+ self._run_check(' <', [('data', '\ufffd <')],
+ convert_charrefs=True)
+
+ self._run_check(' <', [('data', ' '), ('entityref', 'lt')], convert_charrefs=False)
+ self._run_check(' <', [('data', ' <')], convert_charrefs=True)
+ self._run_check(' <', [('data', ' '), ('entityref', 'lt')], convert_charrefs=False)
+ self._run_check(' <', [('data', ' <')], convert_charrefs=True)
+
+ def test_eof_in_charref(self):
+ self._run_check('{', [('charref', '123')], convert_charrefs=False)
+ self._run_check('{', [('data', '{')], convert_charrefs=True)
+ self._run_check('«', [('charref', 'xab')], convert_charrefs=False)
+ self._run_check('«', [('data', '\xab')], convert_charrefs=True)
+
+ self._run_check('', [('charref', '123456789')],
+ convert_charrefs=False)
+ self._run_check('', [('data', '\ufffd')], convert_charrefs=True)
+ self._run_check('', [('charref', 'x123456789')],
+ convert_charrefs=False)
+ self._run_check('', [('data', '\ufffd')], convert_charrefs=True)
+
+ self._run_check('', [('data', '')], convert_charrefs=False)
+ self._run_check('', [('data', '')], convert_charrefs=True)
+ self._run_check('', [('data', '')], convert_charrefs=False)
+ self._run_check('', [('data', '')], convert_charrefs=True)
def test_bad_nesting(self):
# Strangely, this *is* supposed to test that overlapping
@@ -293,30 +383,20 @@ text
'Date().getTime()+\'"><\\/s\'+\'cript>\');\n//]]>'),
'\n\n',
'',
- 'foo = " script>"',
- 'foo = ""',
- 'foo = ""',
- 'foo = ""',
- 'foo = "ſcript>"',
- 'foo = ""',
])
def test_script_content(self, content):
s = f''
- self._run_check(s, [("starttag", "script", []),
- ("data", content),
- ("endtag", "script")])
+ self._run_check(s, [
+ ("starttag", "script", []),
+ ("data", content),
+ ("endtag", "script"),
+ ])
@support.subTests('content', [
'a::before { content: ""; }',
'a::before { content: "¬-an-entity-ref;"; }',
'a::before { content: ""; }',
'a::before { content: "\u2603"; }',
- 'a::before { content: "< /style>"; }',
- 'a::before { content: " style>"; }',
- 'a::before { content: ""; }',
- 'a::before { content: ""; }',
- 'a::before { content: ""; }',
- 'a::before { content: "ſtyle>"; }',
])
def test_style_content(self, content):
s = f''
@@ -324,47 +404,59 @@ text
("data", content),
("endtag", "style")])
- @support.subTests('content', [
- '',
- "",
- '',
- '',
- '',
- '\u2603',
- '< /title>',
- ' title>',
- '',
- '',
- '',
- '',
- ])
- def test_title_content(self, content):
- source = f"{content}"
+ @support.subTests('tag', ['title', 'textarea'])
+ def test_rcdata_content(self, tag):
+ source = f"<{tag}>{SAMPLE_RCDATA}{tag}>"
self._run_check(source, [
- ("starttag", "title", []),
- ("data", content),
- ("endtag", "title"),
+ ("starttag", tag, []),
+ ("data", SAMPLE_RCDATA),
+ ("endtag", tag),
+ ])
+ source = f"<{tag}>&{tag}>"
+ self._run_check(source, [
+ ("starttag", tag, []),
+ ('entityref', 'amp'),
+ ("endtag", tag),
])
- @support.subTests('content', [
- '',
- "",
- '',
- '',
- '',
- '\u2603',
- '< /textarea>',
- ' textarea>',
- '',
- '',
- '',
- ])
- def test_textarea_content(self, content):
- source = f""
+ @support.subTests('tag',
+ ['style', 'xmp', 'iframe', 'noembed', 'noframes', 'script'])
+ def test_rawtext_content(self, tag):
+ source = f"<{tag}>{SAMPLE_RAWTEXT}{tag}>"
self._run_check(source, [
- ("starttag", "textarea", []),
+ ("starttag", tag, []),
+ ("data", SAMPLE_RAWTEXT),
+ ("endtag", tag),
+ ])
+
+ def test_noscript_content(self):
+ source = f""
+ # scripting=False -- normal mode
+ self._run_check(source, [
+ ('starttag', 'noscript', []),
+ ('comment', ' not a comment '),
+ ('starttag', 'not', [('a', 'start tag')]),
+ ('unknown decl', 'CDATA[not a cdata'),
+ ('comment', 'not a bogus comment'),
+ ('endtag', 'not'),
+ ('data', '☃'),
+ ('entityref', 'amp'),
+ ('charref', '9786'),
+ ('endtag', 'noscript'),
+ ])
+ # scripting=True -- RAWTEXT mode
+ self._run_check(source, [
+ ("starttag", "noscript", []),
+ ("data", SAMPLE_RAWTEXT),
+ ("endtag", "noscript"),
+ ], collector=EventCollector(scripting=True))
+
+ def test_plaintext_content(self):
+ content = SAMPLE_RAWTEXT + '' # not closing
+ source = f"{content}"
+ self._run_check(source, [
+ ("starttag", "plaintext", []),
("data", content),
- ("endtag", "textarea"),
])
@support.subTests('endtag', ['script', 'SCRIPT', 'script ', 'script\n',
@@ -381,52 +473,65 @@ text
("endtag", "script")],
collector=EventCollectorNoNormalize(convert_charrefs=False))
- @support.subTests('endtag', ['style', 'STYLE', 'style ', 'style\n',
- 'style/', 'style foo=bar', 'style foo=">"'])
- def test_style_closing_tag(self, endtag):
- content = """
- b::before { content: ""; }
- p::before { content: "¬-an-entity-ref;"; }
- a::before { content: ""; }
- a::after { content: ""; }
- """
- s = f'