From 2eec9194166266c9a87066aad5a9178380dc8067 Mon Sep 17 00:00:00 2001 From: CPython Developers <> Date: Sun, 14 Aug 2022 23:21:01 +0900 Subject: [PATCH] Update xml and test_xml_etree --- Lib/test/test_xml_etree.py | 102 +++++++++++++++----- Lib/xml/dom/expatbuilder.py | 8 +- Lib/xml/dom/minidom.py | 56 ++++++++--- Lib/xml/dom/pulldom.py | 7 ++ Lib/xml/dom/xmlbuilder.py | 25 +---- Lib/xml/etree/ElementInclude.py | 58 ++++++++++-- Lib/xml/etree/ElementPath.py | 34 +++++-- Lib/xml/etree/ElementTree.py | 160 ++++++++++++++++++-------------- Lib/xml/etree/__init__.py | 2 +- Lib/xml/sax/__init__.py | 10 +- Lib/xml/sax/expatreader.py | 14 +-- Lib/xml/sax/handler.py | 45 +++++++++ Lib/xml/sax/saxutils.py | 5 +- 13 files changed, 358 insertions(+), 168 deletions(-) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 4d6710121..b721f028a 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -10,7 +10,6 @@ import functools import html import io import itertools -import locale import operator import os import pickle @@ -130,6 +129,9 @@ def checkwarnings(*filters, quiet=False): return newtest return decorator +def convlinesep(data): + return data.replace(b'\n', os.linesep.encode()) + class ModuleTest(unittest.TestCase): def test_sanity(self): @@ -1023,17 +1025,15 @@ class ElementTreeTest(unittest.TestCase): @unittest.expectedFailure def test_tostring_xml_declaration_unicode_encoding(self): elem = ET.XML('') - preferredencoding = locale.getpreferredencoding() self.assertEqual( - f"\n", - ET.tostring(elem, encoding='unicode', xml_declaration=True) + ET.tostring(elem, encoding='unicode', xml_declaration=True), + "\n" ) # TODO: RUSTPYTHON @unittest.expectedFailure def test_tostring_xml_declaration_cases(self): elem = ET.XML('ø') - preferredencoding = locale.getpreferredencoding() TESTCASES = [ # (expected_retval, encoding, xml_declaration) # ... xml_declaration = None @@ -1060,7 +1060,7 @@ class ElementTreeTest(unittest.TestCase): b"ø", 'US-ASCII', True), (b"\n" b"\xf8", 'ISO-8859-1', True), - (f"\n" + ("\n" "ø", 'unicode', True), ] @@ -1102,11 +1102,10 @@ class ElementTreeTest(unittest.TestCase): b"\n" ) - preferredencoding = locale.getpreferredencoding() stringlist = ET.tostringlist(elem, encoding='unicode', xml_declaration=True) self.assertEqual( ''.join(stringlist), - f"\n" + "\n" ) self.assertRegex(stringlist[0], r"^<\?xml version='1.0' encoding='.+'?>") self.assertEqual(['', '', ''], stringlist[1:]) @@ -3914,38 +3913,91 @@ class IOTest(unittest.TestCase): @unittest.expectedFailure def test_write_to_filename(self): self.addCleanup(os_helper.unlink, TESTFN) - tree = ET.ElementTree(ET.XML('''''')) + tree = ET.ElementTree(ET.XML('''\xf8''')) tree.write(TESTFN) with open(TESTFN, 'rb') as f: - self.assertEqual(f.read(), b'''''') + self.assertEqual(f.read(), b'''ø''') + + def test_write_to_filename_with_encoding(self): + self.addCleanup(os_helper.unlink, TESTFN) + tree = ET.ElementTree(ET.XML('''\xf8''')) + tree.write(TESTFN, encoding='utf-8') + with open(TESTFN, 'rb') as f: + self.assertEqual(f.read(), b'''\xc3\xb8''') + + tree.write(TESTFN, encoding='ISO-8859-1') + with open(TESTFN, 'rb') as f: + self.assertEqual(f.read(), convlinesep( + b'''\n''' + b'''\xf8''')) + + def test_write_to_filename_as_unicode(self): + self.addCleanup(os_helper.unlink, TESTFN) + with open(TESTFN, 'w') as f: + encoding = f.encoding + os_helper.unlink(TESTFN) + + tree = ET.ElementTree(ET.XML('''\xf8''')) + tree.write(TESTFN, encoding='unicode') + with open(TESTFN, 'rb') as f: + self.assertEqual(f.read(), b"\xc3\xb8") # TODO: RUSTPYTHON @unittest.expectedFailure def test_write_to_text_file(self): self.addCleanup(os_helper.unlink, TESTFN) - tree = ET.ElementTree(ET.XML('''''')) + tree = ET.ElementTree(ET.XML('''\xf8''')) with open(TESTFN, 'w', encoding='utf-8') as f: tree.write(f, encoding='unicode') self.assertFalse(f.closed) with open(TESTFN, 'rb') as f: - self.assertEqual(f.read(), b'''''') + self.assertEqual(f.read(), b'''\xc3\xb8''') + + with open(TESTFN, 'w', encoding='ascii', errors='xmlcharrefreplace') as f: + tree.write(f, encoding='unicode') + self.assertFalse(f.closed) + with open(TESTFN, 'rb') as f: + self.assertEqual(f.read(), b'''ø''') + + with open(TESTFN, 'w', encoding='ISO-8859-1') as f: + tree.write(f, encoding='unicode') + self.assertFalse(f.closed) + with open(TESTFN, 'rb') as f: + self.assertEqual(f.read(), b'''\xf8''') # TODO: RUSTPYTHON @unittest.expectedFailure def test_write_to_binary_file(self): self.addCleanup(os_helper.unlink, TESTFN) - tree = ET.ElementTree(ET.XML('''''')) + tree = ET.ElementTree(ET.XML('''\xf8''')) with open(TESTFN, 'wb') as f: tree.write(f) self.assertFalse(f.closed) with open(TESTFN, 'rb') as f: - self.assertEqual(f.read(), b'''''') + self.assertEqual(f.read(), b'''ø''') + + def test_write_to_binary_file_with_encoding(self): + self.addCleanup(os_helper.unlink, TESTFN) + tree = ET.ElementTree(ET.XML('''\xf8''')) + with open(TESTFN, 'wb') as f: + tree.write(f, encoding='utf-8') + self.assertFalse(f.closed) + with open(TESTFN, 'rb') as f: + self.assertEqual(f.read(), b'''\xc3\xb8''') + + with open(TESTFN, 'wb') as f: + tree.write(f, encoding='ISO-8859-1') + self.assertFalse(f.closed) + with open(TESTFN, 'rb') as f: + self.assertEqual(f.read(), + b'''\n''' + b'''\xf8''') # TODO: RUSTPYTHON @unittest.expectedFailure def test_write_to_binary_file_with_bom(self): self.addCleanup(os_helper.unlink, TESTFN) - tree = ET.ElementTree(ET.XML('''''')) + tree = ET.ElementTree(ET.XML('''\xf8''')) # test BOM writing to buffered file with open(TESTFN, 'wb') as f: tree.write(f, encoding='utf-16') @@ -3953,7 +4005,7 @@ class IOTest(unittest.TestCase): with open(TESTFN, 'rb') as f: self.assertEqual(f.read(), '''\n''' - ''''''.encode("utf-16")) + '''\xf8'''.encode("utf-16")) # test BOM writing to non-buffered file with open(TESTFN, 'wb', buffering=0) as f: tree.write(f, encoding='utf-16') @@ -3961,7 +4013,7 @@ class IOTest(unittest.TestCase): with open(TESTFN, 'rb') as f: self.assertEqual(f.read(), '''\n''' - ''''''.encode("utf-16")) + '''\xf8'''.encode("utf-16")) # TODO: RUSTPYTHON @unittest.expectedFailure @@ -3974,10 +4026,10 @@ class IOTest(unittest.TestCase): # TODO: RUSTPYTHON @unittest.expectedFailure def test_write_to_stringio(self): - tree = ET.ElementTree(ET.XML('''''')) + tree = ET.ElementTree(ET.XML('''\xf8''')) stream = io.StringIO() tree.write(stream, encoding='unicode') - self.assertEqual(stream.getvalue(), '''''') + self.assertEqual(stream.getvalue(), '''\xf8''') # TODO: RUSTPYTHON @unittest.expectedFailure @@ -3990,10 +4042,10 @@ class IOTest(unittest.TestCase): # TODO: RUSTPYTHON @unittest.expectedFailure def test_write_to_bytesio(self): - tree = ET.ElementTree(ET.XML('''''')) + tree = ET.ElementTree(ET.XML('''\xf8''')) raw = io.BytesIO() tree.write(raw) - self.assertEqual(raw.getvalue(), b'''''') + self.assertEqual(raw.getvalue(), b'''ø''') class dummy: pass @@ -4011,12 +4063,12 @@ class IOTest(unittest.TestCase): # TODO: RUSTPYTHON @unittest.expectedFailure def test_write_to_user_text_writer(self): - tree = ET.ElementTree(ET.XML('''''')) + tree = ET.ElementTree(ET.XML('''\xf8''')) stream = io.StringIO() writer = self.dummy() writer.write = stream.write tree.write(writer, encoding='unicode') - self.assertEqual(stream.getvalue(), '''''') + self.assertEqual(stream.getvalue(), '''\xf8''') # TODO: RUSTPYTHON @unittest.expectedFailure @@ -4032,12 +4084,12 @@ class IOTest(unittest.TestCase): # TODO: RUSTPYTHON @unittest.expectedFailure def test_write_to_user_binary_writer(self): - tree = ET.ElementTree(ET.XML('''''')) + tree = ET.ElementTree(ET.XML('''\xf8''')) raw = io.BytesIO() writer = self.dummy() writer.write = raw.write tree.write(writer) - self.assertEqual(raw.getvalue(), b'''''') + self.assertEqual(raw.getvalue(), b'''ø''') # TODO: RUSTPYTHON @unittest.expectedFailure diff --git a/Lib/xml/dom/expatbuilder.py b/Lib/xml/dom/expatbuilder.py index 2bd835b03..199c22d0a 100644 --- a/Lib/xml/dom/expatbuilder.py +++ b/Lib/xml/dom/expatbuilder.py @@ -204,11 +204,11 @@ class ExpatBuilder: buffer = file.read(16*1024) if not buffer: break - parser.Parse(buffer, 0) + parser.Parse(buffer, False) if first_buffer and self.document.documentElement: self._setup_subset(buffer) first_buffer = False - parser.Parse("", True) + parser.Parse(b"", True) except ParseEscape: pass doc = self.document @@ -637,7 +637,7 @@ class FragmentBuilder(ExpatBuilder): nsattrs = self._getNSattrs() # get ns decls from node's ancestors document = _FRAGMENT_BUILDER_TEMPLATE % (ident, subset, nsattrs) try: - parser.Parse(document, 1) + parser.Parse(document, True) except: self.reset() raise @@ -697,7 +697,7 @@ class FragmentBuilder(ExpatBuilder): self.fragment = self.document.createDocumentFragment() self.curNode = self.fragment try: - parser.Parse(self._source, 1) + parser.Parse(self._source, True) finally: self.curNode = old_cur_node self.document = old_document diff --git a/Lib/xml/dom/minidom.py b/Lib/xml/dom/minidom.py index 24957ea14..d09ef5e7d 100644 --- a/Lib/xml/dom/minidom.py +++ b/Lib/xml/dom/minidom.py @@ -43,10 +43,11 @@ class Node(xml.dom.Node): def __bool__(self): return True - def toxml(self, encoding=None): - return self.toprettyxml("", "", encoding) + def toxml(self, encoding=None, standalone=None): + return self.toprettyxml("", "", encoding, standalone) - def toprettyxml(self, indent="\t", newl="\n", encoding=None): + def toprettyxml(self, indent="\t", newl="\n", encoding=None, + standalone=None): if encoding is None: writer = io.StringIO() else: @@ -56,7 +57,7 @@ class Node(xml.dom.Node): newline='\n') if self.nodeType == Node.DOCUMENT_NODE: # Can pass encoding only to document, to put it into XML header - self.writexml(writer, "", indent, newl, encoding) + self.writexml(writer, "", indent, newl, encoding, standalone) else: self.writexml(writer, "", indent, newl) if encoding is None: @@ -718,6 +719,14 @@ class Element(Node): Node.unlink(self) def getAttribute(self, attname): + """Returns the value of the specified attribute. + + Returns the value of the element's attribute named attname as + a string. An empty string is returned if the element does not + have such an attribute. Note that an empty string may also be + returned as an explicitly given attribute value, use the + hasAttribute method to distinguish these two cases. + """ if self._attrs is None: return "" try: @@ -823,10 +832,16 @@ class Element(Node): # Restore this since the node is still useful and otherwise # unlinked node.ownerDocument = self.ownerDocument + return node removeAttributeNodeNS = removeAttributeNode def hasAttribute(self, name): + """Checks whether the element has an attribute with the specified name. + + Returns True if the element has an attribute with the specified name. + Otherwise, returns False. + """ if self._attrs is None: return False return name in self._attrs @@ -837,6 +852,11 @@ class Element(Node): return (namespaceURI, localName) in self._attrsNS def getElementsByTagName(self, name): + """Returns all descendant elements with the given tag name. + + Returns the list of all descendant elements (not direct children + only) with the specified tag name. + """ return _get_elements_by_tagName_helper(self, name, NodeList()) def getElementsByTagNameNS(self, namespaceURI, localName): @@ -847,22 +867,27 @@ class Element(Node): return "" % (self.tagName, id(self)) def writexml(self, writer, indent="", addindent="", newl=""): + """Write an XML element to a file-like object + + Write the element to the writer object that must provide + a write method (e.g. a file or StringIO object). + """ # indent = current indentation # addindent = indentation to add to higher levels # newl = newline string writer.write(indent+"<" + self.tagName) attrs = self._get_attributes() - a_names = sorted(attrs.keys()) - for a_name in a_names: + for a_name in attrs.keys(): writer.write(" %s=\"" % a_name) _write_data(writer, attrs[a_name].value) writer.write("\"") if self.childNodes: writer.write(">") if (len(self.childNodes) == 1 and - self.childNodes[0].nodeType == Node.TEXT_NODE): + self.childNodes[0].nodeType in ( + Node.TEXT_NODE, Node.CDATA_SECTION_NODE)): self.childNodes[0].writexml(writer, '', '', '') else: writer.write(newl) @@ -1786,12 +1811,17 @@ class Document(Node, DocumentLS): raise xml.dom.NotSupportedErr("cannot import document type nodes") return _clone_node(node, deep, self) - def writexml(self, writer, indent="", addindent="", newl="", encoding=None): - if encoding is None: - writer.write(''+newl) - else: - writer.write('%s' % ( - encoding, newl)) + def writexml(self, writer, indent="", addindent="", newl="", encoding=None, + standalone=None): + declarations = [] + + if encoding: + declarations.append(f'encoding="{encoding}"') + if standalone is not None: + declarations.append(f'standalone="{"yes" if standalone else "no"}"') + + writer.write(f'{newl}') + for node in self.childNodes: node.writexml(writer, indent, addindent, newl) diff --git a/Lib/xml/dom/pulldom.py b/Lib/xml/dom/pulldom.py index 43504f765..96a8d5951 100644 --- a/Lib/xml/dom/pulldom.py +++ b/Lib/xml/dom/pulldom.py @@ -217,6 +217,13 @@ class DOMEventStream: self.parser.setContentHandler(self.pulldom) def __getitem__(self, pos): + import warnings + warnings.warn( + "DOMEventStream's __getitem__ method ignores 'pos' parameter. " + "Use iterator protocol instead.", + DeprecationWarning, + stacklevel=2 + ) rc = self.getEvent() if rc: return rc diff --git a/Lib/xml/dom/xmlbuilder.py b/Lib/xml/dom/xmlbuilder.py index e9a153647..8a2002634 100644 --- a/Lib/xml/dom/xmlbuilder.py +++ b/Lib/xml/dom/xmlbuilder.py @@ -1,7 +1,6 @@ """Implementation of the DOM Level 3 'LS-Load' feature.""" import copy -import warnings import xml.dom from xml.dom.NodeFilter import NodeFilter @@ -80,7 +79,7 @@ class DOMBuilder: settings = self._settings[(_name_xform(name), state)] except KeyError: raise xml.dom.NotSupportedErr( - "unsupported feature: %r" % (name,)) + "unsupported feature: %r" % (name,)) from None else: for name, value in settings: setattr(self._options, name, value) @@ -332,29 +331,10 @@ class DOMBuilderFilter: del NodeFilter -class _AsyncDeprecatedProperty: - def warn(self, cls): - clsname = cls.__name__ - warnings.warn( - "{cls}.async is deprecated; use {cls}.async_".format(cls=clsname), - DeprecationWarning) - - def __get__(self, instance, cls): - self.warn(cls) - if instance is not None: - return instance.async_ - return False - - def __set__(self, instance, value): - self.warn(type(instance)) - setattr(instance, 'async_', value) - - class DocumentLS: """Mixin to create documents that conform to the load/save spec.""" async_ = False - locals()['async'] = _AsyncDeprecatedProperty() # Avoid DeprecationWarning def _get_async(self): return False @@ -384,9 +364,6 @@ class DocumentLS: return snode.toxml() -del _AsyncDeprecatedProperty - - class DOMImplementationLS: MODE_SYNCHRONOUS = 1 MODE_ASYNCHRONOUS = 2 diff --git a/Lib/xml/etree/ElementInclude.py b/Lib/xml/etree/ElementInclude.py index 963470e3b..40a9b2229 100644 --- a/Lib/xml/etree/ElementInclude.py +++ b/Lib/xml/etree/ElementInclude.py @@ -42,7 +42,7 @@ # -------------------------------------------------------------------- # Licensed to PSF under a Contributor Agreement. -# See http://www.python.org/psf/license for licensing details. +# See https://www.python.org/psf/license for licensing details. ## # Limited XInclude support for the ElementTree package. @@ -50,18 +50,28 @@ import copy from . import ElementTree +from urllib.parse import urljoin XINCLUDE = "{http://www.w3.org/2001/XInclude}" XINCLUDE_INCLUDE = XINCLUDE + "include" XINCLUDE_FALLBACK = XINCLUDE + "fallback" +# For security reasons, the inclusion depth is limited to this read-only value by default. +DEFAULT_MAX_INCLUSION_DEPTH = 6 + + ## # Fatal include error. class FatalIncludeError(SyntaxError): pass + +class LimitedRecursiveIncludeError(FatalIncludeError): + pass + + ## # Default loader. This loader reads an included resource from disk. # @@ -92,13 +102,33 @@ def default_loader(href, parse, encoding=None): # @param loader Optional resource loader. If omitted, it defaults # to {@link default_loader}. If given, it should be a callable # that implements the same interface as default_loader. +# @param base_url The base URL of the original file, to resolve +# relative include file references. +# @param max_depth The maximum number of recursive inclusions. +# Limited to reduce the risk of malicious content explosion. +# Pass a negative value to disable the limitation. +# @throws LimitedRecursiveIncludeError If the {@link max_depth} was exceeded. # @throws FatalIncludeError If the function fails to include a given # resource, or if the tree contains malformed XInclude elements. -# @throws OSError If the function fails to load a given resource. +# @throws IOError If the function fails to load a given resource. +# @returns the node or its replacement if it was an XInclude node -def include(elem, loader=None): +def include(elem, loader=None, base_url=None, + max_depth=DEFAULT_MAX_INCLUSION_DEPTH): + if max_depth is None: + max_depth = -1 + elif max_depth < 0: + raise ValueError("expected non-negative depth or None for 'max_depth', got %r" % max_depth) + + if hasattr(elem, 'getroot'): + elem = elem.getroot() if loader is None: loader = default_loader + + _include(elem, loader, base_url, max_depth, set()) + + +def _include(elem, loader, base_url, max_depth, _parent_hrefs): # look for xinclude elements i = 0 while i < len(elem): @@ -106,14 +136,24 @@ def include(elem, loader=None): if e.tag == XINCLUDE_INCLUDE: # process xinclude directive href = e.get("href") + if base_url: + href = urljoin(base_url, href) parse = e.get("parse", "xml") if parse == "xml": + if href in _parent_hrefs: + raise FatalIncludeError("recursive include of %s" % href) + if max_depth == 0: + raise LimitedRecursiveIncludeError( + "maximum xinclude depth reached when including file %s" % href) + _parent_hrefs.add(href) node = loader(href, parse) if node is None: raise FatalIncludeError( "cannot load %r as %r" % (href, parse) ) - node = copy.copy(node) + node = copy.copy(node) # FIXME: this makes little sense with recursive includes + _include(node, loader, href, max_depth - 1, _parent_hrefs) + _parent_hrefs.remove(href) if e.tail: node.tail = (node.tail or "") + e.tail elem[i] = node @@ -123,11 +163,13 @@ def include(elem, loader=None): raise FatalIncludeError( "cannot load %r as %r" % (href, parse) ) + if e.tail: + text += e.tail if i: node = elem[i-1] - node.tail = (node.tail or "") + text + (e.tail or "") + node.tail = (node.tail or "") + text else: - elem.text = (elem.text or "") + text + (e.tail or "") + elem.text = (elem.text or "") + text del elem[i] continue else: @@ -139,5 +181,5 @@ def include(elem, loader=None): "xi:fallback tag must be child of xi:include (%r)" % e.tag ) else: - include(e, loader) - i = i + 1 + _include(e, loader, base_url, max_depth, _parent_hrefs) + i += 1 diff --git a/Lib/xml/etree/ElementPath.py b/Lib/xml/etree/ElementPath.py index d318e65d8..cd3c354d0 100644 --- a/Lib/xml/etree/ElementPath.py +++ b/Lib/xml/etree/ElementPath.py @@ -48,7 +48,7 @@ # -------------------------------------------------------------------- # Licensed to PSF under a Contributor Agreement. -# See http://www.python.org/psf/license for licensing details. +# See https://www.python.org/psf/license for licensing details. ## # Implementation module for XPath support. There's usually no reason @@ -65,8 +65,9 @@ xpath_tokenizer_re = re.compile( r"//?|" r"\.\.|" r"\(\)|" + r"!=|" r"[/.*:\[\]\(\)@=])|" - r"((?:\{[^}]+\})?[^/\[\]\(\)@=\s]+)|" + r"((?:\{[^}]+\})?[^/\[\]\(\)@!=\s]+)|" r"\s+" ) @@ -225,7 +226,6 @@ def prepare_parent(next, token): def prepare_predicate(next, token): # FIXME: replace with real parser!!! refs: - # http://effbot.org/zone/simple-iterator-parser.htm # http://javascript.crockford.com/tdop/tdop.html signature = [] predicate = [] @@ -253,15 +253,19 @@ def prepare_predicate(next, token): if elem.get(key) is not None: yield elem return select - if signature == "@-='": - # [@attribute='value'] + if signature == "@-='" or signature == "@-!='": + # [@attribute='value'] or [@attribute!='value'] key = predicate[1] value = predicate[-1] def select(context, result): for elem in result: if elem.get(key) == value: yield elem - return select + def select_negated(context, result): + for elem in result: + if (attr_value := elem.get(key)) is not None and attr_value != value: + yield elem + return select_negated if '!=' in signature else select if signature == "-" and not re.match(r"\-?\d+$", predicate[0]): # [tag] tag = predicate[0] @@ -270,8 +274,10 @@ def prepare_predicate(next, token): if elem.find(tag) is not None: yield elem return select - if signature == ".='" or (signature == "-='" and not re.match(r"\-?\d+$", predicate[0])): - # [.='value'] or [tag='value'] + if signature == ".='" or signature == ".!='" or ( + (signature == "-='" or signature == "-!='") + and not re.match(r"\-?\d+$", predicate[0])): + # [.='value'] or [tag='value'] or [.!='value'] or [tag!='value'] tag = predicate[0] value = predicate[-1] if tag: @@ -281,12 +287,22 @@ def prepare_predicate(next, token): if "".join(e.itertext()) == value: yield elem break + def select_negated(context, result): + for elem in result: + for e in elem.iterfind(tag): + if "".join(e.itertext()) != value: + yield elem + break else: def select(context, result): for elem in result: if "".join(elem.itertext()) == value: yield elem - return select + def select_negated(context, result): + for elem in result: + if "".join(elem.itertext()) != value: + yield elem + return select_negated if '!=' in signature else select if signature == "-" or signature == "-()" or signature == "-()-": # [index] or [last()] or [last()-index] if signature == "-": diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index f8538dfb2..2503d9ee7 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -35,7 +35,7 @@ #--------------------------------------------------------------------- # Licensed to PSF under a Contributor Agreement. -# See http://www.python.org/psf/license for licensing details. +# See https://www.python.org/psf/license for licensing details. # # ElementTree # Copyright (c) 1999-2008 by Fredrik Lundh. All rights reserved. @@ -76,7 +76,7 @@ __all__ = [ "dump", "Element", "ElementTree", "fromstring", "fromstringlist", - "iselement", "iterparse", + "indent", "iselement", "iterparse", "parse", "ParseError", "PI", "ProcessingInstruction", "QName", @@ -195,6 +195,13 @@ class Element: original tree. """ + warnings.warn( + "elem.copy() is deprecated. Use copy.copy(elem) instead.", + DeprecationWarning + ) + return self.__copy__() + + def __copy__(self): elem = self.makeelement(self.tag, self.attrib) elem.text = self.text elem.tail = self.tail @@ -273,19 +280,6 @@ class Element: # assert iselement(element) self._children.remove(subelement) - def getchildren(self): - """(Deprecated) Return all subelements. - - Elements are returned in document order. - - """ - warnings.warn( - "This method will be removed in future versions. " - "Use 'list(elem)' or iteration over elem instead.", - DeprecationWarning, stacklevel=2 - ) - return self._children - def find(self, path, namespaces=None): """Find first matching element by tag name or path. @@ -409,15 +403,6 @@ class Element: for e in self._children: yield from e.iter(tag) - # compatibility - def getiterator(self, tag=None): - warnings.warn( - "This method will be removed in future versions. " - "Use 'elem.iter()' or 'list(elem.iter())' instead.", - DeprecationWarning, stacklevel=2 - ) - return list(self.iter(tag)) - def itertext(self): """Create text iterator. @@ -617,15 +602,6 @@ class ElementTree: # assert self._root is not None return self._root.iter(tag) - # compatibility - def getiterator(self, tag=None): - warnings.warn( - "This method will be removed in future versions. " - "Use 'tree.iter()' or 'list(tree.iter())' instead.", - DeprecationWarning, stacklevel=2 - ) - return list(self.iter(tag)) - def find(self, path, namespaces=None): """Find first matching element by tag name or path. @@ -752,16 +728,11 @@ class ElementTree: encoding = "utf-8" else: encoding = "us-ascii" - enc_lower = encoding.lower() - with _get_writer(file_or_filename, enc_lower) as write: + with _get_writer(file_or_filename, encoding) as (write, declared_encoding): if method == "xml" and (xml_declaration or (xml_declaration is None and - enc_lower not in ("utf-8", "us-ascii", "unicode"))): - declared_encoding = encoding - if enc_lower == "unicode": - # Retrieve the default encoding for the xml declaration - import locale - declared_encoding = locale.getpreferredencoding() + encoding.lower() != "unicode" and + declared_encoding.lower() not in ("utf-8", "us-ascii"))): write("\n" % ( declared_encoding,)) if method == "text": @@ -786,19 +757,17 @@ def _get_writer(file_or_filename, encoding): write = file_or_filename.write except AttributeError: # file_or_filename is a file name - if encoding == "unicode": - file = open(file_or_filename, "w") - else: - file = open(file_or_filename, "w", encoding=encoding, - errors="xmlcharrefreplace") - with file: - yield file.write + if encoding.lower() == "unicode": + encoding="utf-8" + with open(file_or_filename, "w", encoding=encoding, + errors="xmlcharrefreplace") as file: + yield file.write, encoding else: # file_or_filename is a file-like object # encoding determines if it is a text or binary writer - if encoding == "unicode": + if encoding.lower() == "unicode": # use a text writer as is - yield write + yield write, getattr(file_or_filename, "encoding", None) or "utf-8" else: # wrap a binary writer with TextIOWrapper with contextlib.ExitStack() as stack: @@ -829,7 +798,7 @@ def _get_writer(file_or_filename, encoding): # Keep the original file open when the TextIOWrapper is # destroyed stack.callback(file.detach) - yield file.write + yield file.write, encoding def _namespaces(elem, default_namespace=None): # identify namespaces used in this tree @@ -1081,15 +1050,15 @@ def _escape_attrib(text): text = text.replace(">", ">") if "\"" in text: text = text.replace("\"", """) - # The following business with carriage returns is to satisfy - # Section 2.11 of the XML specification, stating that - # CR or CR LN should be replaced with just LN + # Although section 2.11 of the XML specification states that CR or + # CR LN should be replaced with just LN, it applies only to EOLNs + # which take part of organizing file into lines. Within attributes, + # we are replacing these with entity numbers, so they do not count. # http://www.w3.org/TR/REC-xml/#sec-line-ends - if "\r\n" in text: - text = text.replace("\r\n", "\n") + # The current solution, contained in following six lines, was + # discussed in issue 17582 and 39011. if "\r" in text: - text = text.replace("\r", "\n") - #The following four lines are issue 17582 + text = text.replace("\r", " ") if "\n" in text: text = text.replace("\n", " ") if "\t" in text: @@ -1185,6 +1154,57 @@ def dump(elem): if not tail or tail[-1] != "\n": sys.stdout.write("\n") + +def indent(tree, space=" ", level=0): + """Indent an XML document by inserting newlines and indentation space + after elements. + + *tree* is the ElementTree or Element to modify. The (root) element + itself will not be changed, but the tail text of all elements in its + subtree will be adapted. + + *space* is the whitespace to insert for each indentation level, two + space characters by default. + + *level* is the initial indentation level. Setting this to a higher + value than 0 can be used for indenting subtrees that are more deeply + nested inside of a document. + """ + if isinstance(tree, ElementTree): + tree = tree.getroot() + if level < 0: + raise ValueError(f"Initial indentation level must be >= 0, got {level}") + if not len(tree): + return + + # Reduce the memory consumption by reusing indentation strings. + indentations = ["\n" + level * space] + + def _indent_children(elem, level): + # Start a new indentation level for the first child. + child_level = level + 1 + try: + child_indentation = indentations[child_level] + except IndexError: + child_indentation = indentations[level] + space + indentations.append(child_indentation) + + if not elem.text or not elem.text.strip(): + elem.text = child_indentation + + for child in elem: + if len(child): + _indent_children(child, child_level) + if not child.tail or not child.tail.strip(): + child.tail = child_indentation + + # Dedent after the last child by overwriting the previous indentation. + if not child.tail.strip(): + child.tail = indentations[level] + + _indent_children(tree, 0) + + # -------------------------------------------------------------------- # parsing @@ -1221,8 +1241,14 @@ def iterparse(source, events=None, parser=None): # Use the internal, undocumented _parser argument for now; When the # parser argument of iterparse is removed, this can be killed. pullparser = XMLPullParser(events=events, _parser=parser) - def iterator(): + + def iterator(source): + close_source = False try: + if not hasattr(source, "read"): + source = open(source, "rb") + close_source = True + yield None while True: yield from pullparser.read_events() # load event buffer @@ -1238,16 +1264,12 @@ def iterparse(source, events=None, parser=None): source.close() class IterParseIterator(collections.abc.Iterator): - __next__ = iterator().__next__ + __next__ = iterator(source).__next__ it = IterParseIterator() it.root = None del iterator, IterParseIterator - close_source = False - if not hasattr(source, "read"): - source = open(source, "rb") - close_source = True - + next(it) return it @@ -1256,7 +1278,7 @@ class XMLPullParser: def __init__(self, events=None, *, _parser=None): # The _parser argument is for internal use only and must not be relied # upon in user code. It will be removed in a future release. - # See http://bugs.python.org/issue17741 for more details. + # See https://bugs.python.org/issue17741 for more details. self._events_queue = collections.deque() self._parser = _parser or XMLParser(target=TreeBuilder()) @@ -1533,7 +1555,6 @@ class XMLParser: # Configure pyexpat: buffering, new-style attribute handling. parser.buffer_text = 1 parser.ordered_attributes = 1 - parser.specified_attributes = 1 self._doctype = None self.entity = {} try: @@ -1553,7 +1574,6 @@ class XMLParser: for event_name in events_to_report: if event_name == "start": parser.ordered_attributes = 1 - parser.specified_attributes = 1 def handler(tag, attrib_in, event=event_name, append=append, start=self._start): append((event, start(tag, attrib_in))) @@ -1690,14 +1710,14 @@ class XMLParser: def feed(self, data): """Feed encoded data to parser.""" try: - self.parser.Parse(data, 0) + self.parser.Parse(data, False) except self._error as v: self._raiseerror(v) def close(self): """Finish feeding data to parser and return element structure.""" try: - self.parser.Parse("", 1) # end of data + self.parser.Parse(b"", True) # end of data except self._error as v: self._raiseerror(v) try: diff --git a/Lib/xml/etree/__init__.py b/Lib/xml/etree/__init__.py index 27fd8f6d4..e2ec53421 100644 --- a/Lib/xml/etree/__init__.py +++ b/Lib/xml/etree/__init__.py @@ -30,4 +30,4 @@ # -------------------------------------------------------------------- # Licensed to PSF under a Contributor Agreement. -# See http://www.python.org/psf/license for licensing details. +# See https://www.python.org/psf/license for licensing details. diff --git a/Lib/xml/sax/__init__.py b/Lib/xml/sax/__init__.py index 13f6cf58d..17b75879e 100644 --- a/Lib/xml/sax/__init__.py +++ b/Lib/xml/sax/__init__.py @@ -67,18 +67,18 @@ if sys.platform[:4] == "java" and sys.registry.containsKey(_key): default_parser_list = sys.registry.getProperty(_key).split(",") -def make_parser(parser_list = []): +def make_parser(parser_list=()): """Creates and returns a SAX parser. Creates the first parser it is able to instantiate of the ones - given in the list created by doing parser_list + - default_parser_list. The lists must contain the names of Python + given in the iterable created by chaining parser_list and + default_parser_list. The iterables must contain the names of Python modules containing both a SAX parser and a create_parser function.""" - for parser_name in parser_list + default_parser_list: + for parser_name in list(parser_list) + default_parser_list: try: return _create_parser(parser_name) - except ImportError as e: + except ImportError: import sys if parser_name in sys.modules: # The parser module was found, but importing it diff --git a/Lib/xml/sax/expatreader.py b/Lib/xml/sax/expatreader.py index 5066ffc2f..e334ac9fe 100644 --- a/Lib/xml/sax/expatreader.py +++ b/Lib/xml/sax/expatreader.py @@ -93,7 +93,7 @@ class ExpatParser(xmlreader.IncrementalParser, xmlreader.Locator): self._parser = None self._namespaces = namespaceHandling self._lex_handler_prop = None - self._parsing = 0 + self._parsing = False self._entity_stack = [] self._external_ges = 0 self._interning = None @@ -203,10 +203,10 @@ class ExpatParser(xmlreader.IncrementalParser, xmlreader.Locator): # IncrementalParser methods - def feed(self, data, isFinal = 0): + def feed(self, data, isFinal=False): if not self._parsing: self.reset() - self._parsing = 1 + self._parsing = True self._cont_handler.startDocument() try: @@ -237,13 +237,13 @@ class ExpatParser(xmlreader.IncrementalParser, xmlreader.Locator): # If we are completing an external entity, do nothing here return try: - self.feed("", isFinal = 1) + self.feed(b"", isFinal=True) self._cont_handler.endDocument() - self._parsing = 0 + self._parsing = False # break cycle created by expat handlers pointing to our methods self._parser = None finally: - self._parsing = 0 + self._parsing = False if self._parser is not None: # Keep ErrorColumnNumber and ErrorLineNumber after closing. parser = _ClosedParser() @@ -307,7 +307,7 @@ class ExpatParser(xmlreader.IncrementalParser, xmlreader.Locator): self._parser.SetParamEntityParsing( expat.XML_PARAM_ENTITY_PARSING_UNLESS_STANDALONE) - self._parsing = 0 + self._parsing = False self._entity_stack = [] # Locator methods diff --git a/Lib/xml/sax/handler.py b/Lib/xml/sax/handler.py index 481733d2c..e8d417e51 100644 --- a/Lib/xml/sax/handler.py +++ b/Lib/xml/sax/handler.py @@ -340,3 +340,48 @@ all_properties = [property_lexical_handler, property_xml_string, property_encoding, property_interning_dict] + + +class LexicalHandler: + """Optional SAX2 handler for lexical events. + + This handler is used to obtain lexical information about an XML + document, that is, information about how the document was encoded + (as opposed to what it contains, which is reported to the + ContentHandler), such as comments and CDATA marked section + boundaries. + + To set the LexicalHandler of an XMLReader, use the setProperty + method with the property identifier + 'http://xml.org/sax/properties/lexical-handler'.""" + + def comment(self, content): + """Reports a comment anywhere in the document (including the + DTD and outside the document element). + + content is a string that holds the contents of the comment.""" + + def startDTD(self, name, public_id, system_id): + """Report the start of the DTD declarations, if the document + has an associated DTD. + + A startEntity event will be reported before declaration events + from the external DTD subset are reported, and this can be + used to infer from which subset DTD declarations derive. + + name is the name of the document element type, public_id the + public identifier of the DTD (or None if none were supplied) + and system_id the system identfier of the external subset (or + None if none were supplied).""" + + def endDTD(self): + """Signals the end of DTD declarations.""" + + def startCDATA(self): + """Reports the beginning of a CDATA marked section. + + The contents of the CDATA marked section will be reported + through the characters event.""" + + def endCDATA(self): + """Reports the end of a CDATA marked section.""" diff --git a/Lib/xml/sax/saxutils.py b/Lib/xml/sax/saxutils.py index a69c7f762..c1612ea1c 100644 --- a/Lib/xml/sax/saxutils.py +++ b/Lib/xml/sax/saxutils.py @@ -56,8 +56,7 @@ def quoteattr(data, entities={}): the optional entities parameter. The keys and values must all be strings; each key will be replaced with its corresponding value. """ - entities = entities.copy() - entities.update({'\n': ' ', '\r': ' ', '\t':' '}) + entities = {**entities, '\n': ' ', '\r': ' ', '\t':' '} data = escape(data, entities) if '"' in data: if "'" in data: @@ -340,6 +339,8 @@ def prepare_input_source(source, base=""): """This function takes an InputSource and an optional base URL and returns a fully resolved InputSource object ready for reading.""" + if isinstance(source, os.PathLike): + source = os.fspath(source) if isinstance(source, str): source = xmlreader.InputSource(source) elif hasattr(source, "read"):