From a12bd0fbedabfb23cf0bbe70d5aa08c7b9f5d49c Mon Sep 17 00:00:00 2001
From: Padraic Fanning <fanninpm@miamioh.edu>
Date: Thu, 22 Apr 2021 20:03:43 -0400
Subject: [PATCH] Add supporting files for test_tokenize

Supporting files not associated with another test:
 - tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txt
 - tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt
 - tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txt
---
 ..._tests-latin1-coding-cookie-and-utf8-bom-sig.txt | 13 +++++++++++++
 ...tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt | 13 +++++++++++++
 ...ze_tests-utf8-coding-cookie-and-utf8-bom-sig.txt | 12 ++++++++++++
 3 files changed, 38 insertions(+)
 create mode 100644 Lib/test/tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txt
 create mode 100644 Lib/test/tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt
 create mode 100644 Lib/test/tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txt

diff --git a/Lib/test/tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txt b/Lib/test/tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txt
new file mode 100644
index 000000000..1b5335b64
--- /dev/null
+++ b/Lib/test/tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txt
@@ -0,0 +1,13 @@
+# -*- coding: latin1 -*-
+# IMPORTANT: this file has the utf-8 BOM signature '\xef\xbb\xbf'
+# at the start of it.  Make sure this is preserved if any changes
+# are made!  Also note that the coding cookie above conflicts with
+# the presence of a utf-8 BOM signature -- this is intended.
+
+# Arbitrary encoded utf-8 text (stolen from test_doctest2.py).
+x = 'ЉЊЈЁЂ'
+def y():
+    """
+    And again in a comment.  ЉЊЈЁЂ
+    """
+    pass
diff --git a/Lib/test/tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt b/Lib/test/tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt
new file mode 100644
index 000000000..04561e484
--- /dev/null
+++ b/Lib/test/tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt
@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+# IMPORTANT: unlike the other test_tokenize-*.txt files, this file
+# does NOT have the utf-8 BOM signature '\xef\xbb\xbf' at the start
+# of it.  Make sure this is not added inadvertently by your editor
+# if any changes are made to this file!
+
+# Arbitrary encoded utf-8 text (stolen from test_doctest2.py).
+x = 'ЉЊЈЁЂ'
+def y():
+    """
+    And again in a comment.  ЉЊЈЁЂ
+    """
+    pass
diff --git a/Lib/test/tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txt b/Lib/test/tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txt
new file mode 100644
index 000000000..4b20ff6ad
--- /dev/null
+++ b/Lib/test/tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txt
@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+# IMPORTANT: this file has the utf-8 BOM signature '\xef\xbb\xbf'
+# at the start of it.  Make sure this is preserved if any changes
+# are made!
+
+# Arbitrary encoded utf-8 text (stolen from test_doctest2.py).
+x = 'ЉЊЈЁЂ'
+def y():
+    """
+    And again in a comment.  ЉЊЈЁЂ
+    """
+    pass