diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index f7b05705d..50ffc0afe 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -6,6 +6,7 @@ from tokenize import (tokenize, _tokenize, untokenize, NUMBER, NAME, OP, NEWLINE) from io import BytesIO, StringIO import unittest +from textwrap import dedent from unittest import TestCase, mock from test.test_grammar import (VALID_UNDERSCORE_LITERALS, INVALID_UNDERSCORE_LITERALS) @@ -45,7 +46,6 @@ class TokenizeTest(TestCase): # The ENDMARKER and final NEWLINE are omitted. f = BytesIO(s.encode('utf-8')) result = stringify_tokens_from_source(tokenize(f.readline), s) - self.assertEqual(result, [" ENCODING 'utf-8' (0, 0) (0, 0)"] + expected.rstrip().splitlines()) @@ -1504,6 +1504,16 @@ class TestTokenize(TestCase): # See http://bugs.python.org/issue16152 self.assertExactTypeEqual('@ ', token.AT) + def test_comment_at_the_end_of_the_source_without_newline(self): + # See http://bugs.python.org/issue44667 + source = 'b = 1\n\n#test' + expected_tokens = [token.NAME, token.EQUAL, token.NUMBER, token.NEWLINE, token.NL, token.COMMENT] + + tokens = list(tokenize(BytesIO(source.encode('utf-8')).readline)) + self.assertEqual(tok_name[tokens[0].exact_type], tok_name[ENCODING]) + for i in range(6): + self.assertEqual(tok_name[tokens[i + 1].exact_type], tok_name[expected_tokens[i]]) + self.assertEqual(tok_name[tokens[-1].exact_type], tok_name[token.ENDMARKER]) class UntokenizeTest(TestCase): diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 215d032f4..d72968e42 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -30,6 +30,7 @@ except ImportError: pass from codecs import lookup, BOM_UTF8 import collections +import functools from io import TextIOWrapper import itertools as _itertools import re @@ -98,6 +99,7 @@ def _all_string_prefixes(): result.add(''.join(u)) return result +@functools.lru_cache def _compile(expr): return re.compile(expr, re.UNICODE) @@ -605,7 +607,7 @@ def _tokenize(readline, encoding): pos += 1 # Add an implicit NEWLINE if the input doesn't end in one - if last_line and last_line[-1] not in '\r\n': + if last_line and last_line[-1] not in '\r\n' and not last_line.strip().startswith("#"): yield TokenInfo(NEWLINE, '', (lnum - 1, len(last_line)), (lnum - 1, len(last_line) + 1), '') for indent in indents[1:]: # pop remaining indent levels yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '')