diff --git a/Lib/tabnanny.py b/Lib/tabnanny.py new file mode 100755 index 000000000..7973f26f9 --- /dev/null +++ b/Lib/tabnanny.py @@ -0,0 +1,333 @@ +#! /usr/bin/env python3 + +"""The Tab Nanny despises ambiguous indentation. She knows no mercy. + +tabnanny -- Detection of ambiguous indentation + +For the time being this module is intended to be called as a script. +However it is possible to import it into an IDE and use the function +check() described below. + +Warning: The API provided by this module is likely to change in future +releases; such changes may not be backward compatible. +""" + +# Released to the public domain, by Tim Peters, 15 April 1998. + +# XXX Note: this is now a standard library module. +# XXX The API needs to undergo changes however; the current code is too +# XXX script-like. This will be addressed later. + +__version__ = "6" + +import os +import sys +import tokenize +if not hasattr(tokenize, 'NL'): + raise ValueError("tokenize.NL doesn't exist -- tokenize module too old") + +__all__ = ["check", "NannyNag", "process_tokens"] + +verbose = 0 +filename_only = 0 + +def errprint(*args): + sep = "" + for arg in args: + sys.stderr.write(sep + str(arg)) + sep = " " + sys.stderr.write("\n") + +def main(): + import getopt + + global verbose, filename_only + try: + opts, args = getopt.getopt(sys.argv[1:], "qv") + except getopt.error as msg: + errprint(msg) + return + for o, a in opts: + if o == '-q': + filename_only = filename_only + 1 + if o == '-v': + verbose = verbose + 1 + if not args: + errprint("Usage:", sys.argv[0], "[-v] file_or_directory ...") + return + for arg in args: + check(arg) + +class NannyNag(Exception): + """ + Raised by process_tokens() if detecting an ambiguous indent. + Captured and handled in check(). + """ + def __init__(self, lineno, msg, line): + self.lineno, self.msg, self.line = lineno, msg, line + def get_lineno(self): + return self.lineno + def get_msg(self): + return self.msg + def get_line(self): + return self.line + +def check(file): + """check(file_or_dir) + + If file_or_dir is a directory and not a symbolic link, then recursively + descend the directory tree named by file_or_dir, checking all .py files + along the way. If file_or_dir is an ordinary Python source file, it is + checked for whitespace related problems. The diagnostic messages are + written to standard output using the print statement. + """ + + if os.path.isdir(file) and not os.path.islink(file): + if verbose: + print("%r: listing directory" % (file,)) + names = os.listdir(file) + for name in names: + fullname = os.path.join(file, name) + if (os.path.isdir(fullname) and + not os.path.islink(fullname) or + os.path.normcase(name[-3:]) == ".py"): + check(fullname) + return + + try: + f = tokenize.open(file) + except OSError as msg: + errprint("%r: I/O Error: %s" % (file, msg)) + return + + if verbose > 1: + print("checking %r ..." % file) + + try: + process_tokens(tokenize.generate_tokens(f.readline)) + + except tokenize.TokenError as msg: + errprint("%r: Token Error: %s" % (file, msg)) + return + + except IndentationError as msg: + errprint("%r: Indentation Error: %s" % (file, msg)) + return + + except NannyNag as nag: + badline = nag.get_lineno() + line = nag.get_line() + if verbose: + print("%r: *** Line %d: trouble in tab city! ***" % (file, badline)) + print("offending line: %r" % (line,)) + print(nag.get_msg()) + else: + if ' ' in file: file = '"' + file + '"' + if filename_only: print(file) + else: print(file, badline, repr(line)) + return + + finally: + f.close() + + if verbose: + print("%r: Clean bill of health." % (file,)) + +class Whitespace: + # the characters used for space and tab + S, T = ' \t' + + # members: + # raw + # the original string + # n + # the number of leading whitespace characters in raw + # nt + # the number of tabs in raw[:n] + # norm + # the normal form as a pair (count, trailing), where: + # count + # a tuple such that raw[:n] contains count[i] + # instances of S * i + T + # trailing + # the number of trailing spaces in raw[:n] + # It's A Theorem that m.indent_level(t) == + # n.indent_level(t) for all t >= 1 iff m.norm == n.norm. + # is_simple + # true iff raw[:n] is of the form (T*)(S*) + + def __init__(self, ws): + self.raw = ws + S, T = Whitespace.S, Whitespace.T + count = [] + b = n = nt = 0 + for ch in self.raw: + if ch == S: + n = n + 1 + b = b + 1 + elif ch == T: + n = n + 1 + nt = nt + 1 + if b >= len(count): + count = count + [0] * (b - len(count) + 1) + count[b] = count[b] + 1 + b = 0 + else: + break + self.n = n + self.nt = nt + self.norm = tuple(count), b + self.is_simple = len(count) <= 1 + + # return length of longest contiguous run of spaces (whether or not + # preceding a tab) + def longest_run_of_spaces(self): + count, trailing = self.norm + return max(len(count)-1, trailing) + + def indent_level(self, tabsize): + # count, il = self.norm + # for i in range(len(count)): + # if count[i]: + # il = il + (i//tabsize + 1)*tabsize * count[i] + # return il + + # quicker: + # il = trailing + sum (i//ts + 1)*ts*count[i] = + # trailing + ts * sum (i//ts + 1)*count[i] = + # trailing + ts * sum i//ts*count[i] + count[i] = + # trailing + ts * [(sum i//ts*count[i]) + (sum count[i])] = + # trailing + ts * [(sum i//ts*count[i]) + num_tabs] + # and note that i//ts*count[i] is 0 when i < ts + + count, trailing = self.norm + il = 0 + for i in range(tabsize, len(count)): + il = il + i//tabsize * count[i] + return trailing + tabsize * (il + self.nt) + + # return true iff self.indent_level(t) == other.indent_level(t) + # for all t >= 1 + def equal(self, other): + return self.norm == other.norm + + # return a list of tuples (ts, i1, i2) such that + # i1 == self.indent_level(ts) != other.indent_level(ts) == i2. + # Intended to be used after not self.equal(other) is known, in which + # case it will return at least one witnessing tab size. + def not_equal_witness(self, other): + n = max(self.longest_run_of_spaces(), + other.longest_run_of_spaces()) + 1 + a = [] + for ts in range(1, n+1): + if self.indent_level(ts) != other.indent_level(ts): + a.append( (ts, + self.indent_level(ts), + other.indent_level(ts)) ) + return a + + # Return True iff self.indent_level(t) < other.indent_level(t) + # for all t >= 1. + # The algorithm is due to Vincent Broman. + # Easy to prove it's correct. + # XXXpost that. + # Trivial to prove n is sharp (consider T vs ST). + # Unknown whether there's a faster general way. I suspected so at + # first, but no longer. + # For the special (but common!) case where M and N are both of the + # form (T*)(S*), M.less(N) iff M.len() < N.len() and + # M.num_tabs() <= N.num_tabs(). Proof is easy but kinda long-winded. + # XXXwrite that up. + # Note that M is of the form (T*)(S*) iff len(M.norm[0]) <= 1. + def less(self, other): + if self.n >= other.n: + return False + if self.is_simple and other.is_simple: + return self.nt <= other.nt + n = max(self.longest_run_of_spaces(), + other.longest_run_of_spaces()) + 1 + # the self.n >= other.n test already did it for ts=1 + for ts in range(2, n+1): + if self.indent_level(ts) >= other.indent_level(ts): + return False + return True + + # return a list of tuples (ts, i1, i2) such that + # i1 == self.indent_level(ts) >= other.indent_level(ts) == i2. + # Intended to be used after not self.less(other) is known, in which + # case it will return at least one witnessing tab size. + def not_less_witness(self, other): + n = max(self.longest_run_of_spaces(), + other.longest_run_of_spaces()) + 1 + a = [] + for ts in range(1, n+1): + if self.indent_level(ts) >= other.indent_level(ts): + a.append( (ts, + self.indent_level(ts), + other.indent_level(ts)) ) + return a + +def format_witnesses(w): + firsts = (str(tup[0]) for tup in w) + prefix = "at tab size" + if len(w) > 1: + prefix = prefix + "s" + return prefix + " " + ', '.join(firsts) + +def process_tokens(tokens): + INDENT = tokenize.INDENT + DEDENT = tokenize.DEDENT + NEWLINE = tokenize.NEWLINE + JUNK = tokenize.COMMENT, tokenize.NL + indents = [Whitespace("")] + check_equal = 0 + + for (type, token, start, end, line) in tokens: + if type == NEWLINE: + # a program statement, or ENDMARKER, will eventually follow, + # after some (possibly empty) run of tokens of the form + # (NL | COMMENT)* (INDENT | DEDENT+)? + # If an INDENT appears, setting check_equal is wrong, and will + # be undone when we see the INDENT. + check_equal = 1 + + elif type == INDENT: + check_equal = 0 + thisguy = Whitespace(token) + if not indents[-1].less(thisguy): + witness = indents[-1].not_less_witness(thisguy) + msg = "indent not greater e.g. " + format_witnesses(witness) + raise NannyNag(start[0], msg, line) + indents.append(thisguy) + + elif type == DEDENT: + # there's nothing we need to check here! what's important is + # that when the run of DEDENTs ends, the indentation of the + # program statement (or ENDMARKER) that triggered the run is + # equal to what's left at the top of the indents stack + + # Ouch! This assert triggers if the last line of the source + # is indented *and* lacks a newline -- then DEDENTs pop out + # of thin air. + # assert check_equal # else no earlier NEWLINE, or an earlier INDENT + check_equal = 1 + + del indents[-1] + + elif check_equal and type not in JUNK: + # this is the first "real token" following a NEWLINE, so it + # must be the first token of the next program statement, or an + # ENDMARKER; the "line" argument exposes the leading whitespace + # for this statement; in the case of ENDMARKER, line is an empty + # string, so will properly match the empty string with which the + # "indents" stack was seeded + check_equal = 0 + thisguy = Whitespace(line) + if not indents[-1].equal(thisguy): + witness = indents[-1].not_equal_witness(thisguy) + msg = "indent not equal e.g. " + format_witnesses(witness) + raise NannyNag(start[0], msg, line) + + +if __name__ == '__main__': + main() diff --git a/Lib/test/test_tabnanny.py b/Lib/test/test_tabnanny.py new file mode 100644 index 000000000..e0a82e95c --- /dev/null +++ b/Lib/test/test_tabnanny.py @@ -0,0 +1,346 @@ +"""Testing `tabnanny` module. + +Glossary: + * errored : Whitespace related problems present in file. +""" +from unittest import TestCase, mock +import errno +import os +import tabnanny +import tokenize +import tempfile +import textwrap +from test.support import (captured_stderr, captured_stdout, script_helper, + findfile) +from test.support.os_helper import unlink + + +SOURCE_CODES = { + "incomplete_expression": ( + 'fruits = [\n' + ' "Apple",\n' + ' "Orange",\n' + ' "Banana",\n' + '\n' + 'print(fruits)\n' + ), + "wrong_indented": ( + 'if True:\n' + ' print("hello")\n' + ' print("world")\n' + 'else:\n' + ' print("else called")\n' + ), + "nannynag_errored": ( + 'if True:\n' + ' \tprint("hello")\n' + '\tprint("world")\n' + 'else:\n' + ' print("else called")\n' + ), + "error_free": ( + 'if True:\n' + ' print("hello")\n' + ' print("world")\n' + 'else:\n' + ' print("else called")\n' + ), + "tab_space_errored_1": ( + 'def my_func():\n' + '\t print("hello world")\n' + '\t if True:\n' + '\t\tprint("If called")' + ), + "tab_space_errored_2": ( + 'def my_func():\n' + '\t\tprint("Hello world")\n' + '\t\tif True:\n' + '\t print("If called")' + ) +} + + +class TemporaryPyFile: + """Create a temporary python source code file.""" + + def __init__(self, source_code='', directory=None): + self.source_code = source_code + self.dir = directory + + def __enter__(self): + with tempfile.NamedTemporaryFile( + mode='w', dir=self.dir, suffix=".py", delete=False + ) as f: + f.write(self.source_code) + self.file_path = f.name + return self.file_path + + def __exit__(self, exc_type, exc_value, exc_traceback): + unlink(self.file_path) + + +class TestFormatWitnesses(TestCase): + """Testing `tabnanny.format_witnesses()`.""" + + def test_format_witnesses(self): + """Asserting formatter result by giving various input samples.""" + tests = [ + ('Test', 'at tab sizes T, e, s, t'), + ('', 'at tab size '), + ('t', 'at tab size t'), + (' t ', 'at tab sizes , , t, , '), + ] + + for words, expected in tests: + with self.subTest(words=words, expected=expected): + self.assertEqual(tabnanny.format_witnesses(words), expected) + + +class TestErrPrint(TestCase): + """Testing `tabnanny.errprint()`.""" + + def test_errprint(self): + """Asserting result of `tabnanny.errprint()` by giving sample inputs.""" + tests = [ + (['first', 'second'], 'first second\n'), + (['first'], 'first\n'), + ([1, 2, 3], '1 2 3\n'), + ([], '\n') + ] + + for args, expected in tests: + with self.subTest(arguments=args, expected=expected): + with captured_stderr() as stderr: + tabnanny.errprint(*args) + self.assertEqual(stderr.getvalue() , expected) + + +class TestNannyNag(TestCase): + def test_all_methods(self): + """Asserting behaviour of `tabnanny.NannyNag` exception.""" + tests = [ + ( + tabnanny.NannyNag(0, "foo", "bar"), + {'lineno': 0, 'msg': 'foo', 'line': 'bar'} + ), + ( + tabnanny.NannyNag(5, "testmsg", "testline"), + {'lineno': 5, 'msg': 'testmsg', 'line': 'testline'} + ) + ] + for nanny, expected in tests: + line_number = nanny.get_lineno() + msg = nanny.get_msg() + line = nanny.get_line() + with self.subTest( + line_number=line_number, expected=expected['lineno'] + ): + self.assertEqual(expected['lineno'], line_number) + with self.subTest(msg=msg, expected=expected['msg']): + self.assertEqual(expected['msg'], msg) + with self.subTest(line=line, expected=expected['line']): + self.assertEqual(expected['line'], line) + + +class TestCheck(TestCase): + """Testing tabnanny.check().""" + + def setUp(self): + self.addCleanup(setattr, tabnanny, 'verbose', tabnanny.verbose) + tabnanny.verbose = 0 # Forcefully deactivating verbose mode. + + def verify_tabnanny_check(self, dir_or_file, out="", err=""): + """Common verification for tabnanny.check(). + + Use this method to assert expected values of `stdout` and `stderr` after + running tabnanny.check() on given `dir` or `file` path. Because + tabnanny.check() captures exceptions and writes to `stdout` and + `stderr`, asserting standard outputs is the only way. + """ + with captured_stdout() as stdout, captured_stderr() as stderr: + tabnanny.check(dir_or_file) + self.assertEqual(stdout.getvalue(), out) + self.assertEqual(stderr.getvalue(), err) + + def test_correct_file(self): + """A python source code file without any errors.""" + with TemporaryPyFile(SOURCE_CODES["error_free"]) as file_path: + self.verify_tabnanny_check(file_path) + + def test_correct_directory_verbose(self): + """Directory containing few error free python source code files. + + Because order of files returned by `os.lsdir()` is not fixed, verify the + existence of each output lines at `stdout` using `in` operator. + `verbose` mode of `tabnanny.verbose` asserts `stdout`. + """ + with tempfile.TemporaryDirectory() as tmp_dir: + lines = [f"{tmp_dir!r}: listing directory\n",] + file1 = TemporaryPyFile(SOURCE_CODES["error_free"], directory=tmp_dir) + file2 = TemporaryPyFile(SOURCE_CODES["error_free"], directory=tmp_dir) + with file1 as file1_path, file2 as file2_path: + for file_path in (file1_path, file2_path): + lines.append(f"{file_path!r}: Clean bill of health.\n") + + tabnanny.verbose = 1 + with captured_stdout() as stdout, captured_stderr() as stderr: + tabnanny.check(tmp_dir) + stdout = stdout.getvalue() + for line in lines: + with self.subTest(line=line): + self.assertIn(line, stdout) + self.assertEqual(stderr.getvalue(), "") + + def test_correct_directory(self): + """Directory which contains few error free python source code files.""" + with tempfile.TemporaryDirectory() as tmp_dir: + with TemporaryPyFile(SOURCE_CODES["error_free"], directory=tmp_dir): + self.verify_tabnanny_check(tmp_dir) + + def test_when_wrong_indented(self): + """A python source code file eligible for raising `IndentationError`.""" + with TemporaryPyFile(SOURCE_CODES["wrong_indented"]) as file_path: + err = ('unindent does not match any outer indentation level' + ' (, line 3)\n') + err = f"{file_path!r}: Indentation Error: {err}" + self.verify_tabnanny_check(file_path, err=err) + + def test_when_tokenize_tokenerror(self): + """A python source code file eligible for raising 'tokenize.TokenError'.""" + with TemporaryPyFile(SOURCE_CODES["incomplete_expression"]) as file_path: + err = "('EOF in multi-line statement', (7, 0))\n" + err = f"{file_path!r}: Token Error: {err}" + self.verify_tabnanny_check(file_path, err=err) + + def test_when_nannynag_error_verbose(self): + """A python source code file eligible for raising `tabnanny.NannyNag`. + + Tests will assert `stdout` after activating `tabnanny.verbose` mode. + """ + with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as file_path: + out = f"{file_path!r}: *** Line 3: trouble in tab city! ***\n" + out += "offending line: '\\tprint(\"world\")\\n'\n" + out += "indent not equal e.g. at tab size 1\n" + + tabnanny.verbose = 1 + self.verify_tabnanny_check(file_path, out=out) + + def test_when_nannynag_error(self): + """A python source code file eligible for raising `tabnanny.NannyNag`.""" + with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as file_path: + out = f"{file_path} 3 '\\tprint(\"world\")\\n'\n" + self.verify_tabnanny_check(file_path, out=out) + + def test_when_no_file(self): + """A python file which does not exist actually in system.""" + path = 'no_file.py' + err = (f"{path!r}: I/O Error: [Errno {errno.ENOENT}] " + f"{os.strerror(errno.ENOENT)}: {path!r}\n") + self.verify_tabnanny_check(path, err=err) + + def test_errored_directory(self): + """Directory containing wrongly indented python source code files.""" + with tempfile.TemporaryDirectory() as tmp_dir: + error_file = TemporaryPyFile( + SOURCE_CODES["wrong_indented"], directory=tmp_dir + ) + code_file = TemporaryPyFile( + SOURCE_CODES["error_free"], directory=tmp_dir + ) + with error_file as e_file, code_file as c_file: + err = ('unindent does not match any outer indentation level' + ' (, line 3)\n') + err = f"{e_file!r}: Indentation Error: {err}" + self.verify_tabnanny_check(tmp_dir, err=err) + + +class TestProcessTokens(TestCase): + """Testing `tabnanny.process_tokens()`.""" + + @mock.patch('tabnanny.NannyNag') + def test_with_correct_code(self, MockNannyNag): + """A python source code without any whitespace related problems.""" + + with TemporaryPyFile(SOURCE_CODES["error_free"]) as file_path: + with open(file_path) as f: + tabnanny.process_tokens(tokenize.generate_tokens(f.readline)) + self.assertFalse(MockNannyNag.called) + + def test_with_errored_codes_samples(self): + """A python source code with whitespace related sampled problems.""" + + # "tab_space_errored_1": executes block under type == tokenize.INDENT + # at `tabnanny.process_tokens()`. + # "tab space_errored_2": executes block under + # `check_equal and type not in JUNK` condition at + # `tabnanny.process_tokens()`. + + for key in ["tab_space_errored_1", "tab_space_errored_2"]: + with self.subTest(key=key): + with TemporaryPyFile(SOURCE_CODES[key]) as file_path: + with open(file_path) as f: + tokens = tokenize.generate_tokens(f.readline) + with self.assertRaises(tabnanny.NannyNag): + tabnanny.process_tokens(tokens) + + +class TestCommandLine(TestCase): + """Tests command line interface of `tabnanny`.""" + + def validate_cmd(self, *args, stdout="", stderr="", partial=False): + """Common function to assert the behaviour of command line interface.""" + _, out, err = script_helper.assert_python_ok('-m', 'tabnanny', *args) + # Note: The `splitlines()` will solve the problem of CRLF(\r) added + # by OS Windows. + out = os.fsdecode(out) + err = os.fsdecode(err) + if partial: + for std, output in ((stdout, out), (stderr, err)): + _output = output.splitlines() + for _std in std.splitlines(): + with self.subTest(std=_std, output=_output): + self.assertIn(_std, _output) + else: + self.assertListEqual(out.splitlines(), stdout.splitlines()) + self.assertListEqual(err.splitlines(), stderr.splitlines()) + + def test_with_errored_file(self): + """Should displays error when errored python file is given.""" + with TemporaryPyFile(SOURCE_CODES["wrong_indented"]) as file_path: + stderr = f"{file_path!r}: Indentation Error: " + stderr += ('unindent does not match any outer indentation level' + ' (, line 3)') + self.validate_cmd(file_path, stderr=stderr) + + def test_with_error_free_file(self): + """Should not display anything if python file is correctly indented.""" + with TemporaryPyFile(SOURCE_CODES["error_free"]) as file_path: + self.validate_cmd(file_path) + + def test_command_usage(self): + """Should display usage on no arguments.""" + path = findfile('tabnanny.py') + stderr = f"Usage: {path} [-v] file_or_directory ..." + self.validate_cmd(stderr=stderr) + + def test_quiet_flag(self): + """Should display less when quite mode is on.""" + with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as file_path: + stdout = f"{file_path}\n" + self.validate_cmd("-q", file_path, stdout=stdout) + + def test_verbose_mode(self): + """Should display more error information if verbose mode is on.""" + with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as path: + stdout = textwrap.dedent( + "offending line: '\\tprint(\"world\")\\n'" + ).strip() + self.validate_cmd("-v", path, stdout=stdout, partial=True) + + def test_double_verbose_mode(self): + """Should display detailed error information if double verbose is on.""" + with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as path: + stdout = textwrap.dedent( + "offending line: '\\tprint(\"world\")\\n'" + ).strip() + self.validate_cmd("-vv", path, stdout=stdout, partial=True)