Add tabnanny module and its test from Python v3.10.4

This commit is contained in:
Jeong Yunwon authored and Jeong YunWon committed 2022-08-06 05:21:55 +09:00
1 parent 5b1e92f793
commit 216fcd8ba5
2 files changed
+679

No files matched your search

Vendored Executable
+333
View File
@@ -0,0 +1,333 @@
#! /usr/bin/env python3
"""The Tab Nanny despises ambiguous indentation. She knows no mercy.
tabnanny -- Detection of ambiguous indentation
For the time being this module is intended to be called as a script.
However it is possible to import it into an IDE and use the function
check() described below.
Warning: The API provided by this module is likely to change in future
releases; such changes may not be backward compatible.
"""
# Released to the public domain, by Tim Peters, 15 April 1998.
# XXX Note: this is now a standard library module.
# XXX The API needs to undergo changes however; the current code is too
# XXX script-like. This will be addressed later.
__version__ = "6"
import os
import sys
import tokenize
if not hasattr(tokenize, 'NL'):
raise ValueError("tokenize.NL doesn't exist -- tokenize module too old")
__all__ = ["check", "NannyNag", "process_tokens"]
verbose = 0
filename_only = 0
def errprint(*args):
sep = ""
for arg in args:
sys.stderr.write(sep + str(arg))
sep = " "
sys.stderr.write("\n")
def main():
import getopt
global verbose, filename_only
try:
opts, args = getopt.getopt(sys.argv[1:], "qv")
except getopt.error as msg:
errprint(msg)
return
for o, a in opts:
if o == '-q':
filename_only = filename_only + 1
if o == '-v':
verbose = verbose + 1
if not args:
errprint("Usage:", sys.argv[0], "[-v] file_or_directory ...")
return
for arg in args:
check(arg)
class NannyNag(Exception):
"""
Raised by process_tokens() if detecting an ambiguous indent.
Captured and handled in check().
"""
def __init__(self, lineno, msg, line):
self.lineno, self.msg, self.line = lineno, msg, line
def get_lineno(self):
return self.lineno
def get_msg(self):
return self.msg
def get_line(self):
return self.line
def check(file):
"""check(file_or_dir)
If file_or_dir is a directory and not a symbolic link, then recursively
descend the directory tree named by file_or_dir, checking all .py files
along the way. If file_or_dir is an ordinary Python source file, it is
checked for whitespace related problems. The diagnostic messages are
written to standard output using the print statement.
"""
if os.path.isdir(file) and not os.path.islink(file):
if verbose:
print("%r: listing directory" % (file,))
names = os.listdir(file)
for name in names:
fullname = os.path.join(file, name)
if (os.path.isdir(fullname) and
not os.path.islink(fullname) or
os.path.normcase(name[-3:]) == ".py"):
check(fullname)
return
try:
f = tokenize.open(file)
except OSError as msg:
errprint("%r: I/O Error: %s" % (file, msg))
return
if verbose > 1:
print("checking %r ..." % file)
try:
process_tokens(tokenize.generate_tokens(f.readline))
except tokenize.TokenError as msg:
errprint("%r: Token Error: %s" % (file, msg))
return
except IndentationError as msg:
errprint("%r: Indentation Error: %s" % (file, msg))
return
except NannyNag as nag:
badline = nag.get_lineno()
line = nag.get_line()
if verbose:
print("%r: *** Line %d: trouble in tab city! ***" % (file, badline))
print("offending line: %r" % (line,))
print(nag.get_msg())
else:
if ' ' in file: file = '"' + file + '"'
if filename_only: print(file)
else: print(file, badline, repr(line))
return
finally:
f.close()
if verbose:
print("%r: Clean bill of health." % (file,))
class Whitespace:
# the characters used for space and tab
S, T = ' \t'
# members:
# raw
# the original string
# n
# the number of leading whitespace characters in raw
# nt
# the number of tabs in raw[:n]
# norm
# the normal form as a pair (count, trailing), where:
# count
# a tuple such that raw[:n] contains count[i]
# instances of S * i + T
# trailing
# the number of trailing spaces in raw[:n]
# It's A Theorem that m.indent_level(t) ==
# n.indent_level(t) for all t >= 1 iff m.norm == n.norm.
# is_simple
# true iff raw[:n] is of the form (T*)(S*)
def __init__(self, ws):
self.raw = ws
S, T = Whitespace.S, Whitespace.T
count = []
b = n = nt = 0
for ch in self.raw:
if ch == S:
n = n + 1
b = b + 1
elif ch == T:
n = n + 1
nt = nt + 1
if b >= len(count):
count = count + [0] * (b - len(count) + 1)
count[b] = count[b] + 1
b = 0
else:
break
self.n = n
self.nt = nt
self.norm = tuple(count), b
self.is_simple = len(count) <= 1
# return length of longest contiguous run of spaces (whether or not
# preceding a tab)
def longest_run_of_spaces(self):
count, trailing = self.norm
return max(len(count)-1, trailing)
def indent_level(self, tabsize):
# count, il = self.norm
# for i in range(len(count)):
# if count[i]:
# il = il + (i//tabsize + 1)*tabsize * count[i]
# return il
# quicker:
# il = trailing + sum (i//ts + 1)*ts*count[i] =
# trailing + ts * sum (i//ts + 1)*count[i] =
# trailing + ts * sum i//ts*count[i] + count[i] =
# trailing + ts * [(sum i//ts*count[i]) + (sum count[i])] =
# trailing + ts * [(sum i//ts*count[i]) + num_tabs]
# and note that i//ts*count[i] is 0 when i < ts
count, trailing = self.norm
il = 0
for i in range(tabsize, len(count)):
il = il + i//tabsize * count[i]
return trailing + tabsize * (il + self.nt)
# return true iff self.indent_level(t) == other.indent_level(t)
# for all t >= 1
def equal(self, other):
return self.norm == other.norm
# return a list of tuples (ts, i1, i2) such that
# i1 == self.indent_level(ts) != other.indent_level(ts) == i2.
# Intended to be used after not self.equal(other) is known, in which
# case it will return at least one witnessing tab size.
def not_equal_witness(self, other):
n = max(self.longest_run_of_spaces(),
other.longest_run_of_spaces()) + 1
a = []
for ts in range(1, n+1):
if self.indent_level(ts) != other.indent_level(ts):
a.append( (ts,
self.indent_level(ts),
other.indent_level(ts)) )
return a
# Return True iff self.indent_level(t) < other.indent_level(t)
# for all t >= 1.
# The algorithm is due to Vincent Broman.
# Easy to prove it's correct.
# XXXpost that.
# Trivial to prove n is sharp (consider T vs ST).
# Unknown whether there's a faster general way. I suspected so at
# first, but no longer.
# For the special (but common!) case where M and N are both of the
# form (T*)(S*), M.less(N) iff M.len() < N.len() and
# M.num_tabs() <= N.num_tabs(). Proof is easy but kinda long-winded.
# XXXwrite that up.
# Note that M is of the form (T*)(S*) iff len(M.norm[0]) <= 1.
def less(self, other):
if self.n >= other.n:
return False
if self.is_simple and other.is_simple:
return self.nt <= other.nt
n = max(self.longest_run_of_spaces(),
other.longest_run_of_spaces()) + 1
# the self.n >= other.n test already did it for ts=1
for ts in range(2, n+1):
if self.indent_level(ts) >= other.indent_level(ts):
return False
return True
# return a list of tuples (ts, i1, i2) such that
# i1 == self.indent_level(ts) >= other.indent_level(ts) == i2.
# Intended to be used after not self.less(other) is known, in which
# case it will return at least one witnessing tab size.
def not_less_witness(self, other):
n = max(self.longest_run_of_spaces(),
other.longest_run_of_spaces()) + 1
a = []
for ts in range(1, n+1):
if self.indent_level(ts) >= other.indent_level(ts):
a.append( (ts,
self.indent_level(ts),
other.indent_level(ts)) )
return a
def format_witnesses(w):
firsts = (str(tup[0]) for tup in w)
prefix = "at tab size"
if len(w) > 1:
prefix = prefix + "s"
return prefix + " " + ', '.join(firsts)
def process_tokens(tokens):
INDENT = tokenize.INDENT
DEDENT = tokenize.DEDENT
NEWLINE = tokenize.NEWLINE
JUNK = tokenize.COMMENT, tokenize.NL
indents = [Whitespace("")]
check_equal = 0
for (type, token, start, end, line) in tokens:
if type == NEWLINE:
# a program statement, or ENDMARKER, will eventually follow,
# after some (possibly empty) run of tokens of the form
# (NL | COMMENT)* (INDENT | DEDENT+)?
# If an INDENT appears, setting check_equal is wrong, and will
# be undone when we see the INDENT.
check_equal = 1
elif type == INDENT:
check_equal = 0
thisguy = Whitespace(token)
if not indents[-1].less(thisguy):
witness = indents[-1].not_less_witness(thisguy)
msg = "indent not greater e.g. " + format_witnesses(witness)
raise NannyNag(start[0], msg, line)
indents.append(thisguy)
elif type == DEDENT:
# there's nothing we need to check here! what's important is
# that when the run of DEDENTs ends, the indentation of the
# program statement (or ENDMARKER) that triggered the run is
# equal to what's left at the top of the indents stack
# Ouch! This assert triggers if the last line of the source
# is indented *and* lacks a newline -- then DEDENTs pop out
# of thin air.
# assert check_equal # else no earlier NEWLINE, or an earlier INDENT
check_equal = 1
del indents[-1]
elif check_equal and type not in JUNK:
# this is the first "real token" following a NEWLINE, so it
# must be the first token of the next program statement, or an
# ENDMARKER; the "line" argument exposes the leading whitespace
# for this statement; in the case of ENDMARKER, line is an empty
# string, so will properly match the empty string with which the
# "indents" stack was seeded
check_equal = 0
thisguy = Whitespace(line)
if not indents[-1].equal(thisguy):
witness = indents[-1].not_equal_witness(thisguy)
msg = "indent not equal e.g. " + format_witnesses(witness)
raise NannyNag(start[0], msg, line)
if __name__ == '__main__':
main()
+346
View File
@@ -0,0 +1,346 @@
"""Testing `tabnanny` module.
Glossary:
* errored : Whitespace related problems present in file.
"""
from unittest import TestCase, mock
import errno
import os
import tabnanny
import tokenize
import tempfile
import textwrap
from test.support import (captured_stderr, captured_stdout, script_helper,
findfile)
from test.support.os_helper import unlink
SOURCE_CODES = {
"incomplete_expression": (
'fruits = [\n'
' "Apple",\n'
' "Orange",\n'
' "Banana",\n'
'\n'
'print(fruits)\n'
),
"wrong_indented": (
'if True:\n'
' print("hello")\n'
' print("world")\n'
'else:\n'
' print("else called")\n'
),
"nannynag_errored": (
'if True:\n'
' \tprint("hello")\n'
'\tprint("world")\n'
'else:\n'
' print("else called")\n'
),
"error_free": (
'if True:\n'
' print("hello")\n'
' print("world")\n'
'else:\n'
' print("else called")\n'
),
"tab_space_errored_1": (
'def my_func():\n'
'\t print("hello world")\n'
'\t if True:\n'
'\t\tprint("If called")'
),
"tab_space_errored_2": (
'def my_func():\n'
'\t\tprint("Hello world")\n'
'\t\tif True:\n'
'\t print("If called")'
)
}
class TemporaryPyFile:
"""Create a temporary python source code file."""
def __init__(self, source_code='', directory=None):
self.source_code = source_code
self.dir = directory
def __enter__(self):
with tempfile.NamedTemporaryFile(
mode='w', dir=self.dir, suffix=".py", delete=False
) as f:
f.write(self.source_code)
self.file_path = f.name
return self.file_path
def __exit__(self, exc_type, exc_value, exc_traceback):
unlink(self.file_path)
class TestFormatWitnesses(TestCase):
"""Testing `tabnanny.format_witnesses()`."""
def test_format_witnesses(self):
"""Asserting formatter result by giving various input samples."""
tests = [
('Test', 'at tab sizes T, e, s, t'),
('', 'at tab size '),
('t', 'at tab size t'),
(' t ', 'at tab sizes , , t, , '),
]
for words, expected in tests:
with self.subTest(words=words, expected=expected):
self.assertEqual(tabnanny.format_witnesses(words), expected)
class TestErrPrint(TestCase):
"""Testing `tabnanny.errprint()`."""
def test_errprint(self):
"""Asserting result of `tabnanny.errprint()` by giving sample inputs."""
tests = [
(['first', 'second'], 'first second\n'),
(['first'], 'first\n'),
([1, 2, 3], '1 2 3\n'),
([], '\n')
]
for args, expected in tests:
with self.subTest(arguments=args, expected=expected):
with captured_stderr() as stderr:
tabnanny.errprint(*args)
self.assertEqual(stderr.getvalue() , expected)
class TestNannyNag(TestCase):
def test_all_methods(self):
"""Asserting behaviour of `tabnanny.NannyNag` exception."""
tests = [
(
tabnanny.NannyNag(0, "foo", "bar"),
{'lineno': 0, 'msg': 'foo', 'line': 'bar'}
),
(
tabnanny.NannyNag(5, "testmsg", "testline"),
{'lineno': 5, 'msg': 'testmsg', 'line': 'testline'}
)
]
for nanny, expected in tests:
line_number = nanny.get_lineno()
msg = nanny.get_msg()
line = nanny.get_line()
with self.subTest(
line_number=line_number, expected=expected['lineno']
):
self.assertEqual(expected['lineno'], line_number)
with self.subTest(msg=msg, expected=expected['msg']):
self.assertEqual(expected['msg'], msg)
with self.subTest(line=line, expected=expected['line']):
self.assertEqual(expected['line'], line)
class TestCheck(TestCase):
"""Testing tabnanny.check()."""
def setUp(self):
self.addCleanup(setattr, tabnanny, 'verbose', tabnanny.verbose)
tabnanny.verbose = 0 # Forcefully deactivating verbose mode.
def verify_tabnanny_check(self, dir_or_file, out="", err=""):
"""Common verification for tabnanny.check().
Use this method to assert expected values of `stdout` and `stderr` after
running tabnanny.check() on given `dir` or `file` path. Because
tabnanny.check() captures exceptions and writes to `stdout` and
`stderr`, asserting standard outputs is the only way.
"""
with captured_stdout() as stdout, captured_stderr() as stderr:
tabnanny.check(dir_or_file)
self.assertEqual(stdout.getvalue(), out)
self.assertEqual(stderr.getvalue(), err)
def test_correct_file(self):
"""A python source code file without any errors."""
with TemporaryPyFile(SOURCE_CODES["error_free"]) as file_path:
self.verify_tabnanny_check(file_path)
def test_correct_directory_verbose(self):
"""Directory containing few error free python source code files.
Because order of files returned by `os.lsdir()` is not fixed, verify the
existence of each output lines at `stdout` using `in` operator.
`verbose` mode of `tabnanny.verbose` asserts `stdout`.
"""
with tempfile.TemporaryDirectory() as tmp_dir:
lines = [f"{tmp_dir!r}: listing directory\n",]
file1 = TemporaryPyFile(SOURCE_CODES["error_free"], directory=tmp_dir)
file2 = TemporaryPyFile(SOURCE_CODES["error_free"], directory=tmp_dir)
with file1 as file1_path, file2 as file2_path:
for file_path in (file1_path, file2_path):
lines.append(f"{file_path!r}: Clean bill of health.\n")
tabnanny.verbose = 1
with captured_stdout() as stdout, captured_stderr() as stderr:
tabnanny.check(tmp_dir)
stdout = stdout.getvalue()
for line in lines:
with self.subTest(line=line):
self.assertIn(line, stdout)
self.assertEqual(stderr.getvalue(), "")
def test_correct_directory(self):
"""Directory which contains few error free python source code files."""
with tempfile.TemporaryDirectory() as tmp_dir:
with TemporaryPyFile(SOURCE_CODES["error_free"], directory=tmp_dir):
self.verify_tabnanny_check(tmp_dir)
def test_when_wrong_indented(self):
"""A python source code file eligible for raising `IndentationError`."""
with TemporaryPyFile(SOURCE_CODES["wrong_indented"]) as file_path:
err = ('unindent does not match any outer indentation level'
' (<tokenize>, line 3)\n')
err = f"{file_path!r}: Indentation Error: {err}"
self.verify_tabnanny_check(file_path, err=err)
def test_when_tokenize_tokenerror(self):
"""A python source code file eligible for raising 'tokenize.TokenError'."""
with TemporaryPyFile(SOURCE_CODES["incomplete_expression"]) as file_path:
err = "('EOF in multi-line statement', (7, 0))\n"
err = f"{file_path!r}: Token Error: {err}"
self.verify_tabnanny_check(file_path, err=err)
def test_when_nannynag_error_verbose(self):
"""A python source code file eligible for raising `tabnanny.NannyNag`.
Tests will assert `stdout` after activating `tabnanny.verbose` mode.
"""
with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as file_path:
out = f"{file_path!r}: *** Line 3: trouble in tab city! ***\n"
out += "offending line: '\\tprint(\"world\")\\n'\n"
out += "indent not equal e.g. at tab size 1\n"
tabnanny.verbose = 1
self.verify_tabnanny_check(file_path, out=out)
def test_when_nannynag_error(self):
"""A python source code file eligible for raising `tabnanny.NannyNag`."""
with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as file_path:
out = f"{file_path} 3 '\\tprint(\"world\")\\n'\n"
self.verify_tabnanny_check(file_path, out=out)
def test_when_no_file(self):
"""A python file which does not exist actually in system."""
path = 'no_file.py'
err = (f"{path!r}: I/O Error: [Errno {errno.ENOENT}] "
f"{os.strerror(errno.ENOENT)}: {path!r}\n")
self.verify_tabnanny_check(path, err=err)
def test_errored_directory(self):
"""Directory containing wrongly indented python source code files."""
with tempfile.TemporaryDirectory() as tmp_dir:
error_file = TemporaryPyFile(
SOURCE_CODES["wrong_indented"], directory=tmp_dir
)
code_file = TemporaryPyFile(
SOURCE_CODES["error_free"], directory=tmp_dir
)
with error_file as e_file, code_file as c_file:
err = ('unindent does not match any outer indentation level'
' (<tokenize>, line 3)\n')
err = f"{e_file!r}: Indentation Error: {err}"
self.verify_tabnanny_check(tmp_dir, err=err)
class TestProcessTokens(TestCase):
"""Testing `tabnanny.process_tokens()`."""
@mock.patch('tabnanny.NannyNag')
def test_with_correct_code(self, MockNannyNag):
"""A python source code without any whitespace related problems."""
with TemporaryPyFile(SOURCE_CODES["error_free"]) as file_path:
with open(file_path) as f:
tabnanny.process_tokens(tokenize.generate_tokens(f.readline))
self.assertFalse(MockNannyNag.called)
def test_with_errored_codes_samples(self):
"""A python source code with whitespace related sampled problems."""
# "tab_space_errored_1": executes block under type == tokenize.INDENT
# at `tabnanny.process_tokens()`.
# "tab space_errored_2": executes block under
# `check_equal and type not in JUNK` condition at
# `tabnanny.process_tokens()`.
for key in ["tab_space_errored_1", "tab_space_errored_2"]:
with self.subTest(key=key):
with TemporaryPyFile(SOURCE_CODES[key]) as file_path:
with open(file_path) as f:
tokens = tokenize.generate_tokens(f.readline)
with self.assertRaises(tabnanny.NannyNag):
tabnanny.process_tokens(tokens)
class TestCommandLine(TestCase):
"""Tests command line interface of `tabnanny`."""
def validate_cmd(self, *args, stdout="", stderr="", partial=False):
"""Common function to assert the behaviour of command line interface."""
_, out, err = script_helper.assert_python_ok('-m', 'tabnanny', *args)
# Note: The `splitlines()` will solve the problem of CRLF(\r) added
# by OS Windows.
out = os.fsdecode(out)
err = os.fsdecode(err)
if partial:
for std, output in ((stdout, out), (stderr, err)):
_output = output.splitlines()
for _std in std.splitlines():
with self.subTest(std=_std, output=_output):
self.assertIn(_std, _output)
else:
self.assertListEqual(out.splitlines(), stdout.splitlines())
self.assertListEqual(err.splitlines(), stderr.splitlines())
def test_with_errored_file(self):
"""Should displays error when errored python file is given."""
with TemporaryPyFile(SOURCE_CODES["wrong_indented"]) as file_path:
stderr = f"{file_path!r}: Indentation Error: "
stderr += ('unindent does not match any outer indentation level'
' (<tokenize>, line 3)')
self.validate_cmd(file_path, stderr=stderr)
def test_with_error_free_file(self):
"""Should not display anything if python file is correctly indented."""
with TemporaryPyFile(SOURCE_CODES["error_free"]) as file_path:
self.validate_cmd(file_path)
def test_command_usage(self):
"""Should display usage on no arguments."""
path = findfile('tabnanny.py')
stderr = f"Usage: {path} [-v] file_or_directory ..."
self.validate_cmd(stderr=stderr)
def test_quiet_flag(self):
"""Should display less when quite mode is on."""
with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as file_path:
stdout = f"{file_path}\n"
self.validate_cmd("-q", file_path, stdout=stdout)
def test_verbose_mode(self):
"""Should display more error information if verbose mode is on."""
with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as path:
stdout = textwrap.dedent(
"offending line: '\\tprint(\"world\")\\n'"
).strip()
self.validate_cmd("-v", path, stdout=stdout, partial=True)
def test_double_verbose_mode(self):
"""Should display detailed error information if double verbose is on."""
with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as path:
stdout = textwrap.dedent(
"offending line: '\\tprint(\"world\")\\n'"
).strip()
self.validate_cmd("-vv", path, stdout=stdout, partial=True)