mirror of
https://github.com/numpy/numpy.git
synced 2026-10-29 08:55:25 +08:00
DWesl
22cd369bec
Makes sure that Python headers always come first (if implicitly) Also adds a script to check this and a CI run to ensure this. NOTE(seberg): If that script/CI run becomes problematic and not super easy to fix maybe we shouldn't worry too much about even deleting it again. It's wortwhile for sure, but I am not sure it is worthwhile to spend too many cycles making pretty.
255 lines
8.3 KiB
Python
Executable File
255 lines
8.3 KiB
Python
Executable File
#!/usr/bin/env python
|
|
"""Check that Python.h is included before any stdlib headers.
|
|
|
|
May be a bit overzealous, but it should get the job done.
|
|
"""
|
|
import argparse
|
|
import fnmatch
|
|
import os.path
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
|
|
from get_submodule_paths import get_submodule_paths
|
|
|
|
HEADER_PATTERN = re.compile(
|
|
r'^\s*#\s*include\s*[<"]((?:\w+/)*\w+(?:\.h[hp+]{0,2})?)[>"]\s*$'
|
|
)
|
|
|
|
PYTHON_INCLUDING_HEADERS = [
|
|
"Python.h",
|
|
# This isn't all of Python.h, but it is the visibility macros
|
|
"pyconfig.h",
|
|
"numpy/npy_common.h",
|
|
"numpy/npy_math.h",
|
|
"numpy/arrayobject.h",
|
|
"numpy/ndarrayobject.h",
|
|
"numpy/ndarraytypes.h",
|
|
"numpy/random/distributions.h",
|
|
"npy_sort.h",
|
|
"npy_config.h",
|
|
"common.h",
|
|
"npy_cpu_features.h",
|
|
# Boost::Python
|
|
"boost/python.hpp",
|
|
]
|
|
LEAF_HEADERS = [
|
|
"numpy/numpyconfig.h",
|
|
"numpy/npy_os.h",
|
|
"numpy/npy_cpu.h",
|
|
"numpy/utils.h",
|
|
]
|
|
|
|
C_CPP_EXTENSIONS = (".c", ".h", ".cpp", ".hpp", ".cc", ".hh", ".cxx", ".hxx")
|
|
# check against list in diff_files
|
|
|
|
PARSER = argparse.ArgumentParser(description=__doc__)
|
|
PARSER.add_argument(
|
|
"files",
|
|
nargs="*",
|
|
help="Lint these files or directories; use **/*.c to lint all files\n"
|
|
"Expects relative paths",
|
|
)
|
|
|
|
|
|
def check_python_h_included_first(name_to_check: str) -> int:
|
|
"""Check that the passed file includes Python.h first if it does at all.
|
|
|
|
Perhaps overzealous, but that should work around concerns with
|
|
recursion.
|
|
|
|
Parameters
|
|
----------
|
|
name_to_check : str
|
|
The name of the file to check.
|
|
|
|
Returns
|
|
-------
|
|
int
|
|
The number of headers before Python.h
|
|
"""
|
|
included_python = False
|
|
included_non_python_header = []
|
|
warned_python_construct = False
|
|
basename_to_check = os.path.basename(name_to_check)
|
|
in_comment = False
|
|
includes_headers = False
|
|
with open(name_to_check) as in_file:
|
|
for i, line in enumerate(in_file, 1):
|
|
# Very basic comment parsing
|
|
# Assumes /*...*/ comments are on their own lines
|
|
if "/*" in line:
|
|
if "*/" not in line:
|
|
in_comment = True
|
|
# else-branch could use regex to remove comment and continue
|
|
continue
|
|
if in_comment:
|
|
if "*/" in line:
|
|
in_comment = False
|
|
continue
|
|
line = line.split("//", 1)[0].strip()
|
|
match = HEADER_PATTERN.match(line)
|
|
if match:
|
|
includes_headers = True
|
|
this_header = match.group(1)
|
|
if this_header in PYTHON_INCLUDING_HEADERS:
|
|
if included_non_python_header and not included_python:
|
|
# Headers before python-including header
|
|
print(
|
|
f"Header before Python.h in file {name_to_check:s}\n"
|
|
f"Python.h on line {i:d}, other header(s) on line(s)"
|
|
f" {included_non_python_header}",
|
|
file=sys.stderr,
|
|
)
|
|
# else: # no headers before python-including header
|
|
included_python = True
|
|
PYTHON_INCLUDING_HEADERS.append(basename_to_check)
|
|
if os.path.dirname(name_to_check).endswith("include/numpy"):
|
|
PYTHON_INCLUDING_HEADERS.append(f"numpy/{basename_to_check:s}")
|
|
# We just found out where Python.h comes in this file
|
|
break
|
|
elif this_header in LEAF_HEADERS:
|
|
# This header is just defines, so it won't include
|
|
# the system headers that cause problems
|
|
continue
|
|
elif not included_python and (
|
|
"numpy/" in this_header
|
|
and this_header not in LEAF_HEADERS
|
|
or "python" in this_header.lower()
|
|
):
|
|
print(
|
|
f"Python.h not included before python-including header "
|
|
f"in file {name_to_check:s}\n"
|
|
f"{this_header:s} on line {i:d}",
|
|
file=sys.stderr,
|
|
)
|
|
included_python = True
|
|
PYTHON_INCLUDING_HEADERS.append(basename_to_check)
|
|
elif not included_python and this_header not in LEAF_HEADERS:
|
|
included_non_python_header.append(i)
|
|
elif (
|
|
not included_python
|
|
and not warned_python_construct
|
|
and ".h" not in basename_to_check
|
|
) and ("py::" in line or "PYBIND11_" in line):
|
|
print(
|
|
"Python-including header not used before python constructs "
|
|
f"in file {name_to_check:s}\nConstruct on line {i:d}",
|
|
file=sys.stderr,
|
|
)
|
|
warned_python_construct = True
|
|
if not includes_headers:
|
|
LEAF_HEADERS.append(basename_to_check)
|
|
return included_python and len(included_non_python_header)
|
|
|
|
|
|
def sort_order(path: str) -> tuple[int, str]:
|
|
if "include/numpy" in path:
|
|
# Want to process numpy/*.h first, to work out which of those
|
|
# include Python.h directly
|
|
priority = 0x00
|
|
elif "h" in os.path.splitext(path)[1].lower():
|
|
# Then other headers, which tend to include numpy/*.h
|
|
priority = 0x10
|
|
else:
|
|
# Source files after headers, to give the best chance of
|
|
# properly checking whether they include Python.h
|
|
priority = 0x20
|
|
if "common" in path:
|
|
priority -= 8
|
|
path_basename = os.path.basename(path)
|
|
if path_basename.startswith("npy_"):
|
|
priority -= 4
|
|
elif path_basename.startswith("npy"):
|
|
priority -= 3
|
|
elif path_basename.startswith("np"):
|
|
priority -= 2
|
|
if "config" in path_basename:
|
|
priority -= 1
|
|
return priority, path
|
|
|
|
|
|
def process_files(file_list: list[str]) -> int:
|
|
n_out_of_order = 0
|
|
submodule_paths = get_submodule_paths()
|
|
root_directory = os.path.dirname(os.path.dirname(__file__))
|
|
for name_to_check in sorted(file_list, key=sort_order):
|
|
name_to_check = os.path.join(root_directory, name_to_check)
|
|
if any(submodule_path in name_to_check for submodule_path in submodule_paths):
|
|
continue
|
|
if ".dispatch." in name_to_check:
|
|
continue
|
|
try:
|
|
n_out_of_order += check_python_h_included_first(name_to_check)
|
|
except UnicodeDecodeError:
|
|
print(f"File {name_to_check:s} not utf-8", sys.stdout)
|
|
return n_out_of_order
|
|
|
|
|
|
def find_c_cpp_files(root: str) -> list[str]:
|
|
|
|
result = []
|
|
|
|
for dirpath, dirnames, filenames in os.walk(root):
|
|
# I'm assuming other people have checked boost
|
|
for name in ("build", ".git", "boost"):
|
|
try:
|
|
dirnames.remove(name)
|
|
except ValueError:
|
|
pass
|
|
for name in fnmatch.filter(dirnames, "*.p"):
|
|
dirnames.remove(name)
|
|
result.extend(
|
|
[
|
|
os.path.join(dirpath, name)
|
|
for name in filenames
|
|
if os.path.splitext(name)[1].lower() in C_CPP_EXTENSIONS
|
|
]
|
|
)
|
|
# Check the headers before the source files
|
|
result.sort(key=lambda path: "h" in os.path.splitext(path)[1], reverse=True)
|
|
return result
|
|
|
|
|
|
def diff_files(sha: str) -> list[str]:
|
|
"""Find the diff since the given SHA.
|
|
|
|
Adapted from lint.py
|
|
"""
|
|
res = subprocess.run(
|
|
[
|
|
"git",
|
|
"diff",
|
|
"--name-only",
|
|
"--diff-filter=ACMR",
|
|
"-z",
|
|
sha,
|
|
"--",
|
|
# Check against C_CPP_EXTENSIONS
|
|
"*.[chCH]",
|
|
"*.[ch]pp",
|
|
"*.[ch]xx",
|
|
"*.cc",
|
|
"*.hh",
|
|
],
|
|
stdout=subprocess.PIPE,
|
|
encoding="utf-8",
|
|
)
|
|
res.check_returncode()
|
|
return [f for f in res.stdout.split("\0") if f]
|
|
|
|
|
|
if __name__ == "__main__":
|
|
args = PARSER.parse_args()
|
|
|
|
if len(args.files) == 0:
|
|
files = find_c_cpp_files("numpy")
|
|
else:
|
|
files = args.files
|
|
if len(files) == 1 and os.path.isdir(files[0]):
|
|
files = find_c_cpp_files(files[0])
|
|
|
|
# See which of the headers include Python.h and add them to the list
|
|
n_out_of_order = process_files(files)
|
|
sys.exit(n_out_of_order)
|