Migrate kudu iwyu code to arrow

This commit is contained in:
fjetter authored and Wes McKinney committed 2017-09-01 11:16:31 -04:00
1 parent c6295f3b74
commit 41edfc2866
13 files changed
+10424

No files matched your search

+2
View File
@@ -29,6 +29,8 @@ pushd $CPP_BUILD_DIR
$TRAVIS_MAKE lint
$TRAVIS_MAKE iwyu
# ARROW-209: checks depending on the LLVM toolchain are disabled temporarily
# until we are able to install the full LLVM toolchain in Travis CI again
+7
View File
@@ -580,6 +580,13 @@ if (${CLANG_TIDY_FOUND})
endif()
############################################################
# "make iwyu" target
############################################################
if (UNIX)
add_custom_target(iwyu ${BUILD_SUPPORT_DIR}/iwyu/iwyu.sh)
endif (UNIX)
############################################################
# Linker and Dependencies
############################################################
+20
View File
@@ -0,0 +1,20 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
---
BasedOnStyle: Google
DerivePointerAlignment: false
ColumnLimit: 90
+29
View File
@@ -0,0 +1,29 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Script which tries to determine the most recent git hash in the current
# branch which was checked in by gerrit. This commit hash is printed to
# stdout.
#
# It does so by looking for the 'Reviewed-on' tag added by gerrit. This is
# more foolproof than trying to guess at the "origin/" branch name, since the
# developer might be working on some local topic branch.
set -e
git log --grep='Reviewed-on: ' -n1 --pretty=format:%H
+91
View File
@@ -0,0 +1,91 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# This is an awk script to process output from the include-what-you-use (IWYU)
# tool. As of now, IWYU is of alpha quality and it gives many incorrect
# recommendations -- obviously invalid or leading to compilation breakage.
# Most of those can be silenced using appropriate IWYU pragmas, but it's not
# the case for the auto-generated files.
#
# Also, it's possible to address invalid recommendation using mappings:
# https://github.com/include-what-you-use/include-what-you-use/blob/master/docs/IWYUMappings.md
#
# Usage:
# 1. Run the CMake with -DCMAKE_CXX_INCLUDE_WHAT_YOU_USE=<iwyu_cmd_line>
#
# The path to the IWYU binary should be absolute. The path to the binary
# and the command-line options should be separated by semicolon
# (that's for feeding it into CMake list variables).
#
# E.g., from the build directory (line breaks are just for readability):
#
# CC=../../thirdparty/clang-toolchain/bin/clang
# CXX=../../thirdparty/clang-toolchain/bin/clang++
# IWYU="`pwd`../../thirdparty/clang-toolchain/bin/include-what-you-use;\
# -Xiwyu;--mapping_file=`pwd`../../build-support/iwyu/mappings/map.imp"
#
# ../../build-support/enable_devtoolset.sh \
# env CC=$CC CXX=$CXX \
# ../../thirdparty/installed/common/bin/cmake \
# -DCMAKE_CXX_INCLUDE_WHAT_YOU_USE=\"$IWYU\" \
# ../..
#
# NOTE:
# Since the arrow code has some 'ifdef NDEBUG' directives, it's possible
# that IWYU would produce different results if run against release, not
# debug build. However, we plan to use the tool only with debug builds.
#
# 2. Run make, separating the output from the IWYU tool into a separate file
# (it's possible to use piping the output from the tool to the script
# but having a file is good for future reference, if necessary):
#
# make -j$(nproc) 2>/tmp/iwyu.log
#
# 3. Process the output from the IWYU tool using the script:
#
# awk -f ../../build-support/iwyu/iwyu-filter.awk /tmp/iwyu.log
#
BEGIN {
# This is the list of the files for which the suggestions from IWYU are
# ignored. Eventually, this list should become empty as soon as all the valid
# suggestions are addressed and invalid ones are taken care either by proper
# IWYU pragmas or adding special mappings (e.g. like boost mappings).
# muted["relative/path/to/file"]
}
# mute all suggestions for the auto-generated files
/.*\.(pb|proxy|service)\.(cc|h) should (add|remove) these lines:/, /^$/ {
next
}
# mute suggestions for the explicitly specified files
/.* should (add|remove) these lines:/ {
do_print = 1
for (path in muted) {
if (index($0, path)) {
do_print = 0
break
}
}
}
/^$/ {
if (do_print) print
do_print = 0
}
{ if (do_print) print }
+62
View File
@@ -0,0 +1,62 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
ROOT=$(cd $(dirname $BASH_SOURCE)/../../..; pwd)
IWYU_LOG=$(mktemp -t arrow-cpp-iwyu.XXXXXX)
trap "rm -f $IWYU_LOG" EXIT
# Build the list of updated files which are of IWYU interest.
file_list_tmp=$(git diff --name-only \
$($ROOT/cpp/build-support/get-upstream-commit.sh) | grep -E '\.(c|cc|h)$')
if [ -z "$file_list_tmp" ]; then
echo "IWYU verification: no updates on related files, declaring success"
exit 0
fi
# Adjust the path for every element in the list. The iwyu_tool.py normalizes
# paths (via realpath) to match the records from the compilation database.
IWYU_FILE_LIST=
for p in $file_list_tmp; do
IWYU_FILE_LIST="$IWYU_FILE_LIST $ROOT/$p"
done
IWYU_MAPPINGS_PATH="$ROOT/cpp/build-support/iwyu/mappings"
IWYU_ARGS="\
--mapping_file=$IWYU_MAPPINGS_PATH/boost-all.imp \
--mapping_file=$IWYU_MAPPINGS_PATH/boost-all-private.imp \
--mapping_file=$IWYU_MAPPINGS_PATH/boost-extra.imp \
--mapping_file=$IWYU_MAPPINGS_PATH/gflags.imp \
--mapping_file=$IWYU_MAPPINGS_PATH/glog.imp \
--mapping_file=$IWYU_MAPPINGS_PATH/gtest.imp"
#PATH="$PATH:$PWD/../../thirdparty/clang-toolchain/bin"
python $ROOT/cpp/build-support/iwyu/iwyu_tool.py -p . $IWYU_FILE_LIST -- \
$IWYU_ARGS | awk -f $ROOT/cpp/build-support/iwyu/iwyu-filter.awk | \
tee $IWYU_LOG
if [ -s "$IWYU_LOG" ]; then
# The output is not empty: the changelist needs correction.
exit 1
fi
# The output is empty: the changelist looks good.
echo "IWYU verification: the changes look good"
exit 0
+272
View File
@@ -0,0 +1,272 @@
#!/usr/bin/env python
# This file has been imported into the apache source tree from
# the IWYU source tree as of version 0.8
# https://github.com/include-what-you-use/include-what-you-use/blob/master/iwyu_tool.py
# and corresponding license has been added:
# https://github.com/include-what-you-use/include-what-you-use/blob/master/LICENSE.TXT
#
# ==============================================================================
# LLVM Release License
# ==============================================================================
# University of Illinois/NCSA
# Open Source License
#
# Copyright (c) 2003-2010 University of Illinois at Urbana-Champaign.
# All rights reserved.
#
# Developed by:
#
# LLVM Team
#
# University of Illinois at Urbana-Champaign
#
# http://llvm.org
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal with
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# * Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimers.
#
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimers in the
# documentation and/or other materials provided with the distribution.
#
# * Neither the names of the LLVM Team, University of Illinois at
# Urbana-Champaign, nor the names of its contributors may be used to
# endorse or promote products derived from this Software without specific
# prior written permission.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
# SOFTWARE.
""" Driver to consume a Clang compilation database and invoke IWYU.
Example usage with CMake:
# Unix systems
$ mkdir build && cd build
$ CC="clang" CXX="clang++" cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ...
$ iwyu_tool.py -p .
# Windows systems
$ mkdir build && cd build
$ cmake -DCMAKE_CXX_COMPILER="%VCINSTALLDIR%/bin/cl.exe" \
-DCMAKE_C_COMPILER="%VCINSTALLDIR%/VC/bin/cl.exe" \
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
-G Ninja ...
$ python iwyu_tool.py -p .
See iwyu_tool.py -h for more details on command-line arguments.
"""
import os
import sys
import json
import argparse
import subprocess
import re
def iwyu_formatter(output):
""" Process iwyu's output, basically a no-op. """
print('\n'.join(output))
CORRECT_RE = re.compile(r'^\((.*?) has correct #includes/fwd-decls\)$')
SHOULD_ADD_RE = re.compile(r'^(.*?) should add these lines:$')
SHOULD_REMOVE_RE = re.compile(r'^(.*?) should remove these lines:$')
FULL_LIST_RE = re.compile(r'The full include-list for (.*?):$')
END_RE = re.compile(r'^---$')
LINES_RE = re.compile(r'^- (.*?) // lines ([0-9]+)-[0-9]+$')
GENERAL, ADD, REMOVE, LIST = range(4)
def clang_formatter(output):
""" Process iwyu's output into something clang-like. """
state = (GENERAL, None)
for line in output:
match = CORRECT_RE.match(line)
if match:
print('%s:1:1: note: #includes/fwd-decls are correct' % match.groups(1))
continue
match = SHOULD_ADD_RE.match(line)
if match:
state = (ADD, match.group(1))
continue
match = SHOULD_REMOVE_RE.match(line)
if match:
state = (REMOVE, match.group(1))
continue
match = FULL_LIST_RE.match(line)
if match:
state = (LIST, match.group(1))
elif END_RE.match(line):
state = (GENERAL, None)
elif not line.strip():
continue
elif state[0] == GENERAL:
print(line)
elif state[0] == ADD:
print('%s:1:1: error: add the following line' % state[1])
print(line)
elif state[0] == REMOVE:
match = LINES_RE.match(line)
line_no = match.group(2) if match else '1'
print('%s:%s:1: error: remove the following line' % (state[1], line_no))
print(match.group(1))
DEFAULT_FORMAT = 'iwyu'
FORMATTERS = {
'iwyu': iwyu_formatter,
'clang': clang_formatter
}
def get_output(cwd, command):
""" Run the given command and return its output as a string. """
process = subprocess.Popen(command,
cwd=cwd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
return process.communicate()[0].decode("utf-8").splitlines()
def run_iwyu(cwd, compile_command, iwyu_args, verbose, formatter):
""" Rewrite compile_command to an IWYU command, and run it. """
compiler, _, args = compile_command.partition(' ')
if compiler.endswith('cl.exe'):
# If the compiler name is cl.exe, let IWYU be cl-compatible
clang_args = ['--driver-mode=cl']
else:
clang_args = []
iwyu_args = ['-Xiwyu ' + a for a in iwyu_args]
command = ['include-what-you-use'] + clang_args + iwyu_args
command = '%s %s' % (' '.join(command), args.strip())
if verbose:
print('%s:' % command)
formatter(get_output(cwd, command))
def main(compilation_db_path, source_files, verbose, formatter, iwyu_args):
""" Entry point. """
# Canonicalize compilation database path
if os.path.isdir(compilation_db_path):
compilation_db_path = os.path.join(compilation_db_path,
'compile_commands.json')
compilation_db_path = os.path.realpath(compilation_db_path)
if not os.path.isfile(compilation_db_path):
print('ERROR: No such file or directory: \'%s\'' % compilation_db_path)
return 1
# Read compilation db from disk
with open(compilation_db_path, 'r') as fileobj:
compilation_db = json.load(fileobj)
# expand symlinks
for entry in compilation_db:
entry['file'] = os.path.realpath(entry['file'])
# Cross-reference source files with compilation database
source_files = [os.path.realpath(s) for s in source_files]
if not source_files:
# No source files specified, analyze entire compilation database
entries = compilation_db
else:
# Source files specified, analyze the ones appearing in compilation db,
# warn for the rest.
entries = []
for source in source_files:
matches = [e for e in compilation_db if e['file'] == source]
if matches:
entries.extend(matches)
else:
print('WARNING: \'%s\' not found in compilation database.' %
source)
# Run analysis
try:
for entry in entries:
cwd, compile_command = entry['directory'], entry['command']
run_iwyu(cwd, compile_command, iwyu_args, verbose, formatter)
except OSError as why:
print('ERROR: Failed to launch include-what-you-use: %s' % why)
return 1
return 0
def _bootstrap():
""" Parse arguments and dispatch to main(). """
# This hackery is necessary to add the forwarded IWYU args to the
# usage and help strings.
def customize_usage(parser):
""" Rewrite the parser's format_usage. """
original_format_usage = parser.format_usage
parser.format_usage = lambda: original_format_usage().rstrip() + \
' -- [<IWYU args>]' + os.linesep
def customize_help(parser):
""" Rewrite the parser's format_help. """
original_format_help = parser.format_help
def custom_help():
""" Customized help string, calls the adjusted format_usage. """
helpmsg = original_format_help()
helplines = helpmsg.splitlines()
helplines[0] = parser.format_usage().rstrip()
return os.linesep.join(helplines) + os.linesep
parser.format_help = custom_help
# Parse arguments
parser = argparse.ArgumentParser(
description='Include-what-you-use compilation database driver.',
epilog='Assumes include-what-you-use is available on the PATH.')
customize_usage(parser)
customize_help(parser)
parser.add_argument('-v', '--verbose', action='store_true',
help='Print IWYU commands')
parser.add_argument('-o', '--output-format', type=str,
choices=FORMATTERS.keys(), default=DEFAULT_FORMAT,
help='Output format (default: %s)' % DEFAULT_FORMAT)
parser.add_argument('-p', metavar='<build-path>', required=True,
help='Compilation database path', dest='dbpath')
parser.add_argument('source', nargs='*',
help='Zero or more source files to run IWYU on. '
'Defaults to all in compilation database.')
def partition_args(argv):
""" Split around '--' into driver args and IWYU args. """
try:
double_dash = argv.index('--')
return argv[:double_dash], argv[double_dash+1:]
except ValueError:
return argv, []
argv, iwyu_args = partition_args(sys.argv[1:])
args = parser.parse_args(argv)
sys.exit(main(args.dbpath, args.source, args.verbose,
FORMATTERS[args.output_format], iwyu_args))
if __name__ == '__main__':
_bootstrap()
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,23 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
[
{ include: ["<boost/core/explicit_operator_bool.hpp>", private, "<boost/optional/optional.hpp>", public ] },
{ include: ["<boost/cstdint.hpp>", private, "<cstdint>", public ] },
{ include: ["<boost/none.hpp>", private, "<boost/optional/optional.hpp>", public ] },
{ include: ["<boost/optional/detail/optional_relops.hpp>", private, "<boost/optional/optional.hpp>", public ] },
{ include: ["<boost/optional/detail/optional_reference_spec.hpp>", private, "<boost/optional/optional.hpp>", public ] }
]
@@ -0,0 +1,20 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
[
# <gflags/gflags_declare.h> confuses the IWYU tool because of the 'using '
{ symbol: [ "fLS::clstring", private, "<string>", public ] }
]
+27
View File
@@ -0,0 +1,27 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
[
{ symbol: [ "LOG", private, "<glog/logging.h>", public ] },
{ symbol: [ "VLOG", private, "<glog/logging.h>", public ] },
{ symbol: [ "CHECK_EQ", private, "<glog/logging.h>", public ] },
{ symbol: [ "CHECK_NE", private, "<glog/logging.h>", public ] },
{ symbol: [ "CHECK_LT", private, "<glog/logging.h>", public ] },
{ symbol: [ "CHECK_GE", private, "<glog/logging.h>", public ] },
{ symbol: [ "CHECK_GT", private, "<glog/logging.h>", public ] },
{ symbol: [ "ErrnoLogMessage", private, "<glog/logging.h>", public ] },
{ symbol: [ "COMPACT_GOOGLE_LOG_0", private, "<glog/logging.h>", public ] }
]
+26
View File
@@ -0,0 +1,26 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
[
{ include: [ "<gtest/internal/gtest-internal.h>", private, "<gtest/gtest.h>", public ] },
{ include: [ "<gtest/internal/gtest-string.h>", private, "<gtest/gtest.h>", public ] },
{ include: [ "<gtest/gtest-death-test.h>", private, "<gtest/gtest.h>", public ] },
{ include: [ "<gtest/gtest-message.h>", private, "<gtest/gtest.h>", public ] },
{ include: [ "<gtest/gtest-param-test.h>", private, "<gtest/gtest.h>", public ] },
{ include: [ "<gtest/gtest-printers.h>", private, "<gtest/gtest.h>", public ] },
{ include: [ "<gtest/gtest-test-part.h>", private, "<gtest/gtest.h>", public ] },
{ include: [ "<gtest/gtest-typed-test.h>", private, "<gtest/gtest.h>", public ] }
]