Clean up IWYU script more. Need to install Python toolchain in Dockerfile

Change-Id: I53282f813575fad213a54e2a2a775e4d604902ea

cpp/build-support/iwyu/iwyu.sh

@@ -27,21 +27,19 @@ trap "rm -f $IWYU_LOG" EXIT
 echo "Logging IWYU to $IWYU_LOG"
 
 IWYU_MAPPINGS_PATH="$ROOT/cpp/build-support/iwyu/mappings"
-IWYU_ARGS="\
-    --mapping_file=$IWYU_MAPPINGS_PATH/boost-all.imp \
+IWYU_ARGS="--mapping_file=$IWYU_MAPPINGS_PATH/boost-all.imp \
     --mapping_file=$IWYU_MAPPINGS_PATH/boost-all-private.imp \
     --mapping_file=$IWYU_MAPPINGS_PATH/boost-extra.imp \
     --mapping_file=$IWYU_MAPPINGS_PATH/gflags.imp \
     --mapping_file=$IWYU_MAPPINGS_PATH/glog.imp \
     --mapping_file=$IWYU_MAPPINGS_PATH/gtest.imp \
-    --mapping_file=$IWYU_MAPPINGS_PATH/arrow-misc.imp"
+     --mapping_file=$IWYU_MAPPINGS_PATH/arrow-misc.imp"
 
 set -e
 
 if [ "$1" == "all" ]; then
-  python $ROOT/cpp/build-support/iwyu/iwyu_tool.py -p . -- \
-       $IWYU_ARGS | awk -f $ROOT/cpp/build-support/iwyu/iwyu-filter.awk | \
-       tee $IWYU_LOG
+    python $ROOT/cpp/build-support/iwyu/iwyu_tool.py -p . \
+        -- $IWYU_ARGS | awk -f $ROOT/cpp/build-support/iwyu/iwyu-filter.awk
 else
   # Build the list of updated files which are of IWYU interest.
   file_list_tmp=$(git diff --name-only \

cpp/build-support/iwyu/mappings/arrow-misc.imp

@@ -15,19 +15,37 @@
 # specific language governing permissions and limitations
 # under the License.
 [
+  { include: ["<ext/alloc_traits.h>", private, "<memory>", public ] },
+  { include: ["<ext/alloc_traits.h>", private, "<condition_variable>", public ] },
+  { include: ["<ext/alloc_traits.h>", private, "<deque>", public ] },
+  { include: ["<ext/alloc_traits.h>", private, "<forward_list>", public ] },
+  { include: ["<ext/alloc_traits.h>", private, "<future>", public ] },
+  { include: ["<ext/alloc_traits.h>", private, "<map>", public ] },
+  { include: ["<ext/alloc_traits.h>", private, "<set>", public ] },
+  { include: ["<ext/alloc_traits.h>", private, "<string>", public ] },
+  { include: ["<ext/alloc_traits.h>", private, "<unordered_map>", public ] },
+  { include: ["<ext/alloc_traits.h>", private, "<unordered_set>", public ] },
+  { include: ["<ext/alloc_traits.h>", private, "<vector>", public ] },
+  { include: ["<bits/stdint-intn.h>", private, "<cstdint>", public ] },
+  { include: ["<bits/stdint-uintn.h>", private, "<cstdint>", public ] },
+  { include: ["<bits/shared_ptr.h>", private, "<memory>", public ] },
   { symbol: ["bool", private, "<cstdint>", public ] },
   { symbol: ["false", private, "<cstdint>", public ] },
   { symbol: ["true", private, "<cstdint>", public ] },
-  { symbol: ["int64_t", private, "<cstdint>", public ] },
+  { symbol: ["int8_t", private, "<cstdint>", public ] },
   { symbol: ["int16_t", private, "<cstdint>", public ] },
   { symbol: ["int32_t", private, "<cstdint>", public ] },
+  { symbol: ["int64_t", private, "<cstdint>", public ] },
   { symbol: ["uint8_t", private, "<cstdint>", public ] },
+  { symbol: ["uint16_t", private, "<cstdint>", public ] },
+  { symbol: ["uint32_t", private, "<cstdint>", public ] },
+  { symbol: ["uint64_t", private, "<cstdint>", public ] },
+  { symbol: ["make_shared", private, "<memory>", public ] },
+  { symbol: ["shared_ptr", private, "<memory>", public ] },
   { symbol: ["_Node_const_iterator", private, "<flatbuffers/flatbuffers.h>", public ] },
   { symbol: ["unordered_map<>::mapped_type", private, "<flatbuffers/flatbuffers.h>", public ] },
   { symbol: ["move", private, "<utility>", public ] },
   { symbol: ["pair", private, "<utility>", public ] },
   { symbol: ["errno", private, "<cerrno>", public ] },
-  { symbol: ["posix_memalign", private, "<cstdlib>", public ] },
-  { include: ["<ext/alloc_traits.h>", private, "<vector>", public ] },
-  { include: ["<string.h>", private, "<cstring>", public ] }
+  { symbol: ["posix_memalign", private, "<cstdlib>", public ] }
 ]

cpp/src/arrow/array.cc

@@ -21,7 +21,6 @@
 #include <cstdint>
 #include <cstring>
 #include <limits>
-#include <set>
 #include <sstream>
 #include <utility>
 
@@ -29,6 +28,7 @@
 #include "arrow/compare.h"
 #include "arrow/pretty_print.h"
 #include "arrow/status.h"
+#include "arrow/type_fwd.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/checked_cast.h"

cpp/src/arrow/array.h

@@ -18,22 +18,21 @@
 #ifndef ARROW_ARRAY_H
 #define ARROW_ARRAY_H
 
-#include <cmath>
 #include <cstdint>
+#include <iosfwd>
 #include <memory>
 #include <string>
+#include <type_traits>
 #include <utility>
 #include <vector>
 
 #include "arrow/buffer.h"
 #include "arrow/type.h"
-#include "arrow/type_fwd.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/checked_cast.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
-#include "arrow/visitor.h"
 
 namespace arrow {
 
@@ -45,12 +44,10 @@ using BufferVector = std::vector<std::shared_ptr<Buffer>>;
 // first time, the null count will be computed. See ARROW-33
 constexpr int64_t kUnknownNullCount = -1;
 
+class ArrayVisitor;
 class MemoryPool;
 class Status;
 
-template <typename T>
-struct Decimal;
-
 // ----------------------------------------------------------------------
 // Generic array data container
 

cpp/src/arrow/buffer.h

@@ -19,6 +19,7 @@
 #define ARROW_BUFFER_H
 
 #include <algorithm>
+#include <array>
 #include <cstdint>
 #include <cstring>
 #include <memory>

cpp/src/arrow/builder.cc

@@ -19,7 +19,6 @@
 #include <algorithm>
 #include <cstdint>
 #include <cstring>
-#include <limits>
 #include <numeric>
 #include <sstream>
 #include <utility>
@@ -27,7 +26,6 @@
 
 #include "arrow/array.h"
 #include "arrow/buffer.h"
-#include "arrow/compare.h"
 #include "arrow/status.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"

cpp/src/arrow/builder.h

@@ -18,13 +18,13 @@
 #ifndef ARROW_BUILDER_H
 #define ARROW_BUILDER_H
 
-#include <algorithm>
 #include <array>
 #include <cstdint>
-#include <functional>
+#include <iterator>
 #include <limits>
 #include <memory>
 #include <string>
+#include <type_traits>
 #include <vector>
 
 #include "arrow/buffer.h"
@@ -35,22 +35,18 @@
 #include "arrow/util/bit-util.h"
 #include "arrow/util/hash.h"
 #include "arrow/util/macros.h"
+#include "arrow/util/type_traits.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
 
 class Array;
+struct ArrayData;
 class Decimal128;
 
 constexpr int64_t kBinaryMemoryLimit = std::numeric_limits<int32_t>::max() - 1;
 constexpr int64_t kListMaximumElements = std::numeric_limits<int32_t>::max() - 1;
 
-namespace internal {
-
-struct ArrayData;
-
-}  // namespace internal
-
 constexpr int64_t kMinBuilderCapacity = 1 << 5;
 
 /// Base class for all data array builders.

cpp/src/arrow/compare.cc

@@ -19,15 +19,19 @@
 
 #include "arrow/compare.h"
 
+#include <cmath>
 #include <cstdint>
+#include <cstring>
+#include <limits>
 #include <memory>
+#include <string>
+#include <type_traits>
 #include <vector>
 
 #include "arrow/array.h"
 #include "arrow/status.h"
 #include "arrow/tensor.h"
 #include "arrow/type.h"
-#include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/checked_cast.h"
 #include "arrow/util/logging.h"

cpp/src/arrow/compare.h

@@ -28,7 +28,6 @@ namespace arrow {
 
 class Array;
 class DataType;
-class Status;
 class Tensor;
 
 /// Returns true if the arrays are exactly equal

cpp/src/arrow/io/file.cc

@@ -36,6 +36,7 @@
 #include <cerrno>
 #include <cstdint>
 #include <cstring>
+#include <memory>
 #include <mutex>
 #include <sstream>
 

cpp/src/arrow/memory_pool-test.cc

@@ -22,6 +22,7 @@
 #include "arrow/memory_pool-test.h"
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
+#include "arrow/test-util.h"
 
 namespace arrow {
 

cpp/src/arrow/pretty_print.cc

@@ -15,10 +15,12 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <cstdint>
 #include <memory>
-#include <ostream>
+#include <iostream>
 #include <sstream>
 #include <string>
+#include <type_traits>
 #include <vector>
 
 #include "arrow/array.h"

cpp/src/arrow/record_batch.h

@@ -23,13 +23,14 @@
 #include <string>
 #include <vector>
 
-#include "arrow/array.h"
 #include "arrow/type.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
 
+class Array;
+struct ArrayData;
 class KeyValueMetadata;
 class Status;
 

cpp/src/arrow/table.cc

@@ -33,6 +33,8 @@
 
 namespace arrow {
 
+class MemoryPool;
+
 // ----------------------------------------------------------------------
 // ChunkedArray and Column methods
 

cpp/src/arrow/table.h

@@ -32,6 +32,7 @@
 namespace arrow {
 
 class KeyValueMetadata;
+class MemoryPool;
 class Status;
 
 /// \class ChunkedArray

cpp/src/arrow/table_builder-test.cc

@@ -15,16 +15,16 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <cstdint>
 #include <memory>
 #include <string>
 #include <vector>
 
 #include "gtest/gtest.h"
 
-#include "arrow/array.h"
+#include "arrow/builder.h"
 #include "arrow/record_batch.h"
 #include "arrow/status.h"
-#include "arrow/table.h"
 #include "arrow/table_builder.h"
 #include "arrow/test-common.h"
 #include "arrow/test-util.h"

cpp/src/arrow/table_builder.cc

@@ -17,17 +17,13 @@
 
 #include "arrow/table_builder.h"
 
-#include <algorithm>
-#include <cstdlib>
 #include <memory>
-#include <sstream>
 #include <utility>
 
 #include "arrow/array.h"
 #include "arrow/builder.h"
 #include "arrow/record_batch.h"
 #include "arrow/status.h"
-#include "arrow/table.h"
 #include "arrow/type.h"
 #include "arrow/util/logging.h"
 

cpp/src/arrow/table_builder.h

@@ -20,9 +20,9 @@
 
 #include <cstdint>
 #include <memory>
-#include <string>
 #include <vector>
 
+#include "arrow/builder.h"
 #include "arrow/status.h"
 #include "arrow/type.h"
 #include "arrow/util/checked_cast.h"
@@ -30,15 +30,13 @@
 
 namespace arrow {
 
-class ArrayBuilder;
 class MemoryPool;
 class RecordBatch;
-class Schema;
 
 /// \class RecordBatchBuilder
 /// \brief Helper class for creating record batches iteratively given a known
 /// schema
-class RecordBatchBuilder {
+class ARROW_EXPORT RecordBatchBuilder {
  public:
   /// \brief Create an initialize a RecordBatchBuilder
   /// \param[in] schema The schema for the record batch

cpp/src/arrow/tensor.cc

@@ -25,7 +25,6 @@
 #include <vector>
 
 #include "arrow/compare.h"
-#include "arrow/status.h"
 #include "arrow/type.h"
 #include "arrow/util/checked_cast.h"
 #include "arrow/util/logging.h"

cpp/src/arrow/type.h

@@ -23,6 +23,7 @@
 #include <memory>
 #include <ostream>
 #include <string>
+#include <type_traits>
 #include <unordered_map>
 #include <vector>
 

cpp/src/arrow/util/compression_zlib.cc

@@ -28,6 +28,7 @@
 
 #include "arrow/status.h"
 #include "arrow/util/logging.h"
+#include "arrow/util/macros.h"
 
 namespace arrow {
 

dev/iwyu/Dockerfile

@@ -19,7 +19,15 @@ ADD . /apache-arrow
 WORKDIR /apache-arrow
 
 RUN apt-get update -qq &&\
-    apt-get install -qq apt-utils wget gnupg software-properties-common
+    apt-get install -qq \
+            apt-utils \
+            wget \
+            gnupg \
+            build-essential \
+            ninja-build \
+            libboost-system-dev \
+            libboost-filesystem-dev \
+            software-properties-common
 
 RUN wget -O llvm-apt.gpg.key http://llvm.org/apt/llvm-snapshot.gpg.key &&\
     apt-key add llvm-apt.gpg.key &&\

dev/iwyu/run_iwyu.sh

@@ -19,12 +19,20 @@
 # Set up environment and working directory
 CLANG_VERSION=6.0
 IWYU_BUILD_DIR=`pwd`/arrow/cpp/docker-iwyu
+IWYU_SH=`pwd`/arrow/cpp/build-support/iwyu/iwyu.sh
 IWYU_URL=https://github.com/include-what-you-use/include-what-you-use/archive/clang_$CLANG_VERSION.tar.gz
 
 rm -rf $IWYU_BUILD_DIR
 mkdir -p $IWYU_BUILD_DIR
 pushd $IWYU_BUILD_DIR
 
+function cleanup {
+    popd
+    rm -rf $IWYU_BUILD_DIR
+}
+
+trap cleanup EXIT
+
 # Build IWYU
 wget -O iwyu.tar.gz $IWYU_URL
 tar xzf iwyu.tar.gz
@@ -32,8 +40,8 @@ rm -f iwyu.tar.gz
 
 IWYU_SRC=`pwd`/include-what-you-use-clang_$CLANG_VERSION
 
-CC=clang-6.0
-CXX=clang++-6.0
+export CC=clang-$CLANG_VERSION
+export CXX=clang++-$CLANG_VERSION
 
 mkdir -p iwyu-build
 pushd iwyu-build
@@ -42,7 +50,11 @@ make -j4
 popd
 
 # Add iwyu and iwyu_tool.py to path
-export PATH=$IWYU_BUILD_DIR/iwyu-build:$IWYU_SRC:$PATH
+export PATH=$IWYU_BUILD_DIR/iwyu-build:$PATH
 
-cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ..
-iwyu_tool.py -p .
+cmake -GNinja -DARROW_PYTHON=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ..
+
+# Make so that vendored bits are built
+ninja
+
+$IWYU_SH all