Enable modular builds and install with ARROW_OPTIONAL_INSTALL option. Remove

ARROW_GANDIVA_BUILD_TESTS. Add helper function for installing header files. Build fewer targets using these options in some Travis CI jobs Change-Id: I3a6c70dec179fb579557f9a4ebe22ef7cc02a9bc
2026-10-28 16:47:19 +08:00 · 2018-12-13 21:47:45 -06:00
1 parent b3bc3384f3
commit 2eca8a7406
@@ -109,6 +109,8 @@ matrix:
    jdk: openjdk8
    env:
    - ARROW_TRAVIS_GANDIVA=1
+    - ARROW_TRAVIS_OPTIONAL_INSTALL=1
+    - ARROW_CPP_BUILD_TARGETS="gandiva"
    - ARROW_TRAVIS_USE_TOOLCHAIN=1
    # ARROW-3979 temporarily disabled.
    - ARROW_TRAVIS_VALGRIND=0
@@ -155,6 +157,8 @@ matrix:
    addons:
    env:
    - ARROW_TRAVIS_GANDIVA=1
+    - ARROW_TRAVIS_OPTIONAL_INSTALL=1
+    - ARROW_CPP_BUILD_TARGETS="gandiva"
    - ARROW_TRAVIS_USE_TOOLCHAIN=1
    - ARROW_BUILD_WARNING_LEVEL=CHECKIN
    before_script:
@@ -119,6 +119,10 @@ if [ $ARROW_TRAVIS_USE_VENDORED_BOOST == "1" ]; then
  CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_BOOST_VENDORED=ON"
 fi

+if [ $ARROW_TRAVIS_OPTIONAL_INSTALL == "1" ]; then
+  CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_OPTIONAL_INSTALL=ON"
+fi
+
 if [ $TRAVIS_OS_NAME == "linux" ]; then
    cmake $CMAKE_COMMON_FLAGS \
          $CMAKE_LINUX_FLAGS \
@@ -139,8 +143,9 @@ else
          $ARROW_CPP_DIR
 fi

-# Build and install libraries
-$TRAVIS_MAKE -j4
+# Build and install libraries. Configure ARROW_CPP_BUILD_TARGETS environment
+# variable to only build certain targets
+$TRAVIS_MAKE -j4 $ARROW_CPP_BUILD_TARGETS
 $TRAVIS_MAKE install

 popd
@@ -86,12 +86,15 @@ rm -rf *
 # XXX Can we simply reuse CMAKE_COMMON_FLAGS from travis_before_script_cpp.sh?
 CMAKE_COMMON_FLAGS="-DARROW_EXTRA_ERROR_CONTEXT=ON"

+PYTHON_CPP_BUILD_TARGETS="arrow_python parquet plasma"
+
 if [ $ARROW_TRAVIS_COVERAGE == "1" ]; then
  CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_GENERATE_COVERAGE=ON"
 fi

 if [ $ARROW_TRAVIS_PYTHON_GANDIVA == "1" ]; then
  CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_GANDIVA=ON -DARROW_GANDIVA_BUILD_TESTS=OFF"
+  PYTHON_CPP_BUILD_TARGETS="$PYTHON_CPP_BUILD_TARGETS gandiva"
 fi

 cmake -GNinja \
@@ -99,6 +102,7 @@ cmake -GNinja \
      -DARROW_BUILD_TESTS=on \
      -DARROW_TEST_INCLUDE_LABELS=python \
      -DARROW_BUILD_UTILITIES=off \
+      -DARROW_OPTIONAL_INSTALL=ON \
      -DARROW_PLASMA=on \
      -DARROW_TENSORFLOW=on \
      -DARROW_PYTHON=on \
@@ -107,7 +111,7 @@ cmake -GNinja \
      -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
      $ARROW_CPP_DIR

-ninja
+ninja $PYTHON_CPP_BUILD_TARGETS
 ninja install

 popd
@@ -98,6 +98,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
  set(ARROW_CXXFLAGS "" CACHE STRING
    "Compiler flags to append when compiling Arrow")

+  option(ARROW_OPTIONAL_INSTALL
+    "If enabled install targets that have already been built"
+    OFF)
+
  option(ARROW_ONLY_LINT
    "Only define the lint and check-format targets"
    OFF)
@@ -323,13 +327,14 @@ Always OFF if building binaries"
  option(ARROW_GANDIVA_JAVA
    "Build the Gandiva JNI wrappers"
    ON)
-
-  option(ARROW_GANDIVA_BUILD_TESTS
-    "Build the Gandiva googletest unit tests"
-    OFF)
-
 endif()

+if (ARROW_OPTIONAL_INSTALL)
+  # Don't make the "install" target depend on the "all" target
+  set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY true)
+
+  set(INSTALL_IS_OPTIONAL OPTIONAL)
+endif()

 ############################################################
 # "make lint" target
@@ -101,6 +101,19 @@ unit tests, and benchmarks (if enabled):
 * `make gandiva` for Gandiva (LLVM expression compiler) libraries
 * `make plasma` for Plasma libraries, server

+If you wish to only build and install one or more project subcomponents, we
+have provided the CMake option `ARROW_OPTIONAL_INSTALL` to only install targets
+that have been built. For example, if you only wish to build the Parquet
+libraries, its tests, and its dependencies, you can run:
+
+```
+cmake .. -DARROW_PARQUET=ON -DARROW_OPTIONAL_INSTALL=ON -DARROW_BUILD_TESTS=ON
+make parquet
+make install
+```
+
+If you omit an explicit target when invoking `make`, all targets will be built.
+
 ## Parquet Development Notes

 To build the C++ libraries for Apache Parquet, add the flag
@@ -269,7 +282,7 @@ The optional `gandiva` libraries and tests can be built by passing
 `-DARROW_GANDIVA=on`.

 ```shell
-cmake .. -DARROW_GANDIVA=ON -DARROW_GANDIVA_BUILD_TESTS=ON
+cmake .. -DARROW_GANDIVA=ON -DARROW_BUILD_TESTS=ON
 make
 ctest -L gandiva
 ```
@@ -226,6 +226,7 @@ function(ADD_ARROW_LIB LIB_NAME)
    endif()

    install(TARGETS ${LIB_NAME}_shared
+      ${INSTALL_IS_OPTIONAL}
      EXPORT ${PROJECT_NAME}-targets
      RUNTIME DESTINATION ${RUNTIME_INSTALL_DIR}
      LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
@@ -272,6 +273,7 @@ function(ADD_ARROW_LIB LIB_NAME)
      LINK_PUBLIC ${ARG_STATIC_LINK_LIBS})

    install(TARGETS ${LIB_NAME}_static
+      ${INSTALL_IS_OPTIONAL}
      EXPORT ${PROJECT_NAME}-targets
      RUNTIME DESTINATION ${RUNTIME_INSTALL_DIR}
      LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
@@ -569,3 +571,17 @@ function(ADD_ARROW_FUZZING REL_FUZZING_NAME)
      PROPERTIES
      LINK_FLAGS "-fsanitize=fuzzer")
 endfunction()
+
+###################################################
+
+function(ARROW_INSTALL_ALL_HEADERS PATH)
+  set(multi_value_args PATTERN)
+  cmake_parse_arguments(ARG "" "" "${multi_value_args}" ${ARGN})
+  if (NOT ${ARG_PATTERN})
+    set(ARG_PATTERN "*.h")
+  endif()
+  file(GLOB_RECURSE CURRENT_DIRECTORY_HEADERS ${ARG_PATTERN})
+  install(FILES
+    ${CURRENT_DIRECTORY_HEADERS}
+    DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/${PATH}")
+endfunction()
@@ -213,8 +213,7 @@ if (ARROW_BUILD_STATIC AND WIN32)
  target_compile_definitions(arrow_static PUBLIC ARROW_STATIC)
 endif()

-if (ARROW_BUILD_TESTS OR ARROW_GANDIVA_BUILD_TESTS
-    OR ARROW_BUILD_BENCHMARKS)
+if (ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS)
  # that depend on gtest
  ADD_ARROW_LIB(arrow_testing
    SOURCES test-util.cc
@@ -244,28 +243,7 @@ foreach(LIB_TARGET ${ARROW_LIBRARIES})
 endforeach()

 # Headers: top level
-install(FILES
-  allocator.h
-  api.h
-  array.h
-  buffer.h
-  builder.h
-  compare.h
-  memory_pool.h
-  pretty_print.h
-  record_batch.h
-  status.h
-  stl.h
-  table.h
-  table_builder.h
-  tensor.h
-  type.h
-  type_fwd.h
-  type_traits.h
-  test-util.h
-  visitor.h
-  visitor_inline.h
-  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow")
+ARROW_INSTALL_ALL_HEADERS("arrow")

 # pkg-config support
 configure_file(arrow.pc.in
@@ -15,7 +15,4 @@
 # specific language governing permissions and limitations
 # under the License.

-# Headers: top level
-install(FILES
-        convert.h
-        DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/adapters/tensorflow")
+ARROW_INSTALL_ALL_HEADERS("arrow/adapters/tensorflow")
@@ -15,12 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.

-# Headers: top level
-install(FILES
-  api.h
-  context.h
-  kernel.h
-  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/compute")
+ARROW_INSTALL_ALL_HEADERS("arrow/compute")

 # pkg-config support
 configure_file(arrow-compute.pc.in
@@ -29,9 +29,4 @@ ADD_ARROW_BENCHMARK(converter-benchmark
 ADD_ARROW_BENCHMARK(parser-benchmark
  PREFIX "arrow-csv")

-# Headers: top level
-file(GLOB_RECURSE ARROW_CSV_HEADERS "*.h")
-
-install(FILES
-  ${ARROW_CSV_HEADERS}
-  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/csv")
+ARROW_INSTALL_ALL_HEADERS("arrow/csv")
@@ -18,15 +18,7 @@
 add_custom_target(arrow_hiveserver2)

 # Headers: top level
-install(FILES
-  api.h
-  columnar-row-set.h
-  operation.h
-  service.h
-  session.h
-  types.h
-  util.h
-  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/dbi/hiveserver2")
+ARROW_INSTALL_ALL_HEADERS("arrow/dbi/hiveserver2")

 set(ARROW_HIVESERVER2_SRCS
  columnar-row-set.cc
@@ -63,12 +63,7 @@ install(FILES
  "${CMAKE_CURRENT_BINARY_DIR}/cuda_version.h"
  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/gpu")

-install(FILES
-  cuda_api.h
-  cuda_arrow_ipc.h
-  cuda_context.h
-  cuda_memory.h
-  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/gpu")
+ARROW_INSTALL_ALL_HEADERS("arrow/gpu")

 # pkg-config support
 configure_file(arrow-cuda.pc.in
@@ -88,16 +88,7 @@ add_custom_command(
 add_custom_target(metadata_fbs DEPENDS ${FBS_OUTPUT_FILES})

 # Headers: top level
-install(FILES
-  api.h
-  dictionary.h
-  feather.h
-  json.h
-  json-simple.h
-  message.h
-  reader.h
-  writer.h
-  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/ipc")
+ARROW_INSTALL_ALL_HEADERS("arrow/ipc")

 if (ARROW_BUILD_STATIC)
  set(ARROW_UTIL_LIB arrow_static)
@@ -91,29 +91,7 @@ if ("${COMPILER_FAMILY}" STREQUAL "clang")
    COMPILE_FLAGS -Wno-parentheses-equality)
 endif()

-install(FILES
-  api.h
-  arrow_to_pandas.h
-  benchmark.h
-  common.h
-  config.h
-  decimal.h
-  deserialize.h
-  helpers.h
-  inference.h
-  init.h
-  io.h
-  iterators.h
-  numpy_convert.h
-  numpy_interop.h
-  numpy_to_arrow.h
-  python_to_arrow.h
-  platform.h
-  pyarrow.h
-  serialize.h
-  type_traits.h
-  visibility.h
-  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/python")
+ARROW_INSTALL_ALL_HEADERS("arrow/python")

 # pkg-config support
 configure_file(arrow-python.pc.in
@@ -20,45 +20,7 @@
 #######################################

 # Headers: top level
-install(FILES
-  bit-stream-utils.h
-  bit-util.h
-  bpacking.h
-  checked_cast.h
-  compiler-util.h
-  compression.h
-  compression_brotli.h
-  compression_bz2.h
-  compression_lz4.h
-  compression_snappy.h
-  compression_zlib.h
-  compression_zstd.h
-  cpu-info.h
-  date.h
-  decimal.h
-  hash-util.h
-  hashing.h
-  io-util.h
-  key_value_metadata.h
-  lazy.h
-  logging.h
-  macros.h
-  memory.h
-  neon-util.h
-  parallel.h
-  rle-encoding.h
-  sse-util.h
-  stl.h
-  stopwatch.h
-  string.h
-  string_view.h
-  thread-pool.h
-  type_traits.h
-  utf8.h
-  variant.h
-  visibility.h
-  windows_compatibility.h
-  DESTINATION include/arrow/util)
+ARROW_INSTALL_ALL_HEADERS("arrow/util")

 #######################################
 # arrow_test_main
@@ -92,31 +92,8 @@ add_dependencies(gandiva ${GANDIVA_LIBRARIES})
 # install for gandiva
 include(GNUInstallDirs)

-# install libgandiva
-install(
-  TARGETS gandiva_shared gandiva_static
-  DESTINATION ${CMAKE_INSTALL_LIBDIR}
-)
-
 # install the header files.
-install(FILES
-  arrow.h
-  condition.h
-  configuration.h
-  expression.h
-  expression_registry.h
-  filter.h
-  func_descriptor.h
-  function_signature.h
-  gandiva_aliases.h
-  literal_holder.h
-  logging.h
-  node.h
-  node_visitor.h
-  projector.h
-  selection_vector.h
-  tree_expr_builder.h
-  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/gandiva")
+ARROW_INSTALL_ALL_HEADERS("gandiva")

 # pkg-config support
 configure_file(gandiva.pc.in
@@ -167,31 +144,24 @@ function(ADD_GANDIVA_TEST REL_TEST_NAME)
  endif()
 endfunction()

-if (ARROW_GANDIVA_BUILD_TESTS)
-  ADD_GANDIVA_TEST(bitmap_accumulator_test)
-  ADD_GANDIVA_TEST(engine_llvm_test)
-  ADD_GANDIVA_TEST(function_signature_test)
-  ADD_GANDIVA_TEST(function_registry_test)
-  ADD_GANDIVA_TEST(llvm_types_test)
-  ADD_GANDIVA_TEST(llvm_generator_test)
-  ADD_GANDIVA_TEST(annotator_test)
-  ADD_GANDIVA_TEST(tree_expr_test)
-  ADD_GANDIVA_TEST(expr_decomposer_test)
-  ADD_GANDIVA_TEST(expression_registry_test)
-  ADD_GANDIVA_TEST(selection_vector_test)
-  ADD_GANDIVA_TEST(lru_cache_test)
-  ADD_GANDIVA_TEST(to_date_holder_test)
-  ADD_GANDIVA_TEST(simple_arena_test)
-endif()
+ADD_GANDIVA_TEST(bitmap_accumulator_test)
+ADD_GANDIVA_TEST(engine_llvm_test)
+ADD_GANDIVA_TEST(function_signature_test)
+ADD_GANDIVA_TEST(function_registry_test)
+ADD_GANDIVA_TEST(llvm_types_test)
+ADD_GANDIVA_TEST(llvm_generator_test)
+ADD_GANDIVA_TEST(annotator_test)
+ADD_GANDIVA_TEST(tree_expr_test)
+ADD_GANDIVA_TEST(expr_decomposer_test)
+ADD_GANDIVA_TEST(expression_registry_test)
+ADD_GANDIVA_TEST(selection_vector_test)
+ADD_GANDIVA_TEST(lru_cache_test)
+ADD_GANDIVA_TEST(to_date_holder_test)
+ADD_GANDIVA_TEST(simple_arena_test)

 if (ARROW_GANDIVA_JAVA)
  add_subdirectory(jni)
 endif()
+
 add_subdirectory(precompiled)
-
-if (ARROW_GANDIVA_BUILD_TESTS)
-  include(CTest)
-  enable_testing()
-
-  add_subdirectory(tests)
-endif()
+add_subdirectory(tests)
@@ -69,7 +69,7 @@ function(add_precompiled_unit_test REL_TEST_NAME)
 endfunction(add_precompiled_unit_test REL_TEST_NAME)

 # testing
-if (ARROW_GANDIVA_BUILD_TESTS)
+if (ARROW_BUILD_TESTS)
  add_precompiled_unit_test(bitmap_test.cc bitmap.cc)
  add_precompiled_unit_test(epoch_time_point_test.cc)
  add_precompiled_unit_test(time_test.cc time.cc timestamp_arithmetic.cc ../context_helper.cc)
@@ -238,26 +238,7 @@ add_subdirectory(api)
 add_subdirectory(arrow)
 add_subdirectory(util)

-# Headers: top level
-install(FILES
-  bloom_filter.h
-  column_reader.h
-  column_page.h
-  column_scanner.h
-  column_writer.h
-  encoding.h
-  exception.h
-  file_reader.h
-  file_writer.h
-  hasher.h
-  metadata.h
-  murmur3.h
-  printer.h
-  properties.h
-  schema.h
-  statistics.h
-  types.h
-  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/parquet")
+ARROW_INSTALL_ALL_HEADERS("parquet")

 configure_file(parquet_version.h.in
  "${CMAKE_CURRENT_BINARY_DIR}/parquet_version.h"
@@ -16,9 +16,4 @@
 # under the License.

 # Headers: public api
-install(FILES
-  io.h
-  reader.h
-  writer.h
-  schema.h
-  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/parquet/api")
+ARROW_INSTALL_ALL_HEADERS("parquet/api")
@@ -22,9 +22,4 @@ ADD_ARROW_BENCHMARK(reader-writer-benchmark
  PREFIX "parquet-arrow"
  EXTRA_LINK_LIBS ${PARQUET_BENCHMARK_LINK_LIBRARIES})

-# Headers: top level
-install(FILES
-  reader.h
-  schema.h
-  writer.h
-  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/parquet/arrow")
+ARROW_INSTALL_ALL_HEADERS("parquet/arrow")
@@ -16,12 +16,7 @@
 # under the License.

 # Headers: util
-install(FILES
-  comparison.h
-  macros.h
-  memory.h
-  visibility.h
-  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/parquet/util")
+ARROW_INSTALL_ALL_HEADERS("parquet/util")

 ADD_PARQUET_TEST(comparison-test)
 ADD_PARQUET_TEST(memory-test)
@@ -138,7 +138,6 @@ if (ARROW_RPATH_ORIGIN)
      INSTALL_RPATH ${_lib_install_rpath})
 endif()

-# Headers: top level
 install(FILES
  common.h
  compat.h
@@ -149,7 +148,9 @@ install(FILES

 # Plasma store
 set_target_properties(plasma_store_server PROPERTIES INSTALL_RPATH_USE_LINK_PATH TRUE)
-install(TARGETS plasma_store_server DESTINATION ${CMAKE_INSTALL_BINDIR})
+install(TARGETS plasma_store_server
+  ${INSTALL_IS_OPTIONAL}
+  DESTINATION ${CMAKE_INSTALL_BINDIR})

 # pkg-config support
 configure_file(plasma.pc.in
@@ -26,7 +26,9 @@ if (PARQUET_BUILD_EXECUTABLES)
    target_link_libraries(${TOOL} parquet_static)
    # Avoid unsetting RPATH when installing
    set_target_properties(${TOOL} PROPERTIES INSTALL_RPATH_USE_LINK_PATH TRUE)
-    install(TARGETS ${TOOL} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
+    install(TARGETS ${TOOL}
+      ${INSTALL_IS_OPTIONAL}
+      RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
  endforeach(TOOL)

  add_dependencies(parquet ${PARQUET_TOOLS})