mirrors/arrow
0
0 Fork 0
mirror of https://github.com/apache/arrow.git synced 2026-10-28 16:47:19 +08:00
Code Activity

ARROW-6610: [C++] Add cmake option to disable filesystem layer

Browse Source
This commit is contained in:
Antoine Pitrou committed 2019-09-30 11:13:27 +02:00
1 parent 789a8e5785
commit b74741dde6
5 files changed
+158 -139
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

cpp/CMakeLists.txt
+5 -1
View File
@@ -268,10 +268,14 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_INTEGRATION)
set(ARROW_JSON ON)
endif()
if(ARROW_FLIGHT OR ARROW_PARQUET OR ARROW_BUILD_TESTS)
if(ARROW_CUDA OR ARROW_FLIGHT OR ARROW_PARQUET OR ARROW_BUILD_TESTS)
set(ARROW_IPC ON)
endif()
if(ARROW_DATASET)
set(ARROW_FILESYSTEM ON)
endif()
if(MSVC)
# ORC doesn't build on windows
set(ARROW_ORC OFF)
cpp/cmake_modules/DefineOptions.cmake
+22 -22
View File
@@ -90,21 +90,19 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
#----------------------------------------------------------------------
set_option_category("Test and benchmark")
define_option(ARROW_BUILD_EXAMPLES "Build the Arrow examples, default OFF" OFF)
define_option(ARROW_BUILD_EXAMPLES "Build the Arrow examples" OFF)
define_option(ARROW_BUILD_TESTS "Build the Arrow googletest unit tests, default OFF"
define_option(ARROW_BUILD_TESTS "Build the Arrow googletest unit tests" OFF)
define_option(ARROW_BUILD_INTEGRATION "Build the Arrow integration test executables"
OFF)
define_option(ARROW_BUILD_INTEGRATION
"Build the Arrow integration test executables, default OFF" OFF)
define_option(ARROW_BUILD_BENCHMARKS "Build the Arrow micro benchmarks, default OFF"
OFF)
define_option(ARROW_BUILD_BENCHMARKS "Build the Arrow micro benchmarks" OFF)
# Reference benchmarks are used to compare to naive implementation, or
# discover various hardware limits.
define_option(ARROW_BUILD_BENCHMARKS_REFERENCE
"Build the Arrow micro reference benchmarks, default OFF." OFF)
"Build the Arrow micro reference benchmarks" OFF)
define_option_string(ARROW_TEST_LINKAGE
"Linkage of Arrow libraries with unit tests executables."
@@ -141,47 +139,49 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
#----------------------------------------------------------------------
set_option_category("Project component")
define_option(ARROW_BUILD_UTILITIES "Build Arrow commandline utilities" ON)
define_option(ARROW_COMPUTE "Build the Arrow Compute Modules" ON)
define_option(ARROW_CUDA "Build the Arrow CUDA extensions (requires CUDA toolkit)" OFF)
define_option(ARROW_DATASET "Build the Arrow Dataset Modules" ON)
define_option(ARROW_FILESYSTEM "Build the Arrow Filesystem Layer" ON)
define_option(ARROW_FLIGHT
"Build the Arrow Flight RPC System (requires GRPC, Protocol Buffers)" OFF)
define_option(ARROW_GANDIVA "Build the Gandiva libraries" OFF)
define_option(ARROW_PARQUET "Build the Parquet libraries" OFF)
define_option(ARROW_HDFS "Build the Arrow HDFS bridge" ON)
define_option(ARROW_HIVESERVER2 "Build the HiveServer2 client and Arrow adapter" OFF)
define_option(ARROW_IPC "Build the Arrow IPC extensions" ON)
define_option(ARROW_BUILD_UTILITIES "Build Arrow commandline utilities" ON)
define_option(ARROW_CUDA "Build the Arrow CUDA extensions (requires CUDA toolkit)" OFF)
define_option(ARROW_ORC "Build the Arrow ORC adapter" OFF)
define_option(ARROW_JEMALLOC "Build the Arrow jemalloc-based allocator" ON)
define_option(ARROW_JNI "Build the Arrow JNI lib" OFF)
define_option(ARROW_TENSORFLOW "Build Arrow with TensorFlow support enabled" OFF)
define_option(ARROW_JEMALLOC "Build the Arrow jemalloc-based allocator" ON)
define_option(ARROW_JSON "Build Arrow with JSON support (requires RapidJSON)" ON)
define_option(ARROW_MIMALLOC "Build the Arrow mimalloc-based allocator" OFF)
define_option(ARROW_HDFS "Build the Arrow HDFS bridge" ON)
define_option(ARROW_PARQUET "Build the Parquet libraries" OFF)
define_option(ARROW_PYTHON "Build the Arrow CPython extensions" OFF)
define_option(ARROW_HIVESERVER2 "Build the HiveServer2 client and Arrow adapter" OFF)
define_option(ARROW_ORC "Build the Arrow ORC adapter" OFF)
define_option(ARROW_PLASMA "Build the plasma object store along with Arrow" OFF)
define_option(ARROW_PLASMA_JAVA_CLIENT "Build the plasma object store java client" OFF)
define_option(ARROW_JSON "Build Arrow with JSON support (requires RapidJSON)" ON)
define_option(ARROW_PYTHON "Build the Arrow CPython extensions" OFF)
define_option(ARROW_S3 "Build Arrow with S3 support (requires the AWS SDK for C++)" OFF)
define_option(ARROW_TENSORFLOW "Build Arrow with TensorFlow support enabled" OFF)
#----------------------------------------------------------------------
set_option_category("Thirdparty toolchain")
cpp/cmake_modules/ThirdpartyToolchain.cmake
+5 -2
View File
@@ -15,6 +15,9 @@
# specific language governing permissions and limitations
# under the License.
include(ProcessorCount)
processorcount(NPROC)
add_custom_target(rapidjson)
add_custom_target(toolchain)
add_custom_target(toolchain-benchmarks)
@@ -500,7 +503,7 @@ if(${CMAKE_GENERATOR} MATCHES "Makefiles")
set(MAKE_BUILD_ARGS "")
else()
# limit the maximum number of jobs for ninja
set(MAKE_BUILD_ARGS "-j4")
set(MAKE_BUILD_ARGS "-j${NPROC}")
endif()
# ----------------------------------------------------------------------
@@ -560,7 +563,7 @@ macro(build_boost)
else()
set(BOOST_BUILD_VARIANT "release")
endif()
set(BOOST_BUILD_COMMAND "./b2" "link=${BOOST_BUILD_LINK}"
set(BOOST_BUILD_COMMAND "./b2" "-j${NPROC}" "link=${BOOST_BUILD_LINK}"
"variant=${BOOST_BUILD_VARIANT}")
if(MSVC)
string(REGEX
cpp/examples/minimal_build/build.sh
+5 -1
View File
@@ -22,6 +22,7 @@
# docker run --rm -t -i -v $PWD:/io -v $ARROW_ROOT:/arrow arrow_cpp_minimal /io/build.sh
BUILD_DIR=/build
NPROC=$(nproc)
mkdir $BUILD_DIR
pushd $BUILD_DIR
@@ -30,6 +31,8 @@ cmake /arrow/cpp -DBOOST_SOURCE=BUNDLED \
-DARROW_BOOST_USE_SHARED=OFF \
-DARROW_COMPUTE=OFF \
-DARROW_DATASET=OFF \
-DARROW_FILESYSTEM=OFF \
-DARROW_HDFS=OFF \
-DARROW_JEMALLOC=OFF \
-DARROW_JSON=OFF \
-DARROW_USE_GLOG=OFF \
@@ -41,6 +44,7 @@ cmake /arrow/cpp -DBOOST_SOURCE=BUNDLED \
-DARROW_WITH_BROTLI=OFF \
-DARROW_BUILD_UTILITIES=OFF
make -j8
make -j$NPROC
make install
popd
cpp/src/arrow/CMakeLists.txt
+121 -113
View File
@@ -114,12 +114,6 @@ set(ARROW_SRCS
csv/options.cc
csv/parser.cc
csv/reader.cc
filesystem/filesystem.cc
filesystem/localfs.cc
filesystem/mockfs.cc
filesystem/path_tree.cc
filesystem/path_util.cc
filesystem/util_internal.cc
io/buffered.cc
io/compressed.cc
io/file.cc
@@ -149,6 +143,13 @@ set(ARROW_SRCS
vendored/base64.cpp
vendored/datetime/tz.cpp)
set(ARROW_TESTING_SRCS
io/test_common.cc
ipc/test_common.cc
testing/gtest_util.cc
testing/random.cc
testing/generator.cc)
# Add dependencies for third-party allocators.
# If possible we only want memory_pool.cc to wait for allocators to finish building,
# but that only works with Ninja
@@ -173,26 +174,6 @@ endif()
unset(_allocator_dependencies)
if(ARROW_JSON)
add_subdirectory(json)
set(ARROW_SRCS
${ARROW_SRCS}
json/options.cc
json/chunked_builder.cc
json/chunker.cc
json/converter.cc
json/parser.cc
json/reader.cc)
endif()
if(ARROW_S3)
set(ARROW_SRCS ${ARROW_SRCS} filesystem/s3fs.cc)
endif()
if(ARROW_WITH_URIPARSER)
set(ARROW_SRCS ${ARROW_SRCS} util/uri.cc)
endif()
if("${COMPILER_FAMILY}" STREQUAL "clang")
set_property(SOURCE util/io_util.cc
APPEND_STRING
@@ -202,94 +183,149 @@ endif()
# Unconditionally install testing headers that are also useful for Arrow consumers.
add_subdirectory(testing)
#
# Add the various components
#
add_subdirectory(array)
add_subdirectory(csv)
add_subdirectory(io)
add_subdirectory(util)
add_subdirectory(vendored)
if(ARROW_COMPUTE)
add_subdirectory(compute)
set(ARROW_SRCS
${ARROW_SRCS}
compute/context.cc
compute/expression.cc
compute/logical_type.cc
compute/operation.cc
compute/kernels/aggregate.cc
compute/kernels/boolean.cc
compute/kernels/cast.cc
compute/kernels/compare.cc
compute/kernels/count.cc
compute/kernels/hash.cc
compute/kernels/filter.cc
compute/kernels/mean.cc
compute/kernels/sort_to_indices.cc
compute/kernels/sum.cc
compute/kernels/take.cc
compute/kernels/isin.cc
compute/kernels/util_internal.cc
compute/operations/cast.cc
compute/operations/literal.cc)
list(APPEND ARROW_SRCS
compute/context.cc
compute/expression.cc
compute/logical_type.cc
compute/operation.cc
compute/kernels/aggregate.cc
compute/kernels/boolean.cc
compute/kernels/cast.cc
compute/kernels/compare.cc
compute/kernels/count.cc
compute/kernels/hash.cc
compute/kernels/filter.cc
compute/kernels/mean.cc
compute/kernels/sort_to_indices.cc
compute/kernels/sum.cc
compute/kernels/take.cc
compute/kernels/isin.cc
compute/kernels/util_internal.cc
compute/operations/cast.cc
compute/operations/literal.cc)
endif()
if(ARROW_CUDA)
# IPC extensions required to build the CUDA library
set(ARROW_IPC ON)
add_subdirectory(gpu)
endif()
if(ARROW_WITH_BROTLI)
add_definitions(-DARROW_WITH_BROTLI)
set(ARROW_SRCS util/compression_brotli.cc ${ARROW_SRCS})
if(ARROW_DATASET)
add_subdirectory(dataset)
endif()
if(ARROW_WITH_BZ2)
add_definitions(-DARROW_WITH_BZ2)
set(ARROW_SRCS util/compression_bz2.cc ${ARROW_SRCS})
if(ARROW_FILESYSTEM)
add_subdirectory(filesystem)
list(APPEND ARROW_SRCS
filesystem/filesystem.cc
filesystem/localfs.cc
filesystem/mockfs.cc
filesystem/path_tree.cc
filesystem/path_util.cc
filesystem/util_internal.cc)
if(ARROW_S3)
list(APPEND ARROW_SRCS filesystem/s3fs.cc)
endif()
list(APPEND ARROW_TESTING_SRCS filesystem/test_util.cc)
endif()
if(ARROW_WITH_LZ4)
add_definitions(-DARROW_WITH_LZ4)
set(ARROW_SRCS util/compression_lz4.cc ${ARROW_SRCS})
if(ARROW_FLIGHT)
add_subdirectory(flight)
endif()
if(ARROW_WITH_SNAPPY)
add_definitions(-DARROW_WITH_SNAPPY)
set(ARROW_SRCS util/compression_snappy.cc ${ARROW_SRCS})
if(ARROW_HIVESERVER2)
add_subdirectory(dbi/hiveserver2)
endif()
if(ARROW_WITH_ZLIB)
add_definitions(-DARROW_WITH_ZLIB)
set(ARROW_SRCS util/compression_zlib.cc ${ARROW_SRCS})
if(ARROW_IPC)
add_subdirectory(ipc)
list(APPEND ARROW_SRCS
ipc/dictionary.cc
ipc/feather.cc
ipc/message.cc
ipc/metadata_internal.cc
ipc/options.cc
ipc/reader.cc
ipc/writer.cc)
if(ARROW_JSON)
list(
APPEND ARROW_SRCS ipc/json_integration.cc ipc/json_internal.cc ipc/json_simple.cc)
endif()
add_dependencies(arrow_dependencies metadata_fbs)
endif()
if(ARROW_WITH_ZSTD)
add_definitions(-DARROW_WITH_ZSTD)
set(ARROW_SRCS util/compression_zstd.cc ${ARROW_SRCS})
if(ARROW_JSON)
add_subdirectory(json)
list(APPEND ARROW_SRCS
json/options.cc
json/chunked_builder.cc
json/chunker.cc
json/converter.cc
json/parser.cc
json/reader.cc)
endif()
if(ARROW_ORC)
add_subdirectory(adapters/orc)
set(ARROW_SRCS adapters/orc/adapter.cc adapters/orc/adapter_util.cc ${ARROW_SRCS})
list(APPEND ARROW_SRCS adapters/orc/adapter.cc adapters/orc/adapter_util.cc)
endif()
if(ARROW_PYTHON)
add_subdirectory(python)
endif()
if(ARROW_TENSORFLOW)
add_subdirectory(adapters/tensorflow)
endif()
if(ARROW_IPC)
add_subdirectory(ipc)
if(ARROW_WITH_URIPARSER)
list(APPEND ARROW_SRCS util/uri.cc)
endif()
set(ARROW_SRCS
${ARROW_SRCS}
ipc/dictionary.cc
ipc/feather.cc
ipc/message.cc
ipc/metadata_internal.cc
ipc/options.cc
ipc/reader.cc
ipc/writer.cc)
if(ARROW_WITH_BROTLI)
add_definitions(-DARROW_WITH_BROTLI)
list(APPEND ARROW_SRCS util/compression_brotli.cc)
endif()
if(ARROW_JSON)
set(ARROW_SRCS ${ARROW_SRCS} ipc/json_integration.cc ipc/json_internal.cc
ipc/json_simple.cc)
endif()
add_dependencies(arrow_dependencies metadata_fbs)
if(ARROW_WITH_BZ2)
add_definitions(-DARROW_WITH_BZ2)
list(APPEND ARROW_SRCS util/compression_bz2.cc)
endif()
if(ARROW_WITH_LZ4)
add_definitions(-DARROW_WITH_LZ4)
list(APPEND ARROW_SRCS util/compression_lz4.cc)
endif()
if(ARROW_WITH_SNAPPY)
add_definitions(-DARROW_WITH_SNAPPY)
list(APPEND ARROW_SRCS util/compression_snappy.cc)
endif()
if(ARROW_WITH_ZLIB)
add_definitions(-DARROW_WITH_ZLIB)
list(APPEND ARROW_SRCS util/compression_zlib.cc)
endif()
if(ARROW_WITH_ZSTD)
add_definitions(-DARROW_WITH_ZSTD)
list(APPEND ARROW_SRCS util/compression_zstd.cc)
endif()
if(NOT APPLE AND NOT MSVC)
@@ -333,12 +369,7 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS OR ARROW_BUILD_INTEGRATION)
# that depend on gtest
add_arrow_lib(arrow_testing
SOURCES
io/test_common.cc
ipc/test_common.cc
filesystem/test_util.cc
testing/gtest_util.cc
testing/random.cc
testing/generator.cc
${ARROW_TESTING_SRCS}
OUTPUTS
ARROW_TESTING_LIBRARIES
DEPENDENCIES
@@ -412,26 +443,3 @@ add_arrow_test(sparse_tensor_test)
add_arrow_benchmark(builder_benchmark)
add_arrow_benchmark(type_benchmark)
add_subdirectory(array)
add_subdirectory(csv)
add_subdirectory(filesystem)
add_subdirectory(io)
add_subdirectory(util)
add_subdirectory(vendored)
if(ARROW_DATASET)
add_subdirectory(dataset)
endif()
if(ARROW_FLIGHT)
add_subdirectory(flight)
endif()
if(ARROW_PYTHON)
add_subdirectory(python)
endif()
if(ARROW_HIVESERVER2)
add_subdirectory(dbi/hiveserver2)
endif()