mirrors/arrow
0
0 Fork 0
mirror of https://github.com/apache/arrow.git synced 2026-10-28 16:47:19 +08:00
Code Activity

ARROW-1123: Make jemalloc the default allocator

Browse Source 
Change-Id: Ib03392431851773df3b59b2c9a4d9a7bd672d2cb
This commit is contained in:
Uwe L. Korn committed 2017-06-18 20:17:46 +02:00
1 parent 5be05ac154
commit e65d0d1dc1
22 files changed
+109 -660
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

cpp/CMakeLists.txt
+66 -13
View File
@@ -437,6 +437,14 @@ if (DEFINED ENV{GFLAGS_HOME})
set(GFLAGS_HOME "$ENV{GFLAGS_HOME}")
endif()
# ----------------------------------------------------------------------
# Find pthreads
if (NOT MSVC)
set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
find_package(Threads REQUIRED)
endif()
# ----------------------------------------------------------------------
# Add Boost dependencies (code adapted from Apache Kudu (incubating))
@@ -756,7 +764,8 @@ if (ARROW_JEMALLOC)
include_directories(SYSTEM ${JEMALLOC_INCLUDE_DIR})
ADD_THIRDPARTY_LIB(jemalloc
STATIC_LIB ${JEMALLOC_STATIC_LIB}
SHARED_LIB ${JEMALLOC_SHARED_LIB})
SHARED_LIB ${JEMALLOC_SHARED_LIB}
DEPS Threads::Threads)
endif()
## Google PerfTools
@@ -803,20 +812,16 @@ include_directories(SYSTEM "${HADOOP_HOME}/include")
# Linker setup
############################################################
set(ARROW_MIN_TEST_LIBS
${ARROW_STATIC_LINK_LIBS}
arrow_static
gtest
gtest_main
${ARROW_BASE_LIBS})
if (APPLE)
if(NOT MSVC)
set(ARROW_MIN_TEST_LIBS
${ARROW_MIN_TEST_LIBS}
${CMAKE_DL_LIBS})
elseif(NOT MSVC)
set(ARROW_MIN_TEST_LIBS
${ARROW_MIN_TEST_LIBS}
pthread
${CMAKE_DL_LIBS})
endif()
set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
@@ -938,8 +943,60 @@ set(ARROW_STATIC_PRIVATE_LINK_LIBS
if (NOT MSVC)
set(ARROW_LINK_LIBS
${ARROW_LINK_LIBS}
${CMAKE_DL_LIBS}
pthread)
${CMAKE_DL_LIBS})
endif()
if (ARROW_JEMALLOC)
add_definitions(-DARROW_JEMALLOC)
# In the case that jemalloc is only available as a shared library also use it to
# link it in the static requirements. In contrast to other libraries we try in
# most cases to use the system provided version of jemalloc to better align with
# other potential users of jemalloc.
if (JEMALLOC_STATIC_LIB)
set(ARROW_JEMALLOC_STATIC_LINK_LIBS
jemalloc_static
)
else()
set(ARROW_JEMALLOC_STATIC_LINK_LIBS
jemalloc_shared
)
endif()
if (NOT APPLE)
set(ARROW_JEMALLOC_STATIC_LINK_LIBS ${ARROW_JEMALLOC_STATIC_LINK_LIBS} Threads::Threads)
endif()
if (ARROW_JEMALLOC_USE_SHARED)
set(ARROW_JEMALLOC_SHARED_LINK_LIBS
jemalloc_shared
)
else()
if (CMAKE_COMPILER_IS_GNUCXX)
set(ARROW_JEMALLOC_SHARED_LINK_LIBS
jemalloc_static
# For glibc <2.17 we need to link to librt.
# As we compile with --as-needed by default, the linker will omit this
# dependency if not required.
rt
)
else()
set(ARROW_JEMALLOC_SHARED_LINK_LIBS
jemalloc_static
)
endif()
endif()
set(ARROW_SHARED_PRIVATE_LINK_LIBS
${ARROW_SHARED_PRIVATE_LINK_LIBS}
${ARROW_JEMALLOC_SHARED_LINK_LIBS})
set(ARROW_STATIC_LINK_LIBS
${ARROW_STATIC_LINK_LIBS}
${ARROW_JEMALLOC_STATIC_LINK_LIBS})
elseif (NOT MSVC)
# We need to separate this as otherwise CMake would mess with the library
# linking order.
set(ARROW_LINK_LIBS
${ARROW_LINK_LIBS}
Threads::Threads)
endif()
if(RAPIDJSON_VENDORED)
@@ -1021,10 +1078,6 @@ ADD_ARROW_LIB(arrow
add_subdirectory(src/arrow/util)
if(ARROW_JEMALLOC)
add_subdirectory(src/arrow/jemalloc)
endif()
if(ARROW_PYTHON)
find_package(PythonLibsNew REQUIRED)
find_package(NumPy REQUIRED)
cpp/cmake_modules/BuildUtils.cmake
+12 -5
View File
@@ -46,6 +46,10 @@ function(ADD_THIRDPARTY_LIB LIB_NAME)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
endif()
if(ARG_DEPS)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES "${ARG_DEPS}")
endif()
message("Added shared library dependency ${LIB_NAME}: ${ARG_SHARED_LIB}")
elseif(ARG_STATIC_LIB)
add_library(${LIB_NAME} STATIC IMPORTED)
@@ -55,6 +59,10 @@ function(ADD_THIRDPARTY_LIB LIB_NAME)
add_library(${AUG_LIB_NAME} STATIC IMPORTED)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
if(ARG_DEPS)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES "${ARG_DEPS}")
endif()
message("Added static library dependency ${LIB_NAME}: ${ARG_STATIC_LIB}")
elseif(ARG_SHARED_LIB)
add_library(${LIB_NAME} SHARED IMPORTED)
@@ -72,14 +80,13 @@ function(ADD_THIRDPARTY_LIB LIB_NAME)
PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
endif()
message("Added shared library dependency ${LIB_NAME}: ${ARG_SHARED_LIB}")
if(ARG_DEPS)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES "${ARG_DEPS}")
endif()
else()
message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
endif()
if(ARG_DEPS)
set_target_properties(${LIB_NAME}
PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES "${ARG_DEPS}")
endif()
endfunction()
function(ADD_ARROW_LIB LIB_NAME)
cpp/src/arrow/jemalloc/CMakeLists.txt
-120
View File
@@ -1,120 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# ----------------------------------------------------------------------
# arrow_jemalloc : Arrow jemalloc-based allocator
include_directories(SYSTEM "{JEMALLOC_INCLUDE_DIR}")
# In the case that jemalloc is only available as a shared library also use it to
# link it in the static requirements. In contrast to other libraries we try in
# most cases to use the system provided version of jemalloc to better align with
# other potential users of jemalloc.
if (JEMALLOC_STATIC_LIB)
set(ARROW_JEMALLOC_STATIC_LINK_LIBS
arrow_static
jemalloc_static
)
else()
set(ARROW_JEMALLOC_STATIC_LINK_LIBS
arrow_static
jemalloc_shared
)
endif()
if (NOT APPLE)
set(ARROW_JEMALLOC_STATIC_LINK_LIBS ${ARROW_JEMALLOC_STATIC_LINK_LIBS} pthread)
endif()
if (ARROW_JEMALLOC_USE_SHARED)
set(ARROW_JEMALLOC_SHARED_LINK_LIBS
arrow_shared
jemalloc_shared
)
else()
if (CMAKE_COMPILER_IS_GNUCXX)
set(ARROW_JEMALLOC_SHARED_LINK_LIBS
arrow_shared
jemalloc_static
# For glibc <2.17 we need to link to librt.
# As we compile with --as-needed by default, the linker will omit this
# dependency if not required.
rt
)
else()
set(ARROW_JEMALLOC_SHARED_LINK_LIBS
arrow_shared
jemalloc_static
)
endif()
endif()
if (ARROW_BUILD_STATIC)
set(ARROW_JEMALLOC_TEST_LINK_LIBS
${ARROW_JEMALLOC_STATIC_LINK_LIBS}
arrow_jemalloc_static)
else()
set(ARROW_JEMALLOC_TEST_LINK_LIBS
arrow_jemalloc_shared)
endif()
set(ARROW_JEMALLOC_SRCS
memory_pool.cc
)
if(NOT APPLE)
# Localize thirdparty symbols using a linker version script. This hides them
# from the client application. The OS X linker does not support the
# version-script option.
set(ARROW_JEMALLOC_LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/symbols.map")
endif()
if (JEMALLOC_VENDORED)
set(JEMALLOC_DEPENDENCIES jemalloc_ep)
endif()
ADD_ARROW_LIB(arrow_jemalloc
SOURCES ${ARROW_JEMALLOC_SRCS}
DEPENDENCIES ${JEMALLOC_DEPENDENCIES}
SHARED_LINK_FLAGS ${ARROW_JEMALLOC_LINK_FLAGS}
SHARED_LINK_LIBS ${ARROW_JEMALLOC_SHARED_LINK_LIBS}
SHARED_PRIVATE_LINK_LIBS ${ARROW_JEMALLOC_SHARED_PRIVATE_LINK_LIBS}
STATIC_LINK_LIBS ${ARROW_JEMALLOC_STATIC_LINK_LIBS}
STATIC_PRIVATE_LINK_LIBS ${ARROW_JEMALLOC_STATIC_PRIVATE_LINK_LIBS}
)
ADD_ARROW_TEST(jemalloc-memory_pool-test)
ARROW_TEST_LINK_LIBRARIES(jemalloc-memory_pool-test
${ARROW_JEMALLOC_TEST_LINK_LIBS})
ADD_ARROW_BENCHMARK(jemalloc-builder-benchmark)
ARROW_BENCHMARK_LINK_LIBRARIES(jemalloc-builder-benchmark
${ARROW_JEMALLOC_TEST_LINK_LIBS})
# Headers: top level
install(FILES
memory_pool.h
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/jemalloc")
# pkg-config support
configure_file(arrow-jemalloc.pc.in
"${CMAKE_CURRENT_BINARY_DIR}/arrow-jemalloc.pc"
@ONLY)
install(
FILES "${CMAKE_CURRENT_BINARY_DIR}/arrow-jemalloc.pc"
DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/")
cpp/src/arrow/jemalloc/arrow-jemalloc.pc.in
-30
View File
@@ -1,30 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
prefix=@CMAKE_INSTALL_PREFIX@
libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@
includedir=${prefix}/include
so_version=@ARROW_SO_VERSION@
abi_version=@ARROW_ABI_VERSION@
Name: Apache Arrow jemalloc-based allocator
Description: jemalloc allocator for Arrow.
Version: @ARROW_VERSION@
Libs: -L${libdir} -larrow_jemalloc
Cflags: -I${includedir}
Requires: arrow
cpp/src/arrow/jemalloc/jemalloc-builder-benchmark.cc
-47
View File
@@ -1,47 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "benchmark/benchmark.h"
#include "arrow/builder.h"
#include "arrow/jemalloc/memory_pool.h"
#include "arrow/test-util.h"
namespace arrow {
constexpr int64_t kFinalSize = 256;
static void BM_BuildPrimitiveArrayNoNulls(
benchmark::State& state) { // NOLINT non-const reference
// 2 MiB block
std::vector<int64_t> data(256 * 1024, 100);
while (state.KeepRunning()) {
Int64Builder builder(jemalloc::MemoryPool::default_pool(), arrow::int64());
for (int i = 0; i < kFinalSize; i++) {
// Build up an array of 512 MiB in size
builder.Append(data.data(), data.size(), nullptr);
}
std::shared_ptr<Array> out;
builder.Finish(&out);
}
state.SetBytesProcessed(
state.iterations() * data.size() * sizeof(int64_t) * kFinalSize);
}
BENCHMARK(BM_BuildPrimitiveArrayNoNulls)->Repetitions(5)->Unit(benchmark::kMillisecond);
} // namespace arrow
cpp/src/arrow/jemalloc/jemalloc-memory_pool-test.cc
-51
View File
@@ -1,51 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <cstdint>
#include <limits>
#include "gtest/gtest.h"
#include "arrow/jemalloc/memory_pool.h"
#include "arrow/memory_pool-test.h"
namespace arrow {
namespace jemalloc {
namespace test {
class TestJemallocMemoryPool : public ::arrow::test::TestMemoryPoolBase {
public:
::arrow::MemoryPool* memory_pool() override {
return ::arrow::jemalloc::MemoryPool::default_pool();
}
};
TEST_F(TestJemallocMemoryPool, MemoryTracking) {
this->TestMemoryTracking();
}
TEST_F(TestJemallocMemoryPool, OOM) {
this->TestOOM();
}
TEST_F(TestJemallocMemoryPool, Reallocate) {
this->TestReallocate();
}
} // namespace test
} // namespace jemalloc
} // namespace arrow
cpp/src/arrow/jemalloc/memory_pool.cc
-76
View File
@@ -1,76 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "arrow/jemalloc/memory_pool.h"
#include <sstream>
// Needed to support jemalloc 3 and 4
#define JEMALLOC_MANGLE
#include <jemalloc/jemalloc.h>
#include "arrow/status.h"
constexpr size_t kAlignment = 64;
namespace arrow {
namespace jemalloc {
MemoryPool* MemoryPool::default_pool() {
static MemoryPool pool;
return &pool;
}
MemoryPool::MemoryPool() : allocated_size_(0) {}
MemoryPool::~MemoryPool() {}
Status MemoryPool::Allocate(int64_t size, uint8_t** out) {
*out = reinterpret_cast<uint8_t*>(mallocx(size, MALLOCX_ALIGN(kAlignment)));
if (*out == NULL) {
std::stringstream ss;
ss << "malloc of size " << size << " failed";
return Status::OutOfMemory(ss.str());
}
allocated_size_ += size;
return Status::OK();
}
Status MemoryPool::Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) {
*ptr = reinterpret_cast<uint8_t*>(rallocx(*ptr, new_size, MALLOCX_ALIGN(kAlignment)));
if (*ptr == NULL) {
std::stringstream ss;
ss << "realloc of size " << new_size << " failed";
return Status::OutOfMemory(ss.str());
}
allocated_size_ += new_size - old_size;
return Status::OK();
}
void MemoryPool::Free(uint8_t* buffer, int64_t size) {
allocated_size_ -= size;
dallocx(buffer, MALLOCX_ALIGN(kAlignment));
}
int64_t MemoryPool::bytes_allocated() const {
return allocated_size_.load();
}
} // namespace jemalloc
} // namespace arrow
cpp/src/arrow/jemalloc/memory_pool.h
-57
View File
@@ -1,57 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
// Public API for the jemalloc-based allocator
#ifndef ARROW_JEMALLOC_MEMORY_POOL_H
#define ARROW_JEMALLOC_MEMORY_POOL_H
#include "arrow/memory_pool.h"
#include <atomic>
namespace arrow {
class Status;
namespace jemalloc {
class ARROW_EXPORT MemoryPool : public ::arrow::MemoryPool {
public:
static MemoryPool* default_pool();
MemoryPool(MemoryPool const&) = delete;
MemoryPool& operator=(MemoryPool const&) = delete;
virtual ~MemoryPool();
Status Allocate(int64_t size, uint8_t** out) override;
Status Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) override;
void Free(uint8_t* buffer, int64_t size) override;
int64_t bytes_allocated() const override;
private:
MemoryPool();
std::atomic<int64_t> allocated_size_;
};
} // namespace jemalloc
} // namespace arrow
#endif // ARROW_JEMALLOC_MEMORY_POOL_H
cpp/src/arrow/jemalloc/symbols.map
-30
View File
@@ -1,30 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License. See accompanying LICENSE file.
{
# Symbols marked as 'local' are not exported by the DSO and thus may not
# be used by client applications.
local:
# devtoolset / static-libstdc++ symbols
__cxa_*;
extern "C++" {
# boost
boost::*;
# devtoolset or -static-libstdc++ - the Red Hat devtoolset statically
# links c++11 symbols into binaries so that the result may be executed on
# a system with an older libstdc++ which doesn't include the necessary
# c++11 symbols.
std::*;
};
};
cpp/src/arrow/memory_pool.cc
+28 -2
View File
@@ -27,14 +27,22 @@
#include "arrow/status.h"
#include "arrow/util/logging.h"
#ifdef ARROW_JEMALLOC
// Needed to support jemalloc 3 and 4
#define JEMALLOC_MANGLE
#include <jemalloc/jemalloc.h>
#endif
namespace arrow {
constexpr size_t kAlignment = 64;
namespace {
// Allocate memory according to the alignment requirements for Arrow
// (as of May 2016 64 bytes)
Status AllocateAligned(int64_t size, uint8_t** out) {
// TODO(emkornfield) find something compatible with windows
constexpr size_t kAlignment = 64;
#ifdef _MSC_VER
// Special code path for MSVC
*out =
@@ -44,6 +52,13 @@ Status AllocateAligned(int64_t size, uint8_t** out) {
ss << "malloc of size " << size << " failed";
return Status::OutOfMemory(ss.str());
}
#elif defined(ARROW_JEMALLOC)
*out = reinterpret_cast<uint8_t*>(mallocx(size, MALLOCX_ALIGN(kAlignment)));
if (*out == NULL) {
std::stringstream ss;
ss << "malloc of size " << size << " failed";
return Status::OutOfMemory(ss.str());
}
#else
const int result = posix_memalign(
reinterpret_cast<void**>(out), kAlignment, static_cast<size_t>(size));
@@ -87,6 +102,14 @@ Status DefaultMemoryPool::Allocate(int64_t size, uint8_t** out) {
}
Status DefaultMemoryPool::Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) {
#ifdef ARROW_JEMALLOC
*ptr = reinterpret_cast<uint8_t*>(rallocx(*ptr, new_size, MALLOCX_ALIGN(kAlignment)));
if (*ptr == NULL) {
std::stringstream ss;
ss << "realloc of size " << new_size << " failed";
return Status::OutOfMemory(ss.str());
}
#else
// Note: We cannot use realloc() here as it doesn't guarantee alignment.
// Allocate new chunk
@@ -98,8 +121,9 @@ Status DefaultMemoryPool::Reallocate(int64_t old_size, int64_t new_size, uint8_t
_aligned_free(*ptr);
#else
std::free(*ptr);
#endif
#endif // defined(_MSC_VER)
*ptr = out;
#endif // defined(ARROW_JEMALLOC)
bytes_allocated_ += new_size - old_size;
{
@@ -118,6 +142,8 @@ void DefaultMemoryPool::Free(uint8_t* buffer, int64_t size) {
DCHECK_GE(bytes_allocated_, size);
#ifdef _MSC_VER
_aligned_free(buffer);
#elif defined(ARROW_JEMALLOC)
dallocx(buffer, MALLOCX_ALIGN(kAlignment));
#else
std::free(buffer);
#endif
python/CMakeLists.txt
-17
View File
@@ -51,9 +51,6 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
option(PYARROW_BUILD_PARQUET
"Build the PyArrow Parquet integration"
OFF)
option(PYARROW_BUILD_JEMALLOC
"Build the PyArrow jemalloc integration"
OFF)
option(PYARROW_BUNDLE_ARROW_CPP
"Bundle the Arrow C++ libraries"
OFF)
@@ -336,20 +333,6 @@ if (PYARROW_BUILD_PARQUET)
_parquet)
endif()
if (PYARROW_BUILD_JEMALLOC)
if (PYARROW_BUNDLE_ARROW_CPP)
bundle_arrow_lib(ARROW_JEMALLOC_SHARED_LIB)
endif()
ADD_THIRDPARTY_LIB(arrow_jemalloc
SHARED_LIB ${ARROW_JEMALLOC_SHARED_LIB})
set(LINK_LIBS
${LINK_LIBS}
arrow_jemalloc_shared)
set(CYTHON_EXTENSIONS
${CYTHON_EXTENSIONS}
_jemalloc)
endif()
############################################################
# Setup and build Cython modules
############################################################
python/cmake_modules/FindArrow.cmake
-10
View File
@@ -61,11 +61,6 @@ find_library(ARROW_LIB_PATH NAMES arrow
NO_DEFAULT_PATH)
get_filename_component(ARROW_LIBS ${ARROW_LIB_PATH} DIRECTORY)
find_library(ARROW_JEMALLOC_LIB_PATH NAMES arrow_jemalloc
PATHS
${ARROW_SEARCH_LIB_PATH}
NO_DEFAULT_PATH)
find_library(ARROW_PYTHON_LIB_PATH NAMES arrow_python
PATHS
${ARROW_SEARCH_LIB_PATH}
@@ -104,10 +99,8 @@ if (ARROW_INCLUDE_DIR AND ARROW_LIBS)
else()
set(ARROW_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/lib${ARROW_LIB_NAME}.a)
set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/lib${ARROW_PYTHON_LIB_NAME}.a)
set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow_jemalloc.a)
set(ARROW_SHARED_LIB ${ARROW_LIBS}/lib${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_LIBS}/libarrow_jemalloc${CMAKE_SHARED_LIBRARY_SUFFIX})
set(ARROW_PYTHON_SHARED_LIB ${ARROW_LIBS}/lib${ARROW_PYTHON_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
endif()
endif()
@@ -116,7 +109,6 @@ if (ARROW_FOUND)
if (NOT Arrow_FIND_QUIETLY)
message(STATUS "Found the Arrow core library: ${ARROW_LIB_PATH}")
message(STATUS "Found the Arrow Python library: ${ARROW_PYTHON_LIB_PATH}")
message(STATUS "Found the Arrow jemalloc library: ${ARROW_JEMALLOC_LIB_PATH}")
endif ()
else ()
if (NOT Arrow_FIND_QUIETLY)
@@ -138,6 +130,4 @@ mark_as_advanced(
ARROW_SHARED_LIB
ARROW_PYTHON_STATIC_LIB
ARROW_PYTHON_SHARED_LIB
ARROW_JEMALLOC_STATIC_LIB
ARROW_JEMALLOC_SHARED_LIB
)
python/doc/source/api.rst
-1
View File
@@ -194,7 +194,6 @@ Memory Pools
MemoryPool
default_memory_pool
jemalloc_memory_pool
total_allocated_bytes
set_memory_pool
python/doc/source/development.rst
+2 -3
View File
@@ -141,7 +141,7 @@ Now, build pyarrow:
cd arrow/python
python setup.py build_ext --build-type=$ARROW_BUILD_TYPE \
--with-parquet --with-jemalloc --inplace
--with-parquet --inplace
If you did not build parquet-cpp, you can omit ``--with-parquet``.
@@ -162,7 +162,6 @@ You should be able to run the unit tests with:
pyarrow/tests/test_hdfs.py sssssssssssssss
pyarrow/tests/test_io.py ..................
pyarrow/tests/test_ipc.py ........
pyarrow/tests/test_jemalloc.py ss
pyarrow/tests/test_parquet.py ....................
pyarrow/tests/test_scalars.py ..........
pyarrow/tests/test_schema.py .........
@@ -175,7 +174,7 @@ You can build a wheel by running:
.. code-block:: shell
python setup.py build_ext --build-type=$ARROW_BUILD_TYPE \
--with-parquet --with-jemalloc --bundle-arrow-cpp bdist_wheel
--with-parquet --bundle-arrow-cpp bdist_wheel
Again, if you did not build parquet-cpp, you should omit ``--with-parquet``.
python/doc/source/index.rst
-6
View File
@@ -44,9 +44,3 @@ structures.
parquet
api
getting_involved
.. toctree::
:maxdepth: 2
:caption: Additional Features
jemalloc MemoryPool <jemalloc.rst>
python/doc/source/jemalloc.rst
-47
View File
@@ -1,47 +0,0 @@
.. Licensed to the Apache Software Foundation (ASF) under one
.. or more contributor license agreements. See the NOTICE file
.. distributed with this work for additional information
.. regarding copyright ownership. The ASF licenses this file
.. to you under the Apache License, Version 2.0 (the
.. "License"); you may not use this file except in compliance
.. with the License. You may obtain a copy of the License at
.. http://www.apache.org/licenses/LICENSE-2.0
.. Unless required by applicable law or agreed to in writing,
.. software distributed under the License is distributed on an
.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
.. KIND, either express or implied. See the License for the
.. specific language governing permissions and limitations
.. under the License.
jemalloc MemoryPool
===================
Arrow's default :class:`~pyarrow.MemoryPool` uses the system's allocator
through the POSIX APIs. Although this already provides aligned allocation, the
POSIX interface doesn't support aligned reallocation. The default reallocation
strategy is to allocate a new region, copy over the old data and free the
previous region. Using `jemalloc <http://jemalloc.net/>`_ we can simply extend
the existing memory allocation to the requested size. While this may still be
linear in the size of allocated memory, it is magnitudes faster as only the page
mapping in the kernel is touched, not the actual data.
The jemalloc-based allocator is not enabled by default to allow the use of the
system allocator and/or other allocators like ``tcmalloc``. You can either
explicitly make it the default allocator or pass it only to single operations.
.. code:: python
import pyarrow as pa
jemalloc_pool = pyarrow.jemalloc_memory_pool()
# Explicitly use jemalloc for allocating memory for an Arrow Table object
array = pa.Array.from_pylist([1, 2, 3], memory_pool=jemalloc_pool)
# Set the global pool
pyarrow.set_memory_pool(jemalloc_pool)
# This operation has no explicit MemoryPool specified and will thus will
# also use jemalloc for its allocations.
array = pa.Array.from_pylist([1, 2, 3])
python/manylinux1/build_arrow.sh
+1 -3
View File
@@ -35,7 +35,6 @@ cd /arrow/python
# PyArrow build configuration
export PYARROW_BUILD_TYPE='release'
export PYARROW_WITH_PARQUET=1
export PYARROW_WITH_JEMALLOC=1
export PYARROW_BUNDLE_ARROW_CPP=1
# Need as otherwise arrow_io is sometimes not linked
export LDFLAGS="-Wl,--no-as-needed"
@@ -60,13 +59,12 @@ for PYTHON in ${PYTHON_VERSIONS}; do
# Clear output directory
rm -rf dist/
echo "=== (${PYTHON}) Building wheel ==="
PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py build_ext --inplace --with-parquet --with-jemalloc --bundle-arrow-cpp
PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py build_ext --inplace --with-parquet --bundle-arrow-cpp
PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py bdist_wheel
echo "=== (${PYTHON}) Test the existence of optional modules ==="
$PIPI_IO -r requirements.txt
PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER -c "import pyarrow.parquet"
PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER -c "import pyarrow._jemalloc"
echo "=== (${PYTHON}) Tag the wheel with manylinux1 ==="
mkdir -p repaired_wheels/
python/pyarrow/__init__.py
-9
View File
@@ -90,15 +90,6 @@ from pyarrow.lib import (ArrowException,
ArrowTypeError)
def jemalloc_memory_pool():
"""
Returns a jemalloc-based memory allocator, which can be passed to
pyarrow.set_memory_pool
"""
from pyarrow._jemalloc import default_pool
return default_pool()
from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem
from pyarrow.ipc import (RecordBatchFileReader, RecordBatchFileWriter,
python/pyarrow/_jemalloc.pyx
-28
View File
@@ -1,28 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
from pyarrow.includes.libarrow_jemalloc cimport CJemallocMemoryPool
from pyarrow.lib cimport MemoryPool
def default_pool():
cdef MemoryPool pool = MemoryPool()
pool.init(CJemallocMemoryPool.default_pool())
return pool
python/pyarrow/includes/libarrow_jemalloc.pxd
-27
View File
@@ -1,27 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# distutils: language = c++
from pyarrow.includes.common cimport *
from pyarrow.includes.libarrow cimport *
cdef extern from "arrow/jemalloc/memory_pool.h" namespace "arrow::jemalloc" nogil:
cdef cppclass CJemallocMemoryPool" arrow::jemalloc::MemoryPool":
int64_t bytes_allocated()
@staticmethod
CMemoryPool* default_pool()
python/pyarrow/tests/test_jemalloc.py
-67
View File
@@ -1,67 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import gc
import pytest
import pyarrow as pa
try:
pa.jemalloc_memory_pool()
HAVE_JEMALLOC = True
except ImportError:
HAVE_JEMALLOC = False
jemalloc = pytest.mark.skipif(not HAVE_JEMALLOC,
reason='jemalloc support not built')
@jemalloc
def test_different_memory_pool():
gc.collect()
bytes_before_default = pa.total_allocated_bytes()
bytes_before_jemalloc = pa.jemalloc_memory_pool().bytes_allocated()
# it works
array = pa.array([1, None, 3, None], # noqa
memory_pool=pa.jemalloc_memory_pool())
gc.collect()
assert pa.total_allocated_bytes() == bytes_before_default
assert (pa.jemalloc_memory_pool().bytes_allocated() >
bytes_before_jemalloc)
@jemalloc
def test_default_memory_pool():
gc.collect()
bytes_before_default = pa.total_allocated_bytes()
bytes_before_jemalloc = pa.jemalloc_memory_pool().bytes_allocated()
old_memory_pool = pa.default_memory_pool()
pa.set_memory_pool(pa.jemalloc_memory_pool())
array = pa.array([1, None, 3, None]) # noqa
pa.set_memory_pool(old_memory_pool)
gc.collect()
assert pa.total_allocated_bytes() == bytes_before_default
assert (pa.jemalloc_memory_pool().bytes_allocated() >
bytes_before_jemalloc)
python/setup.py
-11
View File
@@ -82,7 +82,6 @@ class build_ext(_build_ext):
user_options = ([('extra-cmake-args=', None, 'extra arguments for CMake'),
('build-type=', None, 'build type (debug or release)'),
('with-parquet', None, 'build the Parquet extension'),
('with-jemalloc', None, 'build the jemalloc extension'),
('bundle-arrow-cpp', None,
'bundle the Arrow C++ libraries')] +
_build_ext.user_options)
@@ -100,14 +99,11 @@ class build_ext(_build_ext):
self.with_parquet = strtobool(
os.environ.get('PYARROW_WITH_PARQUET', '0'))
self.with_jemalloc = strtobool(
os.environ.get('PYARROW_WITH_JEMALLOC', '0'))
self.bundle_arrow_cpp = strtobool(
os.environ.get('PYARROW_BUNDLE_ARROW_CPP', '0'))
CYTHON_MODULE_NAMES = [
'lib',
'_jemalloc',
'_parquet']
def _run_cmake(self):
@@ -143,9 +139,6 @@ class build_ext(_build_ext):
if self.with_parquet:
cmake_options.append('-DPYARROW_BUILD_PARQUET=on')
if self.with_jemalloc:
cmake_options.append('-DPYARROW_BUILD_JEMALLOC=on')
if self.bundle_arrow_cpp:
cmake_options.append('-DPYARROW_BUNDLE_ARROW_CPP=ON')
# ARROW-1090: work around CMake rough edges
@@ -249,8 +242,6 @@ class build_ext(_build_ext):
shutil.move(pjoin(build_prefix, 'include'), pjoin(build_lib, 'pyarrow'))
move_lib("arrow")
move_lib("arrow_python")
if self.with_jemalloc:
move_lib("arrow_jemalloc")
if self.with_parquet:
move_lib("parquet")
@@ -284,8 +275,6 @@ class build_ext(_build_ext):
def _failure_permitted(self, name):
if name == '_parquet' and not self.with_parquet:
return True
if name == '_jemalloc' and not self.with_jemalloc:
return True
return False
def _get_inplace_dir(self):