mirror of
https://github.com/apache/arrow.git
synced 2026-10-28 16:47:19 +08:00
ARROW-3209: [C++] Rename libarrow_gpu to libarrow_cuda
Also rename the arrow::gpu namespace to arrow::cuda.
This commit is contained in:
1 parent
ffc5154274
commit
9687346aca
25 files changed
+113
-120
No files matched your search
+2
-2
@@ -150,8 +150,8 @@ Pass multiple labels by dividing with semicolons")
|
||||
"Build the Arrow IPC extensions"
|
||||
ON)
|
||||
|
||||
option(ARROW_GPU
|
||||
"Build the Arrow GPU extensions (requires CUDA installation)"
|
||||
option(ARROW_CUDA
|
||||
"Build the Arrow CUDA extensions (requires CUDA toolkit)"
|
||||
OFF)
|
||||
|
||||
option(ARROW_ORC
|
||||
|
||||
+4
-6
@@ -204,13 +204,11 @@ The Python library must be built against the same Python version for which you
|
||||
are building pyarrow, e.g. Python 2.7 or Python 3.6. NumPy must also be
|
||||
installed.
|
||||
|
||||
### Building GPU extension library (optional)
|
||||
### Building CUDA extension library (optional)
|
||||
|
||||
The optional `arrow_gpu` shared library can be built by passing
|
||||
`-DARROW_GPU=on`. This requires a CUDA installation to build, and to use many
|
||||
of the functions you must have a functioning GPU. Currently only CUDA
|
||||
functionality is supported, though if there is demand we can also add OpenCL
|
||||
interfaces in this library as needed.
|
||||
The optional `arrow_cuda` shared library can be built by passing
|
||||
`-DARROW_CUDA=on`. This requires a CUDA installation to build, and to use many
|
||||
of the functions you must have a functioning CUDA-compatible GPU.
|
||||
|
||||
The CUDA toolchain used to build the library can be customized by using the
|
||||
`$CUDA_HOME` environment variable.
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# - Find ARROW CUDA (arrow/gpu/cuda_api.h, libarrow_gpu.a, libarrow_gpu.so)
|
||||
# - Find ARROW CUDA (arrow/gpu/cuda_api.h, libarrow_cuda.a, libarrow_cuda.so)
|
||||
#
|
||||
# This module requires Arrow from which it uses
|
||||
# ARROW_FOUND
|
||||
@@ -31,10 +31,6 @@
|
||||
# ARROW_CUDA_SHARED_IMP_LIB, path to libarrow's import library (MSVC only)
|
||||
# ARROW_CUDA_FOUND, whether arrow has been found
|
||||
|
||||
#
|
||||
# TODO(ARROW-3209): rename arrow/gpu to arrow/cuda, arrow_gpu to arrow_cuda
|
||||
#
|
||||
|
||||
include(FindPkgConfig)
|
||||
include(GNUInstallDirs)
|
||||
|
||||
@@ -63,14 +59,14 @@ if (NOT (ARROW_CUDA_INCLUDE_DIR STREQUAL ARROW_INCLUDE_DIR))
|
||||
message(WARNING ${ARROW_CUDA_WARN_MSG})
|
||||
endif()
|
||||
|
||||
find_library(ARROW_CUDA_LIB_PATH NAMES arrow_gpu
|
||||
find_library(ARROW_CUDA_LIB_PATH NAMES arrow_cuda
|
||||
PATHS
|
||||
${ARROW_SEARCH_LIB_PATH}
|
||||
NO_DEFAULT_PATH)
|
||||
get_filename_component(ARROW_CUDA_LIBS ${ARROW_CUDA_LIB_PATH} DIRECTORY)
|
||||
|
||||
if (MSVC)
|
||||
find_library(ARROW_CUDA_SHARED_LIBRARIES NAMES arrow_gpu
|
||||
find_library(ARROW_CUDA_SHARED_LIBRARIES NAMES arrow_cuda
|
||||
PATHS ${ARROW_HOME} NO_DEFAULT_PATH
|
||||
PATH_SUFFIXES "bin" )
|
||||
get_filename_component(ARROW_CUDA_SHARED_LIBS ${ARROW_CUDA_SHARED_LIBRARIES} PATH )
|
||||
@@ -79,7 +75,7 @@ endif()
|
||||
|
||||
if (ARROW_CUDA_INCLUDE_DIR AND ARROW_CUDA_LIBS)
|
||||
set(ARROW_CUDA_FOUND TRUE)
|
||||
set(ARROW_CUDA_LIB_NAME arrow_gpu)
|
||||
set(ARROW_CUDA_LIB_NAME arrow_cuda)
|
||||
if (MSVC)
|
||||
set(ARROW_CUDA_STATIC_LIB ${ARROW_CUDA_LIBS}/${ARROW_CUDA_LIB_NAME}${ARROW_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
|
||||
set(ARROW_CUDA_SHARED_LIB ${ARROW_CUDA_SHARED_LIBS}/${ARROW_CUDA_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
|
||||
|
||||
@@ -78,8 +78,8 @@ if (ARROW_COMPUTE)
|
||||
)
|
||||
endif()
|
||||
|
||||
if (ARROW_GPU)
|
||||
# IPC extensions required to build the GPU library
|
||||
if (ARROW_CUDA)
|
||||
# IPC extensions required to build the CUDA library
|
||||
set(ARROW_IPC ON)
|
||||
add_subdirectory(gpu)
|
||||
endif()
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
# under the License.
|
||||
|
||||
#######################################
|
||||
# arrow_gpu
|
||||
# arrow_cuda
|
||||
#######################################
|
||||
|
||||
if (DEFINED ENV{CUDA_HOME})
|
||||
@@ -28,28 +28,28 @@ include_directories(SYSTEM ${CUDA_INCLUDE_DIRS})
|
||||
|
||||
message(STATUS "CUDA Libraries: ${CUDA_LIBRARIES}")
|
||||
|
||||
set(ARROW_GPU_SRCS
|
||||
set(ARROW_CUDA_SRCS
|
||||
cuda_arrow_ipc.cc
|
||||
cuda_context.cc
|
||||
cuda_memory.cc
|
||||
)
|
||||
|
||||
set(ARROW_GPU_SHARED_LINK_LIBS
|
||||
set(ARROW_CUDA_SHARED_LINK_LIBS
|
||||
${CUDA_LIBRARIES}
|
||||
${CUDA_CUDA_LIBRARY}
|
||||
)
|
||||
|
||||
ADD_ARROW_LIB(arrow_gpu
|
||||
SOURCES ${ARROW_GPU_SRCS}
|
||||
OUTPUTS ARROW_GPU_LIBRARIES
|
||||
ADD_ARROW_LIB(arrow_cuda
|
||||
SOURCES ${ARROW_CUDA_SRCS}
|
||||
OUTPUTS ARROW_CUDA_LIBRARIES
|
||||
DEPENDENCIES metadata_fbs
|
||||
SHARED_LINK_FLAGS ""
|
||||
SHARED_LINK_LIBS arrow_shared ${ARROW_GPU_SHARED_LINK_LIBS}
|
||||
# Static arrow_gpu must also link against CUDA shared libs
|
||||
STATIC_LINK_LIBS ${ARROW_GPU_SHARED_LINK_LIBS}
|
||||
SHARED_LINK_LIBS arrow_shared ${ARROW_CUDA_SHARED_LINK_LIBS}
|
||||
# Static arrow_cuda must also link against CUDA shared libs
|
||||
STATIC_LINK_LIBS ${ARROW_CUDA_SHARED_LINK_LIBS}
|
||||
)
|
||||
|
||||
foreach(LIB_TARGET ${ARROW_GPU_LIBRARIES})
|
||||
foreach(LIB_TARGET ${ARROW_CUDA_LIBRARIES})
|
||||
target_compile_definitions(${LIB_TARGET}
|
||||
PRIVATE ARROW_EXPORTING)
|
||||
endforeach()
|
||||
@@ -71,28 +71,28 @@ install(FILES
|
||||
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/gpu")
|
||||
|
||||
# pkg-config support
|
||||
configure_file(arrow-gpu.pc.in
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/arrow-gpu.pc"
|
||||
configure_file(arrow-cuda.pc.in
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/arrow-cuda.pc"
|
||||
@ONLY)
|
||||
|
||||
install(
|
||||
FILES "${CMAKE_CURRENT_BINARY_DIR}/arrow-gpu.pc"
|
||||
FILES "${CMAKE_CURRENT_BINARY_DIR}/arrow-cuda.pc"
|
||||
DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/")
|
||||
|
||||
set(ARROW_GPU_TEST_LINK_LIBS
|
||||
arrow_gpu_shared
|
||||
set(ARROW_CUDA_TEST_LINK_LIBS
|
||||
arrow_cuda_shared
|
||||
${ARROW_TEST_LINK_LIBS})
|
||||
|
||||
if (ARROW_BUILD_TESTS)
|
||||
ADD_ARROW_TEST(cuda-test
|
||||
STATIC_LINK_LIBS ${ARROW_GPU_TEST_LINK_LIBS}
|
||||
STATIC_LINK_LIBS ${ARROW_CUDA_TEST_LINK_LIBS}
|
||||
NO_VALGRIND)
|
||||
endif()
|
||||
|
||||
if (ARROW_BUILD_BENCHMARKS)
|
||||
cuda_add_executable(cuda-benchmark cuda-benchmark.cc)
|
||||
target_link_libraries(cuda-benchmark
|
||||
arrow_gpu_shared
|
||||
arrow_cuda_shared
|
||||
gtest_static
|
||||
${ARROW_BENCHMARK_LINK_LIBS})
|
||||
endif()
|
||||
@@ -18,9 +18,9 @@
|
||||
libdir=@CMAKE_INSTALL_FULL_LIBDIR@
|
||||
includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@
|
||||
|
||||
Name: Apache Arrow GPU
|
||||
Description: GPU integration library for Apache Arrow
|
||||
Name: Apache Arrow CUDA
|
||||
Description: CUDA integration library for Apache Arrow
|
||||
Version: @ARROW_VERSION@
|
||||
Requires: arrow
|
||||
Libs: -L${libdir} -larrow_gpu
|
||||
Libs: -L${libdir} -larrow_cuda
|
||||
Cflags: -I${includedir}
|
||||
@@ -28,7 +28,7 @@
|
||||
#include "arrow/gpu/cuda_api.h"
|
||||
|
||||
namespace arrow {
|
||||
namespace gpu {
|
||||
namespace cuda {
|
||||
|
||||
constexpr int64_t kGpuNumber = 0;
|
||||
|
||||
@@ -94,5 +94,5 @@ BENCHMARK(BM_Writer_Unbuffered)
|
||||
->MinTime(1.0)
|
||||
->UseRealTime();
|
||||
|
||||
} // namespace gpu
|
||||
} // namespace cuda
|
||||
} // namespace arrow
|
||||
@@ -29,7 +29,7 @@
|
||||
#include "arrow/gpu/cuda_api.h"
|
||||
|
||||
namespace arrow {
|
||||
namespace gpu {
|
||||
namespace cuda {
|
||||
|
||||
constexpr int kGpuNumber = 0;
|
||||
|
||||
@@ -323,7 +323,7 @@ TEST_F(TestCudaArrowIpc, BasicWriteRead) {
|
||||
ASSERT_OK(ipc::MakeIntRecordBatch(&batch));
|
||||
|
||||
std::shared_ptr<CudaBuffer> device_serialized;
|
||||
ASSERT_OK(arrow::gpu::SerializeRecordBatch(*batch, context_.get(), &device_serialized));
|
||||
ASSERT_OK(SerializeRecordBatch(*batch, context_.get(), &device_serialized));
|
||||
|
||||
// Test that ReadRecordBatch works properly
|
||||
std::shared_ptr<RecordBatch> device_batch;
|
||||
@@ -343,5 +343,5 @@ TEST_F(TestCudaArrowIpc, BasicWriteRead) {
|
||||
CompareBatch(*batch, *cpu_batch);
|
||||
}
|
||||
|
||||
} // namespace gpu
|
||||
} // namespace cuda
|
||||
} // namespace arrow
|
||||
@@ -38,7 +38,7 @@ namespace arrow {
|
||||
|
||||
namespace flatbuf = org::apache::arrow::flatbuf;
|
||||
|
||||
namespace gpu {
|
||||
namespace cuda {
|
||||
|
||||
Status SerializeRecordBatch(const RecordBatch& batch, CudaContext* ctx,
|
||||
std::shared_ptr<CudaBuffer>* out) {
|
||||
@@ -106,5 +106,5 @@ Status ReadRecordBatch(const std::shared_ptr<Schema>& schema,
|
||||
return ipc::ReadRecordBatch(*message, schema, out);
|
||||
}
|
||||
|
||||
} // namespace gpu
|
||||
} // namespace cuda
|
||||
} // namespace arrow
|
||||
@@ -39,7 +39,7 @@ class Message;
|
||||
|
||||
} // namespace ipc
|
||||
|
||||
namespace gpu {
|
||||
namespace cuda {
|
||||
|
||||
/// \brief Write record batch message to GPU device memory
|
||||
/// \param[in] batch record batch to write
|
||||
@@ -71,7 +71,7 @@ Status ReadRecordBatch(const std::shared_ptr<Schema>& schema,
|
||||
const std::shared_ptr<CudaBuffer>& buffer, MemoryPool* pool,
|
||||
std::shared_ptr<RecordBatch>* out);
|
||||
|
||||
} // namespace gpu
|
||||
} // namespace cuda
|
||||
} // namespace arrow
|
||||
|
||||
#endif // ARROW_GPU_CUDA_ARROW_IPC_H
|
||||
@@ -25,7 +25,7 @@
|
||||
#include <cuda.h>
|
||||
|
||||
namespace arrow {
|
||||
namespace gpu {
|
||||
namespace cuda {
|
||||
|
||||
#define CUDA_DCHECK(STMT) \
|
||||
do { \
|
||||
@@ -45,7 +45,7 @@ namespace gpu {
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
} // namespace gpu
|
||||
} // namespace cuda
|
||||
} // namespace arrow
|
||||
|
||||
#endif // ARROW_GPU_CUDA_COMMON_H
|
||||
@@ -28,8 +28,9 @@
|
||||
|
||||
#include "arrow/gpu/cuda_common.h"
|
||||
#include "arrow/gpu/cuda_memory.h"
|
||||
|
||||
namespace arrow {
|
||||
namespace gpu {
|
||||
namespace cuda {
|
||||
|
||||
struct CudaDevice {
|
||||
int device_num;
|
||||
@@ -342,5 +343,5 @@ void* CudaContext::handle() const { return impl_->context_handle(); }
|
||||
|
||||
int CudaContext::device_number() const { return impl_->device().device_num; }
|
||||
|
||||
} // namespace gpu
|
||||
} // namespace cuda
|
||||
} // namespace arrow
|
||||
@@ -27,7 +27,7 @@
|
||||
#include "arrow/gpu/cuda_memory.h"
|
||||
|
||||
namespace arrow {
|
||||
namespace gpu {
|
||||
namespace cuda {
|
||||
|
||||
// Forward declaration
|
||||
class CudaContext;
|
||||
@@ -138,7 +138,7 @@ class ARROW_EXPORT CudaContext : public std::enable_shared_from_this<CudaContext
|
||||
friend CudaDeviceManager::CudaDeviceManagerImpl;
|
||||
};
|
||||
|
||||
} // namespace gpu
|
||||
} // namespace cuda
|
||||
} // namespace arrow
|
||||
|
||||
#endif // ARROW_GPU_CUDA_CONTEXT_H
|
||||
@@ -34,7 +34,7 @@
|
||||
#include "arrow/gpu/cuda_context.h"
|
||||
|
||||
namespace arrow {
|
||||
namespace gpu {
|
||||
namespace cuda {
|
||||
|
||||
// ----------------------------------------------------------------------
|
||||
// CUDA IPC memory handle
|
||||
@@ -365,5 +365,5 @@ Status AllocateCudaHostBuffer(int device_number, const int64_t size,
|
||||
return manager->AllocateHost(device_number, size, out);
|
||||
}
|
||||
|
||||
} // namespace gpu
|
||||
} // namespace cuda
|
||||
} // namespace arrow
|
||||
@@ -27,7 +27,7 @@
|
||||
#include "arrow/status.h"
|
||||
|
||||
namespace arrow {
|
||||
namespace gpu {
|
||||
namespace cuda {
|
||||
|
||||
class CudaContext;
|
||||
class CudaIpcMemHandle;
|
||||
@@ -215,7 +215,7 @@ ARROW_EXPORT
|
||||
Status AllocateCudaHostBuffer(int device_number, const int64_t size,
|
||||
std::shared_ptr<CudaHostBuffer>* out);
|
||||
|
||||
} // namespace gpu
|
||||
} // namespace cuda
|
||||
} // namespace arrow
|
||||
|
||||
#endif // ARROW_GPU_CUDA_MEMORY_H
|
||||
@@ -83,10 +83,10 @@ set(PLASMA_SRCS
|
||||
set(PLASMA_LINK_LIBS arrow_shared)
|
||||
set(PLASMA_STATIC_LINK_LIBS arrow_static)
|
||||
|
||||
if (ARROW_GPU)
|
||||
set(PLASMA_LINK_LIBS ${PLASMA_LINK_LIBS} arrow_gpu_shared)
|
||||
set(PLASMA_STATIC_LINK_LIBS arrow_gpu_static ${PLASMA_STATIC_LINK_LIBS})
|
||||
add_definitions(-DPLASMA_GPU)
|
||||
if (ARROW_CUDA)
|
||||
set(PLASMA_LINK_LIBS ${PLASMA_LINK_LIBS} arrow_cuda_shared)
|
||||
set(PLASMA_STATIC_LINK_LIBS arrow_cuda_static ${PLASMA_STATIC_LINK_LIBS})
|
||||
add_definitions(-DPLASMA_CUDA)
|
||||
endif()
|
||||
|
||||
ADD_ARROW_LIB(plasma
|
||||
|
||||
+12
-12
@@ -53,13 +53,13 @@
|
||||
#include "plasma/plasma.h"
|
||||
#include "plasma/protocol.h"
|
||||
|
||||
#ifdef PLASMA_GPU
|
||||
#ifdef PLASMA_CUDA
|
||||
#include "arrow/gpu/cuda_api.h"
|
||||
|
||||
using arrow::gpu::CudaBuffer;
|
||||
using arrow::gpu::CudaBufferWriter;
|
||||
using arrow::gpu::CudaContext;
|
||||
using arrow::gpu::CudaDeviceManager;
|
||||
using arrow::cuda::CudaBuffer;
|
||||
using arrow::cuda::CudaBufferWriter;
|
||||
using arrow::cuda::CudaContext;
|
||||
using arrow::cuda::CudaDeviceManager;
|
||||
#endif
|
||||
|
||||
#define XXH_INLINE_ALL 1
|
||||
@@ -89,7 +89,7 @@ constexpr int64_t kL3CacheSizeBytes = 100000000;
|
||||
// ----------------------------------------------------------------------
|
||||
// GPU support
|
||||
|
||||
#ifdef PLASMA_GPU
|
||||
#ifdef PLASMA_CUDA
|
||||
struct GpuProcessHandle {
|
||||
/// Pointer to CUDA buffer that is backing this GPU object.
|
||||
std::shared_ptr<CudaBuffer> ptr;
|
||||
@@ -286,16 +286,16 @@ class PlasmaClient::Impl : public std::enable_shared_from_this<PlasmaClient::Imp
|
||||
/// A hash set to record the ids that users want to delete but still in use.
|
||||
std::unordered_set<ObjectID> deletion_cache_;
|
||||
|
||||
#ifdef PLASMA_GPU
|
||||
#ifdef PLASMA_CUDA
|
||||
/// Cuda Device Manager.
|
||||
arrow::gpu::CudaDeviceManager* manager_;
|
||||
arrow::cuda::CudaDeviceManager* manager_;
|
||||
#endif
|
||||
};
|
||||
|
||||
PlasmaBuffer::~PlasmaBuffer() { ARROW_UNUSED(client_->Release(object_id_)); }
|
||||
|
||||
PlasmaClient::Impl::Impl() {
|
||||
#ifdef PLASMA_GPU
|
||||
#ifdef PLASMA_CUDA
|
||||
DCHECK_OK(CudaDeviceManager::GetInstance(&manager_));
|
||||
#endif
|
||||
}
|
||||
@@ -413,7 +413,7 @@ Status PlasmaClient::Impl::Create(const ObjectID& object_id, int64_t data_size,
|
||||
memcpy((*data)->mutable_data() + object.data_size, metadata, metadata_size);
|
||||
}
|
||||
} else {
|
||||
#ifdef PLASMA_GPU
|
||||
#ifdef PLASMA_CUDA
|
||||
std::lock_guard<std::mutex> lock(gpu_mutex);
|
||||
std::shared_ptr<CudaContext> context;
|
||||
RETURN_NOT_OK(manager_->GetContext(device_num - 1, &context));
|
||||
@@ -497,7 +497,7 @@ Status PlasmaClient::Impl::GetBuffers(
|
||||
physical_buf = std::make_shared<Buffer>(
|
||||
data + object->data_offset, object->data_size + object->metadata_size);
|
||||
} else {
|
||||
#ifdef PLASMA_GPU
|
||||
#ifdef PLASMA_CUDA
|
||||
physical_buf = gpu_object_map.find(object_ids[i])->second->ptr;
|
||||
#else
|
||||
ARROW_LOG(FATAL) << "Arrow GPU library is not enabled.";
|
||||
@@ -560,7 +560,7 @@ Status PlasmaClient::Impl::GetBuffers(
|
||||
physical_buf = std::make_shared<Buffer>(
|
||||
data + object->data_offset, object->data_size + object->metadata_size);
|
||||
} else {
|
||||
#ifdef PLASMA_GPU
|
||||
#ifdef PLASMA_CUDA
|
||||
std::lock_guard<std::mutex> lock(gpu_mutex);
|
||||
auto handle = gpu_object_map.find(object_ids[i]);
|
||||
if (handle == gpu_object_map.end()) {
|
||||
|
||||
@@ -34,7 +34,7 @@
|
||||
|
||||
#include "arrow/status.h"
|
||||
#include "arrow/util/logging.h"
|
||||
#ifdef PLASMA_GPU
|
||||
#ifdef PLASMA_CUDA
|
||||
#include "arrow/gpu/cuda_api.h"
|
||||
#endif
|
||||
|
||||
@@ -118,9 +118,9 @@ struct ObjectTableEntry {
|
||||
int64_t data_size;
|
||||
/// Size of the object metadata in bytes.
|
||||
int64_t metadata_size;
|
||||
#ifdef PLASMA_GPU
|
||||
#ifdef PLASMA_CUDA
|
||||
/// IPC GPU handle to share with clients.
|
||||
std::shared_ptr<::arrow::gpu::CudaIpcMemHandle> ipc_handle;
|
||||
std::shared_ptr<::arrow::cuda::CudaIpcMemHandle> ipc_handle;
|
||||
#endif
|
||||
/// Number of clients currently using this object.
|
||||
int ref_count;
|
||||
|
||||
@@ -40,8 +40,8 @@
|
||||
#include "plasma/common.h"
|
||||
#include "plasma/common_generated.h"
|
||||
|
||||
#ifdef PLASMA_GPU
|
||||
using arrow::gpu::CudaIpcMemHandle;
|
||||
#ifdef PLASMA_CUDA
|
||||
using arrow::cuda::CudaIpcMemHandle;
|
||||
#endif
|
||||
|
||||
namespace plasma {
|
||||
@@ -73,7 +73,7 @@ typedef std::unordered_map<ObjectID, ObjectRequest> ObjectRequestMap;
|
||||
|
||||
// TODO(pcm): Replace this by the flatbuffers message PlasmaObjectSpec.
|
||||
struct PlasmaObject {
|
||||
#ifdef PLASMA_GPU
|
||||
#ifdef PLASMA_CUDA
|
||||
// IPC handle for Cuda.
|
||||
std::shared_ptr<CudaIpcMemHandle> ipc_handle;
|
||||
#endif
|
||||
|
||||
@@ -25,7 +25,7 @@
|
||||
#include "plasma/common.h"
|
||||
#include "plasma/io.h"
|
||||
|
||||
#ifdef ARROW_GPU
|
||||
#ifdef PLASMA_CUDA
|
||||
#include "arrow/gpu/cuda_api.h"
|
||||
#endif
|
||||
|
||||
@@ -129,7 +129,7 @@ Status SendCreateReply(int sock, ObjectID object_id, PlasmaObject* object,
|
||||
object->metadata_offset, object->metadata_size,
|
||||
object->device_num);
|
||||
auto object_string = fbb.CreateString(object_id.binary());
|
||||
#ifdef PLASMA_GPU
|
||||
#ifdef PLASMA_CUDA
|
||||
flatbuffers::Offset<fb::CudaHandle> ipc_handle;
|
||||
if (object->device_num != 0) {
|
||||
std::shared_ptr<arrow::Buffer> handle;
|
||||
@@ -145,7 +145,7 @@ Status SendCreateReply(int sock, ObjectID object_id, PlasmaObject* object,
|
||||
crb.add_store_fd(object->store_fd);
|
||||
crb.add_mmap_size(mmap_size);
|
||||
if (object->device_num != 0) {
|
||||
#ifdef PLASMA_GPU
|
||||
#ifdef PLASMA_CUDA
|
||||
crb.add_ipc_handle(ipc_handle);
|
||||
#else
|
||||
ARROW_LOG(FATAL) << "This should be unreachable.";
|
||||
@@ -171,7 +171,7 @@ Status ReadCreateReply(uint8_t* data, size_t size, ObjectID* object_id,
|
||||
*mmap_size = message->mmap_size();
|
||||
|
||||
object->device_num = message->plasma_object()->device_num();
|
||||
#ifdef PLASMA_GPU
|
||||
#ifdef PLASMA_CUDA
|
||||
if (object->device_num != 0) {
|
||||
RETURN_NOT_OK(CudaIpcMemHandle::FromBuffer(message->ipc_handle()->handle()->data(),
|
||||
&object->ipc_handle));
|
||||
@@ -588,7 +588,7 @@ Status SendGetReply(int sock, ObjectID object_ids[],
|
||||
objects.push_back(PlasmaObjectSpec(object.store_fd, object.data_offset,
|
||||
object.data_size, object.metadata_offset,
|
||||
object.metadata_size, object.device_num));
|
||||
#ifdef PLASMA_GPU
|
||||
#ifdef PLASMA_CUDA
|
||||
if (object.device_num != 0) {
|
||||
std::shared_ptr<arrow::Buffer> handle;
|
||||
RETURN_NOT_OK(object.ipc_handle->Serialize(arrow::default_memory_pool(), &handle));
|
||||
@@ -609,7 +609,7 @@ Status ReadGetReply(uint8_t* data, size_t size, ObjectID object_ids[],
|
||||
std::vector<int>& store_fds, std::vector<int64_t>& mmap_sizes) {
|
||||
DCHECK(data);
|
||||
auto message = flatbuffers::GetRoot<fb::PlasmaGetReply>(data);
|
||||
#ifdef PLASMA_GPU
|
||||
#ifdef PLASMA_CUDA
|
||||
int handle_pos = 0;
|
||||
#endif
|
||||
DCHECK(VerifyFlatbuffer(message, data, size));
|
||||
@@ -624,7 +624,7 @@ Status ReadGetReply(uint8_t* data, size_t size, ObjectID object_ids[],
|
||||
plasma_objects[i].metadata_offset = object->metadata_offset();
|
||||
plasma_objects[i].metadata_size = object->metadata_size();
|
||||
plasma_objects[i].device_num = object->device_num();
|
||||
#ifdef PLASMA_GPU
|
||||
#ifdef PLASMA_CUDA
|
||||
if (object->device_num() != 0) {
|
||||
const void* ipc_handle = message->handles()->Get(handle_pos)->handle()->data();
|
||||
RETURN_NOT_OK(
|
||||
|
||||
@@ -58,12 +58,12 @@
|
||||
#include "plasma/io.h"
|
||||
#include "plasma/malloc.h"
|
||||
|
||||
#ifdef PLASMA_GPU
|
||||
#ifdef PLASMA_CUDA
|
||||
#include "arrow/gpu/cuda_api.h"
|
||||
|
||||
using arrow::gpu::CudaBuffer;
|
||||
using arrow::gpu::CudaContext;
|
||||
using arrow::gpu::CudaDeviceManager;
|
||||
using arrow::cuda::CudaBuffer;
|
||||
using arrow::cuda::CudaContext;
|
||||
using arrow::cuda::CudaDeviceManager;
|
||||
#endif
|
||||
|
||||
using arrow::util::ArrowLog;
|
||||
@@ -117,7 +117,7 @@ PlasmaStore::PlasmaStore(EventLoop* loop, int64_t system_memory, std::string dir
|
||||
store_info_.memory_capacity = system_memory;
|
||||
store_info_.directory = directory;
|
||||
store_info_.hugepages_enabled = hugepages_enabled;
|
||||
#ifdef PLASMA_GPU
|
||||
#ifdef PLASMA_CUDA
|
||||
DCHECK_OK(CudaDeviceManager::GetInstance(&manager_));
|
||||
#endif
|
||||
}
|
||||
@@ -162,7 +162,7 @@ PlasmaError PlasmaStore::CreateObject(const ObjectID& object_id, int64_t data_si
|
||||
}
|
||||
// Try to evict objects until there is enough space.
|
||||
uint8_t* pointer = nullptr;
|
||||
#ifdef PLASMA_GPU
|
||||
#ifdef PLASMA_CUDA
|
||||
std::shared_ptr<CudaBuffer> gpu_handle;
|
||||
std::shared_ptr<CudaContext> context_;
|
||||
if (device_num != 0) {
|
||||
@@ -195,7 +195,7 @@ PlasmaError PlasmaStore::CreateObject(const ObjectID& object_id, int64_t data_si
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
#ifdef PLASMA_GPU
|
||||
#ifdef PLASMA_CUDA
|
||||
DCHECK_OK(context_->Allocate(data_size + metadata_size, &gpu_handle));
|
||||
break;
|
||||
#endif
|
||||
@@ -220,7 +220,7 @@ PlasmaError PlasmaStore::CreateObject(const ObjectID& object_id, int64_t data_si
|
||||
entry->device_num = device_num;
|
||||
entry->create_time = std::time(nullptr);
|
||||
entry->construct_duration = -1;
|
||||
#ifdef PLASMA_GPU
|
||||
#ifdef PLASMA_CUDA
|
||||
if (device_num != 0) {
|
||||
DCHECK_OK(gpu_handle->ExportForIpc(&entry->ipc_handle));
|
||||
result->ipc_handle = entry->ipc_handle;
|
||||
@@ -246,7 +246,7 @@ void PlasmaObject_init(PlasmaObject* object, ObjectTableEntry* entry) {
|
||||
DCHECK(object != nullptr);
|
||||
DCHECK(entry != nullptr);
|
||||
DCHECK(entry->state == ObjectState::PLASMA_SEALED);
|
||||
#ifdef PLASMA_GPU
|
||||
#ifdef PLASMA_CUDA
|
||||
if (entry->device_num != 0) {
|
||||
object->ipc_handle = entry->ipc_handle;
|
||||
}
|
||||
|
||||
@@ -223,8 +223,8 @@ class PlasmaStore {
|
||||
std::unordered_map<int, std::unique_ptr<Client>> connected_clients_;
|
||||
|
||||
std::unordered_set<ObjectID> deletion_cache_;
|
||||
#ifdef PLASMA_GPU
|
||||
arrow::gpu::CudaDeviceManager* manager_;
|
||||
#ifdef PLASMA_CUDA
|
||||
arrow::cuda::CudaDeviceManager* manager_;
|
||||
#endif
|
||||
};
|
||||
|
||||
|
||||
@@ -487,10 +487,10 @@ TEST_F(TestPlasmaStore, ManyObjectTest) {
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef PLASMA_GPU
|
||||
using arrow::gpu::CudaBuffer;
|
||||
using arrow::gpu::CudaBufferReader;
|
||||
using arrow::gpu::CudaBufferWriter;
|
||||
#ifdef PLASMA_CUDA
|
||||
using arrow::cuda::CudaBuffer;
|
||||
using arrow::cuda::CudaBufferReader;
|
||||
using arrow::cuda::CudaBufferWriter;
|
||||
|
||||
namespace {
|
||||
|
||||
@@ -590,7 +590,7 @@ TEST_F(TestPlasmaStore, MultipleClientGPUTest) {
|
||||
AssertCudaRead(object_buffers[0].metadata, {5});
|
||||
}
|
||||
|
||||
#endif // PLASMA_GPU
|
||||
#endif // PLASMA_CUDA
|
||||
|
||||
} // namespace plasma
|
||||
|
||||
|
||||
@@ -17,9 +17,6 @@
|
||||
#
|
||||
# Includes code assembled from BSD/MIT/Apache-licensed code from some 3rd-party
|
||||
# projects, including Kudu, Impala, and libdynd. See python/LICENSE.txt
|
||||
#
|
||||
# TODO(ARROW-3209): rename arrow_gpu to arrow_cuda
|
||||
#
|
||||
|
||||
cmake_minimum_required(VERSION 2.7)
|
||||
project(pyarrow)
|
||||
@@ -393,13 +390,13 @@ if (PYARROW_BUILD_CUDA)
|
||||
endif()
|
||||
endif()
|
||||
if (MSVC)
|
||||
ADD_THIRDPARTY_LIB(arrow_gpu
|
||||
ADD_THIRDPARTY_LIB(arrow_cuda
|
||||
SHARED_LIB ${ARROW_CUDA_SHARED_IMP_LIB})
|
||||
else()
|
||||
ADD_THIRDPARTY_LIB(arrow_gpu
|
||||
ADD_THIRDPARTY_LIB(arrow_cuda
|
||||
SHARED_LIB ${ARROW_CUDA_SHARED_LIB})
|
||||
endif()
|
||||
set(LINK_LIBS ${LINK_LIBS} arrow_gpu_shared)
|
||||
set(LINK_LIBS ${LINK_LIBS} arrow_cuda_shared)
|
||||
set(CYTHON_EXTENSIONS ${CYTHON_EXTENSIONS} _cuda)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
@@ -19,9 +19,9 @@
|
||||
|
||||
from pyarrow.includes.libarrow cimport *
|
||||
|
||||
cdef extern from "arrow/gpu/cuda_api.h" namespace "arrow::gpu" nogil:
|
||||
cdef extern from "arrow/gpu/cuda_api.h" namespace "arrow::cuda" nogil:
|
||||
|
||||
cdef cppclass CCudaDeviceManager" arrow::gpu::CudaDeviceManager":
|
||||
cdef cppclass CCudaDeviceManager" arrow::cuda::CudaDeviceManager":
|
||||
@staticmethod
|
||||
CStatus GetInstance(CCudaDeviceManager** manager)
|
||||
CStatus GetContext(int gpu_number, shared_ptr[CCudaContext]* ctx)
|
||||
@@ -33,7 +33,7 @@ cdef extern from "arrow/gpu/cuda_api.h" namespace "arrow::gpu" nogil:
|
||||
# CStatus FreeHost(void* data, int64_t nbytes)
|
||||
int num_devices() const
|
||||
|
||||
cdef cppclass CCudaContext" arrow::gpu::CudaContext":
|
||||
cdef cppclass CCudaContext" arrow::cuda::CudaContext":
|
||||
shared_ptr[CCudaContext] shared_from_this()
|
||||
# CStatus Close()
|
||||
CStatus Allocate(int64_t nbytes, shared_ptr[CCudaBuffer]* out)
|
||||
@@ -47,13 +47,13 @@ cdef extern from "arrow/gpu/cuda_api.h" namespace "arrow::gpu" nogil:
|
||||
const void* handle() const
|
||||
int device_number() const
|
||||
|
||||
cdef cppclass CCudaIpcMemHandle" arrow::gpu::CudaIpcMemHandle":
|
||||
cdef cppclass CCudaIpcMemHandle" arrow::cuda::CudaIpcMemHandle":
|
||||
@staticmethod
|
||||
CStatus FromBuffer(const void* opaque_handle,
|
||||
shared_ptr[CCudaIpcMemHandle]* handle)
|
||||
CStatus Serialize(CMemoryPool* pool, shared_ptr[CBuffer]* out) const
|
||||
|
||||
cdef cppclass CCudaBuffer" arrow::gpu::CudaBuffer"(CBuffer):
|
||||
cdef cppclass CCudaBuffer" arrow::cuda::CudaBuffer"(CBuffer):
|
||||
CCudaBuffer(uint8_t* data, int64_t size,
|
||||
const shared_ptr[CCudaContext]& context,
|
||||
c_bool own_data=false, c_bool is_ipc=false)
|
||||
@@ -73,17 +73,18 @@ cdef extern from "arrow/gpu/cuda_api.h" namespace "arrow::gpu" nogil:
|
||||
CStatus ExportForIpc(shared_ptr[CCudaIpcMemHandle]* handle)
|
||||
shared_ptr[CCudaContext] context() const
|
||||
|
||||
cdef cppclass CCudaHostBuffer" arrow::gpu::CudaHostBuffer"(CMutableBuffer):
|
||||
cdef cppclass \
|
||||
CCudaHostBuffer" arrow::cuda::CudaHostBuffer"(CMutableBuffer):
|
||||
pass
|
||||
|
||||
cdef cppclass \
|
||||
CCudaBufferReader" arrow::gpu::CudaBufferReader"(CBufferReader):
|
||||
CCudaBufferReader" arrow::cuda::CudaBufferReader"(CBufferReader):
|
||||
CCudaBufferReader(const shared_ptr[CBuffer]& buffer)
|
||||
CStatus Read(int64_t nbytes, int64_t* bytes_read, void* buffer)
|
||||
CStatus Read(int64_t nbytes, shared_ptr[CBuffer]* out)
|
||||
|
||||
cdef cppclass \
|
||||
CCudaBufferWriter" arrow::gpu::CudaBufferWriter"(WritableFile):
|
||||
CCudaBufferWriter" arrow::cuda::CudaBufferWriter"(WritableFile):
|
||||
CCudaBufferWriter(const shared_ptr[CCudaBuffer]& buffer)
|
||||
CStatus Close()
|
||||
CStatus Flush()
|
||||
@@ -98,17 +99,17 @@ cdef extern from "arrow/gpu/cuda_api.h" namespace "arrow::gpu" nogil:
|
||||
CStatus AllocateCudaHostBuffer(int device_number, const int64_t size,
|
||||
shared_ptr[CCudaHostBuffer]* out)
|
||||
|
||||
# Cuda prefix is added to avoid picking up arrow::gpu functions
|
||||
# Cuda prefix is added to avoid picking up arrow::cuda functions
|
||||
# from arrow namespace.
|
||||
CStatus CudaSerializeRecordBatch" arrow::gpu::SerializeRecordBatch"\
|
||||
CStatus CudaSerializeRecordBatch" arrow::cuda::SerializeRecordBatch"\
|
||||
(const CRecordBatch& batch,
|
||||
CCudaContext* ctx,
|
||||
shared_ptr[CCudaBuffer]* out)
|
||||
CStatus CudaReadMessage" arrow::gpu::ReadMessage"\
|
||||
CStatus CudaReadMessage" arrow::cuda::ReadMessage"\
|
||||
(CCudaBufferReader* reader,
|
||||
CMemoryPool* pool,
|
||||
unique_ptr[CMessage]* message)
|
||||
CStatus CudaReadRecordBatch" arrow::gpu::ReadRecordBatch"\
|
||||
CStatus CudaReadRecordBatch" arrow::cuda::ReadRecordBatch"\
|
||||
(const shared_ptr[CSchema]& schema,
|
||||
const shared_ptr[CCudaBuffer]& buffer,
|
||||
CMemoryPool* pool, shared_ptr[CRecordBatch]* out)
|
||||
Reference in new issue
Block a user