mirror of
https://github.com/apache/arrow.git
synced 2026-10-28 16:47:19 +08:00
UnaryKernel::Call returns Status for now for simplicity. Support pre-allocated memory
Change-Id: Ie2d1611ef5b6477680f57dc44653b5784be79f08
This commit is contained in:
1 parent
8903709bad
commit
a9a04c9cac
6 files changed
+170
-143
No files matched your search
+14
-14
@@ -87,48 +87,48 @@ namespace internal {
|
||||
struct ARROW_EXPORT ArrayData {
|
||||
ArrayData() {}
|
||||
|
||||
ArrayData(const std::shared_ptr<DataType>& type, int64_t length,
|
||||
int64_t null_count = kUnknownNullCount, int64_t offset = 0)
|
||||
: type(type), length(length), null_count(null_count), offset(offset) {}
|
||||
|
||||
ArrayData(const std::shared_ptr<DataType>& type, int64_t length,
|
||||
const std::vector<std::shared_ptr<Buffer>>& buffers,
|
||||
int64_t null_count = kUnknownNullCount, int64_t offset = 0)
|
||||
: type(type),
|
||||
length(length),
|
||||
buffers(buffers),
|
||||
null_count(null_count),
|
||||
offset(offset) {}
|
||||
: ArrayData(type, length, null_count, offset) {
|
||||
this->buffers = buffers;
|
||||
}
|
||||
|
||||
ArrayData(const std::shared_ptr<DataType>& type, int64_t length,
|
||||
std::vector<std::shared_ptr<Buffer>>&& buffers,
|
||||
int64_t null_count = kUnknownNullCount, int64_t offset = 0)
|
||||
: type(type),
|
||||
length(length),
|
||||
buffers(std::move(buffers)),
|
||||
null_count(null_count),
|
||||
offset(offset) {}
|
||||
: ArrayData(type, length, null_count, offset) {
|
||||
this->buffers = std::move(buffers);
|
||||
}
|
||||
|
||||
// Move constructor
|
||||
ArrayData(ArrayData&& other) noexcept
|
||||
: type(std::move(other.type)),
|
||||
length(other.length),
|
||||
buffers(std::move(other.buffers)),
|
||||
null_count(other.null_count),
|
||||
offset(other.offset),
|
||||
buffers(std::move(other.buffers)),
|
||||
child_data(std::move(other.child_data)) {}
|
||||
|
||||
ArrayData(const ArrayData& other) noexcept
|
||||
: type(other.type),
|
||||
length(other.length),
|
||||
buffers(other.buffers),
|
||||
null_count(other.null_count),
|
||||
offset(other.offset),
|
||||
buffers(other.buffers),
|
||||
child_data(other.child_data) {}
|
||||
|
||||
// Move assignment
|
||||
ArrayData& operator=(ArrayData&& other) {
|
||||
type = std::move(other.type);
|
||||
length = other.length;
|
||||
buffers = std::move(other.buffers);
|
||||
null_count = other.null_count;
|
||||
offset = other.offset;
|
||||
buffers = std::move(other.buffers);
|
||||
child_data = std::move(other.child_data);
|
||||
return *this;
|
||||
}
|
||||
@@ -139,9 +139,9 @@ struct ARROW_EXPORT ArrayData {
|
||||
|
||||
std::shared_ptr<DataType> type;
|
||||
int64_t length;
|
||||
std::vector<std::shared_ptr<Buffer>> buffers;
|
||||
int64_t null_count;
|
||||
int64_t offset;
|
||||
std::vector<std::shared_ptr<Buffer>> buffers;
|
||||
std::vector<std::shared_ptr<ArrayData>> child_data;
|
||||
};
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
install(FILES
|
||||
cast.h
|
||||
context.h
|
||||
kernel.h
|
||||
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/compute")
|
||||
|
||||
#######################################
|
||||
|
||||
+107
-98
@@ -24,10 +24,12 @@
|
||||
#include <sstream>
|
||||
#include <type_traits>
|
||||
|
||||
#include "arrow/array.h"
|
||||
#include "arrow/type_traits.h"
|
||||
#include "arrow/util/logging.h"
|
||||
|
||||
#include "arrow/compute/context.h"
|
||||
#include "arrow/compute/kernel.h"
|
||||
|
||||
namespace arrow {
|
||||
namespace compute {
|
||||
@@ -64,14 +66,12 @@ struct CastFunctor {};
|
||||
// Indicated no computation required
|
||||
template <typename O, typename I>
|
||||
struct CastFunctor<O, I, typename std::enable_if<is_zero_copy_cast<O, I>::value>::type> {
|
||||
void operator()(FunctionContext* ctx, const CastOptions& options,
|
||||
const ArrayData& input, ArrayData* output) {
|
||||
output->type = input.type;
|
||||
output->buffers = input.buffers;
|
||||
output->length = input.length;
|
||||
output->offset = input.offset;
|
||||
output->null_count = input.null_count;
|
||||
output->child_data = input.child_data;
|
||||
void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
|
||||
ArrayData* output) {
|
||||
auto in_data = input.data();
|
||||
output->null_count = input.null_count();
|
||||
output->buffers = in_data->buffers;
|
||||
output->child_data = in_data->child_data;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -79,10 +79,10 @@ struct CastFunctor<O, I, typename std::enable_if<is_zero_copy_cast<O, I>::value>
|
||||
// Null to other things
|
||||
|
||||
template <typename T>
|
||||
struct CastFunctor<T, NullType,
|
||||
typename std::enable_if<std::is_base_of<FixedWidthType, T>::value>::type> {
|
||||
void operator()(FunctionContext* ctx, const CastOptions& options,
|
||||
const ArrayData& input, ArrayData* output) {
|
||||
struct CastFunctor<T, NullType, typename std::enable_if<
|
||||
std::is_base_of<FixedWidthType, T>::value>::type> {
|
||||
void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
|
||||
ArrayData* output) {
|
||||
// Simply initialize data to 0
|
||||
auto buf = output->buffers[1];
|
||||
memset(buf->mutable_data(), 0, buf->size());
|
||||
@@ -96,14 +96,14 @@ struct CastFunctor<T, NullType,
|
||||
template <typename T>
|
||||
struct CastFunctor<T, BooleanType,
|
||||
typename std::enable_if<std::is_base_of<Number, T>::value>::type> {
|
||||
void operator()(FunctionContext* ctx, const CastOptions& options,
|
||||
const ArrayData& input, ArrayData* output) {
|
||||
void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
|
||||
ArrayData* output) {
|
||||
using c_type = typename T::c_type;
|
||||
const uint8_t* data = input.buffers[1]->data();
|
||||
const uint8_t* data = input.data()->buffers[1]->data();
|
||||
auto out = reinterpret_cast<c_type*>(output->buffers[1]->mutable_data());
|
||||
constexpr auto kOne = static_cast<c_type>(1);
|
||||
constexpr auto kZero = static_cast<c_type>(0);
|
||||
for (int64_t i = 0; i < input.length; ++i) {
|
||||
for (int64_t i = 0; i < input.length(); ++i) {
|
||||
*out++ = BitUtil::GetBit(data, i) ? kOne : kZero;
|
||||
}
|
||||
}
|
||||
@@ -146,12 +146,12 @@ template <typename O, typename I>
|
||||
struct CastFunctor<O, I, typename std::enable_if<std::is_same<BooleanType, O>::value &&
|
||||
std::is_base_of<Number, I>::value &&
|
||||
!std::is_same<O, I>::value>::type> {
|
||||
void operator()(FunctionContext* ctx, const CastOptions& options,
|
||||
const ArrayData& input, ArrayData* output) {
|
||||
void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
|
||||
ArrayData* output) {
|
||||
using in_type = typename I::c_type;
|
||||
auto in_data = reinterpret_cast<const in_type*>(input.buffers[1]->data());
|
||||
auto in_data = reinterpret_cast<const in_type*>(input.data()->buffers[1]->data());
|
||||
uint8_t* out_data = reinterpret_cast<uint8_t*>(output->buffers[1]->mutable_data());
|
||||
for (int64_t i = 0; i < input.length; ++i) {
|
||||
for (int64_t i = 0; i < input.length(); ++i) {
|
||||
BitUtil::SetBitTo(out_data, i, (*in_data++) != 0);
|
||||
}
|
||||
}
|
||||
@@ -160,24 +160,26 @@ struct CastFunctor<O, I, typename std::enable_if<std::is_same<BooleanType, O>::v
|
||||
template <typename O, typename I>
|
||||
struct CastFunctor<O, I,
|
||||
typename std::enable_if<is_integer_downcast<O, I>::value>::type> {
|
||||
void operator()(FunctionContext* ctx, const CastOptions& options,
|
||||
const ArrayData& input, ArrayData* output) {
|
||||
void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
|
||||
ArrayData* output) {
|
||||
using in_type = typename I::c_type;
|
||||
using out_type = typename O::c_type;
|
||||
|
||||
auto in_offset = input.offset;
|
||||
auto in_offset = input.offset();
|
||||
|
||||
auto in_data = reinterpret_cast<const in_type*>(input.buffers[1]->data()) + in_offset;
|
||||
const auto& input_buffers = input.data()->buffers;
|
||||
|
||||
auto in_data = reinterpret_cast<const in_type*>(input_buffers[1]->data()) + in_offset;
|
||||
auto out_data = reinterpret_cast<out_type*>(output->buffers[1]->mutable_data());
|
||||
|
||||
if (!options.allow_int_overflow) {
|
||||
constexpr in_type kMax = static_cast<in_type>(std::numeric_limits<out_type>::max());
|
||||
constexpr in_type kMin = static_cast<in_type>(std::numeric_limits<out_type>::min());
|
||||
|
||||
if (input.null_count > 0) {
|
||||
const uint8_t* is_valid = input.buffers[0]->data();
|
||||
if (input.null_count() > 0) {
|
||||
const uint8_t* is_valid = input_buffers[0]->data();
|
||||
int64_t is_valid_offset = in_offset;
|
||||
for (int64_t i = 0; i < input.length; ++i) {
|
||||
for (int64_t i = 0; i < input.length(); ++i) {
|
||||
if (ARROW_PREDICT_FALSE(BitUtil::GetBit(is_valid, is_valid_offset++) &&
|
||||
(*in_data > kMax || *in_data < kMin))) {
|
||||
ctx->SetStatus(Status::Invalid("Integer value out of bounds"));
|
||||
@@ -185,7 +187,7 @@ struct CastFunctor<O, I,
|
||||
*out_data++ = static_cast<out_type>(*in_data++);
|
||||
}
|
||||
} else {
|
||||
for (int64_t i = 0; i < input.length; ++i) {
|
||||
for (int64_t i = 0; i < input.length(); ++i) {
|
||||
if (ARROW_PREDICT_FALSE(*in_data > kMax || *in_data < kMin)) {
|
||||
ctx->SetStatus(Status::Invalid("Integer value out of bounds"));
|
||||
}
|
||||
@@ -193,7 +195,7 @@ struct CastFunctor<O, I,
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (int64_t i = 0; i < input.length; ++i) {
|
||||
for (int64_t i = 0; i < input.length(); ++i) {
|
||||
*out_data++ = static_cast<out_type>(*in_data++);
|
||||
}
|
||||
}
|
||||
@@ -204,14 +206,14 @@ template <typename O, typename I>
|
||||
struct CastFunctor<O, I,
|
||||
typename std::enable_if<is_numeric_cast<O, I>::value &&
|
||||
!is_integer_downcast<O, I>::value>::type> {
|
||||
void operator()(FunctionContext* ctx, const CastOptions& options,
|
||||
const ArrayData& input, ArrayData* output) {
|
||||
void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
|
||||
ArrayData* output) {
|
||||
using in_type = typename I::c_type;
|
||||
using out_type = typename O::c_type;
|
||||
|
||||
auto in_data = reinterpret_cast<const in_type*>(input.buffers[1]->data());
|
||||
auto in_data = reinterpret_cast<const in_type*>(input.data()->buffers[1]->data());
|
||||
auto out_data = reinterpret_cast<out_type*>(output->buffers[1]->mutable_data());
|
||||
for (int64_t i = 0; i < input.length; ++i) {
|
||||
for (int64_t i = 0; i < input.length(); ++i) {
|
||||
*out_data++ = static_cast<out_type>(*in_data++);
|
||||
}
|
||||
}
|
||||
@@ -219,31 +221,38 @@ struct CastFunctor<O, I,
|
||||
|
||||
// ----------------------------------------------------------------------
|
||||
|
||||
CastKernel::CastKernel(const CastOptions& options, const CastFunction& func,
|
||||
bool is_zero_copy)
|
||||
: options_(options), func_(func), is_zero_copy_(is_zero_copy) {}
|
||||
typedef std::function<void(FunctionContext*, const CastOptions& options, const Array&,
|
||||
ArrayData*)>
|
||||
CastFunction;
|
||||
|
||||
static Status AllocateLike(FunctionContext* ctx, const ArrayData& in_data,
|
||||
ArrayData* out) {
|
||||
static Status AllocateIfNotPreallocated(FunctionContext* ctx, const Array& input,
|
||||
ArrayData* out) {
|
||||
if (!is_primitive(out->type->id())) {
|
||||
return Status::NotImplemented(out->type->ToString());
|
||||
}
|
||||
|
||||
const auto& fw_type = static_cast<const FixedWidthType&>(*out->type);
|
||||
|
||||
out->null_count = in_data.null_count;
|
||||
const int64_t length = input.length();
|
||||
|
||||
// Propagate null bitmap
|
||||
// TODO(wesm): handling null bitmap when input type is NullType
|
||||
out->buffers.clear();
|
||||
out->null_count = input.null_count();
|
||||
|
||||
// Propagate bitmap unless we are null type
|
||||
std::shared_ptr<Buffer> validity_bitmap = in_data.buffers[0];
|
||||
if (in_data.type->id() == Type::NA) {
|
||||
int64_t bitmap_size = BitUtil::BytesForBits(in_data.length);
|
||||
std::shared_ptr<Buffer> validity_bitmap = input.data()->buffers[0];
|
||||
if (input.type_id() == Type::NA) {
|
||||
int64_t bitmap_size = BitUtil::BytesForBits(length);
|
||||
RETURN_NOT_OK(ctx->Allocate(bitmap_size, &validity_bitmap));
|
||||
memset(validity_bitmap->mutable_data(), 0, bitmap_size);
|
||||
}
|
||||
|
||||
if (out->buffers.size() == 2) {
|
||||
// Assuming preallocated, propagage bitmap and move on
|
||||
out->buffers[0] = validity_bitmap;
|
||||
return Status::OK();
|
||||
} else {
|
||||
DCHECK_EQ(0, out->buffers.size());
|
||||
}
|
||||
|
||||
out->buffers.push_back(validity_bitmap);
|
||||
|
||||
std::shared_ptr<Buffer> out_data;
|
||||
@@ -252,9 +261,9 @@ static Status AllocateLike(FunctionContext* ctx, const ArrayData& in_data,
|
||||
int64_t buffer_size = 0;
|
||||
|
||||
if (bit_width == 1) {
|
||||
buffer_size = BitUtil::BytesForBits(in_data.length);
|
||||
buffer_size = BitUtil::BytesForBits(length);
|
||||
} else if (bit_width % 8 == 0) {
|
||||
buffer_size = in_data.length * fw_type.bit_width() / 8;
|
||||
buffer_size = length * fw_type.bit_width() / 8;
|
||||
} else {
|
||||
DCHECK(false);
|
||||
}
|
||||
@@ -267,26 +276,34 @@ static Status AllocateLike(FunctionContext* ctx, const ArrayData& in_data,
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
void CastKernel::operator()(FunctionContext* ctx, const ArrayData& input,
|
||||
ArrayData* out) {
|
||||
if (!is_zero_copy_) {
|
||||
Status s = AllocateLike(ctx, input, out);
|
||||
if (ARROW_PREDICT_FALSE(!s.ok())) {
|
||||
ctx->SetStatus(s);
|
||||
return;
|
||||
}
|
||||
}
|
||||
func_(ctx, options_, input, out);
|
||||
}
|
||||
class CastKernel : public UnaryKernel {
|
||||
public:
|
||||
CastKernel(const CastOptions& options, const CastFunction& func, bool is_zero_copy)
|
||||
: options_(options), func_(func), is_zero_copy_(is_zero_copy) {}
|
||||
|
||||
#define CAST_CASE(InType, OutType) \
|
||||
case OutType::type_id: \
|
||||
is_zero_copy = is_zero_copy_cast<OutType, InType>::value; \
|
||||
func = [](FunctionContext* ctx, const CastOptions& options, const ArrayData& input, \
|
||||
ArrayData* out) { \
|
||||
CastFunctor<OutType, InType> func; \
|
||||
func(ctx, options, input, out); \
|
||||
}; \
|
||||
Status Call(FunctionContext* ctx, const Array& input, ArrayData* out) override {
|
||||
if (!is_zero_copy_) {
|
||||
RETURN_NOT_OK(AllocateIfNotPreallocated(ctx, input, out));
|
||||
}
|
||||
func_(ctx, options_, input, out);
|
||||
RETURN_IF_ERROR(ctx);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
private:
|
||||
CastOptions options_;
|
||||
CastFunction func_;
|
||||
bool is_zero_copy_;
|
||||
};
|
||||
|
||||
#define CAST_CASE(InType, OutType) \
|
||||
case OutType::type_id: \
|
||||
is_zero_copy = is_zero_copy_cast<OutType, InType>::value; \
|
||||
func = [](FunctionContext* ctx, const CastOptions& options, const Array& input, \
|
||||
ArrayData* out) { \
|
||||
CastFunctor<OutType, InType> func; \
|
||||
func(ctx, options, input, out); \
|
||||
}; \
|
||||
break;
|
||||
|
||||
#define NUMERIC_CASES(FN, IN_TYPE) \
|
||||
@@ -302,12 +319,12 @@ void CastKernel::operator()(FunctionContext* ctx, const ArrayData& input,
|
||||
FN(IN_TYPE, FloatType); \
|
||||
FN(IN_TYPE, DoubleType);
|
||||
|
||||
#define NULL_CASES(FN, IN_TYPE) \
|
||||
NUMERIC_CASES(FN, IN_TYPE) \
|
||||
FN(NullType, Time32Type); \
|
||||
FN(NullType, Date32Type); \
|
||||
FN(NullType, TimestampType); \
|
||||
FN(NullType, Time64Type); \
|
||||
#define NULL_CASES(FN, IN_TYPE) \
|
||||
NUMERIC_CASES(FN, IN_TYPE) \
|
||||
FN(NullType, Time32Type); \
|
||||
FN(NullType, Date32Type); \
|
||||
FN(NullType, TimestampType); \
|
||||
FN(NullType, Time64Type); \
|
||||
FN(NullType, Date64Type);
|
||||
|
||||
#define INT32_CASES(FN, IN_TYPE) \
|
||||
@@ -331,20 +348,20 @@ void CastKernel::operator()(FunctionContext* ctx, const ArrayData& input,
|
||||
|
||||
#define TIMESTAMP_CASES(FN, IN_TYPE) FN(TimestampType, TimestampType);
|
||||
|
||||
#define GET_CAST_FUNCTION(CASE_GENERATOR, InType) \
|
||||
static std::unique_ptr<CastKernel> Get##InType##CastFunc( \
|
||||
const std::shared_ptr<DataType>& out_type, const CastOptions& options) { \
|
||||
CastFunction func; \
|
||||
bool is_zero_copy = false; \
|
||||
switch (out_type->id()) { \
|
||||
CASE_GENERATOR(CAST_CASE, InType); \
|
||||
default: \
|
||||
break; \
|
||||
} \
|
||||
if (func != nullptr) { \
|
||||
return std::unique_ptr<CastKernel>(new CastKernel(options, func, is_zero_copy)); \
|
||||
} \
|
||||
return nullptr; \
|
||||
#define GET_CAST_FUNCTION(CASE_GENERATOR, InType) \
|
||||
static std::unique_ptr<UnaryKernel> Get##InType##CastFunc( \
|
||||
const std::shared_ptr<DataType>& out_type, const CastOptions& options) { \
|
||||
CastFunction func; \
|
||||
bool is_zero_copy = false; \
|
||||
switch (out_type->id()) { \
|
||||
CASE_GENERATOR(CAST_CASE, InType); \
|
||||
default: \
|
||||
break; \
|
||||
} \
|
||||
if (func != nullptr) { \
|
||||
return std::unique_ptr<UnaryKernel>(new CastKernel(options, func, is_zero_copy)); \
|
||||
} \
|
||||
return nullptr; \
|
||||
}
|
||||
|
||||
GET_CAST_FUNCTION(NULL_CASES, NullType);
|
||||
@@ -371,7 +388,7 @@ GET_CAST_FUNCTION(TIMESTAMP_CASES, TimestampType);
|
||||
break
|
||||
|
||||
Status GetCastFunction(const DataType& in_type, const std::shared_ptr<DataType>& out_type,
|
||||
const CastOptions& options, std::unique_ptr<CastKernel>* kernel) {
|
||||
const CastOptions& options, std::unique_ptr<UnaryKernel>* kernel) {
|
||||
switch (in_type.id()) {
|
||||
CAST_FUNCTION_CASE(NullType);
|
||||
CAST_FUNCTION_CASE(BooleanType);
|
||||
@@ -406,21 +423,13 @@ Status Cast(FunctionContext* ctx, const Array& array,
|
||||
const std::shared_ptr<DataType>& out_type, const CastOptions& options,
|
||||
std::shared_ptr<Array>* out) {
|
||||
// Dynamic dispatch to obtain right cast function
|
||||
std::unique_ptr<CastKernel> func;
|
||||
std::unique_ptr<UnaryKernel> func;
|
||||
RETURN_NOT_OK(GetCastFunction(*array.type(), out_type, options, &func));
|
||||
|
||||
// TODO(wesm): Must force computation of unknown null counts
|
||||
ARROW_UNUSED(array.null_count());
|
||||
|
||||
// Allocate memory for output
|
||||
auto out_data = std::make_shared<ArrayData>();
|
||||
out_data->type = out_type;
|
||||
out_data->length = array.length();
|
||||
out_data->offset = 0;
|
||||
auto out_data = std::make_shared<ArrayData>(out_type, array.length());
|
||||
|
||||
(*func)(ctx, *array.data(), out_data.get());
|
||||
|
||||
RETURN_IF_ERROR(ctx);
|
||||
RETURN_NOT_OK(func->Call(ctx, array, out_data.get()));
|
||||
return internal::MakeArray(out_data, out);
|
||||
}
|
||||
|
||||
|
||||
@@ -23,6 +23,8 @@
|
||||
#include "arrow/array.h"
|
||||
#include "arrow/util/visibility.h"
|
||||
|
||||
#include "arrow/compute/kernel.h"
|
||||
|
||||
namespace arrow {
|
||||
|
||||
using internal::ArrayData;
|
||||
@@ -30,42 +32,17 @@ using internal::ArrayData;
|
||||
namespace compute {
|
||||
|
||||
class FunctionContext;
|
||||
class UnaryKernel;
|
||||
|
||||
struct CastOptions {
|
||||
bool allow_int_overflow;
|
||||
};
|
||||
|
||||
typedef std::function<void(FunctionContext*, const CastOptions& options, const ArrayData&,
|
||||
ArrayData*)>
|
||||
CastFunction;
|
||||
|
||||
class ARROW_EXPORT OpKernel {
|
||||
public:
|
||||
virtual ~OpKernel() = default;
|
||||
};
|
||||
|
||||
class ARROW_EXPORT UnaryKernel : public OpKernel {
|
||||
public:
|
||||
virtual void operator()(FunctionContext* ctx, const ArrayData& input,
|
||||
ArrayData* out) = 0;
|
||||
};
|
||||
|
||||
class ARROW_EXPORT CastKernel : public UnaryKernel {
|
||||
public:
|
||||
CastKernel(const CastOptions& options, const CastFunction& func, bool is_zero_copy);
|
||||
void operator()(FunctionContext* ctx, const ArrayData& input, ArrayData* out) override;
|
||||
|
||||
private:
|
||||
CastOptions options_;
|
||||
CastFunction func_;
|
||||
bool is_zero_copy_;
|
||||
};
|
||||
|
||||
/// \since 0.7.0
|
||||
/// \note API not yet finalized
|
||||
ARROW_EXPORT
|
||||
Status GetCastFunction(const DataType& in_type, const std::shared_ptr<DataType>& to_type,
|
||||
const CastOptions& options, std::unique_ptr<CastKernel>* kernel);
|
||||
const CastOptions& options, std::unique_ptr<UnaryKernel>* kernel);
|
||||
|
||||
/// \brief Cast from one array type to another
|
||||
/// \param[in] context
|
||||
|
||||
@@ -43,6 +43,9 @@
|
||||
using std::vector;
|
||||
|
||||
namespace arrow {
|
||||
|
||||
using internal::ArrayData;
|
||||
|
||||
namespace compute {
|
||||
|
||||
void AssertArraysEqual(const Array& left, const Array& right) {
|
||||
@@ -99,13 +102,12 @@ class TestCast : public ComputeFixture, public ::testing::Test {
|
||||
ASSERT_RAISES(Invalid, Cast(&ctx_, *input, out_type, options, &result));
|
||||
}
|
||||
|
||||
void CheckZeroCopy(const Array& input,
|
||||
const std::shared_ptr<DataType>& out_type) {
|
||||
void CheckZeroCopy(const Array& input, const std::shared_ptr<DataType>& out_type) {
|
||||
std::shared_ptr<Array> result;
|
||||
ASSERT_OK(Cast(&ctx_, input, out_type, {}, &result));
|
||||
AssertBufferSame(input, *result, 0);
|
||||
AssertBufferSame(input, *result, 1);
|
||||
};
|
||||
}
|
||||
|
||||
template <typename InType, typename I_TYPE, typename OutType, typename O_TYPE>
|
||||
void CheckCase(const std::shared_ptr<DataType>& in_type,
|
||||
@@ -357,5 +359,43 @@ TEST_F(TestCast, FromNull) {
|
||||
AssertArraysEqual(*result, *result);
|
||||
}
|
||||
|
||||
TEST_F(TestCast, PreallocatedMemory) {
|
||||
CastOptions options;
|
||||
options.allow_int_overflow = false;
|
||||
|
||||
vector<bool> is_valid = {true, false, true, true, true};
|
||||
|
||||
const int64_t length = 5;
|
||||
|
||||
std::shared_ptr<Array> arr;
|
||||
vector<int32_t> v1 = {0, 70000, 2000, 1000, 0};
|
||||
vector<int64_t> e1 = {0, 70000, 2000, 1000, 0};
|
||||
ArrayFromVector<Int32Type, int32_t>(int32(), is_valid, v1, &arr);
|
||||
|
||||
auto out_type = int64();
|
||||
|
||||
std::unique_ptr<UnaryKernel> kernel;
|
||||
ASSERT_OK(GetCastFunction(*int32(), out_type, options, &kernel));
|
||||
|
||||
auto out_data = std::make_shared<ArrayData>(out_type, length);
|
||||
|
||||
std::shared_ptr<Buffer> out_values;
|
||||
ASSERT_OK(this->ctx_.Allocate(length * sizeof(int64_t), &out_values));
|
||||
|
||||
out_data->buffers.push_back(nullptr);
|
||||
out_data->buffers.push_back(out_values);
|
||||
|
||||
ASSERT_OK(kernel->Call(&this->ctx_, *arr, out_data.get()));
|
||||
|
||||
// Buffer address unchanged
|
||||
ASSERT_EQ(out_values.get(), out_data->buffers[1].get());
|
||||
|
||||
std::shared_ptr<Array> result, expected;
|
||||
ASSERT_OK(MakeArray(out_data, &result));
|
||||
ArrayFromVector<Int64Type, int64_t>(int64(), is_valid, e1, &expected);
|
||||
|
||||
AssertArraysEqual(*expected, *result);
|
||||
}
|
||||
|
||||
} // namespace compute
|
||||
} // namespace arrow
|
||||
@@ -906,7 +906,7 @@ Status PandasConverter::ConvertObjectsInfer() {
|
||||
return ConvertLists(inferred_type);
|
||||
} else {
|
||||
const std::string supported_types =
|
||||
"string, bool, float, int, date, time, decimal, list, array";
|
||||
"string, bool, float, int, date, time, decimal, list, array";
|
||||
std::stringstream ss;
|
||||
ss << "Error inferring Arrow type for Python object array. ";
|
||||
RETURN_NOT_OK(InvalidConversion(obj, supported_types, &ss));
|
||||
|
||||
Reference in new issue
Block a user