Use unsafe appends for BinaryArray for better performance

This commit is contained in:
Wes McKinney committed 2019-08-29 11:07:07 -05:00
1 parent e6458d2aa2
commit ab2ad26ef4
2 files changed
+31 -11

No files matched your search

+3 -1
View File
@@ -101,7 +101,9 @@ Status BufferOutputStream::Write(const void* data, int64_t nbytes) {
}
DCHECK(buffer_);
if (ARROW_PREDICT_TRUE(nbytes > 0)) {
RETURN_NOT_OK(Reserve(nbytes));
if (ARROW_PREDICT_FALSE(position_ + nbytes >= capacity_)) {
RETURN_NOT_OK(Reserve(nbytes));
}
memcpy(mutable_data_ + position_, data, nbytes);
position_ += nbytes;
}
+28 -10
View File
@@ -146,29 +146,34 @@ void AssertBinary(const arrow::Array& values) {
}
}
template <typename EncoderType>
void PutBinaryArray(const arrow::Array& values, EncoderType* encoder) {
template <>
void PlainEncoder<ByteArrayType>::Put(const arrow::Array& values) {
AssertBinary(values);
const auto& data = checked_cast<const arrow::BinaryArray&>(values);
const int64_t total_bytes = data.value_offset(data.length()) - data.value_offset(0);
PARQUET_THROW_NOT_OK(sink_.Reserve(total_bytes + data.length() * sizeof(uint32_t)));
auto AppendValue = [&](int64_t i) {
auto view = data.GetView(i);
const uint32_t length = static_cast<uint32_t>(view.size());
sink_.UnsafeAppend(&length, sizeof(uint32_t));
sink_.UnsafeAppend(view.data(), view.size());
};
if (data.null_count() == 0) {
// no nulls, just dump the data
for (int64_t i = 0; i < data.length(); i++) {
encoder->Put(ByteArray(data.GetView(i)));
AppendValue(i);
}
} else {
for (int64_t i = 0; i < data.length(); i++) {
if (data.IsValid(i)) {
encoder->Put(ByteArray(data.GetView(i)));
AppendValue(i);
}
}
}
}
template <>
void PlainEncoder<ByteArrayType>::Put(const arrow::Array& values) {
PutBinaryArray(values, this);
}
template <>
inline void PlainEncoder<FLBAType>::Put(const FixedLenByteArray* src, int num_values) {
for (int i = 0; i < num_values; ++i) {
@@ -561,7 +566,20 @@ void DictEncoderImpl<DType>::Put(const arrow::Array& values) {
template <>
void DictEncoderImpl<ByteArrayType>::Put(const arrow::Array& values) {
PutBinaryArray(values, this);
AssertBinary(values);
const auto& data = checked_cast<const arrow::BinaryArray&>(values);
if (data.null_count() == 0) {
// no nulls, just dump the data
for (int64_t i = 0; i < data.length(); i++) {
Put(ByteArray(data.GetView(i)));
}
} else {
for (int64_t i = 0; i < data.length(); i++) {
if (data.IsValid(i)) {
Put(ByteArray(data.GetView(i)));
}
}
}
}
template <typename DType>