mirror of
https://github.com/apache/arrow.git
synced 2026-10-29 09:05:47 +08:00
Use unsafe appends for BinaryArray for better performance
This commit is contained in:
1 parent
e6458d2aa2
commit
ab2ad26ef4
2 files changed
+31
-11
No files matched your search
@@ -101,7 +101,9 @@ Status BufferOutputStream::Write(const void* data, int64_t nbytes) {
|
||||
}
|
||||
DCHECK(buffer_);
|
||||
if (ARROW_PREDICT_TRUE(nbytes > 0)) {
|
||||
RETURN_NOT_OK(Reserve(nbytes));
|
||||
if (ARROW_PREDICT_FALSE(position_ + nbytes >= capacity_)) {
|
||||
RETURN_NOT_OK(Reserve(nbytes));
|
||||
}
|
||||
memcpy(mutable_data_ + position_, data, nbytes);
|
||||
position_ += nbytes;
|
||||
}
|
||||
|
||||
+28
-10
@@ -146,29 +146,34 @@ void AssertBinary(const arrow::Array& values) {
|
||||
}
|
||||
}
|
||||
|
||||
template <typename EncoderType>
|
||||
void PutBinaryArray(const arrow::Array& values, EncoderType* encoder) {
|
||||
template <>
|
||||
void PlainEncoder<ByteArrayType>::Put(const arrow::Array& values) {
|
||||
AssertBinary(values);
|
||||
const auto& data = checked_cast<const arrow::BinaryArray&>(values);
|
||||
const int64_t total_bytes = data.value_offset(data.length()) - data.value_offset(0);
|
||||
PARQUET_THROW_NOT_OK(sink_.Reserve(total_bytes + data.length() * sizeof(uint32_t)));
|
||||
|
||||
auto AppendValue = [&](int64_t i) {
|
||||
auto view = data.GetView(i);
|
||||
const uint32_t length = static_cast<uint32_t>(view.size());
|
||||
sink_.UnsafeAppend(&length, sizeof(uint32_t));
|
||||
sink_.UnsafeAppend(view.data(), view.size());
|
||||
};
|
||||
|
||||
if (data.null_count() == 0) {
|
||||
// no nulls, just dump the data
|
||||
for (int64_t i = 0; i < data.length(); i++) {
|
||||
encoder->Put(ByteArray(data.GetView(i)));
|
||||
AppendValue(i);
|
||||
}
|
||||
} else {
|
||||
for (int64_t i = 0; i < data.length(); i++) {
|
||||
if (data.IsValid(i)) {
|
||||
encoder->Put(ByteArray(data.GetView(i)));
|
||||
AppendValue(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <>
|
||||
void PlainEncoder<ByteArrayType>::Put(const arrow::Array& values) {
|
||||
PutBinaryArray(values, this);
|
||||
}
|
||||
|
||||
template <>
|
||||
inline void PlainEncoder<FLBAType>::Put(const FixedLenByteArray* src, int num_values) {
|
||||
for (int i = 0; i < num_values; ++i) {
|
||||
@@ -561,7 +566,20 @@ void DictEncoderImpl<DType>::Put(const arrow::Array& values) {
|
||||
|
||||
template <>
|
||||
void DictEncoderImpl<ByteArrayType>::Put(const arrow::Array& values) {
|
||||
PutBinaryArray(values, this);
|
||||
AssertBinary(values);
|
||||
const auto& data = checked_cast<const arrow::BinaryArray&>(values);
|
||||
if (data.null_count() == 0) {
|
||||
// no nulls, just dump the data
|
||||
for (int64_t i = 0; i < data.length(); i++) {
|
||||
Put(ByteArray(data.GetView(i)));
|
||||
}
|
||||
} else {
|
||||
for (int64_t i = 0; i < data.length(); i++) {
|
||||
if (data.IsValid(i)) {
|
||||
Put(ByteArray(data.GetView(i)));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename DType>
|
||||
|
||||
Reference in new issue
Block a user