mirrors/arrow
0
0 Fork 0
mirror of https://github.com/apache/arrow.git synced 2026-10-28 16:47:19 +08:00
Code Activity
Files
main
arrow/ruby/red-arrow/test/test-array-builder.rb
T
hypsakata 6ee7f7ecdc GH-48481: [Ruby] Correctly infer types for nested integer arrays (#48699) 
### Rationale for this change

When building an `Arrow::Table` from a Ruby Hash passed to `Arrow::Table.new`, nested Integer arrays are incorrectly inferred as `list<uint8>` or `list<int8>` regardless of the actual values contained. Nested integer arrays should be correctly inferred as the appropriate list type (e.g., `list<int64>`, `list<uint64>`) based on their values, similar to how flat arrays are handled, unless they contain values out of range for any integer type.

### What changes are included in this PR?

This PR modifies the logic in `detect_builder_info()` to fix the inference issue. Specifically:

- **Persist `sub_builder_info` across sub-array elements**: Previously, `sub_builder_info` was recreated for each sub-array element in the Array. The logic has been updated to accumulate and carry over the builder information across elements to ensure correct type inference for the entire list.
- **Refactor Integer builder logic**: Following the pattern used for `BigDecimal`, the logic for determining the Integer builder has been moved to `create_builder()`. `detect_builder_info()` now calls this function.

**Note:** 

- As a side effect of this refactoring, nested lists of `BigDecimal` (which were previously inferred as `string`) may now have their types inferred. However, comprehensive testing and verification for nested `BigDecimal` support will be addressed in a separate issue to keep this PR focused.
- We stopped using `IntArrayBuilder` for inference logic to ensure correctness. This results in a performance overhead (array building is approximately 2x slower) as we can no longer rely on the specialized builder's detection.

```text
                                           user     system      total        real
    array_builder int32 100000         0.085867   0.000194   0.086061 (  0.086369)
int_array_builder int32 100000         0.042163   0.001033   0.043196 (  0.043268)
    array_builder int64 100000         0.086799   0.000015   0.086814 (  0.086828)
int_array_builder int64 100000         0.044493   0.000973   0.045466 (  0.045469)
    array_builder uint32 100000        0.085748   0.000009   0.085757 (  0.085768)
int_array_builder uint32 100000        0.044463   0.001034   0.045497 (  0.045498)
    array_builder uint64 100000        0.084548   0.000987   0.085535 (  0.085537)
int_array_builder uint64 100000        0.044206   0.000017   0.044223 (  0.044225)
```

### Are these changes tested?

Yes. `ruby ruby/red-arrow/test/run-test.rb`

### Are there any user-facing changes?

Yes.

* GitHub Issue: #48481

Authored-by: hypsakata <46911464+hypsakata@users.noreply.github.com>
Signed-off-by: Sutou Kouhei <kou@clear-code.com>
2026-01-03 09:58:34 +09:00

599 lines
18 KiB
Ruby
Raw Permalink Blame History

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
class ArrayBuilderTest < Test::Unit::TestCase
sub_test_case(".build") do
def assert_build(builder_class, raw_array)
array = builder_class.build(raw_array)
assert_equal(raw_array, array.to_a)
end
sub_test_case("generic builder") do
test("strings") do
assert_build(Arrow::ArrayBuilder,
["Hello", nil, "World"])
end
test("symbols") do
array = Arrow::ArrayBuilder.build([:hello, nil, :world])
expected_builder = Arrow::StringDictionaryArrayBuilder.new
assert_equal(expected_builder.build(["hello", nil, "world"]),
array)
end
test("boolean") do
assert_build(Arrow::ArrayBuilder,
[true, nil, false])
end
test("positive integers") do
assert_build(Arrow::ArrayBuilder,
[1, nil, 2, nil, 3])
end
test("negative integers") do
assert_build(Arrow::ArrayBuilder,
[nil, -1, nil, -2, nil, -3])
end
test("times") do
assert_build(Arrow::ArrayBuilder,
[Time.at(0), Time.at(1), Time.at(2)])
end
test("dates") do
assert_build(Arrow::ArrayBuilder,
[Date.new(2018, 1, 4), Date.new(2018, 1, 5)])
end
test("datetimes") do
assert_build(Arrow::ArrayBuilder,
[
DateTime.new(2018, 1, 4, 23, 18, 23),
DateTime.new(2018, 1, 5, 0, 23, 21),
])
end
test("decimal + string") do
raw_array = [BigDecimal("10.1"), "10.1"]
array = Arrow::ArrayBuilder.build(raw_array)
assert_equal(raw_array.collect(&:to_s), array.to_a)
end
test("NaN") do
raw_array = [BigDecimal("10.1"), BigDecimal::NAN]
array = Arrow::ArrayBuilder.build(raw_array)
assert_equal(raw_array.collect(&:to_s), array.to_a)
end
test("Infinity") do
raw_array = [BigDecimal("10.1"), BigDecimal::INFINITY]
array = Arrow::ArrayBuilder.build(raw_array)
assert_equal(raw_array.collect(&:to_s), array.to_a)
end
test("decimal128") do
values = [
BigDecimal("10.1"),
BigDecimal("1.11"),
BigDecimal("1"),
]
array = Arrow::Array.new(values)
data_type = Arrow::Decimal128DataType.new(3, 2)
assert_equal({
data_type: data_type,
values: [
BigDecimal("10.1"),
BigDecimal("1.11"),
BigDecimal("1"),
],
},
{
data_type: array.value_data_type,
values: array.to_a,
})
end
test("decimal256") do
values = [
BigDecimal("1" * 40 + ".1"),
BigDecimal("1" * 38 + ".11"),
BigDecimal("1" * 37),
]
array = Arrow::Array.new(values)
data_type = Arrow::Decimal256DataType.new(41, 2)
assert_equal({
data_type: data_type,
values: [
BigDecimal("1" * 40 + ".1"),
BigDecimal("1" * 38 + ".11"),
BigDecimal("1" * 37),
],
},
{
data_type: array.value_data_type,
values: array.to_a,
})
end
test("list<boolean>s") do
assert_build(Arrow::ArrayBuilder,
[
[nil, true, false],
nil,
[false],
])
end
test("list<string>s") do
assert_build(Arrow::ArrayBuilder,
[
["Hello", "World"],
["Apache Arrow"],
])
end
sub_test_case("nested integer list") do
test("list<uint8>s") do
values = [
[0, 1, 2],
[3, 4],
]
array = Arrow::Array.new(values)
data_type = Arrow::ListDataType.new(:uint8)
assert_equal({
data_type: data_type,
values: [
[0, 1, 2],
[3, 4],
],
},
{
data_type: array.value_data_type,
values: array.to_a,
})
end
test("list<int8>s boundary") do
values = [
[0, GLib::MININT8],
[GLib::MAXINT8],
]
array = Arrow::Array.new(values)
data_type = Arrow::ListDataType.new(:int8)
assert_equal({
data_type: data_type,
values: [
[0, GLib::MININT8],
[GLib::MAXINT8],
],
},
{
data_type: array.value_data_type,
values: array.to_a,
})
end
test("list<int16>s inferred from int8 underflow") do
values = [
[0, GLib::MININT8 - 1],
[GLib::MAXINT8],
]
array = Arrow::Array.new(values)
data_type = Arrow::ListDataType.new(:int16)
assert_equal({
data_type: data_type,
values: [
[0, GLib::MININT8 - 1],
[GLib::MAXINT8],
],
},
{
data_type: array.value_data_type,
values: array.to_a,
})
end
test("list<int16>s inferred from int8 overflow") do
values = [
[0, GLib::MAXINT8 + 1],
[GLib::MININT8],
]
array = Arrow::Array.new(values)
data_type = Arrow::ListDataType.new(:int16)
assert_equal({
data_type: data_type,
values: [
[0, GLib::MAXINT8 + 1],
[GLib::MININT8],
],
},
{
data_type: array.value_data_type,
values: array.to_a,
})
end
test("list<int16>s boundary") do
values = [
[0, GLib::MININT16],
[GLib::MAXINT16],
]
array = Arrow::Array.new(values)
data_type = Arrow::ListDataType.new(:int16)
assert_equal({
data_type: data_type,
values: [
[0, GLib::MININT16],
[GLib::MAXINT16],
],
},
{
data_type: array.value_data_type,
values: array.to_a,
})
end
test("list<int32>s inferred from int16 underflow") do
values = [
[0, GLib::MININT16 - 1],
[GLib::MAXINT16],
]
array = Arrow::Array.new(values)
data_type = Arrow::ListDataType.new(:int32)
assert_equal({
data_type: data_type,
values: [
[0, GLib::MININT16 - 1],
[GLib::MAXINT16],
],
},
{
data_type: array.value_data_type,
values: array.to_a,
})
end
test("list<int32>s inferred from int16 overflow") do
values = [
[0, GLib::MAXINT16 + 1],
[GLib::MININT16],
]
array = Arrow::Array.new(values)
data_type = Arrow::ListDataType.new(:int32)
assert_equal({
data_type: data_type,
values: [
[0, GLib::MAXINT16 + 1],
[GLib::MININT16],
],
},
{
data_type: array.value_data_type,
values: array.to_a,
})
end
test("list<int32>s boundary") do
values = [
[0, GLib::MININT32],
[GLib::MAXINT32],
]
array = Arrow::Array.new(values)
data_type = Arrow::ListDataType.new(:int32)
assert_equal({
data_type: data_type,
values: [
[0, GLib::MININT32],
[GLib::MAXINT32],
],
},
{
data_type: array.value_data_type,
values: array.to_a,
})
end
test("list<int64>s inferred from int32 underflow") do
values = [
[0, GLib::MININT32 - 1],
[GLib::MAXINT32],
]
array = Arrow::Array.new(values)
data_type = Arrow::ListDataType.new(:int64)
assert_equal({
data_type: data_type,
values: [
[0, GLib::MININT32 - 1],
[GLib::MAXINT32],
],
},
{
data_type: array.value_data_type,
values: array.to_a,
})
end
test("list<int64>s inferred from int32 overflow") do
values = [
[0, GLib::MAXINT32 + 1],
[GLib::MININT32],
]
array = Arrow::Array.new(values)
data_type = Arrow::ListDataType.new(:int64)
assert_equal({
data_type: data_type,
values: [
[0, GLib::MAXINT32 + 1],
[GLib::MININT32],
],
},
{
data_type: array.value_data_type,
values: array.to_a,
})
end
test("string fallback from nested int64 array overflow") do
values = [
[0, GLib::MAXINT64 + 1],
[GLib::MININT64],
]
array = Arrow::Array.new(values)
data_type = Arrow::ListDataType.new(:string)
assert_equal({
data_type: data_type,
values: [
["0", "#{GLib::MAXINT64 + 1}"],
["#{GLib::MININT64}"],
],
},
{
data_type: array.value_data_type,
values: array.to_a,
})
end
test("string fallback from nested int64 array underflow") do
values = [
[0, GLib::MININT64 - 1],
[GLib::MAXINT64],
]
array = Arrow::Array.new(values)
data_type = Arrow::ListDataType.new(:string)
assert_equal({
data_type: data_type,
values: [
["0", "#{GLib::MININT64 - 1}"],
["#{GLib::MAXINT64}"],
],
},
{
data_type: array.value_data_type,
values: array.to_a,
})
end
test("list<uint8>s boundary") do
values = [
[0, GLib::MAXUINT8],
[GLib::MAXUINT8],
]
array = Arrow::Array.new(values)
data_type = Arrow::ListDataType.new(:uint8)
assert_equal({
data_type: data_type,
values: [
[0, GLib::MAXUINT8],
[GLib::MAXUINT8],
],
},
{
data_type: array.value_data_type,
values: array.to_a,
})
end
test("list<uint16>s") do
values = [
[0, GLib::MAXUINT8 + 1],
[GLib::MAXUINT8],
]
array = Arrow::Array.new(values)
data_type = Arrow::ListDataType.new(:uint16)
assert_equal({
data_type: data_type,
values: [
[0, GLib::MAXUINT8 + 1],
[GLib::MAXUINT8],
],
},
{
data_type: array.value_data_type,
values: array.to_a,
})
end
test("list<uint16>s boundary") do
values = [
[0, GLib::MAXUINT16],
[GLib::MAXUINT16],
]
array = Arrow::Array.new(values)
data_type = Arrow::ListDataType.new(:uint16)
assert_equal({
data_type: data_type,
values: [
[0, GLib::MAXUINT16],
[GLib::MAXUINT16],
],
},
{
data_type: array.value_data_type,
values: array.to_a,
})
end
test("list<uint32>s") do
values = [
[0, GLib::MAXUINT16 + 1],
[GLib::MAXUINT16],
]
array = Arrow::Array.new(values)
data_type = Arrow::ListDataType.new(:uint32)
assert_equal({
data_type: data_type,
values: [
[0, GLib::MAXUINT16 + 1],
[GLib::MAXUINT16],
],
},
{
data_type: array.value_data_type,
values: array.to_a,
})
end
test("list<uint32>s boundary") do
values = [
[0, GLib::MAXUINT32],
[GLib::MAXUINT32],
]
array = Arrow::Array.new(values)
data_type = Arrow::ListDataType.new(:uint32)
assert_equal({
data_type: data_type,
values: [
[0, GLib::MAXUINT32],
[GLib::MAXUINT32],
],
},
{
data_type: array.value_data_type,
values: array.to_a,
})
end
test("list<uint64>s") do
values = [
[0, GLib::MAXUINT32 + 1],
[GLib::MAXUINT32],
]
array = Arrow::Array.new(values)
data_type = Arrow::ListDataType.new(:uint64)
assert_equal({
data_type: data_type,
values: [
[0, GLib::MAXUINT32 + 1],
[GLib::MAXUINT32],
],
},
{
data_type: array.value_data_type,
values: array.to_a,
})
end
test("string fallback from nested uint64 array overflow") do
values = [
[0, GLib::MAXUINT64 + 1],
[GLib::MAXUINT64],
]
array = Arrow::Array.new(values)
data_type = Arrow::ListDataType.new(:string)
assert_equal({
data_type: data_type,
values: [
["0", "#{GLib::MAXUINT64 + 1}"],
["#{GLib::MAXUINT64}"],
],
},
{
data_type: array.value_data_type,
values: array.to_a,
})
end
end
end
sub_test_case("specific builder") do
test("empty") do
assert_build(Arrow::Int32ArrayBuilder,
[])
end
test("values") do
assert_build(Arrow::Int32ArrayBuilder,
[1, -2])
end
test("values, nils") do
assert_build(Arrow::Int32ArrayBuilder,
[1, -2, nil, nil])
end
test("values, nils, values") do
assert_build(Arrow::Int32ArrayBuilder,
[1, -2, nil, nil, 3, -4])
end
test("values, nils, values, nils") do
assert_build(Arrow::Int32ArrayBuilder,
[1, -2, nil, nil, 3, -4, nil, nil])
end
test("nils") do
assert_build(Arrow::Int32ArrayBuilder,
[nil, nil])
end
test("nils, values") do
assert_build(Arrow::Int32ArrayBuilder,
[nil, nil, 3, -4])
end
test("nils, values, nil") do
assert_build(Arrow::Int32ArrayBuilder,
[nil, nil, 3, -4, nil, nil])
end
test("nils, values, nil, values") do
assert_build(Arrow::Int32ArrayBuilder,
[nil, nil, 3, -4, nil, nil, 5, -6])
end
end
end
end
View Git Blame Copy Permalink