mirror of
https://github.com/apache/arrow.git
synced 2026-10-28 16:47:19 +08:00
Test case for min-max statistics on binary column built with parquet-mr
This commit is contained in:
1 parent
029f73047d
commit
c5ba5fc07f
3 files changed
+32
-4
No files matched your search
@@ -48,7 +48,7 @@ const ApplicationVersion& ApplicationVersion::PARQUET_CPP_FIXED_STATS_VERSION()
|
||||
}
|
||||
|
||||
const ApplicationVersion& ApplicationVersion::PARQUET_MR_FIXED_STATS_VERSION() {
|
||||
static ApplicationVersion version("parquet-cpp", 1, 10, 0);
|
||||
static ApplicationVersion version("parquet-mr", 1, 10, 0);
|
||||
return version;
|
||||
}
|
||||
|
||||
@@ -554,8 +554,8 @@ bool ApplicationVersion::HasCorrectStatistics(Type::type col_type,
|
||||
SortOrder::type sort_order) const {
|
||||
// parquet-cpp version 1.3.0 and parquet-mr 1.10.0 onwards stats are computed
|
||||
// correctly for all types
|
||||
if ((application_ == "parquet-cpp" && VersionLt(PARQUET_CPP_FIXED_STATS_VERSION()))
|
||||
|| (application_ == "parquet-mr" && VersionLt(PARQUET_MR_FIXED_STATS_VERSION()))) {
|
||||
if ((application_ == "parquet-cpp" && VersionLt(PARQUET_CPP_FIXED_STATS_VERSION())) ||
|
||||
(application_ == "parquet-mr" && VersionLt(PARQUET_MR_FIXED_STATS_VERSION()))) {
|
||||
// Only SIGNED are valid unless max and min are the same
|
||||
// (in which case the sort order does not matter)
|
||||
bool max_equals_min = statistics.has_min && statistics.has_max
|
||||
|
||||
@@ -772,5 +772,33 @@ TEST(TestStatisticsDoubleNaN, NaNValues) {
|
||||
ASSERT_EQ(min, -3.0);
|
||||
ASSERT_EQ(max, 4.0);
|
||||
}
|
||||
|
||||
// Test statistics for binary column with UNSIGNED sort order
|
||||
TEST(TestStatisticsMinMax, Unsigned) {
|
||||
std::string dir_string(test::get_data_dir());
|
||||
std::stringstream ss;
|
||||
ss << dir_string << "/binary.parquet";
|
||||
auto path = ss.str();
|
||||
|
||||
// The file is generated by parquet-mr 1.10.0, the first version that
|
||||
// supports correct statistics for binary data (see PARQUET-1025). It
|
||||
// contains a single column of binary type. Data is just single byte values
|
||||
// from 0x00 to 0x0B.
|
||||
auto file_reader = ParquetFileReader::OpenFile(path);
|
||||
auto rg_reader = file_reader->RowGroup(0);
|
||||
auto metadata = rg_reader->metadata();
|
||||
auto column_schema = metadata->schema()->Column(0);
|
||||
ASSERT_EQ(SortOrder::UNSIGNED, column_schema->sort_order());
|
||||
|
||||
auto column_chunk = metadata->ColumnChunk(0);
|
||||
ASSERT_TRUE(column_chunk->is_stats_set());
|
||||
|
||||
std::shared_ptr<RowGroupStatistics> stats = column_chunk->statistics();
|
||||
ASSERT_TRUE(stats != NULL);
|
||||
ASSERT_EQ(0, stats->null_count());
|
||||
ASSERT_EQ(12, stats->num_values());
|
||||
ASSERT_EQ(0x00, stats->EncodeMin()[0]);
|
||||
ASSERT_EQ(0x0b, stats->EncodeMax()[0]);
|
||||
}
|
||||
} // namespace test
|
||||
} // namespace parquet
|
||||
Submodule cpp/submodules/parquet-testing updated: 8eb0213c49...bb7b6abbb3.
Reference in new issue
Block a user