Test case for min-max statistics on binary column built with parquet-mr

This commit is contained in:
Ildar Musin authored and Wes McKinney committed 2019-01-22 18:24:47 -06:00
1 parent 029f73047d
commit c5ba5fc07f
3 files changed
+32 -4

No files matched your search

+3 -3
View File
@@ -48,7 +48,7 @@ const ApplicationVersion& ApplicationVersion::PARQUET_CPP_FIXED_STATS_VERSION()
}
const ApplicationVersion& ApplicationVersion::PARQUET_MR_FIXED_STATS_VERSION() {
static ApplicationVersion version("parquet-cpp", 1, 10, 0);
static ApplicationVersion version("parquet-mr", 1, 10, 0);
return version;
}
@@ -554,8 +554,8 @@ bool ApplicationVersion::HasCorrectStatistics(Type::type col_type,
SortOrder::type sort_order) const {
// parquet-cpp version 1.3.0 and parquet-mr 1.10.0 onwards stats are computed
// correctly for all types
if ((application_ == "parquet-cpp" && VersionLt(PARQUET_CPP_FIXED_STATS_VERSION()))
|| (application_ == "parquet-mr" && VersionLt(PARQUET_MR_FIXED_STATS_VERSION()))) {
if ((application_ == "parquet-cpp" && VersionLt(PARQUET_CPP_FIXED_STATS_VERSION())) ||
(application_ == "parquet-mr" && VersionLt(PARQUET_MR_FIXED_STATS_VERSION()))) {
// Only SIGNED are valid unless max and min are the same
// (in which case the sort order does not matter)
bool max_equals_min = statistics.has_min && statistics.has_max
+28
View File
@@ -772,5 +772,33 @@ TEST(TestStatisticsDoubleNaN, NaNValues) {
ASSERT_EQ(min, -3.0);
ASSERT_EQ(max, 4.0);
}
// Test statistics for binary column with UNSIGNED sort order
TEST(TestStatisticsMinMax, Unsigned) {
std::string dir_string(test::get_data_dir());
std::stringstream ss;
ss << dir_string << "/binary.parquet";
auto path = ss.str();
// The file is generated by parquet-mr 1.10.0, the first version that
// supports correct statistics for binary data (see PARQUET-1025). It
// contains a single column of binary type. Data is just single byte values
// from 0x00 to 0x0B.
auto file_reader = ParquetFileReader::OpenFile(path);
auto rg_reader = file_reader->RowGroup(0);
auto metadata = rg_reader->metadata();
auto column_schema = metadata->schema()->Column(0);
ASSERT_EQ(SortOrder::UNSIGNED, column_schema->sort_order());
auto column_chunk = metadata->ColumnChunk(0);
ASSERT_TRUE(column_chunk->is_stats_set());
std::shared_ptr<RowGroupStatistics> stats = column_chunk->statistics();
ASSERT_TRUE(stats != NULL);
ASSERT_EQ(0, stats->null_count());
ASSERT_EQ(12, stats->num_values());
ASSERT_EQ(0x00, stats->EncodeMin()[0]);
ASSERT_EQ(0x0b, stats->EncodeMax()[0]);
}
} // namespace test
} // namespace parquet