Parquet files with dictionary page offset as 0 is not readable

pyarrow needs to handle dictionary page offset = 0 as a special case
to be compatible with java parquet reader.

Change name of comment
This commit is contained in:
shyam authored and Wes McKinney committed 2019-05-21 11:52:22 -05:00
1 parent 828d18ec37
commit f47762ab03
3 files changed
+9 -2

No files matched your search

@@ -2354,6 +2354,12 @@ TEST(TestArrowReaderAdHoc, CorruptedSchema) {
TryReadDataFile(path, ::arrow::StatusCode::IOError);
}
TEST(TestArrowReaderAdHoc, HandleDictPageOffsetZero) {
// PARQUET-1402: parquet-mr writes files this way which tripped up
// some business logic
TryReadDataFile(test::get_data_file("dict-page-offset-zero.parquet"));
}
class TestArrowReaderAdHocSparkAndHvr
: public ::testing::TestWithParam<
std::tuple<std::string, std::shared_ptr<::DataType>>> {};
+2 -1
View File
@@ -94,7 +94,8 @@ class SerializedRowGroup : public RowGroupReader::Contents {
auto col = row_group_metadata_->ColumnChunk(i);
int64_t col_start = col->data_page_offset();
if (col->has_dictionary_page() && col_start > col->dictionary_page_offset()) {
if (col->has_dictionary_page() && col->dictionary_page_offset() > 0 &&
col_start > col->dictionary_page_offset()) {
col_start = col->dictionary_page_offset();
}