mirror of
https://github.com/apache/arrow.git
synced 2026-10-28 16:47:19 +08:00
Parquet files with dictionary page offset as 0 is not readable
pyarrow needs to handle dictionary page offset = 0 as a special case to be compatible with java parquet reader. Change name of comment
This commit is contained in:
1 parent
828d18ec37
commit
f47762ab03
3 files changed
+9
-2
No files matched your search
@@ -2354,6 +2354,12 @@ TEST(TestArrowReaderAdHoc, CorruptedSchema) {
|
||||
TryReadDataFile(path, ::arrow::StatusCode::IOError);
|
||||
}
|
||||
|
||||
TEST(TestArrowReaderAdHoc, HandleDictPageOffsetZero) {
|
||||
// PARQUET-1402: parquet-mr writes files this way which tripped up
|
||||
// some business logic
|
||||
TryReadDataFile(test::get_data_file("dict-page-offset-zero.parquet"));
|
||||
}
|
||||
|
||||
class TestArrowReaderAdHocSparkAndHvr
|
||||
: public ::testing::TestWithParam<
|
||||
std::tuple<std::string, std::shared_ptr<::DataType>>> {};
|
||||
|
||||
@@ -94,7 +94,8 @@ class SerializedRowGroup : public RowGroupReader::Contents {
|
||||
auto col = row_group_metadata_->ColumnChunk(i);
|
||||
|
||||
int64_t col_start = col->data_page_offset();
|
||||
if (col->has_dictionary_page() && col_start > col->dictionary_page_offset()) {
|
||||
if (col->has_dictionary_page() && col->dictionary_page_offset() > 0 &&
|
||||
col_start > col->dictionary_page_offset()) {
|
||||
col_start = col->dictionary_page_offset();
|
||||
}
|
||||
|
||||
|
||||
Submodule cpp/submodules/parquet-testing updated: bb7b6abbb3...2fc3ade4cc.
Reference in new issue
Block a user