Add method to set file path in a parquet::FileMetaData instance

This commit is contained in:
Wes McKinney committed 2019-05-24 17:41:36 -05:00
1 parent 184b8deb65
commit e77cf7a8ab
5 files changed
+31

No files matched your search

+5
View File
@@ -163,6 +163,11 @@ TEST(Metadata, TestBuildAccess) {
ASSERT_EQ(16, rg2_column2->dictionary_page_offset());
ASSERT_EQ(10, rg2_column1->data_page_offset());
ASSERT_EQ(26, rg2_column2->data_page_offset());
// Test FileMetaData::set_file_path
ASSERT_TRUE(rg2_column1->file_path().empty());
f_accessor->set_file_path("/foo/bar/bar.parquet");
ASSERT_EQ("/foo/bar/bar.parquet", rg2_column1->file_path());
}
TEST(Metadata, TestV1Version) {
+10
View File
@@ -386,6 +386,14 @@ class FileMetaData::FileMetaDataImpl {
return key_value_metadata_;
}
void set_file_path(const std::string& path) {
for (format::RowGroup& row_group : metadata_->row_groups) {
for (format::ColumnChunk& chunk : row_group.columns) {
chunk.__set_file_path(path);
}
}
}
private:
friend FileMetaDataBuilder;
uint32_t metadata_len_;
@@ -483,6 +491,8 @@ std::shared_ptr<const KeyValueMetadata> FileMetaData::key_value_metadata() const
return impl_->key_value_metadata();
}
void FileMetaData::set_file_path(const std::string& path) { impl_->set_file_path(path); }
void FileMetaData::WriteTo(OutputStream* dst) const { return impl_->WriteTo(dst); }
ApplicationVersion::ApplicationVersion(const std::string& application, int major,
+5
View File
@@ -110,8 +110,10 @@ class PARQUET_EXPORT ColumnChunkMetaData {
// column chunk
int64_t file_offset() const;
// parameter is only used when a dataset is spread across multiple files
const std::string& file_path() const;
// column metadata
Type::type type() const;
int64_t num_values() const;
@@ -190,6 +192,9 @@ class PARQUET_EXPORT FileMetaData {
std::shared_ptr<const KeyValueMetadata> key_value_metadata() const;
// Set file_path ColumnChunk fields to a particular value
void set_file_path(const std::string& path);
private:
friend FileMetaDataBuilder;
explicit FileMetaData(const void* serialized_metadata, uint32_t* metadata_len);
+2
View File
@@ -222,6 +222,8 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
const c_string created_by()
int num_schema_elements()
void set_file_path(const c_string& path)
unique_ptr[CRowGroupMetaData] RowGroup(int i)
const SchemaDescriptor* schema()
shared_ptr[const CKeyValueMetadata] key_value_metadata() const
+9
View File
@@ -522,6 +522,15 @@ cdef class FileMetaData:
def row_group(self, int i):
return RowGroupMetaData(self, i)
def set_file_path(self, path):
"""
Modify the file_path field of each ColumnChunk in the
FileMetaData to be a particular value
"""
cdef:
c_string c_path = tobytes(path)
self._metadata.set_file_path(c_path)
cdef class ParquetSchema:
cdef: