ARROW-4028: [Rust] Merge parquet-rs codebase

ci/rust-build-main.bat

@@ -17,6 +17,9 @@
 
 @rem The "main" Rust build script for Windows CI
 
+@rem Retrieve git submodules, configure env var for Parquet unit tests
+git submodule update --init || exit /B
+set PARQUET_TEST_DATA=%CD%\cpp\submodules\parquet-testing\data
 pushd rust
 
 @echo ===================================

ci/travis_script_rust.sh

@@ -19,6 +19,8 @@
 
 set -e
 
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
+
 RUST_DIR=${TRAVIS_BUILD_DIR}/rust
 
 pushd $RUST_DIR

docker-compose.yml

@@ -152,6 +152,8 @@ services:
     build:
       context: .
       dockerfile: rust/Dockerfile
+    environment:
+      PARQUET_TEST_DATA: /arrow/cpp/submodules/parquet-testing/data
     volumes: *ubuntu-volumes
 
   r:

rust/Cargo.toml

@@ -42,10 +42,22 @@ serde_derive = "1.0.80"
 serde_json = "1.0.13"
 rand = "0.5"
 csv = "1.0.0"
+parquet-format = "2.5.0"
+quick-error = "1.2.2"
+byteorder = "1"
+thrift = "0.0.4"
+snap = "0.2"
+brotli = "2.5"
+flate2 = "1.0.2"
+lz4 = "1.23"
+zstd = "0.4"
+chrono = "0.4"
+num-bigint = "0.2"
 num = "0.2"
 
 [dev-dependencies]
 criterion = "0.2"
+lazy_static = "1"
 
 [[bench]]
 name = "array_from_vec"

rust/benches/array_from_vec.rs

@@ -17,7 +17,6 @@
 
 #[macro_use]
 extern crate criterion;
-
 use criterion::Criterion;
 
 extern crate arrow;

rust/benches/builder.rs

@@ -19,11 +19,13 @@ extern crate arrow;
 extern crate criterion;
 extern crate rand;
 
-use arrow::builder::*;
+use std::mem::size_of;
+
 use criterion::*;
 use rand::distributions::Standard;
 use rand::{thread_rng, Rng};
-use std::mem::size_of;
+
+use arrow::builder::*;
 
 // Build arrays with 512k elements.
 const BATCH_SIZE: usize = 8 << 10;

rust/build.rs

@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::process::Command;
+
+fn main() {
+    // Set Parquet version, build hash and "created by" string.
+    let version = env!("CARGO_PKG_VERSION");
+    let mut created_by = format!("parquet-rs version {}", version);
+    if let Ok(git_hash) = run(Command::new("git").arg("rev-parse").arg("HEAD")) {
+        created_by.push_str(format!(" (build {})", git_hash).as_str());
+        println!("cargo:rustc-env=PARQUET_BUILD={}", git_hash);
+    }
+    println!("cargo:rustc-env=PARQUET_VERSION={}", version);
+    println!("cargo:rustc-env=PARQUET_CREATED_BY={}", created_by);
+}
+
+/// Runs command and returns either content of stdout for successful execution,
+/// or an error message otherwise.
+fn run(command: &mut Command) -> Result<String, String> {
+    println!("Running: `{:?}`", command);
+    match command.output() {
+        Ok(ref output) if output.status.success() => {
+            Ok(String::from_utf8_lossy(&output.stdout).trim().to_string())
+        }
+        Ok(ref output) => Err(format!("Failed: `{:?}` ({})", command, output.status)),
+        Err(error) => Err(format!("Failed: `{:?}` ({})", command, error)),
+    }
+}

rust/examples/read_csv.rs

@@ -17,11 +17,12 @@
 
 extern crate arrow;
 
+use std::fs::File;
+use std::sync::Arc;
+
 use arrow::array::{BinaryArray, Float64Array};
 use arrow::csv;
 use arrow::datatypes::{DataType, Field, Schema};
-use std::fs::File;
-use std::sync::Arc;
 
 fn main() {
     let schema = Schema::new(vec![

rust/rustfmt.toml

@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+format_doc_comments = true

rust/src/array.rs

@@ -657,13 +657,15 @@ impl From<Vec<(Field, ArrayRef)>> for StructArray {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::array_data::ArrayData;
-    use crate::buffer::Buffer;
-    use crate::datatypes::{DataType, Field, ToByteSlice};
-    use crate::memory;
+
     use std::sync::Arc;
     use std::thread;
 
+    use crate::array_data::ArrayData;
+    use crate::buffer::Buffer;
+    use crate::datatypes::{DataType, Field};
+    use crate::memory;
+
     #[test]
     fn test_primitive_array_from_vec() {
         let buf = Buffer::from(&[0, 1, 2, 3, 4].to_byte_slice());

rust/src/array_data.rs

@@ -225,9 +225,10 @@ impl ArrayDataBuilder {
 
 #[cfg(test)]
 mod tests {
+    use super::*;
+
     use std::sync::Arc;
 
-    use super::{ArrayData, DataType};
     use crate::buffer::Buffer;
     use crate::util::bit_util;
 

rust/src/builder.rs

@@ -456,10 +456,10 @@ impl BinaryArrayBuilder {
 
 #[cfg(test)]
 mod tests {
-    use crate::array::Array;
-
     use super::*;
 
+    use crate::array::Array;
+
     #[test]
     fn test_builder_i32_empty() {
         let b = Int32BufferBuilder::new(5);
@@ -825,7 +825,6 @@ mod tests {
 
     #[test]
     fn test_binary_array_builder() {
-        use crate::array::BinaryArray;
         let mut builder = BinaryArrayBuilder::new(20);
 
         builder.push(b'h').unwrap();
@@ -860,7 +859,6 @@ mod tests {
 
     #[test]
     fn test_binary_array_builder_push_string() {
-        use crate::array::BinaryArray;
         let mut builder = BinaryArrayBuilder::new(20);
 
         let var = "hello".to_owned();

rust/src/csv/reader.rs

@@ -29,16 +29,16 @@
 //! use std::sync::Arc;
 //!
 //! let schema = Schema::new(vec![
-//!   Field::new("city", DataType::Utf8, false),
-//!   Field::new("lat", DataType::Float64, false),
-//!   Field::new("lng", DataType::Float64, false),
+//!     Field::new("city", DataType::Utf8, false),
+//!     Field::new("lat", DataType::Float64, false),
+//!     Field::new("lng", DataType::Float64, false),
 //! ]);
 //!
 //! let file = File::open("test/data/uk_cities.csv").unwrap();
 //!
 //! let mut csv = csv::Reader::new(file, Arc::new(schema), false, 1024, None);
 //! let batch = csv.next().unwrap().unwrap();
-//!```
+//! ```
 
 use std::fs::File;
 use std::io::BufReader;
@@ -195,8 +195,8 @@ impl Reader {
 
 #[cfg(test)]
 mod tests {
-
     use super::*;
+
     use crate::array::*;
     use crate::datatypes::Field;
 

rust/src/lib.rs

@@ -15,7 +15,12 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#![feature(type_ascription)]
+#![feature(rustc_private)]
 #![feature(specialization)]
+#![feature(try_from)]
+#![allow(dead_code)]
+#![allow(non_camel_case_types)]
 
 pub mod array;
 pub mod array_data;
@@ -27,6 +32,7 @@ pub mod csv;
 pub mod datatypes;
 pub mod error;
 pub mod memory;
+pub mod parquet;
 pub mod record_batch;
 pub mod tensor;
 pub mod util;

rust/src/mod.rs

@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+pub mod array;
+pub mod array_data;
+pub mod bitmap;
+pub mod buffer;
+pub mod builder;
+pub mod csv;
+pub mod datatypes;
+pub mod error;
+pub mod memory;
+pub mod record_batch;
+pub mod tensor;

rust/src/parquet/column/mod.rs

@@ -0,0 +1,124 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Low level column reader and writer APIs.
+//!
+//! This API is designed for reading and writing column values, definition and repetition
+//! levels directly.
+//!
+//! # Example of writing and reading data
+//!
+//! Data has the following format:
+//! ```text
+//! +---------------+
+//! |         values|
+//! +---------------+
+//! |[1, 2]         |
+//! |[3, null, null]|
+//! +---------------+
+//! ```
+//!
+//! The example uses column writer and reader APIs to write raw values, definition and
+//! repetition levels and read them to verify write/read correctness.
+//!
+//! ```rust
+//! use std::{fs, path::Path, rc::Rc};
+//!
+//! use arrow::parquet::{
+//!     column::{reader::ColumnReader, writer::ColumnWriter},
+//!     file::{
+//!         properties::WriterProperties,
+//!         reader::{FileReader, SerializedFileReader},
+//!         writer::{FileWriter, SerializedFileWriter},
+//!     },
+//!     schema::parser::parse_message_type,
+//! };
+//!
+//! let path = Path::new("target/debug/examples/column_sample.parquet");
+//!
+//! // Writing data using column writer API.
+//!
+//! let message_type = "
+//!   message schema {
+//!     optional group values (LIST) {
+//!       repeated group list {
+//!         optional INT32 element;
+//!       }
+//!     }
+//!   }
+//! ";
+//! let schema = Rc::new(parse_message_type(message_type).unwrap());
+//! let props = Rc::new(WriterProperties::builder().build());
+//! let file = fs::File::create(path).unwrap();
+//! let mut writer = SerializedFileWriter::new(file, schema, props).unwrap();
+//! let mut row_group_writer = writer.next_row_group().unwrap();
+//! while let Some(mut col_writer) = row_group_writer.next_column().unwrap() {
+//!     match col_writer {
+//!         // You can also use `get_typed_column_writer` method to extract typed writer.
+//!         ColumnWriter::Int32ColumnWriter(ref mut typed_writer) => {
+//!             typed_writer
+//!                 .write_batch(&[1, 2, 3], Some(&[3, 3, 3, 2, 2]), Some(&[0, 1, 0, 1, 1]))
+//!                 .unwrap();
+//!         }
+//!         _ => {}
+//!     }
+//!     row_group_writer.close_column(col_writer).unwrap();
+//! }
+//! writer.close_row_group(row_group_writer).unwrap();
+//! writer.close().unwrap();
+//!
+//! // Reading data using column reader API.
+//!
+//! let file = fs::File::open(path).unwrap();
+//! let reader = SerializedFileReader::new(file).unwrap();
+//! let metadata = reader.metadata();
+//!
+//! let mut res = Ok((0, 0));
+//! let mut values = vec![0; 8];
+//! let mut def_levels = vec![0; 8];
+//! let mut rep_levels = vec![0; 8];
+//!
+//! for i in 0..metadata.num_row_groups() {
+//!     let row_group_reader = reader.get_row_group(i).unwrap();
+//!     let row_group_metadata = metadata.row_group(i);
+//!
+//!     for j in 0..row_group_metadata.num_columns() {
+//!         let mut column_reader = row_group_reader.get_column_reader(j).unwrap();
+//!         match column_reader {
+//!             // You can also use `get_typed_column_reader` method to extract typed reader.
+//!             ColumnReader::Int32ColumnReader(ref mut typed_reader) => {
+//!                 res = typed_reader.read_batch(
+//!                     8, // batch size
+//!                     Some(&mut def_levels),
+//!                     Some(&mut rep_levels),
+//!                     &mut values,
+//!                 );
+//!             }
+//!             _ => {}
+//!         }
+//!     }
+//! }
+//!
+//! assert_eq!(res, Ok((3, 5)));
+//! assert_eq!(values, vec![1, 2, 3, 0, 0, 0, 0, 0]);
+//! assert_eq!(def_levels, vec![3, 3, 3, 2, 2, 0, 0, 0]);
+//! assert_eq!(rep_levels, vec![0, 1, 0, 1, 1, 0, 0, 0]);
+//! ```
+
+pub mod page;
+pub mod reader;
+pub mod writer;

rust/src/parquet/column/page.rs

@@ -0,0 +1,296 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Contains Parquet Page definitions and page reader interface.
+
+use crate::parquet::basic::{Encoding, PageType};
+use crate::parquet::errors::Result;
+use crate::parquet::file::{metadata::ColumnChunkMetaData, statistics::Statistics};
+use crate::parquet::util::memory::ByteBufferPtr;
+
+/// Parquet Page definition.
+///
+/// List of supported pages.
+/// These are 1-to-1 mapped from the equivalent Thrift definitions, except `buf` which
+/// used to store uncompressed bytes of the page.
+pub enum Page {
+    DataPage {
+        buf: ByteBufferPtr,
+        num_values: u32,
+        encoding: Encoding,
+        def_level_encoding: Encoding,
+        rep_level_encoding: Encoding,
+        statistics: Option<Statistics>,
+    },
+    DataPageV2 {
+        buf: ByteBufferPtr,
+        num_values: u32,
+        encoding: Encoding,
+        num_nulls: u32,
+        num_rows: u32,
+        def_levels_byte_len: u32,
+        rep_levels_byte_len: u32,
+        is_compressed: bool,
+        statistics: Option<Statistics>,
+    },
+    DictionaryPage {
+        buf: ByteBufferPtr,
+        num_values: u32,
+        encoding: Encoding,
+        is_sorted: bool,
+    },
+}
+
+impl Page {
+    /// Returns [`PageType`](`::basic::PageType`) for this page.
+    pub fn page_type(&self) -> PageType {
+        match self {
+            &Page::DataPage { .. } => PageType::DATA_PAGE,
+            &Page::DataPageV2 { .. } => PageType::DATA_PAGE_V2,
+            &Page::DictionaryPage { .. } => PageType::DICTIONARY_PAGE,
+        }
+    }
+
+    /// Returns internal byte buffer reference for this page.
+    pub fn buffer(&self) -> &ByteBufferPtr {
+        match self {
+            &Page::DataPage { ref buf, .. } => &buf,
+            &Page::DataPageV2 { ref buf, .. } => &buf,
+            &Page::DictionaryPage { ref buf, .. } => &buf,
+        }
+    }
+
+    /// Returns number of values in this page.
+    pub fn num_values(&self) -> u32 {
+        match self {
+            &Page::DataPage { num_values, .. } => num_values,
+            &Page::DataPageV2 { num_values, .. } => num_values,
+            &Page::DictionaryPage { num_values, .. } => num_values,
+        }
+    }
+
+    /// Returns this page [`Encoding`](`::basic::Encoding`).
+    pub fn encoding(&self) -> Encoding {
+        match self {
+            &Page::DataPage { encoding, .. } => encoding,
+            &Page::DataPageV2 { encoding, .. } => encoding,
+            &Page::DictionaryPage { encoding, .. } => encoding,
+        }
+    }
+
+    /// Returns optional [`Statistics`](`::file::metadata::Statistics`).
+    pub fn statistics(&self) -> Option<&Statistics> {
+        match self {
+            &Page::DataPage { ref statistics, .. } => statistics.as_ref(),
+            &Page::DataPageV2 { ref statistics, .. } => statistics.as_ref(),
+            &Page::DictionaryPage { .. } => None,
+        }
+    }
+}
+
+/// Helper struct to represent pages with potentially compressed buffer (data page v1) or
+/// compressed and concatenated buffer (def levels + rep levels + compressed values for
+/// data page v2).
+///
+/// The difference with `Page` is that `Page` buffer is always uncompressed.
+pub struct CompressedPage {
+    compressed_page: Page,
+    uncompressed_size: usize,
+}
+
+impl CompressedPage {
+    /// Creates `CompressedPage` from a page with potentially compressed buffer and
+    /// uncompressed size.
+    pub fn new(compressed_page: Page, uncompressed_size: usize) -> Self {
+        Self {
+            compressed_page,
+            uncompressed_size,
+        }
+    }
+
+    /// Returns page type.
+    pub fn page_type(&self) -> PageType {
+        self.compressed_page.page_type()
+    }
+
+    /// Returns underlying page with potentially compressed buffer.
+    pub fn compressed_page(&self) -> &Page {
+        &self.compressed_page
+    }
+
+    /// Returns uncompressed size in bytes.
+    pub fn uncompressed_size(&self) -> usize {
+        self.uncompressed_size
+    }
+
+    /// Returns compressed size in bytes.
+    ///
+    /// Note that it is assumed that buffer is compressed, but it may not be. In this
+    /// case compressed size will be equal to uncompressed size.
+    pub fn compressed_size(&self) -> usize {
+        self.compressed_page.buffer().len()
+    }
+
+    /// Number of values in page.
+    pub fn num_values(&self) -> u32 {
+        self.compressed_page.num_values()
+    }
+
+    /// Returns encoding for values in page.
+    pub fn encoding(&self) -> Encoding {
+        self.compressed_page.encoding()
+    }
+
+    /// Returns slice of compressed buffer in the page.
+    pub fn data(&self) -> &[u8] {
+        self.compressed_page.buffer().data()
+    }
+}
+
+/// Contains page write metrics.
+pub struct PageWriteSpec {
+    pub page_type: PageType,
+    pub uncompressed_size: usize,
+    pub compressed_size: usize,
+    pub num_values: u32,
+    pub offset: u64,
+    pub bytes_written: u64,
+}
+
+impl PageWriteSpec {
+    /// Creates new spec with default page write metrics.
+    pub fn new() -> Self {
+        Self {
+            page_type: PageType::DATA_PAGE,
+            uncompressed_size: 0,
+            compressed_size: 0,
+            num_values: 0,
+            offset: 0,
+            bytes_written: 0,
+        }
+    }
+}
+
+/// API for reading pages from a column chunk.
+/// This offers a iterator like API to get the next page.
+pub trait PageReader {
+    /// Gets the next page in the column chunk associated with this reader.
+    /// Returns `None` if there are no pages left.
+    fn get_next_page(&mut self) -> Result<Option<Page>>;
+}
+
+/// API for writing pages in a column chunk.
+///
+/// It is reasonable to assume that all pages will be written in the correct order, e.g.
+/// dictionary page followed by data pages, or a set of data pages, etc.
+pub trait PageWriter {
+    /// Writes a page into the output stream/sink.
+    /// Returns `PageWriteSpec` that contains information about written page metrics,
+    /// including number of bytes, size, number of values, offset, etc.
+    ///
+    /// This method is called for every compressed page we write into underlying buffer,
+    /// either data page or dictionary page.
+    fn write_page(&mut self, page: CompressedPage) -> Result<PageWriteSpec>;
+
+    /// Writes column chunk metadata into the output stream/sink.
+    ///
+    /// This method is called once before page writer is closed, normally when writes are
+    /// finalised in column writer.
+    fn write_metadata(&mut self, metadata: &ColumnChunkMetaData) -> Result<()>;
+
+    /// Closes resources and flushes underlying sink.
+    /// Page writer should not be used after this method is called.
+    fn close(&mut self) -> Result<()>;
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_page() {
+        let data_page = Page::DataPage {
+            buf: ByteBufferPtr::new(vec![0, 1, 2]),
+            num_values: 10,
+            encoding: Encoding::PLAIN,
+            def_level_encoding: Encoding::RLE,
+            rep_level_encoding: Encoding::RLE,
+            statistics: Some(Statistics::int32(Some(1), Some(2), None, 1, true)),
+        };
+        assert_eq!(data_page.page_type(), PageType::DATA_PAGE);
+        assert_eq!(data_page.buffer().data(), vec![0, 1, 2].as_slice());
+        assert_eq!(data_page.num_values(), 10);
+        assert_eq!(data_page.encoding(), Encoding::PLAIN);
+        assert_eq!(
+            data_page.statistics(),
+            Some(&Statistics::int32(Some(1), Some(2), None, 1, true))
+        );
+
+        let data_page_v2 = Page::DataPageV2 {
+            buf: ByteBufferPtr::new(vec![0, 1, 2]),
+            num_values: 10,
+            encoding: Encoding::PLAIN,
+            num_nulls: 5,
+            num_rows: 20,
+            def_levels_byte_len: 30,
+            rep_levels_byte_len: 40,
+            is_compressed: false,
+            statistics: Some(Statistics::int32(Some(1), Some(2), None, 1, true)),
+        };
+        assert_eq!(data_page_v2.page_type(), PageType::DATA_PAGE_V2);
+        assert_eq!(data_page_v2.buffer().data(), vec![0, 1, 2].as_slice());
+        assert_eq!(data_page_v2.num_values(), 10);
+        assert_eq!(data_page_v2.encoding(), Encoding::PLAIN);
+        assert_eq!(
+            data_page_v2.statistics(),
+            Some(&Statistics::int32(Some(1), Some(2), None, 1, true))
+        );
+
+        let dict_page = Page::DictionaryPage {
+            buf: ByteBufferPtr::new(vec![0, 1, 2]),
+            num_values: 10,
+            encoding: Encoding::PLAIN,
+            is_sorted: false,
+        };
+        assert_eq!(dict_page.page_type(), PageType::DICTIONARY_PAGE);
+        assert_eq!(dict_page.buffer().data(), vec![0, 1, 2].as_slice());
+        assert_eq!(dict_page.num_values(), 10);
+        assert_eq!(dict_page.encoding(), Encoding::PLAIN);
+        assert_eq!(dict_page.statistics(), None);
+    }
+
+    #[test]
+    fn test_compressed_page() {
+        let data_page = Page::DataPage {
+            buf: ByteBufferPtr::new(vec![0, 1, 2]),
+            num_values: 10,
+            encoding: Encoding::PLAIN,
+            def_level_encoding: Encoding::RLE,
+            rep_level_encoding: Encoding::RLE,
+            statistics: Some(Statistics::int32(Some(1), Some(2), None, 1, true)),
+        };
+
+        let cpage = CompressedPage::new(data_page, 5);
+
+        assert_eq!(cpage.page_type(), PageType::DATA_PAGE);
+        assert_eq!(cpage.uncompressed_size(), 5);
+        assert_eq!(cpage.compressed_size(), 3);
+        assert_eq!(cpage.num_values(), 10);
+        assert_eq!(cpage.encoding(), Encoding::PLAIN);
+        assert_eq!(cpage.data(), &[0, 1, 2]);
+    }
+}

rust/src/parquet/compression.rs

@@ -0,0 +1,321 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Contains codec interface and supported codec implementations.
+//!
+//! See [`Compression`](`::basic::Compression`) enum for all available compression
+//! algorithms.
+//!
+//! # Example
+//!
+//! ```rust
+//! use arrow::parquet::{basic::Compression, compression::create_codec};
+//!
+//! let mut codec = match create_codec(Compression::SNAPPY) {
+//!     Ok(Some(codec)) => codec,
+//!     _ => panic!(),
+//! };
+//!
+//! let data = vec![b'p', b'a', b'r', b'q', b'u', b'e', b't'];
+//! let mut compressed = vec![];
+//! codec.compress(&data[..], &mut compressed).unwrap();
+//!
+//! let mut output = vec![];
+//! codec.decompress(&compressed[..], &mut output).unwrap();
+//!
+//! assert_eq!(output, data);
+//! ```
+
+use std::io::{self, Read, Write};
+
+use brotli;
+use flate2::{read, write, Compression};
+use lz4;
+use snap::{decompress_len, max_compress_len, Decoder, Encoder};
+use zstd;
+
+use crate::parquet::basic::Compression as CodecType;
+use crate::parquet::errors::{ParquetError, Result};
+
+/// Parquet compression codec interface.
+pub trait Codec {
+    /// Compresses data stored in slice `input_buf` and writes the compressed result
+    /// to `output_buf`.
+    /// Note that you'll need to call `clear()` before reusing the same `output_buf` across
+    /// different `compress` calls.
+    fn compress(&mut self, input_buf: &[u8], output_buf: &mut Vec<u8>) -> Result<()>;
+
+    /// Decompresses data stored in slice `input_buf` and writes output to `output_buf`.
+    /// Returns the total number of bytes written.
+    fn decompress(&mut self, input_buf: &[u8], output_buf: &mut Vec<u8>) -> Result<usize>;
+}
+
+/// Given the compression type `codec`, returns a codec used to compress and decompress
+/// bytes for the compression type.
+/// This returns `None` if the codec type is `UNCOMPRESSED`.
+pub fn create_codec(codec: CodecType) -> Result<Option<Box<Codec>>> {
+    match codec {
+        CodecType::BROTLI => Ok(Some(Box::new(BrotliCodec::new()))),
+        CodecType::GZIP => Ok(Some(Box::new(GZipCodec::new()))),
+        CodecType::SNAPPY => Ok(Some(Box::new(SnappyCodec::new()))),
+        CodecType::LZ4 => Ok(Some(Box::new(LZ4Codec::new()))),
+        CodecType::ZSTD => Ok(Some(Box::new(ZSTDCodec::new()))),
+        CodecType::UNCOMPRESSED => Ok(None),
+        _ => Err(nyi_err!("The codec type {} is not supported yet", codec)),
+    }
+}
+
+/// Codec for Snappy compression format.
+pub struct SnappyCodec {
+    decoder: Decoder,
+    encoder: Encoder,
+}
+
+impl SnappyCodec {
+    /// Creates new Snappy compression codec.
+    fn new() -> Self {
+        Self {
+            decoder: Decoder::new(),
+            encoder: Encoder::new(),
+        }
+    }
+}
+
+impl Codec for SnappyCodec {
+    fn decompress(&mut self, input_buf: &[u8], output_buf: &mut Vec<u8>) -> Result<usize> {
+        let len = decompress_len(input_buf)?;
+        output_buf.resize(len, 0);
+        self.decoder
+            .decompress(input_buf, output_buf)
+            .map_err(|e| e.into())
+    }
+
+    fn compress(&mut self, input_buf: &[u8], output_buf: &mut Vec<u8>) -> Result<()> {
+        let required_len = max_compress_len(input_buf.len());
+        if output_buf.len() < required_len {
+            output_buf.resize(required_len, 0);
+        }
+        let n = self.encoder.compress(input_buf, &mut output_buf[..])?;
+        output_buf.truncate(n);
+        Ok(())
+    }
+}
+
+/// Codec for GZIP compression algorithm.
+pub struct GZipCodec {}
+
+impl GZipCodec {
+    /// Creates new GZIP compression codec.
+    fn new() -> Self {
+        Self {}
+    }
+}
+
+impl Codec for GZipCodec {
+    fn decompress(&mut self, input_buf: &[u8], output_buf: &mut Vec<u8>) -> Result<usize> {
+        let mut decoder = read::GzDecoder::new(input_buf);
+        decoder.read_to_end(output_buf).map_err(|e| e.into())
+    }
+
+    fn compress(&mut self, input_buf: &[u8], output_buf: &mut Vec<u8>) -> Result<()> {
+        let mut encoder = write::GzEncoder::new(output_buf, Compression::default());
+        encoder.write_all(input_buf)?;
+        encoder.try_finish().map_err(|e| e.into())
+    }
+}
+
+const BROTLI_DEFAULT_BUFFER_SIZE: usize = 4096;
+const BROTLI_DEFAULT_COMPRESSION_QUALITY: u32 = 1; // supported levels 0-9
+const BROTLI_DEFAULT_LG_WINDOW_SIZE: u32 = 22; // recommended between 20-22
+
+/// Codec for Brotli compression algorithm.
+pub struct BrotliCodec {}
+
+impl BrotliCodec {
+    /// Creates new Brotli compression codec.
+    fn new() -> Self {
+        Self {}
+    }
+}
+
+impl Codec for BrotliCodec {
+    fn decompress(&mut self, input_buf: &[u8], output_buf: &mut Vec<u8>) -> Result<usize> {
+        brotli::Decompressor::new(input_buf, BROTLI_DEFAULT_BUFFER_SIZE)
+            .read_to_end(output_buf)
+            .map_err(|e| e.into())
+    }
+
+    fn compress(&mut self, input_buf: &[u8], output_buf: &mut Vec<u8>) -> Result<()> {
+        let mut encoder = brotli::CompressorWriter::new(
+            output_buf,
+            BROTLI_DEFAULT_BUFFER_SIZE,
+            BROTLI_DEFAULT_COMPRESSION_QUALITY,
+            BROTLI_DEFAULT_LG_WINDOW_SIZE,
+        );
+        encoder.write_all(&input_buf[..])?;
+        encoder.flush().map_err(|e| e.into())
+    }
+}
+
+const LZ4_BUFFER_SIZE: usize = 4096;
+
+/// Codec for LZ4 compression algorithm.
+pub struct LZ4Codec {}
+
+impl LZ4Codec {
+    /// Creates new LZ4 compression codec.
+    fn new() -> Self {
+        Self {}
+    }
+}
+
+impl Codec for LZ4Codec {
+    fn decompress(&mut self, input_buf: &[u8], output_buf: &mut Vec<u8>) -> Result<usize> {
+        let mut decoder = lz4::Decoder::new(input_buf)?;
+        let mut buffer: [u8; LZ4_BUFFER_SIZE] = [0; LZ4_BUFFER_SIZE];
+        let mut total_len = 0;
+        loop {
+            let len = decoder.read(&mut buffer)?;
+            if len == 0 {
+                break;
+            }
+            total_len += len;
+            output_buf.write_all(&buffer[0..len])?;
+        }
+        Ok(total_len)
+    }
+
+    fn compress(&mut self, input_buf: &[u8], output_buf: &mut Vec<u8>) -> Result<()> {
+        let mut encoder = lz4::EncoderBuilder::new().build(output_buf)?;
+        let mut from = 0;
+        loop {
+            let to = ::std::cmp::min(from + LZ4_BUFFER_SIZE, input_buf.len());
+            encoder.write_all(&input_buf[from..to])?;
+            from += LZ4_BUFFER_SIZE;
+            if from >= input_buf.len() {
+                break;
+            }
+        }
+        encoder.finish().1.map_err(|e| e.into())
+    }
+}
+
+/// Codec for Zstandard compression algorithm.
+pub struct ZSTDCodec {}
+
+impl ZSTDCodec {
+    /// Creates new Zstandard compression codec.
+    fn new() -> Self {
+        Self {}
+    }
+}
+
+/// Compression level (1-21) for ZSTD. Choose 1 here for better compression speed.
+const ZSTD_COMPRESSION_LEVEL: i32 = 1;
+
+impl Codec for ZSTDCodec {
+    fn decompress(&mut self, input_buf: &[u8], output_buf: &mut Vec<u8>) -> Result<usize> {
+        let mut decoder = zstd::Decoder::new(input_buf)?;
+        match io::copy(&mut decoder, output_buf) {
+            Ok(n) => Ok(n as usize),
+            Err(e) => Err(e.into()),
+        }
+    }
+
+    fn compress(&mut self, input_buf: &[u8], output_buf: &mut Vec<u8>) -> Result<()> {
+        let mut encoder = zstd::Encoder::new(output_buf, ZSTD_COMPRESSION_LEVEL)?;
+        encoder.write_all(&input_buf[..])?;
+        match encoder.finish() {
+            Ok(_) => Ok(()),
+            Err(e) => Err(e.into()),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use crate::parquet::util::test_common::*;
+
+    fn test_roundtrip(c: CodecType, data: &Vec<u8>) {
+        let mut c1 = create_codec(c).unwrap().unwrap();
+        let mut c2 = create_codec(c).unwrap().unwrap();
+
+        // Compress with c1
+        let mut compressed = Vec::new();
+        let mut decompressed = Vec::new();
+        c1.compress(data.as_slice(), &mut compressed)
+            .expect("Error when compressing");
+
+        // Decompress with c2
+        let mut decompressed_size = c2
+            .decompress(compressed.as_slice(), &mut decompressed)
+            .expect("Error when decompressing");
+        assert_eq!(data.len(), decompressed_size);
+        decompressed.truncate(decompressed_size);
+        assert_eq!(*data, decompressed);
+
+        compressed.clear();
+
+        // Compress with c2
+        c2.compress(data.as_slice(), &mut compressed)
+            .expect("Error when compressing");
+
+        // Decompress with c1
+        decompressed_size = c1
+            .decompress(compressed.as_slice(), &mut decompressed)
+            .expect("Error when decompressing");
+        assert_eq!(data.len(), decompressed_size);
+        decompressed.truncate(decompressed_size);
+        assert_eq!(*data, decompressed);
+    }
+
+    fn test_codec(c: CodecType) {
+        let sizes = vec![100, 10000, 100000];
+        for size in sizes {
+            let mut data = random_bytes(size);
+            test_roundtrip(c, &mut data);
+        }
+    }
+
+    #[test]
+    fn test_codec_snappy() {
+        test_codec(CodecType::SNAPPY);
+    }
+
+    #[test]
+    fn test_codec_gzip() {
+        test_codec(CodecType::GZIP);
+    }
+
+    #[test]
+    fn test_codec_brotli() {
+        test_codec(CodecType::BROTLI);
+    }
+
+    #[test]
+    fn test_codec_lz4() {
+        test_codec(CodecType::LZ4);
+    }
+
+    #[test]
+    fn test_codec_zstd() {
+        test_codec(CodecType::ZSTD);
+    }
+
+}

rust/src/parquet/data_type.rs

@@ -0,0 +1,463 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Data types that connect Parquet physical types with their Rust-specific
+//! representations.
+
+use std::mem;
+
+use byteorder::{BigEndian, ByteOrder};
+
+use crate::parquet::basic::Type;
+use crate::parquet::util::memory::{ByteBuffer, ByteBufferPtr};
+
+/// Rust representation for logical type INT96, value is backed by an array of `u32`.
+/// The type only takes 12 bytes, without extra padding.
+#[derive(Clone, Debug)]
+pub struct Int96 {
+    value: Option<[u32; 3]>,
+}
+
+impl Int96 {
+    /// Creates new INT96 type struct with no data set.
+    pub fn new() -> Self {
+        Self { value: None }
+    }
+
+    /// Returns underlying data as slice of [`u32`].
+    pub fn data(&self) -> &[u32] {
+        assert!(self.value.is_some());
+        self.value.as_ref().unwrap()
+    }
+
+    /// Sets data for this INT96 type.
+    pub fn set_data(&mut self, elem0: u32, elem1: u32, elem2: u32) {
+        self.value = Some([elem0, elem1, elem2]);
+    }
+}
+
+impl Default for Int96 {
+    fn default() -> Self {
+        Self { value: None }
+    }
+}
+
+impl PartialEq for Int96 {
+    fn eq(&self, other: &Int96) -> bool {
+        self.data() == other.data()
+    }
+}
+
+impl From<Vec<u32>> for Int96 {
+    fn from(buf: Vec<u32>) -> Self {
+        assert_eq!(buf.len(), 3);
+        let mut result = Self::new();
+        result.set_data(buf[0], buf[1], buf[2]);
+        result
+    }
+}
+
+/// Rust representation for BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY Parquet physical types.
+/// Value is backed by a byte buffer.
+#[derive(Clone, Debug)]
+pub struct ByteArray {
+    data: Option<ByteBufferPtr>,
+}
+
+impl ByteArray {
+    /// Creates new byte array with no data set.
+    pub fn new() -> Self {
+        ByteArray { data: None }
+    }
+
+    /// Gets length of the underlying byte buffer.
+    pub fn len(&self) -> usize {
+        assert!(self.data.is_some());
+        self.data.as_ref().unwrap().len()
+    }
+
+    /// Returns slice of data.
+    pub fn data(&self) -> &[u8] {
+        assert!(self.data.is_some());
+        self.data.as_ref().unwrap().as_ref()
+    }
+
+    /// Set data from another byte buffer.
+    pub fn set_data(&mut self, data: ByteBufferPtr) {
+        self.data = Some(data);
+    }
+
+    /// Returns `ByteArray` instance with slice of values for a data.
+    pub fn slice(&self, start: usize, len: usize) -> Self {
+        assert!(self.data.is_some());
+        Self::from(self.data.as_ref().unwrap().range(start, len))
+    }
+}
+
+impl From<Vec<u8>> for ByteArray {
+    fn from(buf: Vec<u8>) -> ByteArray {
+        Self {
+            data: Some(ByteBufferPtr::new(buf)),
+        }
+    }
+}
+
+impl<'a> From<&'a str> for ByteArray {
+    fn from(s: &'a str) -> ByteArray {
+        let mut v = Vec::new();
+        v.extend_from_slice(s.as_bytes());
+        Self {
+            data: Some(ByteBufferPtr::new(v)),
+        }
+    }
+}
+
+impl From<ByteBufferPtr> for ByteArray {
+    fn from(ptr: ByteBufferPtr) -> ByteArray {
+        Self { data: Some(ptr) }
+    }
+}
+
+impl From<ByteBuffer> for ByteArray {
+    fn from(mut buf: ByteBuffer) -> ByteArray {
+        Self {
+            data: Some(buf.consume()),
+        }
+    }
+}
+
+impl Default for ByteArray {
+    fn default() -> Self {
+        ByteArray { data: None }
+    }
+}
+
+impl PartialEq for ByteArray {
+    fn eq(&self, other: &ByteArray) -> bool {
+        self.data() == other.data()
+    }
+}
+
+/// Rust representation for Decimal values.
+///
+/// This is not a representation of Parquet physical type, but rather a wrapper for
+/// DECIMAL logical type, and serves as container for raw parts of decimal values:
+/// unscaled value in bytes, precision and scale.
+#[derive(Clone, Debug)]
+pub enum Decimal {
+    /// Decimal backed by `i32`.
+    Int32 {
+        value: [u8; 4],
+        precision: i32,
+        scale: i32,
+    },
+    /// Decimal backed by `i64`.
+    Int64 {
+        value: [u8; 8],
+        precision: i32,
+        scale: i32,
+    },
+    /// Decimal backed by byte array.
+    Bytes {
+        value: ByteArray,
+        precision: i32,
+        scale: i32,
+    },
+}
+
+impl Decimal {
+    /// Creates new decimal value from `i32`.
+    pub fn from_i32(value: i32, precision: i32, scale: i32) -> Self {
+        let mut bytes = [0; 4];
+        BigEndian::write_i32(&mut bytes, value);
+        Decimal::Int32 {
+            value: bytes,
+            precision,
+            scale,
+        }
+    }
+
+    /// Creates new decimal value from `i64`.
+    pub fn from_i64(value: i64, precision: i32, scale: i32) -> Self {
+        let mut bytes = [0; 8];
+        BigEndian::write_i64(&mut bytes, value);
+        Decimal::Int64 {
+            value: bytes,
+            precision,
+            scale,
+        }
+    }
+
+    /// Creates new decimal value from `ByteArray`.
+    pub fn from_bytes(value: ByteArray, precision: i32, scale: i32) -> Self {
+        Decimal::Bytes {
+            value,
+            precision,
+            scale,
+        }
+    }
+
+    /// Returns bytes of unscaled value.
+    pub fn data(&self) -> &[u8] {
+        match *self {
+            Decimal::Int32 { ref value, .. } => value,
+            Decimal::Int64 { ref value, .. } => value,
+            Decimal::Bytes { ref value, .. } => value.data(),
+        }
+    }
+
+    /// Returns decimal precision.
+    pub fn precision(&self) -> i32 {
+        match *self {
+            Decimal::Int32 { precision, .. } => precision,
+            Decimal::Int64 { precision, .. } => precision,
+            Decimal::Bytes { precision, .. } => precision,
+        }
+    }
+
+    /// Returns decimal scale.
+    pub fn scale(&self) -> i32 {
+        match *self {
+            Decimal::Int32 { scale, .. } => scale,
+            Decimal::Int64 { scale, .. } => scale,
+            Decimal::Bytes { scale, .. } => scale,
+        }
+    }
+}
+
+impl Default for Decimal {
+    fn default() -> Self {
+        Self::from_i32(0, 0, 0)
+    }
+}
+
+impl PartialEq for Decimal {
+    fn eq(&self, other: &Decimal) -> bool {
+        self.precision() == other.precision()
+            && self.scale() == other.scale()
+            && self.data() == other.data()
+    }
+}
+
+/// Converts an instance of data type to a slice of bytes as `u8`.
+pub trait AsBytes {
+    /// Returns slice of bytes for this data type.
+    fn as_bytes(&self) -> &[u8];
+}
+
+macro_rules! gen_as_bytes {
+    ($source_ty:ident) => {
+        impl AsBytes for $source_ty {
+            fn as_bytes(&self) -> &[u8] {
+                unsafe {
+                    ::std::slice::from_raw_parts(
+                        self as *const $source_ty as *const u8,
+                        ::std::mem::size_of::<$source_ty>(),
+                    )
+                }
+            }
+        }
+    };
+}
+
+gen_as_bytes!(bool);
+gen_as_bytes!(u8);
+gen_as_bytes!(i32);
+gen_as_bytes!(u32);
+gen_as_bytes!(i64);
+gen_as_bytes!(f32);
+gen_as_bytes!(f64);
+
+impl AsBytes for Int96 {
+    fn as_bytes(&self) -> &[u8] {
+        unsafe { ::std::slice::from_raw_parts(self.data() as *const [u32] as *const u8, 12) }
+    }
+}
+
+impl AsBytes for ByteArray {
+    fn as_bytes(&self) -> &[u8] {
+        self.data()
+    }
+}
+
+impl AsBytes for Decimal {
+    fn as_bytes(&self) -> &[u8] {
+        self.data()
+    }
+}
+
+impl AsBytes for Vec<u8> {
+    fn as_bytes(&self) -> &[u8] {
+        self.as_slice()
+    }
+}
+
+impl<'a> AsBytes for &'a str {
+    fn as_bytes(&self) -> &[u8] {
+        (self as &str).as_bytes()
+    }
+}
+
+impl AsBytes for str {
+    fn as_bytes(&self) -> &[u8] {
+        (self as &str).as_bytes()
+    }
+}
+
+/// Contains the Parquet physical type information as well as the Rust primitive type
+/// presentation.
+pub trait DataType: 'static {
+    type T: ::std::cmp::PartialEq
+        + ::std::fmt::Debug
+        + ::std::default::Default
+        + ::std::clone::Clone
+        + AsBytes;
+
+    /// Returns Parquet physical type.
+    fn get_physical_type() -> Type;
+
+    /// Returns size in bytes for Rust representation of the physical type.
+    fn get_type_size() -> usize;
+}
+
+macro_rules! make_type {
+    ($name:ident, $physical_ty:path, $native_ty:ty, $size:expr) => {
+        pub struct $name {}
+
+        impl DataType for $name {
+            type T = $native_ty;
+
+            fn get_physical_type() -> Type {
+                $physical_ty
+            }
+
+            fn get_type_size() -> usize {
+                $size
+            }
+        }
+    };
+}
+
+/// Generate struct definitions for all physical types
+
+make_type!(BoolType, Type::BOOLEAN, bool, 1);
+make_type!(Int32Type, Type::INT32, i32, 4);
+make_type!(Int64Type, Type::INT64, i64, 8);
+make_type!(Int96Type, Type::INT96, Int96, mem::size_of::<Int96>());
+make_type!(FloatType, Type::FLOAT, f32, 4);
+make_type!(DoubleType, Type::DOUBLE, f64, 8);
+make_type!(
+    ByteArrayType,
+    Type::BYTE_ARRAY,
+    ByteArray,
+    mem::size_of::<ByteArray>()
+);
+make_type!(
+    FixedLenByteArrayType,
+    Type::FIXED_LEN_BYTE_ARRAY,
+    ByteArray,
+    mem::size_of::<ByteArray>()
+);
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_as_bytes() {
+        assert_eq!(false.as_bytes(), &[0]);
+        assert_eq!(true.as_bytes(), &[1]);
+        assert_eq!((7 as i32).as_bytes(), &[7, 0, 0, 0]);
+        assert_eq!((555 as i32).as_bytes(), &[43, 2, 0, 0]);
+        assert_eq!((555 as u32).as_bytes(), &[43, 2, 0, 0]);
+        assert_eq!(i32::max_value().as_bytes(), &[255, 255, 255, 127]);
+        assert_eq!(i32::min_value().as_bytes(), &[0, 0, 0, 128]);
+        assert_eq!((7 as i64).as_bytes(), &[7, 0, 0, 0, 0, 0, 0, 0]);
+        assert_eq!((555 as i64).as_bytes(), &[43, 2, 0, 0, 0, 0, 0, 0]);
+        assert_eq!(
+            (i64::max_value()).as_bytes(),
+            &[255, 255, 255, 255, 255, 255, 255, 127]
+        );
+        assert_eq!((i64::min_value()).as_bytes(), &[0, 0, 0, 0, 0, 0, 0, 128]);
+        assert_eq!((3.14 as f32).as_bytes(), &[195, 245, 72, 64]);
+        assert_eq!(
+            (3.14 as f64).as_bytes(),
+            &[31, 133, 235, 81, 184, 30, 9, 64]
+        );
+        assert_eq!("hello".as_bytes(), &[b'h', b'e', b'l', b'l', b'o']);
+        assert_eq!(
+            Vec::from("hello".as_bytes()).as_bytes(),
+            &[b'h', b'e', b'l', b'l', b'o']
+        );
+
+        // Test Int96
+        let i96 = Int96::from(vec![1, 2, 3]);
+        assert_eq!(i96.as_bytes(), &[1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0]);
+
+        // Test ByteArray
+        let ba = ByteArray::from(vec![1, 2, 3]);
+        assert_eq!(ba.as_bytes(), &[1, 2, 3]);
+
+        // Test Decimal
+        let decimal = Decimal::from_i32(123, 5, 2);
+        assert_eq!(decimal.as_bytes(), &[0, 0, 0, 123]);
+        let decimal = Decimal::from_i64(123, 5, 2);
+        assert_eq!(decimal.as_bytes(), &[0, 0, 0, 0, 0, 0, 0, 123]);
+        let decimal = Decimal::from_bytes(ByteArray::from(vec![1, 2, 3]), 5, 2);
+        assert_eq!(decimal.as_bytes(), &[1, 2, 3]);
+    }
+
+    #[test]
+    fn test_int96_from() {
+        assert_eq!(
+            Int96::from(vec![1, 12345, 1234567890]).data(),
+            &[1, 12345, 1234567890]
+        );
+    }
+
+    #[test]
+    fn test_byte_array_from() {
+        assert_eq!(
+            ByteArray::from(vec![b'A', b'B', b'C']).data(),
+            &[b'A', b'B', b'C']
+        );
+        assert_eq!(ByteArray::from("ABC").data(), &[b'A', b'B', b'C']);
+        assert_eq!(
+            ByteArray::from(ByteBufferPtr::new(vec![1u8, 2u8, 3u8, 4u8, 5u8])).data(),
+            &[1u8, 2u8, 3u8, 4u8, 5u8]
+        );
+        let mut buf = ByteBuffer::new();
+        buf.set_data(vec![6u8, 7u8, 8u8, 9u8, 10u8]);
+        assert_eq!(ByteArray::from(buf).data(), &[6u8, 7u8, 8u8, 9u8, 10u8]);
+    }
+
+    #[test]
+    fn test_decimal_partial_eq() {
+        assert_eq!(Decimal::default(), Decimal::from_i32(0, 0, 0));
+        assert_eq!(Decimal::from_i32(222, 5, 2), Decimal::from_i32(222, 5, 2));
+        assert_eq!(
+            Decimal::from_bytes(ByteArray::from(vec![0, 0, 0, 3]), 5, 2),
+            Decimal::from_i32(3, 5, 2)
+        );
+
+        assert!(Decimal::from_i32(222, 5, 2) != Decimal::from_i32(111, 5, 2));
+        assert!(Decimal::from_i32(222, 5, 2) != Decimal::from_i32(222, 6, 2));
+        assert!(Decimal::from_i32(222, 5, 2) != Decimal::from_i32(222, 5, 3));
+
+        assert!(Decimal::from_i64(222, 5, 2) != Decimal::from_i32(222, 5, 2));
+    }
+}

rust/src/parquet/encodings/levels.rs

@@ -0,0 +1,529 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::{cmp, mem};
+
+use super::rle::{RleDecoder, RleEncoder};
+
+use crate::parquet::basic::Encoding;
+use crate::parquet::data_type::AsBytes;
+use crate::parquet::errors::{ParquetError, Result};
+use crate::parquet::util::{
+    bit_util::{ceil, log2, BitReader, BitWriter},
+    memory::ByteBufferPtr,
+};
+
+/// Computes max buffer size for level encoder/decoder based on encoding, max
+/// repetition/definition level and number of total buffered values (includes null
+/// values).
+#[inline]
+pub fn max_buffer_size(encoding: Encoding, max_level: i16, num_buffered_values: usize) -> usize {
+    let bit_width = log2(max_level as u64 + 1) as u8;
+    match encoding {
+        Encoding::RLE => {
+            RleEncoder::max_buffer_size(bit_width, num_buffered_values)
+                + RleEncoder::min_buffer_size(bit_width)
+        }
+        Encoding::BIT_PACKED => ceil((num_buffered_values * bit_width as usize) as i64, 8) as usize,
+        _ => panic!("Unsupported encoding type {}", encoding),
+    }
+}
+
+/// Encoder for definition/repetition levels.
+/// Currently only supports RLE and BIT_PACKED (dev/null) encoding, including v2.
+pub enum LevelEncoder {
+    RLE(RleEncoder),
+    RLE_V2(RleEncoder),
+    BIT_PACKED(u8, BitWriter),
+}
+
+impl LevelEncoder {
+    /// Creates new level encoder based on encoding, max level and underlying byte buffer.
+    /// For bit packed encoding it is assumed that buffer is already allocated with
+    /// `levels::max_buffer_size` method.
+    ///
+    /// Used to encode levels for Data Page v1.
+    ///
+    /// Panics, if encoding is not supported.
+    pub fn v1(encoding: Encoding, max_level: i16, byte_buffer: Vec<u8>) -> Self {
+        let bit_width = log2(max_level as u64 + 1) as u8;
+        match encoding {
+            Encoding::RLE => LevelEncoder::RLE(RleEncoder::new_from_buf(
+                bit_width,
+                byte_buffer,
+                mem::size_of::<i32>(),
+            )),
+            Encoding::BIT_PACKED => {
+                // Here we set full byte buffer without adjusting for num_buffered_values,
+                // because byte buffer will already be allocated with size from
+                // `max_buffer_size()` method.
+                LevelEncoder::BIT_PACKED(bit_width, BitWriter::new_from_buf(byte_buffer, 0))
+            }
+            _ => panic!("Unsupported encoding type {}", encoding),
+        }
+    }
+
+    /// Creates new level encoder based on RLE encoding. Used to encode Data Page v2
+    /// repetition and definition levels.
+    pub fn v2(max_level: i16, byte_buffer: Vec<u8>) -> Self {
+        let bit_width = log2(max_level as u64 + 1) as u8;
+        LevelEncoder::RLE_V2(RleEncoder::new_from_buf(bit_width, byte_buffer, 0))
+    }
+
+    /// Put/encode levels vector into this level encoder.
+    /// Returns number of encoded values that are less than or equal to length of the input
+    /// buffer.
+    ///
+    /// RLE and BIT_PACKED level encoders return Err() when internal buffer overflows or
+    /// flush fails.
+    #[inline]
+    pub fn put(&mut self, buffer: &[i16]) -> Result<usize> {
+        let mut num_encoded = 0;
+        match *self {
+            LevelEncoder::RLE(ref mut encoder) | LevelEncoder::RLE_V2(ref mut encoder) => {
+                for value in buffer {
+                    if !encoder.put(*value as u64)? {
+                        return Err(general_err!("RLE buffer is full"));
+                    }
+                    num_encoded += 1;
+                }
+                encoder.flush()?;
+            }
+            LevelEncoder::BIT_PACKED(bit_width, ref mut encoder) => {
+                for value in buffer {
+                    if !encoder.put_value(*value as u64, bit_width as usize) {
+                        return Err(general_err!("Not enough bytes left"));
+                    }
+                    num_encoded += 1;
+                }
+                encoder.flush();
+            }
+        }
+        Ok(num_encoded)
+    }
+
+    /// Finalizes level encoder, flush all intermediate buffers and return resulting
+    /// encoded buffer. Returned buffer is already truncated to encoded bytes only.
+    #[inline]
+    pub fn consume(self) -> Result<Vec<u8>> {
+        match self {
+            LevelEncoder::RLE(encoder) => {
+                let mut encoded_data = encoder.consume()?;
+                // Account for the buffer offset
+                let encoded_len = encoded_data.len() - mem::size_of::<i32>();
+                let len = (encoded_len as i32).to_le();
+                let len_bytes = len.as_bytes();
+                encoded_data[0..len_bytes.len()].copy_from_slice(len_bytes);
+                Ok(encoded_data)
+            }
+            LevelEncoder::RLE_V2(encoder) => encoder.consume(),
+            LevelEncoder::BIT_PACKED(_, encoder) => Ok(encoder.consume()),
+        }
+    }
+}
+
+/// Decoder for definition/repetition levels.
+/// Currently only supports RLE and BIT_PACKED encoding for Data Page v1 and
+/// RLE for Data Page v2.
+pub enum LevelDecoder {
+    RLE(Option<usize>, RleDecoder),
+    RLE_V2(Option<usize>, RleDecoder),
+    BIT_PACKED(Option<usize>, u8, BitReader),
+}
+
+impl LevelDecoder {
+    /// Creates new level decoder based on encoding and max definition/repetition level.
+    /// This method only initializes level decoder, `set_data` method must be called
+    /// before reading any value.
+    ///
+    /// Used to encode levels for Data Page v1.
+    ///
+    /// Panics if encoding is not supported
+    pub fn v1(encoding: Encoding, max_level: i16) -> Self {
+        let bit_width = log2(max_level as u64 + 1) as u8;
+        match encoding {
+            Encoding::RLE => LevelDecoder::RLE(None, RleDecoder::new(bit_width)),
+            Encoding::BIT_PACKED => {
+                LevelDecoder::BIT_PACKED(None, bit_width, BitReader::from(Vec::new()))
+            }
+            _ => panic!("Unsupported encoding type {}", encoding),
+        }
+    }
+
+    /// Creates new level decoder based on RLE encoding.
+    /// Used to decode Data Page v2 repetition and definition levels.
+    ///
+    /// To set data for this decoder, use `set_data_range` method.
+    pub fn v2(max_level: i16) -> Self {
+        let bit_width = log2(max_level as u64 + 1) as u8;
+        LevelDecoder::RLE_V2(None, RleDecoder::new(bit_width))
+    }
+
+    /// Sets data for this level decoder, and returns total number of bytes set.
+    /// This is used for Data Page v1 levels.
+    ///
+    /// `data` is encoded data as byte buffer, `num_buffered_values` represents total
+    /// number of values that is expected.
+    ///
+    /// Both RLE and BIT_PACKED level decoders set `num_buffered_values` as total number of
+    /// values that they can return and track num values.
+    #[inline]
+    pub fn set_data(&mut self, num_buffered_values: usize, data: ByteBufferPtr) -> usize {
+        match *self {
+            LevelDecoder::RLE(ref mut num_values, ref mut decoder) => {
+                *num_values = Some(num_buffered_values);
+                let i32_size = mem::size_of::<i32>();
+                let data_size = read_num_bytes!(i32, i32_size, data.as_ref()) as usize;
+                decoder.set_data(data.range(i32_size, data_size));
+                i32_size + data_size
+            }
+            LevelDecoder::BIT_PACKED(ref mut num_values, bit_width, ref mut decoder) => {
+                *num_values = Some(num_buffered_values);
+                // Set appropriate number of bytes: if max size is larger than buffer - set full
+                // buffer
+                let num_bytes = ceil((num_buffered_values * bit_width as usize) as i64, 8);
+                let data_size = cmp::min(num_bytes as usize, data.len());
+                decoder.reset(data.range(data.start(), data_size));
+                data_size
+            }
+            _ => panic!(),
+        }
+    }
+
+    /// Sets byte array explicitly when start position `start` and length `len` are known
+    /// in advance. Only supported by RLE level decoder and used for Data Page v2 levels.
+    /// Returns number of total bytes set for this decoder (len).
+    #[inline]
+    pub fn set_data_range(
+        &mut self,
+        num_buffered_values: usize,
+        data: &ByteBufferPtr,
+        start: usize,
+        len: usize,
+    ) -> usize {
+        match *self {
+            LevelDecoder::RLE_V2(ref mut num_values, ref mut decoder) => {
+                decoder.set_data(data.range(start, len));
+                *num_values = Some(num_buffered_values);
+                len
+            }
+            _ => panic!("set_data_range() method is only supported by RLE v2 encoding type"),
+        }
+    }
+
+    /// Returns true if data is set for decoder, false otherwise.
+    #[inline]
+    pub fn is_data_set(&self) -> bool {
+        match self {
+            LevelDecoder::RLE(ref num_values, _) => num_values.is_some(),
+            LevelDecoder::RLE_V2(ref num_values, _) => num_values.is_some(),
+            LevelDecoder::BIT_PACKED(ref num_values, ..) => num_values.is_some(),
+        }
+    }
+
+    /// Decodes values and puts them into `buffer`.
+    /// Returns number of values that were successfully decoded (less than or equal to
+    /// buffer length).
+    #[inline]
+    pub fn get(&mut self, buffer: &mut [i16]) -> Result<usize> {
+        assert!(self.is_data_set(), "No data set for decoding");
+        match *self {
+            LevelDecoder::RLE(ref mut num_values, ref mut decoder)
+            | LevelDecoder::RLE_V2(ref mut num_values, ref mut decoder) => {
+                // Max length we can read
+                let len = cmp::min(num_values.unwrap(), buffer.len());
+                let values_read = decoder.get_batch::<i16>(&mut buffer[0..len])?;
+                *num_values = num_values.map(|len| len - values_read);
+                Ok(values_read)
+            }
+            LevelDecoder::BIT_PACKED(ref mut num_values, bit_width, ref mut decoder) => {
+                // When extracting values from bit reader, it might return more values than left
+                // because of padding to a full byte, we use num_values to track precise number
+                // of values.
+                let len = cmp::min(num_values.unwrap(), buffer.len());
+                let values_read = decoder.get_batch::<i16>(&mut buffer[..len], bit_width as usize);
+                *num_values = num_values.map(|len| len - values_read);
+                Ok(values_read)
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use crate::parquet::util::test_common::random_numbers_range;
+
+    fn test_internal_roundtrip(enc: Encoding, levels: &[i16], max_level: i16, v2: bool) {
+        let size = max_buffer_size(enc, max_level, levels.len());
+        let mut encoder = if v2 {
+            LevelEncoder::v2(max_level, vec![0; size])
+        } else {
+            LevelEncoder::v1(enc, max_level, vec![0; size])
+        };
+        encoder.put(&levels).expect("put() should be OK");
+        let encoded_levels = encoder.consume().expect("consume() should be OK");
+
+        let byte_buf = ByteBufferPtr::new(encoded_levels);
+        let mut decoder;
+        if v2 {
+            decoder = LevelDecoder::v2(max_level);
+            decoder.set_data_range(levels.len(), &byte_buf, 0, byte_buf.len());
+        } else {
+            decoder = LevelDecoder::v1(enc, max_level);
+            decoder.set_data(levels.len(), byte_buf);
+        };
+
+        let mut buffer = vec![0; levels.len()];
+        let num_decoded = decoder.get(&mut buffer).expect("get() should be OK");
+        assert_eq!(num_decoded, levels.len());
+        assert_eq!(buffer, levels);
+    }
+
+    // Performs incremental read until all bytes are read
+    fn test_internal_roundtrip_incremental(
+        enc: Encoding,
+        levels: &[i16],
+        max_level: i16,
+        v2: bool,
+    ) {
+        let size = max_buffer_size(enc, max_level, levels.len());
+        let mut encoder = if v2 {
+            LevelEncoder::v2(max_level, vec![0; size])
+        } else {
+            LevelEncoder::v1(enc, max_level, vec![0; size])
+        };
+        encoder.put(&levels).expect("put() should be OK");
+        let encoded_levels = encoder.consume().expect("consume() should be OK");
+
+        let byte_buf = ByteBufferPtr::new(encoded_levels);
+        let mut decoder;
+        if v2 {
+            decoder = LevelDecoder::v2(max_level);
+            decoder.set_data_range(levels.len(), &byte_buf, 0, byte_buf.len());
+        } else {
+            decoder = LevelDecoder::v1(enc, max_level);
+            decoder.set_data(levels.len(), byte_buf);
+        }
+
+        let mut buffer = vec![0; levels.len() * 2];
+        let mut total_decoded = 0;
+        let mut safe_stop = levels.len() * 2; // still terminate in case of issues in the code
+        while safe_stop > 0 {
+            safe_stop -= 1;
+            let num_decoded = decoder
+                .get(&mut buffer[total_decoded..total_decoded + 1])
+                .expect("get() should be OK");
+            if num_decoded == 0 {
+                break;
+            }
+            total_decoded += num_decoded;
+        }
+        assert!(
+            safe_stop > 0,
+            "Failed to read values incrementally, reached safe stop"
+        );
+        assert_eq!(total_decoded, levels.len());
+        assert_eq!(&buffer[0..levels.len()], levels);
+    }
+
+    // Tests encoding/decoding of values when output buffer is larger than number of
+    // encoded values
+    fn test_internal_roundtrip_underflow(enc: Encoding, levels: &[i16], max_level: i16, v2: bool) {
+        let size = max_buffer_size(enc, max_level, levels.len());
+        let mut encoder = if v2 {
+            LevelEncoder::v2(max_level, vec![0; size])
+        } else {
+            LevelEncoder::v1(enc, max_level, vec![0; size])
+        };
+        // Encode only one value
+        let num_encoded = encoder.put(&levels[0..1]).expect("put() should be OK");
+        let encoded_levels = encoder.consume().expect("consume() should be OK");
+        assert_eq!(num_encoded, 1);
+
+        let byte_buf = ByteBufferPtr::new(encoded_levels);
+        let mut decoder;
+        // Set one encoded value as `num_buffered_values`
+        if v2 {
+            decoder = LevelDecoder::v2(max_level);
+            decoder.set_data_range(1, &byte_buf, 0, byte_buf.len());
+        } else {
+            decoder = LevelDecoder::v1(enc, max_level);
+            decoder.set_data(1, byte_buf);
+        }
+
+        let mut buffer = vec![0; levels.len()];
+        let num_decoded = decoder.get(&mut buffer).expect("get() should be OK");
+        assert_eq!(num_decoded, num_encoded);
+        assert_eq!(buffer[0..num_decoded], levels[0..num_decoded]);
+    }
+
+    // Tests when encoded values are larger than encoder's buffer
+    fn test_internal_roundtrip_overflow(enc: Encoding, levels: &[i16], max_level: i16, v2: bool) {
+        let size = max_buffer_size(enc, max_level, levels.len());
+        let mut encoder = if v2 {
+            LevelEncoder::v2(max_level, vec![0; size])
+        } else {
+            LevelEncoder::v1(enc, max_level, vec![0; size])
+        };
+        let mut found_err = false;
+        // Insert a large number of values, so we run out of space
+        for _ in 0..100 {
+            match encoder.put(&levels) {
+                Err(err) => {
+                    assert!(format!("{}", err).contains("Not enough bytes left"));
+                    found_err = true;
+                    break;
+                }
+                Ok(_) => {}
+            }
+        }
+        if !found_err {
+            panic!("Failed test: no buffer overflow");
+        }
+    }
+
+    #[test]
+    fn test_roundtrip_one() {
+        let levels = vec![0, 1, 1, 1, 1, 0, 0, 0, 0, 1];
+        let max_level = 1;
+        test_internal_roundtrip(Encoding::RLE, &levels, max_level, false);
+        test_internal_roundtrip(Encoding::BIT_PACKED, &levels, max_level, false);
+        test_internal_roundtrip(Encoding::RLE, &levels, max_level, true);
+    }
+
+    #[test]
+    fn test_roundtrip() {
+        let levels = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9];
+        let max_level = 10;
+        test_internal_roundtrip(Encoding::RLE, &levels, max_level, false);
+        test_internal_roundtrip(Encoding::BIT_PACKED, &levels, max_level, false);
+        test_internal_roundtrip(Encoding::RLE, &levels, max_level, true);
+    }
+
+    #[test]
+    fn test_roundtrip_incremental() {
+        let levels = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9];
+        let max_level = 10;
+        test_internal_roundtrip_incremental(Encoding::RLE, &levels, max_level, false);
+        test_internal_roundtrip_incremental(Encoding::BIT_PACKED, &levels, max_level, false);
+        test_internal_roundtrip_incremental(Encoding::RLE, &levels, max_level, true);
+    }
+
+    #[test]
+    fn test_roundtrip_all_zeros() {
+        let levels = vec![0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
+        let max_level = 1;
+        test_internal_roundtrip(Encoding::RLE, &levels, max_level, false);
+        test_internal_roundtrip(Encoding::BIT_PACKED, &levels, max_level, false);
+        test_internal_roundtrip(Encoding::RLE, &levels, max_level, true);
+    }
+
+    #[test]
+    fn test_roundtrip_random() {
+        // This test is mainly for bit packed level encoder/decoder
+        let mut levels = Vec::new();
+        let max_level = 5;
+        random_numbers_range::<i16>(120, 0, max_level, &mut levels);
+        test_internal_roundtrip(Encoding::RLE, &levels, max_level, false);
+        test_internal_roundtrip(Encoding::BIT_PACKED, &levels, max_level, false);
+        test_internal_roundtrip(Encoding::RLE, &levels, max_level, true);
+    }
+
+    #[test]
+    fn test_roundtrip_underflow() {
+        let levels = vec![1, 1, 2, 3, 2, 1, 1, 2, 3, 1];
+        let max_level = 3;
+        test_internal_roundtrip_underflow(Encoding::RLE, &levels, max_level, false);
+        test_internal_roundtrip_underflow(Encoding::BIT_PACKED, &levels, max_level, false);
+        test_internal_roundtrip_underflow(Encoding::RLE, &levels, max_level, true);
+    }
+
+    #[test]
+    fn test_roundtrip_overflow() {
+        let levels = vec![1, 1, 2, 3, 2, 1, 1, 2, 3, 1];
+        let max_level = 3;
+        test_internal_roundtrip_overflow(Encoding::RLE, &levels, max_level, false);
+        test_internal_roundtrip_overflow(Encoding::BIT_PACKED, &levels, max_level, false);
+        test_internal_roundtrip_overflow(Encoding::RLE, &levels, max_level, true);
+    }
+
+    #[test]
+    fn test_rle_decoder_set_data_range() {
+        // Buffer containing both repetition and definition levels
+        let buffer = ByteBufferPtr::new(vec![5, 198, 2, 5, 42, 168, 10, 0, 2, 3, 36, 73]);
+
+        let max_rep_level = 1;
+        let mut decoder = LevelDecoder::v2(max_rep_level);
+        assert_eq!(decoder.set_data_range(10, &buffer, 0, 3), 3);
+        let mut result = vec![0; 10];
+        let num_decoded = decoder.get(&mut result).expect("get() should be OK");
+        assert_eq!(num_decoded, 10);
+        assert_eq!(result, vec![0, 1, 1, 0, 0, 0, 1, 1, 0, 1]);
+
+        let max_def_level = 2;
+        let mut decoder = LevelDecoder::v2(max_def_level);
+        assert_eq!(decoder.set_data_range(10, &buffer, 3, 5), 5);
+        let mut result = vec![0; 10];
+        let num_decoded = decoder.get(&mut result).expect("get() should be OK");
+        assert_eq!(num_decoded, 10);
+        assert_eq!(result, vec![2, 2, 2, 0, 0, 2, 2, 2, 2, 2]);
+    }
+
+    #[test]
+    #[should_panic(expected = "set_data_range() method is only supported by RLE v2 encoding type")]
+    fn test_bit_packed_decoder_set_data_range() {
+        // Buffer containing both repetition and definition levels
+        let buffer = ByteBufferPtr::new(vec![1, 2, 3, 4, 5]);
+        let max_level = 1;
+        let mut decoder = LevelDecoder::v1(Encoding::BIT_PACKED, max_level);
+        decoder.set_data_range(10, &buffer, 0, 3);
+    }
+
+    #[test]
+    fn test_bit_packed_decoder_set_data() {
+        // Test the maximum size that is assigned based on number of values and buffer length
+        let buffer = ByteBufferPtr::new(vec![1, 2, 3, 4, 5]);
+        let max_level = 1;
+        let mut decoder = LevelDecoder::v1(Encoding::BIT_PACKED, max_level);
+        // This should reset to entire buffer
+        assert_eq!(decoder.set_data(1024, buffer.all()), buffer.len());
+        // This should set smallest num bytes
+        assert_eq!(decoder.set_data(3, buffer.all()), 1);
+    }
+
+    #[test]
+    #[should_panic(expected = "No data set for decoding")]
+    fn test_rle_level_decoder_get_no_set_data() {
+        // `get()` normally panics because bit_reader is not set for RLE decoding
+        // we have explicit check now in set_data
+        let max_rep_level = 2;
+        let mut decoder = LevelDecoder::v1(Encoding::RLE, max_rep_level);
+        let mut buffer = vec![0; 16];
+        decoder.get(&mut buffer).unwrap();
+    }
+
+    #[test]
+    #[should_panic(expected = "No data set for decoding")]
+    fn test_bit_packed_level_decoder_get_no_set_data() {
+        let max_rep_level = 2;
+        let mut decoder = LevelDecoder::v1(Encoding::BIT_PACKED, max_rep_level);
+        let mut buffer = vec![0; 16];
+        decoder.get(&mut buffer).unwrap();
+    }
+}

rust/src/parquet/encodings/mod.rs

@@ -0,0 +1,21 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+pub mod decoding;
+pub mod encoding;
+pub mod levels;
+mod rle;

rust/src/parquet/encodings/rle.rs

@@ -0,0 +1,839 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::{
+    cmp,
+    mem::{size_of, transmute_copy},
+};
+
+use crate::parquet::errors::{ParquetError, Result};
+use crate::parquet::util::{
+    bit_util::{self, BitReader, BitWriter},
+    memory::ByteBufferPtr,
+};
+
+/// Rle/Bit-Packing Hybrid Encoding
+/// The grammar for this encoding looks like the following (copied verbatim
+/// from https://github.com/Parquet/parquet-format/blob/master/Encodings.md):
+///
+/// rle-bit-packed-hybrid: <length> <encoded-data>
+/// length := length of the <encoded-data> in bytes stored as 4 bytes little endian
+/// encoded-data := <run>*
+/// run := <bit-packed-run> | <rle-run>
+/// bit-packed-run := <bit-packed-header> <bit-packed-values>
+/// bit-packed-header := varint-encode(<bit-pack-count> << 1 | 1)
+/// we always bit-pack a multiple of 8 values at a time, so we only store the number of
+/// values / 8
+/// bit-pack-count := (number of values in this run) / 8
+/// bit-packed-values := *see 1 below*
+/// rle-run := <rle-header> <repeated-value>
+/// rle-header := varint-encode( (number of times repeated) << 1)
+/// repeated-value := value that is repeated, using a fixed-width of
+/// round-up-to-next-byte(bit-width)
+
+/// Maximum groups per bit-packed run. Current value is 64.
+const MAX_GROUPS_PER_BIT_PACKED_RUN: usize = 1 << 6;
+const MAX_VALUES_PER_BIT_PACKED_RUN: usize = MAX_GROUPS_PER_BIT_PACKED_RUN * 8;
+const MAX_WRITER_BUF_SIZE: usize = 1 << 10;
+
+/// A RLE/Bit-Packing hybrid encoder.
+// TODO: tracking memory usage
+pub struct RleEncoder {
+    // Number of bits needed to encode the value. Must be in the range of [0, 64].
+    bit_width: u8,
+
+    // Underlying writer which holds an internal buffer.
+    bit_writer: BitWriter,
+
+    // If this is true, the buffer is full and subsequent `put()` calls will fail.
+    buffer_full: bool,
+
+    // The maximum byte size a single run can take.
+    max_run_byte_size: usize,
+
+    // Buffered values for bit-packed runs.
+    buffered_values: [u64; 8],
+
+    // Number of current buffered values. Must be less than 8.
+    num_buffered_values: usize,
+
+    // The current (also last) value that was written and the count of how many
+    // times in a row that value has been seen.
+    current_value: u64,
+
+    // The number of repetitions for `current_value`. If this gets too high we'd
+    // switch to use RLE encoding.
+    repeat_count: usize,
+
+    // Number of bit-packed values in the current run. This doesn't include values
+    // in `buffered_values`.
+    bit_packed_count: usize,
+
+    // The position of the indicator byte in the `bit_writer`.
+    indicator_byte_pos: i64,
+}
+
+impl RleEncoder {
+    pub fn new(bit_width: u8, buffer_len: usize) -> Self {
+        let buffer = vec![0; buffer_len];
+        RleEncoder::new_from_buf(bit_width, buffer, 0)
+    }
+
+    /// Initialize the encoder from existing `buffer` and the starting offset `start`.
+    pub fn new_from_buf(bit_width: u8, buffer: Vec<u8>, start: usize) -> Self {
+        assert!(bit_width <= 64, "bit_width ({}) out of range.", bit_width);
+        let max_run_byte_size = RleEncoder::min_buffer_size(bit_width);
+        assert!(
+            buffer.len() >= max_run_byte_size,
+            "buffer length {} must be greater than {}",
+            buffer.len(),
+            max_run_byte_size
+        );
+        let bit_writer = BitWriter::new_from_buf(buffer, start);
+        RleEncoder {
+            bit_width,
+            bit_writer,
+            buffer_full: false,
+            max_run_byte_size,
+            buffered_values: [0; 8],
+            num_buffered_values: 0,
+            current_value: 0,
+            repeat_count: 0,
+            bit_packed_count: 0,
+            indicator_byte_pos: -1,
+        }
+    }
+
+    /// Returns the minimum buffer size needed to use the encoder for `bit_width`.
+    /// This is the maximum length of a single run for `bit_width`.
+    pub fn min_buffer_size(bit_width: u8) -> usize {
+        let max_bit_packed_run_size = 1 + bit_util::ceil(
+            (MAX_VALUES_PER_BIT_PACKED_RUN * bit_width as usize) as i64,
+            8,
+        );
+        let max_rle_run_size =
+            bit_util::MAX_VLQ_BYTE_LEN + bit_util::ceil(bit_width as i64, 8) as usize;
+        ::std::cmp::max(max_bit_packed_run_size as usize, max_rle_run_size)
+    }
+
+    /// Returns the maximum buffer size takes to encode `num_values` values with
+    /// `bit_width`.
+    pub fn max_buffer_size(bit_width: u8, num_values: usize) -> usize {
+        // First the maximum size for bit-packed run
+        let bytes_per_run = bit_width;
+        let num_runs = bit_util::ceil(num_values as i64, 8) as usize;
+        let bit_packed_max_size = num_runs + num_runs * bytes_per_run as usize;
+
+        // Second the maximum size for RLE run
+        let min_rle_run_size = 1 + bit_util::ceil(bit_width as i64, 8) as usize;
+        let rle_max_size = bit_util::ceil(num_values as i64, 8) as usize * min_rle_run_size;
+        ::std::cmp::max(bit_packed_max_size, rle_max_size) as usize
+    }
+
+    /// Encodes `value`, which must be representable with `bit_width` bits.
+    /// Returns true if the value fits in buffer, false if it doesn't, or
+    /// error if something is wrong.
+    #[inline]
+    pub fn put(&mut self, value: u64) -> Result<bool> {
+        // This function buffers 8 values at a time. After seeing 8 values, it
+        // decides whether the current run should be encoded in bit-packed or RLE.
+        if self.buffer_full {
+            // The value cannot fit in the current buffer.
+            return Ok(false);
+        }
+        if self.current_value == value {
+            self.repeat_count += 1;
+            if self.repeat_count > 8 {
+                // A continuation of last value. No need to buffer.
+                return Ok(true);
+            }
+        } else {
+            if self.repeat_count >= 8 {
+                // The current RLE run has ended and we've gathered enough. Flush first.
+                assert_eq!(self.bit_packed_count, 0);
+                self.flush_rle_run()?;
+            }
+            self.repeat_count = 1;
+            self.current_value = value;
+        }
+
+        self.buffered_values[self.num_buffered_values] = value;
+        self.num_buffered_values += 1;
+        if self.num_buffered_values == 8 {
+            // Buffered values are full. Flush them.
+            assert_eq!(self.bit_packed_count % 8, 0);
+            self.flush_buffered_values()?;
+        }
+
+        Ok(true)
+    }
+
+    #[inline]
+    pub fn buffer(&self) -> &[u8] {
+        self.bit_writer.buffer()
+    }
+
+    #[inline]
+    pub fn len(&self) -> usize {
+        self.bit_writer.bytes_written()
+    }
+
+    #[inline]
+    pub fn consume(mut self) -> Result<Vec<u8>> {
+        self.flush()?;
+        Ok(self.bit_writer.consume())
+    }
+
+    /// Borrow equivalent of the `consume` method.
+    /// Call `clear()` after invoking this method.
+    #[inline]
+    pub fn flush_buffer(&mut self) -> Result<&[u8]> {
+        self.flush()?;
+        Ok(self.bit_writer.flush_buffer())
+    }
+
+    /// Clears the internal state so this encoder can be reused (e.g., after becoming full).
+    #[inline]
+    pub fn clear(&mut self) {
+        self.bit_writer.clear();
+        self.buffer_full = false;
+        self.num_buffered_values = 0;
+        self.current_value = 0;
+        self.repeat_count = 0;
+        self.bit_packed_count = 0;
+        self.indicator_byte_pos = -1;
+    }
+
+    /// Flushes all remaining values and return the final byte buffer maintained by the
+    /// internal writer.
+    #[inline]
+    pub fn flush(&mut self) -> Result<()> {
+        if self.bit_packed_count > 0 || self.repeat_count > 0 || self.num_buffered_values > 0 {
+            let all_repeat = self.bit_packed_count == 0
+                && (self.repeat_count == self.num_buffered_values || self.num_buffered_values == 0);
+            if self.repeat_count > 0 && all_repeat {
+                self.flush_rle_run()?;
+            } else {
+                // Buffer the last group of bit-packed values to 8 by padding with 0s.
+                if self.num_buffered_values > 0 {
+                    while self.num_buffered_values < 8 {
+                        self.buffered_values[self.num_buffered_values] = 0;
+                        self.num_buffered_values += 1;
+                    }
+                }
+                self.bit_packed_count += self.num_buffered_values;
+                self.flush_bit_packed_run(true)?;
+                self.repeat_count = 0;
+            }
+        }
+        Ok(())
+    }
+
+    #[inline]
+    fn flush_rle_run(&mut self) -> Result<()> {
+        assert!(self.repeat_count > 0);
+        let indicator_value = self.repeat_count << 1 | 0;
+        let mut result = self.bit_writer.put_vlq_int(indicator_value as u64);
+        result &= self.bit_writer.put_aligned(
+            self.current_value,
+            bit_util::ceil(self.bit_width as i64, 8) as usize,
+        );
+        if !result {
+            return Err(general_err!("Failed to write RLE run"));
+        }
+        self.num_buffered_values = 0;
+        self.repeat_count = 0;
+        Ok(())
+    }
+
+    #[inline]
+    fn flush_bit_packed_run(&mut self, update_indicator_byte: bool) -> Result<()> {
+        if self.indicator_byte_pos < 0 {
+            self.indicator_byte_pos = self.bit_writer.skip(1)? as i64;
+        }
+
+        // Write all buffered values as bit-packed literals
+        for i in 0..self.num_buffered_values {
+            let _ = self
+                .bit_writer
+                .put_value(self.buffered_values[i], self.bit_width as usize);
+        }
+        self.num_buffered_values = 0;
+        if update_indicator_byte {
+            // Write the indicator byte to the reserved position in `bit_writer`
+            let num_groups = self.bit_packed_count / 8;
+            let indicator_byte = ((num_groups << 1) | 1) as u8;
+            if !self.bit_writer.put_aligned_offset(
+                indicator_byte,
+                1,
+                self.indicator_byte_pos as usize,
+            ) {
+                return Err(general_err!("Not enough space to write indicator byte"));
+            }
+            self.indicator_byte_pos = -1;
+            self.bit_packed_count = 0;
+        }
+        Ok(())
+    }
+
+    #[inline]
+    fn flush_buffered_values(&mut self) -> Result<()> {
+        if self.repeat_count >= 8 {
+            self.num_buffered_values = 0;
+            if self.bit_packed_count > 0 {
+                // In this case we choose RLE encoding. Flush the current buffered values
+                // as bit-packed encoding.
+                assert_eq!(self.bit_packed_count % 8, 0);
+                self.flush_bit_packed_run(true)?
+            }
+            return Ok(());
+        }
+
+        self.bit_packed_count += self.num_buffered_values;
+        let num_groups = self.bit_packed_count / 8;
+        if num_groups + 1 >= MAX_GROUPS_PER_BIT_PACKED_RUN {
+            // We've reached the maximum value that can be hold in a single bit-packed run.
+            assert!(self.indicator_byte_pos >= 0);
+            self.flush_bit_packed_run(true)?;
+        } else {
+            self.flush_bit_packed_run(false)?;
+        }
+        self.repeat_count = 0;
+        Ok(())
+    }
+}
+
+/// A RLE/Bit-Packing hybrid decoder.
+pub struct RleDecoder {
+    // Number of bits used to encode the value. Must be between [0, 64].
+    bit_width: u8,
+
+    // Bit reader loaded with input buffer.
+    bit_reader: Option<BitReader>,
+
+    // Buffer used when `bit_reader` is not `None`, for batch reading.
+    index_buf: Option<[i32; 1024]>,
+
+    // The remaining number of values in RLE for this run
+    rle_left: u32,
+
+    // The remaining number of values in Bit-Packing for this run
+    bit_packed_left: u32,
+
+    // The current value for the case of RLE mode
+    current_value: Option<u64>,
+}
+
+impl RleDecoder {
+    pub fn new(bit_width: u8) -> Self {
+        RleDecoder {
+            bit_width,
+            rle_left: 0,
+            bit_packed_left: 0,
+            bit_reader: None,
+            index_buf: None,
+            current_value: None,
+        }
+    }
+
+    pub fn set_data(&mut self, data: ByteBufferPtr) {
+        if let Some(ref mut bit_reader) = self.bit_reader {
+            bit_reader.reset(data);
+        } else {
+            self.bit_reader = Some(BitReader::new(data));
+            self.index_buf = Some([0; 1024]);
+        }
+
+        let _ = self.reload();
+    }
+
+    #[inline]
+    pub fn get<T: Default>(&mut self) -> Result<Option<T>> {
+        assert!(size_of::<T>() <= 8);
+
+        while self.rle_left <= 0 && self.bit_packed_left <= 0 {
+            if !self.reload() {
+                return Ok(None);
+            }
+        }
+
+        let value = if self.rle_left > 0 {
+            let rle_value = unsafe {
+                transmute_copy::<u64, T>(
+                    self.current_value
+                        .as_mut()
+                        .expect("current_value should be Some"),
+                )
+            };
+            self.rle_left -= 1;
+            rle_value
+        } else {
+            // self.bit_packed_left > 0
+            let bit_reader = self.bit_reader.as_mut().expect("bit_reader should be Some");
+            let bit_packed_value = bit_reader
+                .get_value(self.bit_width as usize)
+                .ok_or(eof_err!("Not enough data for 'bit_packed_value'"))?;
+            self.bit_packed_left -= 1;
+            bit_packed_value
+        };
+
+        Ok(Some(value))
+    }
+
+    #[inline]
+    pub fn get_batch<T: Default>(&mut self, buffer: &mut [T]) -> Result<usize> {
+        assert!(self.bit_reader.is_some());
+        assert!(size_of::<T>() <= 8);
+
+        let mut values_read = 0;
+        while values_read < buffer.len() {
+            if self.rle_left > 0 {
+                assert!(self.current_value.is_some());
+                let num_values = cmp::min(buffer.len() - values_read, self.rle_left as usize);
+                for i in 0..num_values {
+                    let repeated_value =
+                        unsafe { transmute_copy::<u64, T>(self.current_value.as_mut().unwrap()) };
+                    buffer[values_read + i] = repeated_value;
+                }
+                self.rle_left -= num_values as u32;
+                values_read += num_values;
+            } else if self.bit_packed_left > 0 {
+                assert!(self.bit_reader.is_some());
+                let mut num_values =
+                    cmp::min(buffer.len() - values_read, self.bit_packed_left as usize);
+                if let Some(ref mut bit_reader) = self.bit_reader {
+                    num_values = bit_reader.get_batch::<T>(
+                        &mut buffer[values_read..values_read + num_values],
+                        self.bit_width as usize,
+                    );
+                    self.bit_packed_left -= num_values as u32;
+                    values_read += num_values;
+                }
+            } else {
+                if !self.reload() {
+                    break;
+                }
+            }
+        }
+
+        Ok(values_read)
+    }
+
+    #[inline]
+    pub fn get_batch_with_dict<T>(
+        &mut self,
+        dict: &[T],
+        buffer: &mut [T],
+        max_values: usize,
+    ) -> Result<usize>
+    where
+        T: Default + Clone,
+    {
+        assert!(buffer.len() >= max_values);
+
+        let mut values_read = 0;
+        while values_read < max_values {
+            if self.rle_left > 0 {
+                assert!(self.current_value.is_some());
+                let num_values = cmp::min(max_values - values_read, self.rle_left as usize);
+                let dict_idx = self.current_value.unwrap() as usize;
+                for i in 0..num_values {
+                    buffer[values_read + i] = dict[dict_idx].clone();
+                }
+                self.rle_left -= num_values as u32;
+                values_read += num_values;
+            } else if self.bit_packed_left > 0 {
+                assert!(self.bit_reader.is_some());
+                let mut num_values =
+                    cmp::min(max_values - values_read, self.bit_packed_left as usize);
+                if let Some(ref mut bit_reader) = self.bit_reader {
+                    let mut index_buf = self.index_buf.unwrap();
+                    num_values = cmp::min(num_values, index_buf.len());
+                    loop {
+                        num_values = bit_reader.get_batch::<i32>(
+                            &mut index_buf[..num_values],
+                            self.bit_width as usize,
+                        );
+                        for i in 0..num_values {
+                            buffer[values_read + i] = dict[index_buf[i] as usize].clone();
+                        }
+                        self.bit_packed_left -= num_values as u32;
+                        values_read += num_values;
+                        if num_values < index_buf.len() {
+                            break;
+                        }
+                    }
+                }
+            } else {
+                if !self.reload() {
+                    break;
+                }
+            }
+        }
+
+        Ok(values_read)
+    }
+
+    #[inline]
+    fn reload(&mut self) -> bool {
+        assert!(self.bit_reader.is_some());
+        if let Some(ref mut bit_reader) = self.bit_reader {
+            if let Some(indicator_value) = bit_reader.get_vlq_int() {
+                if indicator_value & 1 == 1 {
+                    self.bit_packed_left = ((indicator_value >> 1) * 8) as u32;
+                } else {
+                    self.rle_left = (indicator_value >> 1) as u32;
+                    let value_width = bit_util::ceil(self.bit_width as i64, 8);
+                    self.current_value = bit_reader.get_aligned::<u64>(value_width as usize);
+                    assert!(self.current_value.is_some());
+                }
+                return true;
+            } else {
+                return false;
+            }
+        }
+        return false;
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use rand::{
+        self,
+        distributions::{Distribution, Standard},
+        thread_rng, Rng, SeedableRng,
+    };
+
+    use crate::parquet::util::memory::ByteBufferPtr;
+
+    const MAX_WIDTH: usize = 32;
+
+    #[test]
+    fn test_rle_decode_int32() {
+        // Test data: 0-7 with bit width 3
+        // 00000011 10001000 11000110 11111010
+        let data = ByteBufferPtr::new(vec![0x03, 0x88, 0xC6, 0xFA]);
+        let mut decoder: RleDecoder = RleDecoder::new(3);
+        decoder.set_data(data);
+        let mut buffer = vec![0; 8];
+        let expected = vec![0, 1, 2, 3, 4, 5, 6, 7];
+        let result = decoder.get_batch::<i32>(&mut buffer);
+        assert!(result.is_ok());
+        assert_eq!(buffer, expected);
+    }
+
+    #[test]
+    fn test_rle_consume_flush_buffer() {
+        let data = vec![1, 1, 1, 2, 2, 3, 3, 3];
+        let mut encoder1 = RleEncoder::new(3, 256);
+        let mut encoder2 = RleEncoder::new(3, 256);
+        for value in data {
+            encoder1.put(value as u64).unwrap();
+            encoder2.put(value as u64).unwrap();
+        }
+        let res1 = encoder1.flush_buffer().unwrap();
+        let res2 = encoder2.consume().unwrap();
+        assert_eq!(res1, &res2[..]);
+    }
+
+    #[test]
+    fn test_rle_decode_bool() {
+        // RLE test data: 50 1s followed by 50 0s
+        // 01100100 00000001 01100100 00000000
+        let data1 = ByteBufferPtr::new(vec![0x64, 0x01, 0x64, 0x00]);
+
+        // Bit-packing test data: alternating 1s and 0s, 100 total
+        // 100 / 8 = 13 groups
+        // 00011011 10101010 ... 00001010
+        let data2 = ByteBufferPtr::new(vec![
+            0x1B, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0x0A,
+        ]);
+
+        let mut decoder: RleDecoder = RleDecoder::new(1);
+        decoder.set_data(data1);
+        let mut buffer = vec![false; 100];
+        let mut expected = vec![];
+        for i in 0..100 {
+            if i < 50 {
+                expected.push(true);
+            } else {
+                expected.push(false);
+            }
+        }
+        let result = decoder.get_batch::<bool>(&mut buffer);
+        assert!(result.is_ok());
+        assert_eq!(buffer, expected);
+
+        decoder.set_data(data2);
+        let mut buffer = vec![false; 100];
+        let mut expected = vec![];
+        for i in 0..100 {
+            if i % 2 == 0 {
+                expected.push(false);
+            } else {
+                expected.push(true);
+            }
+        }
+        let result = decoder.get_batch::<bool>(&mut buffer);
+        assert!(result.is_ok());
+        assert_eq!(buffer, expected);
+    }
+
+    #[test]
+    fn test_rle_decode_with_dict_int32() {
+        // Test RLE encoding: 3 0s followed by 4 1s followed by 5 2s
+        // 00000110 00000000 00001000 00000001 00001010 00000010
+        let dict = vec![10, 20, 30];
+        let data = ByteBufferPtr::new(vec![0x06, 0x00, 0x08, 0x01, 0x0A, 0x02]);
+        let mut decoder: RleDecoder = RleDecoder::new(3);
+        decoder.set_data(data);
+        let mut buffer = vec![0; 12];
+        let expected = vec![10, 10, 10, 20, 20, 20, 20, 30, 30, 30, 30, 30];
+        let result = decoder.get_batch_with_dict::<i32>(&dict, &mut buffer, 12);
+        assert!(result.is_ok());
+        assert_eq!(buffer, expected);
+
+        // Test bit-pack encoding: 345345345455 (2 groups: 8 and 4)
+        // 011 100 101 011 100 101 011 100 101 100 101 101
+        // 00000011 01100011 11000111 10001110 00000011 01100101 00001011
+        let dict = vec!["aaa", "bbb", "ccc", "ddd", "eee", "fff"];
+        let data = ByteBufferPtr::new(vec![0x03, 0x63, 0xC7, 0x8E, 0x03, 0x65, 0x0B]);
+        let mut decoder: RleDecoder = RleDecoder::new(3);
+        decoder.set_data(data);
+        let mut buffer = vec![""; 12];
+        let expected = vec![
+            "ddd", "eee", "fff", "ddd", "eee", "fff", "ddd", "eee", "fff", "eee", "fff", "fff",
+        ];
+        let result =
+            decoder.get_batch_with_dict::<&str>(dict.as_slice(), buffer.as_mut_slice(), 12);
+        assert!(result.is_ok());
+        assert_eq!(buffer, expected);
+    }
+
+    fn validate_rle(
+        values: &[i64],
+        bit_width: u8,
+        expected_encoding: Option<&[u8]>,
+        expected_len: i32,
+    ) {
+        let buffer_len = 64 * 1024;
+        let mut encoder = RleEncoder::new(bit_width, buffer_len);
+        for v in values {
+            let result = encoder.put(*v as u64);
+            assert!(result.is_ok());
+        }
+        let buffer = ByteBufferPtr::new(encoder.consume().expect("Expect consume() OK"));
+        if expected_len != -1 {
+            assert_eq!(buffer.len(), expected_len as usize);
+        }
+        match expected_encoding {
+            Some(b) => assert_eq!(buffer.as_ref(), b),
+            _ => (),
+        }
+
+        // Verify read
+        let mut decoder = RleDecoder::new(bit_width);
+        decoder.set_data(buffer.all());
+        for v in values {
+            let val: i64 = decoder
+                .get()
+                .expect("get() should be OK")
+                .expect("get() should return more value");
+            assert_eq!(val, *v);
+        }
+
+        // Verify batch read
+        decoder.set_data(buffer);
+        let mut values_read: Vec<i64> = vec![0; values.len()];
+        decoder
+            .get_batch(&mut values_read[..])
+            .expect("get_batch() should be OK");
+        assert_eq!(&values_read[..], values);
+    }
+
+    #[test]
+    fn test_rle_specific_sequences() {
+        let mut expected_buffer = Vec::new();
+        let mut values = Vec::new();
+        for _ in 0..50 {
+            values.push(0);
+        }
+        for _ in 0..50 {
+            values.push(1);
+        }
+        expected_buffer.push(50 << 1);
+        expected_buffer.push(0);
+        expected_buffer.push(50 << 1);
+        expected_buffer.push(1);
+
+        for width in 1..9 {
+            validate_rle(&values[..], width, Some(&expected_buffer[..]), 4);
+        }
+        for width in 9..MAX_WIDTH + 1 {
+            validate_rle(
+                &values[..],
+                width as u8,
+                None,
+                2 * (1 + bit_util::ceil(width as i64, 8) as i32),
+            );
+        }
+
+        // Test 100 0's and 1's alternating
+        values.clear();
+        expected_buffer.clear();
+        for i in 0..101 {
+            values.push(i % 2);
+        }
+        let num_groups = bit_util::ceil(100, 8) as u8;
+        expected_buffer.push(((num_groups << 1) as u8) | 1);
+        for _ in 1..(100 / 8) + 1 {
+            expected_buffer.push(0b10101010);
+        }
+        // For the last 4 0 and 1's, padded with 0.
+        expected_buffer.push(0b00001010);
+        validate_rle(
+            &values,
+            1,
+            Some(&expected_buffer[..]),
+            1 + num_groups as i32,
+        );
+        for width in 2..MAX_WIDTH + 1 {
+            let num_values = bit_util::ceil(100, 8) * 8;
+            validate_rle(
+                &values,
+                width as u8,
+                None,
+                1 + bit_util::ceil(width as i64 * num_values, 8) as i32,
+            );
+        }
+    }
+
+    // `validate_rle` on `num_vals` with width `bit_width`. If `value` is -1, that value
+    // is used, otherwise alternating values are used.
+    fn test_rle_values(bit_width: usize, num_vals: usize, value: i32) {
+        let mod_val = if bit_width == 64 {
+            1
+        } else {
+            1u64 << bit_width
+        };
+        let mut values: Vec<i64> = vec![];
+        for v in 0..num_vals {
+            let val = if value == -1 {
+                v as i64 % mod_val as i64
+            } else {
+                value as i64
+            };
+            values.push(val);
+        }
+        validate_rle(&values, bit_width as u8, None, -1);
+    }
+
+    #[test]
+    fn test_values() {
+        for width in 1..MAX_WIDTH + 1 {
+            test_rle_values(width, 1, -1);
+            test_rle_values(width, 1024, -1);
+            test_rle_values(width, 1024, 0);
+            test_rle_values(width, 1024, 1);
+        }
+    }
+
+    #[test]
+    fn test_rle_specific_roundtrip() {
+        let bit_width = 1;
+        let buffer_len = RleEncoder::min_buffer_size(bit_width);
+        let values: Vec<i16> = vec![0, 1, 1, 1, 1, 0, 0, 0, 0, 1];
+        let mut encoder = RleEncoder::new(bit_width, buffer_len);
+        for v in &values {
+            assert!(encoder.put(*v as u64).expect("put() should be OK"));
+        }
+        let buffer = encoder.consume().expect("consume() should be OK");
+        let mut decoder = RleDecoder::new(bit_width);
+        decoder.set_data(ByteBufferPtr::new(buffer));
+        let mut actual_values: Vec<i16> = vec![0; values.len()];
+        decoder
+            .get_batch(&mut actual_values)
+            .expect("get_batch() should be OK");
+        assert_eq!(actual_values, values);
+    }
+
+    fn test_round_trip(values: &[i32], bit_width: u8) {
+        let buffer_len = 64 * 1024;
+        let mut encoder = RleEncoder::new(bit_width, buffer_len);
+        for v in values {
+            let result = encoder.put(*v as u64).expect("put() should be OK");
+            assert!(result, "put() should not return false");
+        }
+
+        let buffer = ByteBufferPtr::new(encoder.consume().expect("consume() should be OK"));
+
+        // Verify read
+        let mut decoder = RleDecoder::new(bit_width);
+        decoder.set_data(buffer.all());
+        for v in values {
+            let val = decoder
+                .get::<i32>()
+                .expect("get() should be OK")
+                .expect("get() should return value");
+            assert_eq!(val, *v);
+        }
+
+        // Verify batch read
+        let mut decoder = RleDecoder::new(bit_width);
+        decoder.set_data(buffer);
+        let mut values_read: Vec<i32> = vec![0; values.len()];
+        decoder
+            .get_batch(&mut values_read[..])
+            .expect("get_batch() should be OK");
+        assert_eq!(&values_read[..], values);
+    }
+
+    #[test]
+    fn test_random() {
+        let seed_len = 32;
+        let niters = 50;
+        let ngroups = 1000;
+        let max_group_size = 15;
+        let mut values = vec![];
+
+        for _ in 0..niters {
+            values.clear();
+            let mut rng = thread_rng();
+            let seed_vec: Vec<u8> = Standard.sample_iter(&mut rng).take(seed_len).collect();
+            let mut seed = [0u8; 32];
+            seed.copy_from_slice(&seed_vec[0..seed_len]);
+            let mut gen = rand::StdRng::from_seed(seed);
+
+            let mut parity = false;
+            for _ in 0..ngroups {
+                let mut group_size = gen.gen_range::<u32>(1, 20);
+                if group_size > max_group_size {
+                    group_size = 1;
+                }
+                for _ in 0..group_size {
+                    values.push(parity as i32);
+                }
+                parity = !parity;
+            }
+            let bit_width = bit_util::num_required_bits(values.len() as u64);
+            assert!(bit_width < 64);
+            test_round_trip(&values[..], bit_width as u8);
+        }
+    }
+}

rust/src/parquet/errors.rs

@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Common Parquet errors and macros.
+
+use std::{cell, convert, io, result};
+
+use quick_error::quick_error;
+use snap;
+use thrift;
+
+quick_error! {
+  /// Set of errors that can be produced during different operations in Parquet.
+  #[derive(Debug, PartialEq)]
+  pub enum ParquetError {
+      /// General Parquet error.
+      /// Returned when code violates normal workflow of working with Parquet files.
+      General(message: String) {
+          display("Parquet error: {}", message)
+              description(message)
+              from(e: io::Error) -> (format!("underlying IO error: {}", e))
+              from(e: snap::Error) -> (format!("underlying snap error: {}", e))
+              from(e: thrift::Error) -> (format!("underlying Thrift error: {}", e))
+              from(e: cell::BorrowMutError) -> (format!("underlying borrow error: {}", e))
+      }
+      /// "Not yet implemented" Parquet error.
+      /// Returned when functionality is not yet available.
+      NYI(message: String) {
+          display("NYI: {}", message)
+              description(message)
+      }
+      /// "End of file" Parquet error.
+      /// Returned when IO related failures occur, e.g. when there are not enough bytes to
+      /// decode.
+      EOF(message: String) {
+          display("EOF: {}", message)
+              description(message)
+      }
+  }
+}
+
+/// A specialized `Result` for Parquet errors.
+pub type Result<T> = result::Result<T, ParquetError>;
+
+// ----------------------------------------------------------------------
+// Conversion from `ParquetError` to other types of `Error`s
+
+impl convert::From<ParquetError> for io::Error {
+    fn from(e: ParquetError) -> Self {
+        io::Error::new(io::ErrorKind::Other, e)
+    }
+}
+
+// ----------------------------------------------------------------------
+// Convenient macros for different errors
+
+macro_rules! general_err {
+    ($fmt:expr) => (ParquetError::General($fmt.to_owned()));
+    ($fmt:expr, $($args:expr),*) => (ParquetError::General(format!($fmt, $($args),*)));
+    ($e:expr, $fmt:expr) => (ParquetError::General($fmt.to_owned(), $e));
+    ($e:ident, $fmt:expr, $($args:tt),*) => (
+        ParquetError::General(&format!($fmt, $($args),*), $e));
+}
+
+macro_rules! nyi_err {
+    ($fmt:expr) => (ParquetError::NYI($fmt.to_owned()));
+    ($fmt:expr, $($args:expr),*) => (ParquetError::NYI(format!($fmt, $($args),*)));
+}
+
+macro_rules! eof_err {
+    ($fmt:expr) => (ParquetError::EOF($fmt.to_owned()));
+    ($fmt:expr, $($args:expr),*) => (ParquetError::EOF(format!($fmt, $($args),*)));
+}

rust/src/parquet/file/metadata.rs

@@ -0,0 +1,736 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Contains information about available Parquet metadata.
+//!
+//! The hierarchy of metadata is as follows:
+//!
+//! [`ParquetMetaData`](struct.ParquetMetaData.html) contains
+//! [`FileMetaData`](struct.FileMetaData.html) and zero or more
+//! [`RowGroupMetaData`](struct.RowGroupMetaData.html) for each row group.
+//!
+//! [`FileMetaData`](struct.FileMetaData.html) includes file version, application specific
+//! metadata.
+//!
+//! Each [`RowGroupMetaData`](struct.RowGroupMetaData.html) contains information about row
+//! group and one or more [`ColumnChunkMetaData`](struct.ColumnChunkMetaData.html) for
+//! each column chunk.
+//!
+//! [`ColumnChunkMetaData`](struct.ColumnChunkMetaData.html) has information about column
+//! chunk (primitive leaf column), including encoding/compression, number of values, etc.
+
+use std::rc::Rc;
+
+use parquet_format::{ColumnChunk, ColumnMetaData, RowGroup};
+
+use crate::parquet::basic::{ColumnOrder, Compression, Encoding, Type};
+use crate::parquet::errors::{ParquetError, Result};
+use crate::parquet::file::statistics::{self, Statistics};
+use crate::parquet::schema::types::{
+    ColumnDescPtr, ColumnDescriptor, ColumnPath, SchemaDescPtr, SchemaDescriptor,
+    Type as SchemaType, TypePtr,
+};
+
+/// Reference counted pointer for [`ParquetMetaData`].
+pub type ParquetMetaDataPtr = Rc<ParquetMetaData>;
+
+/// Global Parquet metadata.
+pub struct ParquetMetaData {
+    file_metadata: FileMetaDataPtr,
+    row_groups: Vec<RowGroupMetaDataPtr>,
+}
+
+impl ParquetMetaData {
+    /// Creates Parquet metadata from file metadata and a list of row group metadata `Rc`s
+    /// for each available row group.
+    pub fn new(file_metadata: FileMetaData, row_group_ptrs: Vec<RowGroupMetaDataPtr>) -> Self {
+        ParquetMetaData {
+            file_metadata: Rc::new(file_metadata),
+            row_groups: row_group_ptrs,
+        }
+    }
+
+    /// Returns file metadata as reference counted clone.
+    pub fn file_metadata(&self) -> FileMetaDataPtr {
+        self.file_metadata.clone()
+    }
+
+    /// Returns number of row groups in this file.
+    pub fn num_row_groups(&self) -> usize {
+        self.row_groups.len()
+    }
+
+    /// Returns row group metadata for `i`th position.
+    /// Position should be less than number of row groups `num_row_groups`.
+    pub fn row_group(&self, i: usize) -> RowGroupMetaDataPtr {
+        self.row_groups[i].clone()
+    }
+
+    /// Returns slice of row group reference counted pointers in this file.
+    pub fn row_groups(&self) -> &[RowGroupMetaDataPtr] {
+        &self.row_groups.as_slice()
+    }
+}
+
+/// Reference counted pointer for [`FileMetaData`].
+pub type FileMetaDataPtr = Rc<FileMetaData>;
+
+/// Metadata for a Parquet file.
+pub struct FileMetaData {
+    version: i32,
+    num_rows: i64,
+    created_by: Option<String>,
+    schema: TypePtr,
+    schema_descr: SchemaDescPtr,
+    column_orders: Option<Vec<ColumnOrder>>,
+}
+
+impl FileMetaData {
+    /// Creates new file metadata.
+    pub fn new(
+        version: i32,
+        num_rows: i64,
+        created_by: Option<String>,
+        schema: TypePtr,
+        schema_descr: SchemaDescPtr,
+        column_orders: Option<Vec<ColumnOrder>>,
+    ) -> Self {
+        FileMetaData {
+            version,
+            num_rows,
+            created_by,
+            schema,
+            schema_descr,
+            column_orders,
+        }
+    }
+
+    /// Returns version of this file.
+    pub fn version(&self) -> i32 {
+        self.version
+    }
+
+    /// Returns number of rows in the file.
+    pub fn num_rows(&self) -> i64 {
+        self.num_rows
+    }
+
+    /// String message for application that wrote this file.
+    ///
+    /// This should have the following format:
+    /// `<application> version <application version> (build <application build hash>)`.
+    ///
+    /// ```shell
+    /// parquet-mr version 1.8.0 (build 0fda28af84b9746396014ad6a415b90592a98b3b)
+    /// ```
+    pub fn created_by(&self) -> &Option<String> {
+        &self.created_by
+    }
+
+    /// Returns Parquet ['Type`] that describes schema in this file.
+    pub fn schema(&self) -> &SchemaType {
+        self.schema.as_ref()
+    }
+
+    /// Returns a reference to schema descriptor.
+    pub fn schema_descr(&self) -> &SchemaDescriptor {
+        &self.schema_descr
+    }
+
+    /// Returns reference counted clone for schema descriptor.
+    pub fn schema_descr_ptr(&self) -> SchemaDescPtr {
+        self.schema_descr.clone()
+    }
+
+    /// Column (sort) order used for `min` and `max` values of each column in this file.
+    ///
+    /// Each column order corresponds to one column, determined by its position in the list,
+    /// matching the position of the column in the schema.
+    ///
+    /// When `None` is returned, there are no column orders available, and each column
+    /// should be assumed to have undefined (legacy) column order.
+    pub fn column_orders(&self) -> Option<&Vec<ColumnOrder>> {
+        self.column_orders.as_ref()
+    }
+
+    /// Returns column order for `i`th column in this file.
+    /// If column orders are not available, returns undefined (legacy) column order.
+    pub fn column_order(&self, i: usize) -> ColumnOrder {
+        self.column_orders
+            .as_ref()
+            .map(|data| data[i])
+            .unwrap_or(ColumnOrder::UNDEFINED)
+    }
+}
+
+/// Reference counted pointer for [`RowGroupMetaData`].
+pub type RowGroupMetaDataPtr = Rc<RowGroupMetaData>;
+
+/// Metadata for a row group.
+pub struct RowGroupMetaData {
+    columns: Vec<ColumnChunkMetaDataPtr>,
+    num_rows: i64,
+    total_byte_size: i64,
+    schema_descr: SchemaDescPtr,
+}
+
+impl RowGroupMetaData {
+    /// Returns builer for row group metadata.
+    pub fn builder(schema_descr: SchemaDescPtr) -> RowGroupMetaDataBuilder {
+        RowGroupMetaDataBuilder::new(schema_descr)
+    }
+
+    /// Number of columns in this row group.
+    pub fn num_columns(&self) -> usize {
+        self.columns.len()
+    }
+
+    /// Returns column chunk metadata for `i`th column.
+    pub fn column(&self, i: usize) -> &ColumnChunkMetaData {
+        &self.columns[i]
+    }
+
+    /// Returns slice of column chunk metadata [`Rc`] pointers.
+    pub fn columns(&self) -> &[ColumnChunkMetaDataPtr] {
+        &self.columns
+    }
+
+    /// Number of rows in this row group.
+    pub fn num_rows(&self) -> i64 {
+        self.num_rows
+    }
+
+    /// Total byte size of all uncompressed column data in this row group.
+    pub fn total_byte_size(&self) -> i64 {
+        self.total_byte_size
+    }
+
+    /// Returns reference to a schema descriptor.
+    pub fn schema_descr(&self) -> &SchemaDescriptor {
+        self.schema_descr.as_ref()
+    }
+
+    /// Returns reference counted clone of schema descriptor.
+    pub fn schema_descr_ptr(&self) -> SchemaDescPtr {
+        self.schema_descr.clone()
+    }
+
+    /// Method to convert from Thrift.
+    pub fn from_thrift(schema_descr: SchemaDescPtr, mut rg: RowGroup) -> Result<RowGroupMetaData> {
+        assert_eq!(schema_descr.num_columns(), rg.columns.len());
+        let total_byte_size = rg.total_byte_size;
+        let num_rows = rg.num_rows;
+        let mut columns = vec![];
+        for (c, d) in rg.columns.drain(0..).zip(schema_descr.columns()) {
+            let cc = ColumnChunkMetaData::from_thrift(d.clone(), c)?;
+            columns.push(Rc::new(cc));
+        }
+        Ok(RowGroupMetaData {
+            columns,
+            num_rows,
+            total_byte_size,
+            schema_descr,
+        })
+    }
+
+    /// Method to convert to Thrift.
+    pub fn to_thrift(&self) -> RowGroup {
+        RowGroup {
+            columns: self.columns().into_iter().map(|v| v.to_thrift()).collect(),
+            total_byte_size: self.total_byte_size,
+            num_rows: self.num_rows,
+            sorting_columns: None,
+        }
+    }
+}
+
+/// Builder for row group metadata.
+pub struct RowGroupMetaDataBuilder {
+    columns: Vec<ColumnChunkMetaDataPtr>,
+    schema_descr: SchemaDescPtr,
+    num_rows: i64,
+    total_byte_size: i64,
+}
+
+impl RowGroupMetaDataBuilder {
+    /// Creates new builder from schema descriptor.
+    fn new(schema_descr: SchemaDescPtr) -> Self {
+        Self {
+            columns: Vec::with_capacity(schema_descr.num_columns()),
+            schema_descr,
+            num_rows: 0,
+            total_byte_size: 0,
+        }
+    }
+
+    /// Sets number of rows in this row group.
+    pub fn set_num_rows(mut self, value: i64) -> Self {
+        self.num_rows = value;
+        self
+    }
+
+    /// Sets total size in bytes for this row group.
+    pub fn set_total_byte_size(mut self, value: i64) -> Self {
+        self.total_byte_size = value;
+        self
+    }
+
+    /// Sets column metadata for this row group.
+    pub fn set_column_metadata(mut self, value: Vec<ColumnChunkMetaDataPtr>) -> Self {
+        self.columns = value;
+        self
+    }
+
+    /// Builds row group metadata.
+    pub fn build(self) -> Result<RowGroupMetaData> {
+        if self.schema_descr.num_columns() != self.columns.len() {
+            return Err(general_err!(
+                "Column length mismatch: {} != {}",
+                self.schema_descr.num_columns(),
+                self.columns.len()
+            ));
+        }
+
+        Ok(RowGroupMetaData {
+            columns: self.columns,
+            num_rows: self.num_rows,
+            total_byte_size: self.total_byte_size,
+            schema_descr: self.schema_descr,
+        })
+    }
+}
+
+/// Reference counted pointer for [`ColumnChunkMetaData`].
+pub type ColumnChunkMetaDataPtr = Rc<ColumnChunkMetaData>;
+
+/// Metadata for a column chunk.
+pub struct ColumnChunkMetaData {
+    column_type: Type,
+    column_path: ColumnPath,
+    column_descr: ColumnDescPtr,
+    encodings: Vec<Encoding>,
+    file_path: Option<String>,
+    file_offset: i64,
+    num_values: i64,
+    compression: Compression,
+    total_compressed_size: i64,
+    total_uncompressed_size: i64,
+    data_page_offset: i64,
+    index_page_offset: Option<i64>,
+    dictionary_page_offset: Option<i64>,
+    statistics: Option<Statistics>,
+}
+
+/// Represents common operations for a column chunk.
+impl ColumnChunkMetaData {
+    /// Returns builder for column chunk metadata.
+    pub fn builder(column_descr: ColumnDescPtr) -> ColumnChunkMetaDataBuilder {
+        ColumnChunkMetaDataBuilder::new(column_descr)
+    }
+
+    /// File where the column chunk is stored.
+    ///
+    /// If not set, assumed to belong to the same file as the metadata.
+    /// This path is relative to the current file.
+    pub fn file_path(&self) -> Option<&String> {
+        self.file_path.as_ref()
+    }
+
+    /// Byte offset in `file_path()`.
+    pub fn file_offset(&self) -> i64 {
+        self.file_offset
+    }
+
+    /// Type of this column. Must be primitive.
+    pub fn column_type(&self) -> Type {
+        self.column_type
+    }
+
+    /// Path (or identifier) of this column.
+    pub fn column_path(&self) -> &ColumnPath {
+        &self.column_path
+    }
+
+    /// Descriptor for this column.
+    pub fn column_descr(&self) -> &ColumnDescriptor {
+        self.column_descr.as_ref()
+    }
+
+    /// Reference counted clone of descriptor for this column.
+    pub fn column_descr_ptr(&self) -> ColumnDescPtr {
+        self.column_descr.clone()
+    }
+
+    /// All encodings used for this column.
+    pub fn encodings(&self) -> &Vec<Encoding> {
+        &self.encodings
+    }
+
+    /// Total number of values in this column chunk.
+    pub fn num_values(&self) -> i64 {
+        self.num_values
+    }
+
+    /// Compression for this column.
+    pub fn compression(&self) -> Compression {
+        self.compression
+    }
+
+    /// Returns the total compressed data size of this column chunk.
+    pub fn compressed_size(&self) -> i64 {
+        self.total_compressed_size
+    }
+
+    /// Returns the total uncompressed data size of this column chunk.
+    pub fn uncompressed_size(&self) -> i64 {
+        self.total_uncompressed_size
+    }
+
+    /// Returns the offset for the column data.
+    pub fn data_page_offset(&self) -> i64 {
+        self.data_page_offset
+    }
+
+    /// Returns `true` if this column chunk contains a index page, `false` otherwise.
+    pub fn has_index_page(&self) -> bool {
+        self.index_page_offset.is_some()
+    }
+
+    /// Returns the offset for the index page.
+    pub fn index_page_offset(&self) -> Option<i64> {
+        self.index_page_offset
+    }
+
+    /// Returns `true` if this column chunk contains a dictionary page, `false` otherwise.
+    pub fn has_dictionary_page(&self) -> bool {
+        self.dictionary_page_offset.is_some()
+    }
+
+    /// Returns the offset for the dictionary page, if any.
+    pub fn dictionary_page_offset(&self) -> Option<i64> {
+        self.dictionary_page_offset
+    }
+
+    /// Returns statistics that are set for this column chunk,
+    /// or `None` if no statistics are available.
+    pub fn statistics(&self) -> Option<&Statistics> {
+        self.statistics.as_ref()
+    }
+
+    /// Method to convert from Thrift.
+    pub fn from_thrift(column_descr: ColumnDescPtr, cc: ColumnChunk) -> Result<Self> {
+        if cc.meta_data.is_none() {
+            return Err(general_err!("Expected to have column metadata"));
+        }
+        let mut col_metadata: ColumnMetaData = cc.meta_data.unwrap();
+        let column_type = Type::from(col_metadata.type_);
+        let column_path = ColumnPath::new(col_metadata.path_in_schema);
+        let encodings = col_metadata
+            .encodings
+            .drain(0..)
+            .map(Encoding::from)
+            .collect();
+        let compression = Compression::from(col_metadata.codec);
+        let file_path = cc.file_path;
+        let file_offset = cc.file_offset;
+        let num_values = col_metadata.num_values;
+        let total_compressed_size = col_metadata.total_compressed_size;
+        let total_uncompressed_size = col_metadata.total_uncompressed_size;
+        let data_page_offset = col_metadata.data_page_offset;
+        let index_page_offset = col_metadata.index_page_offset;
+        let dictionary_page_offset = col_metadata.dictionary_page_offset;
+        let statistics = statistics::from_thrift(column_type, col_metadata.statistics);
+        let result = ColumnChunkMetaData {
+            column_type,
+            column_path,
+            column_descr,
+            encodings,
+            file_path,
+            file_offset,
+            num_values,
+            compression,
+            total_compressed_size,
+            total_uncompressed_size,
+            data_page_offset,
+            index_page_offset,
+            dictionary_page_offset,
+            statistics,
+        };
+        Ok(result)
+    }
+
+    /// Method to convert to Thrift.
+    pub fn to_thrift(&self) -> ColumnChunk {
+        let column_metadata = ColumnMetaData {
+            type_: self.column_type.into(),
+            encodings: self.encodings().into_iter().map(|&v| v.into()).collect(),
+            path_in_schema: Vec::from(self.column_path.as_ref()),
+            codec: self.compression.into(),
+            num_values: self.num_values,
+            total_uncompressed_size: self.total_uncompressed_size,
+            total_compressed_size: self.total_compressed_size,
+            key_value_metadata: None,
+            data_page_offset: self.data_page_offset,
+            index_page_offset: self.index_page_offset,
+            dictionary_page_offset: self.dictionary_page_offset,
+            statistics: statistics::to_thrift(self.statistics.as_ref()),
+            encoding_stats: None,
+        };
+
+        ColumnChunk {
+            file_path: self.file_path().map(|v| v.clone()),
+            file_offset: self.file_offset,
+            meta_data: Some(column_metadata),
+            offset_index_offset: None,
+            offset_index_length: None,
+            column_index_offset: None,
+            column_index_length: None,
+        }
+    }
+}
+
+/// Builder for column chunk metadata.
+pub struct ColumnChunkMetaDataBuilder {
+    column_descr: ColumnDescPtr,
+    encodings: Vec<Encoding>,
+    file_path: Option<String>,
+    file_offset: i64,
+    num_values: i64,
+    compression: Compression,
+    total_compressed_size: i64,
+    total_uncompressed_size: i64,
+    data_page_offset: i64,
+    index_page_offset: Option<i64>,
+    dictionary_page_offset: Option<i64>,
+    statistics: Option<Statistics>,
+}
+
+impl ColumnChunkMetaDataBuilder {
+    /// Creates new column chunk metadata builder.
+    fn new(column_descr: ColumnDescPtr) -> Self {
+        Self {
+            column_descr,
+            encodings: Vec::new(),
+            file_path: None,
+            file_offset: 0,
+            num_values: 0,
+            compression: Compression::UNCOMPRESSED,
+            total_compressed_size: 0,
+            total_uncompressed_size: 0,
+            data_page_offset: 0,
+            index_page_offset: None,
+            dictionary_page_offset: None,
+            statistics: None,
+        }
+    }
+
+    /// Sets list of encodings for this column chunk.
+    pub fn set_encodings(mut self, encodings: Vec<Encoding>) -> Self {
+        self.encodings = encodings;
+        self
+    }
+
+    /// Sets optional file path for this column chunk.
+    pub fn set_file_path(mut self, value: String) -> Self {
+        self.file_path = Some(value);
+        self
+    }
+
+    /// Sets file offset in bytes.
+    pub fn set_file_offset(mut self, value: i64) -> Self {
+        self.file_offset = value;
+        self
+    }
+
+    /// Sets number of values.
+    pub fn set_num_values(mut self, value: i64) -> Self {
+        self.num_values = value;
+        self
+    }
+
+    /// Sets compression.
+    pub fn set_compression(mut self, value: Compression) -> Self {
+        self.compression = value;
+        self
+    }
+
+    /// Sets total compressed size in bytes.
+    pub fn set_total_compressed_size(mut self, value: i64) -> Self {
+        self.total_compressed_size = value;
+        self
+    }
+
+    /// Sets total uncompressed size in bytes.
+    pub fn set_total_uncompressed_size(mut self, value: i64) -> Self {
+        self.total_uncompressed_size = value;
+        self
+    }
+
+    /// Sets data page offset in bytes.
+    pub fn set_data_page_offset(mut self, value: i64) -> Self {
+        self.data_page_offset = value;
+        self
+    }
+
+    /// Sets optional dictionary page ofset in bytes.
+    pub fn set_dictionary_page_offset(mut self, value: Option<i64>) -> Self {
+        self.dictionary_page_offset = value;
+        self
+    }
+
+    /// Sets optional index page offset in bytes.
+    pub fn set_index_page_offset(mut self, value: Option<i64>) -> Self {
+        self.index_page_offset = value;
+        self
+    }
+
+    /// Sets statistics for this column chunk.
+    pub fn set_statistics(mut self, value: Statistics) -> Self {
+        self.statistics = Some(value);
+        self
+    }
+
+    /// Builds column chunk metadata.
+    pub fn build(self) -> Result<ColumnChunkMetaData> {
+        Ok(ColumnChunkMetaData {
+            column_type: self.column_descr.physical_type(),
+            column_path: self.column_descr.path().clone(),
+            column_descr: self.column_descr,
+            encodings: self.encodings,
+            file_path: self.file_path,
+            file_offset: self.file_offset,
+            num_values: self.num_values,
+            compression: self.compression,
+            total_compressed_size: self.total_compressed_size,
+            total_uncompressed_size: self.total_uncompressed_size,
+            data_page_offset: self.data_page_offset,
+            index_page_offset: self.index_page_offset,
+            dictionary_page_offset: self.dictionary_page_offset,
+            statistics: self.statistics,
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_row_group_metadata_thrift_conversion() {
+        let schema_descr = get_test_schema_descr();
+
+        let mut columns = vec![];
+        for ptr in schema_descr.columns() {
+            let column = ColumnChunkMetaData::builder(ptr.clone()).build().unwrap();
+            columns.push(Rc::new(column));
+        }
+        let row_group_meta = RowGroupMetaData::builder(schema_descr.clone())
+            .set_num_rows(1000)
+            .set_total_byte_size(2000)
+            .set_column_metadata(columns)
+            .build()
+            .unwrap();
+
+        let row_group_exp = row_group_meta.to_thrift();
+        let row_group_res =
+            RowGroupMetaData::from_thrift(schema_descr.clone(), row_group_exp.clone())
+                .unwrap()
+                .to_thrift();
+
+        assert_eq!(row_group_res, row_group_exp);
+    }
+
+    #[test]
+    fn test_row_group_metadata_thrift_conversion_empty() {
+        let schema_descr = get_test_schema_descr();
+
+        let row_group_meta = RowGroupMetaData::builder(schema_descr.clone()).build();
+
+        assert!(row_group_meta.is_err());
+        if let Err(e) = row_group_meta {
+            assert_eq!(
+                e.to_string(),
+                "Parquet error: Column length mismatch: 2 != 0"
+            );
+        }
+    }
+
+    #[test]
+    fn test_column_chunk_metadata_thrift_conversion() {
+        let column_descr = get_test_schema_descr().column(0);
+
+        let col_metadata = ColumnChunkMetaData::builder(column_descr.clone())
+            .set_encodings(vec![Encoding::PLAIN, Encoding::RLE])
+            .set_file_path("file_path".to_owned())
+            .set_file_offset(100)
+            .set_num_values(1000)
+            .set_compression(Compression::SNAPPY)
+            .set_total_compressed_size(2000)
+            .set_total_uncompressed_size(3000)
+            .set_data_page_offset(4000)
+            .set_dictionary_page_offset(Some(5000))
+            .build()
+            .unwrap();
+
+        let col_chunk_exp = col_metadata.to_thrift();
+
+        let col_chunk_res =
+            ColumnChunkMetaData::from_thrift(column_descr.clone(), col_chunk_exp.clone())
+                .unwrap()
+                .to_thrift();
+
+        assert_eq!(col_chunk_res, col_chunk_exp);
+    }
+
+    #[test]
+    fn test_column_chunk_metadata_thrift_conversion_empty() {
+        let column_descr = get_test_schema_descr().column(0);
+
+        let col_metadata = ColumnChunkMetaData::builder(column_descr.clone())
+            .build()
+            .unwrap();
+
+        let col_chunk_exp = col_metadata.to_thrift();
+        let col_chunk_res =
+            ColumnChunkMetaData::from_thrift(column_descr.clone(), col_chunk_exp.clone())
+                .unwrap()
+                .to_thrift();
+
+        assert_eq!(col_chunk_res, col_chunk_exp);
+    }
+
+    /// Returns sample schema descriptor so we can create column metadata.
+    fn get_test_schema_descr() -> SchemaDescPtr {
+        let schema = SchemaType::group_type_builder("schema")
+            .with_fields(&mut vec![
+                Rc::new(
+                    SchemaType::primitive_type_builder("a", Type::INT32)
+                        .build()
+                        .unwrap(),
+                ),
+                Rc::new(
+                    SchemaType::primitive_type_builder("b", Type::INT32)
+                        .build()
+                        .unwrap(),
+                ),
+            ])
+            .build()
+            .unwrap();
+
+        Rc::new(SchemaDescriptor::new(Rc::new(schema)))
+    }
+}

rust/src/parquet/file/mod.rs

@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Main entrypoint for working with Parquet API.
+//!
+//! Provides access to file and row group readers and writers, record API, metadata, etc.
+//!
+//! See [`reader::SerializedFileReader`](reader/struct.SerializedFileReader.html) or
+//! [`writer::SerializedFileWriter`](writer/struct.SerializedFileWriter.html) for a
+//! starting reference, [`metadata::ParquetMetaData`](metadata/index.html) for file
+//! metadata, and [`statistics`](statistics/index.html) for working with statistics.
+//!
+//! # Example of writing a new file
+//!
+//! ```rust
+//! use std::{fs, path::Path, rc::Rc};
+//!
+//! use arrow::parquet::{
+//!     file::{
+//!         properties::WriterProperties,
+//!         writer::{FileWriter, SerializedFileWriter},
+//!     },
+//!     schema::parser::parse_message_type,
+//! };
+//!
+//! let path = Path::new("target/debug/examples/sample.parquet");
+//!
+//! let message_type = "
+//!   message schema {
+//!     REQUIRED INT32 b;
+//!   }
+//! ";
+//! let schema = Rc::new(parse_message_type(message_type).unwrap());
+//! let props = Rc::new(WriterProperties::builder().build());
+//! let file = fs::File::create(&path).unwrap();
+//! let mut writer = SerializedFileWriter::new(file, schema, props).unwrap();
+//! let mut row_group_writer = writer.next_row_group().unwrap();
+//! while let Some(mut col_writer) = row_group_writer.next_column().unwrap() {
+//!     // ... write values to a column writer
+//!     row_group_writer.close_column(col_writer).unwrap();
+//! }
+//! writer.close_row_group(row_group_writer).unwrap();
+//! writer.close().unwrap();
+//!
+//! let bytes = fs::read(&path).unwrap();
+//! assert_eq!(&bytes[0..4], &[b'P', b'A', b'R', b'1']);
+//! ```
+//! # Example of reading an existing file
+//!
+//! ```rust
+//! use arrow::parquet::file::reader::{FileReader, SerializedFileReader};
+//! use std::{fs::File, path::Path};
+//!
+//! let path = Path::new("target/debug/examples/sample.parquet");
+//! if let Ok(file) = File::open(&path) {
+//!     let file = File::open(&path).unwrap();
+//!     let reader = SerializedFileReader::new(file).unwrap();
+//!
+//!     let parquet_metadata = reader.metadata();
+//!     assert_eq!(parquet_metadata.num_row_groups(), 1);
+//!
+//!     let row_group_reader = reader.get_row_group(0).unwrap();
+//!     assert_eq!(row_group_reader.num_columns(), 1);
+//! }
+//! ```
+
+pub mod metadata;
+pub mod properties;
+pub mod reader;
+pub mod statistics;
+pub mod writer;
+
+const FOOTER_SIZE: usize = 8;
+const PARQUET_MAGIC: [u8; 4] = [b'P', b'A', b'R', b'1'];

rust/src/parquet/file/properties.rs

@@ -0,0 +1,648 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Writer properties.
+//!
+//! # Usage
+//!
+//! ```rust
+//! use arrow::parquet::{
+//!     basic::{Compression, Encoding},
+//!     file::properties::*,
+//!     schema::types::ColumnPath,
+//! };
+//!
+//! // Create properties with default configuration.
+//! let props = WriterProperties::builder().build();
+//!
+//! // Use properties builder to set certain options and assemble the configuration.
+//! let props = WriterProperties::builder()
+//!     .set_writer_version(WriterVersion::PARQUET_1_0)
+//!     .set_encoding(Encoding::PLAIN)
+//!     .set_column_encoding(ColumnPath::from("col1"), Encoding::DELTA_BINARY_PACKED)
+//!     .set_compression(Compression::SNAPPY)
+//!     .build();
+//!
+//! assert_eq!(props.writer_version(), WriterVersion::PARQUET_1_0);
+//! assert_eq!(
+//!     props.encoding(&ColumnPath::from("col1")),
+//!     Some(Encoding::DELTA_BINARY_PACKED)
+//! );
+//! assert_eq!(
+//!     props.encoding(&ColumnPath::from("col2")),
+//!     Some(Encoding::PLAIN)
+//! );
+//! ```
+
+use std::{collections::HashMap, rc::Rc};
+
+use crate::parquet::basic::{Compression, Encoding};
+use crate::parquet::schema::types::ColumnPath;
+
+const DEFAULT_PAGE_SIZE: usize = 1024 * 1024;
+const DEFAULT_WRITE_BATCH_SIZE: usize = 1024;
+const DEFAULT_WRITER_VERSION: WriterVersion = WriterVersion::PARQUET_1_0;
+const DEFAULT_COMPRESSION: Compression = Compression::UNCOMPRESSED;
+const DEFAULT_DICTIONARY_ENABLED: bool = true;
+const DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT: usize = DEFAULT_PAGE_SIZE;
+const DEFAULT_STATISTICS_ENABLED: bool = true;
+const DEFAULT_MAX_STATISTICS_SIZE: usize = 4096;
+const DEFAULT_MAX_ROW_GROUP_SIZE: usize = 128 * 1024 * 1024;
+const DEFAULT_CREATED_BY: &str = env!("PARQUET_CREATED_BY");
+
+/// Parquet writer version.
+///
+/// Basic constant, which is not part of the Thrift definition.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum WriterVersion {
+    PARQUET_1_0,
+    PARQUET_2_0,
+}
+
+impl WriterVersion {
+    /// Returns writer version as `i32`.
+    pub fn as_num(&self) -> i32 {
+        match self {
+            WriterVersion::PARQUET_1_0 => 1,
+            WriterVersion::PARQUET_2_0 => 2,
+        }
+    }
+}
+
+/// Reference counted writer properties.
+pub type WriterPropertiesPtr = Rc<WriterProperties>;
+
+/// Writer properties.
+///
+/// It is created as an immutable data structure, use [`WriterPropertiesBuilder`] to
+/// assemble the properties.
+#[derive(Debug, Clone)]
+pub struct WriterProperties {
+    data_pagesize_limit: usize,
+    dictionary_pagesize_limit: usize,
+    write_batch_size: usize,
+    max_row_group_size: usize,
+    writer_version: WriterVersion,
+    created_by: String,
+    default_column_properties: ColumnProperties,
+    column_properties: HashMap<ColumnPath, ColumnProperties>,
+}
+
+impl WriterProperties {
+    /// Returns builder for writer properties with default values.
+    pub fn builder() -> WriterPropertiesBuilder {
+        WriterPropertiesBuilder::with_defaults()
+    }
+
+    /// Returns data page size limit.
+    pub fn data_pagesize_limit(&self) -> usize {
+        self.data_pagesize_limit
+    }
+
+    /// Returns dictionary page size limit.
+    pub fn dictionary_pagesize_limit(&self) -> usize {
+        self.dictionary_pagesize_limit
+    }
+
+    /// Returns configured batch size for writes.
+    ///
+    /// When writing a batch of data, this setting allows to split it internally into
+    /// smaller batches so we can better estimate the size of a page currently being
+    /// written.
+    pub fn write_batch_size(&self) -> usize {
+        self.write_batch_size
+    }
+
+    /// Returns max size for a row group.
+    pub fn max_row_group_size(&self) -> usize {
+        self.max_row_group_size
+    }
+
+    /// Returns configured writer version.
+    pub fn writer_version(&self) -> WriterVersion {
+        self.writer_version
+    }
+
+    /// Returns `created_by` string.
+    pub fn created_by(&self) -> &str {
+        &self.created_by
+    }
+
+    /// Returns encoding for a data page, when dictionary encoding is enabled.
+    /// This is not configurable.
+    #[inline]
+    pub fn dictionary_data_page_encoding(&self) -> Encoding {
+        // PLAIN_DICTIONARY encoding is deprecated in writer version 1.
+        // Dictionary values are encoded using RLE_DICTIONARY encoding.
+        Encoding::RLE_DICTIONARY
+    }
+
+    /// Returns encoding for dictionary page, when dictionary encoding is enabled.
+    /// This is not configurable.
+    #[inline]
+    pub fn dictionary_page_encoding(&self) -> Encoding {
+        // PLAIN_DICTIONARY is deprecated in writer version 1.
+        // Dictionary is encoded using plain encoding.
+        Encoding::PLAIN
+    }
+
+    /// Returns encoding for a column, if set.
+    /// In case when dictionary is enabled, returns fallback encoding.
+    ///
+    /// If encoding is not set, then column writer will choose the best encoding
+    /// based on the column type.
+    pub fn encoding(&self, col: &ColumnPath) -> Option<Encoding> {
+        self.column_properties
+            .get(col)
+            .and_then(|c| c.encoding())
+            .or_else(|| self.default_column_properties.encoding())
+    }
+
+    /// Returns compression codec for a column.
+    pub fn compression(&self, col: &ColumnPath) -> Compression {
+        self.column_properties
+            .get(col)
+            .and_then(|c| c.compression())
+            .or_else(|| self.default_column_properties.compression())
+            .unwrap_or(DEFAULT_COMPRESSION)
+    }
+
+    /// Returns `true` if dictionary encoding is enabled for a column.
+    pub fn dictionary_enabled(&self, col: &ColumnPath) -> bool {
+        self.column_properties
+            .get(col)
+            .and_then(|c| c.dictionary_enabled())
+            .or_else(|| self.default_column_properties.dictionary_enabled())
+            .unwrap_or(DEFAULT_DICTIONARY_ENABLED)
+    }
+
+    /// Returns `true` if statistics are enabled for a column.
+    pub fn statistics_enabled(&self, col: &ColumnPath) -> bool {
+        self.column_properties
+            .get(col)
+            .and_then(|c| c.statistics_enabled())
+            .or_else(|| self.default_column_properties.statistics_enabled())
+            .unwrap_or(DEFAULT_STATISTICS_ENABLED)
+    }
+
+    /// Returns max size for statistics.
+    /// Only applicable if statistics are enabled.
+    pub fn max_statistics_size(&self, col: &ColumnPath) -> usize {
+        self.column_properties
+            .get(col)
+            .and_then(|c| c.max_statistics_size())
+            .or_else(|| self.default_column_properties.max_statistics_size())
+            .unwrap_or(DEFAULT_MAX_STATISTICS_SIZE)
+    }
+}
+
+/// Writer properties builder.
+pub struct WriterPropertiesBuilder {
+    data_pagesize_limit: usize,
+    dictionary_pagesize_limit: usize,
+    write_batch_size: usize,
+    max_row_group_size: usize,
+    writer_version: WriterVersion,
+    created_by: String,
+    default_column_properties: ColumnProperties,
+    column_properties: HashMap<ColumnPath, ColumnProperties>,
+}
+
+impl WriterPropertiesBuilder {
+    /// Returns default state of the builder.
+    fn with_defaults() -> Self {
+        Self {
+            data_pagesize_limit: DEFAULT_PAGE_SIZE,
+            dictionary_pagesize_limit: DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT,
+            write_batch_size: DEFAULT_WRITE_BATCH_SIZE,
+            max_row_group_size: DEFAULT_MAX_ROW_GROUP_SIZE,
+            writer_version: DEFAULT_WRITER_VERSION,
+            created_by: DEFAULT_CREATED_BY.to_string(),
+            default_column_properties: ColumnProperties::new(),
+            column_properties: HashMap::new(),
+        }
+    }
+
+    /// Finalizes the configuration and returns immutable writer properties struct.
+    pub fn build(self) -> WriterProperties {
+        WriterProperties {
+            data_pagesize_limit: self.data_pagesize_limit,
+            dictionary_pagesize_limit: self.dictionary_pagesize_limit,
+            write_batch_size: self.write_batch_size,
+            max_row_group_size: self.max_row_group_size,
+            writer_version: self.writer_version,
+            created_by: self.created_by,
+            default_column_properties: self.default_column_properties,
+            column_properties: self.column_properties,
+        }
+    }
+
+    // ----------------------------------------------------------------------
+    // Writer properies related to a file
+
+    /// Sets writer version.
+    pub fn set_writer_version(mut self, value: WriterVersion) -> Self {
+        self.writer_version = value;
+        self
+    }
+
+    /// Sets data page size limit.
+    pub fn set_data_pagesize_limit(mut self, value: usize) -> Self {
+        self.data_pagesize_limit = value;
+        self
+    }
+
+    /// Sets dictionary page size limit.
+    pub fn set_dictionary_pagesize_limit(mut self, value: usize) -> Self {
+        self.dictionary_pagesize_limit = value;
+        self
+    }
+
+    /// Sets write batch size.
+    pub fn set_write_batch_size(mut self, value: usize) -> Self {
+        self.write_batch_size = value;
+        self
+    }
+
+    /// Sets max size for a row group.
+    pub fn set_max_row_group_size(mut self, value: usize) -> Self {
+        self.max_row_group_size = value;
+        self
+    }
+
+    /// Sets "created by" property.
+    pub fn set_created_by(mut self, value: String) -> Self {
+        self.created_by = value;
+        self
+    }
+
+    // ----------------------------------------------------------------------
+    // Setters for any column (global)
+
+    /// Sets encoding for any column.
+    ///
+    /// If dictionary is not enabled, this is treated as a primary encoding for all columns.
+    /// In case when dictionary is enabled for any column, this value is considered to
+    /// be a fallback encoding for that column.
+    ///
+    /// Panics if user tries to set dictionary encoding here, regardless of dictinoary
+    /// encoding flag being set.
+    pub fn set_encoding(mut self, value: Encoding) -> Self {
+        self.default_column_properties.set_encoding(value);
+        self
+    }
+
+    /// Sets compression codec for any column.
+    pub fn set_compression(mut self, value: Compression) -> Self {
+        self.default_column_properties.set_compression(value);
+        self
+    }
+
+    /// Sets flag to enable/disable dictionary encoding for any column.
+    ///
+    /// Use this method to set dictionary encoding, instead of explicitly specifying
+    /// encoding in `set_encoding` method.
+    pub fn set_dictionary_enabled(mut self, value: bool) -> Self {
+        self.default_column_properties.set_dictionary_enabled(value);
+        self
+    }
+
+    /// Sets flag to enable/disable statistics for any column.
+    pub fn set_statistics_enabled(mut self, value: bool) -> Self {
+        self.default_column_properties.set_statistics_enabled(value);
+        self
+    }
+
+    /// Sets max statistics size for any column.
+    /// Applicable only if statistics are enabled.
+    pub fn set_max_statistics_size(mut self, value: usize) -> Self {
+        self.default_column_properties
+            .set_max_statistics_size(value);
+        self
+    }
+
+    // ----------------------------------------------------------------------
+    // Setters for a specific column
+
+    /// Helper method to get existing or new mutable reference of column properties.
+    #[inline]
+    fn get_mut_props(&mut self, col: ColumnPath) -> &mut ColumnProperties {
+        self.column_properties
+            .entry(col)
+            .or_insert(ColumnProperties::new())
+    }
+
+    /// Sets encoding for a column.
+    /// Takes precedence over globally defined settings.
+    ///
+    /// If dictionary is not enabled, this is treated as a primary encoding for this column.
+    /// In case when dictionary is enabled for this column, either through global defaults
+    /// or explicitly, this value is considered to be a fallback encoding for this column.
+    ///
+    /// Panics if user tries to set dictionary encoding here, regardless of dictinoary
+    /// encoding flag being set.
+    pub fn set_column_encoding(mut self, col: ColumnPath, value: Encoding) -> Self {
+        self.get_mut_props(col).set_encoding(value);
+        self
+    }
+
+    /// Sets compression codec for a column.
+    /// Takes precedence over globally defined settings.
+    pub fn set_column_compression(mut self, col: ColumnPath, value: Compression) -> Self {
+        self.get_mut_props(col).set_compression(value);
+        self
+    }
+
+    /// Sets flag to enable/disable dictionary encoding for a column.
+    /// Takes precedence over globally defined settings.
+    pub fn set_column_dictionary_enabled(mut self, col: ColumnPath, value: bool) -> Self {
+        self.get_mut_props(col).set_dictionary_enabled(value);
+        self
+    }
+
+    /// Sets flag to enable/disable statistics for a column.
+    /// Takes precedence over globally defined settings.
+    pub fn set_column_statistics_enabled(mut self, col: ColumnPath, value: bool) -> Self {
+        self.get_mut_props(col).set_statistics_enabled(value);
+        self
+    }
+
+    /// Sets max size for statistics for a column.
+    /// Takes precedence over globally defined settings.
+    pub fn set_column_max_statistics_size(mut self, col: ColumnPath, value: usize) -> Self {
+        self.get_mut_props(col).set_max_statistics_size(value);
+        self
+    }
+}
+
+/// Container for column properties that can be changed as part of writer.
+///
+/// If a field is `None`, it means that no specific value has been set for this column,
+/// so some subsequent or default value must be used.
+#[derive(Debug, Clone, PartialEq)]
+struct ColumnProperties {
+    encoding: Option<Encoding>,
+    codec: Option<Compression>,
+    dictionary_enabled: Option<bool>,
+    statistics_enabled: Option<bool>,
+    max_statistics_size: Option<usize>,
+}
+
+impl ColumnProperties {
+    /// Initialise column properties with default values.
+    fn new() -> Self {
+        Self {
+            encoding: None,
+            codec: None,
+            dictionary_enabled: None,
+            statistics_enabled: None,
+            max_statistics_size: None,
+        }
+    }
+
+    /// Sets encoding for this column.
+    ///
+    /// If dictionary is not enabled, this is treated as a primary encoding for a column.
+    /// In case when dictionary is enabled for a column, this value is considered to
+    /// be a fallback encoding.
+    ///
+    /// Panics if user tries to set dictionary encoding here, regardless of dictinoary
+    /// encoding flag being set. Use `set_dictionary_enabled` method to enable dictionary
+    /// for a column.
+    fn set_encoding(&mut self, value: Encoding) {
+        if value == Encoding::PLAIN_DICTIONARY || value == Encoding::RLE_DICTIONARY {
+            panic!("Dictionary encoding can not be used as fallback encoding");
+        }
+        self.encoding = Some(value);
+    }
+
+    /// Sets compression codec for this column.
+    fn set_compression(&mut self, value: Compression) {
+        self.codec = Some(value);
+    }
+
+    /// Sets whether or not dictionary encoding is enabled for this column.
+    fn set_dictionary_enabled(&mut self, enabled: bool) {
+        self.dictionary_enabled = Some(enabled);
+    }
+
+    /// Sets whether or not statistics are enabled for this column.
+    fn set_statistics_enabled(&mut self, enabled: bool) {
+        self.statistics_enabled = Some(enabled);
+    }
+
+    /// Sets max size for statistics for this column.
+    fn set_max_statistics_size(&mut self, value: usize) {
+        self.max_statistics_size = Some(value);
+    }
+
+    /// Returns optional encoding for this column.
+    fn encoding(&self) -> Option<Encoding> {
+        self.encoding
+    }
+
+    /// Returns optional compression codec for this column.
+    fn compression(&self) -> Option<Compression> {
+        self.codec
+    }
+
+    /// Returns `Some(true)` if dictionary encoding is enabled for this column, if disabled
+    /// then returns `Some(false)`. If result is `None`, then no setting has been provided.
+    fn dictionary_enabled(&self) -> Option<bool> {
+        self.dictionary_enabled
+    }
+
+    /// Returns `Some(true)` if statistics are enabled for this column, if disabled then
+    /// returns `Some(false)`. If result is `None`, then no setting has been provided.
+    fn statistics_enabled(&self) -> Option<bool> {
+        self.statistics_enabled
+    }
+
+    /// Returns optional max size in bytes for statistics.
+    fn max_statistics_size(&self) -> Option<usize> {
+        self.max_statistics_size
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_writer_version() {
+        assert_eq!(WriterVersion::PARQUET_1_0.as_num(), 1);
+        assert_eq!(WriterVersion::PARQUET_2_0.as_num(), 2);
+    }
+
+    #[test]
+    fn test_writer_properties_default_settings() {
+        let props = WriterProperties::builder().build();
+        assert_eq!(props.data_pagesize_limit(), DEFAULT_PAGE_SIZE);
+        assert_eq!(
+            props.dictionary_pagesize_limit(),
+            DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT
+        );
+        assert_eq!(props.write_batch_size(), DEFAULT_WRITE_BATCH_SIZE);
+        assert_eq!(props.max_row_group_size(), DEFAULT_MAX_ROW_GROUP_SIZE);
+        assert_eq!(props.writer_version(), DEFAULT_WRITER_VERSION);
+        assert_eq!(props.created_by(), DEFAULT_CREATED_BY);
+        assert_eq!(props.encoding(&ColumnPath::from("col")), None);
+        assert_eq!(
+            props.compression(&ColumnPath::from("col")),
+            DEFAULT_COMPRESSION
+        );
+        assert_eq!(
+            props.dictionary_enabled(&ColumnPath::from("col")),
+            DEFAULT_DICTIONARY_ENABLED
+        );
+        assert_eq!(
+            props.statistics_enabled(&ColumnPath::from("col")),
+            DEFAULT_STATISTICS_ENABLED
+        );
+        assert_eq!(
+            props.max_statistics_size(&ColumnPath::from("col")),
+            DEFAULT_MAX_STATISTICS_SIZE
+        );
+    }
+
+    #[test]
+    fn test_writer_properties_dictionary_encoding() {
+        // dictionary encoding is not configurable, and it should be the same for both
+        // writer version 1 and 2.
+        for version in vec![WriterVersion::PARQUET_1_0, WriterVersion::PARQUET_2_0] {
+            let props = WriterProperties::builder()
+                .set_writer_version(version)
+                .build();
+            assert_eq!(props.dictionary_page_encoding(), Encoding::PLAIN);
+            assert_eq!(
+                props.dictionary_data_page_encoding(),
+                Encoding::RLE_DICTIONARY
+            );
+        }
+    }
+
+    #[test]
+    #[should_panic(expected = "Dictionary encoding can not be used as fallback encoding")]
+    fn test_writer_properties_panic_when_plain_dictionary_is_fallback() {
+        // Should panic when user specifies dictionary encoding as fallback encoding.
+        WriterProperties::builder()
+            .set_encoding(Encoding::PLAIN_DICTIONARY)
+            .build();
+    }
+
+    #[test]
+    #[should_panic(expected = "Dictionary encoding can not be used as fallback encoding")]
+    fn test_writer_properties_panic_when_rle_dictionary_is_fallback() {
+        // Should panic when user specifies dictionary encoding as fallback encoding.
+        WriterProperties::builder()
+            .set_encoding(Encoding::RLE_DICTIONARY)
+            .build();
+    }
+
+    #[test]
+    #[should_panic(expected = "Dictionary encoding can not be used as fallback encoding")]
+    fn test_writer_properties_panic_when_dictionary_is_enabled() {
+        WriterProperties::builder()
+            .set_dictionary_enabled(true)
+            .set_column_encoding(ColumnPath::from("col"), Encoding::RLE_DICTIONARY)
+            .build();
+    }
+
+    #[test]
+    #[should_panic(expected = "Dictionary encoding can not be used as fallback encoding")]
+    fn test_writer_properties_panic_when_dictionary_is_disabled() {
+        WriterProperties::builder()
+            .set_dictionary_enabled(false)
+            .set_column_encoding(ColumnPath::from("col"), Encoding::RLE_DICTIONARY)
+            .build();
+    }
+
+    #[test]
+    fn test_writer_properties_builder() {
+        let props = WriterProperties::builder()
+            // file settings
+            .set_writer_version(WriterVersion::PARQUET_2_0)
+            .set_data_pagesize_limit(10)
+            .set_dictionary_pagesize_limit(20)
+            .set_write_batch_size(30)
+            .set_max_row_group_size(40)
+            .set_created_by("default".to_owned())
+            // global column settings
+            .set_encoding(Encoding::DELTA_BINARY_PACKED)
+            .set_compression(Compression::GZIP)
+            .set_dictionary_enabled(false)
+            .set_statistics_enabled(false)
+            .set_max_statistics_size(50)
+            // specific column settings
+            .set_column_encoding(ColumnPath::from("col"), Encoding::RLE)
+            .set_column_compression(ColumnPath::from("col"), Compression::SNAPPY)
+            .set_column_dictionary_enabled(ColumnPath::from("col"), true)
+            .set_column_statistics_enabled(ColumnPath::from("col"), true)
+            .set_column_max_statistics_size(ColumnPath::from("col"), 123)
+            .build();
+
+        assert_eq!(props.writer_version(), WriterVersion::PARQUET_2_0);
+        assert_eq!(props.data_pagesize_limit(), 10);
+        assert_eq!(props.dictionary_pagesize_limit(), 20);
+        assert_eq!(props.write_batch_size(), 30);
+        assert_eq!(props.max_row_group_size(), 40);
+        assert_eq!(props.created_by(), "default");
+
+        assert_eq!(
+            props.encoding(&ColumnPath::from("a")),
+            Some(Encoding::DELTA_BINARY_PACKED)
+        );
+        assert_eq!(props.compression(&ColumnPath::from("a")), Compression::GZIP);
+        assert_eq!(props.dictionary_enabled(&ColumnPath::from("a")), false);
+        assert_eq!(props.statistics_enabled(&ColumnPath::from("a")), false);
+        assert_eq!(props.max_statistics_size(&ColumnPath::from("a")), 50);
+
+        assert_eq!(
+            props.encoding(&ColumnPath::from("col")),
+            Some(Encoding::RLE)
+        );
+        assert_eq!(
+            props.compression(&ColumnPath::from("col")),
+            Compression::SNAPPY
+        );
+        assert_eq!(props.dictionary_enabled(&ColumnPath::from("col")), true);
+        assert_eq!(props.statistics_enabled(&ColumnPath::from("col")), true);
+        assert_eq!(props.max_statistics_size(&ColumnPath::from("col")), 123);
+    }
+
+    #[test]
+    fn test_writer_properties_builder_partial_defaults() {
+        let props = WriterProperties::builder()
+            .set_encoding(Encoding::DELTA_BINARY_PACKED)
+            .set_compression(Compression::GZIP)
+            .set_column_encoding(ColumnPath::from("col"), Encoding::RLE)
+            .build();
+
+        assert_eq!(
+            props.encoding(&ColumnPath::from("col")),
+            Some(Encoding::RLE)
+        );
+        assert_eq!(
+            props.compression(&ColumnPath::from("col")),
+            Compression::GZIP
+        );
+        assert_eq!(
+            props.dictionary_enabled(&ColumnPath::from("col")),
+            DEFAULT_DICTIONARY_ENABLED
+        );
+    }
+}

rust/src/parquet/file/reader.rs

@@ -0,0 +1,899 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Contains file reader API and provides methods to access file metadata, row group
+//! readers to read individual column chunks, or access record iterator.
+
+use std::{
+    convert::TryFrom,
+    fs::File,
+    io::{BufReader, Cursor, Read, Seek, SeekFrom},
+    path::Path,
+    rc::Rc,
+};
+
+use byteorder::{ByteOrder, LittleEndian};
+use parquet_format::{
+    ColumnOrder as TColumnOrder, FileMetaData as TFileMetaData, PageHeader, PageType,
+};
+use thrift::protocol::TCompactInputProtocol;
+
+use crate::parquet::basic::{ColumnOrder, Compression, Encoding, Type};
+use crate::parquet::column::{
+    page::{Page, PageReader},
+    reader::{ColumnReader, ColumnReaderImpl},
+};
+use crate::parquet::compression::{create_codec, Codec};
+use crate::parquet::errors::{ParquetError, Result};
+use crate::parquet::file::{metadata::*, statistics, FOOTER_SIZE, PARQUET_MAGIC};
+use crate::parquet::record::reader::RowIter;
+use crate::parquet::schema::types::{self, SchemaDescriptor, Type as SchemaType};
+use crate::parquet::util::{io::FileSource, memory::ByteBufferPtr};
+
+// ----------------------------------------------------------------------
+// APIs for file & row group readers
+
+/// Parquet file reader API. With this, user can get metadata information about the
+/// Parquet file, can get reader for each row group, and access record iterator.
+pub trait FileReader {
+    /// Get metadata information about this file.
+    fn metadata(&self) -> ParquetMetaDataPtr;
+
+    /// Get the total number of row groups for this file.
+    fn num_row_groups(&self) -> usize;
+
+    /// Get the `i`th row group reader. Note this doesn't do bound check.
+    fn get_row_group(&self, i: usize) -> Result<Box<RowGroupReader>>;
+
+    /// Get full iterator of `Row`s from a file (over all row groups).
+    ///
+    /// Iterator will automatically load the next row group to advance.
+    ///
+    /// Projected schema can be a subset of or equal to the file schema, when it is None,
+    /// full file schema is assumed.
+    fn get_row_iter(&self, projection: Option<SchemaType>) -> Result<RowIter>;
+}
+
+/// Parquet row group reader API. With this, user can get metadata information about the
+/// row group, as well as readers for each individual column chunk.
+pub trait RowGroupReader {
+    /// Get metadata information about this row group.
+    fn metadata(&self) -> RowGroupMetaDataPtr;
+
+    /// Get the total number of column chunks in this row group.
+    fn num_columns(&self) -> usize;
+
+    /// Get page reader for the `i`th column chunk.
+    fn get_column_page_reader(&self, i: usize) -> Result<Box<PageReader>>;
+
+    /// Get value reader for the `i`th column chunk.
+    fn get_column_reader(&self, i: usize) -> Result<ColumnReader>;
+
+    /// Get iterator of `Row`s from this row group.
+    ///
+    /// Projected schema can be a subset of or equal to the file schema, when it is None,
+    /// full file schema is assumed.
+    fn get_row_iter(&self, projection: Option<SchemaType>) -> Result<RowIter>;
+}
+
+// ----------------------------------------------------------------------
+// Serialized impl for file & row group readers
+
+/// Length should return the amount of bytes that implementor contains.
+/// It's mainly used to read the metadata, which is at the end of the source.
+pub trait Length {
+    /// Returns the amount of bytes of the inner source.
+    fn len(&self) -> u64;
+}
+
+/// TryClone tries to clone the type and should maintain the `Seek` position of the given
+/// instance.
+pub trait TryClone: Sized {
+    /// Clones the type returning a new instance or an error if it's not possible
+    /// to clone it.
+    fn try_clone(&self) -> Result<Self>;
+}
+
+impl Length for File {
+    fn len(&self) -> u64 {
+        self.metadata().map(|m| m.len()).unwrap_or(0u64)
+    }
+}
+
+impl TryClone for File {
+    fn try_clone(&self) -> Result<Self> {
+        self.try_clone().map_err(|e| e.into())
+    }
+}
+
+impl<'a> Length for Cursor<&'a [u8]> {
+    fn len(&self) -> u64 {
+        self.get_ref().len() as u64
+    }
+}
+
+impl<'a> TryClone for Cursor<&'a [u8]> {
+    fn try_clone(&self) -> Result<Self> {
+        Ok(self.clone())
+    }
+}
+
+/// ParquetReader is the interface which needs to be fulfilled to be able to parse a
+/// parquet source.
+pub trait ParquetReader: Read + Seek + Length + TryClone {}
+impl<T: Read + Seek + Length + TryClone> ParquetReader for T {}
+
+/// A serialized implementation for Parquet [`FileReader`].
+pub struct SerializedFileReader<R: ParquetReader> {
+    buf: BufReader<R>,
+    metadata: ParquetMetaDataPtr,
+}
+
+impl<R: ParquetReader> SerializedFileReader<R> {
+    /// Creates file reader from a Parquet file.
+    /// Returns error if Parquet file does not exist or is corrupt.
+    pub fn new(reader: R) -> Result<Self> {
+        let mut buf = BufReader::new(reader);
+        let metadata = Self::parse_metadata(&mut buf)?;
+        Ok(Self {
+            buf,
+            metadata: Rc::new(metadata),
+        })
+    }
+
+    // Layout of Parquet file
+    // +---------------------------+---+-----+
+    // |      Rest of file         | B |  A  |
+    // +---------------------------+---+-----+
+    // where A: parquet footer, B: parquet metadata.
+    //
+    fn parse_metadata(buf: &mut BufReader<R>) -> Result<ParquetMetaData> {
+        let file_size = buf.get_ref().len();
+        if file_size < (FOOTER_SIZE as u64) {
+            return Err(general_err!(
+                "Invalid Parquet file. Size is smaller than footer"
+            ));
+        }
+        let mut footer_buffer: [u8; FOOTER_SIZE] = [0; FOOTER_SIZE];
+        buf.seek(SeekFrom::End(-(FOOTER_SIZE as i64)))?;
+        buf.read_exact(&mut footer_buffer)?;
+        if footer_buffer[4..] != PARQUET_MAGIC {
+            return Err(general_err!("Invalid Parquet file. Corrupt footer"));
+        }
+        let metadata_len = LittleEndian::read_i32(&footer_buffer[0..4]) as i64;
+        if metadata_len < 0 {
+            return Err(general_err!(
+                "Invalid Parquet file. Metadata length is less than zero ({})",
+                metadata_len
+            ));
+        }
+        let metadata_start: i64 = file_size as i64 - FOOTER_SIZE as i64 - metadata_len;
+        if metadata_start < 0 {
+            return Err(general_err!(
+                "Invalid Parquet file. Metadata start is less than zero ({})",
+                metadata_start
+            ));
+        }
+        buf.seek(SeekFrom::Start(metadata_start as u64))?;
+        let metadata_buf = buf.take(metadata_len as u64).into_inner();
+
+        // TODO: row group filtering
+        let mut prot = TCompactInputProtocol::new(metadata_buf);
+        let mut t_file_metadata: TFileMetaData = TFileMetaData::read_from_in_protocol(&mut prot)
+            .map_err(|e| ParquetError::General(format!("Could not parse metadata: {}", e)))?;
+        let schema = types::from_thrift(&mut t_file_metadata.schema)?;
+        let schema_descr = Rc::new(SchemaDescriptor::new(schema.clone()));
+        let mut row_groups = Vec::new();
+        for rg in t_file_metadata.row_groups {
+            row_groups.push(Rc::new(RowGroupMetaData::from_thrift(
+                schema_descr.clone(),
+                rg,
+            )?));
+        }
+        let column_orders = Self::parse_column_orders(t_file_metadata.column_orders, &schema_descr);
+
+        let file_metadata = FileMetaData::new(
+            t_file_metadata.version,
+            t_file_metadata.num_rows,
+            t_file_metadata.created_by,
+            schema,
+            schema_descr,
+            column_orders,
+        );
+        Ok(ParquetMetaData::new(file_metadata, row_groups))
+    }
+
+    /// Parses column orders from Thrift definition.
+    /// If no column orders are defined, returns `None`.
+    fn parse_column_orders(
+        t_column_orders: Option<Vec<TColumnOrder>>,
+        schema_descr: &SchemaDescriptor,
+    ) -> Option<Vec<ColumnOrder>> {
+        match t_column_orders {
+            Some(orders) => {
+                // Should always be the case
+                assert_eq!(
+                    orders.len(),
+                    schema_descr.num_columns(),
+                    "Column order length mismatch"
+                );
+                let mut res = Vec::new();
+                for (i, column) in schema_descr.columns().iter().enumerate() {
+                    match orders[i] {
+                        TColumnOrder::TYPEORDER(_) => {
+                            let sort_order = ColumnOrder::get_sort_order(
+                                column.logical_type(),
+                                column.physical_type(),
+                            );
+                            res.push(ColumnOrder::TYPE_DEFINED_ORDER(sort_order));
+                        }
+                    }
+                }
+                Some(res)
+            }
+            None => None,
+        }
+    }
+}
+
+impl<R: 'static + ParquetReader> FileReader for SerializedFileReader<R> {
+    fn metadata(&self) -> ParquetMetaDataPtr {
+        self.metadata.clone()
+    }
+
+    fn num_row_groups(&self) -> usize {
+        self.metadata.num_row_groups()
+    }
+
+    fn get_row_group(&self, i: usize) -> Result<Box<RowGroupReader>> {
+        let row_group_metadata = self.metadata.row_group(i);
+        // Row groups should be processed sequentially.
+        let f = self.buf.get_ref().try_clone()?;
+        Ok(Box::new(SerializedRowGroupReader::new(
+            f,
+            row_group_metadata,
+        )))
+    }
+
+    fn get_row_iter(&self, projection: Option<SchemaType>) -> Result<RowIter> {
+        RowIter::from_file(projection, self)
+    }
+}
+
+impl TryFrom<File> for SerializedFileReader<File> {
+    type Error = ParquetError;
+
+    fn try_from(file: File) -> Result<Self> {
+        Self::new(file)
+    }
+}
+
+impl<'a> TryFrom<&'a Path> for SerializedFileReader<File> {
+    type Error = ParquetError;
+
+    fn try_from(path: &Path) -> Result<Self> {
+        let file = File::open(path)?;
+        Self::try_from(file)
+    }
+}
+
+impl TryFrom<String> for SerializedFileReader<File> {
+    type Error = ParquetError;
+
+    fn try_from(path: String) -> Result<Self> {
+        Self::try_from(Path::new(&path))
+    }
+}
+
+impl<'a> TryFrom<&'a str> for SerializedFileReader<File> {
+    type Error = ParquetError;
+
+    fn try_from(path: &str) -> Result<Self> {
+        Self::try_from(Path::new(&path))
+    }
+}
+
+/// A serialized implementation for Parquet [`RowGroupReader`].
+pub struct SerializedRowGroupReader<R: ParquetReader> {
+    buf: BufReader<R>,
+    metadata: RowGroupMetaDataPtr,
+}
+
+impl<R: 'static + ParquetReader> SerializedRowGroupReader<R> {
+    /// Creates new row group reader from a file and row group metadata.
+    fn new(file: R, metadata: RowGroupMetaDataPtr) -> Self {
+        let buf = BufReader::new(file);
+        Self { buf, metadata }
+    }
+}
+
+impl<R: 'static + ParquetReader> RowGroupReader for SerializedRowGroupReader<R> {
+    fn metadata(&self) -> RowGroupMetaDataPtr {
+        self.metadata.clone()
+    }
+
+    fn num_columns(&self) -> usize {
+        self.metadata.num_columns()
+    }
+
+    // TODO: fix PARQUET-816
+    fn get_column_page_reader(&self, i: usize) -> Result<Box<PageReader>> {
+        let col = self.metadata.column(i);
+        let mut col_start = col.data_page_offset();
+        if col.has_dictionary_page() {
+            col_start = col.dictionary_page_offset().unwrap();
+        }
+        let col_length = col.compressed_size();
+        let file_chunk = FileSource::new(self.buf.get_ref(), col_start as u64, col_length as usize);
+        let page_reader = SerializedPageReader::new(
+            file_chunk,
+            col.num_values(),
+            col.compression(),
+            col.column_descr().physical_type(),
+        )?;
+        Ok(Box::new(page_reader))
+    }
+
+    fn get_column_reader(&self, i: usize) -> Result<ColumnReader> {
+        let schema_descr = self.metadata.schema_descr();
+        let col_descr = schema_descr.column(i);
+        let col_page_reader = self.get_column_page_reader(i)?;
+        let col_reader = match col_descr.physical_type() {
+            Type::BOOLEAN => {
+                ColumnReader::BoolColumnReader(ColumnReaderImpl::new(col_descr, col_page_reader))
+            }
+            Type::INT32 => {
+                ColumnReader::Int32ColumnReader(ColumnReaderImpl::new(col_descr, col_page_reader))
+            }
+            Type::INT64 => {
+                ColumnReader::Int64ColumnReader(ColumnReaderImpl::new(col_descr, col_page_reader))
+            }
+            Type::INT96 => {
+                ColumnReader::Int96ColumnReader(ColumnReaderImpl::new(col_descr, col_page_reader))
+            }
+            Type::FLOAT => {
+                ColumnReader::FloatColumnReader(ColumnReaderImpl::new(col_descr, col_page_reader))
+            }
+            Type::DOUBLE => {
+                ColumnReader::DoubleColumnReader(ColumnReaderImpl::new(col_descr, col_page_reader))
+            }
+            Type::BYTE_ARRAY => ColumnReader::ByteArrayColumnReader(ColumnReaderImpl::new(
+                col_descr,
+                col_page_reader,
+            )),
+            Type::FIXED_LEN_BYTE_ARRAY => ColumnReader::FixedLenByteArrayColumnReader(
+                ColumnReaderImpl::new(col_descr, col_page_reader),
+            ),
+        };
+        Ok(col_reader)
+    }
+
+    fn get_row_iter(&self, projection: Option<SchemaType>) -> Result<RowIter> {
+        RowIter::from_row_group(projection, self)
+    }
+}
+
+/// A serialized implementation for Parquet [`PageReader`].
+pub struct SerializedPageReader<T: Read> {
+    // The file source buffer which references exactly the bytes for the column trunk
+    // to be read by this page reader.
+    buf: T,
+
+    // The compression codec for this column chunk. Only set for non-PLAIN codec.
+    decompressor: Option<Box<Codec>>,
+
+    // The number of values we have seen so far.
+    seen_num_values: i64,
+
+    // The number of total values in this column chunk.
+    total_num_values: i64,
+
+    // Column chunk type.
+    physical_type: Type,
+}
+
+impl<T: Read> SerializedPageReader<T> {
+    /// Creates a new serialized page reader from file source.
+    pub fn new(
+        buf: T,
+        total_num_values: i64,
+        compression: Compression,
+        physical_type: Type,
+    ) -> Result<Self> {
+        let decompressor = create_codec(compression)?;
+        let result = Self {
+            buf,
+            total_num_values,
+            seen_num_values: 0,
+            decompressor,
+            physical_type,
+        };
+        Ok(result)
+    }
+
+    /// Reads Page header from Thrift.
+    fn read_page_header(&mut self) -> Result<PageHeader> {
+        let mut prot = TCompactInputProtocol::new(&mut self.buf);
+        let page_header = PageHeader::read_from_in_protocol(&mut prot)?;
+        Ok(page_header)
+    }
+}
+
+impl<T: Read> PageReader for SerializedPageReader<T> {
+    fn get_next_page(&mut self) -> Result<Option<Page>> {
+        while self.seen_num_values < self.total_num_values {
+            let page_header = self.read_page_header()?;
+
+            // When processing data page v2, depending on enabled compression for the page, we
+            // should account for uncompressed data ('offset') of repetition and definition
+            // levels.
+            //
+            // We always use 0 offset for other pages other than v2, `true` flag means that
+            // compression will be applied if decompressor is defined
+            let mut offset: usize = 0;
+            let mut can_decompress = true;
+
+            if let Some(ref header_v2) = page_header.data_page_header_v2 {
+                offset = (header_v2.definition_levels_byte_length
+                    + header_v2.repetition_levels_byte_length) as usize;
+                // When is_compressed flag is missing the page is considered compressed
+                can_decompress = header_v2.is_compressed.unwrap_or(true);
+            }
+
+            let compressed_len = page_header.compressed_page_size as usize - offset;
+            let uncompressed_len = page_header.uncompressed_page_size as usize - offset;
+            // We still need to read all bytes from buffered stream
+            let mut buffer = vec![0; offset + compressed_len];
+            self.buf.read_exact(&mut buffer)?;
+
+            // TODO: page header could be huge because of statistics. We should set a maximum
+            // page header size and abort if that is exceeded.
+            if let Some(decompressor) = self.decompressor.as_mut() {
+                if can_decompress {
+                    let mut decompressed_buffer = Vec::with_capacity(uncompressed_len);
+                    let decompressed_size =
+                        decompressor.decompress(&buffer[offset..], &mut decompressed_buffer)?;
+                    if decompressed_size != uncompressed_len {
+                        return Err(general_err!(
+                            "Actual decompressed size doesn't match the expected one ({} vs {})",
+                            decompressed_size,
+                            uncompressed_len
+                        ));
+                    }
+                    if offset == 0 {
+                        buffer = decompressed_buffer;
+                    } else {
+                        // Prepend saved offsets to the buffer
+                        buffer.truncate(offset);
+                        buffer.append(&mut decompressed_buffer);
+                    }
+                }
+            }
+
+            let result = match page_header.type_ {
+                PageType::DICTIONARY_PAGE => {
+                    assert!(page_header.dictionary_page_header.is_some());
+                    let dict_header = page_header.dictionary_page_header.as_ref().unwrap();
+                    let is_sorted = dict_header.is_sorted.unwrap_or(false);
+                    Page::DictionaryPage {
+                        buf: ByteBufferPtr::new(buffer),
+                        num_values: dict_header.num_values as u32,
+                        encoding: Encoding::from(dict_header.encoding),
+                        is_sorted,
+                    }
+                }
+                PageType::DATA_PAGE => {
+                    assert!(page_header.data_page_header.is_some());
+                    let header = page_header.data_page_header.unwrap();
+                    self.seen_num_values += header.num_values as i64;
+                    Page::DataPage {
+                        buf: ByteBufferPtr::new(buffer),
+                        num_values: header.num_values as u32,
+                        encoding: Encoding::from(header.encoding),
+                        def_level_encoding: Encoding::from(header.definition_level_encoding),
+                        rep_level_encoding: Encoding::from(header.repetition_level_encoding),
+                        statistics: statistics::from_thrift(self.physical_type, header.statistics),
+                    }
+                }
+                PageType::DATA_PAGE_V2 => {
+                    assert!(page_header.data_page_header_v2.is_some());
+                    let header = page_header.data_page_header_v2.unwrap();
+                    let is_compressed = header.is_compressed.unwrap_or(true);
+                    self.seen_num_values += header.num_values as i64;
+                    Page::DataPageV2 {
+                        buf: ByteBufferPtr::new(buffer),
+                        num_values: header.num_values as u32,
+                        encoding: Encoding::from(header.encoding),
+                        num_nulls: header.num_nulls as u32,
+                        num_rows: header.num_rows as u32,
+                        def_levels_byte_len: header.definition_levels_byte_length as u32,
+                        rep_levels_byte_len: header.repetition_levels_byte_length as u32,
+                        is_compressed,
+                        statistics: statistics::from_thrift(self.physical_type, header.statistics),
+                    }
+                }
+                _ => {
+                    // For unknown page type (e.g., INDEX_PAGE), skip and read next.
+                    continue;
+                }
+            };
+            return Ok(Some(result));
+        }
+
+        // We are at the end of this column chunk and no more page left. Return None.
+        Ok(None)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use parquet_format::TypeDefinedOrder;
+
+    use crate::parquet::basic::SortOrder;
+    use crate::parquet::util::test_common::{get_temp_file, get_test_file, get_test_path};
+
+    #[test]
+    fn test_file_reader_metadata_size_smaller_than_footer() {
+        let test_file = get_temp_file("corrupt-1.parquet", &[]);
+        let reader_result = SerializedFileReader::new(test_file);
+        assert!(reader_result.is_err());
+        assert_eq!(
+            reader_result.err().unwrap(),
+            general_err!("Invalid Parquet file. Size is smaller than footer")
+        );
+    }
+
+    // #[test]
+    // fn test_cursor_and_file_has_the_same_behaviour() {
+    //     let path = get_test_path("alltypes_plain.parquet");
+    //     let buffer = include_bytes!(path);
+    //     let cursor = Cursor::new(buffer.as_ref());
+
+    //     let read_from_file =
+    //         SerializedFileReader::new(File::open("testdata/alltypes_plain.parquet").unwrap())
+    //             .unwrap();
+    //     let read_from_cursor = SerializedFileReader::new(cursor).unwrap();
+
+    //     let file_iter = read_from_file.get_row_iter(None).unwrap();
+    //     let cursor_iter = read_from_cursor.get_row_iter(None).unwrap();
+
+    //     assert!(file_iter.eq(cursor_iter));
+    // }
+
+    #[test]
+    fn test_file_reader_metadata_corrupt_footer() {
+        let test_file = get_temp_file("corrupt-2.parquet", &[1, 2, 3, 4, 5, 6, 7, 8]);
+        let reader_result = SerializedFileReader::new(test_file);
+        assert!(reader_result.is_err());
+        assert_eq!(
+            reader_result.err().unwrap(),
+            general_err!("Invalid Parquet file. Corrupt footer")
+        );
+    }
+
+    #[test]
+    fn test_file_reader_metadata_invalid_length() {
+        let test_file = get_temp_file("corrupt-3.parquet", &[0, 0, 0, 255, b'P', b'A', b'R', b'1']);
+        let reader_result = SerializedFileReader::new(test_file);
+        assert!(reader_result.is_err());
+        assert_eq!(
+            reader_result.err().unwrap(),
+            general_err!("Invalid Parquet file. Metadata length is less than zero (-16777216)")
+        );
+    }
+
+    #[test]
+    fn test_file_reader_metadata_invalid_start() {
+        let test_file = get_temp_file("corrupt-4.parquet", &[255, 0, 0, 0, b'P', b'A', b'R', b'1']);
+        let reader_result = SerializedFileReader::new(test_file);
+        assert!(reader_result.is_err());
+        assert_eq!(
+            reader_result.err().unwrap(),
+            general_err!("Invalid Parquet file. Metadata start is less than zero (-255)")
+        );
+    }
+
+    #[test]
+    fn test_file_reader_column_orders_parse() {
+        // Define simple schema, we do not need to provide logical types.
+        let mut fields = vec![
+            Rc::new(
+                SchemaType::primitive_type_builder("col1", Type::INT32)
+                    .build()
+                    .unwrap(),
+            ),
+            Rc::new(
+                SchemaType::primitive_type_builder("col2", Type::FLOAT)
+                    .build()
+                    .unwrap(),
+            ),
+        ];
+        let schema = SchemaType::group_type_builder("schema")
+            .with_fields(&mut fields)
+            .build()
+            .unwrap();
+        let schema_descr = SchemaDescriptor::new(Rc::new(schema));
+
+        let t_column_orders = Some(vec![
+            TColumnOrder::TYPEORDER(TypeDefinedOrder::new()),
+            TColumnOrder::TYPEORDER(TypeDefinedOrder::new()),
+        ]);
+
+        assert_eq!(
+            SerializedFileReader::<File>::parse_column_orders(t_column_orders, &schema_descr),
+            Some(vec![
+                ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::SIGNED),
+                ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::SIGNED)
+            ])
+        );
+
+        // Test when no column orders are defined.
+        assert_eq!(
+            SerializedFileReader::<File>::parse_column_orders(None, &schema_descr),
+            None
+        );
+    }
+
+    #[test]
+    #[should_panic(expected = "Column order length mismatch")]
+    fn test_file_reader_column_orders_len_mismatch() {
+        let schema = SchemaType::group_type_builder("schema").build().unwrap();
+        let schema_descr = SchemaDescriptor::new(Rc::new(schema));
+
+        let t_column_orders = Some(vec![TColumnOrder::TYPEORDER(TypeDefinedOrder::new())]);
+
+        SerializedFileReader::<File>::parse_column_orders(t_column_orders, &schema_descr);
+    }
+
+    #[test]
+    fn test_file_reader_try_from() {
+        // Valid file path
+        let test_file = get_test_file("alltypes_plain.parquet");
+        let test_path_buf = get_test_path("alltypes_plain.parquet");
+        let test_path = test_path_buf.as_path();
+        let test_path_str = test_path.to_str().unwrap();
+
+        let reader = SerializedFileReader::try_from(test_file);
+        assert!(reader.is_ok());
+
+        let reader = SerializedFileReader::try_from(test_path);
+        assert!(reader.is_ok());
+
+        let reader = SerializedFileReader::try_from(test_path_str);
+        assert!(reader.is_ok());
+
+        let reader = SerializedFileReader::try_from(test_path_str.to_string());
+        assert!(reader.is_ok());
+
+        // Invalid file path
+        let test_path = Path::new("invalid.parquet");
+        let test_path_str = test_path.to_str().unwrap();
+
+        let reader = SerializedFileReader::try_from(test_path);
+        assert!(reader.is_err());
+
+        let reader = SerializedFileReader::try_from(test_path_str);
+        assert!(reader.is_err());
+
+        let reader = SerializedFileReader::try_from(test_path_str.to_string());
+        assert!(reader.is_err());
+    }
+
+    #[test]
+    fn test_reuse_file_chunk() {
+        // This test covers the case of maintaining the correct start position in a file
+        // stream for each column reader after initializing and moving to the next one
+        // (without necessarily reading the entire column).
+        let test_file = get_test_file("alltypes_plain.parquet");
+        let reader = SerializedFileReader::new(test_file).unwrap();
+        let row_group = reader.get_row_group(0).unwrap();
+
+        let mut page_readers = Vec::new();
+        for i in 0..row_group.num_columns() {
+            page_readers.push(row_group.get_column_page_reader(i).unwrap());
+        }
+
+        // Now buffer each col reader, we do not expect any failures like:
+        // General("underlying Thrift error: end of file")
+        for mut page_reader in page_readers {
+            assert!(page_reader.get_next_page().is_ok());
+        }
+    }
+
+    #[test]
+    fn test_file_reader() {
+        let test_file = get_test_file("alltypes_plain.parquet");
+        let reader_result = SerializedFileReader::new(test_file);
+        assert!(reader_result.is_ok());
+        let reader = reader_result.unwrap();
+
+        // Test contents in Parquet metadata
+        let metadata = reader.metadata();
+        assert_eq!(metadata.num_row_groups(), 1);
+
+        // Test contents in file metadata
+        let file_metadata = metadata.file_metadata();
+        assert!(file_metadata.created_by().is_some());
+        assert_eq!(
+            file_metadata.created_by().as_ref().unwrap(),
+            "impala version 1.3.0-INTERNAL (build 8a48ddb1eff84592b3fc06bc6f51ec120e1fffc9)"
+        );
+        assert_eq!(file_metadata.num_rows(), 8);
+        assert_eq!(file_metadata.version(), 1);
+        assert_eq!(file_metadata.column_orders(), None);
+
+        // Test contents in row group metadata
+        let row_group_metadata = metadata.row_group(0);
+        assert_eq!(row_group_metadata.num_columns(), 11);
+        assert_eq!(row_group_metadata.num_rows(), 8);
+        assert_eq!(row_group_metadata.total_byte_size(), 671);
+        // Check each column order
+        for i in 0..row_group_metadata.num_columns() {
+            assert_eq!(file_metadata.column_order(i), ColumnOrder::UNDEFINED);
+        }
+
+        // Test row group reader
+        let row_group_reader_result = reader.get_row_group(0);
+        assert!(row_group_reader_result.is_ok());
+        let row_group_reader: Box<RowGroupReader> = row_group_reader_result.unwrap();
+        assert_eq!(
+            row_group_reader.num_columns(),
+            row_group_metadata.num_columns()
+        );
+        assert_eq!(
+            row_group_reader.metadata().total_byte_size(),
+            row_group_metadata.total_byte_size()
+        );
+
+        // Test page readers
+        // TODO: test for every column
+        let page_reader_0_result = row_group_reader.get_column_page_reader(0);
+        assert!(page_reader_0_result.is_ok());
+        let mut page_reader_0: Box<PageReader> = page_reader_0_result.unwrap();
+        let mut page_count = 0;
+        while let Ok(Some(page)) = page_reader_0.get_next_page() {
+            let is_expected_page = match page {
+                Page::DictionaryPage {
+                    buf,
+                    num_values,
+                    encoding,
+                    is_sorted,
+                } => {
+                    assert_eq!(buf.len(), 32);
+                    assert_eq!(num_values, 8);
+                    assert_eq!(encoding, Encoding::PLAIN_DICTIONARY);
+                    assert_eq!(is_sorted, false);
+                    true
+                }
+                Page::DataPage {
+                    buf,
+                    num_values,
+                    encoding,
+                    def_level_encoding,
+                    rep_level_encoding,
+                    statistics,
+                } => {
+                    assert_eq!(buf.len(), 11);
+                    assert_eq!(num_values, 8);
+                    assert_eq!(encoding, Encoding::PLAIN_DICTIONARY);
+                    assert_eq!(def_level_encoding, Encoding::RLE);
+                    assert_eq!(rep_level_encoding, Encoding::BIT_PACKED);
+                    assert!(statistics.is_none());
+                    true
+                }
+                _ => false,
+            };
+            assert!(is_expected_page);
+            page_count += 1;
+        }
+        assert_eq!(page_count, 2);
+    }
+
+    #[test]
+    fn test_file_reader_datapage_v2() {
+        let test_file = get_test_file("datapage_v2.snappy.parquet");
+        let reader_result = SerializedFileReader::new(test_file);
+        assert!(reader_result.is_ok());
+        let reader = reader_result.unwrap();
+
+        // Test contents in Parquet metadata
+        let metadata = reader.metadata();
+        assert_eq!(metadata.num_row_groups(), 1);
+
+        // Test contents in file metadata
+        let file_metadata = metadata.file_metadata();
+        assert!(file_metadata.created_by().is_some());
+        assert_eq!(
+            file_metadata.created_by().as_ref().unwrap(),
+            "parquet-mr version 1.8.1 (build 4aba4dae7bb0d4edbcf7923ae1339f28fd3f7fcf)"
+        );
+        assert_eq!(file_metadata.num_rows(), 5);
+        assert_eq!(file_metadata.version(), 1);
+        assert_eq!(file_metadata.column_orders(), None);
+
+        let row_group_metadata = metadata.row_group(0);
+
+        // Check each column order
+        for i in 0..row_group_metadata.num_columns() {
+            assert_eq!(file_metadata.column_order(i), ColumnOrder::UNDEFINED);
+        }
+
+        // Test row group reader
+        let row_group_reader_result = reader.get_row_group(0);
+        assert!(row_group_reader_result.is_ok());
+        let row_group_reader: Box<RowGroupReader> = row_group_reader_result.unwrap();
+        assert_eq!(
+            row_group_reader.num_columns(),
+            row_group_metadata.num_columns()
+        );
+        assert_eq!(
+            row_group_reader.metadata().total_byte_size(),
+            row_group_metadata.total_byte_size()
+        );
+
+        // Test page readers
+        // TODO: test for every column
+        let page_reader_0_result = row_group_reader.get_column_page_reader(0);
+        assert!(page_reader_0_result.is_ok());
+        let mut page_reader_0: Box<PageReader> = page_reader_0_result.unwrap();
+        let mut page_count = 0;
+        while let Ok(Some(page)) = page_reader_0.get_next_page() {
+            let is_expected_page = match page {
+                Page::DictionaryPage {
+                    buf,
+                    num_values,
+                    encoding,
+                    is_sorted,
+                } => {
+                    assert_eq!(buf.len(), 7);
+                    assert_eq!(num_values, 1);
+                    assert_eq!(encoding, Encoding::PLAIN);
+                    assert_eq!(is_sorted, false);
+                    true
+                }
+                Page::DataPageV2 {
+                    buf,
+                    num_values,
+                    encoding,
+                    num_nulls,
+                    num_rows,
+                    def_levels_byte_len,
+                    rep_levels_byte_len,
+                    is_compressed,
+                    statistics,
+                } => {
+                    assert_eq!(buf.len(), 4);
+                    assert_eq!(num_values, 5);
+                    assert_eq!(encoding, Encoding::RLE_DICTIONARY);
+                    assert_eq!(num_nulls, 1);
+                    assert_eq!(num_rows, 5);
+                    assert_eq!(def_levels_byte_len, 2);
+                    assert_eq!(rep_levels_byte_len, 0);
+                    assert_eq!(is_compressed, true);
+                    assert!(statistics.is_some());
+                    true
+                }
+                _ => false,
+            };
+            assert!(is_expected_page);
+            page_count += 1;
+        }
+        assert_eq!(page_count, 2);
+    }
+}

rust/src/parquet/file/statistics.rs

@@ -0,0 +1,692 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Contains definitions for working with Parquet statistics.
+//!
+//! Though some common methods are available on enum, use pattern match to extract
+//! actual min and max values from statistics, see below:
+//!
+//! ```rust
+//! use arrow::parquet::file::statistics::Statistics;
+//!
+//! let stats = Statistics::int32(Some(1), Some(10), None, 3, true);
+//! assert_eq!(stats.null_count(), 3);
+//! assert!(stats.has_min_max_set());
+//! assert!(stats.is_min_max_deprecated());
+//!
+//! match stats {
+//!     Statistics::Int32(ref typed) => {
+//!         assert_eq!(*typed.min(), 1);
+//!         assert_eq!(*typed.max(), 10);
+//!     }
+//!     _ => {}
+//! }
+//! ```
+
+use std::{cmp, fmt};
+
+use byteorder::{ByteOrder, LittleEndian};
+use parquet_format::Statistics as TStatistics;
+
+use crate::parquet::basic::Type;
+use crate::parquet::data_type::*;
+
+// Macro to generate methods create Statistics.
+macro_rules! statistics_new_func {
+    ($func:ident, $vtype:ty, $stat:ident) => {
+        pub fn $func(
+            min: $vtype,
+            max: $vtype,
+            distinct: Option<u64>,
+            nulls: u64,
+            is_deprecated: bool,
+        ) -> Self {
+            Statistics::$stat(TypedStatistics::new(
+                min,
+                max,
+                distinct,
+                nulls,
+                is_deprecated,
+            ))
+        }
+    };
+}
+
+// Macro to generate getter functions for Statistics.
+macro_rules! statistics_enum_func {
+    ($self:ident, $func:ident) => {{
+        match *$self {
+            Statistics::Boolean(ref typed) => typed.$func(),
+            Statistics::Int32(ref typed) => typed.$func(),
+            Statistics::Int64(ref typed) => typed.$func(),
+            Statistics::Int96(ref typed) => typed.$func(),
+            Statistics::Float(ref typed) => typed.$func(),
+            Statistics::Double(ref typed) => typed.$func(),
+            Statistics::ByteArray(ref typed) => typed.$func(),
+            Statistics::FixedLenByteArray(ref typed) => typed.$func(),
+        }
+    }};
+}
+
+/// Converts Thrift definition into `Statistics`.
+pub fn from_thrift(physical_type: Type, thrift_stats: Option<TStatistics>) -> Option<Statistics> {
+    match thrift_stats {
+        Some(stats) => {
+            // Number of nulls recorded, when it is not available, we just mark it as 0.
+            let null_count = stats.null_count.unwrap_or(0);
+            assert!(
+                null_count >= 0,
+                "Statistics null count is negative ({})",
+                null_count
+            );
+
+            // Generic null count.
+            let null_count = null_count as u64;
+            // Generic distinct count (count of distinct values occurring)
+            let distinct_count = stats.distinct_count.map(|value| value as u64);
+            // Whether or not statistics use deprecated min/max fields.
+            let old_format = stats.min_value.is_none() && stats.max_value.is_none();
+            // Generic min value as bytes.
+            let min = if old_format {
+                stats.min
+            } else {
+                stats.min_value
+            };
+            // Generic max value as bytes.
+            let max = if old_format {
+                stats.max
+            } else {
+                stats.max_value
+            };
+
+            // Values are encoded using PLAIN encoding definition, except that
+            // variable-length byte arrays do not include a length prefix.
+            //
+            // Instead of using actual decoder, we manually convert values.
+            let res = match physical_type {
+                Type::BOOLEAN => Statistics::boolean(
+                    min.map(|data| data[0] != 0),
+                    max.map(|data| data[0] != 0),
+                    distinct_count,
+                    null_count,
+                    old_format,
+                ),
+                Type::INT32 => Statistics::int32(
+                    min.map(|data| LittleEndian::read_i32(&data)),
+                    max.map(|data| LittleEndian::read_i32(&data)),
+                    distinct_count,
+                    null_count,
+                    old_format,
+                ),
+                Type::INT64 => Statistics::int64(
+                    min.map(|data| LittleEndian::read_i64(&data)),
+                    max.map(|data| LittleEndian::read_i64(&data)),
+                    distinct_count,
+                    null_count,
+                    old_format,
+                ),
+                Type::INT96 => {
+                    // INT96 statistics may not be correct, because comparison is signed
+                    // byte-wise, not actual timestamps. It is recommended to ignore min/max
+                    // statistics for INT96 columns.
+                    let min = min.map(|data| {
+                        assert_eq!(data.len(), 12);
+                        unsafe {
+                            let raw = ::std::slice::from_raw_parts(data.as_ptr() as *mut u32, 3);
+                            Int96::from(Vec::from(raw))
+                        }
+                    });
+                    let max = max.map(|data| {
+                        assert_eq!(data.len(), 12);
+                        unsafe {
+                            let raw = ::std::slice::from_raw_parts(data.as_ptr() as *mut u32, 3);
+                            Int96::from(Vec::from(raw))
+                        }
+                    });
+                    Statistics::int96(min, max, distinct_count, null_count, old_format)
+                }
+                Type::FLOAT => Statistics::float(
+                    min.map(|data| LittleEndian::read_f32(&data)),
+                    max.map(|data| LittleEndian::read_f32(&data)),
+                    distinct_count,
+                    null_count,
+                    old_format,
+                ),
+                Type::DOUBLE => Statistics::double(
+                    min.map(|data| LittleEndian::read_f64(&data)),
+                    max.map(|data| LittleEndian::read_f64(&data)),
+                    distinct_count,
+                    null_count,
+                    old_format,
+                ),
+                Type::BYTE_ARRAY => Statistics::byte_array(
+                    min.map(|data| ByteArray::from(data)),
+                    max.map(|data| ByteArray::from(data)),
+                    distinct_count,
+                    null_count,
+                    old_format,
+                ),
+                Type::FIXED_LEN_BYTE_ARRAY => Statistics::fixed_len_byte_array(
+                    min.map(|data| ByteArray::from(data)),
+                    max.map(|data| ByteArray::from(data)),
+                    distinct_count,
+                    null_count,
+                    old_format,
+                ),
+            };
+
+            Some(res)
+        }
+        None => None,
+    }
+}
+
+// Convert Statistics into Thrift definition.
+pub fn to_thrift(stats: Option<&Statistics>) -> Option<TStatistics> {
+    if stats.is_none() {
+        return None;
+    }
+
+    let stats = stats.unwrap();
+
+    let mut thrift_stats = TStatistics {
+        max: None,
+        min: None,
+        null_count: if stats.has_nulls() {
+            Some(stats.null_count() as i64)
+        } else {
+            None
+        },
+        distinct_count: stats.distinct_count().map(|value| value as i64),
+        max_value: None,
+        min_value: None,
+    };
+
+    // Get min/max if set.
+    let (min, max) = if stats.has_min_max_set() {
+        (
+            Some(stats.min_bytes().to_vec()),
+            Some(stats.max_bytes().to_vec()),
+        )
+    } else {
+        (None, None)
+    };
+
+    if stats.is_min_max_deprecated() {
+        thrift_stats.min = min;
+        thrift_stats.max = max;
+    } else {
+        thrift_stats.min_value = min;
+        thrift_stats.max_value = max;
+    }
+
+    Some(thrift_stats)
+}
+
+/// Statistics for a column chunk and data page.
+#[derive(Debug, PartialEq)]
+pub enum Statistics {
+    Boolean(TypedStatistics<BoolType>),
+    Int32(TypedStatistics<Int32Type>),
+    Int64(TypedStatistics<Int64Type>),
+    Int96(TypedStatistics<Int96Type>),
+    Float(TypedStatistics<FloatType>),
+    Double(TypedStatistics<DoubleType>),
+    ByteArray(TypedStatistics<ByteArrayType>),
+    FixedLenByteArray(TypedStatistics<FixedLenByteArrayType>),
+}
+
+impl Statistics {
+    statistics_new_func![boolean, Option<bool>, Boolean];
+
+    statistics_new_func![int32, Option<i32>, Int32];
+
+    statistics_new_func![int64, Option<i64>, Int64];
+
+    statistics_new_func![int96, Option<Int96>, Int96];
+
+    statistics_new_func![float, Option<f32>, Float];
+
+    statistics_new_func![double, Option<f64>, Double];
+
+    statistics_new_func![byte_array, Option<ByteArray>, ByteArray];
+
+    statistics_new_func![fixed_len_byte_array, Option<ByteArray>, FixedLenByteArray];
+
+    /// Returns `true` if statistics have old `min` and `max` fields set.
+    /// This means that the column order is likely to be undefined, which, for old files
+    /// could mean a signed sort order of values.
+    ///
+    /// Refer to [`ColumnOrder`](`::basic::ColumnOrder`) and
+    /// [`SortOrder`](`::basic::SortOrder`) for more information.
+    pub fn is_min_max_deprecated(&self) -> bool {
+        statistics_enum_func![self, is_min_max_deprecated]
+    }
+
+    /// Returns optional value of number of distinct values occurring.
+    /// When it is `None`, the value should be ignored.
+    pub fn distinct_count(&self) -> Option<u64> {
+        statistics_enum_func![self, distinct_count]
+    }
+
+    /// Returns number of null values for the column.
+    /// Note that this includes all nulls when column is part of the complex type.
+    pub fn null_count(&self) -> u64 {
+        statistics_enum_func![self, null_count]
+    }
+
+    /// Returns `true` if statistics collected any null values, `false` otherwise.
+    pub fn has_nulls(&self) -> bool {
+        self.null_count() > 0
+    }
+
+    /// Returns `true` if min value and max value are set.
+    /// Normally both min/max values will be set to `Some(value)` or `None`.
+    pub fn has_min_max_set(&self) -> bool {
+        statistics_enum_func![self, has_min_max_set]
+    }
+
+    /// Returns slice of bytes that represent min value.
+    /// Panics if min value is not set.
+    pub fn min_bytes(&self) -> &[u8] {
+        statistics_enum_func![self, min_bytes]
+    }
+
+    /// Returns slice of bytes that represent max value.
+    /// Panics if max value is not set.
+    pub fn max_bytes(&self) -> &[u8] {
+        statistics_enum_func![self, max_bytes]
+    }
+
+    /// Returns physical type associated with statistics.
+    pub fn physical_type(&self) -> Type {
+        match self {
+            Statistics::Boolean(_) => Type::BOOLEAN,
+            Statistics::Int32(_) => Type::INT32,
+            Statistics::Int64(_) => Type::INT64,
+            Statistics::Int96(_) => Type::INT96,
+            Statistics::Float(_) => Type::FLOAT,
+            Statistics::Double(_) => Type::DOUBLE,
+            Statistics::ByteArray(_) => Type::BYTE_ARRAY,
+            Statistics::FixedLenByteArray(_) => Type::FIXED_LEN_BYTE_ARRAY,
+        }
+    }
+}
+
+impl fmt::Display for Statistics {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            Statistics::Boolean(typed) => write!(f, "{}", typed),
+            Statistics::Int32(typed) => write!(f, "{}", typed),
+            Statistics::Int64(typed) => write!(f, "{}", typed),
+            Statistics::Int96(typed) => write!(f, "{}", typed),
+            Statistics::Float(typed) => write!(f, "{}", typed),
+            Statistics::Double(typed) => write!(f, "{}", typed),
+            Statistics::ByteArray(typed) => write!(f, "{}", typed),
+            Statistics::FixedLenByteArray(typed) => write!(f, "{}", typed),
+        }
+    }
+}
+
+/// Typed implementation for [`Statistics`].
+pub struct TypedStatistics<T: DataType> {
+    min: Option<T::T>,
+    max: Option<T::T>,
+    // Distinct count could be omitted in some cases
+    distinct_count: Option<u64>,
+    null_count: u64,
+    is_min_max_deprecated: bool,
+}
+
+impl<T: DataType> TypedStatistics<T> {
+    /// Creates new typed statistics.
+    pub fn new(
+        min: Option<T::T>,
+        max: Option<T::T>,
+        distinct_count: Option<u64>,
+        null_count: u64,
+        is_min_max_deprecated: bool,
+    ) -> Self {
+        Self {
+            min,
+            max,
+            distinct_count,
+            null_count,
+            is_min_max_deprecated,
+        }
+    }
+
+    /// Returns min value of the statistics.
+    ///
+    /// Panics if min value is not set, e.g. all values are `null`.
+    /// Use `has_min_max_set` method to check that.
+    pub fn min(&self) -> &T::T {
+        self.min.as_ref().unwrap()
+    }
+
+    /// Returns max value of the statistics.
+    ///
+    /// Panics if max value is not set, e.g. all values are `null`.
+    /// Use `has_min_max_set` method to check that.
+    pub fn max(&self) -> &T::T {
+        self.max.as_ref().unwrap()
+    }
+
+    /// Returns min value as bytes of the statistics.
+    ///
+    /// Panics if min value is not set, use `has_min_max_set` method to check
+    /// if values are set.
+    pub fn min_bytes(&self) -> &[u8] {
+        self.min().as_bytes()
+    }
+
+    /// Returns max value as bytes of the statistics.
+    ///
+    /// Panics if max value is not set, use `has_min_max_set` method to check
+    /// if values are set.
+    pub fn max_bytes(&self) -> &[u8] {
+        self.max().as_bytes()
+    }
+
+    /// Whether or not min and max values are set.
+    /// Normally both min/max values will be set to `Some(value)` or `None`.
+    fn has_min_max_set(&self) -> bool {
+        self.min.is_some() && self.max.is_some()
+    }
+
+    /// Returns optional value of number of distinct values occurring.
+    fn distinct_count(&self) -> Option<u64> {
+        self.distinct_count
+    }
+
+    /// Returns null count.
+    fn null_count(&self) -> u64 {
+        self.null_count
+    }
+
+    /// Returns `true` if statistics were created using old min/max fields.
+    fn is_min_max_deprecated(&self) -> bool {
+        self.is_min_max_deprecated
+    }
+}
+
+impl<T: DataType> fmt::Display for TypedStatistics<T> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{{")?;
+        write!(f, "min: ")?;
+        match self.min {
+            Some(ref value) => self.value_fmt(f, value)?,
+            None => write!(f, "N/A")?,
+        }
+        write!(f, ", max: ")?;
+        match self.max {
+            Some(ref value) => self.value_fmt(f, value)?,
+            None => write!(f, "N/A")?,
+        }
+        write!(f, ", distinct_count: ")?;
+        match self.distinct_count {
+            Some(value) => write!(f, "{}", value)?,
+            None => write!(f, "N/A")?,
+        }
+        write!(f, ", null_count: {}", self.null_count)?;
+        write!(f, ", min_max_deprecated: {}", self.is_min_max_deprecated)?;
+        write!(f, "}}")
+    }
+}
+
+impl<T: DataType> fmt::Debug for TypedStatistics<T> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            f,
+            "{{min: {:?}, max: {:?}, distinct_count: {:?}, null_count: {}, \
+             min_max_deprecated: {}}}",
+            self.min, self.max, self.distinct_count, self.null_count, self.is_min_max_deprecated
+        )
+    }
+}
+
+impl<T: DataType> cmp::PartialEq for TypedStatistics<T> {
+    fn eq(&self, other: &TypedStatistics<T>) -> bool {
+        self.min == other.min
+            && self.max == other.max
+            && self.distinct_count == other.distinct_count
+            && self.null_count == other.null_count
+            && self.is_min_max_deprecated == other.is_min_max_deprecated
+    }
+}
+
+/// Trait to provide a specific write format for values.
+/// For example, we should display vector slices for byte array types, and original
+/// values for other types.
+trait ValueDisplay<T: DataType> {
+    fn value_fmt(&self, f: &mut fmt::Formatter, value: &T::T) -> fmt::Result;
+}
+
+impl<T: DataType> ValueDisplay<T> for TypedStatistics<T> {
+    default fn value_fmt(&self, f: &mut fmt::Formatter, value: &T::T) -> fmt::Result {
+        write!(f, "{:?}", value)
+    }
+}
+
+impl ValueDisplay<Int96Type> for TypedStatistics<Int96Type> {
+    fn value_fmt(&self, f: &mut fmt::Formatter, value: &Int96) -> fmt::Result {
+        write!(f, "{:?}", value.data())
+    }
+}
+
+impl ValueDisplay<ByteArrayType> for TypedStatistics<ByteArrayType> {
+    fn value_fmt(&self, f: &mut fmt::Formatter, value: &ByteArray) -> fmt::Result {
+        write!(f, "{:?}", value.data())
+    }
+}
+
+impl ValueDisplay<FixedLenByteArrayType> for TypedStatistics<FixedLenByteArrayType> {
+    fn value_fmt(&self, f: &mut fmt::Formatter, value: &ByteArray) -> fmt::Result {
+        write!(f, "{:?}", value.data())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_statistics_min_max_bytes() {
+        let stats = Statistics::int32(Some(-123), Some(234), None, 1, false);
+        assert!(stats.has_min_max_set());
+        assert_eq!(stats.min_bytes(), (-123).as_bytes());
+        assert_eq!(stats.max_bytes(), 234.as_bytes());
+
+        let stats = Statistics::byte_array(
+            Some(ByteArray::from(vec![1, 2, 3])),
+            Some(ByteArray::from(vec![3, 4, 5])),
+            None,
+            1,
+            true,
+        );
+        assert!(stats.has_min_max_set());
+        assert_eq!(stats.min_bytes(), &[1, 2, 3]);
+        assert_eq!(stats.max_bytes(), &[3, 4, 5]);
+    }
+
+    #[test]
+    #[should_panic(expected = "Statistics null count is negative (-10)")]
+    fn test_statistics_negative_null_count() {
+        let thrift_stats = TStatistics {
+            max: None,
+            min: None,
+            null_count: Some(-10),
+            distinct_count: None,
+            max_value: None,
+            min_value: None,
+        };
+
+        from_thrift(Type::INT32, Some(thrift_stats));
+    }
+
+    #[test]
+    fn test_statistics_thrift_none() {
+        assert_eq!(from_thrift(Type::INT32, None), None);
+        assert_eq!(from_thrift(Type::BYTE_ARRAY, None), None);
+    }
+
+    #[test]
+    fn test_statistics_debug() {
+        let stats = Statistics::int32(Some(1), Some(12), None, 12, true);
+        assert_eq!(
+            format!("{:?}", stats),
+            "Int32({min: Some(1), max: Some(12), distinct_count: None, null_count: 12, \
+             min_max_deprecated: true})"
+        );
+
+        let stats = Statistics::int32(None, None, None, 7, false);
+        assert_eq!(
+            format!("{:?}", stats),
+            "Int32({min: None, max: None, distinct_count: None, null_count: 7, \
+             min_max_deprecated: false})"
+        )
+    }
+
+    #[test]
+    fn test_statistics_display() {
+        let stats = Statistics::int32(Some(1), Some(12), None, 12, true);
+        assert_eq!(
+            format!("{}", stats),
+            "{min: 1, max: 12, distinct_count: N/A, null_count: 12, min_max_deprecated: true}"
+        );
+
+        let stats = Statistics::int64(None, None, None, 7, false);
+        assert_eq!(
+            format!("{}", stats),
+            "{min: N/A, max: N/A, distinct_count: N/A, null_count: 7, min_max_deprecated: \
+             false}"
+        );
+
+        let stats = Statistics::int96(
+            Some(Int96::from(vec![1, 0, 0])),
+            Some(Int96::from(vec![2, 3, 4])),
+            None,
+            3,
+            true,
+        );
+        assert_eq!(
+            format!("{}", stats),
+            "{min: [1, 0, 0], max: [2, 3, 4], distinct_count: N/A, null_count: 3, \
+             min_max_deprecated: true}"
+        );
+
+        let stats = Statistics::byte_array(
+            Some(ByteArray::from(vec![1u8])),
+            Some(ByteArray::from(vec![2u8])),
+            Some(5),
+            7,
+            false,
+        );
+        assert_eq!(
+            format!("{}", stats),
+            "{min: [1], max: [2], distinct_count: 5, null_count: 7, min_max_deprecated: false}"
+        );
+    }
+
+    #[test]
+    fn test_statistics_partial_eq() {
+        let expected = Statistics::int32(Some(12), Some(45), None, 11, true);
+
+        assert!(Statistics::int32(Some(12), Some(45), None, 11, true) == expected);
+        assert!(Statistics::int32(Some(11), Some(45), None, 11, true) != expected);
+        assert!(Statistics::int32(Some(12), Some(44), None, 11, true) != expected);
+        assert!(Statistics::int32(Some(12), Some(45), None, 23, true) != expected);
+        assert!(Statistics::int32(Some(12), Some(45), None, 11, false) != expected);
+
+        assert!(
+            Statistics::int32(Some(12), Some(45), None, 11, false)
+                != Statistics::int64(Some(12), Some(45), None, 11, false)
+        );
+
+        assert!(
+            Statistics::boolean(Some(false), Some(true), None, 0, true)
+                != Statistics::double(Some(1.2), Some(4.5), None, 0, true)
+        );
+
+        assert!(
+            Statistics::byte_array(
+                Some(ByteArray::from(vec![1, 2, 3])),
+                Some(ByteArray::from(vec![1, 2, 3])),
+                None,
+                0,
+                true
+            ) != Statistics::fixed_len_byte_array(
+                Some(ByteArray::from(vec![1, 2, 3])),
+                Some(ByteArray::from(vec![1, 2, 3])),
+                None,
+                0,
+                true
+            )
+        );
+    }
+
+    #[test]
+    fn test_statistics_from_thrift() {
+        // Helper method to check statistics conversion.
+        fn check_stats(stats: Statistics) {
+            let tpe = stats.physical_type();
+            let thrift_stats = to_thrift(Some(&stats));
+            assert_eq!(from_thrift(tpe, thrift_stats), Some(stats));
+        }
+
+        check_stats(Statistics::boolean(Some(false), Some(true), None, 7, true));
+        check_stats(Statistics::boolean(Some(false), Some(true), None, 7, true));
+        check_stats(Statistics::boolean(Some(false), Some(true), None, 0, false));
+        check_stats(Statistics::boolean(Some(true), Some(true), None, 7, true));
+        check_stats(Statistics::boolean(Some(false), Some(false), None, 7, true));
+        check_stats(Statistics::boolean(None, None, None, 7, true));
+
+        check_stats(Statistics::int32(Some(-100), Some(500), None, 7, true));
+        check_stats(Statistics::int32(Some(-100), Some(500), None, 0, false));
+        check_stats(Statistics::int32(None, None, None, 7, true));
+
+        check_stats(Statistics::int64(Some(-100), Some(200), None, 7, true));
+        check_stats(Statistics::int64(Some(-100), Some(200), None, 0, false));
+        check_stats(Statistics::int64(None, None, None, 7, true));
+
+        check_stats(Statistics::float(Some(1.2), Some(3.4), None, 7, true));
+        check_stats(Statistics::float(Some(1.2), Some(3.4), None, 0, false));
+        check_stats(Statistics::float(None, None, None, 7, true));
+
+        check_stats(Statistics::double(Some(1.2), Some(3.4), None, 7, true));
+        check_stats(Statistics::double(Some(1.2), Some(3.4), None, 0, false));
+        check_stats(Statistics::double(None, None, None, 7, true));
+
+        check_stats(Statistics::byte_array(
+            Some(ByteArray::from(vec![1, 2, 3])),
+            Some(ByteArray::from(vec![3, 4, 5])),
+            None,
+            7,
+            true,
+        ));
+        check_stats(Statistics::byte_array(None, None, None, 7, true));
+
+        check_stats(Statistics::fixed_len_byte_array(
+            Some(ByteArray::from(vec![1, 2, 3])),
+            Some(ByteArray::from(vec![3, 4, 5])),
+            None,
+            7,
+            true,
+        ));
+        check_stats(Statistics::fixed_len_byte_array(None, None, None, 7, true));
+    }
+}

rust/src/parquet/file/writer.rs

@@ -0,0 +1,936 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Contains file writer API, and provides methods to write row groups and columns by
+//! using row group writers and column writers respectively.
+
+use std::{
+    fs::File,
+    io::{Seek, SeekFrom, Write},
+    rc::Rc,
+};
+
+use byteorder::{ByteOrder, LittleEndian};
+use parquet_format as parquet;
+use thrift::protocol::{TCompactOutputProtocol, TOutputProtocol};
+
+use crate::parquet::basic::PageType;
+use crate::parquet::column::{
+    page::{CompressedPage, Page, PageWriteSpec, PageWriter},
+    writer::{get_column_writer, ColumnWriter},
+};
+use crate::parquet::errors::{ParquetError, Result};
+use crate::parquet::file::{
+    metadata::*, properties::WriterPropertiesPtr, statistics::to_thrift as statistics_to_thrift,
+    FOOTER_SIZE, PARQUET_MAGIC,
+};
+use crate::parquet::schema::types::{self, SchemaDescPtr, SchemaDescriptor, TypePtr};
+use crate::parquet::util::io::{FileSink, Position};
+
+// ----------------------------------------------------------------------
+// APIs for file & row group writers
+
+/// Parquet file writer API.
+/// Provides methods to write row groups sequentially.
+///
+/// The main workflow should be as following:
+/// - Create file writer, this will open a new file and potentially write some metadata.
+/// - Request a new row group writer by calling `next_row_group`.
+/// - Once finished writing row group, close row group writer by passing it into
+/// `close_row_group` method - this will finalise row group metadata and update metrics.
+/// - Write subsequent row groups, if necessary.
+/// - After all row groups have been written, close the file writer using `close` method.
+pub trait FileWriter {
+    /// Creates new row group from this file writer.
+    /// In case of IO error or Thrift error, returns `Err`.
+    ///
+    /// There is no limit on a number of row groups in a file; however, row groups have
+    /// to be written sequentially. Every time the next row group is requested, the
+    /// previous row group must be finalised and closed using `close_row_group` method.
+    fn next_row_group(&mut self) -> Result<Box<RowGroupWriter>>;
+
+    /// Finalises and closes row group that was created using `next_row_group` method.
+    /// After calling this method, the next row group is available for writes.
+    fn close_row_group(&mut self, row_group_writer: Box<RowGroupWriter>) -> Result<()>;
+
+    /// Closes and finalises file writer.
+    ///
+    /// All row groups must be appended before this method is called.
+    /// No writes are allowed after this point.
+    ///
+    /// Can be called multiple times. It is up to implementation to either result in no-op,
+    /// or return an `Err` for subsequent calls.
+    fn close(&mut self) -> Result<()>;
+}
+
+/// Parquet row group writer API.
+/// Provides methods to access column writers in an iterator-like fashion, order is
+/// guaranteed to match the order of schema leaves (column descriptors).
+///
+/// All columns should be written sequentially; the main workflow is:
+/// - Request the next column using `next_column` method - this will return `None` if no
+/// more columns are available to write.
+/// - Once done writing a column, close column writer with `close_column` method - this
+/// will finalise column chunk metadata and update row group metrics.
+/// - Once all columns have been written, close row group writer with `close` method -
+/// it will return row group metadata and is no-op on already closed row group.
+pub trait RowGroupWriter {
+    /// Returns the next column writer, if available; otherwise returns `None`.
+    /// In case of any IO error or Thrift error, or if row group writer has already been
+    /// closed returns `Err`.
+    ///
+    /// To request the next column writer, the previous one must be finalised and closed
+    /// using `close_column`.
+    fn next_column(&mut self) -> Result<Option<ColumnWriter>>;
+
+    /// Closes column writer that was created using `next_column` method.
+    /// This should be called before requesting the next column writer.
+    fn close_column(&mut self, column_writer: ColumnWriter) -> Result<()>;
+
+    /// Closes this row group writer and returns row group metadata.
+    /// After calling this method row group writer must not be used.
+    ///
+    /// It is recommended to call this method before requesting another row group, but it
+    /// will be closed automatically before returning a new row group.
+    ///
+    /// Can be called multiple times. In subsequent calls will result in no-op and return
+    /// already created row group metadata.
+    fn close(&mut self) -> Result<RowGroupMetaDataPtr>;
+}
+
+// ----------------------------------------------------------------------
+// Serialized impl for file & row group writers
+
+/// A serialized implementation for Parquet [`FileWriter`].
+/// See documentation on file writer for more information.
+pub struct SerializedFileWriter {
+    file: File,
+    schema: TypePtr,
+    descr: SchemaDescPtr,
+    props: WriterPropertiesPtr,
+    total_num_rows: u64,
+    row_groups: Vec<RowGroupMetaDataPtr>,
+    previous_writer_closed: bool,
+    is_closed: bool,
+}
+
+impl SerializedFileWriter {
+    /// Creates new file writer.
+    pub fn new(mut file: File, schema: TypePtr, properties: WriterPropertiesPtr) -> Result<Self> {
+        Self::start_file(&mut file)?;
+        Ok(Self {
+            file,
+            schema: schema.clone(),
+            descr: Rc::new(SchemaDescriptor::new(schema)),
+            props: properties,
+            total_num_rows: 0,
+            row_groups: Vec::new(),
+            previous_writer_closed: true,
+            is_closed: false,
+        })
+    }
+
+    /// Writes magic bytes at the beginning of the file.
+    fn start_file(file: &mut File) -> Result<()> {
+        file.write(&PARQUET_MAGIC)?;
+        Ok(())
+    }
+
+    /// Finalises active row group writer, otherwise no-op.
+    fn finalise_row_group_writer(
+        &mut self,
+        mut row_group_writer: Box<RowGroupWriter>,
+    ) -> Result<()> {
+        let row_group_metadata = row_group_writer.close()?;
+        self.row_groups.push(row_group_metadata);
+        Ok(())
+    }
+
+    /// Assembles and writes metadata at the end of the file.
+    fn write_metadata(&mut self) -> Result<()> {
+        let file_metadata = parquet::FileMetaData {
+            version: self.props.writer_version().as_num(),
+            schema: types::to_thrift(self.schema.as_ref())?,
+            num_rows: self.total_num_rows as i64,
+            row_groups: self
+                .row_groups
+                .as_slice()
+                .into_iter()
+                .map(|v| v.to_thrift())
+                .collect(),
+            key_value_metadata: None,
+            created_by: Some(self.props.created_by().to_owned()),
+            column_orders: None,
+        };
+
+        // Write file metadata
+        let start_pos = self.file.seek(SeekFrom::Current(0))?;
+        {
+            let mut protocol = TCompactOutputProtocol::new(&mut self.file);
+            file_metadata.write_to_out_protocol(&mut protocol)?;
+            protocol.flush()?;
+        }
+        let end_pos = self.file.seek(SeekFrom::Current(0))?;
+
+        // Write footer
+        let mut footer_buffer: [u8; FOOTER_SIZE] = [0; FOOTER_SIZE];
+        let metadata_len = (end_pos - start_pos) as i32;
+        LittleEndian::write_i32(&mut footer_buffer, metadata_len);
+        (&mut footer_buffer[4..]).write(&PARQUET_MAGIC)?;
+        self.file.write(&footer_buffer)?;
+        Ok(())
+    }
+
+    #[inline]
+    fn assert_closed(&self) -> Result<()> {
+        if self.is_closed {
+            Err(general_err!("File writer is closed"))
+        } else {
+            Ok(())
+        }
+    }
+
+    #[inline]
+    fn assert_previous_writer_closed(&self) -> Result<()> {
+        if !self.previous_writer_closed {
+            Err(general_err!("Previous row group writer was not closed"))
+        } else {
+            Ok(())
+        }
+    }
+}
+
+impl FileWriter for SerializedFileWriter {
+    #[inline]
+    fn next_row_group(&mut self) -> Result<Box<RowGroupWriter>> {
+        self.assert_closed()?;
+        self.assert_previous_writer_closed()?;
+        let row_group_writer =
+            SerializedRowGroupWriter::new(self.descr.clone(), self.props.clone(), &self.file);
+        self.previous_writer_closed = false;
+        Ok(Box::new(row_group_writer))
+    }
+
+    #[inline]
+    fn close_row_group(&mut self, row_group_writer: Box<RowGroupWriter>) -> Result<()> {
+        self.assert_closed()?;
+        let res = self.finalise_row_group_writer(row_group_writer);
+        self.previous_writer_closed = res.is_ok();
+        res
+    }
+
+    #[inline]
+    fn close(&mut self) -> Result<()> {
+        self.assert_closed()?;
+        self.assert_previous_writer_closed()?;
+        self.write_metadata()?;
+        self.is_closed = true;
+        Ok(())
+    }
+}
+
+/// A serialized implementation for Parquet [`RowGroupWriter`].
+/// Coordinates writing of a row group with column writers.
+/// See documentation on row group writer for more information.
+pub struct SerializedRowGroupWriter {
+    descr: SchemaDescPtr,
+    props: WriterPropertiesPtr,
+    file: File,
+    total_rows_written: Option<u64>,
+    total_bytes_written: u64,
+    column_index: usize,
+    previous_writer_closed: bool,
+    row_group_metadata: Option<RowGroupMetaDataPtr>,
+    column_chunks: Vec<ColumnChunkMetaDataPtr>,
+}
+
+impl SerializedRowGroupWriter {
+    pub fn new(schema_descr: SchemaDescPtr, properties: WriterPropertiesPtr, file: &File) -> Self {
+        let num_columns = schema_descr.num_columns();
+        Self {
+            descr: schema_descr,
+            props: properties,
+            file: file.try_clone().unwrap(),
+            total_rows_written: None,
+            total_bytes_written: 0,
+            column_index: 0,
+            previous_writer_closed: true,
+            row_group_metadata: None,
+            column_chunks: Vec::with_capacity(num_columns),
+        }
+    }
+
+    /// Checks and finalises current column writer.
+    fn finalise_column_writer(&mut self, writer: ColumnWriter) -> Result<()> {
+        let (bytes_written, rows_written, metadata) = match writer {
+            ColumnWriter::BoolColumnWriter(typed) => typed.close()?,
+            ColumnWriter::Int32ColumnWriter(typed) => typed.close()?,
+            ColumnWriter::Int64ColumnWriter(typed) => typed.close()?,
+            ColumnWriter::Int96ColumnWriter(typed) => typed.close()?,
+            ColumnWriter::FloatColumnWriter(typed) => typed.close()?,
+            ColumnWriter::DoubleColumnWriter(typed) => typed.close()?,
+            ColumnWriter::ByteArrayColumnWriter(typed) => typed.close()?,
+            ColumnWriter::FixedLenByteArrayColumnWriter(typed) => typed.close()?,
+        };
+
+        // Update row group writer metrics
+        self.total_bytes_written += bytes_written;
+        self.column_chunks.push(Rc::new(metadata));
+        if let Some(rows) = self.total_rows_written {
+            if rows != rows_written {
+                return Err(general_err!(
+                    "Incorrect number of rows, expected {} != {} rows",
+                    rows,
+                    rows_written
+                ));
+            }
+        } else {
+            self.total_rows_written = Some(rows_written);
+        }
+
+        Ok(())
+    }
+
+    #[inline]
+    fn assert_closed(&self) -> Result<()> {
+        if self.row_group_metadata.is_some() {
+            Err(general_err!("Row group writer is closed"))
+        } else {
+            Ok(())
+        }
+    }
+
+    #[inline]
+    fn assert_previous_writer_closed(&self) -> Result<()> {
+        if !self.previous_writer_closed {
+            Err(general_err!("Previous column writer was not closed"))
+        } else {
+            Ok(())
+        }
+    }
+}
+
+impl RowGroupWriter for SerializedRowGroupWriter {
+    #[inline]
+    fn next_column(&mut self) -> Result<Option<ColumnWriter>> {
+        self.assert_closed()?;
+        self.assert_previous_writer_closed()?;
+
+        if self.column_index >= self.descr.num_columns() {
+            return Ok(None);
+        }
+        let sink = FileSink::new(&self.file);
+        let page_writer = Box::new(SerializedPageWriter::new(sink));
+        let column_writer = get_column_writer(
+            self.descr.column(self.column_index),
+            self.props.clone(),
+            page_writer,
+        );
+        self.column_index += 1;
+        self.previous_writer_closed = false;
+
+        Ok(Some(column_writer))
+    }
+
+    #[inline]
+    fn close_column(&mut self, column_writer: ColumnWriter) -> Result<()> {
+        let res = self.finalise_column_writer(column_writer);
+        self.previous_writer_closed = res.is_ok();
+        res
+    }
+
+    #[inline]
+    fn close(&mut self) -> Result<RowGroupMetaDataPtr> {
+        if self.row_group_metadata.is_none() {
+            self.assert_previous_writer_closed()?;
+
+            let row_group_metadata = RowGroupMetaData::builder(self.descr.clone())
+                .set_column_metadata(self.column_chunks.clone())
+                .set_total_byte_size(self.total_bytes_written as i64)
+                .set_num_rows(self.total_rows_written.unwrap_or(0) as i64)
+                .build()?;
+
+            self.row_group_metadata = Some(Rc::new(row_group_metadata));
+        }
+
+        let metadata = self.row_group_metadata.as_ref().unwrap().clone();
+        Ok(metadata)
+    }
+}
+
+/// A serialized implementation for Parquet [`PageWriter`].
+/// Writes and serializes pages and metadata into output stream.
+///
+/// `SerializedPageWriter` should not be used after calling `close()`.
+pub struct SerializedPageWriter<T: Write + Position> {
+    sink: T,
+}
+
+impl<T: Write + Position> SerializedPageWriter<T> {
+    /// Creates new page writer.
+    pub fn new(sink: T) -> Self {
+        Self { sink }
+    }
+
+    /// Serializes page header into Thrift.
+    /// Returns number of bytes that have been written into the sink.
+    #[inline]
+    fn serialize_page_header(&mut self, header: parquet::PageHeader) -> Result<usize> {
+        let start_pos = self.sink.pos();
+        {
+            let mut protocol = TCompactOutputProtocol::new(&mut self.sink);
+            header.write_to_out_protocol(&mut protocol)?;
+            protocol.flush()?;
+        }
+        Ok((self.sink.pos() - start_pos) as usize)
+    }
+
+    /// Serializes column chunk into Thrift.
+    /// Returns Ok() if there are not errors serializing and writing data into the sink.
+    #[inline]
+    fn serialize_column_chunk(&mut self, chunk: parquet::ColumnChunk) -> Result<()> {
+        let mut protocol = TCompactOutputProtocol::new(&mut self.sink);
+        chunk.write_to_out_protocol(&mut protocol)?;
+        protocol.flush()?;
+        Ok(())
+    }
+}
+
+impl<T: Write + Position> PageWriter for SerializedPageWriter<T> {
+    fn write_page(&mut self, page: CompressedPage) -> Result<PageWriteSpec> {
+        let uncompressed_size = page.uncompressed_size();
+        let compressed_size = page.compressed_size();
+        let num_values = page.num_values();
+        let encoding = page.encoding();
+        let page_type = page.page_type();
+
+        let mut page_header = parquet::PageHeader {
+            type_: page_type.into(),
+            uncompressed_page_size: uncompressed_size as i32,
+            compressed_page_size: compressed_size as i32,
+            // TODO: Add support for crc checksum
+            crc: None,
+            data_page_header: None,
+            index_page_header: None,
+            dictionary_page_header: None,
+            data_page_header_v2: None,
+        };
+
+        match page.compressed_page() {
+            &Page::DataPage {
+                def_level_encoding,
+                rep_level_encoding,
+                ref statistics,
+                ..
+            } => {
+                let data_page_header = parquet::DataPageHeader {
+                    num_values: num_values as i32,
+                    encoding: encoding.into(),
+                    definition_level_encoding: def_level_encoding.into(),
+                    repetition_level_encoding: rep_level_encoding.into(),
+                    statistics: statistics_to_thrift(statistics.as_ref()),
+                };
+                page_header.data_page_header = Some(data_page_header);
+            }
+            &Page::DataPageV2 {
+                num_nulls,
+                num_rows,
+                def_levels_byte_len,
+                rep_levels_byte_len,
+                is_compressed,
+                ref statistics,
+                ..
+            } => {
+                let data_page_header_v2 = parquet::DataPageHeaderV2 {
+                    num_values: num_values as i32,
+                    num_nulls: num_nulls as i32,
+                    num_rows: num_rows as i32,
+                    encoding: encoding.into(),
+                    definition_levels_byte_length: def_levels_byte_len as i32,
+                    repetition_levels_byte_length: rep_levels_byte_len as i32,
+                    is_compressed: Some(is_compressed),
+                    statistics: statistics_to_thrift(statistics.as_ref()),
+                };
+                page_header.data_page_header_v2 = Some(data_page_header_v2);
+            }
+            &Page::DictionaryPage { is_sorted, .. } => {
+                let dictionary_page_header = parquet::DictionaryPageHeader {
+                    num_values: num_values as i32,
+                    encoding: encoding.into(),
+                    is_sorted: Some(is_sorted),
+                };
+                page_header.dictionary_page_header = Some(dictionary_page_header);
+            }
+        }
+
+        let start_pos = self.sink.pos();
+
+        let header_size = self.serialize_page_header(page_header)?;
+        self.sink.write_all(page.data())?;
+
+        let mut spec = PageWriteSpec::new();
+        spec.page_type = page_type;
+        spec.uncompressed_size = uncompressed_size + header_size;
+        spec.compressed_size = compressed_size + header_size;
+        spec.offset = start_pos;
+        spec.bytes_written = self.sink.pos() - start_pos;
+        // Number of values is incremented for data pages only
+        if page_type == PageType::DATA_PAGE || page_type == PageType::DATA_PAGE_V2 {
+            spec.num_values = num_values;
+        }
+
+        Ok(spec)
+    }
+
+    fn write_metadata(&mut self, metadata: &ColumnChunkMetaData) -> Result<()> {
+        self.serialize_column_chunk(metadata.to_thrift())
+    }
+
+    fn close(&mut self) -> Result<()> {
+        self.sink.flush()?;
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use std::{error::Error, io::Cursor};
+
+    use crate::parquet::basic::{Compression, Encoding, Repetition, Type};
+    use crate::parquet::column::page::PageReader;
+    use crate::parquet::compression::{create_codec, Codec};
+    use crate::parquet::file::{
+        properties::WriterProperties,
+        reader::{FileReader, SerializedFileReader, SerializedPageReader},
+        statistics::{from_thrift, to_thrift, Statistics},
+    };
+    use crate::parquet::record::RowAccessor;
+    use crate::parquet::util::{memory::ByteBufferPtr, test_common::get_temp_file};
+
+    #[test]
+    fn test_file_writer_error_after_close() {
+        let file = get_temp_file("test_file_writer_error_after_close", &[]);
+        let schema = Rc::new(types::Type::group_type_builder("schema").build().unwrap());
+        let props = Rc::new(WriterProperties::builder().build());
+        let mut writer = SerializedFileWriter::new(file, schema, props).unwrap();
+        writer.close().unwrap();
+        {
+            let res = writer.next_row_group();
+            assert!(res.is_err());
+            if let Err(err) = res {
+                assert_eq!(err.description(), "File writer is closed");
+            }
+        }
+        {
+            let res = writer.close();
+            assert!(res.is_err());
+            if let Err(err) = res {
+                assert_eq!(err.description(), "File writer is closed");
+            }
+        }
+    }
+
+    #[test]
+    fn test_row_group_writer_error_after_close() {
+        let file = get_temp_file("test_file_writer_row_group_error_after_close", &[]);
+        let schema = Rc::new(types::Type::group_type_builder("schema").build().unwrap());
+        let props = Rc::new(WriterProperties::builder().build());
+        let mut writer = SerializedFileWriter::new(file, schema, props).unwrap();
+        let mut row_group_writer = writer.next_row_group().unwrap();
+        row_group_writer.close().unwrap();
+
+        let res = row_group_writer.next_column();
+        assert!(res.is_err());
+        if let Err(err) = res {
+            assert_eq!(err.description(), "Row group writer is closed");
+        }
+    }
+
+    #[test]
+    fn test_row_group_writer_error_not_all_columns_written() {
+        let file = get_temp_file("test_row_group_writer_error_not_all_columns_written", &[]);
+        let schema = Rc::new(
+            types::Type::group_type_builder("schema")
+                .with_fields(&mut vec![Rc::new(
+                    types::Type::primitive_type_builder("col1", Type::INT32)
+                        .build()
+                        .unwrap(),
+                )])
+                .build()
+                .unwrap(),
+        );
+        let props = Rc::new(WriterProperties::builder().build());
+        let mut writer = SerializedFileWriter::new(file, schema, props).unwrap();
+        let mut row_group_writer = writer.next_row_group().unwrap();
+        let res = row_group_writer.close();
+        assert!(res.is_err());
+        if let Err(err) = res {
+            assert_eq!(err.description(), "Column length mismatch: 1 != 0");
+        }
+    }
+
+    #[test]
+    fn test_row_group_writer_num_records_mismatch() {
+        let file = get_temp_file("test_row_group_writer_num_records_mismatch", &[]);
+        let schema = Rc::new(
+            types::Type::group_type_builder("schema")
+                .with_fields(&mut vec![
+                    Rc::new(
+                        types::Type::primitive_type_builder("col1", Type::INT32)
+                            .with_repetition(Repetition::REQUIRED)
+                            .build()
+                            .unwrap(),
+                    ),
+                    Rc::new(
+                        types::Type::primitive_type_builder("col2", Type::INT32)
+                            .with_repetition(Repetition::REQUIRED)
+                            .build()
+                            .unwrap(),
+                    ),
+                ])
+                .build()
+                .unwrap(),
+        );
+        let props = Rc::new(WriterProperties::builder().build());
+        let mut writer = SerializedFileWriter::new(file, schema, props).unwrap();
+        let mut row_group_writer = writer.next_row_group().unwrap();
+
+        let mut col_writer = row_group_writer.next_column().unwrap().unwrap();
+        if let ColumnWriter::Int32ColumnWriter(ref mut typed) = col_writer {
+            typed.write_batch(&[1, 2, 3], None, None).unwrap();
+        }
+        row_group_writer.close_column(col_writer).unwrap();
+
+        let mut col_writer = row_group_writer.next_column().unwrap().unwrap();
+        if let ColumnWriter::Int32ColumnWriter(ref mut typed) = col_writer {
+            typed.write_batch(&[1, 2], None, None).unwrap();
+        }
+
+        let res = row_group_writer.close_column(col_writer);
+        assert!(res.is_err());
+        if let Err(err) = res {
+            assert_eq!(
+                err.description(),
+                "Incorrect number of rows, expected 3 != 2 rows"
+            );
+        }
+    }
+
+    #[test]
+    fn test_file_writer_empty_file() {
+        let file = get_temp_file("test_file_writer_write_empty_file", &[]);
+
+        let schema = Rc::new(
+            types::Type::group_type_builder("schema")
+                .with_fields(&mut vec![Rc::new(
+                    types::Type::primitive_type_builder("col1", Type::INT32)
+                        .build()
+                        .unwrap(),
+                )])
+                .build()
+                .unwrap(),
+        );
+        let props = Rc::new(WriterProperties::builder().build());
+        let mut writer =
+            SerializedFileWriter::new(file.try_clone().unwrap(), schema, props).unwrap();
+        writer.close().unwrap();
+
+        let reader = SerializedFileReader::new(file).unwrap();
+        assert_eq!(reader.get_row_iter(None).unwrap().count(), 0);
+    }
+
+    #[test]
+    fn test_file_writer_empty_row_groups() {
+        let file = get_temp_file("test_file_writer_write_empty_row_groups", &[]);
+        test_file_roundtrip(file, vec![]);
+    }
+
+    #[test]
+    fn test_file_writer_single_row_group() {
+        let file = get_temp_file("test_file_writer_write_single_row_group", &[]);
+        test_file_roundtrip(file, vec![vec![1, 2, 3, 4, 5]]);
+    }
+
+    #[test]
+    fn test_file_writer_multiple_row_groups() {
+        let file = get_temp_file("test_file_writer_write_multiple_row_groups", &[]);
+        test_file_roundtrip(
+            file,
+            vec![
+                vec![1, 2, 3, 4, 5],
+                vec![1, 2, 3],
+                vec![1],
+                vec![1, 2, 3, 4, 5, 6],
+            ],
+        );
+    }
+
+    #[test]
+    fn test_file_writer_multiple_large_row_groups() {
+        let file = get_temp_file("test_file_writer_multiple_large_row_groups", &[]);
+        test_file_roundtrip(
+            file,
+            vec![vec![123; 1024], vec![124; 1000], vec![125; 15], vec![]],
+        );
+    }
+
+    #[test]
+    fn test_page_writer_data_pages() {
+        let pages = vec![
+            Page::DataPage {
+                buf: ByteBufferPtr::new(vec![1, 2, 3, 4, 5, 6, 7, 8]),
+                num_values: 10,
+                encoding: Encoding::DELTA_BINARY_PACKED,
+                def_level_encoding: Encoding::RLE,
+                rep_level_encoding: Encoding::RLE,
+                statistics: Some(Statistics::int32(Some(1), Some(3), None, 7, true)),
+            },
+            Page::DataPageV2 {
+                buf: ByteBufferPtr::new(vec![4; 128]),
+                num_values: 10,
+                encoding: Encoding::DELTA_BINARY_PACKED,
+                num_nulls: 2,
+                num_rows: 12,
+                def_levels_byte_len: 24,
+                rep_levels_byte_len: 32,
+                is_compressed: false,
+                statistics: Some(Statistics::int32(Some(1), Some(3), None, 7, true)),
+            },
+        ];
+
+        test_page_roundtrip(&pages[..], Compression::SNAPPY, Type::INT32);
+        test_page_roundtrip(&pages[..], Compression::UNCOMPRESSED, Type::INT32);
+    }
+
+    #[test]
+    fn test_page_writer_dict_pages() {
+        let pages = vec![
+            Page::DictionaryPage {
+                buf: ByteBufferPtr::new(vec![1, 2, 3, 4, 5]),
+                num_values: 5,
+                encoding: Encoding::RLE_DICTIONARY,
+                is_sorted: false,
+            },
+            Page::DataPage {
+                buf: ByteBufferPtr::new(vec![1, 2, 3, 4, 5, 6, 7, 8]),
+                num_values: 10,
+                encoding: Encoding::DELTA_BINARY_PACKED,
+                def_level_encoding: Encoding::RLE,
+                rep_level_encoding: Encoding::RLE,
+                statistics: Some(Statistics::int32(Some(1), Some(3), None, 7, true)),
+            },
+            Page::DataPageV2 {
+                buf: ByteBufferPtr::new(vec![4; 128]),
+                num_values: 10,
+                encoding: Encoding::DELTA_BINARY_PACKED,
+                num_nulls: 2,
+                num_rows: 12,
+                def_levels_byte_len: 24,
+                rep_levels_byte_len: 32,
+                is_compressed: false,
+                statistics: None,
+            },
+        ];
+
+        test_page_roundtrip(&pages[..], Compression::SNAPPY, Type::INT32);
+        test_page_roundtrip(&pages[..], Compression::UNCOMPRESSED, Type::INT32);
+    }
+
+    /// Tests writing and reading pages.
+    /// Physical type is for statistics only, should match any defined statistics type in
+    /// pages.
+    fn test_page_roundtrip(pages: &[Page], codec: Compression, physical_type: Type) {
+        let mut compressed_pages = vec![];
+        let mut total_num_values = 0i64;
+        let mut compressor = create_codec(codec).unwrap();
+
+        for page in pages {
+            let uncompressed_len = page.buffer().len();
+
+            let compressed_page = match page {
+                &Page::DataPage {
+                    ref buf,
+                    num_values,
+                    encoding,
+                    def_level_encoding,
+                    rep_level_encoding,
+                    ref statistics,
+                } => {
+                    total_num_values += num_values as i64;
+                    let output_buf = compress_helper(compressor.as_mut(), buf.data());
+
+                    Page::DataPage {
+                        buf: ByteBufferPtr::new(output_buf),
+                        num_values,
+                        encoding,
+                        def_level_encoding,
+                        rep_level_encoding,
+                        statistics: from_thrift(physical_type, to_thrift(statistics.as_ref())),
+                    }
+                }
+                &Page::DataPageV2 {
+                    ref buf,
+                    num_values,
+                    encoding,
+                    num_nulls,
+                    num_rows,
+                    def_levels_byte_len,
+                    rep_levels_byte_len,
+                    ref statistics,
+                    ..
+                } => {
+                    total_num_values += num_values as i64;
+                    let offset = (def_levels_byte_len + rep_levels_byte_len) as usize;
+                    let cmp_buf = compress_helper(compressor.as_mut(), &buf.data()[offset..]);
+                    let mut output_buf = Vec::from(&buf.data()[..offset]);
+                    output_buf.extend_from_slice(&cmp_buf[..]);
+
+                    Page::DataPageV2 {
+                        buf: ByteBufferPtr::new(output_buf),
+                        num_values,
+                        encoding,
+                        num_nulls,
+                        num_rows,
+                        def_levels_byte_len,
+                        rep_levels_byte_len,
+                        is_compressed: compressor.is_some(),
+                        statistics: from_thrift(physical_type, to_thrift(statistics.as_ref())),
+                    }
+                }
+                &Page::DictionaryPage {
+                    ref buf,
+                    num_values,
+                    encoding,
+                    is_sorted,
+                } => {
+                    let output_buf = compress_helper(compressor.as_mut(), buf.data());
+
+                    Page::DictionaryPage {
+                        buf: ByteBufferPtr::new(output_buf),
+                        num_values,
+                        encoding,
+                        is_sorted,
+                    }
+                }
+            };
+
+            let compressed_page = CompressedPage::new(compressed_page, uncompressed_len);
+            compressed_pages.push(compressed_page);
+        }
+
+        let mut buffer: Vec<u8> = vec![];
+        let mut result_pages: Vec<Page> = vec![];
+        {
+            let cursor = Cursor::new(&mut buffer);
+            let mut page_writer = SerializedPageWriter::new(cursor);
+
+            for page in compressed_pages {
+                page_writer.write_page(page).unwrap();
+            }
+            page_writer.close().unwrap();
+        }
+        {
+            let mut page_reader = SerializedPageReader::new(
+                Cursor::new(&buffer),
+                total_num_values,
+                codec,
+                physical_type,
+            )
+            .unwrap();
+
+            while let Some(page) = page_reader.get_next_page().unwrap() {
+                result_pages.push(page);
+            }
+        }
+
+        assert_eq!(result_pages.len(), pages.len());
+        for i in 0..result_pages.len() {
+            assert_page(&result_pages[i], &pages[i]);
+        }
+    }
+
+    /// Helper function to compress a slice
+    fn compress_helper(compressor: Option<&mut Box<Codec>>, data: &[u8]) -> Vec<u8> {
+        let mut output_buf = vec![];
+        if let Some(cmpr) = compressor {
+            cmpr.compress(data, &mut output_buf).unwrap();
+        } else {
+            output_buf.extend_from_slice(data);
+        }
+        output_buf
+    }
+
+    /// Check if pages match.
+    fn assert_page(left: &Page, right: &Page) {
+        assert_eq!(left.page_type(), right.page_type());
+        assert_eq!(left.buffer().data(), right.buffer().data());
+        assert_eq!(left.num_values(), right.num_values());
+        assert_eq!(left.encoding(), right.encoding());
+        assert_eq!(to_thrift(left.statistics()), to_thrift(right.statistics()));
+    }
+
+    /// File write-read roundtrip.
+    /// `data` consists of arrays of values for each row group.
+    fn test_file_roundtrip(file: File, data: Vec<Vec<i32>>) {
+        let schema = Rc::new(
+            types::Type::group_type_builder("schema")
+                .with_fields(&mut vec![Rc::new(
+                    types::Type::primitive_type_builder("col1", Type::INT32)
+                        .with_repetition(Repetition::REQUIRED)
+                        .build()
+                        .unwrap(),
+                )])
+                .build()
+                .unwrap(),
+        );
+        let props = Rc::new(WriterProperties::builder().build());
+        let mut file_writer =
+            SerializedFileWriter::new(file.try_clone().unwrap(), schema, props).unwrap();
+
+        for subset in &data {
+            let mut row_group_writer = file_writer.next_row_group().unwrap();
+            let col_writer = row_group_writer.next_column().unwrap();
+            if let Some(mut writer) = col_writer {
+                match writer {
+                    ColumnWriter::Int32ColumnWriter(ref mut typed) => {
+                        typed.write_batch(&subset[..], None, None).unwrap();
+                    }
+                    _ => {
+                        unimplemented!();
+                    }
+                }
+                row_group_writer.close_column(writer).unwrap();
+            }
+            file_writer.close_row_group(row_group_writer).unwrap();
+        }
+
+        file_writer.close().unwrap();
+
+        let reader = SerializedFileReader::new(file).unwrap();
+        assert_eq!(reader.num_row_groups(), data.len());
+        for i in 0..reader.num_row_groups() {
+            let row_group_reader = reader.get_row_group(i).unwrap();
+            let iter = row_group_reader.get_row_iter(None).unwrap();
+            let res = iter
+                .map(|elem| elem.get_int(0).unwrap())
+                .collect::<Vec<i32>>();
+            assert_eq!(res, data[i]);
+        }
+    }
+}

rust/src/parquet/mod.rs

@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#[macro_use]
+pub mod errors;
+pub mod basic;
+pub mod data_type;
+
+// Exported for external use, such as benchmarks
+pub use self::encodings::{decoding, encoding};
+pub use self::util::memory;
+
+#[macro_use]
+mod util;
+pub mod column;
+pub mod compression;
+mod encodings;
+pub mod file;
+pub mod record;
+pub mod schema;

rust/src/parquet/record/mod.rs

@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Contains record-based API for reading Parquet files.
+
+mod api;
+pub mod reader;
+mod triplet;
+
+pub use self::api::{List, ListAccessor, Map, MapAccessor, Row, RowAccessor};

rust/src/parquet/record/triplet.rs

@@ -0,0 +1,561 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::parquet::basic::Type as PhysicalType;
+use crate::parquet::column::reader::{get_typed_column_reader, ColumnReader, ColumnReaderImpl};
+use crate::parquet::data_type::*;
+use crate::parquet::errors::{ParquetError, Result};
+use crate::parquet::record::api::Field;
+use crate::parquet::schema::types::ColumnDescPtr;
+
+/// Macro to generate simple functions that cover all types of triplet iterator.
+/// $func is a function of a typed triplet iterator and $token is a either {`ref`} or
+/// {`ref`, `mut`}
+macro_rules! triplet_enum_func {
+  ($self:ident, $func:ident, $( $token:tt ),*) => ({
+    match *$self {
+      TripletIter::BoolTripletIter($($token)* typed) => typed.$func(),
+      TripletIter::Int32TripletIter($($token)* typed) => typed.$func(),
+      TripletIter::Int64TripletIter($($token)* typed) => typed.$func(),
+      TripletIter::Int96TripletIter($($token)* typed) => typed.$func(),
+      TripletIter::FloatTripletIter($($token)* typed) => typed.$func(),
+      TripletIter::DoubleTripletIter($($token)* typed) => typed.$func(),
+      TripletIter::ByteArrayTripletIter($($token)* typed) => typed.$func(),
+      TripletIter::FixedLenByteArrayTripletIter($($token)* typed) => typed.$func()
+    }
+  });
+}
+
+/// High level API wrapper on column reader.
+/// Provides per-element access for each primitive column.
+pub enum TripletIter {
+    BoolTripletIter(TypedTripletIter<BoolType>),
+    Int32TripletIter(TypedTripletIter<Int32Type>),
+    Int64TripletIter(TypedTripletIter<Int64Type>),
+    Int96TripletIter(TypedTripletIter<Int96Type>),
+    FloatTripletIter(TypedTripletIter<FloatType>),
+    DoubleTripletIter(TypedTripletIter<DoubleType>),
+    ByteArrayTripletIter(TypedTripletIter<ByteArrayType>),
+    FixedLenByteArrayTripletIter(TypedTripletIter<FixedLenByteArrayType>),
+}
+
+impl TripletIter {
+    /// Creates new triplet for column reader
+    pub fn new(descr: ColumnDescPtr, reader: ColumnReader, batch_size: usize) -> Self {
+        match descr.physical_type() {
+            PhysicalType::BOOLEAN => {
+                TripletIter::BoolTripletIter(TypedTripletIter::new(descr, batch_size, reader))
+            }
+            PhysicalType::INT32 => {
+                TripletIter::Int32TripletIter(TypedTripletIter::new(descr, batch_size, reader))
+            }
+            PhysicalType::INT64 => {
+                TripletIter::Int64TripletIter(TypedTripletIter::new(descr, batch_size, reader))
+            }
+            PhysicalType::INT96 => {
+                TripletIter::Int96TripletIter(TypedTripletIter::new(descr, batch_size, reader))
+            }
+            PhysicalType::FLOAT => {
+                TripletIter::FloatTripletIter(TypedTripletIter::new(descr, batch_size, reader))
+            }
+            PhysicalType::DOUBLE => {
+                TripletIter::DoubleTripletIter(TypedTripletIter::new(descr, batch_size, reader))
+            }
+            PhysicalType::BYTE_ARRAY => {
+                TripletIter::ByteArrayTripletIter(TypedTripletIter::new(descr, batch_size, reader))
+            }
+            PhysicalType::FIXED_LEN_BYTE_ARRAY => TripletIter::FixedLenByteArrayTripletIter(
+                TypedTripletIter::new(descr, batch_size, reader),
+            ),
+        }
+    }
+
+    /// Invokes underlying typed triplet iterator to buffer current value.
+    /// Should be called once - either before `is_null` or `current_value`.
+    #[inline]
+    pub fn read_next(&mut self) -> Result<bool> {
+        triplet_enum_func!(self, read_next, ref, mut)
+    }
+
+    /// Provides check on values/levels left without invoking the underlying typed triplet
+    /// iterator.
+    /// Returns true if more values/levels exist, false otherwise.
+    /// It is always in sync with `read_next` method.
+    #[inline]
+    pub fn has_next(&self) -> bool {
+        triplet_enum_func!(self, has_next, ref)
+    }
+
+    /// Returns current definition level for a leaf triplet iterator
+    #[inline]
+    pub fn current_def_level(&self) -> i16 {
+        triplet_enum_func!(self, current_def_level, ref)
+    }
+
+    /// Returns max definition level for a leaf triplet iterator
+    #[inline]
+    pub fn max_def_level(&self) -> i16 {
+        triplet_enum_func!(self, max_def_level, ref)
+    }
+
+    /// Returns current repetition level for a leaf triplet iterator
+    #[inline]
+    pub fn current_rep_level(&self) -> i16 {
+        triplet_enum_func!(self, current_rep_level, ref)
+    }
+
+    /// Returns max repetition level for a leaf triplet iterator
+    #[inline]
+    pub fn max_rep_level(&self) -> i16 {
+        triplet_enum_func!(self, max_rep_level, ref)
+    }
+
+    /// Returns true, if current value is null.
+    /// Based on the fact that for non-null value current definition level
+    /// equals to max definition level.
+    #[inline]
+    pub fn is_null(&self) -> bool {
+        self.current_def_level() < self.max_def_level()
+    }
+
+    /// Updates non-null value for current row.
+    pub fn current_value(&self) -> Field {
+        assert!(!self.is_null(), "Value is null");
+        match *self {
+            TripletIter::BoolTripletIter(ref typed) => {
+                Field::convert_bool(typed.column_descr(), *typed.current_value())
+            }
+            TripletIter::Int32TripletIter(ref typed) => {
+                Field::convert_int32(typed.column_descr(), *typed.current_value())
+            }
+            TripletIter::Int64TripletIter(ref typed) => {
+                Field::convert_int64(typed.column_descr(), *typed.current_value())
+            }
+            TripletIter::Int96TripletIter(ref typed) => {
+                Field::convert_int96(typed.column_descr(), typed.current_value().clone())
+            }
+            TripletIter::FloatTripletIter(ref typed) => {
+                Field::convert_float(typed.column_descr(), *typed.current_value())
+            }
+            TripletIter::DoubleTripletIter(ref typed) => {
+                Field::convert_double(typed.column_descr(), *typed.current_value())
+            }
+            TripletIter::ByteArrayTripletIter(ref typed) => {
+                Field::convert_byte_array(typed.column_descr(), typed.current_value().clone())
+            }
+            TripletIter::FixedLenByteArrayTripletIter(ref typed) => {
+                Field::convert_byte_array(typed.column_descr(), typed.current_value().clone())
+            }
+        }
+    }
+}
+
+/// Internal typed triplet iterator as a wrapper for column reader
+/// (primitive leaf column), provides per-element access.
+pub struct TypedTripletIter<T: DataType> {
+    reader: ColumnReaderImpl<T>,
+    column_descr: ColumnDescPtr,
+    batch_size: usize,
+    // type properties
+    max_def_level: i16,
+    max_rep_level: i16,
+    // values and levels
+    values: Vec<T::T>,
+    def_levels: Option<Vec<i16>>,
+    rep_levels: Option<Vec<i16>>,
+    // current index for the triplet (value, def, rep)
+    curr_triplet_index: usize,
+    // how many triplets are left before we need to buffer
+    triplets_left: usize,
+    // helper flag to quickly check if we have more values/levels to read
+    has_next: bool,
+}
+
+impl<T: DataType> TypedTripletIter<T> {
+    /// Creates new typed triplet iterator based on provided column reader.
+    /// Use batch size to specify the amount of values to buffer from column reader.
+    fn new(descr: ColumnDescPtr, batch_size: usize, column_reader: ColumnReader) -> Self {
+        assert!(
+            batch_size > 0,
+            "Expected positive batch size, found: {}",
+            batch_size
+        );
+
+        let max_def_level = descr.max_def_level();
+        let max_rep_level = descr.max_rep_level();
+
+        let def_levels = if max_def_level == 0 {
+            None
+        } else {
+            Some(vec![0; batch_size])
+        };
+        let rep_levels = if max_rep_level == 0 {
+            None
+        } else {
+            Some(vec![0; batch_size])
+        };
+
+        Self {
+            reader: get_typed_column_reader(column_reader),
+            column_descr: descr,
+            batch_size,
+            max_def_level,
+            max_rep_level,
+            values: vec![T::T::default(); batch_size],
+            def_levels,
+            rep_levels,
+            curr_triplet_index: 0,
+            triplets_left: 0,
+            has_next: false,
+        }
+    }
+
+    /// Returns column descriptor reference for the current typed triplet iterator.
+    #[inline]
+    pub fn column_descr(&self) -> &ColumnDescPtr {
+        &self.column_descr
+    }
+
+    /// Returns maximum definition level for the triplet iterator (leaf column).
+    #[inline]
+    fn max_def_level(&self) -> i16 {
+        self.max_def_level
+    }
+
+    /// Returns maximum repetition level for the triplet iterator (leaf column).
+    #[inline]
+    fn max_rep_level(&self) -> i16 {
+        self.max_rep_level
+    }
+
+    /// Returns current value.
+    /// Method does not advance the iterator, therefore can be called multiple times.
+    #[inline]
+    fn current_value(&self) -> &T::T {
+        assert!(
+            self.current_def_level() == self.max_def_level(),
+            "Cannot extract value, max definition level: {}, current level: {}",
+            self.max_def_level(),
+            self.current_def_level()
+        );
+        &self.values[self.curr_triplet_index]
+    }
+
+    /// Returns current definition level.
+    /// If field is required, then maximum definition level is returned.
+    #[inline]
+    fn current_def_level(&self) -> i16 {
+        match self.def_levels {
+            Some(ref vec) => vec[self.curr_triplet_index],
+            None => self.max_def_level,
+        }
+    }
+
+    /// Returns current repetition level.
+    /// If field is required, then maximum repetition level is returned.
+    #[inline]
+    fn current_rep_level(&self) -> i16 {
+        match self.rep_levels {
+            Some(ref vec) => vec[self.curr_triplet_index],
+            None => self.max_rep_level,
+        }
+    }
+
+    /// Quick check if iterator has more values/levels to read.
+    /// It is updated as a result of `read_next` method, so they are synchronized.
+    #[inline]
+    fn has_next(&self) -> bool {
+        self.has_next
+    }
+
+    /// Advances to the next triplet.
+    /// Returns true, if there are more records to read, false there are no records left.
+    fn read_next(&mut self) -> Result<bool> {
+        self.curr_triplet_index += 1;
+
+        if self.curr_triplet_index >= self.triplets_left {
+            let (values_read, levels_read) = {
+                // Get slice of definition levels, if available
+                let def_levels = match self.def_levels {
+                    Some(ref mut vec) => Some(&mut vec[..]),
+                    None => None,
+                };
+
+                // Get slice of repetition levels, if available
+                let rep_levels = match self.rep_levels {
+                    Some(ref mut vec) => Some(&mut vec[..]),
+                    None => None,
+                };
+
+                // Buffer triplets
+                self.reader
+                    .read_batch(self.batch_size, def_levels, rep_levels, &mut self.values)?
+            };
+
+            // No more values or levels to read
+            if values_read == 0 && levels_read == 0 {
+                self.has_next = false;
+                return Ok(false);
+            }
+
+            // We never read values more than levels
+            if levels_read == 0 || values_read == levels_read {
+                // There are no definition levels to read, column is required
+                // or definition levels match values, so it does not require spacing
+                self.curr_triplet_index = 0;
+                self.triplets_left = values_read;
+            } else if values_read < levels_read {
+                // Add spacing for triplets.
+                // The idea is setting values for positions in def_levels when current definition
+                // level equals to maximum definition level. Values and levels are guaranteed to
+                // line up, because of the column reader method.
+
+                // Note: if values_read == 0, then spacing will not be triggered
+                let mut idx = values_read;
+                let def_levels = self.def_levels.as_ref().unwrap();
+                for i in 0..levels_read {
+                    if def_levels[levels_read - i - 1] == self.max_def_level {
+                        idx -= 1; // This is done to avoid usize becoming a negative value
+                        self.values.swap(levels_read - i - 1, idx);
+                    }
+                }
+                self.curr_triplet_index = 0;
+                self.triplets_left = levels_read;
+            } else {
+                return Err(general_err!(
+                    "Spacing of values/levels is wrong, values_read: {}, levels_read: {}",
+                    values_read,
+                    levels_read
+                ));
+            }
+        }
+
+        self.has_next = true;
+        Ok(true)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use crate::parquet::file::reader::{FileReader, SerializedFileReader};
+    use crate::parquet::schema::types::ColumnPath;
+    use crate::parquet::util::test_common::get_test_file;
+
+    #[test]
+    #[should_panic(expected = "Expected positive batch size, found: 0")]
+    fn test_triplet_zero_batch_size() {
+        let column_path = ColumnPath::from(vec!["b_struct".to_string(), "b_c_int".to_string()]);
+        test_column_in_file(
+            "nulls.snappy.parquet",
+            0,
+            &column_path,
+            &vec![],
+            &vec![],
+            &vec![],
+        );
+    }
+
+    #[test]
+    fn test_triplet_null_column() {
+        let path = vec!["b_struct", "b_c_int"];
+        let values = vec![];
+        let def_levels = vec![1, 1, 1, 1, 1, 1, 1, 1];
+        let rep_levels = vec![0, 0, 0, 0, 0, 0, 0, 0];
+        test_triplet_iter(
+            "nulls.snappy.parquet",
+            path,
+            &values,
+            &def_levels,
+            &rep_levels,
+        );
+    }
+
+    #[test]
+    fn test_triplet_required_column() {
+        let path = vec!["ID"];
+        let values = vec![Field::Long(8)];
+        let def_levels = vec![0];
+        let rep_levels = vec![0];
+        test_triplet_iter(
+            "nonnullable.impala.parquet",
+            path,
+            &values,
+            &def_levels,
+            &rep_levels,
+        );
+    }
+
+    #[test]
+    fn test_triplet_optional_column() {
+        let path = vec!["nested_struct", "A"];
+        let values = vec![Field::Int(1), Field::Int(7)];
+        let def_levels = vec![2, 1, 1, 1, 1, 0, 2];
+        let rep_levels = vec![0, 0, 0, 0, 0, 0, 0];
+        test_triplet_iter(
+            "nullable.impala.parquet",
+            path,
+            &values,
+            &def_levels,
+            &rep_levels,
+        );
+    }
+
+    #[test]
+    fn test_triplet_optional_list_column() {
+        let path = vec!["a", "list", "element", "list", "element", "list", "element"];
+        let values = vec![
+            Field::Str("a".to_string()),
+            Field::Str("b".to_string()),
+            Field::Str("c".to_string()),
+            Field::Str("d".to_string()),
+            Field::Str("a".to_string()),
+            Field::Str("b".to_string()),
+            Field::Str("c".to_string()),
+            Field::Str("d".to_string()),
+            Field::Str("e".to_string()),
+            Field::Str("a".to_string()),
+            Field::Str("b".to_string()),
+            Field::Str("c".to_string()),
+            Field::Str("d".to_string()),
+            Field::Str("e".to_string()),
+            Field::Str("f".to_string()),
+        ];
+        let def_levels = vec![7, 7, 7, 4, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 4, 7];
+        let rep_levels = vec![0, 3, 2, 1, 2, 0, 3, 2, 3, 1, 2, 0, 3, 2, 3, 2, 1, 2];
+        test_triplet_iter(
+            "nested_lists.snappy.parquet",
+            path,
+            &values,
+            &def_levels,
+            &rep_levels,
+        );
+    }
+
+    #[test]
+    fn test_triplet_optional_map_column() {
+        let path = vec!["a", "key_value", "value", "key_value", "key"];
+        let values = vec![
+            Field::Int(1),
+            Field::Int(2),
+            Field::Int(1),
+            Field::Int(1),
+            Field::Int(3),
+            Field::Int(4),
+            Field::Int(5),
+        ];
+        let def_levels = vec![4, 4, 4, 2, 3, 4, 4, 4, 4];
+        let rep_levels = vec![0, 2, 0, 0, 0, 0, 0, 2, 2];
+        test_triplet_iter(
+            "nested_maps.snappy.parquet",
+            path,
+            &values,
+            &def_levels,
+            &rep_levels,
+        );
+    }
+
+    // Check triplet iterator across different batch sizes
+    fn test_triplet_iter(
+        file_name: &str,
+        column_path: Vec<&str>,
+        expected_values: &[Field],
+        expected_def_levels: &[i16],
+        expected_rep_levels: &[i16],
+    ) {
+        // Convert path into column path
+        let path: Vec<String> = column_path.iter().map(|x| x.to_string()).collect();
+        let column_path = ColumnPath::from(path);
+
+        let batch_sizes = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 128, 256];
+        for batch_size in batch_sizes {
+            test_column_in_file(
+                file_name,
+                batch_size,
+                &column_path,
+                expected_values,
+                expected_def_levels,
+                expected_rep_levels,
+            );
+        }
+    }
+
+    // Check values of a selectd column in a file
+    fn test_column_in_file(
+        file_name: &str,
+        batch_size: usize,
+        column_path: &ColumnPath,
+        expected_values: &[Field],
+        expected_def_levels: &[i16],
+        expected_rep_levels: &[i16],
+    ) {
+        let file = get_test_file(file_name);
+        let file_reader = SerializedFileReader::new(file).unwrap();
+        // Get schema descriptor
+        let file_metadata = file_reader.metadata().file_metadata();
+        let schema = file_metadata.schema_descr();
+        // Get first row group
+        let row_group_reader = file_reader.get_row_group(0).unwrap();
+
+        for i in 0..schema.num_columns() {
+            let descr = schema.column(i);
+            if descr.path() == column_path {
+                let reader = row_group_reader.get_column_reader(i).unwrap();
+                test_triplet_column(
+                    descr,
+                    reader,
+                    batch_size,
+                    expected_values,
+                    expected_def_levels,
+                    expected_rep_levels,
+                );
+            }
+        }
+    }
+
+    // Check values for individual triplet iterator
+    fn test_triplet_column(
+        descr: ColumnDescPtr,
+        reader: ColumnReader,
+        batch_size: usize,
+        expected_values: &[Field],
+        expected_def_levels: &[i16],
+        expected_rep_levels: &[i16],
+    ) {
+        let mut iter = TripletIter::new(descr.clone(), reader, batch_size);
+        let mut values: Vec<Field> = Vec::new();
+        let mut def_levels: Vec<i16> = Vec::new();
+        let mut rep_levels: Vec<i16> = Vec::new();
+
+        assert_eq!(iter.max_def_level(), descr.max_def_level());
+        assert_eq!(iter.max_rep_level(), descr.max_rep_level());
+
+        while let Ok(true) = iter.read_next() {
+            assert!(iter.has_next());
+            if !iter.is_null() {
+                values.push(iter.current_value());
+            }
+            def_levels.push(iter.current_def_level());
+            rep_levels.push(iter.current_rep_level());
+        }
+
+        assert_eq!(values, expected_values);
+        assert_eq!(def_levels, expected_def_levels);
+        assert_eq!(rep_levels, expected_rep_levels);
+    }
+}

rust/src/parquet/schema/mod.rs

@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Parquet schema definitions and methods to print and parse schema.
+//!
+//! # Example
+//!
+//! ```rust
+//! use arrow::parquet::{
+//!     basic::{LogicalType, Repetition, Type as PhysicalType},
+//!     schema::{parser, printer, types::Type},
+//! };
+//! use std::rc::Rc;
+//!
+//! // Create the following schema:
+//! //
+//! // message schema {
+//! //   OPTIONAL BYTE_ARRAY a (UTF8);
+//! //   REQUIRED INT32 b;
+//! // }
+//!
+//! let field_a = Type::primitive_type_builder("a", PhysicalType::BYTE_ARRAY)
+//!     .with_logical_type(LogicalType::UTF8)
+//!     .with_repetition(Repetition::OPTIONAL)
+//!     .build()
+//!     .unwrap();
+//!
+//! let field_b = Type::primitive_type_builder("b", PhysicalType::INT32)
+//!     .with_repetition(Repetition::REQUIRED)
+//!     .build()
+//!     .unwrap();
+//!
+//! let schema = Type::group_type_builder("schema")
+//!     .with_fields(&mut vec![Rc::new(field_a), Rc::new(field_b)])
+//!     .build()
+//!     .unwrap();
+//!
+//! let mut buf = Vec::new();
+//!
+//! // Print schema into buffer
+//! printer::print_schema(&mut buf, &schema);
+//!
+//! // Parse schema from the string
+//! let string_schema = String::from_utf8(buf).unwrap();
+//! let parsed_schema = parser::parse_message_type(&string_schema).unwrap();
+//!
+//! assert_eq!(schema, parsed_schema);
+//! ```
+
+pub mod parser;
+pub mod printer;
+pub mod types;

rust/src/parquet/schema/parser.rs

@@ -0,0 +1,764 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Parquet schema parser.
+//! Provides methods to parse and validate string message type into Parquet
+//! [`Type`](`::schema::types::Type`).
+//!
+//! # Example
+//!
+//! ```rust
+//! use arrow::parquet::schema::parser::parse_message_type;
+//!
+//! let message_type = "
+//!   message spark_schema {
+//!     OPTIONAL BYTE_ARRAY a (UTF8);
+//!     REQUIRED INT32 b;
+//!     REQUIRED DOUBLE c;
+//!     REQUIRED BOOLEAN d;
+//!     OPTIONAL group e (LIST) {
+//!       REPEATED group list {
+//!         REQUIRED INT32 element;
+//!       }
+//!     }
+//!   }
+//! ";
+//!
+//! let schema = parse_message_type(message_type).expect("Expected valid schema");
+//! println!("{:?}", schema);
+//! ```
+
+use std::rc::Rc;
+
+use crate::parquet::basic::{LogicalType, Repetition, Type as PhysicalType};
+use crate::parquet::errors::{ParquetError, Result};
+use crate::parquet::schema::types::{Type, TypePtr};
+
+/// Parses message type as string into a Parquet [`Type`](`::schema::types::Type`) which,
+/// for example, could be used to extract individual columns. Returns Parquet general
+/// error when parsing or validation fails.
+pub fn parse_message_type<'a>(message_type: &'a str) -> Result<Type> {
+    let mut parser = Parser {
+        tokenizer: &mut Tokenizer::from_str(message_type),
+    };
+    parser.parse_message_type()
+}
+
+/// Tokenizer to split message type string into tokens that are separated using characters
+/// defined in `is_schema_delim` method. Tokenizer also preserves delimiters as tokens.
+/// Tokenizer provides Iterator interface to process tokens; it also allows to step back
+/// to reprocess previous tokens.
+struct Tokenizer<'a> {
+    // List of all tokens for a string
+    tokens: Vec<&'a str>,
+    // Current index of vector
+    index: usize,
+}
+
+impl<'a> Tokenizer<'a> {
+    // Create tokenizer from message type string
+    pub fn from_str(string: &'a str) -> Self {
+        let vec = string
+            .split_whitespace()
+            .flat_map(|t| Self::split_token(t))
+            .collect();
+        Tokenizer {
+            tokens: vec,
+            index: 0,
+        }
+    }
+
+    // List of all special characters in schema
+    fn is_schema_delim(c: char) -> bool {
+        c == ';' || c == '{' || c == '}' || c == '(' || c == ')' || c == '=' || c == ','
+    }
+
+    /// Splits string into tokens; input string can already be token or can contain
+    /// delimiters, e.g. required" -> Vec("required") and
+    /// "(UTF8);" -> Vec("(", "UTF8", ")", ";")
+    fn split_token(string: &str) -> Vec<&str> {
+        let mut buffer: Vec<&str> = Vec::new();
+        let mut tail = string;
+        while let Some(index) = tail.find(Self::is_schema_delim) {
+            let (h, t) = tail.split_at(index);
+            if !h.is_empty() {
+                buffer.push(h);
+            }
+            buffer.push(&t[0..1]);
+            tail = &t[1..];
+        }
+        if !tail.is_empty() {
+            buffer.push(tail);
+        }
+        buffer
+    }
+
+    // Move pointer to a previous element
+    fn backtrack(&mut self) {
+        self.index -= 1;
+    }
+}
+
+impl<'a> Iterator for Tokenizer<'a> {
+    type Item = &'a str;
+
+    fn next(&mut self) -> Option<&'a str> {
+        if self.index < self.tokens.len() {
+            self.index += 1;
+            Some(self.tokens[self.index - 1])
+        } else {
+            None
+        }
+    }
+}
+
+/// Internal Schema parser.
+/// Traverses message type using tokenizer and parses each group/primitive type
+/// recursively.
+struct Parser<'a> {
+    tokenizer: &'a mut Tokenizer<'a>,
+}
+
+// Utility function to assert token on validity.
+fn assert_token(token: Option<&str>, expected: &str) -> Result<()> {
+    match token {
+        Some(value) if value == expected => Ok(()),
+        Some(other) => Err(general_err!(
+            "Expected '{}', found token '{}'",
+            expected,
+            other
+        )),
+        None => Err(general_err!(
+            "Expected '{}', but no token found (None)",
+            expected
+        )),
+    }
+}
+
+// Utility function to parse i32 or return general error.
+fn parse_i32(value: Option<&str>, not_found_msg: &str, parse_fail_msg: &str) -> Result<i32> {
+    value
+        .ok_or(general_err!(not_found_msg))
+        .and_then(|v| v.parse::<i32>().map_err(|_| general_err!(parse_fail_msg)))
+}
+
+impl<'a> Parser<'a> {
+    // Entry function to parse message type, uses internal tokenizer.
+    fn parse_message_type(&mut self) -> Result<Type> {
+        // Check that message type starts with "message".
+        match self.tokenizer.next() {
+            Some("message") => {
+                let name = self
+                    .tokenizer
+                    .next()
+                    .ok_or(general_err!("Expected name, found None"))?;
+                let mut fields = self.parse_child_types()?;
+                Type::group_type_builder(name)
+                    .with_fields(&mut fields)
+                    .build()
+            }
+            _ => Err(general_err!("Message type does not start with 'message'")),
+        }
+    }
+
+    // Parses child types for a current group type.
+    // This is only invoked on root and group types.
+    fn parse_child_types(&mut self) -> Result<Vec<TypePtr>> {
+        assert_token(self.tokenizer.next(), "{")?;
+        let mut vec = Vec::new();
+        while let Some(value) = self.tokenizer.next() {
+            if value == "}" {
+                break;
+            } else {
+                self.tokenizer.backtrack();
+                vec.push(Rc::new(self.add_type()?));
+            }
+        }
+        Ok(vec)
+    }
+
+    fn add_type(&mut self) -> Result<Type> {
+        // Parse repetition
+        let repetition = self
+            .tokenizer
+            .next()
+            .ok_or(general_err!("Expected repetition, found None"))
+            .and_then(|v| v.to_uppercase().parse::<Repetition>())?;
+
+        match self.tokenizer.next() {
+            Some(group) if group.to_uppercase() == "GROUP" => self.add_group_type(Some(repetition)),
+            Some(type_string) => {
+                let physical_type = type_string.to_uppercase().parse::<PhysicalType>()?;
+                self.add_primitive_type(repetition, physical_type)
+            }
+            None => Err(general_err!("Invalid type, could not extract next token")),
+        }
+    }
+
+    fn add_group_type(&mut self, repetition: Option<Repetition>) -> Result<Type> {
+        // Parse name of the group type
+        let name = self
+            .tokenizer
+            .next()
+            .ok_or(general_err!("Expected name, found None"))?;
+
+        // Parse logical type if exists
+        let logical_type = if let Some("(") = self.tokenizer.next() {
+            let tpe = self
+                .tokenizer
+                .next()
+                .ok_or(general_err!("Expected logical type, found None"))
+                .and_then(|v| v.to_uppercase().parse::<LogicalType>())?;
+            assert_token(self.tokenizer.next(), ")")?;
+            tpe
+        } else {
+            self.tokenizer.backtrack();
+            LogicalType::NONE
+        };
+
+        // Parse optional id
+        let id = if let Some("=") = self.tokenizer.next() {
+            self.tokenizer.next().and_then(|v| v.parse::<i32>().ok())
+        } else {
+            self.tokenizer.backtrack();
+            None
+        };
+
+        let mut fields = self.parse_child_types()?;
+        let mut builder = Type::group_type_builder(name)
+            .with_logical_type(logical_type)
+            .with_fields(&mut fields);
+        if let Some(rep) = repetition {
+            builder = builder.with_repetition(rep);
+        }
+        if let Some(id) = id {
+            builder = builder.with_id(id);
+        }
+        builder.build()
+    }
+
+    fn add_primitive_type(
+        &mut self,
+        repetition: Repetition,
+        physical_type: PhysicalType,
+    ) -> Result<Type> {
+        // Read type length if the type is FIXED_LEN_BYTE_ARRAY.
+        let mut length: i32 = -1;
+        if physical_type == PhysicalType::FIXED_LEN_BYTE_ARRAY {
+            assert_token(self.tokenizer.next(), "(")?;
+            length = parse_i32(
+                self.tokenizer.next(),
+                "Expected length for FIXED_LEN_BYTE_ARRAY, found None",
+                "Failed to parse length for FIXED_LEN_BYTE_ARRAY",
+            )?;
+            assert_token(self.tokenizer.next(), ")")?;
+        }
+
+        // Parse name of the primitive type
+        let name = self
+            .tokenizer
+            .next()
+            .ok_or(general_err!("Expected name, found None"))?;
+
+        // Parse logical type
+        let (logical_type, precision, scale) = if let Some("(") = self.tokenizer.next() {
+            let tpe = self
+                .tokenizer
+                .next()
+                .ok_or(general_err!("Expected logical type, found None"))
+                .and_then(|v| v.to_uppercase().parse::<LogicalType>())?;
+
+            // Parse precision and scale for decimals
+            let mut precision: i32 = -1;
+            let mut scale: i32 = -1;
+
+            if tpe == LogicalType::DECIMAL {
+                if let Some("(") = self.tokenizer.next() {
+                    // Parse precision
+                    precision = parse_i32(
+                        self.tokenizer.next(),
+                        "Expected precision, found None",
+                        "Failed to parse precision for DECIMAL type",
+                    )?;
+
+                    // Parse scale
+                    scale = if let Some(",") = self.tokenizer.next() {
+                        parse_i32(
+                            self.tokenizer.next(),
+                            "Expected scale, found None",
+                            "Failed to parse scale for DECIMAL type",
+                        )?
+                    } else {
+                        // Scale is not provided, set it to 0.
+                        self.tokenizer.backtrack();
+                        0
+                    };
+
+                    assert_token(self.tokenizer.next(), ")")?;
+                } else {
+                    self.tokenizer.backtrack();
+                }
+            }
+
+            assert_token(self.tokenizer.next(), ")")?;
+            (tpe, precision, scale)
+        } else {
+            self.tokenizer.backtrack();
+            (LogicalType::NONE, -1, -1)
+        };
+
+        // Parse optional id
+        let id = if let Some("=") = self.tokenizer.next() {
+            self.tokenizer.next().and_then(|v| v.parse::<i32>().ok())
+        } else {
+            self.tokenizer.backtrack();
+            None
+        };
+        assert_token(self.tokenizer.next(), ";")?;
+
+        let mut builder = Type::primitive_type_builder(name, physical_type)
+            .with_repetition(repetition)
+            .with_logical_type(logical_type)
+            .with_length(length)
+            .with_precision(precision)
+            .with_scale(scale);
+        if let Some(id) = id {
+            builder = builder.with_id(id);
+        }
+        Ok(builder.build()?)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_tokenize_empty_string() {
+        assert_eq!(Tokenizer::from_str("").next(), None);
+    }
+
+    #[test]
+    fn test_tokenize_delimiters() {
+        let mut iter = Tokenizer::from_str(",;{}()=");
+        assert_eq!(iter.next(), Some(","));
+        assert_eq!(iter.next(), Some(";"));
+        assert_eq!(iter.next(), Some("{"));
+        assert_eq!(iter.next(), Some("}"));
+        assert_eq!(iter.next(), Some("("));
+        assert_eq!(iter.next(), Some(")"));
+        assert_eq!(iter.next(), Some("="));
+        assert_eq!(iter.next(), None);
+    }
+
+    #[test]
+    fn test_tokenize_delimiters_with_whitespaces() {
+        let mut iter = Tokenizer::from_str(" , ; { } ( ) = ");
+        assert_eq!(iter.next(), Some(","));
+        assert_eq!(iter.next(), Some(";"));
+        assert_eq!(iter.next(), Some("{"));
+        assert_eq!(iter.next(), Some("}"));
+        assert_eq!(iter.next(), Some("("));
+        assert_eq!(iter.next(), Some(")"));
+        assert_eq!(iter.next(), Some("="));
+        assert_eq!(iter.next(), None);
+    }
+
+    #[test]
+    fn test_tokenize_words() {
+        let mut iter = Tokenizer::from_str("abc def ghi jkl mno");
+        assert_eq!(iter.next(), Some("abc"));
+        assert_eq!(iter.next(), Some("def"));
+        assert_eq!(iter.next(), Some("ghi"));
+        assert_eq!(iter.next(), Some("jkl"));
+        assert_eq!(iter.next(), Some("mno"));
+        assert_eq!(iter.next(), None);
+    }
+
+    #[test]
+    fn test_tokenize_backtrack() {
+        let mut iter = Tokenizer::from_str("abc;");
+        assert_eq!(iter.next(), Some("abc"));
+        assert_eq!(iter.next(), Some(";"));
+        iter.backtrack();
+        assert_eq!(iter.next(), Some(";"));
+        assert_eq!(iter.next(), None);
+    }
+
+    #[test]
+    fn test_tokenize_message_type() {
+        let schema = "
+    message schema {
+      required int32 a;
+      optional binary c (UTF8);
+      required group d {
+        required int32 a;
+        optional binary c (UTF8);
+      }
+      required group e (LIST) {
+        repeated group list {
+          required int32 element;
+        }
+      }
+    }
+    ";
+        let mut iter = Tokenizer::from_str(schema);
+        let mut res = Vec::new();
+        while let Some(token) = iter.next() {
+            res.push(token);
+        }
+        assert_eq!(
+            res,
+            vec![
+                "message", "schema", "{", "required", "int32", "a", ";", "optional", "binary", "c",
+                "(", "UTF8", ")", ";", "required", "group", "d", "{", "required", "int32", "a",
+                ";", "optional", "binary", "c", "(", "UTF8", ")", ";", "}", "required", "group",
+                "e", "(", "LIST", ")", "{", "repeated", "group", "list", "{", "required", "int32",
+                "element", ";", "}", "}", "}"
+            ]
+        );
+    }
+
+    #[test]
+    fn test_assert_token() {
+        assert!(assert_token(Some("a"), "a").is_ok());
+        assert!(assert_token(Some("a"), "b").is_err());
+        assert!(assert_token(None, "b").is_err());
+    }
+
+    #[test]
+    fn test_parse_message_type_invalid() {
+        let mut iter = Tokenizer::from_str("test");
+        let result = Parser {
+            tokenizer: &mut iter,
+        }
+        .parse_message_type();
+        assert!(result.is_err());
+        assert_eq!(
+            result.unwrap_err().to_string(),
+            "Parquet error: Message type does not start with 'message'"
+        );
+    }
+
+    #[test]
+    fn test_parse_message_type_no_name() {
+        let mut iter = Tokenizer::from_str("message");
+        let result = Parser {
+            tokenizer: &mut iter,
+        }
+        .parse_message_type();
+        assert!(result.is_err());
+        assert_eq!(
+            result.unwrap_err().to_string(),
+            "Parquet error: Expected name, found None"
+        );
+    }
+
+    #[test]
+    fn test_parse_message_type_fixed_byte_array() {
+        let schema = "
+    message schema {
+      REQUIRED FIXED_LEN_BYTE_ARRAY col;
+    }
+    ";
+        let mut iter = Tokenizer::from_str(schema);
+        let result = Parser {
+            tokenizer: &mut iter,
+        }
+        .parse_message_type();
+        assert!(result.is_err());
+
+        let schema = "
+    message schema {
+      REQUIRED FIXED_LEN_BYTE_ARRAY(16) col;
+    }
+    ";
+        let mut iter = Tokenizer::from_str(schema);
+        let result = Parser {
+            tokenizer: &mut iter,
+        }
+        .parse_message_type();
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_parse_message_type_decimal() {
+        // It is okay for decimal to omit precision and scale with right syntax.
+        // Here we test wrong syntax of decimal type
+
+        // Invalid decimal syntax
+        let schema = "
+    message root {
+      optional int32 f1 (DECIMAL();
+    }
+    ";
+        let mut iter = Tokenizer::from_str(schema);
+        let result = Parser {
+            tokenizer: &mut iter,
+        }
+        .parse_message_type();
+        assert!(result.is_err());
+
+        // Invalid decimal, need precision and scale
+        let schema = "
+    message root {
+      optional int32 f1 (DECIMAL());
+    }
+    ";
+        let mut iter = Tokenizer::from_str(schema);
+        let result = Parser {
+            tokenizer: &mut iter,
+        }
+        .parse_message_type();
+        assert!(result.is_err());
+
+        // Invalid decimal because of `,` - has precision, needs scale
+        let schema = "
+    message root {
+      optional int32 f1 (DECIMAL(8,));
+    }
+    ";
+        let mut iter = Tokenizer::from_str(schema);
+        let result = Parser {
+            tokenizer: &mut iter,
+        }
+        .parse_message_type();
+        assert!(result.is_err());
+
+        // Invalid decimal because, we always require either precision or scale to be
+        // specified as part of logical type
+        let schema = "
+    message root {
+      optional int32 f3 (DECIMAL);
+    }
+    ";
+        let mut iter = Tokenizer::from_str(schema);
+        let result = Parser {
+            tokenizer: &mut iter,
+        }
+        .parse_message_type();
+        assert!(result.is_err());
+
+        // Valid decimal (precision, scale)
+        let schema = "
+    message root {
+      optional int32 f1 (DECIMAL(8, 3));
+      optional int32 f2 (DECIMAL(8));
+    }
+    ";
+        let mut iter = Tokenizer::from_str(schema);
+        let result = Parser {
+            tokenizer: &mut iter,
+        }
+        .parse_message_type();
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_parse_message_type_compare_1() {
+        let schema = "
+    message root {
+      optional fixed_len_byte_array(5) f1 (DECIMAL(9, 3));
+      optional fixed_len_byte_array (16) f2 (DECIMAL (38, 18));
+    }
+    ";
+        let mut iter = Tokenizer::from_str(schema);
+        let message = Parser {
+            tokenizer: &mut iter,
+        }
+        .parse_message_type()
+        .unwrap();
+
+        let expected = Type::group_type_builder("root")
+            .with_fields(&mut vec![
+                Rc::new(
+                    Type::primitive_type_builder("f1", PhysicalType::FIXED_LEN_BYTE_ARRAY)
+                        .with_logical_type(LogicalType::DECIMAL)
+                        .with_length(5)
+                        .with_precision(9)
+                        .with_scale(3)
+                        .build()
+                        .unwrap(),
+                ),
+                Rc::new(
+                    Type::primitive_type_builder("f2", PhysicalType::FIXED_LEN_BYTE_ARRAY)
+                        .with_logical_type(LogicalType::DECIMAL)
+                        .with_length(16)
+                        .with_precision(38)
+                        .with_scale(18)
+                        .build()
+                        .unwrap(),
+                ),
+            ])
+            .build()
+            .unwrap();
+
+        assert_eq!(message, expected);
+    }
+
+    #[test]
+    fn test_parse_message_type_compare_2() {
+        let schema = "
+    message root {
+      required group a0 {
+        optional group a1 (LIST) {
+          repeated binary a2 (UTF8);
+        }
+
+        optional group b1 (LIST) {
+          repeated group b2 {
+            optional int32 b3;
+            optional double b4;
+          }
+        }
+      }
+    }
+    ";
+        let mut iter = Tokenizer::from_str(schema);
+        let message = Parser {
+            tokenizer: &mut iter,
+        }
+        .parse_message_type()
+        .unwrap();
+
+        let expected = Type::group_type_builder("root")
+            .with_fields(&mut vec![Rc::new(
+                Type::group_type_builder("a0")
+                    .with_repetition(Repetition::REQUIRED)
+                    .with_fields(&mut vec![
+                        Rc::new(
+                            Type::group_type_builder("a1")
+                                .with_repetition(Repetition::OPTIONAL)
+                                .with_logical_type(LogicalType::LIST)
+                                .with_fields(&mut vec![Rc::new(
+                                    Type::primitive_type_builder("a2", PhysicalType::BYTE_ARRAY)
+                                        .with_repetition(Repetition::REPEATED)
+                                        .with_logical_type(LogicalType::UTF8)
+                                        .build()
+                                        .unwrap(),
+                                )])
+                                .build()
+                                .unwrap(),
+                        ),
+                        Rc::new(
+                            Type::group_type_builder("b1")
+                                .with_repetition(Repetition::OPTIONAL)
+                                .with_logical_type(LogicalType::LIST)
+                                .with_fields(&mut vec![Rc::new(
+                                    Type::group_type_builder("b2")
+                                        .with_repetition(Repetition::REPEATED)
+                                        .with_fields(&mut vec![
+                                            Rc::new(
+                                                Type::primitive_type_builder(
+                                                    "b3",
+                                                    PhysicalType::INT32,
+                                                )
+                                                .build()
+                                                .unwrap(),
+                                            ),
+                                            Rc::new(
+                                                Type::primitive_type_builder(
+                                                    "b4",
+                                                    PhysicalType::DOUBLE,
+                                                )
+                                                .build()
+                                                .unwrap(),
+                                            ),
+                                        ])
+                                        .build()
+                                        .unwrap(),
+                                )])
+                                .build()
+                                .unwrap(),
+                        ),
+                    ])
+                    .build()
+                    .unwrap(),
+            )])
+            .build()
+            .unwrap();
+
+        assert_eq!(message, expected);
+    }
+
+    #[test]
+    fn test_parse_message_type_compare_3() {
+        let schema = "
+    message root {
+      required int32 _1 (INT_8);
+      required int32 _2 (INT_16);
+      required float _3;
+      required double _4;
+      optional int32 _5 (DATE);
+      optional binary _6 (UTF8);
+    }
+    ";
+        let mut iter = Tokenizer::from_str(schema);
+        let message = Parser {
+            tokenizer: &mut iter,
+        }
+        .parse_message_type()
+        .unwrap();
+
+        let mut fields = vec![
+            Rc::new(
+                Type::primitive_type_builder("_1", PhysicalType::INT32)
+                    .with_repetition(Repetition::REQUIRED)
+                    .with_logical_type(LogicalType::INT_8)
+                    .build()
+                    .unwrap(),
+            ),
+            Rc::new(
+                Type::primitive_type_builder("_2", PhysicalType::INT32)
+                    .with_repetition(Repetition::REQUIRED)
+                    .with_logical_type(LogicalType::INT_16)
+                    .build()
+                    .unwrap(),
+            ),
+            Rc::new(
+                Type::primitive_type_builder("_3", PhysicalType::FLOAT)
+                    .with_repetition(Repetition::REQUIRED)
+                    .build()
+                    .unwrap(),
+            ),
+            Rc::new(
+                Type::primitive_type_builder("_4", PhysicalType::DOUBLE)
+                    .with_repetition(Repetition::REQUIRED)
+                    .build()
+                    .unwrap(),
+            ),
+            Rc::new(
+                Type::primitive_type_builder("_5", PhysicalType::INT32)
+                    .with_logical_type(LogicalType::DATE)
+                    .build()
+                    .unwrap(),
+            ),
+            Rc::new(
+                Type::primitive_type_builder("_6", PhysicalType::BYTE_ARRAY)
+                    .with_logical_type(LogicalType::UTF8)
+                    .build()
+                    .unwrap(),
+            ),
+        ];
+
+        let expected = Type::group_type_builder("root")
+            .with_fields(&mut fields)
+            .build()
+            .unwrap();
+        assert_eq!(message, expected);
+    }
+}

rust/src/parquet/schema/printer.rs

@@ -0,0 +1,467 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Parquet schema printer.
+//! Provides methods to print Parquet file schema and list file metadata.
+//!
+//! # Example
+//!
+//! ```rust
+//! use arrow::parquet::{
+//!     file::reader::{FileReader, SerializedFileReader},
+//!     schema::printer::{print_file_metadata, print_parquet_metadata, print_schema},
+//! };
+//! use std::{fs::File, path::Path};
+//!
+//! // Open a file
+//! let path = Path::new("test.parquet");
+//! if let Ok(file) = File::open(&path) {
+//!     let reader = SerializedFileReader::new(file).unwrap();
+//!     let parquet_metadata = reader.metadata();
+//!
+//!     print_parquet_metadata(&mut std::io::stdout(), &parquet_metadata);
+//!     print_file_metadata(&mut std::io::stdout(), &parquet_metadata.file_metadata());
+//!
+//!     print_schema(
+//!         &mut std::io::stdout(),
+//!         &parquet_metadata.file_metadata().schema(),
+//!     );
+//! }
+//! ```
+
+use std::{fmt, io};
+
+use crate::parquet::basic::{LogicalType, Type as PhysicalType};
+use crate::parquet::file::metadata::{
+    ColumnChunkMetaData, FileMetaData, ParquetMetaData, RowGroupMetaData,
+};
+use crate::parquet::schema::types::Type;
+
+/// Prints Parquet metadata [`ParquetMetaData`](`::file::metadata::ParquetMetaData`)
+/// information.
+#[allow(unused_must_use)]
+pub fn print_parquet_metadata(out: &mut io::Write, metadata: &ParquetMetaData) {
+    print_file_metadata(out, &metadata.file_metadata());
+    writeln!(out, "");
+    writeln!(out, "");
+    writeln!(out, "num of row groups: {}", metadata.num_row_groups());
+    writeln!(out, "row groups:");
+    writeln!(out, "");
+    for (i, rg) in metadata.row_groups().iter().enumerate() {
+        writeln!(out, "row group {}:", i);
+        print_dashes(out, 80);
+        print_row_group_metadata(out, rg);
+    }
+}
+
+/// Prints file metadata [`FileMetaData`](`::file::metadata::FileMetaData`) information.
+#[allow(unused_must_use)]
+pub fn print_file_metadata(out: &mut io::Write, file_metadata: &FileMetaData) {
+    writeln!(out, "version: {}", file_metadata.version());
+    writeln!(out, "num of rows: {}", file_metadata.num_rows());
+    if let Some(created_by) = file_metadata.created_by().as_ref() {
+        writeln!(out, "created by: {}", created_by);
+    }
+    let schema = file_metadata.schema();
+    print_schema(out, schema);
+}
+
+/// Prints Parquet [`Type`](`::schema::types::Type`) information.
+#[allow(unused_must_use)]
+pub fn print_schema(out: &mut io::Write, tp: &Type) {
+    // TODO: better if we can pass fmt::Write to Printer.
+    // But how can we make it to accept both io::Write & fmt::Write?
+    let mut s = String::new();
+    {
+        let mut printer = Printer::new(&mut s);
+        printer.print(tp);
+    }
+    writeln!(out, "{}", s);
+}
+
+#[allow(unused_must_use)]
+fn print_row_group_metadata(out: &mut io::Write, rg_metadata: &RowGroupMetaData) {
+    writeln!(out, "total byte size: {}", rg_metadata.total_byte_size());
+    writeln!(out, "num of rows: {}", rg_metadata.num_rows());
+    writeln!(out, "");
+    writeln!(out, "num of columns: {}", rg_metadata.num_columns());
+    writeln!(out, "columns: ");
+    for (i, cc) in rg_metadata.columns().iter().enumerate() {
+        writeln!(out, "");
+        writeln!(out, "column {}:", i);
+        print_dashes(out, 80);
+        print_column_chunk_metadata(out, cc);
+    }
+}
+
+#[allow(unused_must_use)]
+fn print_column_chunk_metadata(out: &mut io::Write, cc_metadata: &ColumnChunkMetaData) {
+    writeln!(out, "column type: {}", cc_metadata.column_type());
+    writeln!(out, "column path: {}", cc_metadata.column_path());
+    let encoding_strs: Vec<_> = cc_metadata
+        .encodings()
+        .iter()
+        .map(|e| format!("{}", e))
+        .collect();
+    writeln!(out, "encodings: {}", encoding_strs.join(" "));
+    let file_path_str = match cc_metadata.file_path() {
+        None => "N/A",
+        Some(ref fp) => *fp,
+    };
+    writeln!(out, "file path: {}", file_path_str);
+    writeln!(out, "file offset: {}", cc_metadata.file_offset());
+    writeln!(out, "num of values: {}", cc_metadata.num_values());
+    writeln!(
+        out,
+        "total compressed size (in bytes): {}",
+        cc_metadata.compressed_size()
+    );
+    writeln!(
+        out,
+        "total uncompressed size (in bytes): {}",
+        cc_metadata.uncompressed_size()
+    );
+    writeln!(out, "data page offset: {}", cc_metadata.data_page_offset());
+    let index_page_offset_str = match cc_metadata.index_page_offset() {
+        None => "N/A".to_owned(),
+        Some(ipo) => ipo.to_string(),
+    };
+    writeln!(out, "index page offset: {}", index_page_offset_str);
+    let dict_page_offset_str = match cc_metadata.dictionary_page_offset() {
+        None => "N/A".to_owned(),
+        Some(dpo) => dpo.to_string(),
+    };
+    writeln!(out, "dictionary page offset: {}", dict_page_offset_str);
+    let statistics_str = match cc_metadata.statistics() {
+        None => "N/A".to_owned(),
+        Some(stats) => stats.to_string(),
+    };
+    writeln!(out, "statistics: {}", statistics_str);
+    writeln!(out, "");
+}
+
+#[allow(unused_must_use)]
+fn print_dashes(out: &mut io::Write, num: i32) {
+    for _ in 0..num {
+        write!(out, "-");
+    }
+    writeln!(out, "");
+}
+
+const INDENT_WIDTH: i32 = 2;
+
+/// Struct for printing Parquet message type.
+struct Printer<'a> {
+    output: &'a mut fmt::Write,
+    indent: i32,
+}
+
+#[allow(unused_must_use)]
+impl<'a> Printer<'a> {
+    fn new(output: &'a mut fmt::Write) -> Self {
+        Printer { output, indent: 0 }
+    }
+
+    fn print_indent(&mut self) {
+        for _ in 0..self.indent {
+            write!(self.output, " ");
+        }
+    }
+}
+
+#[allow(unused_must_use)]
+impl<'a> Printer<'a> {
+    pub fn print(&mut self, tp: &Type) {
+        self.print_indent();
+        match tp {
+            &Type::PrimitiveType {
+                ref basic_info,
+                physical_type,
+                type_length,
+                scale,
+                precision,
+            } => {
+                let phys_type_str = match physical_type {
+                    PhysicalType::FIXED_LEN_BYTE_ARRAY => {
+                        // We need to include length for fixed byte array
+                        format!("{} ({})", physical_type, type_length)
+                    }
+                    _ => format!("{}", physical_type),
+                };
+                // Also print logical type if it is available
+                let logical_type_str = match basic_info.logical_type() {
+                    LogicalType::NONE => format!(""),
+                    decimal @ LogicalType::DECIMAL => {
+                        // For decimal type we should print precision and scale if they are > 0, e.g.
+                        // DECIMAL(9, 2) - DECIMAL(9) - DECIMAL
+                        let precision_scale = match (precision, scale) {
+                            (p, s) if p > 0 && s > 0 => format!(" ({}, {})", p, s),
+                            (p, 0) if p > 0 => format!(" ({})", p),
+                            _ => format!(""),
+                        };
+                        format!(" ({}{})", decimal, precision_scale)
+                    }
+                    other_logical_type => format!(" ({})", other_logical_type),
+                };
+                write!(
+                    self.output,
+                    "{} {} {}{};",
+                    basic_info.repetition(),
+                    phys_type_str,
+                    basic_info.name(),
+                    logical_type_str
+                );
+            }
+            &Type::GroupType {
+                ref basic_info,
+                ref fields,
+            } => {
+                if basic_info.has_repetition() {
+                    let r = basic_info.repetition();
+                    write!(self.output, "{} group {} ", r, basic_info.name());
+                    if basic_info.logical_type() != LogicalType::NONE {
+                        write!(self.output, "({}) ", basic_info.logical_type());
+                    }
+                    writeln!(self.output, "{{");
+                } else {
+                    writeln!(self.output, "message {} {{", basic_info.name());
+                }
+
+                self.indent += INDENT_WIDTH;
+                for c in fields {
+                    self.print(&c);
+                    writeln!(self.output, "");
+                }
+                self.indent -= INDENT_WIDTH;
+                self.print_indent();
+                write!(self.output, "}}");
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use std::rc::Rc;
+
+    use crate::parquet::basic::{Repetition, Type as PhysicalType};
+    use crate::parquet::schema::{parser::parse_message_type, types::Type};
+
+    fn assert_print_parse_message(message: Type) {
+        let mut s = String::new();
+        {
+            let mut p = Printer::new(&mut s);
+            p.print(&message);
+        }
+        let parsed = parse_message_type(&s).unwrap();
+        assert_eq!(message, parsed);
+    }
+
+    #[test]
+    fn test_print_primitive_type() {
+        let mut s = String::new();
+        {
+            let mut p = Printer::new(&mut s);
+            let foo = Type::primitive_type_builder("foo", PhysicalType::INT32)
+                .with_repetition(Repetition::REQUIRED)
+                .with_logical_type(LogicalType::INT_32)
+                .build()
+                .unwrap();
+            p.print(&foo);
+        }
+        assert_eq!(&mut s, "REQUIRED INT32 foo (INT_32);");
+    }
+
+    #[test]
+    fn test_print_primitive_type_without_logical() {
+        let mut s = String::new();
+        {
+            let mut p = Printer::new(&mut s);
+            let foo = Type::primitive_type_builder("foo", PhysicalType::DOUBLE)
+                .with_repetition(Repetition::REQUIRED)
+                .build()
+                .unwrap();
+            p.print(&foo);
+        }
+        assert_eq!(&mut s, "REQUIRED DOUBLE foo;");
+    }
+
+    #[test]
+    fn test_print_group_type() {
+        let mut s = String::new();
+        {
+            let mut p = Printer::new(&mut s);
+            let f1 = Type::primitive_type_builder("f1", PhysicalType::INT32)
+                .with_repetition(Repetition::REQUIRED)
+                .with_logical_type(LogicalType::INT_32)
+                .with_id(0)
+                .build();
+            let f2 = Type::primitive_type_builder("f2", PhysicalType::BYTE_ARRAY)
+                .with_logical_type(LogicalType::UTF8)
+                .with_id(1)
+                .build();
+            let f3 = Type::primitive_type_builder("f3", PhysicalType::FIXED_LEN_BYTE_ARRAY)
+                .with_repetition(Repetition::REPEATED)
+                .with_logical_type(LogicalType::INTERVAL)
+                .with_length(12)
+                .with_id(2)
+                .build();
+            let mut struct_fields = Vec::new();
+            struct_fields.push(Rc::new(f1.unwrap()));
+            struct_fields.push(Rc::new(f2.unwrap()));
+            let foo = Type::group_type_builder("foo")
+                .with_repetition(Repetition::OPTIONAL)
+                .with_fields(&mut struct_fields)
+                .with_id(1)
+                .build()
+                .unwrap();
+            let mut fields = Vec::new();
+            fields.push(Rc::new(foo));
+            fields.push(Rc::new(f3.unwrap()));
+            let message = Type::group_type_builder("schema")
+                .with_fields(&mut fields)
+                .with_id(2)
+                .build()
+                .unwrap();
+            p.print(&message);
+        }
+        let expected = "message schema {
+  OPTIONAL group foo {
+    REQUIRED INT32 f1 (INT_32);
+    OPTIONAL BYTE_ARRAY f2 (UTF8);
+  }
+  REPEATED FIXED_LEN_BYTE_ARRAY (12) f3 (INTERVAL);
+}";
+        assert_eq!(&mut s, expected);
+    }
+
+    #[test]
+    fn test_print_and_parse_primitive() {
+        let a2 = Type::primitive_type_builder("a2", PhysicalType::BYTE_ARRAY)
+            .with_repetition(Repetition::REPEATED)
+            .with_logical_type(LogicalType::UTF8)
+            .build()
+            .unwrap();
+
+        let a1 = Type::group_type_builder("a1")
+            .with_repetition(Repetition::OPTIONAL)
+            .with_logical_type(LogicalType::LIST)
+            .with_fields(&mut vec![Rc::new(a2)])
+            .build()
+            .unwrap();
+
+        let b3 = Type::primitive_type_builder("b3", PhysicalType::INT32)
+            .with_repetition(Repetition::OPTIONAL)
+            .build()
+            .unwrap();
+
+        let b4 = Type::primitive_type_builder("b4", PhysicalType::DOUBLE)
+            .with_repetition(Repetition::OPTIONAL)
+            .build()
+            .unwrap();
+
+        let b2 = Type::group_type_builder("b2")
+            .with_repetition(Repetition::REPEATED)
+            .with_logical_type(LogicalType::NONE)
+            .with_fields(&mut vec![Rc::new(b3), Rc::new(b4)])
+            .build()
+            .unwrap();
+
+        let b1 = Type::group_type_builder("b1")
+            .with_repetition(Repetition::OPTIONAL)
+            .with_logical_type(LogicalType::LIST)
+            .with_fields(&mut vec![Rc::new(b2)])
+            .build()
+            .unwrap();
+
+        let a0 = Type::group_type_builder("a0")
+            .with_repetition(Repetition::REQUIRED)
+            .with_fields(&mut vec![Rc::new(a1), Rc::new(b1)])
+            .build()
+            .unwrap();
+
+        let message = Type::group_type_builder("root")
+            .with_fields(&mut vec![Rc::new(a0)])
+            .build()
+            .unwrap();
+
+        assert_print_parse_message(message);
+    }
+
+    #[test]
+    fn test_print_and_parse_nested() {
+        let f1 = Type::primitive_type_builder("f1", PhysicalType::INT32)
+            .with_repetition(Repetition::REQUIRED)
+            .with_logical_type(LogicalType::INT_32)
+            .build()
+            .unwrap();
+
+        let f2 = Type::primitive_type_builder("f2", PhysicalType::BYTE_ARRAY)
+            .with_repetition(Repetition::OPTIONAL)
+            .with_logical_type(LogicalType::UTF8)
+            .build()
+            .unwrap();
+
+        let foo = Type::group_type_builder("foo")
+            .with_repetition(Repetition::OPTIONAL)
+            .with_fields(&mut vec![Rc::new(f1), Rc::new(f2)])
+            .build()
+            .unwrap();
+
+        let f3 = Type::primitive_type_builder("f3", PhysicalType::FIXED_LEN_BYTE_ARRAY)
+            .with_repetition(Repetition::REPEATED)
+            .with_logical_type(LogicalType::INTERVAL)
+            .with_length(12)
+            .build()
+            .unwrap();
+
+        let message = Type::group_type_builder("schema")
+            .with_fields(&mut vec![Rc::new(foo), Rc::new(f3)])
+            .build()
+            .unwrap();
+
+        assert_print_parse_message(message);
+    }
+
+    #[test]
+    fn test_print_and_parse_decimal() {
+        let f1 = Type::primitive_type_builder("f1", PhysicalType::INT32)
+            .with_repetition(Repetition::OPTIONAL)
+            .with_logical_type(LogicalType::DECIMAL)
+            .with_precision(9)
+            .with_scale(2)
+            .build()
+            .unwrap();
+
+        let f2 = Type::primitive_type_builder("f2", PhysicalType::INT32)
+            .with_repetition(Repetition::OPTIONAL)
+            .with_logical_type(LogicalType::DECIMAL)
+            .with_precision(9)
+            .with_scale(0)
+            .build()
+            .unwrap();
+
+        let message = Type::group_type_builder("schema")
+            .with_fields(&mut vec![Rc::new(f1), Rc::new(f2)])
+            .build()
+            .unwrap();
+
+        assert_print_parse_message(message);
+    }
+}

rust/src/parquet/util/hash_util.rs

@@ -0,0 +1,160 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::parquet::data_type::AsBytes;
+
+/// Computes hash value for `data`, with a seed value `seed`.
+/// The data type `T` must implement the `AsBytes` trait.
+pub fn hash<T: AsBytes>(data: &T, seed: u32) -> u32 {
+    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+    {
+        if is_x86_feature_detected!("sse4.2") {
+            unsafe { crc32_hash(data, seed) }
+        } else {
+            murmur_hash2_64a(data, seed as u64) as u32
+        }
+    }
+}
+
+const MURMUR_PRIME: u64 = 0xc6a4a7935bd1e995;
+const MURMUR_R: i32 = 47;
+
+/// Rust implementation of MurmurHash2, 64-bit version for 64-bit platforms
+fn murmur_hash2_64a<T: AsBytes>(data: &T, seed: u64) -> u64 {
+    let data_bytes = data.as_bytes();
+    let len = data_bytes.len();
+    let len_64 = (len / 8) * 8;
+    let data_bytes_64 = unsafe {
+        ::std::slice::from_raw_parts(&data_bytes[0..len_64] as *const [u8] as *const u64, len / 8)
+    };
+
+    let mut h = seed ^ (MURMUR_PRIME.wrapping_mul(data_bytes.len() as u64));
+    for v in data_bytes_64 {
+        let mut k = *v;
+        k = k.wrapping_mul(MURMUR_PRIME);
+        k ^= k >> MURMUR_R;
+        k = k.wrapping_mul(MURMUR_PRIME);
+        h ^= k;
+        h = h.wrapping_mul(MURMUR_PRIME);
+    }
+
+    let data2 = &data_bytes[len_64..];
+
+    let v = len & 7;
+    if v == 7 {
+        h ^= (data2[6] as u64) << 48;
+    }
+    if v >= 6 {
+        h ^= (data2[5] as u64) << 40;
+    }
+    if v >= 5 {
+        h ^= (data2[4] as u64) << 32;
+    }
+    if v >= 4 {
+        h ^= (data2[3] as u64) << 24;
+    }
+    if v >= 3 {
+        h ^= (data2[2] as u64) << 16;
+    }
+    if v >= 2 {
+        h ^= (data2[1] as u64) << 8;
+    }
+    if v >= 1 {
+        h ^= data2[0] as u64;
+    }
+    if v > 0 {
+        h = h.wrapping_mul(MURMUR_PRIME);
+    }
+
+    h ^= h >> MURMUR_R;
+    h = h.wrapping_mul(MURMUR_PRIME);
+    h ^= h >> MURMUR_R;
+    h
+}
+
+/// CRC32 hash implementation using SSE4 instructions. Borrowed from Impala.
+#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+#[target_feature(enable = "sse4.2")]
+unsafe fn crc32_hash<T: AsBytes>(data: &T, seed: u32) -> u32 {
+    #[cfg(target_arch = "x86")]
+    use std::arch::x86::*;
+    #[cfg(target_arch = "x86_64")]
+    use std::arch::x86_64::*;
+
+    let bytes: &[u8] = data.as_bytes();
+    let u32_num_bytes = ::std::mem::size_of::<u32>();
+    let mut num_bytes = bytes.len();
+    let num_words = num_bytes / u32_num_bytes;
+    num_bytes %= u32_num_bytes;
+
+    let bytes_u32: &[u32] = ::std::slice::from_raw_parts(
+        &bytes[0..num_words * u32_num_bytes] as *const [u8] as *const u32,
+        num_words,
+    );
+
+    let mut offset = 0;
+    let mut hash = seed;
+    while offset < num_words {
+        hash = _mm_crc32_u32(hash, bytes_u32[offset]);
+        offset += 1;
+    }
+
+    offset = num_words * u32_num_bytes;
+    while offset < num_bytes {
+        hash = _mm_crc32_u8(hash, bytes[offset]);
+        offset += 1;
+    }
+
+    // The lower half of the CRC hash has poor uniformity, so swap the halves
+    // for anyone who only uses the first several bits of the hash.
+    hash = (hash << 16) | (hash >> 16);
+    hash
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_murmur2_64a() {
+        let result = murmur_hash2_64a(&"hello", 123);
+        assert_eq!(result, 2597646618390559622);
+
+        let result = murmur_hash2_64a(&"helloworld", 123);
+        assert_eq!(result, 4934371746140206573);
+
+        let result = murmur_hash2_64a(&"helloworldparquet", 123);
+        assert_eq!(result, 2392198230801491746);
+    }
+
+    #[test]
+    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+    fn test_crc32() {
+        if is_x86_feature_detected!("sse4.2") {
+            unsafe {
+                let result = crc32_hash(&"hello", 123);
+                assert_eq!(result, 2927487359);
+
+                let result = crc32_hash(&"helloworld", 123);
+                assert_eq!(result, 314229527);
+
+                let result = crc32_hash(&"helloworldparquet", 123);
+                assert_eq!(result, 667078870);
+            }
+        }
+    }
+}

rust/src/parquet/util/io.rs

@@ -0,0 +1,220 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::{cmp, fs::File, io::*, sync::Mutex};
+
+use crate::parquet::file::reader::ParquetReader;
+
+// ----------------------------------------------------------------------
+// Read/Write wrappers for `File`.
+
+/// Position trait returns the current position in the stream.
+/// Should be viewed as a lighter version of `Seek` that does not allow seek operations,
+/// and does not require mutable reference for the current position.
+pub trait Position {
+    /// Returns position in the stream.
+    fn pos(&self) -> u64;
+}
+
+/// Struct that represents a slice of a file data with independent start position and
+/// length. Internally clones provided file handle, wraps with BufReader and resets
+/// position before any read.
+///
+/// This is workaround and alternative for `file.try_clone()` method. It clones `File`
+/// while preserving independent position, which is not available with `try_clone()`.
+///
+/// Designed after `arrow::io::RandomAccessFile`.
+pub struct FileSource<R: ParquetReader> {
+    reader: Mutex<BufReader<R>>,
+    start: u64, // start position in a file
+    end: u64,   // end position in a file
+}
+
+impl<R: ParquetReader> FileSource<R> {
+    /// Creates new file reader with start and length from a file handle
+    pub fn new(fd: &R, start: u64, length: usize) -> Self {
+        Self {
+            reader: Mutex::new(BufReader::new(fd.try_clone().unwrap())),
+            start,
+            end: start + length as u64,
+        }
+    }
+}
+
+impl<R: ParquetReader> Read for FileSource<R> {
+    fn read(&mut self, buf: &mut [u8]) -> Result<usize> {
+        let mut reader = self
+            .reader
+            .lock()
+            .map_err(|err| Error::new(ErrorKind::Other, err.to_string()))?;
+
+        let bytes_to_read = cmp::min(buf.len(), (self.end - self.start) as usize);
+        let buf = &mut buf[0..bytes_to_read];
+
+        reader.seek(SeekFrom::Start(self.start as u64))?;
+        let res = reader.read(buf);
+        if let Ok(bytes_read) = res {
+            self.start += bytes_read as u64;
+        }
+
+        res
+    }
+}
+
+impl<R: ParquetReader> Position for FileSource<R> {
+    fn pos(&self) -> u64 {
+        self.start
+    }
+}
+
+/// Struct that represents `File` output stream with position tracking.
+/// Used as a sink in file writer.
+pub struct FileSink {
+    buf: BufWriter<File>,
+    // This is not necessarily position in the underlying file,
+    // but rather current position in the sink.
+    pos: u64,
+}
+
+impl FileSink {
+    /// Creates new file sink.
+    /// Position is set to whatever position file has.
+    pub fn new(file: &File) -> Self {
+        let mut owned_file = file.try_clone().unwrap();
+        let pos = owned_file.seek(SeekFrom::Current(0)).unwrap();
+        Self {
+            buf: BufWriter::new(owned_file),
+            pos,
+        }
+    }
+}
+
+impl Write for FileSink {
+    fn write(&mut self, buf: &[u8]) -> Result<usize> {
+        let num_bytes = self.buf.write(buf)?;
+        self.pos += num_bytes as u64;
+        Ok(num_bytes)
+    }
+
+    fn flush(&mut self) -> Result<()> {
+        self.buf.flush()
+    }
+}
+
+impl Position for FileSink {
+    fn pos(&self) -> u64 {
+        self.pos
+    }
+}
+
+// Position implementation for Cursor to use in various tests.
+impl<'a> Position for Cursor<&'a mut Vec<u8>> {
+    fn pos(&self) -> u64 {
+        self.position()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use crate::parquet::util::test_common::{get_temp_file, get_test_file};
+
+    #[test]
+    fn test_io_read_fully() {
+        let mut buf = vec![0; 8];
+        let mut src = FileSource::new(&get_test_file("alltypes_plain.parquet"), 0, 4);
+
+        let bytes_read = src.read(&mut buf[..]).unwrap();
+        assert_eq!(bytes_read, 4);
+        assert_eq!(buf, vec![b'P', b'A', b'R', b'1', 0, 0, 0, 0]);
+    }
+
+    #[test]
+    fn test_io_read_in_chunks() {
+        let mut buf = vec![0; 4];
+        let mut src = FileSource::new(&get_test_file("alltypes_plain.parquet"), 0, 4);
+
+        let bytes_read = src.read(&mut buf[0..2]).unwrap();
+        assert_eq!(bytes_read, 2);
+        let bytes_read = src.read(&mut buf[2..]).unwrap();
+        assert_eq!(bytes_read, 2);
+        assert_eq!(buf, vec![b'P', b'A', b'R', b'1']);
+    }
+
+    #[test]
+    fn test_io_read_pos() {
+        let mut src = FileSource::new(&get_test_file("alltypes_plain.parquet"), 0, 4);
+
+        src.read(&mut vec![0; 1]).unwrap();
+        assert_eq!(src.pos(), 1);
+
+        src.read(&mut vec![0; 4]).unwrap();
+        assert_eq!(src.pos(), 4);
+    }
+
+    #[test]
+    fn test_io_read_over_limit() {
+        let mut src = FileSource::new(&get_test_file("alltypes_plain.parquet"), 0, 4);
+
+        // Read all bytes from source
+        src.read(&mut vec![0; 128]).unwrap();
+        assert_eq!(src.pos(), 4);
+
+        // Try reading again, should return 0 bytes.
+        let bytes_read = src.read(&mut vec![0; 128]).unwrap();
+        assert_eq!(bytes_read, 0);
+        assert_eq!(src.pos(), 4);
+    }
+
+    #[test]
+    fn test_io_seek_switch() {
+        let mut buf = vec![0; 4];
+        let mut file = get_test_file("alltypes_plain.parquet");
+        let mut src = FileSource::new(&file, 0, 4);
+
+        file.seek(SeekFrom::Start(5 as u64))
+            .expect("File seek to a position");
+
+        let bytes_read = src.read(&mut buf[..]).unwrap();
+        assert_eq!(bytes_read, 4);
+        assert_eq!(buf, vec![b'P', b'A', b'R', b'1']);
+    }
+
+    #[test]
+    fn test_io_write_with_pos() {
+        let mut file = get_temp_file("file_sink_test", &[b'a', b'b', b'c']);
+        file.seek(SeekFrom::Current(3)).unwrap();
+
+        // Write into sink
+        let mut sink = FileSink::new(&file);
+        assert_eq!(sink.pos(), 3);
+
+        sink.write(&[b'd', b'e', b'f', b'g']).unwrap();
+        assert_eq!(sink.pos(), 7);
+
+        sink.flush().unwrap();
+        assert_eq!(sink.pos(), file.seek(SeekFrom::Current(0)).unwrap());
+
+        // Read data using file chunk
+        let mut res = vec![0u8; 7];
+        let mut chunk = FileSource::new(&file, 0, file.metadata().unwrap().len() as usize);
+        chunk.read(&mut res[..]).unwrap();
+
+        assert_eq!(res, vec![b'a', b'b', b'c', b'd', b'e', b'f', b'g']);
+    }
+}

rust/src/parquet/util/memory.rs

@@ -0,0 +1,524 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Utility methods and structs for working with memory.
+
+use std::{
+    cell::Cell,
+    fmt::{Debug, Display, Formatter, Result as FmtResult},
+    io::{Result as IoResult, Write},
+    mem,
+    ops::{Index, IndexMut},
+    rc::{Rc, Weak},
+};
+
+// ----------------------------------------------------------------------
+// Memory Tracker classes
+
+/// Reference counted pointer for [`MemTracker`].
+pub type MemTrackerPtr = Rc<MemTracker>;
+/// Non-owning reference for [`MemTracker`].
+pub type WeakMemTrackerPtr = Weak<MemTracker>;
+
+/// Struct to track memory usage information.
+#[derive(Debug)]
+pub struct MemTracker {
+    // In the tuple, the first element is the current memory allocated (in bytes),
+    // and the second element is the maximum memory allocated so far (in bytes).
+    memory_usage: Cell<(i64, i64)>,
+}
+
+impl MemTracker {
+    /// Creates new memory tracker.
+    #[inline]
+    pub fn new() -> MemTracker {
+        MemTracker {
+            memory_usage: Cell::new((0, 0)),
+        }
+    }
+
+    /// Returns the current memory consumption, in bytes.
+    pub fn memory_usage(&self) -> i64 {
+        self.memory_usage.get().0
+    }
+
+    /// Returns the maximum memory consumption so far, in bytes.
+    pub fn max_memory_usage(&self) -> i64 {
+        self.memory_usage.get().1
+    }
+
+    /// Adds `num_bytes` to the memory consumption tracked by this memory tracker.
+    #[inline]
+    pub fn alloc(&self, num_bytes: i64) {
+        let (current, mut maximum) = self.memory_usage.get();
+        let new_current = current + num_bytes;
+        if new_current > maximum {
+            maximum = new_current
+        }
+        self.memory_usage.set((new_current, maximum));
+    }
+}
+
+// ----------------------------------------------------------------------
+// Buffer classes
+
+/// Type alias for [`Buffer`].
+pub type ByteBuffer = Buffer<u8>;
+/// Type alias for [`BufferPtr`].
+pub type ByteBufferPtr = BufferPtr<u8>;
+
+/// A resize-able buffer class with generic member, with optional memory tracker.
+///
+/// Note that a buffer has two attributes:
+/// `capacity` and `size`: the former is the total number of space reserved for
+/// the buffer, while the latter is the actual number of elements.
+/// Invariant: `capacity` >= `size`.
+/// The total allocated bytes for a buffer equals to `capacity * sizeof<T>()`.
+pub struct Buffer<T: Clone> {
+    data: Vec<T>,
+    mem_tracker: Option<MemTrackerPtr>,
+    type_length: usize,
+}
+
+impl<T: Clone> Buffer<T> {
+    /// Creates new empty buffer.
+    pub fn new() -> Self {
+        Buffer {
+            data: vec![],
+            mem_tracker: None,
+            type_length: ::std::mem::size_of::<T>(),
+        }
+    }
+
+    /// Adds [`MemTracker`] for this buffer.
+    #[inline]
+    pub fn with_mem_tracker(mut self, mc: MemTrackerPtr) -> Self {
+        mc.alloc((self.data.capacity() * self.type_length) as i64);
+        self.mem_tracker = Some(mc);
+        self
+    }
+
+    /// Returns slice of data in this buffer.
+    #[inline]
+    pub fn data(&self) -> &[T] {
+        self.data.as_slice()
+    }
+
+    /// Sets data for this buffer.
+    #[inline]
+    pub fn set_data(&mut self, new_data: Vec<T>) {
+        if let Some(ref mc) = self.mem_tracker {
+            let capacity_diff = new_data.capacity() as i64 - self.data.capacity() as i64;
+            mc.alloc(capacity_diff * self.type_length as i64);
+        }
+        self.data = new_data;
+    }
+
+    /// Resizes underlying data in place to a new length `new_size`.
+    ///
+    /// If `new_size` is less than current length, data is truncated, otherwise, it is
+    /// extended to `new_size` with provided default value `init_value`.
+    ///
+    /// Memory tracker is also updated, if available.
+    #[inline]
+    pub fn resize(&mut self, new_size: usize, init_value: T) {
+        let old_capacity = self.data.capacity();
+        self.data.resize(new_size, init_value);
+        if let Some(ref mc) = self.mem_tracker {
+            let capacity_diff = self.data.capacity() as i64 - old_capacity as i64;
+            mc.alloc(capacity_diff * self.type_length as i64);
+        }
+    }
+
+    /// Clears underlying data.
+    #[inline]
+    pub fn clear(&mut self) {
+        self.data.clear()
+    }
+
+    /// Reserves capacity `additional_capacity` for underlying data vector.
+    ///
+    /// Memory tracker is also updated, if available.
+    #[inline]
+    pub fn reserve(&mut self, additional_capacity: usize) {
+        let old_capacity = self.data.capacity();
+        self.data.reserve(additional_capacity);
+        if self.data.capacity() > old_capacity {
+            if let Some(ref mc) = self.mem_tracker {
+                let capacity_diff = self.data.capacity() as i64 - old_capacity as i64;
+                mc.alloc(capacity_diff * self.type_length as i64);
+            }
+        }
+    }
+
+    /// Returns [`BufferPtr`] with buffer data.
+    /// Buffer data is reset.
+    #[inline]
+    pub fn consume(&mut self) -> BufferPtr<T> {
+        let old_data = mem::replace(&mut self.data, vec![]);
+        let mut result = BufferPtr::new(old_data);
+        if let Some(ref mc) = self.mem_tracker {
+            result = result.with_mem_tracker(mc.clone());
+        }
+        result
+    }
+
+    /// Adds `value` to the buffer.
+    #[inline]
+    pub fn push(&mut self, value: T) {
+        self.data.push(value)
+    }
+
+    /// Returns current capacity for the buffer.
+    #[inline]
+    pub fn capacity(&self) -> usize {
+        self.data.capacity()
+    }
+
+    /// Returns current size for the buffer.
+    #[inline]
+    pub fn size(&self) -> usize {
+        self.data.len()
+    }
+
+    /// Returns `true` if memory tracker is added to buffer, `false` otherwise.
+    #[inline]
+    pub fn is_mem_tracked(&self) -> bool {
+        self.mem_tracker.is_some()
+    }
+
+    /// Returns memory tracker associated with this buffer.
+    /// This may panic, if memory tracker is not set, use method above to check if
+    /// memory tracker is available.
+    #[inline]
+    pub fn mem_tracker(&self) -> &MemTrackerPtr {
+        self.mem_tracker.as_ref().unwrap()
+    }
+}
+
+impl<T: Sized + Clone> Index<usize> for Buffer<T> {
+    type Output = T;
+
+    fn index(&self, index: usize) -> &T {
+        &self.data[index]
+    }
+}
+
+impl<T: Sized + Clone> IndexMut<usize> for Buffer<T> {
+    fn index_mut(&mut self, index: usize) -> &mut T {
+        &mut self.data[index]
+    }
+}
+
+// TODO: implement this for other types
+impl Write for Buffer<u8> {
+    #[inline]
+    fn write(&mut self, buf: &[u8]) -> IoResult<usize> {
+        let old_capacity = self.data.capacity();
+        let bytes_written = self.data.write(buf)?;
+        if let Some(ref mc) = self.mem_tracker {
+            if self.data.capacity() - old_capacity > 0 {
+                mc.alloc((self.data.capacity() - old_capacity) as i64)
+            }
+        }
+        Ok(bytes_written)
+    }
+
+    fn flush(&mut self) -> IoResult<()> {
+        // No-op
+        self.data.flush()
+    }
+}
+
+impl AsRef<[u8]> for Buffer<u8> {
+    fn as_ref(&self) -> &[u8] {
+        self.data.as_slice()
+    }
+}
+
+impl<T: Clone> Drop for Buffer<T> {
+    #[inline]
+    fn drop(&mut self) {
+        if let Some(ref mc) = self.mem_tracker {
+            mc.alloc(-((self.data.capacity() * self.type_length) as i64));
+        }
+    }
+}
+
+// ----------------------------------------------------------------------
+// Immutable Buffer (BufferPtr) classes
+
+/// An representation of a slice on a reference-counting and read-only byte array.
+/// Sub-slices can be further created from this. The byte array will be released
+/// when all slices are dropped.
+#[derive(Clone, Debug)]
+pub struct BufferPtr<T> {
+    data: Rc<Vec<T>>,
+    start: usize,
+    len: usize,
+    // TODO: will this create too many references? rethink about this.
+    mem_tracker: Option<MemTrackerPtr>,
+}
+
+impl<T> BufferPtr<T> {
+    /// Creates new buffer from a vector.
+    pub fn new(v: Vec<T>) -> Self {
+        let len = v.len();
+        Self {
+            data: Rc::new(v),
+            start: 0,
+            len,
+            mem_tracker: None,
+        }
+    }
+
+    /// Returns slice of data in this buffer.
+    pub fn data(&self) -> &[T] {
+        &self.data[self.start..self.start + self.len]
+    }
+
+    /// Updates this buffer with new `start` position and length `len`.
+    ///
+    /// Range should be within current start position and length.
+    pub fn with_range(mut self, start: usize, len: usize) -> Self {
+        assert!(start <= self.len);
+        assert!(start + len <= self.len);
+        self.start = start;
+        self.len = len;
+        self
+    }
+
+    /// Adds memory tracker to this buffer.
+    pub fn with_mem_tracker(mut self, mc: MemTrackerPtr) -> Self {
+        self.mem_tracker = Some(mc);
+        self
+    }
+
+    /// Returns start position of this buffer.
+    pub fn start(&self) -> usize {
+        self.start
+    }
+
+    /// Returns length of this buffer
+    pub fn len(&self) -> usize {
+        self.len
+    }
+
+    /// Returns `true` if this buffer has memory tracker, `false` otherwise.
+    pub fn is_mem_tracked(&self) -> bool {
+        self.mem_tracker.is_some()
+    }
+
+    /// Returns a shallow copy of the buffer.
+    /// Reference counted pointer to the data is copied.
+    pub fn all(&self) -> BufferPtr<T> {
+        BufferPtr {
+            data: self.data.clone(),
+            start: self.start,
+            len: self.len,
+            mem_tracker: self.mem_tracker.as_ref().map(|p| p.clone()),
+        }
+    }
+
+    /// Returns a shallow copy of the buffer that starts with `start` position.
+    pub fn start_from(&self, start: usize) -> BufferPtr<T> {
+        assert!(start <= self.len);
+        BufferPtr {
+            data: self.data.clone(),
+            start: self.start + start,
+            len: self.len - start,
+            mem_tracker: self.mem_tracker.as_ref().map(|p| p.clone()),
+        }
+    }
+
+    /// Returns a shallow copy that is a range slice within this buffer.
+    pub fn range(&self, start: usize, len: usize) -> BufferPtr<T> {
+        assert!(start + len <= self.len);
+        BufferPtr {
+            data: self.data.clone(),
+            start: self.start + start,
+            len,
+            mem_tracker: self.mem_tracker.as_ref().map(|p| p.clone()),
+        }
+    }
+}
+
+impl<T: Sized> Index<usize> for BufferPtr<T> {
+    type Output = T;
+
+    fn index(&self, index: usize) -> &T {
+        assert!(index < self.len);
+        &self.data[self.start + index]
+    }
+}
+
+impl<T: Debug> Display for BufferPtr<T> {
+    fn fmt(&self, f: &mut Formatter) -> FmtResult {
+        write!(f, "{:?}", self.data)
+    }
+}
+
+impl<T> Drop for BufferPtr<T> {
+    fn drop(&mut self) {
+        if self.is_mem_tracked()
+            && Rc::strong_count(&self.data) == 1
+            && Rc::weak_count(&self.data) == 0
+        {
+            let mc = self.mem_tracker.as_ref().unwrap();
+            mc.alloc(-(self.data.capacity() as i64));
+        }
+    }
+}
+
+impl AsRef<[u8]> for BufferPtr<u8> {
+    fn as_ref(&self) -> &[u8] {
+        &self.data[self.start..self.start + self.len]
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_byte_buffer_mem_tracker() {
+        let mem_tracker = Rc::new(MemTracker::new());
+
+        let mut buffer = ByteBuffer::new().with_mem_tracker(mem_tracker.clone());
+        buffer.set_data(vec![0; 10]);
+        assert_eq!(mem_tracker.memory_usage(), buffer.capacity() as i64);
+        buffer.set_data(vec![0; 20]);
+        let capacity = buffer.capacity() as i64;
+        assert_eq!(mem_tracker.memory_usage(), capacity);
+
+        let max_capacity = {
+            let mut buffer2 = ByteBuffer::new().with_mem_tracker(mem_tracker.clone());
+            buffer2.reserve(30);
+            assert_eq!(
+                mem_tracker.memory_usage(),
+                buffer2.capacity() as i64 + capacity
+            );
+            buffer2.set_data(vec![0; 100]);
+            assert_eq!(
+                mem_tracker.memory_usage(),
+                buffer2.capacity() as i64 + capacity
+            );
+            buffer2.capacity() as i64 + capacity
+        };
+
+        assert_eq!(mem_tracker.memory_usage(), capacity);
+        assert_eq!(mem_tracker.max_memory_usage(), max_capacity);
+
+        buffer.reserve(40);
+        assert_eq!(mem_tracker.memory_usage(), buffer.capacity() as i64);
+
+        buffer.consume();
+        assert_eq!(mem_tracker.memory_usage(), buffer.capacity() as i64);
+    }
+
+    #[test]
+    fn test_byte_ptr_mem_tracker() {
+        let mem_tracker = Rc::new(MemTracker::new());
+
+        let mut buffer = ByteBuffer::new().with_mem_tracker(mem_tracker.clone());
+        buffer.set_data(vec![0; 60]);
+
+        {
+            let buffer_capacity = buffer.capacity() as i64;
+            let buf_ptr = buffer.consume();
+            assert_eq!(mem_tracker.memory_usage(), buffer_capacity);
+            {
+                let buf_ptr1 = buf_ptr.all();
+                {
+                    let _ = buf_ptr.start_from(20);
+                    assert_eq!(mem_tracker.memory_usage(), buffer_capacity);
+                }
+                assert_eq!(mem_tracker.memory_usage(), buffer_capacity);
+                let _ = buf_ptr1.range(30, 20);
+                assert_eq!(mem_tracker.memory_usage(), buffer_capacity);
+            }
+            assert_eq!(mem_tracker.memory_usage(), buffer_capacity);
+        }
+        assert_eq!(mem_tracker.memory_usage(), buffer.capacity() as i64);
+    }
+
+    #[test]
+    fn test_byte_buffer() {
+        let mut buffer = ByteBuffer::new();
+        assert_eq!(buffer.size(), 0);
+        assert_eq!(buffer.capacity(), 0);
+
+        let mut buffer2 = ByteBuffer::new();
+        buffer2.reserve(40);
+        assert_eq!(buffer2.size(), 0);
+        assert_eq!(buffer2.capacity(), 40);
+
+        buffer.set_data((0..5).collect());
+        assert_eq!(buffer.size(), 5);
+        assert_eq!(buffer[4], 4);
+
+        buffer.set_data((0..20).collect());
+        assert_eq!(buffer.size(), 20);
+        assert_eq!(buffer[10], 10);
+
+        let expected: Vec<u8> = (0..20).collect();
+        {
+            let data = buffer.data();
+            assert_eq!(data, expected.as_slice());
+        }
+
+        buffer.reserve(40);
+        assert!(buffer.capacity() >= 40);
+
+        let byte_ptr = buffer.consume();
+        assert_eq!(buffer.size(), 0);
+        assert_eq!(byte_ptr.as_ref(), expected.as_slice());
+
+        let values: Vec<u8> = (0..30).collect();
+        let _ = buffer.write(values.as_slice());
+        let _ = buffer.flush();
+
+        assert_eq!(buffer.data(), values.as_slice());
+    }
+
+    #[test]
+    fn test_byte_ptr() {
+        let values = (0..50).collect();
+        let ptr = ByteBufferPtr::new(values);
+        assert_eq!(ptr.len(), 50);
+        assert_eq!(ptr.start(), 0);
+        assert_eq!(ptr[40], 40);
+
+        let ptr2 = ptr.all();
+        assert_eq!(ptr2.len(), 50);
+        assert_eq!(ptr2.start(), 0);
+        assert_eq!(ptr2[40], 40);
+
+        let ptr3 = ptr.start_from(20);
+        assert_eq!(ptr3.len(), 30);
+        assert_eq!(ptr3.start(), 20);
+        assert_eq!(ptr3[0], 20);
+
+        let ptr4 = ptr3.range(10, 10);
+        assert_eq!(ptr4.len(), 10);
+        assert_eq!(ptr4.start(), 30);
+        assert_eq!(ptr4[0], 30);
+
+        let expected: Vec<u8> = (30..40).collect();
+        assert_eq!(ptr4.as_ref(), expected.as_slice());
+    }
+}

rust/src/parquet/util/mod.rs

@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+pub mod io;
+pub mod memory;
+#[macro_use]
+pub mod bit_util;
+mod bit_packing;
+pub mod hash_util;
+
+#[cfg(test)]
+pub mod test_common;

rust/src/parquet/util/test_common.rs

@@ -0,0 +1,190 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use rand::{
+    distributions::{range::SampleRange, Distribution, Standard},
+    thread_rng, Rng,
+};
+use std::{env, fs, io::Write, path::PathBuf, str::FromStr};
+
+use crate::parquet::data_type::*;
+use crate::parquet::util::memory::ByteBufferPtr;
+
+/// Random generator of data type `T` values and sequences.
+pub trait RandGen<T: DataType> {
+    fn gen(len: i32) -> T::T;
+
+    fn gen_vec(len: i32, total: usize) -> Vec<T::T> {
+        let mut result = vec![];
+        for _ in 0..total {
+            result.push(Self::gen(len))
+        }
+        result
+    }
+}
+
+impl<T: DataType> RandGen<T> for T {
+    default fn gen(_: i32) -> T::T {
+        panic!("Unsupported data type");
+    }
+}
+
+impl RandGen<BoolType> for BoolType {
+    fn gen(_: i32) -> bool {
+        thread_rng().gen::<bool>()
+    }
+}
+
+impl RandGen<Int32Type> for Int32Type {
+    fn gen(_: i32) -> i32 {
+        thread_rng().gen::<i32>()
+    }
+}
+
+impl RandGen<Int64Type> for Int64Type {
+    fn gen(_: i32) -> i64 {
+        thread_rng().gen::<i64>()
+    }
+}
+
+impl RandGen<Int96Type> for Int96Type {
+    fn gen(_: i32) -> Int96 {
+        let mut rng = thread_rng();
+        let mut result = Int96::new();
+        result.set_data(rng.gen::<u32>(), rng.gen::<u32>(), rng.gen::<u32>());
+        result
+    }
+}
+
+impl RandGen<FloatType> for FloatType {
+    fn gen(_: i32) -> f32 {
+        thread_rng().gen::<f32>()
+    }
+}
+
+impl RandGen<DoubleType> for DoubleType {
+    fn gen(_: i32) -> f64 {
+        thread_rng().gen::<f64>()
+    }
+}
+
+impl RandGen<ByteArrayType> for ByteArrayType {
+    fn gen(_: i32) -> ByteArray {
+        let mut rng = thread_rng();
+        let mut result = ByteArray::new();
+        let mut value = vec![];
+        let len = rng.gen_range::<usize>(0, 128);
+        for _ in 0..len {
+            value.push(rng.gen_range(0, 255) & 0xFF);
+        }
+        result.set_data(ByteBufferPtr::new(value));
+        result
+    }
+}
+
+impl RandGen<FixedLenByteArrayType> for FixedLenByteArrayType {
+    fn gen(len: i32) -> ByteArray {
+        let mut rng = thread_rng();
+        let value_len = if len < 0 {
+            rng.gen_range::<usize>(0, 128)
+        } else {
+            len as usize
+        };
+        let value = random_bytes(value_len);
+        ByteArray::from(value)
+    }
+}
+
+pub fn random_bytes(n: usize) -> Vec<u8> {
+    let mut result = vec![];
+    let mut rng = thread_rng();
+    for _ in 0..n {
+        result.push(rng.gen_range(0, 255) & 0xFF);
+    }
+    result
+}
+
+pub fn random_bools(n: usize) -> Vec<bool> {
+    let mut result = vec![];
+    let mut rng = thread_rng();
+    for _ in 0..n {
+        result.push(rng.gen::<bool>());
+    }
+    result
+}
+
+pub fn random_numbers<T>(n: usize) -> Vec<T>
+where
+    Standard: Distribution<T>,
+{
+    let mut rng = thread_rng();
+    Standard.sample_iter(&mut rng).take(n).collect()
+}
+
+pub fn random_numbers_range<T>(n: usize, low: T, high: T, result: &mut Vec<T>)
+where
+    T: PartialOrd + SampleRange + Copy,
+{
+    let mut rng = thread_rng();
+    for _ in 0..n {
+        result.push(rng.gen_range(low, high));
+    }
+}
+
+/// Returns path to the test parquet file in 'data' directory
+pub fn get_test_path(file_name: &str) -> PathBuf {
+    let result = env::var("PARQUET_TEST_DATA");
+    if result.is_err() {
+        panic!("Please point PARQUET_TEST_DATA environment variable to the test data directory");
+    }
+    let mut pathbuf = PathBuf::from_str(result.unwrap().as_str()).unwrap();
+    pathbuf.push(file_name);
+    pathbuf
+}
+
+/// Returns file handle for a test parquet file from 'data' directory
+pub fn get_test_file(file_name: &str) -> fs::File {
+    let file = fs::File::open(get_test_path(file_name).as_path());
+    if file.is_err() {
+        panic!("Test file {} not found", file_name)
+    }
+    file.unwrap()
+}
+
+/// Returns file handle for a temp file in 'target' directory with a provided content
+pub fn get_temp_file(file_name: &str, content: &[u8]) -> fs::File {
+    // build tmp path to a file in "target/debug/testdata"
+    let mut path_buf = env::current_dir().unwrap();
+    path_buf.push("target");
+    path_buf.push("debug");
+    path_buf.push("testdata");
+    fs::create_dir_all(&path_buf).unwrap();
+    path_buf.push(file_name);
+
+    // write file content
+    let mut tmp_file = fs::File::create(path_buf.as_path()).unwrap();
+    tmp_file.write_all(content).unwrap();
+    tmp_file.sync_all().unwrap();
+
+    // return file handle for both read and write
+    let file = fs::OpenOptions::new()
+        .read(true)
+        .write(true)
+        .open(path_buf.as_path());
+    assert!(file.is_ok());
+    file.unwrap()
+}

rust/src/record_batch.rs

@@ -15,9 +15,10 @@
 // specific language governing permissions and limitations
 // under the License.
 
+use std::sync::Arc;
+
 use crate::array::*;
 use crate::datatypes::*;
-use std::sync::Arc;
 
 /// A batch of column-oriented data
 pub struct RecordBatch {
@@ -67,6 +68,7 @@ unsafe impl Sync for RecordBatch {}
 #[cfg(test)]
 mod tests {
     use super::*;
+
     use crate::array_data::*;
     use crate::buffer::*;
 

rust/src/tensor.rs

@@ -216,6 +216,7 @@ impl<'a, T: ArrowPrimitiveType> Tensor<'a, T> {
 #[cfg(test)]
 mod tests {
     use super::*;
+
     use crate::buffer::Buffer;
     use crate::builder::*;