Merge remote-tracking branch 'upstream/master' into parquet-column-selection

appveyor.yml

@@ -40,6 +40,10 @@ environment:
       PYTHON: "3.5"
       ARCH: "64"
       CONFIGURATION: "Release"
+    - JOB: "Static_Crt_Build"
+      GENERATOR: Visual Studio 14 2015 Win64
+      PYTHON: "3.5"
+      ARCH: "64"
     - JOB: "Toolchain"
       GENERATOR: Visual Studio 14 2015 Win64
       PYTHON: "3.5"

c_glib/arrow-glib/Makefile.am

@@ -45,10 +45,12 @@ libarrow_glib_la_headers =			\
 	array.h					\
 	array-builder.h				\
 	arrow-glib.h				\
+	basic-array.h				\
 	basic-data-type.h			\
 	buffer.h				\
 	chunked-array.h				\
 	column.h				\
+	composite-array.h			\
 	composite-data-type.h			\
 	data-type.h				\
 	error.h					\
@@ -86,12 +88,13 @@ libarrow_glib_la_generated_sources =		\
 	$(libarrow_glib_la_generated_headers)
 
 libarrow_glib_la_sources =			\
-	array.cpp				\
 	array-builder.cpp			\
+	basic-array.cpp				\
 	basic-data-type.cpp			\
 	buffer.cpp				\
 	chunked-array.cpp			\
 	column.cpp				\
+	composite-array.cpp			\
 	composite-data-type.cpp			\
 	error.cpp				\
 	field.cpp				\
@@ -125,6 +128,7 @@ libarrow_glib_la_cpp_headers =			\
 	array.hpp				\
 	array-builder.hpp			\
 	arrow-glib.hpp				\
+	basic-array.hpp				\
 	basic-data-type.hpp			\
 	buffer.hpp				\
 	chunked-array.hpp			\
@@ -207,6 +211,11 @@ if HAVE_INTROSPECTION
 -include $(INTROSPECTION_MAKEFILE)
 INTROSPECTION_GIRS =
 INTROSPECTION_SCANNER_ARGS =
+INTROSPECTION_SCANNER_ENV =
+if OS_MACOS
+INTROSPECTION_SCANNER_ENV +=			\
+	ARCHFLAGS=
+endif
 INTROSPECTION_COMPILER_ARGS =
 
 Arrow-1.0.gir: libarrow-glib.la

c_glib/arrow-glib/array.hpp

@@ -19,9 +19,5 @@
 
 #pragma once
 
-#include <arrow/api.h>
-
-#include <arrow-glib/array.h>
-
-GArrowArray *garrow_array_new_raw(std::shared_ptr<arrow::Array> *arrow_array);
-std::shared_ptr<arrow::Array> garrow_array_get_raw(GArrowArray *array);
+#include <arrow-glib/basic-array.hpp>
+#include <arrow-glib/composite-array.h>

c_glib/arrow-glib/basic-array.cpp

@@ -24,7 +24,7 @@
 #include <arrow-glib/array.hpp>
 #include <arrow-glib/buffer.hpp>
 #include <arrow-glib/compute.hpp>
-#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/basic-data-type.hpp>
 #include <arrow-glib/error.hpp>
 #include <arrow-glib/type.hpp>
 
@@ -86,9 +86,9 @@ garrow_primitive_array_new(GArrowDataType *data_type,
 G_BEGIN_DECLS
 
 /**
- * SECTION: array
- * @section_id: array-classes
- * @title: Array classes
+ * SECTION: basic-array
+ * @section_id: basic-array-classes
+ * @title: Basic array classes
  * @include: arrow-glib/arrow-glib.h
  *
  * #GArrowArray is a base class for all array classes such as
@@ -194,15 +194,6 @@ G_BEGIN_DECLS
  * nanoseconds since midnight in 64-bit signed integer array. It can
  * store zero or more time data. If you don't have Arrow format data,
  * you need to use #GArrowTime64ArrayBuilder to create a new array.
- *
- * #GArrowListArray is a class for list array. It can store zero or
- * more list data. If you don't have Arrow format data, you need to
- * use #GArrowListArrayBuilder to create a new array.
- *
- * #GArrowStructArray is a class for struct array. It can store zero
- * or more structs. One struct has zero or more fields. If you don't
- * have Arrow format data, you need to use #GArrowStructArrayBuilder
- * to create a new array.
  */
 
 typedef struct GArrowArrayPrivate_ {
@@ -572,6 +563,77 @@ garrow_array_cast(GArrowArray *array,
   return garrow_array_new_raw(&arrow_casted_array);
 }
 
+/**
+ * garrow_array_unique:
+ * @array: A #GArrowArray.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (nullable) (transfer full):
+ *   A newly created unique elements array on success, %NULL on error.
+ *
+ * Since: 0.8.0
+ */
+GArrowArray *
+garrow_array_unique(GArrowArray *array,
+                    GError **error)
+{
+  auto arrow_array = garrow_array_get_raw(array);
+  auto memory_pool = arrow::default_memory_pool();
+  arrow::compute::FunctionContext context(memory_pool);
+  std::shared_ptr<arrow::Array> arrow_unique_array;
+  auto status = arrow::compute::Unique(&context,
+                                       arrow::compute::Datum(arrow_array),
+                                       &arrow_unique_array);
+  if (!status.ok()) {
+    std::stringstream message;
+    message << "[array][unique] <";
+    message << arrow_array->type()->ToString();
+    message << ">";
+    garrow_error_check(error, status, message.str().c_str());
+    return NULL;
+  }
+
+  return garrow_array_new_raw(&arrow_unique_array);
+}
+
+/**
+ * garrow_array_dictionary_encode:
+ * @array: A #GArrowArray.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (nullable) (transfer full):
+ *   A newly created #GArrowDictionarArray for the @array on success,
+ *   %NULL on error.
+ *
+ * Since: 0.8.0
+ */
+GArrowArray *
+garrow_array_dictionary_encode(GArrowArray *array,
+                               GError **error)
+{
+  auto arrow_array = garrow_array_get_raw(array);
+  auto memory_pool = arrow::default_memory_pool();
+  arrow::compute::FunctionContext context(memory_pool);
+  arrow::compute::Datum dictionary_encoded_datum;
+  auto status =
+    arrow::compute::DictionaryEncode(&context,
+                                     arrow::compute::Datum(arrow_array),
+                                     &dictionary_encoded_datum);
+  if (!status.ok()) {
+    std::stringstream message;
+    message << "[array][dictionary-encode] <";
+    message << arrow_array->type()->ToString();
+    message << ">";
+    garrow_error_check(error, status, message.str().c_str());
+    return NULL;
+  }
+
+  auto arrow_dictionary_encoded_array =
+    arrow::MakeArray(dictionary_encoded_datum.array());
+
+  return garrow_array_new_raw(&arrow_dictionary_encoded_array);
+}
+
 
 G_DEFINE_TYPE(GArrowNullArray,               \
               garrow_null_array,             \
@@ -2028,195 +2090,6 @@ garrow_time64_array_get_values(GArrowTime64Array *array,
   return reinterpret_cast<const gint64 *>(values);
 }
 
-
-G_DEFINE_TYPE(GArrowListArray,               \
-              garrow_list_array,             \
-              GARROW_TYPE_ARRAY)
-
-static void
-garrow_list_array_init(GArrowListArray *object)
-{
-}
-
-static void
-garrow_list_array_class_init(GArrowListArrayClass *klass)
-{
-}
-
-/**
- * garrow_list_array_new:
- * @length: The number of elements.
- * @value_offsets: The offsets of @values in Arrow format.
- * @values: The values as #GArrowArray.
- * @null_bitmap: (nullable): The bitmap that shows null elements. The
- *   N-th element is null when the N-th bit is 0, not null otherwise.
- *   If the array has no null elements, the bitmap must be %NULL and
- *   @n_nulls is 0.
- * @n_nulls: The number of null elements. If -1 is specified, the
- *   number of nulls are computed from @null_bitmap.
- *
- * Returns: A newly created #GArrowListArray.
- *
- * Since: 0.4.0
- */
-GArrowListArray *
-garrow_list_array_new(gint64 length,
-                      GArrowBuffer *value_offsets,
-                      GArrowArray *values,
-                      GArrowBuffer *null_bitmap,
-                      gint64 n_nulls)
-{
-  const auto arrow_value_offsets = garrow_buffer_get_raw(value_offsets);
-  const auto arrow_values = garrow_array_get_raw(values);
-  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
-  auto arrow_data_type = arrow::list(arrow_values->type());
-  auto arrow_list_array =
-    std::make_shared<arrow::ListArray>(arrow_data_type,
-                                       length,
-                                       arrow_value_offsets,
-                                       arrow_values,
-                                       arrow_bitmap,
-                                       n_nulls);
-  auto arrow_array =
-    std::static_pointer_cast<arrow::Array>(arrow_list_array);
-  return GARROW_LIST_ARRAY(garrow_array_new_raw(&arrow_array));
-}
-
-/**
- * garrow_list_array_get_value_type:
- * @array: A #GArrowListArray.
- *
- * Returns: (transfer full): The data type of value in each list.
- */
-GArrowDataType *
-garrow_list_array_get_value_type(GArrowListArray *array)
-{
-  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
-  auto arrow_list_array =
-    static_cast<arrow::ListArray *>(arrow_array.get());
-  auto arrow_value_type = arrow_list_array->value_type();
-  return garrow_data_type_new_raw(&arrow_value_type);
-}
-
-/**
- * garrow_list_array_get_value:
- * @array: A #GArrowListArray.
- * @i: The index of the target value.
- *
- * Returns: (transfer full): The i-th list.
- */
-GArrowArray *
-garrow_list_array_get_value(GArrowListArray *array,
-                            gint64 i)
-{
-  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
-  auto arrow_list_array =
-    static_cast<arrow::ListArray *>(arrow_array.get());
-  auto arrow_list =
-    arrow_list_array->values()->Slice(arrow_list_array->value_offset(i),
-                                      arrow_list_array->value_length(i));
-  return garrow_array_new_raw(&arrow_list);
-}
-
-
-G_DEFINE_TYPE(GArrowStructArray,               \
-              garrow_struct_array,             \
-              GARROW_TYPE_ARRAY)
-
-static void
-garrow_struct_array_init(GArrowStructArray *object)
-{
-}
-
-static void
-garrow_struct_array_class_init(GArrowStructArrayClass *klass)
-{
-}
-
-/**
- * garrow_struct_array_new:
- * @data_type: The data type of the struct.
- * @length: The number of elements.
- * @children: (element-type GArrowArray): The arrays for each field
- *   as #GList of #GArrowArray.
- * @null_bitmap: (nullable): The bitmap that shows null elements. The
- *   N-th element is null when the N-th bit is 0, not null otherwise.
- *   If the array has no null elements, the bitmap must be %NULL and
- *   @n_nulls is 0.
- * @n_nulls: The number of null elements. If -1 is specified, the
- *   number of nulls are computed from @null_bitmap.
- *
- * Returns: A newly created #GArrowStructArray.
- *
- * Since: 0.4.0
- */
-GArrowStructArray *
-garrow_struct_array_new(GArrowDataType *data_type,
-                        gint64 length,
-                        GList *children,
-                        GArrowBuffer *null_bitmap,
-                        gint64 n_nulls)
-{
-  const auto arrow_data_type = garrow_data_type_get_raw(data_type);
-  std::vector<std::shared_ptr<arrow::Array>> arrow_children;
-  for (GList *node = children; node; node = node->next) {
-    GArrowArray *child = GARROW_ARRAY(node->data);
-    arrow_children.push_back(garrow_array_get_raw(child));
-  }
-  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
-  auto arrow_struct_array =
-    std::make_shared<arrow::StructArray>(arrow_data_type,
-                                         length,
-                                         arrow_children,
-                                         arrow_bitmap,
-                                         n_nulls);
-  auto arrow_array =
-    std::static_pointer_cast<arrow::Array>(arrow_struct_array);
-  return GARROW_STRUCT_ARRAY(garrow_array_new_raw(&arrow_array));
-}
-
-/**
- * garrow_struct_array_get_field
- * @array: A #GArrowStructArray.
- * @i: The index of the field in the struct.
- *
- * Returns: (transfer full): The i-th field.
- */
-GArrowArray *
-garrow_struct_array_get_field(GArrowStructArray *array,
-                              gint i)
-{
-  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
-  auto arrow_struct_array =
-    static_cast<arrow::StructArray *>(arrow_array.get());
-  auto arrow_field = arrow_struct_array->field(i);
-  return garrow_array_new_raw(&arrow_field);
-}
-
-/**
- * garrow_struct_array_get_fields
- * @array: A #GArrowStructArray.
- *
- * Returns: (element-type GArrowArray) (transfer full):
- *   The fields in the struct.
- */
-GList *
-garrow_struct_array_get_fields(GArrowStructArray *array)
-{
-  const auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
-  const auto arrow_struct_array =
-    static_cast<const arrow::StructArray *>(arrow_array.get());
-
-  GList *fields = NULL;
-  for (int i = 0; i < arrow_struct_array->num_fields(); ++i) {
-    auto arrow_field = arrow_struct_array->field(i);
-    GArrowArray *field = garrow_array_new_raw(&arrow_field);
-    fields = g_list_prepend(fields, field);
-  }
-
-  return g_list_reverse(fields);
-}
-
 G_END_DECLS
 
 GArrowArray *
@@ -2289,6 +2162,9 @@ garrow_array_new_raw(std::shared_ptr<arrow::Array> *arrow_array)
   case arrow::Type::type::STRUCT:
     type = GARROW_TYPE_STRUCT_ARRAY;
     break;
+  case arrow::Type::type::DICTIONARY:
+    type = GARROW_TYPE_DICTIONARY_ARRAY;
+    break;
   default:
     type = GARROW_TYPE_ARRAY;
     break;

c_glib/arrow-glib/basic-array.hpp

@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/api.h>
+
+#include <arrow-glib/basic-array.h>
+
+GArrowArray *garrow_array_new_raw(std::shared_ptr<arrow::Array> *arrow_array);
+std::shared_ptr<arrow::Array> garrow_array_get_raw(GArrowArray *array);

c_glib/arrow-glib/basic-data-type.cpp

@@ -220,6 +220,37 @@ garrow_data_type_get_id(GArrowDataType *data_type)
 }
 
 
+G_DEFINE_ABSTRACT_TYPE(GArrowFixedWidthDataType,                 \
+                       garrow_fixed_width_data_type,             \
+                       GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_fixed_width_data_type_init(GArrowFixedWidthDataType *object)
+{
+}
+
+static void
+garrow_fixed_width_data_type_class_init(GArrowFixedWidthDataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_fixed_width_data_type_get_id:
+ * @data_type: A #GArrowFixedWidthDataType.
+ *
+ * Returns: The number of bits for one data.
+ */
+gint
+garrow_fixed_width_data_type_get_bit_width(GArrowFixedWidthDataType *data_type)
+{
+  const auto arrow_data_type =
+    garrow_data_type_get_raw(GARROW_DATA_TYPE(data_type));
+  const auto arrow_fixed_width_type =
+    std::static_pointer_cast<arrow::FixedWidthType>(arrow_data_type);
+  return arrow_fixed_width_type->bit_width();
+}
+
+
 G_DEFINE_TYPE(GArrowNullDataType,                \
               garrow_null_data_type,             \
               GARROW_TYPE_DATA_TYPE)
@@ -254,7 +285,7 @@ garrow_null_data_type_new(void)
 
 G_DEFINE_TYPE(GArrowBooleanDataType,                \
               garrow_boolean_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
+              GARROW_TYPE_FIXED_WIDTH_DATA_TYPE)
 
 static void
 garrow_boolean_data_type_init(GArrowBooleanDataType *object)
@@ -1015,12 +1046,15 @@ garrow_data_type_new_raw(std::shared_ptr<arrow::DataType> *arrow_data_type)
   case arrow::Type::type::STRUCT:
     type = GARROW_TYPE_STRUCT_DATA_TYPE;
     break;
+  case arrow::Type::type::DICTIONARY:
+    type = GARROW_TYPE_DICTIONARY_DATA_TYPE;
+    break;
   default:
     type = GARROW_TYPE_DATA_TYPE;
     break;
   }
   data_type = GARROW_DATA_TYPE(g_object_new(type,
-                                            "data_type", arrow_data_type,
+                                            "data-type", arrow_data_type,
                                             NULL));
   return data_type;
 }

c_glib/arrow-glib/basic-data-type.h

@@ -19,57 +19,45 @@
 
 #pragma once
 
+#include <arrow-glib/gobject-type.h>
 #include <arrow-glib/type.h>
 
 G_BEGIN_DECLS
 
-#define GARROW_TYPE_DATA_TYPE                   \
-  (garrow_data_type_get_type())
-#define GARROW_DATA_TYPE(obj)                           \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_DATA_TYPE,    \
-                              GArrowDataType))
-#define GARROW_DATA_TYPE_CLASS(klass)                   \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_DATA_TYPE,       \
-                           GArrowDataTypeClass))
-#define GARROW_IS_DATA_TYPE(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_DATA_TYPE))
-#define GARROW_IS_DATA_TYPE_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_DATA_TYPE))
-#define GARROW_DATA_TYPE_GET_CLASS(obj)                 \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_DATA_TYPE,     \
-                             GArrowDataTypeClass))
-
-typedef struct _GArrowDataType         GArrowDataType;
-typedef struct _GArrowDataTypeClass    GArrowDataTypeClass;
-
-/**
- * GArrowDataType:
- *
- * It wraps `arrow::DataType`.
- */
-struct _GArrowDataType
-{
-  /*< private >*/
-  GObject parent_instance;
-};
-
+#define GARROW_TYPE_DATA_TYPE (garrow_data_type_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowDataType,
+                         garrow_data_type,
+                         GARROW,
+                         DATA_TYPE,
+                         GObject)
 struct _GArrowDataTypeClass
 {
   GObjectClass parent_class;
 };
 
-GType      garrow_data_type_get_type  (void) G_GNUC_CONST;
 gboolean   garrow_data_type_equal     (GArrowDataType *data_type,
                                        GArrowDataType *other_data_type);
 gchar     *garrow_data_type_to_string (GArrowDataType *data_type);
 GArrowType garrow_data_type_get_id    (GArrowDataType *data_type);
 
 
+#define GARROW_TYPE_FIXED_WIDTH_DATA_TYPE (garrow_fixed_width_data_type_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowFixedWidthDataType,
+                         garrow_fixed_width_data_type,
+                         GARROW,
+                         FIXED_WIDTH_DATA_TYPE,
+                         GArrowDataType)
+struct _GArrowFixedWidthDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+gint garrow_fixed_width_data_type_get_bit_width(GArrowFixedWidthDataType *data_type);
+/* TODO:
+GList *garrow_fixed_width_data_type_get_buffer_layout(GArrowFixedWidthDataType *data_type);
+*/
+
+
 #define GARROW_TYPE_NULL_DATA_TYPE              \
   (garrow_null_data_type_get_type())
 #define GARROW_NULL_DATA_TYPE(obj)                              \
@@ -114,47 +102,17 @@ GType               garrow_null_data_type_get_type (void) G_GNUC_CONST;
 GArrowNullDataType *garrow_null_data_type_new      (void);
 
 
-#define GARROW_TYPE_BOOLEAN_DATA_TYPE           \
-  (garrow_boolean_data_type_get_type())
-#define GARROW_BOOLEAN_DATA_TYPE(obj)                           \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_BOOLEAN_DATA_TYPE,    \
-                              GArrowBooleanDataType))
-#define GARROW_BOOLEAN_DATA_TYPE_CLASS(klass)                   \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_BOOLEAN_DATA_TYPE,       \
-                           GArrowBooleanDataTypeClass))
-#define GARROW_IS_BOOLEAN_DATA_TYPE(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_BOOLEAN_DATA_TYPE))
-#define GARROW_IS_BOOLEAN_DATA_TYPE_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_BOOLEAN_DATA_TYPE))
-#define GARROW_BOOLEAN_DATA_TYPE_GET_CLASS(obj)                 \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_BOOLEAN_DATA_TYPE,     \
-                             GArrowBooleanDataTypeClass))
-
-typedef struct _GArrowBooleanDataType         GArrowBooleanDataType;
-typedef struct _GArrowBooleanDataTypeClass    GArrowBooleanDataTypeClass;
-
-/**
- * GArrowBooleanDataType:
- *
- * It wraps `arrow::BooleanType`.
- */
-struct _GArrowBooleanDataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
+#define GARROW_TYPE_BOOLEAN_DATA_TYPE (garrow_boolean_data_type_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowBooleanDataType,
+                         garrow_boolean_data_type,
+                         GARROW,
+                         BOOLEAN_DATA_TYPE,
+                         GArrowFixedWidthDataType)
 struct _GArrowBooleanDataTypeClass
 {
-  GArrowDataTypeClass parent_class;
+  GArrowFixedWidthDataTypeClass parent_class;
 };
 
-GType                  garrow_boolean_data_type_get_type (void) G_GNUC_CONST;
 GArrowBooleanDataType *garrow_boolean_data_type_new      (void);
 
 

c_glib/arrow-glib/composite-array.cpp

@@ -0,0 +1,344 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array.hpp>
+#include <arrow-glib/buffer.hpp>
+#include <arrow-glib/compute.hpp>
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/type.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: composite-array
+ * @section_id: composite-array-classes
+ * @title: Composite array classes
+ * @include: arrow-glib/arrow-glib.h
+ *
+ * #GArrowListArray is a class for list array. It can store zero or
+ * more list data. If you don't have Arrow format data, you need to
+ * use #GArrowListArrayBuilder to create a new array.
+ *
+ * #GArrowStructArray is a class for struct array. It can store zero
+ * or more structs. One struct has zero or more fields. If you don't
+ * have Arrow format data, you need to use #GArrowStructArrayBuilder
+ * to create a new array.
+ *
+ * #GArrowDictionaryArray is a class for dictionary array. It can
+ * store data with dictionary and indices. It's space effective than
+ * normal array when the array has many same values. You can convert a
+ * normal array to dictionary array by garrow_array_dictionary_encode().
+ */
+
+G_DEFINE_TYPE(GArrowListArray,               \
+              garrow_list_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_list_array_init(GArrowListArray *object)
+{
+}
+
+static void
+garrow_list_array_class_init(GArrowListArrayClass *klass)
+{
+}
+
+/**
+ * garrow_list_array_new:
+ * @length: The number of elements.
+ * @value_offsets: The offsets of @values in Arrow format.
+ * @values: The values as #GArrowArray.
+ * @null_bitmap: (nullable): The bitmap that shows null elements. The
+ *   N-th element is null when the N-th bit is 0, not null otherwise.
+ *   If the array has no null elements, the bitmap must be %NULL and
+ *   @n_nulls is 0.
+ * @n_nulls: The number of null elements. If -1 is specified, the
+ *   number of nulls are computed from @null_bitmap.
+ *
+ * Returns: A newly created #GArrowListArray.
+ *
+ * Since: 0.4.0
+ */
+GArrowListArray *
+garrow_list_array_new(gint64 length,
+                      GArrowBuffer *value_offsets,
+                      GArrowArray *values,
+                      GArrowBuffer *null_bitmap,
+                      gint64 n_nulls)
+{
+  const auto arrow_value_offsets = garrow_buffer_get_raw(value_offsets);
+  const auto arrow_values = garrow_array_get_raw(values);
+  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
+  auto arrow_data_type = arrow::list(arrow_values->type());
+  auto arrow_list_array =
+    std::make_shared<arrow::ListArray>(arrow_data_type,
+                                       length,
+                                       arrow_value_offsets,
+                                       arrow_values,
+                                       arrow_bitmap,
+                                       n_nulls);
+  auto arrow_array =
+    std::static_pointer_cast<arrow::Array>(arrow_list_array);
+  return GARROW_LIST_ARRAY(garrow_array_new_raw(&arrow_array));
+}
+
+/**
+ * garrow_list_array_get_value_type:
+ * @array: A #GArrowListArray.
+ *
+ * Returns: (transfer full): The data type of value in each list.
+ */
+GArrowDataType *
+garrow_list_array_get_value_type(GArrowListArray *array)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto arrow_list_array =
+    static_cast<arrow::ListArray *>(arrow_array.get());
+  auto arrow_value_type = arrow_list_array->value_type();
+  return garrow_data_type_new_raw(&arrow_value_type);
+}
+
+/**
+ * garrow_list_array_get_value:
+ * @array: A #GArrowListArray.
+ * @i: The index of the target value.
+ *
+ * Returns: (transfer full): The i-th list.
+ */
+GArrowArray *
+garrow_list_array_get_value(GArrowListArray *array,
+                            gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto arrow_list_array =
+    static_cast<arrow::ListArray *>(arrow_array.get());
+  auto arrow_list =
+    arrow_list_array->values()->Slice(arrow_list_array->value_offset(i),
+                                      arrow_list_array->value_length(i));
+  return garrow_array_new_raw(&arrow_list);
+}
+
+
+G_DEFINE_TYPE(GArrowStructArray,               \
+              garrow_struct_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_struct_array_init(GArrowStructArray *object)
+{
+}
+
+static void
+garrow_struct_array_class_init(GArrowStructArrayClass *klass)
+{
+}
+
+/**
+ * garrow_struct_array_new:
+ * @data_type: The data type of the struct.
+ * @length: The number of elements.
+ * @children: (element-type GArrowArray): The arrays for each field
+ *   as #GList of #GArrowArray.
+ * @null_bitmap: (nullable): The bitmap that shows null elements. The
+ *   N-th element is null when the N-th bit is 0, not null otherwise.
+ *   If the array has no null elements, the bitmap must be %NULL and
+ *   @n_nulls is 0.
+ * @n_nulls: The number of null elements. If -1 is specified, the
+ *   number of nulls are computed from @null_bitmap.
+ *
+ * Returns: A newly created #GArrowStructArray.
+ *
+ * Since: 0.4.0
+ */
+GArrowStructArray *
+garrow_struct_array_new(GArrowDataType *data_type,
+                        gint64 length,
+                        GList *children,
+                        GArrowBuffer *null_bitmap,
+                        gint64 n_nulls)
+{
+  const auto arrow_data_type = garrow_data_type_get_raw(data_type);
+  std::vector<std::shared_ptr<arrow::Array>> arrow_children;
+  for (GList *node = children; node; node = node->next) {
+    GArrowArray *child = GARROW_ARRAY(node->data);
+    arrow_children.push_back(garrow_array_get_raw(child));
+  }
+  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
+  auto arrow_struct_array =
+    std::make_shared<arrow::StructArray>(arrow_data_type,
+                                         length,
+                                         arrow_children,
+                                         arrow_bitmap,
+                                         n_nulls);
+  auto arrow_array =
+    std::static_pointer_cast<arrow::Array>(arrow_struct_array);
+  return GARROW_STRUCT_ARRAY(garrow_array_new_raw(&arrow_array));
+}
+
+/**
+ * garrow_struct_array_get_field
+ * @array: A #GArrowStructArray.
+ * @i: The index of the field in the struct.
+ *
+ * Returns: (transfer full): The i-th field.
+ */
+GArrowArray *
+garrow_struct_array_get_field(GArrowStructArray *array,
+                              gint i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto arrow_struct_array =
+    static_cast<arrow::StructArray *>(arrow_array.get());
+  auto arrow_field = arrow_struct_array->field(i);
+  return garrow_array_new_raw(&arrow_field);
+}
+
+/**
+ * garrow_struct_array_get_fields
+ * @array: A #GArrowStructArray.
+ *
+ * Returns: (element-type GArrowArray) (transfer full):
+ *   The fields in the struct.
+ */
+GList *
+garrow_struct_array_get_fields(GArrowStructArray *array)
+{
+  const auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  const auto arrow_struct_array =
+    static_cast<const arrow::StructArray *>(arrow_array.get());
+
+  GList *fields = NULL;
+  for (int i = 0; i < arrow_struct_array->num_fields(); ++i) {
+    auto arrow_field = arrow_struct_array->field(i);
+    GArrowArray *field = garrow_array_new_raw(&arrow_field);
+    fields = g_list_prepend(fields, field);
+  }
+
+  return g_list_reverse(fields);
+}
+
+
+G_DEFINE_TYPE(GArrowDictionaryArray,            \
+              garrow_dictionary_array,          \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_dictionary_array_init(GArrowDictionaryArray *object)
+{
+}
+
+static void
+garrow_dictionary_array_class_init(GArrowDictionaryArrayClass *klass)
+{
+}
+
+/**
+ * garrow_dictionary_array_new:
+ * @data_type: The data type of dictionary.
+ * @indices: The indices of values in dictionary.
+ *
+ * Returns: A newly created #GArrowDictionaryArray.
+ *
+ * Since: 0.8.0
+ */
+GArrowDictionaryArray *
+garrow_dictionary_array_new(GArrowDataType *data_type,
+                            GArrowArray *indices)
+{
+  const auto arrow_data_type = garrow_data_type_get_raw(data_type);
+  const auto arrow_indices = garrow_array_get_raw(indices);
+  auto arrow_dictionary_array =
+    std::make_shared<arrow::DictionaryArray>(arrow_data_type,
+                                             arrow_indices);
+  auto arrow_array =
+    std::static_pointer_cast<arrow::Array>(arrow_dictionary_array);
+  return GARROW_DICTIONARY_ARRAY(garrow_array_new_raw(&arrow_array));
+}
+
+/**
+ * garrow_dictionary_array_get_indices:
+ * @array: A #GArrowDictionaryArray.
+ *
+ * Returns: (transfer full): The indices of values in dictionary.
+ *
+ * Since: 0.8.0
+ */
+GArrowArray *
+garrow_dictionary_array_get_indices(GArrowDictionaryArray *array)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto arrow_dictionary_array =
+    std::static_pointer_cast<arrow::DictionaryArray>(arrow_array);
+  auto arrow_indices = arrow_dictionary_array->indices();
+  return garrow_array_new_raw(&arrow_indices);
+}
+
+/**
+ * garrow_dictionary_array_get_dictionary:
+ * @array: A #GArrowDictionaryArray.
+ *
+ * Returns: (transfer full): The dictionary of this array.
+ *
+ * Since: 0.8.0
+ */
+GArrowArray *
+garrow_dictionary_array_get_dictionary(GArrowDictionaryArray *array)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto arrow_dictionary_array =
+    std::static_pointer_cast<arrow::DictionaryArray>(arrow_array);
+  auto arrow_dictionary = arrow_dictionary_array->dictionary();
+  return garrow_array_new_raw(&arrow_dictionary);
+}
+
+/**
+ * garrow_dictionary_array_get_dictionary_data_type:
+ * @array: A #GArrowDictionaryArray.
+ *
+ * Returns: (transfer full): The dictionary data type of this array.
+ *
+ * Since: 0.8.0
+ */
+GArrowDictionaryDataType *
+garrow_dictionary_array_get_dictionary_data_type(GArrowDictionaryArray *array)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto arrow_dictionary_array =
+    std::static_pointer_cast<arrow::DictionaryArray>(arrow_array);
+  auto arrow_dictionary_data_type = arrow_dictionary_array->dict_type();
+  auto const_arrow_data_type =
+    static_cast<const arrow::DataType *>(arrow_dictionary_data_type);
+  auto arrow_data_type = const_cast<arrow::DataType *>(const_arrow_data_type);
+  struct NullDeleter {
+    void operator()(arrow::DataType *data_type) {
+    }
+  };
+  std::shared_ptr<arrow::DataType>
+    shared_arrow_data_type(arrow_data_type, NullDeleter());
+  auto data_type = garrow_data_type_new_raw(&shared_arrow_data_type);
+  return GARROW_DICTIONARY_DATA_TYPE(data_type);
+}
+
+G_END_DECLS

c_glib/arrow-glib/composite-array.h

@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/basic-array.h>
+#include <arrow-glib/data-type.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_LIST_ARRAY                  \
+  (garrow_list_array_get_type())
+#define GARROW_LIST_ARRAY(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_LIST_ARRAY,   \
+                              GArrowListArray))
+#define GARROW_LIST_ARRAY_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_LIST_ARRAY,      \
+                           GArrowListArrayClass))
+#define GARROW_IS_LIST_ARRAY(obj)                       \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_LIST_ARRAY))
+#define GARROW_IS_LIST_ARRAY_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_LIST_ARRAY))
+#define GARROW_LIST_ARRAY_GET_CLASS(obj)                \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_LIST_ARRAY,    \
+                             GArrowListArrayClass))
+
+typedef struct _GArrowListArray         GArrowListArray;
+typedef struct _GArrowListArrayClass    GArrowListArrayClass;
+
+/**
+ * GArrowListArray:
+ *
+ * It wraps `arrow::ListArray`.
+ */
+struct _GArrowListArray
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowListArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_list_array_get_type(void) G_GNUC_CONST;
+
+GArrowListArray *garrow_list_array_new(gint64 length,
+                                       GArrowBuffer *value_offsets,
+                                       GArrowArray *values,
+                                       GArrowBuffer *null_bitmap,
+                                       gint64 n_nulls);
+
+GArrowDataType *garrow_list_array_get_value_type(GArrowListArray *array);
+GArrowArray *garrow_list_array_get_value(GArrowListArray *array,
+                                         gint64 i);
+
+
+#define GARROW_TYPE_STRUCT_ARRAY                \
+  (garrow_struct_array_get_type())
+#define GARROW_STRUCT_ARRAY(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_STRUCT_ARRAY, \
+                              GArrowStructArray))
+#define GARROW_STRUCT_ARRAY_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_STRUCT_ARRAY,    \
+                           GArrowStructArrayClass))
+#define GARROW_IS_STRUCT_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_STRUCT_ARRAY))
+#define GARROW_IS_STRUCT_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_STRUCT_ARRAY))
+#define GARROW_STRUCT_ARRAY_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_STRUCT_ARRAY,  \
+                             GArrowStructArrayClass))
+
+typedef struct _GArrowStructArray         GArrowStructArray;
+typedef struct _GArrowStructArrayClass    GArrowStructArrayClass;
+
+/**
+ * GArrowStructArray:
+ *
+ * It wraps `arrow::StructArray`.
+ */
+struct _GArrowStructArray
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowStructArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_struct_array_get_type(void) G_GNUC_CONST;
+
+GArrowStructArray *garrow_struct_array_new(GArrowDataType *data_type,
+                                           gint64 length,
+                                           GList *children,
+                                           GArrowBuffer *null_bitmap,
+                                           gint64 n_nulls);
+
+GArrowArray *garrow_struct_array_get_field(GArrowStructArray *array,
+                                           gint i);
+GList *garrow_struct_array_get_fields(GArrowStructArray *array);
+
+
+#define GARROW_TYPE_DICTIONARY_ARRAY (garrow_dictionary_array_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowDictionaryArray,
+                         garrow_dictionary_array,
+                         GARROW,
+                         DICTIONARY_ARRAY,
+                         GArrowArray)
+struct _GArrowDictionaryArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GArrowDictionaryArray *
+garrow_dictionary_array_new(GArrowDataType *data_type, GArrowArray *indices);
+GArrowArray *
+garrow_dictionary_array_get_indices(GArrowDictionaryArray *array);
+GArrowArray *
+garrow_dictionary_array_get_dictionary(GArrowDictionaryArray *array);
+GArrowDictionaryDataType *
+garrow_dictionary_array_get_dictionary_data_type(GArrowDictionaryArray *array);
+
+G_END_DECLS

c_glib/arrow-glib/composite-data-type.cpp

@@ -21,6 +21,7 @@
 #  include <config.h>
 #endif
 
+#include <arrow-glib/basic-array.hpp>
 #include <arrow-glib/data-type.hpp>
 #include <arrow-glib/enums.h>
 #include <arrow-glib/error.hpp>
@@ -38,6 +39,8 @@ G_BEGIN_DECLS
  * #GArrowListDataType is a class for list data type.
  *
  * #GArrowStructDataType is a class for struct data type.
+ *
+ * #GArrowDictionaryDataType is a class for dictionary data type.
  */
 
 G_DEFINE_TYPE(GArrowListDataType,                \
@@ -133,4 +136,95 @@ garrow_struct_data_type_new(GList *fields)
   return data_type;
 }
 
+
+G_DEFINE_TYPE(GArrowDictionaryDataType,                \
+              garrow_dictionary_data_type,             \
+              GARROW_TYPE_FIXED_WIDTH_DATA_TYPE)
+
+static void
+garrow_dictionary_data_type_init(GArrowDictionaryDataType *object)
+{
+}
+
+static void
+garrow_dictionary_data_type_class_init(GArrowDictionaryDataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_dictionary_data_type_new:
+ * @index_data_type: The data type of index.
+ * @dictionary: The dictionary.
+ * @ordered: Whether dictionary contents are ordered or not.
+ *
+ * Returns: The newly created dictionary data type.
+ *
+ * Since: 0.8.0
+ */
+GArrowDictionaryDataType *
+garrow_dictionary_data_type_new(GArrowDataType *index_data_type,
+                                GArrowArray *dictionary,
+                                gboolean ordered)
+{
+  auto arrow_index_data_type = garrow_data_type_get_raw(index_data_type);
+  auto arrow_dictionary = garrow_array_get_raw(dictionary);
+  auto arrow_data_type = arrow::dictionary(arrow_index_data_type,
+                                           arrow_dictionary,
+                                           ordered);
+  return GARROW_DICTIONARY_DATA_TYPE(garrow_data_type_new_raw(&arrow_data_type));
+}
+
+/**
+ * garrow_dictionary_data_type_get_index_data_type:
+ * @data_type: The #GArrowDictionaryDataType.
+ *
+ * Returns: (transfer full): The #GArrowDataType of index.
+ *
+ * Since: 0.8.0
+ */
+GArrowDataType *
+garrow_dictionary_data_type_get_index_data_type(GArrowDictionaryDataType *data_type)
+{
+  auto arrow_data_type = garrow_data_type_get_raw(GARROW_DATA_TYPE(data_type));
+  auto arrow_dictionary_data_type =
+    std::static_pointer_cast<arrow::DictionaryType>(arrow_data_type);
+  auto arrow_index_data_type = arrow_dictionary_data_type->index_type();
+  return garrow_data_type_new_raw(&arrow_index_data_type);
+}
+
+/**
+ * garrow_dictionary_data_type_get_dictionary:
+ * @data_type: The #GArrowDictionaryDataType.
+ *
+ * Returns: (transfer full): The dictionary as #GArrowArray.
+ *
+ * Since: 0.8.0
+ */
+GArrowArray *
+garrow_dictionary_data_type_get_dictionary(GArrowDictionaryDataType *data_type)
+{
+  auto arrow_data_type = garrow_data_type_get_raw(GARROW_DATA_TYPE(data_type));
+  auto arrow_dictionary_data_type =
+    std::static_pointer_cast<arrow::DictionaryType>(arrow_data_type);
+  auto arrow_dictionary = arrow_dictionary_data_type->dictionary();
+  return garrow_array_new_raw(&arrow_dictionary);
+}
+
+/**
+ * garrow_dictionary_data_type_is_ordered:
+ * @data_type: The #GArrowDictionaryDataType.
+ *
+ * Returns: Whether dictionary contents are ordered or not.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_dictionary_data_type_is_ordered(GArrowDictionaryDataType *data_type)
+{
+  auto arrow_data_type = garrow_data_type_get_raw(GARROW_DATA_TYPE(data_type));
+  auto arrow_dictionary_data_type =
+    std::static_pointer_cast<arrow::DictionaryType>(arrow_data_type);
+  return arrow_dictionary_data_type->ordered();
+}
+
 G_END_DECLS

c_glib/arrow-glib/composite-data-type.h

@@ -19,6 +19,7 @@
 
 #pragma once
 
+#include <arrow-glib/basic-array.h>
 #include <arrow-glib/basic-data-type.h>
 #include <arrow-glib/field.h>
 
@@ -112,4 +113,28 @@ struct _GArrowStructDataTypeClass
 GType                 garrow_struct_data_type_get_type (void) G_GNUC_CONST;
 GArrowStructDataType *garrow_struct_data_type_new      (GList *fields);
 
+
+#define GARROW_TYPE_DICTIONARY_DATA_TYPE (garrow_dictionary_data_type_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowDictionaryDataType,
+                         garrow_dictionary_data_type,
+                         GARROW,
+                         DICTIONARY_DATA_TYPE,
+                         GArrowFixedWidthDataType)
+struct _GArrowDictionaryDataTypeClass
+{
+  GArrowFixedWidthDataTypeClass parent_class;
+};
+
+GArrowDictionaryDataType *
+garrow_dictionary_data_type_new(GArrowDataType *index_data_type,
+                                GArrowArray *dictionary,
+                                gboolean ordered);
+GArrowDataType *
+garrow_dictionary_data_type_get_index_data_type(GArrowDictionaryDataType *data_type);
+GArrowArray *
+garrow_dictionary_data_type_get_dictionary(GArrowDictionaryDataType *data_type);
+gboolean
+garrow_dictionary_data_type_is_ordered(GArrowDictionaryDataType *data_type);
+
+
 G_END_DECLS

c_glib/arrow-glib/input-stream.cpp

@@ -420,7 +420,7 @@ namespace garrow {
 
     arrow::Status Read(int64_t n_bytes,
                        int64_t *n_read_bytes,
-                       uint8_t *out) override {
+                       void *out) override {
       GError *error = NULL;
       *n_read_bytes = g_input_stream_read(input_stream_,
                                           out,
@@ -437,7 +437,7 @@ namespace garrow {
     }
 
     arrow::Status ReadAt(int64_t position, int64_t n_bytes,
-			 int64_t *n_read_bytes, uint8_t* out) override {
+			 int64_t *n_read_bytes, void* out) override {
 	return arrow::io::RandomAccessFile::ReadAt(
 	    position, n_bytes, n_read_bytes, out);
     }

c_glib/arrow-glib/meson.build

@@ -18,12 +18,13 @@
 # under the License.
 
 sources = files(
-  'array.cpp',
   'array-builder.cpp',
+  'basic-array.cpp',
   'basic-data-type.cpp',
   'buffer.cpp',
   'chunked-array.cpp',
   'column.cpp',
+  'composite-array.cpp',
   'composite-data-type.cpp',
   'error.cpp',
   'field.cpp',
@@ -59,10 +60,12 @@ c_headers = files(
   'array.h',
   'array-builder.h',
   'arrow-glib.h',
+  'basic-array.h',
   'basic-data-type.h',
   'buffer.h',
   'chunked-array.h',
   'column.h',
+  'composite-array.h',
   'composite-data-type.h',
   'data-type.h',
   'error.h',
@@ -102,6 +105,7 @@ cpp_headers = files(
   'array.hpp',
   'array-builder.hpp',
   'arrow-glib.hpp',
+  'basic-array.hpp',
   'basic-data-type.hpp',
   'buffer.hpp',
   'chunked-array.hpp',

c_glib/arrow-glib/output-stream.cpp

@@ -76,7 +76,7 @@ garrow_output_stream_file_interface_init(GArrowFileInterface *iface)
   iface->get_raw = garrow_output_stream_get_raw_file_interface;
 }
 
-static std::shared_ptr<arrow::io::Writeable>
+static std::shared_ptr<arrow::io::Writable>
 garrow_output_stream_get_raw_writeable_interface(GArrowWriteable *writeable)
 {
   auto output_stream = GARROW_OUTPUT_STREAM(writeable);
@@ -325,7 +325,7 @@ namespace garrow {
       return arrow::Status::OK();
     }
 
-    arrow::Status Write(const uint8_t *data,
+    arrow::Status Write(const void *data,
                         int64_t n_bytes) override {
       GError *error = NULL;
       gsize n_written_bytes;

c_glib/arrow-glib/record-batch.cpp

@@ -28,6 +28,23 @@
 
 #include <sstream>
 
+static inline bool
+garrow_record_batch_adjust_index(const std::shared_ptr<arrow::RecordBatch> arrow_record_batch,
+                                 gint &i)
+{
+  auto n_columns = arrow_record_batch->num_columns();
+  if (i < 0) {
+    i += n_columns;
+    if (i < 0) {
+      return false;
+    }
+  }
+  if (i >= n_columns) {
+    return false;
+  }
+  return true;
+}
+
 G_BEGIN_DECLS
 
 /**
@@ -135,13 +152,15 @@ garrow_record_batch_class_init(GArrowRecordBatchClass *klass)
  * @schema: The schema of the record batch.
  * @n_rows: The number of the rows in the record batch.
  * @columns: (element-type GArrowArray): The columns in the record batch.
+ * @error: (nullable): Return location for a #GError or %NULL.
  *
- * Returns: A newly created #GArrowRecordBatch.
+ * Returns: (nullable): A newly created #GArrowRecordBatch or %NULL on error.
  */
 GArrowRecordBatch *
 garrow_record_batch_new(GArrowSchema *schema,
                         guint32 n_rows,
-                        GList *columns)
+                        GList *columns,
+                        GError **error)
 {
   std::vector<std::shared_ptr<arrow::Array>> arrow_columns;
   for (GList *node = columns; node; node = node->next) {
@@ -152,7 +171,12 @@ garrow_record_batch_new(GArrowSchema *schema,
   auto arrow_record_batch =
     arrow::RecordBatch::Make(garrow_schema_get_raw(schema),
                              n_rows, arrow_columns);
-  return garrow_record_batch_new_raw(&arrow_record_batch);
+  auto status = arrow_record_batch->Validate();
+  if (garrow_error_check(error, status, "[record-batch][new]")) {
+    return garrow_record_batch_new_raw(&arrow_record_batch);
+  } else {
+    return NULL;
+  }
 }
 
 /**
@@ -192,15 +216,21 @@ garrow_record_batch_get_schema(GArrowRecordBatch *record_batch)
 /**
  * garrow_record_batch_get_column:
  * @record_batch: A #GArrowRecordBatch.
- * @i: The index of the target column.
+ * @i: The index of the target column. If it's negative, index is
+ *   counted backward from the end of the columns. `-1` means the last
+ *   column.
  *
- * Returns: (transfer full): The i-th column in the record batch.
+ * Returns: (transfer full) (nullable): The i-th column in the record batch
+ *   on success, %NULL on out of index.
  */
 GArrowArray *
 garrow_record_batch_get_column(GArrowRecordBatch *record_batch,
-                               guint i)
+                               gint i)
 {
   const auto arrow_record_batch = garrow_record_batch_get_raw(record_batch);
+  if (!garrow_record_batch_adjust_index(arrow_record_batch, i)) {
+    return NULL;
+  }
   auto arrow_column = arrow_record_batch->column(i);
   return garrow_array_new_raw(&arrow_column);
 }
@@ -230,15 +260,21 @@ garrow_record_batch_get_columns(GArrowRecordBatch *record_batch)
 /**
  * garrow_record_batch_get_column_name:
  * @record_batch: A #GArrowRecordBatch.
- * @i: The index of the target column.
+ * @i: The index of the target column. If it's negative, index is
+ *   counted backward from the end of the columns. `-1` means the last
+ *   column.
  *
- * Returns: The name of the i-th column in the record batch.
+ * Returns: (nullable): The name of the i-th column in the record batch
+ *   on success, %NULL on out of index
  */
 const gchar *
 garrow_record_batch_get_column_name(GArrowRecordBatch *record_batch,
-                                    guint i)
+                                    gint i)
 {
   const auto arrow_record_batch = garrow_record_batch_get_raw(record_batch);
+  if (!garrow_record_batch_adjust_index(arrow_record_batch, i)) {
+    return NULL;
+  }
   return arrow_record_batch->column_name(i).c_str();
 }
 

c_glib/arrow-glib/record-batch.h

@@ -68,17 +68,18 @@ GType garrow_record_batch_get_type(void) G_GNUC_CONST;
 
 GArrowRecordBatch *garrow_record_batch_new(GArrowSchema *schema,
                                            guint32 n_rows,
-                                           GList *columns);
+                                           GList *columns,
+                                           GError **error);
 
 gboolean garrow_record_batch_equal(GArrowRecordBatch *record_batch,
                                    GArrowRecordBatch *other_record_batch);
 
 GArrowSchema *garrow_record_batch_get_schema     (GArrowRecordBatch *record_batch);
 GArrowArray  *garrow_record_batch_get_column     (GArrowRecordBatch *record_batch,
-                                                  guint i);
+                                                  gint i);
 GList        *garrow_record_batch_get_columns    (GArrowRecordBatch *record_batch);
 const gchar  *garrow_record_batch_get_column_name(GArrowRecordBatch *record_batch,
-                                                  guint i);
+                                                  gint i);
 guint         garrow_record_batch_get_n_columns  (GArrowRecordBatch *record_batch);
 gint64        garrow_record_batch_get_n_rows     (GArrowRecordBatch *record_batch);
 GArrowRecordBatch *garrow_record_batch_slice     (GArrowRecordBatch *record_batch,

c_glib/arrow-glib/writeable.cpp

@@ -88,7 +88,7 @@ garrow_writeable_flush(GArrowWriteable *writeable,
 
 G_END_DECLS
 
-std::shared_ptr<arrow::io::Writeable>
+std::shared_ptr<arrow::io::Writable>
 garrow_writeable_get_raw(GArrowWriteable *writeable)
 {
   auto *iface = GARROW_WRITEABLE_GET_IFACE(writeable);

c_glib/arrow-glib/writeable.hpp

@@ -26,13 +26,13 @@
 /**
  * GArrowWriteableInterface:
  *
- * It wraps `arrow::io::Writeable`.
+ * It wraps `arrow::io::Writable`.
  */
 struct _GArrowWriteableInterface
 {
   GTypeInterface parent_iface;
 
-  std::shared_ptr<arrow::io::Writeable> (*get_raw)(GArrowWriteable *file);
+  std::shared_ptr<arrow::io::Writable> (*get_raw)(GArrowWriteable *file);
 };
 
-std::shared_ptr<arrow::io::Writeable> garrow_writeable_get_raw(GArrowWriteable *writeable);
+std::shared_ptr<arrow::io::Writable> garrow_writeable_get_raw(GArrowWriteable *writeable);

c_glib/arrow-gpu-glib/Makefile.am

@@ -72,6 +72,10 @@ INTROSPECTION_GIRS =
 INTROSPECTION_SCANNER_ARGS =
 INTROSPECTION_SCANNER_ENV =			\
 	PKG_CONFIG_PATH=${abs_builddir}/../arrow-glib:$${PKG_CONFIG_PATH}
+if OS_MACOS
+INTROSPECTION_SCANNER_ENV +=			\
+	ARCHFLAGS=
+endif
 INTROSPECTION_COMPILER_ARGS =			\
 	--includedir=$(abs_builddir)/../arrow-glib
 

c_glib/configure.ac

@@ -34,6 +34,19 @@ AC_CONFIG_HEADERS([config.h])
 AM_INIT_AUTOMAKE([1.13 foreign])
 AM_SILENT_RULES([yes])
 
+AC_CANONICAL_HOST
+AC_MSG_CHECKING([for macOS])
+case "$host_os" in
+darwin*)
+  os_macos=yes
+  ;;
+*)
+  os_macos=no
+  ;;
+esac
+AC_MSG_RESULT([$os_macos])
+AM_CONDITIONAL(OS_MACOS, test "$os_macos" = "yes")
+
 AC_PROG_CC
 AC_PROG_CXX
 AX_CXX_COMPILE_STDCXX_11([ext], [mandatory])

c_glib/doc/reference/arrow-glib-docs.xml

@@ -40,7 +40,8 @@
     <title>Data</title>
     <chapter id="array">
       <title>Array</title>
-      <xi:include href="xml/array.xml"/>
+      <xi:include href="xml/basic-array.xml"/>
+      <xi:include href="xml/composite-array.xml"/>
     </chapter>
     <chapter id="array-builder">
       <title>Array builder</title>

c_glib/example/go/read-batch.go

@@ -57,8 +57,8 @@ func PrintColumnValue(column *arrow.Array, i int64) {
 func PrintRecordBatch(recordBatch *arrow.RecordBatch) {
 	nColumns := recordBatch.GetNColumns()
 	for i := uint32(0); i < nColumns; i++ {
-		column := recordBatch.GetColumn(i)
-		columnName := recordBatch.GetColumnName(i)
+		column := recordBatch.GetColumn(int32(i))
+		columnName := recordBatch.GetColumnName(int32(i))
 		fmt.Printf("  %s: [", columnName)
 		nRows := recordBatch.GetNRows()
 		for j := int64(0); j < nRows; j++ {

c_glib/example/go/read-stream.go

@@ -57,8 +57,8 @@ func PrintColumnValue(column *arrow.Array, i int64) {
 func PrintRecordBatch(recordBatch *arrow.RecordBatch) {
 	nColumns := recordBatch.GetNColumns()
 	for i := uint32(0); i < nColumns; i++ {
-		column := recordBatch.GetColumn(i)
-		columnName := recordBatch.GetColumnName(i)
+		column := recordBatch.GetColumn(int32(i))
+		columnName := recordBatch.GetColumnName(int32(i))
 		fmt.Printf("  %s: [", columnName)
 		nRows := recordBatch.GetNRows()
 		for j := int64(0); j < nRows; j++ {

c_glib/example/go/write-batch.go

@@ -188,7 +188,10 @@ func main() {
 		BuildDoubleArray(),
 	}
 
-	recordBatch := arrow.NewRecordBatch(schema, 4, columns)
+	recordBatch, err := arrow.NewRecordBatch(schema, 4, columns)
+	if err != nil {
+		log.Fatalf("Failed to create record batch #1: %v", err)
+	}
 	_, err = writer.WriteRecordBatch(recordBatch)
 	if err != nil {
 		log.Fatalf("Failed to write record batch #1: %v", err)
@@ -198,7 +201,10 @@ func main() {
 	for i, column := range columns {
 		slicedColumns[i] = column.Slice(1, 3)
 	}
-	recordBatch = arrow.NewRecordBatch(schema, 3, slicedColumns)
+	recordBatch, err = arrow.NewRecordBatch(schema, 3, slicedColumns)
+	if err != nil {
+		log.Fatalf("Failed to create record batch #2: %v", err)
+	}
 	_, err = writer.WriteRecordBatch(recordBatch)
 	if err != nil {
 		log.Fatalf("Failed to write record batch #2: %v", err)

c_glib/example/go/write-stream.go

@@ -188,7 +188,10 @@ func main() {
 		BuildDoubleArray(),
 	}
 
-	recordBatch := arrow.NewRecordBatch(schema, 4, columns)
+	recordBatch, err := arrow.NewRecordBatch(schema, 4, columns)
+	if err != nil {
+		log.Fatalf("Failed to create record batch #1: %v", err)
+	}
 	_, err = writer.WriteRecordBatch(recordBatch)
 	if err != nil {
 		log.Fatalf("Failed to write record batch #1: %v", err)
@@ -198,7 +201,10 @@ func main() {
 	for i, column := range columns {
 		slicedColumns[i] = column.Slice(1, 3)
 	}
-	recordBatch = arrow.NewRecordBatch(schema, 3, slicedColumns)
+	recordBatch, err = arrow.NewRecordBatch(schema, 3, slicedColumns)
+	if err != nil {
+		log.Fatalf("Failed to create record batch #2: %v", err)
+	}
 	writer.WriteRecordBatch(recordBatch)
 	_, err = writer.WriteRecordBatch(recordBatch)
 	if err != nil {

c_glib/test/test-dictionary-array.rb

@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestDictionaryArray < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def setup
+    @index_data_type = Arrow::Int32DataType.new
+    @dictionary = build_string_array(["C", "C++", "Ruby"])
+    @ordered = false
+    @data_type = Arrow::DictionaryDataType.new(@index_data_type,
+                                               @dictionary,
+                                               @ordered)
+  end
+
+  sub_test_case(".new") do
+    def test_new
+      indices = build_int32_array([0, 2, 2, 1, 0])
+      dictionary_array = Arrow::DictionaryArray.new(@data_type, indices)
+      assert_equal(<<-STRING.chomp, dictionary_array.to_s)
+
+-- is_valid: all not null
+-- dictionary: ["C", "C++", "Ruby"]
+-- indices: [0, 2, 2, 1, 0]
+      STRING
+    end
+  end
+
+  sub_test_case("instance methods") do
+    def setup
+      super
+      @indices = build_int32_array([0, 2, 2, 1, 0])
+      @dictionary_array = Arrow::DictionaryArray.new(@data_type, @indices)
+    end
+
+    def test_indices
+      assert_equal(@indices, @dictionary_array.indices)
+    end
+
+    def test_dictionary
+      assert_equal(@dictionary, @dictionary_array.dictionary)
+    end
+
+    def test_dictionary_data_type
+      assert_equal(@data_type,
+                   @dictionary_array.dictionary_data_type)
+    end
+  end
+end

c_glib/test/test-dictionary-data-type.rb

@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestDictionaryDataType < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def setup
+    @index_data_type = Arrow::Int32DataType.new
+    @dictionary = build_string_array(["C", "C++", "Ruby"])
+    @ordered = true
+    @data_type = Arrow::DictionaryDataType.new(@index_data_type,
+                                               @dictionary,
+                                               @ordered)
+  end
+
+  def test_type
+    assert_equal(Arrow::Type::DICTIONARY, @data_type.id)
+  end
+
+  def test_to_s
+    assert_equal("dictionary<values=string, indices=int32, ordered=1>",
+                 @data_type.to_s)
+  end
+
+  def test_bit_width
+    assert_equal(32, @data_type.bit_width)
+  end
+
+  def test_index_data_type
+    assert_equal(@index_data_type, @data_type.index_data_type)
+  end
+
+  def test_dictionary
+    assert_equal(@dictionary, @data_type.dictionary)
+  end
+
+  def test_ordered?
+    assert do
+      @data_type.ordered?
+    end
+  end
+end

c_glib/test/test-dictionary-encode.rb

@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestDictionaryEncode < Test::Unit::TestCase
+  include Helper::Buildable
+  include Helper::Omittable
+
+  def test_int32
+    array = build_int32_array([1, 3, 1, -1, -3, -1])
+    assert_equal(<<-STRING.chomp, array.dictionary_encode.to_s)
+
+-- is_valid: all not null
+-- dictionary: [1, 3, -1, -3]
+-- indices: [0, 1, 0, 2, 3, 2]
+    STRING
+  end
+
+  def test_string
+    array = build_string_array(["Ruby", "Python", "Ruby"])
+    assert_equal(<<-STRING.chomp, array.dictionary_encode.to_s)
+
+-- is_valid: all not null
+-- dictionary: ["Ruby", "Python"]
+-- indices: [0, 1, 0]
+    STRING
+  end
+end

c_glib/test/test-record-batch.rb

@@ -18,32 +18,53 @@
 class TestTable < Test::Unit::TestCase
   include Helper::Buildable
 
-  def test_new
-    fields = [
-      Arrow::Field.new("visible", Arrow::BooleanDataType.new),
-      Arrow::Field.new("valid", Arrow::BooleanDataType.new),
-    ]
-    schema = Arrow::Schema.new(fields)
-    columns = [
-      build_boolean_array([true]),
-      build_boolean_array([false]),
-    ]
-    record_batch = Arrow::RecordBatch.new(schema, 1, columns)
-    assert_equal(1, record_batch.n_rows)
-  end
-
-  sub_test_case("instance methods") do
-    def setup
+  sub_test_case(".new") do
+    def test_valid
       fields = [
         Arrow::Field.new("visible", Arrow::BooleanDataType.new),
         Arrow::Field.new("valid", Arrow::BooleanDataType.new),
       ]
       schema = Arrow::Schema.new(fields)
       columns = [
-        build_boolean_array([true, false, true, false, true, false]),
-        build_boolean_array([false, true, false, true, false]),
+        build_boolean_array([true]),
+        build_boolean_array([false]),
       ]
-      @record_batch = Arrow::RecordBatch.new(schema, 5, columns)
+      record_batch = Arrow::RecordBatch.new(schema, 1, columns)
+      assert_equal(1, record_batch.n_rows)
+    end
+
+    def test_no_columns
+      fields = [
+        Arrow::Field.new("visible", Arrow::BooleanDataType.new),
+      ]
+      schema = Arrow::Schema.new(fields)
+      message = "[record-batch][new]: " +
+        "Invalid: Number of columns did not match schema"
+      assert_raise(Arrow::Error::Invalid.new(message)) do
+        Arrow::RecordBatch.new(schema, 0, [])
+      end
+    end
+  end
+
+  sub_test_case("instance methods") do
+    def setup
+      @visible_field = Arrow::Field.new("visible", Arrow::BooleanDataType.new)
+      @visible_values = [true, false, true, false, true]
+      @valid_field = Arrow::Field.new("valid", Arrow::BooleanDataType.new)
+      @valid_values = [false, true, false, true, false]
+
+      fields = [
+        @visible_field,
+        @valid_field,
+      ]
+      schema = Arrow::Schema.new(fields)
+      columns = [
+        build_boolean_array(@visible_values),
+        build_boolean_array(@valid_values),
+      ]
+      @record_batch = Arrow::RecordBatch.new(schema,
+                                             @visible_values.size,
+                                             columns)
     end
 
     def test_equal
@@ -53,7 +74,7 @@ class TestTable < Test::Unit::TestCase
       ]
       schema = Arrow::Schema.new(fields)
       columns = [
-        build_boolean_array([true, false, true, false, true, false]),
+        build_boolean_array([true, false, true, false, true]),
         build_boolean_array([false, true, false, true, false]),
       ]
       other_record_batch = Arrow::RecordBatch.new(schema, 5, columns)
@@ -66,12 +87,28 @@ class TestTable < Test::Unit::TestCase
                    @record_batch.schema.fields.collect(&:name))
     end
 
-    def test_column
-      assert_equal(5, @record_batch.get_column(1).length)
+    sub_test_case("#column") do
+      def test_positive
+        assert_equal(build_boolean_array(@valid_values),
+                     @record_batch.get_column(1))
+      end
+
+      def test_negative
+        assert_equal(build_boolean_array(@visible_values),
+                     @record_batch.get_column(-2))
+      end
+
+      def test_positive_out_of_index
+        assert_nil(@record_batch.get_column(2))
+      end
+
+      def test_negative_out_of_index
+        assert_nil(@record_batch.get_column(-3))
+      end
     end
 
     def test_columns
-      assert_equal([6, 5],
+      assert_equal([5, 5],
                    @record_batch.columns.collect(&:length))
     end
 
@@ -94,7 +131,7 @@ class TestTable < Test::Unit::TestCase
 
     def test_to_s
       assert_equal(<<-PRETTY_PRINT, @record_batch.to_s)
-visible: [true, false, true, false, true, false]
+visible: [true, false, true, false, true]
 valid: [false, true, false, true, false]
       PRETTY_PRINT
     end

c_glib/test/test-unique.rb

@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestUnique < Test::Unit::TestCase
+  include Helper::Buildable
+  include Helper::Omittable
+
+  def test_int32
+    assert_equal(build_int32_array([1, 3, -1, -3]),
+                 build_int32_array([1, 3, 1, -1, -3, -1]).unique)
+  end
+
+  def test_string
+    assert_equal(build_string_array(["Ruby", "Python"]),
+                 build_string_array(["Ruby", "Python", "Ruby"]).unique)
+  end
+end

ci/msvc-build.bat

@@ -17,6 +17,38 @@
 
 @echo on
 
+if "%JOB%" == "Static_Crt_Build" (
+  mkdir cpp\build-debug
+  pushd cpp\build-debug
+
+  cmake -G "%GENERATOR%" ^
+        -DARROW_USE_STATIC_CRT=ON ^
+        -DARROW_BOOST_USE_SHARED=OFF ^
+        -DCMAKE_BUILD_TYPE=Debug ^
+        -DARROW_CXXFLAGS="/MP" ^
+        ..  || exit /B
+
+  cmake --build . --config Debug || exit /B
+  popd
+
+  mkdir cpp\build-release
+  pushd cpp\build-release
+
+  cmake -G "%GENERATOR%" ^
+        -DARROW_USE_STATIC_CRT=ON ^
+        -DARROW_BOOST_USE_SHARED=OFF ^
+        -DCMAKE_BUILD_TYPE=Release ^
+        -DARROW_CXXFLAGS="/WX /MP" ^
+        ..  || exit /B
+
+  cmake --build . --config Release || exit /B
+  ctest -VV  || exit /B
+  popd
+
+  @rem Finish Static_Crt_Build build successfully
+  exit /B 0
+)
+
 if "%JOB%" == "Build_Debug" (
   mkdir cpp\build-debug
   pushd cpp\build-debug

ci/travis_before_script_c_glib.sh

@@ -97,13 +97,7 @@ if [ $BUILD_SYSTEM = "autotools" ]; then
   CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS CXXFLAGS=-DARROW_NO_DEPRECATED_API"
 
   ./configure $CONFIGURE_OPTIONS
-
-  if [ "$TRAVIS_OS_NAME" = "osx" ]; then
-    ARCHFLAGS="-arch x86_64" make -j4
-  else
-    make -j4
-  fi
-
+  make -j4
   make install
 else
   MESON_OPTIONS="--prefix=$ARROW_C_GLIB_INSTALL"

cpp/CMakeLists.txt

@@ -210,6 +210,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
       "Lz4 static lib suffix used on Windows with MSVC (default _static)")
     set(ZSTD_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING
       "ZStd static lib suffix used on Windows with MSVC (default _static)")
+
+    option(ARROW_USE_STATIC_CRT
+      "Build Arrow with statically linked CRT"
+      OFF)
   endif()
 endif()
 

cpp/build-support/lz4_msbuild_gl_runtimelibrary_params.patch

@@ -0,0 +1,601 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+diff --git a/visual/VS2010/datagen/datagen.vcxproj b/visual/VS2010/datagen/datagen.vcxproj
+index aaf81ad..096741a 100644
+--- a/visual/VS2010/datagen/datagen.vcxproj
++++ b/visual/VS2010/datagen/datagen.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -113,6 +117,46 @@
+       <GenerateDebugInformation>true</GenerateDebugInformation>
+     </Link>
+   </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+     <ClCompile>
+       <WarningLevel>Level4</WarningLevel>
+diff --git a/visual/VS2010/frametest/frametest.vcxproj b/visual/VS2010/frametest/frametest.vcxproj
+index 76d12c9..34ca686 100644
+--- a/visual/VS2010/frametest/frametest.vcxproj
++++ b/visual/VS2010/frametest/frametest.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -113,6 +117,46 @@
+       <GenerateDebugInformation>true</GenerateDebugInformation>
+     </Link>
+   </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+     <ClCompile>
+       <WarningLevel>Level4</WarningLevel>
+diff --git a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj
+index c10552a..c1ca231 100644
+--- a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj
++++ b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -117,6 +121,46 @@
+       <AdditionalDependencies>liblz4.lib;%(AdditionalDependencies)</AdditionalDependencies>
+     </Link>
+   </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+     <ClCompile>
+       <WarningLevel>Level4</WarningLevel>
+diff --git a/visual/VS2010/fullbench/fullbench.vcxproj b/visual/VS2010/fullbench/fullbench.vcxproj
+index e2d95c9..613ff3c 100644
+--- a/visual/VS2010/fullbench/fullbench.vcxproj
++++ b/visual/VS2010/fullbench/fullbench.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -113,6 +117,46 @@
+       <GenerateDebugInformation>true</GenerateDebugInformation>
+     </Link>
+   </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+     <ClCompile>
+       <WarningLevel>Level4</WarningLevel>
+diff --git a/visual/VS2010/fuzzer/fuzzer.vcxproj b/visual/VS2010/fuzzer/fuzzer.vcxproj
+index 85d6c9b..be8b9a1 100644
+--- a/visual/VS2010/fuzzer/fuzzer.vcxproj
++++ b/visual/VS2010/fuzzer/fuzzer.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -113,6 +117,46 @@
+       <GenerateDebugInformation>true</GenerateDebugInformation>
+     </Link>
+   </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+     <ClCompile>
+       <WarningLevel>Level4</WarningLevel>
+diff --git a/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj b/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj
+index 389f13c..e30e667 100644
+--- a/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj
++++ b/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj
+@@ -40,15 +40,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>DynamicLibrary</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>DynamicLibrary</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -116,6 +120,46 @@
+       <GenerateDebugInformation>true</GenerateDebugInformation>
+     </Link>
+   </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+     <ClCompile>
+       <WarningLevel>Level4</WarningLevel>
+diff --git a/visual/VS2010/liblz4/liblz4.vcxproj b/visual/VS2010/liblz4/liblz4.vcxproj
+index a0b8000..38d2ce2 100644
+--- a/visual/VS2010/liblz4/liblz4.vcxproj
++++ b/visual/VS2010/liblz4/liblz4.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>StaticLibrary</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>StaticLibrary</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -115,6 +119,46 @@
+       <GenerateDebugInformation>true</GenerateDebugInformation>
+     </Link>
+   </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+     <ClCompile>
+       <WarningLevel>Level4</WarningLevel>
+diff --git a/visual/VS2010/lz4/lz4.vcxproj b/visual/VS2010/lz4/lz4.vcxproj
+index 693e121..9eb005b 100644
+--- a/visual/VS2010/lz4/lz4.vcxproj
++++ b/visual/VS2010/lz4/lz4.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -115,6 +119,46 @@
+       <AdditionalDependencies>setargv.obj;%(AdditionalDependencies)</AdditionalDependencies>
+     </Link>
+   </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+     <ClCompile>
+       <WarningLevel>Level4</WarningLevel>

cpp/build-support/lz4_msbuild_wholeprogramoptimization_param.patch

@@ -1,225 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-diff --git a/visual/VS2010/datagen/datagen.vcxproj b/visual/VS2010/datagen/datagen.vcxproj
-index aaf81ad..67b716f 100644
---- a/visual/VS2010/datagen/datagen.vcxproj
-+++ b/visual/VS2010/datagen/datagen.vcxproj
-@@ -39,15 +39,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>
-diff --git a/visual/VS2010/frametest/frametest.vcxproj b/visual/VS2010/frametest/frametest.vcxproj
-index 76d12c9..723571d 100644
---- a/visual/VS2010/frametest/frametest.vcxproj
-+++ b/visual/VS2010/frametest/frametest.vcxproj
-@@ -39,15 +39,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>
-diff --git a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj
-index c10552a..0c8f293 100644
---- a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj
-+++ b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj
-@@ -39,15 +39,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>
-diff --git a/visual/VS2010/fullbench/fullbench.vcxproj b/visual/VS2010/fullbench/fullbench.vcxproj
-index e2d95c9..4cd88d0 100644
---- a/visual/VS2010/fullbench/fullbench.vcxproj
-+++ b/visual/VS2010/fullbench/fullbench.vcxproj
-@@ -39,15 +39,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>
-diff --git a/visual/VS2010/fuzzer/fuzzer.vcxproj b/visual/VS2010/fuzzer/fuzzer.vcxproj
-index 85d6c9b..3ddc77d 100644
---- a/visual/VS2010/fuzzer/fuzzer.vcxproj
-+++ b/visual/VS2010/fuzzer/fuzzer.vcxproj
-@@ -39,15 +39,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>
-diff --git a/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj b/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj
-index 389f13c..038a4d2 100644
---- a/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj
-+++ b/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj
-@@ -40,15 +40,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>DynamicLibrary</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>DynamicLibrary</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>
-diff --git a/visual/VS2010/liblz4/liblz4.vcxproj b/visual/VS2010/liblz4/liblz4.vcxproj
-index a0b8000..9aad8c2 100644
---- a/visual/VS2010/liblz4/liblz4.vcxproj
-+++ b/visual/VS2010/liblz4/liblz4.vcxproj
-@@ -39,15 +39,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>StaticLibrary</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>StaticLibrary</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>
-diff --git a/visual/VS2010/lz4/lz4.vcxproj b/visual/VS2010/lz4/lz4.vcxproj
-index 693e121..7e63f1e 100644
---- a/visual/VS2010/lz4/lz4.vcxproj
-+++ b/visual/VS2010/lz4/lz4.vcxproj
-@@ -39,15 +39,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>

cpp/build-support/zstd_msbuild_gl_runtimelibrary_params.patch

@@ -0,0 +1,528 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+diff --git a/build/VS2010/datagen/datagen.vcxproj b/build/VS2010/datagen/datagen.vcxproj
+index bd8a213d..691d39fe 100644
+--- a/build/VS2010/datagen/datagen.vcxproj
++++ b/build/VS2010/datagen/datagen.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -84,6 +88,46 @@
+     <RunCodeAnalysis>false</RunCodeAnalysis>
+     <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+   </PropertyGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+     <ClCompile>
+       <PrecompiledHeader>
+diff --git a/build/VS2010/fullbench-dll/fullbench-dll.vcxproj b/build/VS2010/fullbench-dll/fullbench-dll.vcxproj
+index e697318e..a5720189 100644
+--- a/build/VS2010/fullbench-dll/fullbench-dll.vcxproj
++++ b/build/VS2010/fullbench-dll/fullbench-dll.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -84,6 +88,46 @@
+     <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+     <RunCodeAnalysis>false</RunCodeAnalysis>
+   </PropertyGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+     <ClCompile>
+       <PrecompiledHeader>
+diff --git a/build/VS2010/fullbench/fullbench.vcxproj b/build/VS2010/fullbench/fullbench.vcxproj
+index 2bff4ca4..d64fac81 100644
+--- a/build/VS2010/fullbench/fullbench.vcxproj
++++ b/build/VS2010/fullbench/fullbench.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -84,6 +88,46 @@
+     <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+     <RunCodeAnalysis>false</RunCodeAnalysis>
+   </PropertyGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+     <ClCompile>
+       <PrecompiledHeader>
+diff --git a/build/VS2010/fuzzer/fuzzer.vcxproj b/build/VS2010/fuzzer/fuzzer.vcxproj
+index 12a4b931..1ffc5941 100644
+--- a/build/VS2010/fuzzer/fuzzer.vcxproj
++++ b/build/VS2010/fuzzer/fuzzer.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -84,6 +88,46 @@
+     <RunCodeAnalysis>false</RunCodeAnalysis>
+     <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress;$(UniversalCRT_IncludePath);</IncludePath>
+   </PropertyGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+     <ClCompile>
+       <PrecompiledHeader>
+diff --git a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
+index 364b3bea..1f34e956 100644
+--- a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
++++ b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
+@@ -94,15 +94,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>DynamicLibrary</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>DynamicLibrary</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -143,6 +147,46 @@
+     <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
+     <RunCodeAnalysis>false</RunCodeAnalysis>
+   </PropertyGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+     <ClCompile>
+       <PrecompiledHeader>
+diff --git a/build/VS2010/libzstd/libzstd.vcxproj b/build/VS2010/libzstd/libzstd.vcxproj
+index 6087d737..0d2a6fe0 100644
+--- a/build/VS2010/libzstd/libzstd.vcxproj
++++ b/build/VS2010/libzstd/libzstd.vcxproj
+@@ -91,15 +91,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>StaticLibrary</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>StaticLibrary</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -140,6 +144,46 @@
+     <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
+     <RunCodeAnalysis>false</RunCodeAnalysis>
+   </PropertyGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+     <ClCompile>
+       <PrecompiledHeader>
+diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj
+index 438dc617..3fb5bba8 100644
+--- a/build/VS2010/zstd/zstd.vcxproj
++++ b/build/VS2010/zstd/zstd.vcxproj
+@@ -100,15 +100,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -149,6 +153,46 @@
+     <RunCodeAnalysis>false</RunCodeAnalysis>
+     <LibraryPath>$(LibraryPath);</LibraryPath>
+   </PropertyGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+     <ClCompile>
+       <PrecompiledHeader>

cpp/build-support/zstd_msbuild_wholeprogramoptimization_param.patch

@@ -1,199 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-diff --git a/build/VS2010/datagen/datagen.vcxproj b/build/VS2010/datagen/datagen.vcxproj
-index bd8a213..8e4dc89 100644
---- a/build/VS2010/datagen/datagen.vcxproj
-+++ b/build/VS2010/datagen/datagen.vcxproj
-@@ -39,15 +39,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>MultiByte</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>MultiByte</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>
-diff --git a/build/VS2010/fullbench-dll/fullbench-dll.vcxproj b/build/VS2010/fullbench-dll/fullbench-dll.vcxproj
-index e697318..82cd4ab 100644
---- a/build/VS2010/fullbench-dll/fullbench-dll.vcxproj
-+++ b/build/VS2010/fullbench-dll/fullbench-dll.vcxproj
-@@ -39,15 +39,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>MultiByte</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>MultiByte</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>
-diff --git a/build/VS2010/fullbench/fullbench.vcxproj b/build/VS2010/fullbench/fullbench.vcxproj
-index 2bff4ca..ced4047 100644
---- a/build/VS2010/fullbench/fullbench.vcxproj
-+++ b/build/VS2010/fullbench/fullbench.vcxproj
-@@ -39,15 +39,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>MultiByte</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>MultiByte</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>
-diff --git a/build/VS2010/fuzzer/fuzzer.vcxproj b/build/VS2010/fuzzer/fuzzer.vcxproj
-index 12a4b93..227efd1 100644
---- a/build/VS2010/fuzzer/fuzzer.vcxproj
-+++ b/build/VS2010/fuzzer/fuzzer.vcxproj
-@@ -39,15 +39,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>MultiByte</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>MultiByte</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>
-diff --git a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
-index 364b3be..b227320 100644
---- a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
-+++ b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
-@@ -94,15 +94,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>DynamicLibrary</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>MultiByte</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>DynamicLibrary</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>MultiByte</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>
-diff --git a/build/VS2010/libzstd/libzstd.vcxproj b/build/VS2010/libzstd/libzstd.vcxproj
-index 6087d73..51a0572 100644
---- a/build/VS2010/libzstd/libzstd.vcxproj
-+++ b/build/VS2010/libzstd/libzstd.vcxproj
-@@ -91,15 +91,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>StaticLibrary</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>MultiByte</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>StaticLibrary</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>MultiByte</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>
-diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj
-index 438dc61..834ae01 100644
---- a/build/VS2010/zstd/zstd.vcxproj
-+++ b/build/VS2010/zstd/zstd.vcxproj
-@@ -100,15 +100,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>MultiByte</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>MultiByte</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>

cpp/cmake_modules/SetupCxxFlags.cmake

@@ -47,6 +47,15 @@ if (MSVC)
     # Set desired warning level (e.g. set /W4 for more warnings)
     set(CXX_COMMON_FLAGS "/W3")
   endif()
+
+  if (ARROW_USE_STATIC_CRT)
+    foreach (c_flag CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_RELEASE CMAKE_CXX_FLAGS_DEBUG
+                    CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO
+                    CMAKE_C_FLAGS CMAKE_C_FLAGS_RELEASE CMAKE_C_FLAGS_DEBUG
+                    CMAKE_C_FLAGS_MINSIZEREL CMAKE_C_FLAGS_RELWITHDEBINFO)
+      string(REPLACE "/MD" "-MT" ${c_flag} "${${c_flag}}")
+    endforeach()
+  endif()
 else()
   # Common flags set below with warning level
   set(CXX_COMMON_FLAGS "")
@@ -68,9 +77,6 @@ if ("${UPPERCASE_BUILD_WARNING_LEVEL}" STREQUAL "CHECKIN")
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /W3")
     # Treat all compiler warnings as errors
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /WX")
-
-    # MSVC version of -Wno-deprecated
-    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /wd4996")
   elseif ("${COMPILER_FAMILY}" STREQUAL "clang")
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Weverything -Wno-c++98-compat \
 -Wno-c++98-compat-pedantic -Wno-deprecated -Wno-weak-vtables -Wno-padded \
@@ -118,9 +124,6 @@ elseif ("${UPPERCASE_BUILD_WARNING_LEVEL}" STREQUAL "EVERYTHING")
     # /wdnnnn disables a warning where "nnnn" is a warning number
     # Treat all compiler warnings as errors
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS}  /WX")
-
-    # MSVC version of -Wno-deprecated
-    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /wd4996")
   elseif ("${COMPILER_FAMILY}" STREQUAL "clang")
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Weverything -Wno-c++98-compat -Wno-c++98-compat-pedantic")
     # Treat all compiler warnings as errors
@@ -140,9 +143,6 @@ else()
     # /wdnnnn disables a warning where "nnnn" is a warning number
     string(REPLACE "/W3" "" CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS}")
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /W3")
-
-    # MSVC version of -Wno-deprecated
-    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /wd4996")
   elseif ("${COMPILER_FAMILY}" STREQUAL "clang")
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall")
   elseif ("${COMPILER_FAMILY}" STREQUAL "gcc")

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -130,6 +130,9 @@ endif()
 
 set(Boost_DEBUG TRUE)
 set(Boost_USE_MULTITHREADED ON)
+if (MSVC AND ARROW_USE_STATIC_CRT)
+  set(Boost_USE_STATIC_RUNTIME ON)
+endif()
 set(Boost_ADDITIONAL_VERSIONS
   "1.65.0" "1.65"
   "1.64.0" "1.64"
@@ -204,11 +207,11 @@ else()
     else()
       find_package(Boost COMPONENTS system filesystem REQUIRED)
       if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
-	set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
-	set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
+        set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
+        set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
       else()
-	set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
-	set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
+        set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
+        set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
       endif()
       set(BOOST_SYSTEM_LIBRARY boost_system_shared)
       set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_shared)
@@ -222,11 +225,11 @@ else()
     else()
       find_package(Boost COMPONENTS system filesystem REQUIRED)
       if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
-	set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
-	set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
+        set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
+        set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
       else()
-	set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
-	set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
+        set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
+        set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
       endif()
       set(BOOST_SYSTEM_LIBRARY boost_system_static)
       set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static)
@@ -272,8 +275,10 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS)
     set(GTEST_VENDORED 1)
     set(GTEST_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
                          -DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX}
-                         -Dgtest_force_shared_crt=ON
                          -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS})
+    if (MSVC AND NOT ARROW_USE_STATIC_CRT)
+      set(GTEST_CMAKE_ARGS ${GTEST_CMAKE_ARGS} -Dgtest_force_shared_crt=ON)
+    endif()
 
     ExternalProject_Add(googletest_ep
       URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
@@ -320,6 +325,8 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS)
                           -DBUILD_TESTING=OFF
                           -BUILD_CONFIG_TESTS=OFF
                           -DINSTALL_HEADERS=ON
+                          -DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}
+                          -DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS}
                           -DCMAKE_CXX_FLAGS=${GFLAGS_CMAKE_CXX_FLAGS})
 
     ExternalProject_Add(gflags_ep
@@ -434,6 +441,8 @@ if (ARROW_IPC)
       "-DCMAKE_CXX_FLAGS=${FLATBUFFERS_CMAKE_CXX_FLAGS}"
       "-DCMAKE_INSTALL_PREFIX:PATH=${FLATBUFFERS_PREFIX}"
       "-DFLATBUFFERS_BUILD_TESTS=OFF"
+      "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}"
+      "-DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS}"
       ${EP_LOG_OPTIONS})
 
     set(FLATBUFFERS_INCLUDE_DIR "${FLATBUFFERS_PREFIX}/include")
@@ -550,6 +559,8 @@ if (ARROW_WITH_ZLIB)
     set(ZLIB_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
                         -DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX}
                         -DCMAKE_C_FLAGS=${EP_C_FLAGS}
+                        -DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}
+                        -DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS}
                         -DBUILD_SHARED_LIBS=OFF)
 
     ExternalProject_Add(zlib_ep
@@ -599,7 +610,9 @@ if (ARROW_WITH_SNAPPY)
     if (MSVC)
       set(SNAPPY_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
                             "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}"
-                            "-DCMAKE_C_FLAGS=${EX_C_FLAGS}"
+                            "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}"
+                            "-DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS}"
+                            "-DCMAKE_C_FLAGS=${EP_C_FLAGS}"
                             "-DCMAKE_INSTALL_PREFIX=${SNAPPY_PREFIX}")
       set(SNAPPY_UPDATE_COMMAND ${CMAKE_COMMAND} -E copy
                         ${CMAKE_SOURCE_DIR}/cmake_modules/SnappyCMakeLists.txt
@@ -659,7 +672,9 @@ if (ARROW_WITH_BROTLI)
     set(BROTLI_STATIC_LIBRARY_COMMON "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon${CMAKE_STATIC_LIBRARY_SUFFIX}")
     set(BROTLI_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
                           "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}"
-                          "-DCMAKE_C_FLAGS=${EX_C_FLAGS}"
+                          "-DCMAKE_C_FLAGS=${EP_C_FLAGS}"
+                          "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}"
+                          "-DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS}"
                           -DCMAKE_INSTALL_PREFIX=${BROTLI_PREFIX}
                           -DCMAKE_INSTALL_LIBDIR=lib/${CMAKE_LIBRARY_ARCHITECTURE}
                           -DBUILD_SHARED_LIBS=OFF)
@@ -709,9 +724,17 @@ if (ARROW_WITH_LZ4)
     set(LZ4_INCLUDE_DIR "${LZ4_BUILD_DIR}/lib")
 
     if (MSVC)
+      if (ARROW_USE_STATIC_CRT)
+        if (${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG")
+          set(LZ4_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreadedDebug")
+        else()
+          set(LZ4_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreaded")
+        endif()
+      endif()
       set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/visual/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/liblz4_static.lib")
-      set(LZ4_BUILD_COMMAND BUILD_COMMAND msbuild.exe /m /p:Configuration=${CMAKE_BUILD_TYPE} /p:Platform=x64 /p:PlatformToolset=v140 /t:Build ${LZ4_BUILD_DIR}/visual/VS2010/lz4.sln)
-      set(LZ4_PATCH_COMMAND PATCH_COMMAND git --git-dir=. apply --verbose --whitespace=fix ${CMAKE_SOURCE_DIR}/build-support/lz4_msbuild_wholeprogramoptimization_param.patch)
+      set(LZ4_BUILD_COMMAND BUILD_COMMAND msbuild.exe /m /p:Configuration=${CMAKE_BUILD_TYPE} /p:Platform=x64 /p:PlatformToolset=v140
+                                          ${LZ4_RUNTIME_LIBRARY_LINKAGE} /t:Build ${LZ4_BUILD_DIR}/visual/VS2010/lz4.sln)
+      set(LZ4_PATCH_COMMAND PATCH_COMMAND git --git-dir=. apply --verbose --whitespace=fix ${CMAKE_SOURCE_DIR}/build-support/lz4_msbuild_gl_runtimelibrary_params.patch)
     else()
       set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/lib/liblz4.a")
       set(LZ4_BUILD_COMMAND BUILD_COMMAND ${CMAKE_SOURCE_DIR}/build-support/build-lz4-lib.sh)
@@ -753,9 +776,18 @@ if (ARROW_WITH_ZSTD)
     set(ZSTD_INCLUDE_DIR "${ZSTD_BUILD_DIR}/lib")
 
     if (MSVC)
+      if (ARROW_USE_STATIC_CRT)
+        if (${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG")
+          set(ZSTD_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreadedDebug")
+        else()
+          set(ZSTD_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreaded")
+        endif()
+      endif()
       set(ZSTD_STATIC_LIB "${ZSTD_BUILD_DIR}/build/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/libzstd_static.lib")
-      set(ZSTD_BUILD_COMMAND BUILD_COMMAND msbuild ${ZSTD_BUILD_DIR}/build/VS2010/zstd.sln /t:Build /v:minimal /p:Configuration=${CMAKE_BUILD_TYPE} /p:Platform=x64 /p:PlatformToolset=v140 /p:OutDir=${ZSTD_BUILD_DIR}/build/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/ /p:SolutionDir=${ZSTD_BUILD_DIR}/build/VS2010/ )
-      set(ZSTD_PATCH_COMMAND PATCH_COMMAND git --git-dir=. apply --verbose --whitespace=fix ${CMAKE_SOURCE_DIR}/build-support/zstd_msbuild_wholeprogramoptimization_param.patch)
+      set(ZSTD_BUILD_COMMAND BUILD_COMMAND msbuild ${ZSTD_BUILD_DIR}/build/VS2010/zstd.sln /t:Build /v:minimal /p:Configuration=${CMAKE_BUILD_TYPE}
+                             ${ZSTD_RUNTIME_LIBRARY_LINKAGE} /p:Platform=x64 /p:PlatformToolset=v140
+                             /p:OutDir=${ZSTD_BUILD_DIR}/build/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/ /p:SolutionDir=${ZSTD_BUILD_DIR}/build/VS2010/ )
+      set(ZSTD_PATCH_COMMAND PATCH_COMMAND git --git-dir=. apply --verbose --whitespace=fix ${CMAKE_SOURCE_DIR}/build-support/zstd_msbuild_gl_runtimelibrary_params.patch)
     else()
       set(ZSTD_STATIC_LIB "${ZSTD_BUILD_DIR}/lib/libzstd.a")
       set(ZSTD_BUILD_COMMAND BUILD_COMMAND ${CMAKE_SOURCE_DIR}/build-support/build-zstd-lib.sh)
@@ -802,7 +834,7 @@ if (ARROW_WITH_GRPC)
     set(GRPC_STATIC_LIBRARY_GRPCPP "${GRPC_BUILD_DIR}/${CMAKE_CFG_INTDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}grpc++${CMAKE_STATIC_LIBRARY_SUFFIX}")
     set(GRPC_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
                           "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}"
-                          "-DCMAKE_C_FLAGS=${EX_C_FLAGS}"
+                          "-DCMAKE_C_FLAGS=${EP_C_FLAGS}"
                           -DCMAKE_INSTALL_PREFIX=${GRPC_PREFIX}
                           -DBUILD_SHARED_LIBS=OFF)
 

cpp/src/arrow/CMakeLists.txt

@@ -38,6 +38,7 @@ set(ARROW_SRCS
   util/compression.cc
   util/cpu-info.cc
   util/decimal.cc
+  util/hash.cc
   util/key_value_metadata.cc
 )
 

cpp/src/arrow/array-test.cc

@@ -1619,6 +1619,353 @@ TEST_F(TestAdaptiveUIntBuilder, TestAppendVector) {
   ASSERT_TRUE(expected_->Equals(result_));
 }
 
+// ----------------------------------------------------------------------
+// Dictionary tests
+
+template <typename Type>
+class TestDictionaryBuilder : public TestBuilder {};
+
+typedef ::testing::Types<Int8Type, UInt8Type, Int16Type, UInt16Type, Int32Type,
+                         UInt32Type, Int64Type, UInt64Type, FloatType, DoubleType>
+    PrimitiveDictionaries;
+
+TYPED_TEST_CASE(TestDictionaryBuilder, PrimitiveDictionaries);
+
+TYPED_TEST(TestDictionaryBuilder, Basic) {
+  DictionaryBuilder<TypeParam> builder(default_memory_pool());
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(1)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(2)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(1)));
+
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+
+  // Build expected data
+  NumericBuilder<TypeParam> dict_builder;
+  ASSERT_OK(dict_builder.Append(static_cast<typename TypeParam::c_type>(1)));
+  ASSERT_OK(dict_builder.Append(static_cast<typename TypeParam::c_type>(2)));
+  std::shared_ptr<Array> dict_array;
+  ASSERT_OK(dict_builder.Finish(&dict_array));
+  auto dtype = std::make_shared<DictionaryType>(int8(), dict_array);
+
+  Int8Builder int_builder;
+  ASSERT_OK(int_builder.Append(0));
+  ASSERT_OK(int_builder.Append(1));
+  ASSERT_OK(int_builder.Append(0));
+  std::shared_ptr<Array> int_array;
+  ASSERT_OK(int_builder.Finish(&int_array));
+
+  DictionaryArray expected(dtype, int_array);
+  ASSERT_TRUE(expected.Equals(result));
+}
+
+TYPED_TEST(TestDictionaryBuilder, ArrayConversion) {
+  NumericBuilder<TypeParam> builder;
+  // DictionaryBuilder<TypeParam> builder;
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(1)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(2)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(1)));
+
+  std::shared_ptr<Array> intermediate_result;
+  ASSERT_OK(builder.Finish(&intermediate_result));
+  DictionaryBuilder<TypeParam> dictionary_builder(default_memory_pool());
+  ASSERT_OK(dictionary_builder.AppendArray(*intermediate_result));
+  std::shared_ptr<Array> result;
+  ASSERT_OK(dictionary_builder.Finish(&result));
+
+  // Build expected data
+  NumericBuilder<TypeParam> dict_builder;
+  ASSERT_OK(dict_builder.Append(static_cast<typename TypeParam::c_type>(1)));
+  ASSERT_OK(dict_builder.Append(static_cast<typename TypeParam::c_type>(2)));
+  std::shared_ptr<Array> dict_array;
+  ASSERT_OK(dict_builder.Finish(&dict_array));
+  auto dtype = std::make_shared<DictionaryType>(int8(), dict_array);
+
+  Int8Builder int_builder;
+  ASSERT_OK(int_builder.Append(0));
+  ASSERT_OK(int_builder.Append(1));
+  ASSERT_OK(int_builder.Append(0));
+  std::shared_ptr<Array> int_array;
+  ASSERT_OK(int_builder.Finish(&int_array));
+
+  DictionaryArray expected(dtype, int_array);
+  ASSERT_TRUE(expected.Equals(result));
+}
+
+TYPED_TEST(TestDictionaryBuilder, DoubleTableSize) {
+  using Scalar = typename TypeParam::c_type;
+  // Skip this test for (u)int8
+  if (sizeof(Scalar) > 1) {
+    // Build the dictionary Array
+    DictionaryBuilder<TypeParam> builder(default_memory_pool());
+    // Build expected data
+    NumericBuilder<TypeParam> dict_builder;
+    Int16Builder int_builder;
+
+    // Fill with 1024 different values
+    for (int64_t i = 0; i < 1024; i++) {
+      ASSERT_OK(builder.Append(static_cast<Scalar>(i)));
+      ASSERT_OK(dict_builder.Append(static_cast<Scalar>(i)));
+      ASSERT_OK(int_builder.Append(static_cast<uint16_t>(i)));
+    }
+    // Fill with an already existing value
+    for (int64_t i = 0; i < 1024; i++) {
+      ASSERT_OK(builder.Append(static_cast<Scalar>(1)));
+      ASSERT_OK(int_builder.Append(1));
+    }
+
+    // Finalize result
+    std::shared_ptr<Array> result;
+    ASSERT_OK(builder.Finish(&result));
+
+    // Finalize expected data
+    std::shared_ptr<Array> dict_array;
+    ASSERT_OK(dict_builder.Finish(&dict_array));
+    auto dtype = std::make_shared<DictionaryType>(int16(), dict_array);
+    std::shared_ptr<Array> int_array;
+    ASSERT_OK(int_builder.Finish(&int_array));
+
+    DictionaryArray expected(dtype, int_array);
+    ASSERT_TRUE(expected.Equals(result));
+  }
+}
+
+TEST(TestStringDictionaryBuilder, Basic) {
+  // Build the dictionary Array
+  StringDictionaryBuilder builder(default_memory_pool());
+  ASSERT_OK(builder.Append("test"));
+  ASSERT_OK(builder.Append("test2"));
+  ASSERT_OK(builder.Append("test"));
+
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+
+  // Build expected data
+  StringBuilder str_builder;
+  ASSERT_OK(str_builder.Append("test"));
+  ASSERT_OK(str_builder.Append("test2"));
+  std::shared_ptr<Array> str_array;
+  ASSERT_OK(str_builder.Finish(&str_array));
+  auto dtype = std::make_shared<DictionaryType>(int8(), str_array);
+
+  Int8Builder int_builder;
+  ASSERT_OK(int_builder.Append(0));
+  ASSERT_OK(int_builder.Append(1));
+  ASSERT_OK(int_builder.Append(0));
+  std::shared_ptr<Array> int_array;
+  ASSERT_OK(int_builder.Finish(&int_array));
+
+  DictionaryArray expected(dtype, int_array);
+  ASSERT_TRUE(expected.Equals(result));
+}
+
+TEST(TestStringDictionaryBuilder, DoubleTableSize) {
+  // Build the dictionary Array
+  StringDictionaryBuilder builder(default_memory_pool());
+  // Build expected data
+  StringBuilder str_builder;
+  Int16Builder int_builder;
+
+  // Fill with 1024 different values
+  for (int64_t i = 0; i < 1024; i++) {
+    std::stringstream ss;
+    ss << "test" << i;
+    ASSERT_OK(builder.Append(ss.str()));
+    ASSERT_OK(str_builder.Append(ss.str()));
+    ASSERT_OK(int_builder.Append(static_cast<uint16_t>(i)));
+  }
+  // Fill with an already existing value
+  for (int64_t i = 0; i < 1024; i++) {
+    ASSERT_OK(builder.Append("test1"));
+    ASSERT_OK(int_builder.Append(1));
+  }
+
+  // Finalize result
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+
+  // Finalize expected data
+  std::shared_ptr<Array> str_array;
+  ASSERT_OK(str_builder.Finish(&str_array));
+  auto dtype = std::make_shared<DictionaryType>(int16(), str_array);
+  std::shared_ptr<Array> int_array;
+  ASSERT_OK(int_builder.Finish(&int_array));
+
+  DictionaryArray expected(dtype, int_array);
+  ASSERT_TRUE(expected.Equals(result));
+}
+
+TEST(TestFixedSizeBinaryDictionaryBuilder, Basic) {
+  // Build the dictionary Array
+  DictionaryBuilder<FixedSizeBinaryType> builder(arrow::fixed_size_binary(4),
+                                                 default_memory_pool());
+  std::vector<uint8_t> test{12, 12, 11, 12};
+  std::vector<uint8_t> test2{12, 12, 11, 11};
+  ASSERT_OK(builder.Append(test.data()));
+  ASSERT_OK(builder.Append(test2.data()));
+  ASSERT_OK(builder.Append(test.data()));
+
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+
+  // Build expected data
+  FixedSizeBinaryBuilder fsb_builder(arrow::fixed_size_binary(4));
+  ASSERT_OK(fsb_builder.Append(test.data()));
+  ASSERT_OK(fsb_builder.Append(test2.data()));
+  std::shared_ptr<Array> fsb_array;
+  ASSERT_OK(fsb_builder.Finish(&fsb_array));
+  auto dtype = std::make_shared<DictionaryType>(int8(), fsb_array);
+
+  Int8Builder int_builder;
+  ASSERT_OK(int_builder.Append(0));
+  ASSERT_OK(int_builder.Append(1));
+  ASSERT_OK(int_builder.Append(0));
+  std::shared_ptr<Array> int_array;
+  ASSERT_OK(int_builder.Finish(&int_array));
+
+  DictionaryArray expected(dtype, int_array);
+  ASSERT_TRUE(expected.Equals(result));
+}
+
+TEST(TestFixedSizeBinaryDictionaryBuilder, DoubleTableSize) {
+  // Build the dictionary Array
+  DictionaryBuilder<FixedSizeBinaryType> builder(arrow::fixed_size_binary(4),
+                                                 default_memory_pool());
+  // Build expected data
+  FixedSizeBinaryBuilder fsb_builder(arrow::fixed_size_binary(4));
+  Int16Builder int_builder;
+
+  // Fill with 1024 different values
+  for (int64_t i = 0; i < 1024; i++) {
+    std::vector<uint8_t> value{12, 12, static_cast<uint8_t>(i / 128),
+                               static_cast<uint8_t>(i % 128)};
+    ASSERT_OK(builder.Append(value.data()));
+    ASSERT_OK(fsb_builder.Append(value.data()));
+    ASSERT_OK(int_builder.Append(static_cast<uint16_t>(i)));
+  }
+  // Fill with an already existing value
+  std::vector<uint8_t> known_value{12, 12, 0, 1};
+  for (int64_t i = 0; i < 1024; i++) {
+    ASSERT_OK(builder.Append(known_value.data()));
+    ASSERT_OK(int_builder.Append(1));
+  }
+
+  // Finalize result
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+
+  // Finalize expected data
+  std::shared_ptr<Array> fsb_array;
+  ASSERT_OK(fsb_builder.Finish(&fsb_array));
+  auto dtype = std::make_shared<DictionaryType>(int16(), fsb_array);
+  std::shared_ptr<Array> int_array;
+  ASSERT_OK(int_builder.Finish(&int_array));
+
+  DictionaryArray expected(dtype, int_array);
+  ASSERT_TRUE(expected.Equals(result));
+}
+
+TEST(TestFixedSizeBinaryDictionaryBuilder, InvalidTypeAppend) {
+  // Build the dictionary Array
+  DictionaryBuilder<FixedSizeBinaryType> builder(arrow::fixed_size_binary(4),
+                                                 default_memory_pool());
+  // Build an array with different byte width
+  FixedSizeBinaryBuilder fsb_builder(arrow::fixed_size_binary(5));
+  std::vector<uint8_t> value{100, 1, 1, 1, 1};
+  ASSERT_OK(fsb_builder.Append(value.data()));
+  std::shared_ptr<Array> fsb_array;
+  ASSERT_OK(fsb_builder.Finish(&fsb_array));
+
+  ASSERT_RAISES(Invalid, builder.AppendArray(*fsb_array));
+}
+
+TEST(TestDecimalDictionaryBuilder, Basic) {
+  // Build the dictionary Array
+  const auto& decimal_type = arrow::decimal(2, 0);
+  DictionaryBuilder<FixedSizeBinaryType> builder(decimal_type, default_memory_pool());
+
+  // Test data
+  std::vector<Decimal128> test{12, 12, 11, 12};
+  for (const auto& value : test) {
+    ASSERT_OK(builder.Append(value.ToBytes().data()));
+  }
+
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+
+  // Build expected data
+  FixedSizeBinaryBuilder decimal_builder(decimal_type);
+  ASSERT_OK(decimal_builder.Append(Decimal128(12).ToBytes()));
+  ASSERT_OK(decimal_builder.Append(Decimal128(11).ToBytes()));
+
+  std::shared_ptr<Array> decimal_array;
+  ASSERT_OK(decimal_builder.Finish(&decimal_array));
+  auto dtype = arrow::dictionary(int8(), decimal_array);
+
+  Int8Builder int_builder;
+  ASSERT_OK(int_builder.Append({0, 0, 1, 0}));
+  std::shared_ptr<Array> int_array;
+  ASSERT_OK(int_builder.Finish(&int_array));
+
+  DictionaryArray expected(dtype, int_array);
+  ASSERT_TRUE(expected.Equals(result));
+}
+
+TEST(TestDecimalDictionaryBuilder, DoubleTableSize) {
+  const auto& decimal_type = arrow::decimal(21, 0);
+
+  // Build the dictionary Array
+  DictionaryBuilder<FixedSizeBinaryType> builder(decimal_type, default_memory_pool());
+
+  // Build expected data
+  FixedSizeBinaryBuilder fsb_builder(decimal_type);
+  Int16Builder int_builder;
+
+  // Fill with 1024 different values
+  for (int64_t i = 0; i < 1024; i++) {
+    const uint8_t bytes[] = {0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             12,
+                             12,
+                             static_cast<uint8_t>(i / 128),
+                             static_cast<uint8_t>(i % 128)};
+    ASSERT_OK(builder.Append(bytes));
+    ASSERT_OK(fsb_builder.Append(bytes));
+    ASSERT_OK(int_builder.Append(static_cast<uint16_t>(i)));
+  }
+  // Fill with an already existing value
+  const uint8_t known_value[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 12, 0, 1};
+  for (int64_t i = 0; i < 1024; i++) {
+    ASSERT_OK(builder.Append(known_value));
+    ASSERT_OK(int_builder.Append(1));
+  }
+
+  // Finalize result
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+
+  // Finalize expected data
+  std::shared_ptr<Array> fsb_array;
+  ASSERT_OK(fsb_builder.Finish(&fsb_array));
+
+  auto dtype = std::make_shared<DictionaryType>(int16(), fsb_array);
+  std::shared_ptr<Array> int_array;
+  ASSERT_OK(int_builder.Finish(&int_array));
+
+  DictionaryArray expected(dtype, int_array);
+  ASSERT_TRUE(expected.Equals(result));
+}
+
 // ----------------------------------------------------------------------
 // List tests
 

cpp/src/arrow/array.cc

@@ -36,6 +36,20 @@
 
 namespace arrow {
 
+std::shared_ptr<ArrayData> ArrayData::Make(const std::shared_ptr<DataType>& type,
+                                           int64_t length,
+                                           std::vector<std::shared_ptr<Buffer>>&& buffers,
+                                           int64_t null_count, int64_t offset) {
+  return std::make_shared<ArrayData>(type, length, std::move(buffers), null_count,
+                                     offset);
+}
+
+std::shared_ptr<ArrayData> ArrayData::Make(const std::shared_ptr<DataType>& type,
+                                           int64_t length, int64_t null_count,
+                                           int64_t offset) {
+  return std::make_shared<ArrayData>(type, length, null_count, offset);
+}
+
 // ----------------------------------------------------------------------
 // Base array class
 
@@ -89,7 +103,7 @@ static inline std::shared_ptr<ArrayData> SliceData(const ArrayData& data, int64_
   length = std::min(data.length - offset, length);
   offset += data.offset;
 
-  auto new_data = data.ShallowCopy();
+  auto new_data = data.Copy();
   new_data->length = length;
   new_data->offset = offset;
   new_data->null_count = data.null_count != 0 ? kUnknownNullCount : 0;
@@ -112,8 +126,7 @@ std::string Array::ToString() const {
 }
 
 NullArray::NullArray(int64_t length) {
-  BufferVector buffers = {nullptr};
-  SetData(std::make_shared<ArrayData>(null(), length, std::move(buffers), length));
+  SetData(ArrayData::Make(null(), length, {nullptr}, length));
 }
 
 // ----------------------------------------------------------------------
@@ -123,9 +136,7 @@ PrimitiveArray::PrimitiveArray(const std::shared_ptr<DataType>& type, int64_t le
                                const std::shared_ptr<Buffer>& data,
                                const std::shared_ptr<Buffer>& null_bitmap,
                                int64_t null_count, int64_t offset) {
-  BufferVector buffers = {null_bitmap, data};
-  SetData(
-      std::make_shared<ArrayData>(type, length, std::move(buffers), null_count, offset));
+  SetData(ArrayData::Make(type, length, {null_bitmap, data}, null_count, offset));
 }
 
 const uint8_t* PrimitiveArray::raw_values() const {
@@ -165,9 +176,8 @@ ListArray::ListArray(const std::shared_ptr<DataType>& type, int64_t length,
                      const std::shared_ptr<Array>& values,
                      const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count,
                      int64_t offset) {
-  BufferVector buffers = {null_bitmap, value_offsets};
   auto internal_data =
-      std::make_shared<ArrayData>(type, length, std::move(buffers), null_count, offset);
+      ArrayData::Make(type, length, {null_bitmap, value_offsets}, null_count, offset);
   internal_data->child_data.emplace_back(values->data());
   SetData(internal_data);
 }
@@ -219,9 +229,8 @@ Status ListArray::FromArrays(const Array& offsets, const Array& values, MemoryPo
   }
 
   auto list_type = list(values.type());
-  auto internal_data =
-      std::make_shared<ArrayData>(list_type, num_offsets - 1, std::move(buffers),
-                                  offsets.null_count(), offsets.offset());
+  auto internal_data = ArrayData::Make(list_type, num_offsets - 1, std::move(buffers),
+                                       offsets.null_count(), offsets.offset());
   internal_data->child_data.push_back(values.data());
 
   *out = std::make_shared<ListArray>(internal_data);
@@ -276,9 +285,8 @@ BinaryArray::BinaryArray(const std::shared_ptr<DataType>& type, int64_t length,
                          const std::shared_ptr<Buffer>& data,
                          const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count,
                          int64_t offset) {
-  BufferVector buffers = {null_bitmap, value_offsets, data};
-  SetData(
-      std::make_shared<ArrayData>(type, length, std::move(buffers), null_count, offset));
+  SetData(ArrayData::Make(type, length, {null_bitmap, value_offsets, data}, null_count,
+                          offset));
 }
 
 StringArray::StringArray(const std::shared_ptr<ArrayData>& data) {
@@ -338,9 +346,7 @@ StructArray::StructArray(const std::shared_ptr<DataType>& type, int64_t length,
                          const std::vector<std::shared_ptr<Array>>& children,
                          std::shared_ptr<Buffer> null_bitmap, int64_t null_count,
                          int64_t offset) {
-  BufferVector buffers = {null_bitmap};
-  SetData(
-      std::make_shared<ArrayData>(type, length, std::move(buffers), null_count, offset));
+  SetData(ArrayData::Make(type, length, {null_bitmap}, null_count, offset));
   for (const auto& child : children) {
     data_->child_data.push_back(child->data());
   }
@@ -384,9 +390,8 @@ UnionArray::UnionArray(const std::shared_ptr<DataType>& type, int64_t length,
                        const std::shared_ptr<Buffer>& value_offsets,
                        const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count,
                        int64_t offset) {
-  BufferVector buffers = {null_bitmap, type_ids, value_offsets};
-  auto internal_data =
-      std::make_shared<ArrayData>(type, length, std::move(buffers), null_count, offset);
+  auto internal_data = ArrayData::Make(
+      type, length, {null_bitmap, type_ids, value_offsets}, null_count, offset);
   for (const auto& child : children) {
     internal_data->child_data.push_back(child->data());
   }
@@ -416,9 +421,8 @@ Status UnionArray::MakeDense(const Array& type_ids, const Array& value_offsets,
                           static_cast<const UInt8Array&>(type_ids).values(),
                           static_cast<const Int32Array&>(value_offsets).values()};
   auto union_type = union_(children, UnionMode::DENSE);
-  auto internal_data =
-      std::make_shared<ArrayData>(union_type, type_ids.length(), std::move(buffers),
-                                  type_ids.null_count(), type_ids.offset());
+  auto internal_data = ArrayData::Make(union_type, type_ids.length(), std::move(buffers),
+                                       type_ids.null_count(), type_ids.offset());
   for (const auto& child : children) {
     internal_data->child_data.push_back(child->data());
   }
@@ -435,9 +439,8 @@ Status UnionArray::MakeSparse(const Array& type_ids,
   BufferVector buffers = {type_ids.null_bitmap(),
                           static_cast<const UInt8Array&>(type_ids).values(), nullptr};
   auto union_type = union_(children, UnionMode::SPARSE);
-  auto internal_data =
-      std::make_shared<ArrayData>(union_type, type_ids.length(), std::move(buffers),
-                                  type_ids.null_count(), type_ids.offset());
+  auto internal_data = ArrayData::Make(union_type, type_ids.length(), std::move(buffers),
+                                       type_ids.null_count(), type_ids.offset());
   for (const auto& child : children) {
     internal_data->child_data.push_back(child->data());
     if (child->length() != type_ids.length()) {
@@ -479,14 +482,14 @@ DictionaryArray::DictionaryArray(const std::shared_ptr<DataType>& type,
     : dict_type_(static_cast<const DictionaryType*>(type.get())) {
   DCHECK_EQ(type->id(), Type::DICTIONARY);
   DCHECK_EQ(indices->type_id(), dict_type_->index_type()->id());
-  auto data = indices->data()->ShallowCopy();
+  auto data = indices->data()->Copy();
   data->type = type;
   SetData(data);
 }
 
 void DictionaryArray::SetData(const std::shared_ptr<ArrayData>& data) {
   this->Array::SetData(data);
-  auto indices_data = data_->ShallowCopy();
+  auto indices_data = data_->Copy();
   indices_data->type = dict_type_->index_type();
   std::shared_ptr<Array> result;
   indices_ = MakeArray(indices_data);

cpp/src/arrow/array.h

@@ -104,6 +104,17 @@ struct ARROW_EXPORT ArrayData {
     this->buffers = std::move(buffers);
   }
 
+  static std::shared_ptr<ArrayData> Make(const std::shared_ptr<DataType>& type,
+                                         int64_t length,
+                                         std::vector<std::shared_ptr<Buffer>>&& buffers,
+                                         int64_t null_count = kUnknownNullCount,
+                                         int64_t offset = 0);
+
+  static std::shared_ptr<ArrayData> Make(const std::shared_ptr<DataType>& type,
+                                         int64_t length,
+                                         int64_t null_count = kUnknownNullCount,
+                                         int64_t offset = 0);
+
   // Move constructor
   ArrayData(ArrayData&& other) noexcept
       : type(std::move(other.type)),
@@ -132,9 +143,14 @@ struct ARROW_EXPORT ArrayData {
     return *this;
   }
 
-  std::shared_ptr<ArrayData> ShallowCopy() const {
-    return std::make_shared<ArrayData>(*this);
-  }
+  std::shared_ptr<ArrayData> Copy() const { return std::make_shared<ArrayData>(*this); }
+
+#ifndef ARROW_NO_DEPRECATED_API
+
+  // Deprecated since 0.8.0
+  std::shared_ptr<ArrayData> ShallowCopy() const { return Copy(); }
+
+#endif
 
   std::shared_ptr<DataType> type;
   int64_t length;

cpp/src/arrow/builder.cc

@@ -34,6 +34,7 @@
 #include "arrow/util/cpu-info.h"
 #include "arrow/util/decimal.h"
 #include "arrow/util/hash-util.h"
+#include "arrow/util/hash.h"
 #include "arrow/util/logging.h"
 
 namespace arrow {
@@ -219,8 +220,7 @@ void ArrayBuilder::UnsafeSetNotNull(int64_t length) {
 // Null builder
 
 Status NullBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
-  BufferVector buffers = {nullptr};
-  *out = std::make_shared<ArrayData>(null(), length_, std::move(buffers), length_);
+  *out = ArrayData::Make(null(), length_, {nullptr}, length_);
   length_ = null_count_ = 0;
   return Status::OK();
 }
@@ -314,8 +314,7 @@ Status PrimitiveBuilder<T>::FinishInternal(std::shared_ptr<ArrayData>* out) {
     // Trim buffers
     RETURN_NOT_OK(data_->Resize(bytes_required));
   }
-  BufferVector buffers = {null_bitmap_, data_};
-  *out = std::make_shared<ArrayData>(type_, length_, std::move(buffers), null_count_);
+  *out = ArrayData::Make(type_, length_, {null_bitmap_, data_}, null_count_);
 
   data_ = null_bitmap_ = nullptr;
   capacity_ = length_ = null_count_ = 0;
@@ -404,9 +403,7 @@ Status AdaptiveIntBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
       return Status::NotImplemented("Only ints of size 1,2,4,8 are supported");
   }
 
-  BufferVector buffers = {null_bitmap_, data_};
-  *out =
-      std::make_shared<ArrayData>(output_type, length_, std::move(buffers), null_count_);
+  *out = ArrayData::Make(output_type, length_, {null_bitmap_, data_}, null_count_);
 
   data_ = null_bitmap_ = nullptr;
   capacity_ = length_ = null_count_ = 0;
@@ -562,9 +559,7 @@ Status AdaptiveUIntBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
       return Status::NotImplemented("Only ints of size 1,2,4,8 are supported");
   }
 
-  BufferVector buffers = {null_bitmap_, data_};
-  *out =
-      std::make_shared<ArrayData>(output_type, length_, std::move(buffers), null_count_);
+  *out = ArrayData::Make(output_type, length_, {null_bitmap_, data_}, null_count_);
 
   data_ = null_bitmap_ = nullptr;
   capacity_ = length_ = null_count_ = 0;
@@ -741,8 +736,7 @@ Status BooleanBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
     // Trim buffers
     RETURN_NOT_OK(data_->Resize(bytes_required));
   }
-  BufferVector buffers = {null_bitmap_, data_};
-  *out = std::make_shared<ArrayData>(boolean(), length_, std::move(buffers), null_count_);
+  *out = ArrayData::Make(boolean(), length_, {null_bitmap_, data_}, null_count_);
 
   data_ = null_bitmap_ = nullptr;
   capacity_ = length_ = null_count_ = 0;
@@ -812,6 +806,305 @@ Status BooleanBuilder::Append(const std::vector<bool>& values) {
   return Status::OK();
 }
 
+// ----------------------------------------------------------------------
+// DictionaryBuilder
+
+using internal::WrappedBinary;
+
+template <typename T>
+DictionaryBuilder<T>::DictionaryBuilder(const std::shared_ptr<DataType>& type,
+                                        MemoryPool* pool)
+    : ArrayBuilder(type, pool),
+      hash_slots_(nullptr),
+      dict_builder_(type, pool),
+      values_builder_(pool),
+      byte_width_(-1) {
+  if (!::arrow::CpuInfo::initialized()) {
+    ::arrow::CpuInfo::Init();
+  }
+}
+
+DictionaryBuilder<NullType>::DictionaryBuilder(const std::shared_ptr<DataType>& type,
+                                               MemoryPool* pool)
+    : ArrayBuilder(type, pool), values_builder_(pool) {
+  if (!::arrow::CpuInfo::initialized()) {
+    ::arrow::CpuInfo::Init();
+  }
+}
+
+DictionaryBuilder<NullType>::~DictionaryBuilder() {}
+
+template <>
+DictionaryBuilder<FixedSizeBinaryType>::DictionaryBuilder(
+    const std::shared_ptr<DataType>& type, MemoryPool* pool)
+    : ArrayBuilder(type, pool),
+      hash_slots_(nullptr),
+      dict_builder_(type, pool),
+      values_builder_(pool),
+      byte_width_(static_cast<const FixedSizeBinaryType&>(*type).byte_width()) {
+  if (!::arrow::CpuInfo::initialized()) {
+    ::arrow::CpuInfo::Init();
+  }
+}
+
+template <typename T>
+Status DictionaryBuilder<T>::Init(int64_t elements) {
+  RETURN_NOT_OK(ArrayBuilder::Init(elements));
+
+  // Fill the initial hash table
+  RETURN_NOT_OK(internal::NewHashTable(kInitialHashTableSize, pool_, &hash_table_));
+  hash_slots_ = reinterpret_cast<int32_t*>(hash_table_->mutable_data());
+  hash_table_size_ = kInitialHashTableSize;
+  mod_bitmask_ = kInitialHashTableSize - 1;
+  hash_table_load_threshold_ =
+      static_cast<int64_t>(static_cast<double>(elements) * kMaxHashTableLoad);
+
+  return values_builder_.Init(elements);
+}
+
+Status DictionaryBuilder<NullType>::Init(int64_t elements) {
+  RETURN_NOT_OK(ArrayBuilder::Init(elements));
+  return values_builder_.Init(elements);
+}
+
+template <typename T>
+Status DictionaryBuilder<T>::Resize(int64_t capacity) {
+  if (capacity < kMinBuilderCapacity) {
+    capacity = kMinBuilderCapacity;
+  }
+
+  if (capacity_ == 0) {
+    return Init(capacity);
+  } else {
+    return ArrayBuilder::Resize(capacity);
+  }
+}
+
+Status DictionaryBuilder<NullType>::Resize(int64_t capacity) {
+  if (capacity < kMinBuilderCapacity) {
+    capacity = kMinBuilderCapacity;
+  }
+
+  if (capacity_ == 0) {
+    return Init(capacity);
+  } else {
+    return ArrayBuilder::Resize(capacity);
+  }
+}
+
+template <typename T>
+Status DictionaryBuilder<T>::FinishInternal(std::shared_ptr<ArrayData>* out) {
+  std::shared_ptr<Array> dictionary;
+  RETURN_NOT_OK(dict_builder_.Finish(&dictionary));
+
+  RETURN_NOT_OK(values_builder_.FinishInternal(out));
+  (*out)->type = std::make_shared<DictionaryType>((*out)->type, dictionary);
+  return Status::OK();
+}
+
+Status DictionaryBuilder<NullType>::FinishInternal(std::shared_ptr<ArrayData>* out) {
+  std::shared_ptr<Array> dictionary = std::make_shared<NullArray>(0);
+
+  RETURN_NOT_OK(values_builder_.FinishInternal(out));
+  (*out)->type = std::make_shared<DictionaryType>((*out)->type, dictionary);
+  return Status::OK();
+}
+
+template <typename T>
+Status DictionaryBuilder<T>::Append(const Scalar& value) {
+  RETURN_NOT_OK(Reserve(1));
+  // Based on DictEncoder<DType>::Put
+  int64_t j = HashValue(value) & mod_bitmask_;
+  hash_slot_t index = hash_slots_[j];
+
+  // Find an empty slot
+  while (kHashSlotEmpty != index && SlotDifferent(index, value)) {
+    // Linear probing
+    ++j;
+    if (j == hash_table_size_) {
+      j = 0;
+    }
+    index = hash_slots_[j];
+  }
+
+  if (index == kHashSlotEmpty) {
+    // Not in the hash table, so we insert it now
+    index = static_cast<hash_slot_t>(dict_builder_.length());
+    hash_slots_[j] = index;
+    RETURN_NOT_OK(AppendDictionary(value));
+
+    if (ARROW_PREDICT_FALSE(static_cast<int32_t>(dict_builder_.length()) >
+                            hash_table_load_threshold_)) {
+      RETURN_NOT_OK(DoubleTableSize());
+    }
+  }
+
+  RETURN_NOT_OK(values_builder_.Append(index));
+
+  return Status::OK();
+}
+
+template <typename T>
+Status DictionaryBuilder<T>::AppendArray(const Array& array) {
+  const auto& numeric_array = static_cast<const NumericArray<T>&>(array);
+  for (int64_t i = 0; i < array.length(); i++) {
+    if (array.IsNull(i)) {
+      RETURN_NOT_OK(AppendNull());
+    } else {
+      RETURN_NOT_OK(Append(numeric_array.Value(i)));
+    }
+  }
+  return Status::OK();
+}
+
+Status DictionaryBuilder<NullType>::AppendArray(const Array& array) {
+  for (int64_t i = 0; i < array.length(); i++) {
+    RETURN_NOT_OK(AppendNull());
+  }
+  return Status::OK();
+}
+
+template <>
+Status DictionaryBuilder<FixedSizeBinaryType>::AppendArray(const Array& array) {
+  if (!type_->Equals(*array.type())) {
+    return Status::Invalid("Cannot append FixedSizeBinary array with non-matching type");
+  }
+
+  const auto& numeric_array = static_cast<const FixedSizeBinaryArray&>(array);
+  for (int64_t i = 0; i < array.length(); i++) {
+    if (array.IsNull(i)) {
+      RETURN_NOT_OK(AppendNull());
+    } else {
+      RETURN_NOT_OK(Append(numeric_array.Value(i)));
+    }
+  }
+  return Status::OK();
+}
+
+template <typename T>
+Status DictionaryBuilder<T>::AppendNull() {
+  return values_builder_.AppendNull();
+}
+
+Status DictionaryBuilder<NullType>::AppendNull() { return values_builder_.AppendNull(); }
+
+template <typename T>
+Status DictionaryBuilder<T>::DoubleTableSize() {
+#define INNER_LOOP                                                \
+  Scalar value = GetDictionaryValue(static_cast<int64_t>(index)); \
+  int64_t j = HashValue(value) & new_mod_bitmask;
+
+  DOUBLE_TABLE_SIZE(, INNER_LOOP);
+
+  return Status::OK();
+}
+
+template <typename T>
+typename DictionaryBuilder<T>::Scalar DictionaryBuilder<T>::GetDictionaryValue(
+    int64_t index) {
+  const Scalar* data = reinterpret_cast<const Scalar*>(dict_builder_.data()->data());
+  return data[index];
+}
+
+template <>
+const uint8_t* DictionaryBuilder<FixedSizeBinaryType>::GetDictionaryValue(int64_t index) {
+  return dict_builder_.GetValue(index);
+}
+
+template <typename T>
+int64_t DictionaryBuilder<T>::HashValue(const Scalar& value) {
+  return HashUtil::Hash(&value, sizeof(Scalar), 0);
+}
+
+template <>
+int64_t DictionaryBuilder<FixedSizeBinaryType>::HashValue(const Scalar& value) {
+  return HashUtil::Hash(value, byte_width_, 0);
+}
+
+template <typename T>
+bool DictionaryBuilder<T>::SlotDifferent(hash_slot_t index, const Scalar& value) {
+  const Scalar other = GetDictionaryValue(static_cast<int64_t>(index));
+  return other != value;
+}
+
+template <>
+bool DictionaryBuilder<FixedSizeBinaryType>::SlotDifferent(hash_slot_t index,
+                                                           const Scalar& value) {
+  int32_t width = static_cast<const FixedSizeBinaryType&>(*type_).byte_width();
+  const Scalar other = GetDictionaryValue(static_cast<int64_t>(index));
+  return memcmp(other, value, width) != 0;
+}
+
+template <typename T>
+Status DictionaryBuilder<T>::AppendDictionary(const Scalar& value) {
+  return dict_builder_.Append(value);
+}
+
+#define BINARY_DICTIONARY_SPECIALIZATIONS(Type)                                     \
+  template <>                                                                       \
+  WrappedBinary DictionaryBuilder<Type>::GetDictionaryValue(int64_t index) {        \
+    int32_t v_len;                                                                  \
+    const uint8_t* v = dict_builder_.GetValue(static_cast<int64_t>(index), &v_len); \
+    return WrappedBinary(v, v_len);                                                 \
+  }                                                                                 \
+                                                                                    \
+  template <>                                                                       \
+  Status DictionaryBuilder<Type>::AppendDictionary(const WrappedBinary& value) {    \
+    return dict_builder_.Append(value.ptr_, value.length_);                         \
+  }                                                                                 \
+                                                                                    \
+  template <>                                                                       \
+  Status DictionaryBuilder<Type>::AppendArray(const Array& array) {                 \
+    const BinaryArray& binary_array = static_cast<const BinaryArray&>(array);       \
+    WrappedBinary value(nullptr, 0);                                                \
+    for (int64_t i = 0; i < array.length(); i++) {                                  \
+      if (array.IsNull(i)) {                                                        \
+        RETURN_NOT_OK(AppendNull());                                                \
+      } else {                                                                      \
+        value.ptr_ = binary_array.GetValue(i, &value.length_);                      \
+        RETURN_NOT_OK(Append(value));                                               \
+      }                                                                             \
+    }                                                                               \
+    return Status::OK();                                                            \
+  }                                                                                 \
+                                                                                    \
+  template <>                                                                       \
+  int64_t DictionaryBuilder<Type>::HashValue(const WrappedBinary& value) {          \
+    return HashUtil::Hash(value.ptr_, value.length_, 0);                            \
+  }                                                                                 \
+                                                                                    \
+  template <>                                                                       \
+  bool DictionaryBuilder<Type>::SlotDifferent(hash_slot_t index,                    \
+                                              const WrappedBinary& value) {         \
+    int32_t other_length;                                                           \
+    const uint8_t* other_value =                                                    \
+        dict_builder_.GetValue(static_cast<int64_t>(index), &other_length);         \
+    return !(other_length == value.length_ &&                                       \
+             0 == memcmp(other_value, value.ptr_, value.length_));                  \
+  }
+
+BINARY_DICTIONARY_SPECIALIZATIONS(StringType);
+BINARY_DICTIONARY_SPECIALIZATIONS(BinaryType);
+
+template class DictionaryBuilder<UInt8Type>;
+template class DictionaryBuilder<UInt16Type>;
+template class DictionaryBuilder<UInt32Type>;
+template class DictionaryBuilder<UInt64Type>;
+template class DictionaryBuilder<Int8Type>;
+template class DictionaryBuilder<Int16Type>;
+template class DictionaryBuilder<Int32Type>;
+template class DictionaryBuilder<Int64Type>;
+template class DictionaryBuilder<Date32Type>;
+template class DictionaryBuilder<Date64Type>;
+template class DictionaryBuilder<Time32Type>;
+template class DictionaryBuilder<Time64Type>;
+template class DictionaryBuilder<TimestampType>;
+template class DictionaryBuilder<FloatType>;
+template class DictionaryBuilder<DoubleType>;
+template class DictionaryBuilder<FixedSizeBinaryType>;
+template class DictionaryBuilder<BinaryType>;
+template class DictionaryBuilder<StringType>;
+
 // ----------------------------------------------------------------------
 // Decimal128Builder
 
@@ -828,8 +1121,7 @@ Status Decimal128Builder::FinishInternal(std::shared_ptr<ArrayData>* out) {
   std::shared_ptr<Buffer> data;
   RETURN_NOT_OK(byte_builder_.Finish(&data));
 
-  BufferVector buffers = {null_bitmap_, data};
-  *out = std::make_shared<ArrayData>(type_, length_, std::move(buffers), null_count_);
+  *out = ArrayData::Make(type_, length_, {null_bitmap_, data}, null_count_);
   return Status::OK();
 }
 
@@ -896,8 +1188,7 @@ Status ListBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
     RETURN_NOT_OK(value_builder_->FinishInternal(&items));
   }
 
-  BufferVector buffers = {null_bitmap_, offsets};
-  *out = std::make_shared<ArrayData>(type_, length_, std::move(buffers), null_count_);
+  *out = ArrayData::Make(type_, length_, {null_bitmap_, offsets}, null_count_);
   (*out)->child_data.emplace_back(std::move(items));
   Reset();
   return Status::OK();
@@ -969,8 +1260,8 @@ Status BinaryBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
   RETURN_NOT_OK(offsets_builder_.Finish(&offsets));
   RETURN_NOT_OK(value_data_builder_.Finish(&value_data));
 
-  BufferVector buffers = {null_bitmap_, offsets, value_data};
-  *out = std::make_shared<ArrayData>(type_, length_, std::move(buffers), null_count_, 0);
+  *out = ArrayData::Make(type_, length_, {null_bitmap_, offsets, value_data}, null_count_,
+                         0);
   Reset();
   return Status::OK();
 }
@@ -1040,8 +1331,7 @@ Status FixedSizeBinaryBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
   std::shared_ptr<Buffer> data;
   RETURN_NOT_OK(byte_builder_.Finish(&data));
 
-  BufferVector buffers = {null_bitmap_, data};
-  *out = std::make_shared<ArrayData>(type_, length_, std::move(buffers), null_count_);
+  *out = ArrayData::Make(type_, length_, {null_bitmap_, data}, null_count_);
   return Status::OK();
 }
 
@@ -1060,8 +1350,7 @@ StructBuilder::StructBuilder(const std::shared_ptr<DataType>& type, MemoryPool*
 }
 
 Status StructBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
-  BufferVector buffers = {null_bitmap_};
-  *out = std::make_shared<ArrayData>(type_, length_, std::move(buffers), null_count_);
+  *out = ArrayData::Make(type_, length_, {null_bitmap_}, null_count_);
 
   (*out)->child_data.resize(field_builders_.size());
   for (size_t i = 0; i < field_builders_.size(); ++i) {

cpp/src/arrow/builder.h

@@ -32,6 +32,7 @@
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
+#include "arrow/util/hash.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
 
@@ -809,6 +810,158 @@ class ARROW_EXPORT StructBuilder : public ArrayBuilder {
   std::vector<std::unique_ptr<ArrayBuilder>> field_builders_;
 };
 
+// ----------------------------------------------------------------------
+// Dictionary builder
+
+namespace internal {
+
+// TODO(ARROW-1176): Use Tensorflow's StringPiece instead of this here.
+struct WrappedBinary {
+  WrappedBinary(const uint8_t* ptr, int32_t length) : ptr_(ptr), length_(length) {}
+
+  const uint8_t* ptr_;
+  int32_t length_;
+};
+
+template <typename T>
+struct DictionaryScalar {
+  using type = typename T::c_type;
+};
+
+template <>
+struct DictionaryScalar<BinaryType> {
+  using type = WrappedBinary;
+};
+
+template <>
+struct DictionaryScalar<StringType> {
+  using type = WrappedBinary;
+};
+
+template <>
+struct DictionaryScalar<FixedSizeBinaryType> {
+  using type = uint8_t const*;
+};
+
+}  // namespace internal
+
+/// \brief Array builder for created encoded DictionaryArray from dense array
+/// data
+template <typename T>
+class ARROW_EXPORT DictionaryBuilder : public ArrayBuilder {
+ public:
+  using Scalar = typename internal::DictionaryScalar<T>::type;
+
+  ~DictionaryBuilder() {}
+
+  DictionaryBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool);
+
+  template <typename T1 = T>
+  explicit DictionaryBuilder(
+      typename std::enable_if<TypeTraits<T1>::is_parameter_free, MemoryPool*>::type pool)
+      : DictionaryBuilder<T1>(TypeTraits<T1>::type_singleton(), pool) {}
+
+  /// \brief Append a scalar value
+  Status Append(const Scalar& value);
+
+  /// \brief Append a scalar null value
+  Status AppendNull();
+
+  /// \brief Append a whole dense array to the builder
+  Status AppendArray(const Array& array);
+
+  Status Init(int64_t elements) override;
+  Status Resize(int64_t capacity) override;
+  Status FinishInternal(std::shared_ptr<ArrayData>* out) override;
+
+ protected:
+  Status DoubleTableSize();
+  Scalar GetDictionaryValue(int64_t index);
+  int64_t HashValue(const Scalar& value);
+  bool SlotDifferent(hash_slot_t slot, const Scalar& value);
+  Status AppendDictionary(const Scalar& value);
+
+  std::shared_ptr<Buffer> hash_table_;
+  int32_t* hash_slots_;
+
+  /// Size of the table. Must be a power of 2.
+  int64_t hash_table_size_;
+
+  // Store hash_table_size_ - 1, so that j & mod_bitmask_ is equivalent to j %
+  // hash_table_size_, but uses far fewer CPU cycles
+  int64_t mod_bitmask_;
+
+  typename TypeTraits<T>::BuilderType dict_builder_;
+  AdaptiveIntBuilder values_builder_;
+  int32_t byte_width_;
+
+  /// Size at which we decide to resize
+  int64_t hash_table_load_threshold_;
+};
+
+template <>
+class ARROW_EXPORT DictionaryBuilder<NullType> : public ArrayBuilder {
+ public:
+  ~DictionaryBuilder();
+
+  DictionaryBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool);
+  explicit DictionaryBuilder(MemoryPool* pool);
+
+  /// \brief Append a scalar null value
+  Status AppendNull();
+
+  /// \brief Append a whole dense array to the builder
+  Status AppendArray(const Array& array);
+
+  Status Init(int64_t elements) override;
+  Status Resize(int64_t capacity) override;
+  Status FinishInternal(std::shared_ptr<ArrayData>* out) override;
+
+ protected:
+  AdaptiveIntBuilder values_builder_;
+};
+
+class ARROW_EXPORT BinaryDictionaryBuilder : public DictionaryBuilder<BinaryType> {
+ public:
+  using DictionaryBuilder::Append;
+  using DictionaryBuilder::DictionaryBuilder;
+
+  Status Append(const uint8_t* value, int32_t length) {
+    return Append(internal::WrappedBinary(value, length));
+  }
+
+  Status Append(const char* value, int32_t length) {
+    return Append(
+        internal::WrappedBinary(reinterpret_cast<const uint8_t*>(value), length));
+  }
+
+  Status Append(const std::string& value) {
+    return Append(internal::WrappedBinary(reinterpret_cast<const uint8_t*>(value.c_str()),
+                                          static_cast<int32_t>(value.size())));
+  }
+};
+
+/// \brief Dictionary array builder with convenience methods for strings
+class ARROW_EXPORT StringDictionaryBuilder : public DictionaryBuilder<StringType> {
+ public:
+  using DictionaryBuilder::Append;
+  using DictionaryBuilder::DictionaryBuilder;
+
+  Status Append(const uint8_t* value, int32_t length) {
+    return Append(internal::WrappedBinary(value, length));
+  }
+
+  Status Append(const char* value, int32_t length) {
+    return Append(
+        internal::WrappedBinary(reinterpret_cast<const uint8_t*>(value), length));
+  }
+
+  Status Append(const std::string& value) {
+    return Append(internal::WrappedBinary(reinterpret_cast<const uint8_t*>(value.c_str()),
+                                          static_cast<int32_t>(value.size())));
+  }
+};
+
 // ----------------------------------------------------------------------
 // Helper functions
 

cpp/src/arrow/compute/compute-benchmark.cc

@@ -81,8 +81,131 @@ static void BM_BuildStringDictionary(
   state.SetBytesProcessed(state.iterations() * total_bytes);
 }
 
-BENCHMARK(BM_BuildDictionary)->Repetitions(3)->Unit(benchmark::kMicrosecond);
-BENCHMARK(BM_BuildStringDictionary)->Repetitions(3)->Unit(benchmark::kMicrosecond);
+template <typename Type>
+struct HashParams {
+  using T = typename Type::c_type;
+
+  double null_percent;
+
+  void GenerateTestData(const int64_t length, const int64_t num_unique,
+                        std::shared_ptr<Array>* arr) const {
+    std::vector<int64_t> draws;
+    std::vector<T> values;
+    std::vector<bool> is_valid;
+    test::randint<int64_t>(length, 0, num_unique, &draws);
+    for (int64_t draw : draws) {
+      values.push_back(draw);
+    }
+
+    if (this->null_percent > 0) {
+      test::random_is_valid(length, this->null_percent, &is_valid);
+      ArrayFromVector<Type, T>(is_valid, values, arr);
+    } else {
+      ArrayFromVector<Type, T>(values, arr);
+    }
+  }
+
+  int64_t GetBytesProcessed(int64_t length) const { return length * sizeof(T); }
+};
+
+template <>
+struct HashParams<StringType> {
+  double null_percent;
+  int32_t byte_width;
+  void GenerateTestData(const int64_t length, const int64_t num_unique,
+                        std::shared_ptr<Array>* arr) const {
+    std::vector<int64_t> draws;
+    test::randint<int64_t>(length, 0, num_unique, &draws);
+
+    const int64_t total_bytes = this->byte_width * num_unique;
+    std::vector<uint8_t> uniques(total_bytes);
+    const uint32_t seed = 0;
+    test::random_bytes(total_bytes, seed, uniques.data());
+
+    std::vector<bool> is_valid;
+    if (this->null_percent > 0) {
+      test::random_is_valid(length, this->null_percent, &is_valid);
+    }
+
+    StringBuilder builder;
+    for (int64_t i = 0; i < length; ++i) {
+      if (this->null_percent == 0 || is_valid[i]) {
+        ABORT_NOT_OK(builder.Append(uniques.data() + this->byte_width * draws[i],
+                                    this->byte_width));
+      } else {
+        ABORT_NOT_OK(builder.AppendNull());
+      }
+    }
+    ABORT_NOT_OK(builder.Finish(arr));
+  }
+
+  int64_t GetBytesProcessed(int64_t length) const { return length * byte_width; }
+};
+
+template <typename ParamType>
+void BenchUnique(benchmark::State& state, const ParamType& params, int64_t length,
+                 int64_t num_unique) {
+  std::shared_ptr<Array> arr;
+  params.GenerateTestData(length, num_unique, &arr);
+
+  FunctionContext ctx;
+  while (state.KeepRunning()) {
+    std::shared_ptr<Array> out;
+    ABORT_NOT_OK(Unique(&ctx, Datum(arr), &out));
+  }
+  state.SetBytesProcessed(state.iterations() * params.GetBytesProcessed(length));
+}
+
+template <typename ParamType>
+void BenchDictionaryEncode(benchmark::State& state, const ParamType& params,
+                           int64_t length, int64_t num_unique) {
+  std::shared_ptr<Array> arr;
+  params.GenerateTestData(length, num_unique, &arr);
+
+  FunctionContext ctx;
+  while (state.KeepRunning()) {
+    Datum out;
+    ABORT_NOT_OK(DictionaryEncode(&ctx, Datum(arr), &out));
+  }
+  state.SetBytesProcessed(state.iterations() * params.GetBytesProcessed(length));
+}
+
+static void BM_UniqueInt64NoNulls(benchmark::State& state) {
+  BenchUnique(state, HashParams<Int64Type>{0}, state.range(0), state.range(1));
+}
+
+static void BM_UniqueInt64WithNulls(benchmark::State& state) {
+  BenchUnique(state, HashParams<Int64Type>{0.05}, state.range(0), state.range(1));
+}
+
+static void BM_UniqueString10bytes(benchmark::State& state) {
+  // Byte strings with 10 bytes each
+  BenchUnique(state, HashParams<StringType>{0.05, 10}, state.range(0), state.range(1));
+}
+
+static void BM_UniqueString100bytes(benchmark::State& state) {
+  // Byte strings with 100 bytes each
+  BenchUnique(state, HashParams<StringType>{0.05, 100}, state.range(0), state.range(1));
+}
+
+BENCHMARK(BM_BuildDictionary)->MinTime(1.0)->Unit(benchmark::kMicrosecond);
+BENCHMARK(BM_BuildStringDictionary)->MinTime(1.0)->Unit(benchmark::kMicrosecond);
+
+constexpr int64_t kHashBenchmarkLength = 1 << 24;
+
+#define ADD_HASH_ARGS(WHAT)                        \
+  WHAT->Args({kHashBenchmarkLength, 50})           \
+      ->Args({kHashBenchmarkLength, 1 << 10})      \
+      ->Args({kHashBenchmarkLength, 10 * 1 << 10}) \
+      ->Args({kHashBenchmarkLength, 1 << 20})      \
+      ->MinTime(1.0)                               \
+      ->Unit(benchmark::kMicrosecond)              \
+      ->UseRealTime()
+
+ADD_HASH_ARGS(BENCHMARK(BM_UniqueInt64NoNulls));
+ADD_HASH_ARGS(BENCHMARK(BM_UniqueInt64WithNulls));
+ADD_HASH_ARGS(BENCHMARK(BM_UniqueString10bytes));
+ADD_HASH_ARGS(BENCHMARK(BM_UniqueString100bytes));
 
 }  // namespace compute
 }  // namespace arrow

cpp/src/arrow/compute/compute-test.cc

@@ -688,7 +688,7 @@ TEST_F(TestCast, PreallocatedMemory) {
   std::unique_ptr<UnaryKernel> kernel;
   ASSERT_OK(GetCastFunction(*int32(), out_type, options, &kernel));
 
-  auto out_data = std::make_shared<ArrayData>(out_type, length);
+  auto out_data = ArrayData::Make(out_type, length);
 
   shared_ptr<Buffer> out_values;
   ASSERT_OK(this->ctx_.Allocate(length * sizeof(int64_t), &out_values));
@@ -709,6 +709,66 @@ TEST_F(TestCast, PreallocatedMemory) {
   ASSERT_ARRAYS_EQUAL(*expected, *result);
 }
 
+template <typename InType, typename InT, typename OutType, typename OutT>
+void CheckOffsetOutputCase(FunctionContext* ctx, const std::shared_ptr<DataType>& in_type,
+                           const vector<InT>& in_values,
+                           const std::shared_ptr<DataType>& out_type,
+                           const vector<OutT>& out_values) {
+  using OutTraits = TypeTraits<OutType>;
+
+  CastOptions options;
+
+  const int64_t length = static_cast<int64_t>(in_values.size());
+
+  shared_ptr<Array> arr, expected;
+  ArrayFromVector<InType, InT>(in_type, in_values, &arr);
+  ArrayFromVector<OutType, OutT>(out_type, out_values, &expected);
+
+  shared_ptr<Buffer> out_buffer;
+  ASSERT_OK(ctx->Allocate(OutTraits::bytes_required(length), &out_buffer));
+
+  std::unique_ptr<UnaryKernel> kernel;
+  ASSERT_OK(GetCastFunction(*in_type, out_type, options, &kernel));
+
+  const int64_t first_half = length / 2;
+
+  auto out_data = ArrayData::Make(out_type, length, {nullptr, out_buffer});
+  auto out_second_data = out_data->Copy();
+  out_second_data->offset = first_half;
+
+  Datum out_first(out_data);
+  Datum out_second(out_second_data);
+
+  // Cast each bit
+  ASSERT_OK(kernel->Call(ctx, Datum(arr->Slice(0, first_half)), &out_first));
+  ASSERT_OK(kernel->Call(ctx, Datum(arr->Slice(first_half)), &out_second));
+
+  shared_ptr<Array> result = MakeArray(out_data);
+
+  ASSERT_ARRAYS_EQUAL(*expected, *result);
+}
+
+TEST_F(TestCast, OffsetOutputBuffer) {
+  // ARROW-1735
+  vector<int32_t> v1 = {0, 10000, 2000, 1000, 0};
+  vector<int64_t> e1 = {0, 10000, 2000, 1000, 0};
+
+  auto in_type = int32();
+  auto out_type = int64();
+  CheckOffsetOutputCase<Int32Type, int32_t, Int64Type, int64_t>(&this->ctx_, in_type, v1,
+                                                                out_type, e1);
+
+  vector<bool> e2 = {false, true, true, true, false};
+
+  out_type = boolean();
+  CheckOffsetOutputCase<Int32Type, int32_t, BooleanType, bool>(&this->ctx_, in_type, v1,
+                                                               boolean(), e2);
+
+  vector<int16_t> e3 = {0, 10000, 2000, 1000, 0};
+  CheckOffsetOutputCase<Int32Type, int32_t, Int16Type, int16_t>(&this->ctx_, in_type, v1,
+                                                                int16(), e3);
+}
+
 template <typename TestType>
 class TestDictionaryCast : public TestCast {};
 
@@ -741,6 +801,47 @@ TYPED_TEST(TestDictionaryCast, Basic) {
   this->CheckPass(*plain_array, *dict_array, dict_array->type(), options);
 }*/
 
+TEST_F(TestCast, ListToList) {
+  CastOptions options;
+  std::shared_ptr<Array> offsets;
+
+  vector<int32_t> offsets_values = {0, 1, 2, 5, 7, 7, 8, 10};
+  std::vector<bool> offsets_is_valid = {true, true, true, true, false, true, true, true};
+  ArrayFromVector<Int32Type, int32_t>(offsets_is_valid, offsets_values, &offsets);
+
+  shared_ptr<Array> int32_plain_array =
+      TestBase::MakeRandomArray<typename TypeTraits<Int32Type>::ArrayType>(10, 2);
+  std::shared_ptr<Array> int32_list_array;
+  ASSERT_OK(
+      ListArray::FromArrays(*offsets, *int32_plain_array, pool_, &int32_list_array));
+
+  std::shared_ptr<Array> int64_plain_array;
+  ASSERT_OK(Cast(&this->ctx_, *int32_plain_array, int64(), options, &int64_plain_array));
+  std::shared_ptr<Array> int64_list_array;
+  ASSERT_OK(
+      ListArray::FromArrays(*offsets, *int64_plain_array, pool_, &int64_list_array));
+
+  std::shared_ptr<Array> float64_plain_array;
+  ASSERT_OK(
+      Cast(&this->ctx_, *int32_plain_array, float64(), options, &float64_plain_array));
+  std::shared_ptr<Array> float64_list_array;
+  ASSERT_OK(
+      ListArray::FromArrays(*offsets, *float64_plain_array, pool_, &float64_list_array));
+
+  this->CheckPass(*int32_list_array, *int64_list_array, int64_list_array->type(),
+                  options);
+  this->CheckPass(*int32_list_array, *float64_list_array, float64_list_array->type(),
+                  options);
+  this->CheckPass(*int64_list_array, *int32_list_array, int32_list_array->type(),
+                  options);
+  this->CheckPass(*int64_list_array, *float64_list_array, float64_list_array->type(),
+                  options);
+  this->CheckPass(*float64_list_array, *int32_list_array, int32_list_array->type(),
+                  options);
+  this->CheckPass(*float64_list_array, *int64_list_array, int64_list_array->type(),
+                  options);
+}
+
 // ----------------------------------------------------------------------
 // Dictionary tests
 
@@ -809,8 +910,8 @@ TYPED_TEST(TestHashKernelPrimitive, PrimitiveResizeTable) {
     return;
   }
 
-  const int64_t kTotalValues = 10000;
-  const int64_t kRepeats = 10;
+  const int64_t kTotalValues = 1000000;
+  const int64_t kRepeats = 5;
 
   vector<T> values;
   vector<T> uniques;

cpp/src/arrow/compute/kernels/cast.cc

@@ -124,12 +124,7 @@ template <typename T>
 struct CastFunctor<T, NullType, typename std::enable_if<
                                     std::is_base_of<FixedWidthType, T>::value>::type> {
   void operator()(FunctionContext* ctx, const CastOptions& options,
-                  const ArrayData& input, ArrayData* output) {
-    // Simply initialize data to 0
-    auto buf = output->buffers[1];
-    DCHECK_EQ(output->offset, 0);
-    memset(buf->mutable_data(), 0, buf->size());
-  }
+                  const ArrayData& input, ArrayData* output) {}
 };
 
 template <>
@@ -199,14 +194,19 @@ struct CastFunctor<O, I, typename std::enable_if<std::is_same<BooleanType, O>::v
                                                  !std::is_same<O, I>::value>::type> {
   void operator()(FunctionContext* ctx, const CastOptions& options,
                   const ArrayData& input, ArrayData* output) {
-    using in_type = typename I::c_type;
-    DCHECK_EQ(output->offset, 0);
+    auto in_data = GetValues<typename I::c_type>(input, 1);
+    internal::BitmapWriter writer(output->buffers[1]->mutable_data(), output->offset,
+                                  input.length);
 
-    const in_type* in_data = GetValues<in_type>(input, 1);
-    uint8_t* out_data = GetMutableValues<uint8_t>(output, 1);
     for (int64_t i = 0; i < input.length; ++i) {
-      BitUtil::SetBitTo(out_data, i, (*in_data++) != 0);
+      if (*in_data++ != 0) {
+        writer.Set();
+      } else {
+        writer.Clear();
+      }
+      writer.Next();
     }
+    writer.Finish();
   }
 };
 
@@ -217,7 +217,6 @@ struct CastFunctor<O, I,
                   const ArrayData& input, ArrayData* output) {
     using in_type = typename I::c_type;
     using out_type = typename O::c_type;
-    DCHECK_EQ(output->offset, 0);
 
     auto in_offset = input.offset;
 
@@ -461,6 +460,49 @@ struct CastFunctor<Date32Type, Date64Type> {
   }
 };
 
+// ----------------------------------------------------------------------
+// List to List
+
+class ListCastKernel : public UnaryKernel {
+ public:
+  ListCastKernel(std::unique_ptr<UnaryKernel> child_caster,
+                 const std::shared_ptr<DataType>& out_type)
+      : child_caster_(std::move(child_caster)), out_type_(out_type) {}
+
+  Status Call(FunctionContext* ctx, const Datum& input, Datum* out) override {
+    DCHECK_EQ(Datum::ARRAY, input.kind());
+
+    const ArrayData& in_data = *input.array();
+    DCHECK_EQ(Type::LIST, in_data.type->id());
+    ArrayData* result;
+
+    if (in_data.offset != 0) {
+      return Status::NotImplemented(
+          "Casting sliced lists (non-zero offset) not yet implemented");
+    }
+
+    if (out->kind() == Datum::NONE) {
+      out->value = ArrayData::Make(out_type_, in_data.length);
+    }
+
+    result = out->array().get();
+
+    // Copy buffers from parent
+    result->buffers = in_data.buffers;
+
+    Datum casted_child;
+    RETURN_NOT_OK(child_caster_->Call(ctx, Datum(in_data.child_data[0]), &casted_child));
+    result->child_data.push_back(casted_child.array());
+
+    RETURN_IF_ERROR(ctx);
+    return Status::OK();
+  }
+
+ private:
+  std::unique_ptr<UnaryKernel> child_caster_;
+  std::shared_ptr<DataType> out_type_;
+};
+
 // ----------------------------------------------------------------------
 // Dictionary to other things
 
@@ -475,9 +517,10 @@ void UnpackFixedSizeBinaryDictionary(FunctionContext* ctx, const Array& indices,
 
   const index_c_type* in = GetValues<index_c_type>(*indices.data(), 1);
 
-  uint8_t* out = output->buffers[1]->mutable_data();
   int32_t byte_width =
       static_cast<const FixedSizeBinaryType&>(*output->type).byte_width();
+
+  uint8_t* out = output->buffers[1]->mutable_data() + byte_width * output->offset;
   for (int64_t i = 0; i < indices.length(); ++i) {
     if (valid_bits_reader.IsSet()) {
       const uint8_t* value = dictionary.Value(in[i]);
@@ -493,7 +536,7 @@ struct CastFunctor<
     typename std::enable_if<std::is_base_of<FixedSizeBinaryType, T>::value>::type> {
   void operator()(FunctionContext* ctx, const CastOptions& options,
                   const ArrayData& input, ArrayData* output) {
-    DictionaryArray dict_array(input.ShallowCopy());
+    DictionaryArray dict_array(input.Copy());
 
     const DictionaryType& type = static_cast<const DictionaryType&>(*input.type);
     const DataType& values_type = *type.dictionary()->type();
@@ -565,7 +608,7 @@ struct CastFunctor<T, DictionaryType,
                    typename std::enable_if<std::is_base_of<BinaryType, T>::value>::type> {
   void operator()(FunctionContext* ctx, const CastOptions& options,
                   const ArrayData& input, ArrayData* output) {
-    DictionaryArray dict_array(input.ShallowCopy());
+    DictionaryArray dict_array(input.Copy());
 
     const DictionaryType& type = static_cast<const DictionaryType&>(*input.type);
     const DataType& values_type = *type.dictionary()->type();
@@ -605,12 +648,10 @@ struct CastFunctor<T, DictionaryType,
 template <typename IndexType, typename c_type>
 void UnpackPrimitiveDictionary(const Array& indices, const c_type* dictionary,
                                c_type* out) {
-  using index_c_type = typename IndexType::c_type;
-
   internal::BitmapReader valid_bits_reader(indices.null_bitmap_data(), indices.offset(),
                                            indices.length());
 
-  const index_c_type* in = GetValues<index_c_type>(*indices.data(), 1);
+  auto in = GetValues<typename IndexType::c_type>(*indices.data(), 1);
   for (int64_t i = 0; i < indices.length(); ++i) {
     if (valid_bits_reader.IsSet()) {
       out[i] = dictionary[in[i]];
@@ -627,7 +668,7 @@ struct CastFunctor<T, DictionaryType,
                   const ArrayData& input, ArrayData* output) {
     using c_type = typename T::c_type;
 
-    DictionaryArray dict_array(input.ShallowCopy());
+    DictionaryArray dict_array(input.Copy());
 
     const DictionaryType& type = static_cast<const DictionaryType&>(*input.type);
     const DataType& values_type = *type.dictionary()->type();
@@ -638,7 +679,7 @@ struct CastFunctor<T, DictionaryType,
 
     const c_type* dictionary = GetValues<c_type>(*type.dictionary()->data(), 1);
 
-    auto out = reinterpret_cast<c_type*>(output->buffers[1]->mutable_data());
+    auto out = GetMutableValues<c_type>(output, 1);
     const Array& indices = *dict_array.indices();
     switch (indices.type()->id()) {
       case Type::INT8:
@@ -747,7 +788,7 @@ class CastKernel : public UnaryKernel {
     ArrayData* result;
 
     if (out->kind() == Datum::NONE) {
-      out->value = std::make_shared<ArrayData>(out_type_, in_data.length);
+      out->value = ArrayData::Make(out_type_, in_data.length);
     }
 
     result = out->array().get();
@@ -897,7 +938,6 @@ GET_CAST_FUNCTION(DATE64_CASES, Date64Type);
 GET_CAST_FUNCTION(TIME32_CASES, Time32Type);
 GET_CAST_FUNCTION(TIME64_CASES, Time64Type);
 GET_CAST_FUNCTION(TIMESTAMP_CASES, TimestampType);
-
 GET_CAST_FUNCTION(DICTIONARY_CASES, DictionaryType);
 
 #define CAST_FUNCTION_CASE(InType)                      \
@@ -905,6 +945,26 @@ GET_CAST_FUNCTION(DICTIONARY_CASES, DictionaryType);
     *kernel = Get##InType##CastFunc(out_type, options); \
     break
 
+namespace {
+
+Status GetListCastFunc(const DataType& in_type, const std::shared_ptr<DataType>& out_type,
+                       const CastOptions& options, std::unique_ptr<UnaryKernel>* kernel) {
+  if (out_type->id() != Type::LIST) {
+    // Kernel will be null
+    return Status::OK();
+  }
+  const DataType& in_value_type = *static_cast<const ListType&>(in_type).value_type();
+  std::shared_ptr<DataType> out_value_type =
+      static_cast<const ListType&>(*out_type).value_type();
+  std::unique_ptr<UnaryKernel> child_caster;
+  RETURN_NOT_OK(GetCastFunction(in_value_type, out_value_type, options, &child_caster));
+  *kernel =
+      std::unique_ptr<UnaryKernel>(new ListCastKernel(std::move(child_caster), out_type));
+  return Status::OK();
+}
+
+}  // namespace
+
 Status GetCastFunction(const DataType& in_type, const std::shared_ptr<DataType>& out_type,
                        const CastOptions& options, std::unique_ptr<UnaryKernel>* kernel) {
   switch (in_type.id()) {
@@ -926,6 +986,9 @@ Status GetCastFunction(const DataType& in_type, const std::shared_ptr<DataType>&
     CAST_FUNCTION_CASE(Time64Type);
     CAST_FUNCTION_CASE(TimestampType);
     CAST_FUNCTION_CASE(DictionaryType);
+    case Type::LIST:
+      RETURN_NOT_OK(GetListCastFunc(in_type, out_type, options, kernel));
+      break;
     default:
       break;
   }

cpp/src/arrow/compute/kernels/hash.cc

@@ -30,21 +30,13 @@
 #include "arrow/compute/kernel.h"
 #include "arrow/compute/kernels/util-internal.h"
 #include "arrow/util/hash-util.h"
+#include "arrow/util/hash.h"
 
 namespace arrow {
 namespace compute {
 
 namespace {
 
-// Initially 1024 elements
-static constexpr int64_t kInitialHashTableSize = 1 << 10;
-
-typedef int32_t hash_slot_t;
-static constexpr hash_slot_t kHashSlotEmpty = std::numeric_limits<int32_t>::max();
-
-// The maximum load factor for the hash table before resizing.
-static constexpr double kMaxHashTableLoad = 0.7;
-
 enum class SIMDMode : char { NOSIMD, SSE4, AVX2 };
 
 #define CHECK_IMPLEMENTED(KERNEL, FUNCNAME, TYPE)                  \
@@ -54,17 +46,6 @@ enum class SIMDMode : char { NOSIMD, SSE4, AVX2 };
     return Status::NotImplemented(ss.str());                       \
   }
 
-Status NewHashTable(int64_t size, MemoryPool* pool, std::shared_ptr<Buffer>* out) {
-  auto hash_table = std::make_shared<PoolBuffer>(pool);
-
-  RETURN_NOT_OK(hash_table->Resize(sizeof(hash_slot_t) * size));
-  int32_t* slots = reinterpret_cast<hash_slot_t*>(hash_table->mutable_data());
-  std::fill(slots, slots + size, kHashSlotEmpty);
-
-  *out = hash_table;
-  return Status::OK();
-}
-
 // This is a slight design concession -- some hash actions have the possibility
 // of failure. Rather than introduce extra error checking into all actions, we
 // will raise an internal exception so that only the actions where errors can
@@ -129,7 +110,7 @@ class HashTable {
 
 Status HashTable::Init(int64_t elements) {
   DCHECK_EQ(elements, BitUtil::NextPower2(elements));
-  RETURN_NOT_OK(NewHashTable(elements, pool_, &hash_table_));
+  RETURN_NOT_OK(internal::NewHashTable(elements, pool_, &hash_table_));
   hash_slots_ = reinterpret_cast<hash_slot_t*>(hash_table_->mutable_data());
   hash_table_size_ = elements;
   hash_table_load_threshold_ =
@@ -238,44 +219,6 @@ struct HashDictionary<Type, enable_if_has_c_type<Type>> {
     }                                                                                    \
   }
 
-#define DOUBLE_TABLE_SIZE(SETUP_CODE, COMPUTE_HASH)                              \
-  do {                                                                           \
-    int64_t new_size = hash_table_size_ * 2;                                     \
-                                                                                 \
-    std::shared_ptr<Buffer> new_hash_table;                                      \
-    RETURN_NOT_OK(NewHashTable(new_size, pool_, &new_hash_table));               \
-    int32_t* new_hash_slots =                                                    \
-        reinterpret_cast<hash_slot_t*>(new_hash_table->mutable_data());          \
-    int64_t new_mod_bitmask = new_size - 1;                                      \
-                                                                                 \
-    SETUP_CODE;                                                                  \
-                                                                                 \
-    for (int i = 0; i < hash_table_size_; ++i) {                                 \
-      hash_slot_t index = hash_slots_[i];                                        \
-                                                                                 \
-      if (index == kHashSlotEmpty) {                                             \
-        continue;                                                                \
-      }                                                                          \
-                                                                                 \
-      COMPUTE_HASH;                                                              \
-      while (kHashSlotEmpty != new_hash_slots[j]) {                              \
-        ++j;                                                                     \
-        if (ARROW_PREDICT_FALSE(j == hash_table_size_)) {                        \
-          j = 0;                                                                 \
-        }                                                                        \
-      }                                                                          \
-                                                                                 \
-      new_hash_slots[j] = index;                                                 \
-    }                                                                            \
-                                                                                 \
-    hash_table_ = new_hash_table;                                                \
-    hash_slots_ = reinterpret_cast<hash_slot_t*>(hash_table_->mutable_data());   \
-    hash_table_size_ = new_size;                                                 \
-    hash_table_load_threshold_ =                                                 \
-        static_cast<int64_t>(static_cast<double>(new_size) * kMaxHashTableLoad); \
-    mod_bitmask_ = new_size - 1;                                                 \
-  } while (false)
-
 template <typename Type, typename Action>
 class HashTableKernel<Type, Action, enable_if_has_c_type<Type>> : public HashTable {
  public:
@@ -342,8 +285,7 @@ class HashTableKernel<Type, Action, enable_if_has_c_type<Type>> : public HashTab
     auto dict_data = dict_.buffer;
     RETURN_NOT_OK(dict_data->Resize(dict_.size * sizeof(T), false));
 
-    BufferVector buffers = {nullptr, dict_data};
-    *out = std::make_shared<ArrayData>(type_, dict_.size, std::move(buffers), 0);
+    *out = ArrayData::Make(type_, dict_.size, {nullptr, dict_data}, 0);
     return Status::OK();
   }
 
@@ -528,7 +470,7 @@ class HashTableKernel<Type, Action, enable_if_binary<Type>> : public HashTable {
     RETURN_NOT_OK(dict_offsets_.Finish(&buffers[1]));
     RETURN_NOT_OK(dict_data_.Finish(&buffers[2]));
 
-    *out = std::make_shared<ArrayData>(type_, dict_size_, std::move(buffers), 0);
+    *out = ArrayData::Make(type_, dict_size_, std::move(buffers), 0);
     return Status::OK();
   }
 
@@ -634,7 +576,7 @@ class HashTableKernel<Type, Action, enable_if_fixed_size_binary<Type>>
     BufferVector buffers = {nullptr, nullptr};
     RETURN_NOT_OK(dict_data_.Finish(&buffers[1]));
 
-    *out = std::make_shared<ArrayData>(type_, dict_size_, std::move(buffers), 0);
+    *out = ArrayData::Make(type_, dict_size_, std::move(buffers), 0);
     return Status::OK();
   }
 

cpp/src/arrow/gpu/CMakeLists.txt

@@ -54,7 +54,7 @@ configure_file(cuda_version.h.in
   @ONLY)
 
 install(FILES
-  "${CMAKE_CURRENT_SOURCE_DIR}/cuda_version.h"
+  "${CMAKE_CURRENT_BINARY_DIR}/cuda_version.h"
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/gpu")
 
 install(FILES

cpp/src/arrow/gpu/cuda_context.cc

@@ -69,28 +69,27 @@ class CudaContext::CudaContextImpl {
     return Status::OK();
   }
 
-  Status CopyHostToDevice(uint8_t* dst, const uint8_t* src, int64_t nbytes) {
+  Status CopyHostToDevice(void* dst, const void* src, int64_t nbytes) {
     CU_RETURN_NOT_OK(cuCtxSetCurrent(context_));
-    CU_RETURN_NOT_OK(cuMemcpyHtoD(reinterpret_cast<CUdeviceptr>(dst),
-                                  reinterpret_cast<const void*>(src),
+    CU_RETURN_NOT_OK(cuMemcpyHtoD(reinterpret_cast<CUdeviceptr>(dst), src,
                                   static_cast<size_t>(nbytes)));
     return Status::OK();
   }
 
-  Status CopyDeviceToHost(uint8_t* dst, const uint8_t* src, int64_t nbytes) {
+  Status CopyDeviceToHost(void* dst, const void* src, int64_t nbytes) {
     CU_RETURN_NOT_OK(cuCtxSetCurrent(context_));
     CU_RETURN_NOT_OK(cuMemcpyDtoH(dst, reinterpret_cast<const CUdeviceptr>(src),
                                   static_cast<size_t>(nbytes)));
     return Status::OK();
   }
 
-  Status Free(uint8_t* device_ptr, int64_t nbytes) {
+  Status Free(void* device_ptr, int64_t nbytes) {
     CU_RETURN_NOT_OK(cuMemFree(reinterpret_cast<CUdeviceptr>(device_ptr)));
     bytes_allocated_ -= nbytes;
     return Status::OK();
   }
 
-  Status ExportIpcBuffer(uint8_t* data, std::unique_ptr<CudaIpcMemHandle>* handle) {
+  Status ExportIpcBuffer(void* data, std::unique_ptr<CudaIpcMemHandle>* handle) {
     CU_RETURN_NOT_OK(cuCtxSetCurrent(context_));
     CUipcMemHandle cu_handle;
     CU_RETURN_NOT_OK(cuIpcGetMemHandle(&cu_handle, reinterpret_cast<CUdeviceptr>(data)));
@@ -145,7 +144,7 @@ class CudaDeviceManager::CudaDeviceManagerImpl {
     return Status::OK();
   }
 
-  Status FreeHost(uint8_t* data, int64_t nbytes) {
+  Status FreeHost(void* data, int64_t nbytes) {
     CU_RETURN_NOT_OK(cuMemFreeHost(data));
     host_bytes_allocated_ -= nbytes;
     return Status::OK();
@@ -221,7 +220,7 @@ Status CudaDeviceManager::AllocateHost(int64_t nbytes,
   return Status::OK();
 }
 
-Status CudaDeviceManager::FreeHost(uint8_t* data, int64_t nbytes) {
+Status CudaDeviceManager::FreeHost(void* data, int64_t nbytes) {
   return impl_->FreeHost(data, nbytes);
 }
 
@@ -241,22 +240,22 @@ Status CudaContext::Allocate(int64_t nbytes, std::shared_ptr<CudaBuffer>* out) {
   return Status::OK();
 }
 
-Status CudaContext::ExportIpcBuffer(uint8_t* data,
+Status CudaContext::ExportIpcBuffer(void* data,
                                     std::unique_ptr<CudaIpcMemHandle>* handle) {
   return impl_->ExportIpcBuffer(data, handle);
 }
 
-Status CudaContext::CopyHostToDevice(uint8_t* dst, const uint8_t* src, int64_t nbytes) {
+Status CudaContext::CopyHostToDevice(void* dst, const void* src, int64_t nbytes) {
   return impl_->CopyHostToDevice(dst, src, nbytes);
 }
 
-Status CudaContext::CopyDeviceToHost(uint8_t* dst, const uint8_t* src, int64_t nbytes) {
+Status CudaContext::CopyDeviceToHost(void* dst, const void* src, int64_t nbytes) {
   return impl_->CopyDeviceToHost(dst, src, nbytes);
 }
 
 Status CudaContext::Close() { return impl_->Close(); }
 
-Status CudaContext::Free(uint8_t* device_ptr, int64_t nbytes) {
+Status CudaContext::Free(void* device_ptr, int64_t nbytes) {
   return impl_->Free(device_ptr, nbytes);
 }
 

cpp/src/arrow/gpu/cuda_context.h

@@ -46,7 +46,7 @@ class ARROW_EXPORT CudaDeviceManager {
 
   Status AllocateHost(int64_t nbytes, std::shared_ptr<CudaHostBuffer>* buffer);
 
-  Status FreeHost(uint8_t* data, int64_t nbytes);
+  Status FreeHost(void* data, int64_t nbytes);
 
   int num_devices() const;
 
@@ -88,10 +88,10 @@ class ARROW_EXPORT CudaContext : public std::enable_shared_from_this<CudaContext
  private:
   CudaContext();
 
-  Status ExportIpcBuffer(uint8_t* data, std::unique_ptr<CudaIpcMemHandle>* handle);
-  Status CopyHostToDevice(uint8_t* dst, const uint8_t* src, int64_t nbytes);
-  Status CopyDeviceToHost(uint8_t* dst, const uint8_t* src, int64_t nbytes);
-  Status Free(uint8_t* device_ptr, int64_t nbytes);
+  Status ExportIpcBuffer(void* data, std::unique_ptr<CudaIpcMemHandle>* handle);
+  Status CopyHostToDevice(void* dst, const void* src, int64_t nbytes);
+  Status CopyDeviceToHost(void* dst, const void* src, int64_t nbytes);
+  Status Free(void* device_ptr, int64_t nbytes);
 
   class CudaContextImpl;
   std::unique_ptr<CudaContextImpl> impl_;

cpp/src/arrow/gpu/cuda_memory.cc

@@ -101,11 +101,11 @@ CudaBuffer::CudaBuffer(const std::shared_ptr<CudaBuffer>& parent, const int64_t
       is_ipc_(false) {}
 
 Status CudaBuffer::CopyToHost(const int64_t position, const int64_t nbytes,
-                              uint8_t* out) const {
+                              void* out) const {
   return context_->CopyDeviceToHost(out, data_ + position, nbytes);
 }
 
-Status CudaBuffer::CopyFromHost(const int64_t position, const uint8_t* data,
+Status CudaBuffer::CopyFromHost(const int64_t position, const void* data,
                                 int64_t nbytes) {
   DCHECK_LE(nbytes, size_ - position) << "Copy would overflow buffer";
   return context_->CopyHostToDevice(mutable_data_ + position, data, nbytes);
@@ -134,7 +134,7 @@ CudaBufferReader::CudaBufferReader(const std::shared_ptr<CudaBuffer>& buffer)
 
 CudaBufferReader::~CudaBufferReader() {}
 
-Status CudaBufferReader::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
+Status CudaBufferReader::Read(int64_t nbytes, int64_t* bytes_read, void* buffer) {
   nbytes = std::min(nbytes, size_ - position_);
   *bytes_read = nbytes;
   RETURN_NOT_OK(context_->CopyDeviceToHost(buffer, data_ + position_, nbytes));
@@ -190,7 +190,7 @@ class CudaBufferWriter::CudaBufferWriterImpl {
     return Status::OK();
   }
 
-  Status Write(const uint8_t* data, int64_t nbytes) {
+  Status Write(const void* data, int64_t nbytes) {
     if (nbytes == 0) {
       return Status::OK();
     }
@@ -214,7 +214,7 @@ class CudaBufferWriter::CudaBufferWriterImpl {
     return Status::OK();
   }
 
-  Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) {
+  Status WriteAt(int64_t position, const void* data, int64_t nbytes) {
     std::lock_guard<std::mutex> guard(lock_);
     RETURN_NOT_OK(Seek(position));
     return Write(data, nbytes);
@@ -269,11 +269,11 @@ Status CudaBufferWriter::Seek(int64_t position) {
 
 Status CudaBufferWriter::Tell(int64_t* position) const { return impl_->Tell(position); }
 
-Status CudaBufferWriter::Write(const uint8_t* data, int64_t nbytes) {
+Status CudaBufferWriter::Write(const void* data, int64_t nbytes) {
   return impl_->Write(data, nbytes);
 }
 
-Status CudaBufferWriter::WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) {
+Status CudaBufferWriter::WriteAt(int64_t position, const void* data, int64_t nbytes) {
   return impl_->WriteAt(position, data, nbytes);
 }
 

cpp/src/arrow/gpu/cuda_memory.h

@@ -49,14 +49,14 @@ class ARROW_EXPORT CudaBuffer : public Buffer {
   /// \brief Copy memory from GPU device to CPU host
   /// \param[out] out a pre-allocated output buffer
   /// \return Status
-  Status CopyToHost(const int64_t position, const int64_t nbytes, uint8_t* out) const;
+  Status CopyToHost(const int64_t position, const int64_t nbytes, void* out) const;
 
   /// \brief Copy memory to device at position
   /// \param[in] position start position to copy bytes
   /// \param[in] data the host data to copy
   /// \param[in] nbytes number of bytes to copy
   /// \return Status
-  Status CopyFromHost(const int64_t position, const uint8_t* data, int64_t nbytes);
+  Status CopyFromHost(const int64_t position, const void* data, int64_t nbytes);
 
   /// \brief Expose this device buffer as IPC memory which can be used in other processes
   /// \param[out] handle the exported IPC handle
@@ -130,7 +130,7 @@ class ARROW_EXPORT CudaBufferReader : public io::BufferReader {
   /// \param[in] nbytes number of bytes to read
   /// \param[out] bytes_read actual number of bytes read
   /// \param[out] buffer pre-allocated memory to write into
-  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
+  Status Read(int64_t nbytes, int64_t* bytes_read, void* buffer) override;
 
   /// \brief Zero-copy read from device memory
   /// \param[in] nbytes number of bytes to read
@@ -158,9 +158,9 @@ class ARROW_EXPORT CudaBufferWriter : public io::WriteableFile {
 
   Status Seek(int64_t position) override;
 
-  Status Write(const uint8_t* data, int64_t nbytes) override;
+  Status Write(const void* data, int64_t nbytes) override;
 
-  Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) override;
+  Status WriteAt(int64_t position, const void* data, int64_t nbytes) override;
 
   Status Tell(int64_t* position) const override;
 

cpp/src/arrow/io/file.cc

@@ -394,11 +394,11 @@ class OSFile {
     return Status::OK();
   }
 
-  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
-    return FileRead(fd_, out, nbytes, bytes_read);
+  Status Read(int64_t nbytes, int64_t* bytes_read, void* out) {
+    return FileRead(fd_, reinterpret_cast<uint8_t*>(out), nbytes, bytes_read);
   }
 
-  Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
+  Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, void* out) {
     std::lock_guard<std::mutex> guard(lock_);
     RETURN_NOT_OK(Seek(position));
     return Read(nbytes, bytes_read, out);
@@ -413,12 +413,12 @@ class OSFile {
 
   Status Tell(int64_t* pos) const { return FileTell(fd_, pos); }
 
-  Status Write(const uint8_t* data, int64_t length) {
+  Status Write(const void* data, int64_t length) {
     std::lock_guard<std::mutex> guard(lock_);
     if (length < 0) {
       return Status::IOError("Length must be non-negative");
     }
-    return FileWrite(fd_, data, length);
+    return FileWrite(fd_, reinterpret_cast<const uint8_t*>(data), length);
   }
 
   int fd() const { return fd_; }
@@ -504,13 +504,13 @@ Status ReadableFile::Close() { return impl_->Close(); }
 
 Status ReadableFile::Tell(int64_t* pos) const { return impl_->Tell(pos); }
 
-Status ReadableFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
+Status ReadableFile::Read(int64_t nbytes, int64_t* bytes_read, void* out) {
   std::lock_guard<std::mutex> guard(impl_->lock());
   return impl_->Read(nbytes, bytes_read, out);
 }
 
 Status ReadableFile::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
-                            uint8_t* out) {
+                            void* out) {
   return impl_->ReadAt(position, nbytes, bytes_read, out);
 }
 
@@ -570,7 +570,7 @@ Status FileOutputStream::Close() { return impl_->Close(); }
 
 Status FileOutputStream::Tell(int64_t* pos) const { return impl_->Tell(pos); }
 
-Status FileOutputStream::Write(const uint8_t* data, int64_t length) {
+Status FileOutputStream::Write(const void* data, int64_t length) {
   return impl_->Write(data, length);
 }
 
@@ -710,7 +710,7 @@ Status MemoryMappedFile::Close() {
   return Status::OK();
 }
 
-Status MemoryMappedFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
+Status MemoryMappedFile::Read(int64_t nbytes, int64_t* bytes_read, void* out) {
   nbytes = std::max<int64_t>(
       0, std::min(nbytes, memory_map_->size() - memory_map_->position()));
   if (nbytes > 0) {
@@ -735,7 +735,7 @@ Status MemoryMappedFile::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
 }
 
 Status MemoryMappedFile::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
-                                uint8_t* out) {
+                                void* out) {
   std::lock_guard<std::mutex> guard(memory_map_->lock());
   RETURN_NOT_OK(Seek(position));
   return Read(nbytes, bytes_read, out);
@@ -750,7 +750,7 @@ Status MemoryMappedFile::ReadAt(int64_t position, int64_t nbytes,
 
 bool MemoryMappedFile::supports_zero_copy() const { return true; }
 
-Status MemoryMappedFile::WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) {
+Status MemoryMappedFile::WriteAt(int64_t position, const void* data, int64_t nbytes) {
   std::lock_guard<std::mutex> guard(memory_map_->lock());
 
   if (!memory_map_->opened() || !memory_map_->writable()) {
@@ -761,7 +761,7 @@ Status MemoryMappedFile::WriteAt(int64_t position, const uint8_t* data, int64_t
   return WriteInternal(data, nbytes);
 }
 
-Status MemoryMappedFile::Write(const uint8_t* data, int64_t nbytes) {
+Status MemoryMappedFile::Write(const void* data, int64_t nbytes) {
   std::lock_guard<std::mutex> guard(memory_map_->lock());
 
   if (!memory_map_->opened() || !memory_map_->writable()) {
@@ -773,7 +773,7 @@ Status MemoryMappedFile::Write(const uint8_t* data, int64_t nbytes) {
   return WriteInternal(data, nbytes);
 }
 
-Status MemoryMappedFile::WriteInternal(const uint8_t* data, int64_t nbytes) {
+Status MemoryMappedFile::WriteInternal(const void* data, int64_t nbytes) {
   memcpy(memory_map_->head(), data, static_cast<size_t>(nbytes));
   memory_map_->advance(nbytes);
   return Status::OK();

cpp/src/arrow/io/file.h

@@ -59,7 +59,7 @@ class ARROW_EXPORT FileOutputStream : public OutputStream {
   Status Tell(int64_t* position) const override;
 
   // Write bytes to the stream. Thread-safe
-  Status Write(const uint8_t* data, int64_t nbytes) override;
+  Status Write(const void* data, int64_t nbytes) override;
 
   int file_descriptor() const;
 
@@ -93,12 +93,12 @@ class ARROW_EXPORT ReadableFile : public RandomAccessFile {
   Status Tell(int64_t* position) const override;
 
   // Read bytes from the file. Thread-safe
-  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
+  Status Read(int64_t nbytes, int64_t* bytes_read, void* buffer) override;
   Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
 
   /// \brief Thread-safe implementation of ReadAt
   Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
-                uint8_t* out) override;
+                void* out) override;
 
   /// \brief Thread-safe implementation of ReadAt
   Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) override;
@@ -141,13 +141,13 @@ class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface {
   Status Seek(int64_t position) override;
 
   // Required by RandomAccessFile, copies memory into out. Not thread-safe
-  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) override;
+  Status Read(int64_t nbytes, int64_t* bytes_read, void* out) override;
 
   // Zero copy read. Not thread-safe
   Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
 
   Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
-                uint8_t* out) override;
+                void* out) override;
 
   /// Default implementation is thread-safe
   Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) override;
@@ -155,10 +155,10 @@ class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface {
   bool supports_zero_copy() const override;
 
   /// Write data at the current position in the file. Thread-safe
-  Status Write(const uint8_t* data, int64_t nbytes) override;
+  Status Write(const void* data, int64_t nbytes) override;
 
   /// Write data at a particular position in the file. Thread-safe
-  Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) override;
+  Status WriteAt(int64_t position, const void* data, int64_t nbytes) override;
 
   // @return: the size in bytes of the memory source
   Status GetSize(int64_t* size) override;
@@ -168,7 +168,7 @@ class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface {
  private:
   MemoryMappedFile();
 
-  Status WriteInternal(const uint8_t* data, int64_t nbytes);
+  Status WriteInternal(const void* data, int64_t nbytes);
 
   class ARROW_NO_EXPORT MemoryMap;
   std::shared_ptr<MemoryMap> memory_map_;

cpp/src/arrow/io/hdfs.cc

@@ -119,7 +119,7 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
     return Status::OK();
   }
 
-  Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
+  Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, void* buffer) {
     tSize ret;
     if (driver_->HasPread()) {
       ret = driver_->Pread(fs_, file_, static_cast<tOffset>(position),
@@ -149,11 +149,11 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
     return Status::OK();
   }
 
-  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
+  Status Read(int64_t nbytes, int64_t* bytes_read, void* buffer) {
     int64_t total_bytes = 0;
     while (total_bytes < nbytes) {
       tSize ret = driver_->Read(
-          fs_, file_, reinterpret_cast<void*>(buffer + total_bytes),
+          fs_, file_, reinterpret_cast<uint8_t*>(buffer) + total_bytes,
           static_cast<tSize>(std::min<int64_t>(buffer_size_, nbytes - total_bytes)));
       RETURN_NOT_OK(CheckReadResult(ret));
       total_bytes += ret;
@@ -212,7 +212,7 @@ HdfsReadableFile::~HdfsReadableFile() { DCHECK(impl_->Close().ok()); }
 Status HdfsReadableFile::Close() { return impl_->Close(); }
 
 Status HdfsReadableFile::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
-                                uint8_t* buffer) {
+                                void* buffer) {
   return impl_->ReadAt(position, nbytes, bytes_read, buffer);
 }
 
@@ -223,7 +223,7 @@ Status HdfsReadableFile::ReadAt(int64_t position, int64_t nbytes,
 
 bool HdfsReadableFile::supports_zero_copy() const { return false; }
 
-Status HdfsReadableFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
+Status HdfsReadableFile::Read(int64_t nbytes, int64_t* bytes_read, void* buffer) {
   return impl_->Read(nbytes, bytes_read, buffer);
 }
 
@@ -261,7 +261,7 @@ class HdfsOutputStream::HdfsOutputStreamImpl : public HdfsAnyFileImpl {
     return Status::OK();
   }
 
-  Status Write(const uint8_t* buffer, int64_t nbytes, int64_t* bytes_written) {
+  Status Write(const void* buffer, int64_t nbytes, int64_t* bytes_written) {
     std::lock_guard<std::mutex> guard(lock_);
     tSize ret = driver_->Write(fs_, file_, reinterpret_cast<const void*>(buffer),
                                static_cast<tSize>(nbytes));
@@ -277,12 +277,11 @@ HdfsOutputStream::~HdfsOutputStream() { DCHECK(impl_->Close().ok()); }
 
 Status HdfsOutputStream::Close() { return impl_->Close(); }
 
-Status HdfsOutputStream::Write(const uint8_t* buffer, int64_t nbytes,
-                               int64_t* bytes_read) {
+Status HdfsOutputStream::Write(const void* buffer, int64_t nbytes, int64_t* bytes_read) {
   return impl_->Write(buffer, nbytes, bytes_read);
 }
 
-Status HdfsOutputStream::Write(const uint8_t* buffer, int64_t nbytes) {
+Status HdfsOutputStream::Write(const void* buffer, int64_t nbytes) {
   int64_t bytes_written_dummy = 0;
   return Write(buffer, nbytes, &bytes_written_dummy);
 }

cpp/src/arrow/io/hdfs.h

@@ -182,12 +182,12 @@ class ARROW_EXPORT HdfsReadableFile : public RandomAccessFile {
 
   // NOTE: If you wish to read a particular range of a file in a multithreaded
   // context, you may prefer to use ReadAt to avoid locking issues
-  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
+  Status Read(int64_t nbytes, int64_t* bytes_read, void* buffer) override;
 
   Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
 
   Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
-                uint8_t* buffer) override;
+                void* buffer) override;
 
   Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) override;
 
@@ -217,9 +217,9 @@ class ARROW_EXPORT HdfsOutputStream : public OutputStream {
 
   Status Close() override;
 
-  Status Write(const uint8_t* buffer, int64_t nbytes) override;
+  Status Write(const void* buffer, int64_t nbytes) override;
 
-  Status Write(const uint8_t* buffer, int64_t nbytes, int64_t* bytes_written);
+  Status Write(const void* buffer, int64_t nbytes, int64_t* bytes_written);
 
   Status Flush() override;
 

cpp/src/arrow/io/interfaces.cc

@@ -38,7 +38,7 @@ RandomAccessFile::RandomAccessFile()
     : impl_(new RandomAccessFile::RandomAccessFileImpl()) {}
 
 Status RandomAccessFile::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
-                                uint8_t* out) {
+                                void* out) {
   std::lock_guard<std::mutex> lock(impl_->lock_);
   RETURN_NOT_OK(Seek(position));
   return Read(nbytes, bytes_read, out);
@@ -51,12 +51,11 @@ Status RandomAccessFile::ReadAt(int64_t position, int64_t nbytes,
   return Read(nbytes, out);
 }
 
-Status Writeable::Write(const std::string& data) {
-  return Write(reinterpret_cast<const uint8_t*>(data.c_str()),
-               static_cast<int64_t>(data.size()));
+Status Writable::Write(const std::string& data) {
+  return Write(data.c_str(), static_cast<int64_t>(data.size()));
 }
 
-Status Writeable::Flush() { return Status::OK(); }
+Status Writable::Flush() { return Status::OK(); }
 
 }  // namespace io
 }  // namespace arrow

cpp/src/arrow/io/interfaces.h

@@ -86,11 +86,11 @@ class ARROW_EXPORT Seekable {
   virtual Status Seek(int64_t position) = 0;
 };
 
-class ARROW_EXPORT Writeable {
+class ARROW_EXPORT Writable {
  public:
-  virtual ~Writeable() = default;
+  virtual ~Writable() = default;
 
-  virtual Status Write(const uint8_t* data, int64_t nbytes) = 0;
+  virtual Status Write(const void* data, int64_t nbytes) = 0;
 
   /// \brief Flush buffered bytes, if any
   virtual Status Flush();
@@ -102,13 +102,13 @@ class ARROW_EXPORT Readable {
  public:
   virtual ~Readable() = default;
 
-  virtual Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) = 0;
+  virtual Status Read(int64_t nbytes, int64_t* bytes_read, void* out) = 0;
 
   // Does not copy if not necessary
   virtual Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) = 0;
 };
 
-class ARROW_EXPORT OutputStream : virtual public FileInterface, public Writeable {
+class ARROW_EXPORT OutputStream : virtual public FileInterface, public Writable {
  protected:
   OutputStream() = default;
 };
@@ -138,7 +138,7 @@ class ARROW_EXPORT RandomAccessFile : public InputStream, public Seekable {
   /// \param[out] out The buffer to read bytes into
   /// \return Status
   virtual Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
-                        uint8_t* out) = 0;
+                        void* out) = 0;
 
   /// \brief Read nbytes at position, provide default implementations using Read(...), but
   /// can be overridden. Default implementation is thread-safe.
@@ -162,7 +162,7 @@ class ARROW_EXPORT RandomAccessFile : public InputStream, public Seekable {
 
 class ARROW_EXPORT WriteableFile : public OutputStream, public Seekable {
  public:
-  virtual Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) = 0;
+  virtual Status WriteAt(int64_t position, const void* data, int64_t nbytes) = 0;
 
  protected:
   WriteableFile() = default;

cpp/src/arrow/io/io-file-test.cc

@@ -135,7 +135,7 @@ TEST_F(TestFileOutputStream, Close) {
   OpenFile();
 
   const char* data = "testdata";
-  ASSERT_OK(file_->Write(reinterpret_cast<const uint8_t*>(data), strlen(data)));
+  ASSERT_OK(file_->Write(data, strlen(data)));
 
   int fd = file_->file_descriptor();
   ASSERT_OK(file_->Close());
@@ -158,7 +158,7 @@ TEST_F(TestFileOutputStream, InvalidWrites) {
 
   const char* data = "";
 
-  ASSERT_RAISES(IOError, file_->Write(reinterpret_cast<const uint8_t*>(data), -1));
+  ASSERT_RAISES(IOError, file_->Write(data, -1));
 }
 
 TEST_F(TestFileOutputStream, Tell) {
@@ -170,7 +170,7 @@ TEST_F(TestFileOutputStream, Tell) {
   ASSERT_EQ(0, position);
 
   const char* data = "testdata";
-  ASSERT_OK(file_->Write(reinterpret_cast<const uint8_t*>(data), 8));
+  ASSERT_OK(file_->Write(data, 8));
   ASSERT_OK(file_->Tell(&position));
   ASSERT_EQ(8, position);
 }
@@ -179,7 +179,7 @@ TEST_F(TestFileOutputStream, TruncatesNewFile) {
   ASSERT_OK(FileOutputStream::Open(path_, &file_));
 
   const char* data = "testdata";
-  ASSERT_OK(file_->Write(reinterpret_cast<const uint8_t*>(data), strlen(data)));
+  ASSERT_OK(file_->Write(data, strlen(data)));
   ASSERT_OK(file_->Close());
 
   ASSERT_OK(FileOutputStream::Open(path_, &file_));
@@ -583,8 +583,7 @@ TEST_F(TestMemoryMappedFile, ThreadSafety) {
 
   std::shared_ptr<MemoryMappedFile> file;
   ASSERT_OK(MemoryMappedFile::Open(path, FileMode::READWRITE, &file));
-  ASSERT_OK(file->Write(reinterpret_cast<const uint8_t*>(data.c_str()),
-                        static_cast<int64_t>(data.size())));
+  ASSERT_OK(file->Write(data.c_str(), static_cast<int64_t>(data.size())));
 
   std::atomic<int> correct_count(0);
   constexpr int niter = 10000;

cpp/src/arrow/io/io-memory-test.cc

@@ -93,7 +93,7 @@ TEST(TestFixedSizeBufferWriter, Basics) {
 
   std::string data = "data123456";
   auto nbytes = static_cast<int64_t>(data.size());
-  ASSERT_OK(writer.Write(reinterpret_cast<const uint8_t*>(data.c_str()), nbytes));
+  ASSERT_OK(writer.Write(data.c_str(), nbytes));
 
   ASSERT_OK(writer.Tell(&position));
   ASSERT_EQ(nbytes, position);

cpp/src/arrow/io/memory.cc

@@ -79,7 +79,7 @@ Status BufferOutputStream::Tell(int64_t* position) const {
   return Status::OK();
 }
 
-Status BufferOutputStream::Write(const uint8_t* data, int64_t nbytes) {
+Status BufferOutputStream::Write(const void* data, int64_t nbytes) {
   if (ARROW_PREDICT_FALSE(!is_open_)) {
     return Status::IOError("OutputStream is closed");
   }
@@ -116,7 +116,7 @@ Status MockOutputStream::Tell(int64_t* position) const {
   return Status::OK();
 }
 
-Status MockOutputStream::Write(const uint8_t* data, int64_t nbytes) {
+Status MockOutputStream::Write(const void* data, int64_t nbytes) {
   extent_bytes_written_ += nbytes;
   return Status::OK();
 }
@@ -162,9 +162,10 @@ class FixedSizeBufferWriter::FixedSizeBufferWriterImpl {
     return Status::OK();
   }
 
-  Status Write(const uint8_t* data, int64_t nbytes) {
+  Status Write(const void* data, int64_t nbytes) {
     if (nbytes > memcopy_threshold_ && memcopy_num_threads_ > 1) {
-      internal::parallel_memcopy(mutable_data_ + position_, data, nbytes,
+      internal::parallel_memcopy(mutable_data_ + position_,
+                                 reinterpret_cast<const uint8_t*>(data), nbytes,
                                  memcopy_blocksize_, memcopy_num_threads_);
     } else {
       memcpy(mutable_data_ + position_, data, nbytes);
@@ -173,7 +174,7 @@ class FixedSizeBufferWriter::FixedSizeBufferWriterImpl {
     return Status::OK();
   }
 
-  Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) {
+  Status WriteAt(int64_t position, const void* data, int64_t nbytes) {
     std::lock_guard<std::mutex> guard(lock_);
     RETURN_NOT_OK(Seek(position));
     return Write(data, nbytes);
@@ -210,11 +211,11 @@ Status FixedSizeBufferWriter::Tell(int64_t* position) const {
   return impl_->Tell(position);
 }
 
-Status FixedSizeBufferWriter::Write(const uint8_t* data, int64_t nbytes) {
+Status FixedSizeBufferWriter::Write(const void* data, int64_t nbytes) {
   return impl_->Write(data, nbytes);
 }
 
-Status FixedSizeBufferWriter::WriteAt(int64_t position, const uint8_t* data,
+Status FixedSizeBufferWriter::WriteAt(int64_t position, const void* data,
                                       int64_t nbytes) {
   return impl_->WriteAt(position, data, nbytes);
 }
@@ -240,6 +241,9 @@ BufferReader::BufferReader(const std::shared_ptr<Buffer>& buffer)
 BufferReader::BufferReader(const uint8_t* data, int64_t size)
     : buffer_(nullptr), data_(data), size_(size), position_(0) {}
 
+BufferReader::BufferReader(const Buffer& buffer)
+    : BufferReader(buffer.data(), buffer.size()) {}
+
 Status BufferReader::Close() {
   // no-op
   return Status::OK();
@@ -252,7 +256,7 @@ Status BufferReader::Tell(int64_t* position) const {
 
 bool BufferReader::supports_zero_copy() const { return true; }
 
-Status BufferReader::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
+Status BufferReader::Read(int64_t nbytes, int64_t* bytes_read, void* buffer) {
   memcpy(buffer, data_ + position_, nbytes);
   *bytes_read = std::min(nbytes, size_ - position_);
   position_ += *bytes_read;
@@ -273,7 +277,7 @@ Status BufferReader::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
 }
 
 Status BufferReader::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
-                            uint8_t* out) {
+                            void* out) {
   return RandomAccessFile::ReadAt(position, nbytes, bytes_read, out);
 }
 

cpp/src/arrow/io/memory.h

@@ -48,7 +48,7 @@ class ARROW_EXPORT BufferOutputStream : public OutputStream {
   // Implement the OutputStream interface
   Status Close() override;
   Status Tell(int64_t* position) const override;
-  Status Write(const uint8_t* data, int64_t nbytes) override;
+  Status Write(const void* data, int64_t nbytes) override;
 
   /// Close the stream and return the buffer
   Status Finish(std::shared_ptr<Buffer>* result);
@@ -72,7 +72,7 @@ class ARROW_EXPORT MockOutputStream : public OutputStream {
   // Implement the OutputStream interface
   Status Close() override;
   Status Tell(int64_t* position) const override;
-  Status Write(const uint8_t* data, int64_t nbytes) override;
+  Status Write(const void* data, int64_t nbytes) override;
 
   int64_t GetExtentBytesWritten() const { return extent_bytes_written_; }
 
@@ -90,8 +90,8 @@ class ARROW_EXPORT FixedSizeBufferWriter : public WriteableFile {
   Status Close() override;
   Status Seek(int64_t position) override;
   Status Tell(int64_t* position) const override;
-  Status Write(const uint8_t* data, int64_t nbytes) override;
-  Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) override;
+  Status Write(const void* data, int64_t nbytes) override;
+  Status WriteAt(int64_t position, const void* data, int64_t nbytes) override;
 
   void set_memcopy_threads(int num_threads);
   void set_memcopy_blocksize(int64_t blocksize);
@@ -107,16 +107,17 @@ class ARROW_EXPORT FixedSizeBufferWriter : public WriteableFile {
 class ARROW_EXPORT BufferReader : public RandomAccessFile {
  public:
   explicit BufferReader(const std::shared_ptr<Buffer>& buffer);
+  explicit BufferReader(const Buffer& buffer);
   BufferReader(const uint8_t* data, int64_t size);
 
   Status Close() override;
   Status Tell(int64_t* position) const override;
-  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
+  Status Read(int64_t nbytes, int64_t* bytes_read, void* buffer) override;
 
   // Zero copy read
   Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
   Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
-                uint8_t* out) override;
+                void* out) override;
 
   /// Default implementation is thread-safe
   Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) override;

cpp/src/arrow/ipc/feather-test.cc

@@ -366,15 +366,14 @@ TEST_F(TestTableWriter, TimeTypes) {
   ArrayFromVector<Date32Type, int32_t>(is_valid, date_values_vec, &date_array);
 
   const auto& prim_values = static_cast<const PrimitiveArray&>(*values);
-  std::vector<std::shared_ptr<Buffer>> buffers = {prim_values.null_bitmap(),
-                                                  prim_values.values()};
+  BufferVector buffers = {prim_values.null_bitmap(), prim_values.values()};
 
   std::vector<std::shared_ptr<ArrayData>> arrays;
   arrays.push_back(date_array->data());
 
   for (int i = 1; i < schema->num_fields(); ++i) {
-    arrays.emplace_back(std::make_shared<ArrayData>(
-        schema->field(i)->type(), values->length(), buffers, values->null_count(), 0));
+    arrays.emplace_back(ArrayData::Make(schema->field(i)->type(), values->length(),
+                                        BufferVector(buffers), values->null_count(), 0));
   }
 
   auto batch = RecordBatch::Make(schema, values->length(), std::move(arrays));

cpp/src/arrow/ipc/feather.cc

@@ -371,7 +371,7 @@ class TableReader::TableReaderImpl {
     buffers.push_back(SliceBuffer(buffer, offset, buffer->size() - offset));
 
     auto arr_data =
-        std::make_shared<ArrayData>(type, meta->length(), buffers, meta->null_count());
+        ArrayData::Make(type, meta->length(), std::move(buffers), meta->null_count());
     *out = MakeArray(arr_data);
     return Status::OK();
   }
@@ -523,10 +523,8 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
     uint32_t buffer_size = static_cast<uint32_t>(bytes_written);
 
     // Footer: metadata length, magic bytes
-    RETURN_NOT_OK(
-        stream_->Write(reinterpret_cast<const uint8_t*>(&buffer_size), sizeof(uint32_t)));
-    return stream_->Write(reinterpret_cast<const uint8_t*>(kFeatherMagicBytes),
-                          strlen(kFeatherMagicBytes));
+    RETURN_NOT_OK(stream_->Write(&buffer_size, sizeof(uint32_t)));
+    return stream_->Write(kFeatherMagicBytes, strlen(kFeatherMagicBytes));
   }
 
   Status LoadArrayMetadata(const Array& values, ArrayMetadata* meta) {

cpp/src/arrow/ipc/ipc-json-test.cc

@@ -39,6 +39,7 @@
 
 namespace arrow {
 namespace ipc {
+namespace internal {
 namespace json {
 
 void TestSchemaRoundTrip(const Schema& schema) {
@@ -46,7 +47,7 @@ void TestSchemaRoundTrip(const Schema& schema) {
   rj::Writer<rj::StringBuffer> writer(sb);
 
   writer.StartObject();
-  ASSERT_OK(internal::WriteSchema(schema, &writer));
+  ASSERT_OK(WriteSchema(schema, &writer));
   writer.EndObject();
 
   std::string json_schema = sb.GetString();
@@ -55,7 +56,7 @@ void TestSchemaRoundTrip(const Schema& schema) {
   d.Parse(json_schema);
 
   std::shared_ptr<Schema> out;
-  if (!internal::ReadSchema(d, default_memory_pool(), &out).ok()) {
+  if (!ReadSchema(d, default_memory_pool(), &out).ok()) {
     FAIL() << "Unable to read JSON schema: " << json_schema;
   }
 
@@ -70,7 +71,7 @@ void TestArrayRoundTrip(const Array& array) {
   rj::StringBuffer sb;
   rj::Writer<rj::StringBuffer> writer(sb);
 
-  ASSERT_OK(internal::WriteArray(name, array, &writer));
+  ASSERT_OK(WriteArray(name, array, &writer));
 
   std::string array_as_json = sb.GetString();
 
@@ -82,7 +83,7 @@ void TestArrayRoundTrip(const Array& array) {
   }
 
   std::shared_ptr<Array> out;
-  ASSERT_OK(internal::ReadArray(default_memory_pool(), d, array.type(), &out));
+  ASSERT_OK(ReadArray(default_memory_pool(), d, array.type(), &out));
 
   // std::cout << array_as_json << std::endl;
   CompareArraysDetailed(0, *out, array);
@@ -415,5 +416,6 @@ TEST_P(TestJsonRoundTrip, RoundTrip) {
 INSTANTIATE_TEST_CASE_P(TestJsonRoundTrip, TestJsonRoundTrip, BATCH_CASES());
 
 }  // namespace json
+}  // namespace internal
 }  // namespace ipc
 }  // namespace arrow

cpp/src/arrow/ipc/json-integration-test.cc

@@ -50,8 +50,7 @@ DEFINE_bool(verbose, true, "Verbose output");
 namespace fs = boost::filesystem;
 
 namespace arrow {
-
-class Buffer;
+namespace ipc {
 
 bool file_exists(const char* path) {
   std::ifstream handle(path);
@@ -73,16 +72,15 @@ static Status ConvertJsonToArrow(const std::string& json_path,
   std::shared_ptr<Buffer> json_buffer;
   RETURN_NOT_OK(in_file->Read(file_size, &json_buffer));
 
-  std::unique_ptr<ipc::JsonReader> reader;
-  RETURN_NOT_OK(ipc::JsonReader::Open(json_buffer, &reader));
+  std::unique_ptr<internal::json::JsonReader> reader;
+  RETURN_NOT_OK(internal::json::JsonReader::Open(json_buffer, &reader));
 
   if (FLAGS_verbose) {
     std::cout << "Found schema: " << reader->schema()->ToString() << std::endl;
   }
 
-  std::shared_ptr<ipc::RecordBatchWriter> writer;
-  RETURN_NOT_OK(
-      ipc::RecordBatchFileWriter::Open(out_file.get(), reader->schema(), &writer));
+  std::shared_ptr<RecordBatchWriter> writer;
+  RETURN_NOT_OK(RecordBatchFileWriter::Open(out_file.get(), reader->schema(), &writer));
 
   for (int i = 0; i < reader->num_record_batches(); ++i) {
     std::shared_ptr<RecordBatch> batch;
@@ -101,15 +99,15 @@ static Status ConvertArrowToJson(const std::string& arrow_path,
   RETURN_NOT_OK(io::ReadableFile::Open(arrow_path, &in_file));
   RETURN_NOT_OK(io::FileOutputStream::Open(json_path, &out_file));
 
-  std::shared_ptr<ipc::RecordBatchFileReader> reader;
-  RETURN_NOT_OK(ipc::RecordBatchFileReader::Open(in_file.get(), &reader));
+  std::shared_ptr<RecordBatchFileReader> reader;
+  RETURN_NOT_OK(RecordBatchFileReader::Open(in_file.get(), &reader));
 
   if (FLAGS_verbose) {
     std::cout << "Found schema: " << reader->schema()->ToString() << std::endl;
   }
 
-  std::unique_ptr<ipc::JsonWriter> writer;
-  RETURN_NOT_OK(ipc::JsonWriter::Open(reader->schema(), &writer));
+  std::unique_ptr<internal::json::JsonWriter> writer;
+  RETURN_NOT_OK(internal::json::JsonWriter::Open(reader->schema(), &writer));
 
   for (int i = 0; i < reader->num_record_batches(); ++i) {
     std::shared_ptr<RecordBatch> batch;
@@ -119,8 +117,7 @@ static Status ConvertArrowToJson(const std::string& arrow_path,
 
   std::string result;
   RETURN_NOT_OK(writer->Finish(&result));
-  return out_file->Write(reinterpret_cast<const uint8_t*>(result.c_str()),
-                         static_cast<int64_t>(result.size()));
+  return out_file->Write(result.c_str(), static_cast<int64_t>(result.size()));
 }
 
 static Status ValidateArrowVsJson(const std::string& arrow_path,
@@ -135,15 +132,15 @@ static Status ValidateArrowVsJson(const std::string& arrow_path,
   std::shared_ptr<Buffer> json_buffer;
   RETURN_NOT_OK(json_file->Read(file_size, &json_buffer));
 
-  std::unique_ptr<ipc::JsonReader> json_reader;
-  RETURN_NOT_OK(ipc::JsonReader::Open(json_buffer, &json_reader));
+  std::unique_ptr<internal::json::JsonReader> json_reader;
+  RETURN_NOT_OK(internal::json::JsonReader::Open(json_buffer, &json_reader));
 
   // Construct Arrow reader
   std::shared_ptr<io::ReadableFile> arrow_file;
   RETURN_NOT_OK(io::ReadableFile::Open(arrow_path, &arrow_file));
 
-  std::shared_ptr<ipc::RecordBatchFileReader> arrow_reader;
-  RETURN_NOT_OK(ipc::RecordBatchFileReader::Open(arrow_file.get(), &arrow_reader));
+  std::shared_ptr<RecordBatchFileReader> arrow_reader;
+  RETURN_NOT_OK(RecordBatchFileReader::Open(arrow_file.get(), &arrow_reader));
 
   auto json_schema = json_reader->schema();
   auto arrow_schema = arrow_reader->schema();
@@ -250,8 +247,7 @@ class TestJSONIntegration : public ::testing::Test {
     do {
       std::shared_ptr<io::FileOutputStream> out;
       RETURN_NOT_OK(io::FileOutputStream::Open(path, &out));
-      RETURN_NOT_OK(out->Write(reinterpret_cast<const uint8_t*>(data),
-                               static_cast<int64_t>(strlen(data))));
+      RETURN_NOT_OK(out->Write(data, static_cast<int64_t>(strlen(data))));
     } while (0);
     return Status::OK();
   }
@@ -401,6 +397,7 @@ TEST_F(TestJSONIntegration, ErrorStates) {
   ASSERT_RAISES(Invalid, RunCommand(json_path, "", "VALIDATE"));
 }
 
+}  // namespace ipc
 }  // namespace arrow
 
 int main(int argc, char** argv) {
@@ -409,7 +406,7 @@ int main(int argc, char** argv) {
   int ret = 0;
 
   if (FLAGS_integration) {
-    arrow::Status result = arrow::RunCommand(FLAGS_json, FLAGS_arrow, FLAGS_mode);
+    arrow::Status result = arrow::ipc::RunCommand(FLAGS_json, FLAGS_arrow, FLAGS_mode);
     if (!result.ok()) {
       std::cout << "Error message: " << result.ToString() << std::endl;
       ret = 1;

cpp/src/arrow/ipc/json-internal.cc

@@ -40,24 +40,11 @@
 
 namespace arrow {
 namespace ipc {
-namespace json {
 namespace internal {
+namespace json {
 
-static std::string GetBufferTypeName(BufferType type) {
-  switch (type) {
-    case BufferType::DATA:
-      return "DATA";
-    case BufferType::OFFSET:
-      return "OFFSET";
-    case BufferType::TYPE:
-      return "TYPE";
-    case BufferType::VALIDITY:
-      return "VALIDITY";
-    default:
-      break;
-  }
-  return "UNKNOWN";
-}
+using ::arrow::ipc::DictionaryMemo;
+using ::arrow::ipc::DictionaryTypeMap;
 
 static std::string GetFloatingPrecisionName(FloatingPoint::Precision precision) {
   switch (precision) {
@@ -174,12 +161,9 @@ class SchemaWriter {
       RETURN_NOT_OK(WriteDictionaryMetadata(dict_type));
 
       const DataType& dictionary_type = *dict_type.dictionary()->type();
-      const DataType& index_type = *dict_type.index_type();
       RETURN_NOT_OK(WriteChildren(dictionary_type.children()));
-      WriteBufferLayout(index_type.GetBufferLayout());
     } else {
       RETURN_NOT_OK(WriteChildren(type.children()));
-      WriteBufferLayout(type.GetBufferLayout());
     }
 
     writer_->EndObject();
@@ -301,26 +285,6 @@ class SchemaWriter {
     return Status::OK();
   }
 
-  void WriteBufferLayout(const std::vector<BufferDescr>& buffer_layout) {
-    writer_->Key("typeLayout");
-    writer_->StartObject();
-    writer_->Key("vectors");
-    writer_->StartArray();
-
-    for (const BufferDescr& buffer : buffer_layout) {
-      writer_->StartObject();
-      writer_->Key("type");
-      writer_->String(GetBufferTypeName(buffer.type()));
-
-      writer_->Key("typeBitWidth");
-      writer_->Int(buffer.bit_width());
-
-      writer_->EndObject();
-    }
-    writer_->EndArray();
-    writer_->EndObject();
-  }
-
   Status WriteChildren(const std::vector<std::shared_ptr<Field>>& children) {
     writer_->Key("children");
     writer_->StartArray();
@@ -1502,7 +1466,7 @@ Status ReadArray(MemoryPool* pool, const rj::Value& json_array, const Schema& sc
   return ReadArray(pool, json_array, result->type(), array);
 }
 
-}  // namespace internal
 }  // namespace json
+}  // namespace internal
 }  // namespace ipc
 }  // namespace arrow

cpp/src/arrow/ipc/json-internal.h

@@ -92,8 +92,8 @@ using RjObject = rj::Value::ConstObject;
 
 namespace arrow {
 namespace ipc {
-namespace json {
 namespace internal {
+namespace json {
 
 Status WriteSchema(const Schema& schema, RjWriter* writer);
 Status WriteRecordBatch(const RecordBatch& batch, RjWriter* writer);
@@ -111,8 +111,8 @@ Status ReadArray(MemoryPool* pool, const rj::Value& json_obj,
 Status ReadArray(MemoryPool* pool, const rj::Value& json_obj, const Schema& schema,
                  std::shared_ptr<Array>* array);
 
-}  // namespace internal
 }  // namespace json
+}  // namespace internal
 }  // namespace ipc
 }  // namespace arrow
 

cpp/src/arrow/ipc/json.cc

@@ -33,6 +33,8 @@ using std::size_t;
 
 namespace arrow {
 namespace ipc {
+namespace internal {
+namespace json {
 
 // ----------------------------------------------------------------------
 // Writer implementation
@@ -45,7 +47,7 @@ class JsonWriter::JsonWriterImpl {
 
   Status Start() {
     writer_->StartObject();
-    RETURN_NOT_OK(json::internal::WriteSchema(*schema_, writer_.get()));
+    RETURN_NOT_OK(json::WriteSchema(*schema_, writer_.get()));
 
     // Record batches
     writer_->Key("batches");
@@ -63,7 +65,7 @@ class JsonWriter::JsonWriterImpl {
 
   Status WriteRecordBatch(const RecordBatch& batch) {
     DCHECK_EQ(batch.num_columns(), schema_->num_fields());
-    return json::internal::WriteRecordBatch(batch, writer_.get());
+    return json::WriteRecordBatch(batch, writer_.get());
   }
 
  private:
@@ -106,7 +108,7 @@ class JsonReader::JsonReaderImpl {
       return Status::IOError("JSON parsing failed");
     }
 
-    RETURN_NOT_OK(json::internal::ReadSchema(doc_, pool_, &schema_));
+    RETURN_NOT_OK(json::ReadSchema(doc_, pool_, &schema_));
 
     auto it = doc_.FindMember("batches");
     RETURN_NOT_ARRAY("batches", it, doc_);
@@ -120,8 +122,7 @@ class JsonReader::JsonReaderImpl {
     DCHECK_LT(i, static_cast<int>(record_batches_->GetArray().Size()))
         << "i out of bounds";
 
-    return json::internal::ReadRecordBatch(record_batches_->GetArray()[i], schema_, pool_,
-                                           batch);
+    return json::ReadRecordBatch(record_batches_->GetArray()[i], schema_, pool_, batch);
   }
 
   std::shared_ptr<Schema> schema() const { return schema_; }
@@ -164,5 +165,7 @@ Status JsonReader::ReadRecordBatch(int i, std::shared_ptr<RecordBatch>* batch) c
   return impl_->ReadRecordBatch(i, batch);
 }
 
+}  // namespace json
+}  // namespace internal
 }  // namespace ipc
 }  // namespace arrow

cpp/src/arrow/ipc/json.h

@@ -34,12 +34,14 @@ class RecordBatch;
 class Schema;
 
 namespace ipc {
+namespace internal {
+namespace json {
 
 /// \class JsonWriter
 /// \brief Write the JSON representation of an Arrow record batch file or stream
 ///
 /// This is used for integration testing
-class ARROW_EXPORT JsonWriter {
+class JsonWriter {
  public:
   ~JsonWriter();
 
@@ -72,7 +74,7 @@ class ARROW_EXPORT JsonWriter {
 /// \brief Read the JSON representation of an Arrow record batch file or stream
 ///
 /// This is used for integration testing
-class ARROW_EXPORT JsonReader {
+class JsonReader {
  public:
   ~JsonReader();
 
@@ -113,6 +115,8 @@ class ARROW_EXPORT JsonReader {
   std::unique_ptr<JsonReaderImpl> impl_;
 };
 
+}  // namespace json
+}  // namespace internal
 }  // namespace ipc
 }  // namespace arrow
 

cpp/src/arrow/ipc/message.cc

@@ -236,11 +236,35 @@ Status ReadMessage(io::InputStream* file, std::unique_ptr<Message>* message) {
 // ----------------------------------------------------------------------
 // Implement InputStream message reader
 
-Status InputStreamMessageReader::ReadNextMessage(std::unique_ptr<Message>* message) {
-  return ReadMessage(stream_, message);
+/// \brief Implementation of MessageReader that reads from InputStream
+class InputStreamMessageReader : public MessageReader {
+ public:
+  explicit InputStreamMessageReader(io::InputStream* stream) : stream_(stream) {}
+
+  explicit InputStreamMessageReader(const std::shared_ptr<io::InputStream>& owned_stream)
+      : InputStreamMessageReader(owned_stream.get()) {
+    owned_stream_ = owned_stream;
+  }
+
+  ~InputStreamMessageReader() {}
+
+  Status ReadNextMessage(std::unique_ptr<Message>* message) {
+    return ReadMessage(stream_, message);
+  }
+
+ private:
+  io::InputStream* stream_;
+  std::shared_ptr<io::InputStream> owned_stream_;
+};
+
+std::unique_ptr<MessageReader> MessageReader::Open(io::InputStream* stream) {
+  return std::unique_ptr<MessageReader>(new InputStreamMessageReader(stream));
 }
 
-InputStreamMessageReader::~InputStreamMessageReader() {}
+std::unique_ptr<MessageReader> MessageReader::Open(
+    const std::shared_ptr<io::InputStream>& owned_stream) {
+  return std::unique_ptr<MessageReader>(new InputStreamMessageReader(owned_stream));
+}
 
 }  // namespace ipc
 }  // namespace arrow

cpp/src/arrow/ipc/message.h

@@ -144,6 +144,13 @@ class ARROW_EXPORT MessageReader {
  public:
   virtual ~MessageReader() = default;
 
+  /// \brief Create MessageReader that reads from InputStream
+  static std::unique_ptr<MessageReader> Open(io::InputStream* stream);
+
+  /// \brief Create MessageReader that reads from owned InputStream
+  static std::unique_ptr<MessageReader> Open(
+      const std::shared_ptr<io::InputStream>& owned_stream);
+
   /// \brief Read next Message from the interface
   ///
   /// \param[out] message an arrow::ipc::Message instance
@@ -151,26 +158,6 @@ class ARROW_EXPORT MessageReader {
   virtual Status ReadNextMessage(std::unique_ptr<Message>* message) = 0;
 };
 
-/// \brief Implementation of MessageReader that reads from InputStream
-/// \since 0.5.0
-class ARROW_EXPORT InputStreamMessageReader : public MessageReader {
- public:
-  explicit InputStreamMessageReader(io::InputStream* stream) : stream_(stream) {}
-
-  explicit InputStreamMessageReader(const std::shared_ptr<io::InputStream>& owned_stream)
-      : InputStreamMessageReader(owned_stream.get()) {
-    owned_stream_ = owned_stream;
-  }
-
-  ~InputStreamMessageReader();
-
-  Status ReadNextMessage(std::unique_ptr<Message>* message) override;
-
- private:
-  io::InputStream* stream_;
-  std::shared_ptr<io::InputStream> owned_stream_;
-};
-
 /// \brief Read encapulated RPC message from position in file
 ///
 /// Read a length-prefixed message flatbuffer starting at the indicated file

cpp/src/arrow/ipc/metadata-internal.cc

@@ -54,7 +54,6 @@ using DictionaryOffset = flatbuffers::Offset<flatbuf::DictionaryEncoding>;
 using FieldOffset = flatbuffers::Offset<flatbuf::Field>;
 using KeyValueOffset = flatbuffers::Offset<flatbuf::KeyValue>;
 using RecordBatchOffset = flatbuffers::Offset<flatbuf::RecordBatch>;
-using VectorLayoutOffset = flatbuffers::Offset<arrow::flatbuf::VectorLayout>;
 using Offset = flatbuffers::Offset<void>;
 using FBString = flatbuffers::Offset<flatbuffers::String>;
 
@@ -341,34 +340,8 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
 // TODO(wesm): Convert this to visitor pattern
 static Status TypeToFlatbuffer(FBB& fbb, const DataType& type,
                                std::vector<FieldOffset>* children,
-                               std::vector<VectorLayoutOffset>* layout,
                                flatbuf::Type* out_type, DictionaryMemo* dictionary_memo,
                                Offset* offset) {
-  std::vector<BufferDescr> buffer_layout = type.GetBufferLayout();
-  for (const BufferDescr& descr : buffer_layout) {
-    flatbuf::VectorType vector_type;
-    switch (descr.type()) {
-      case BufferType::OFFSET:
-        vector_type = flatbuf::VectorType_OFFSET;
-        break;
-      case BufferType::DATA:
-        vector_type = flatbuf::VectorType_DATA;
-        break;
-      case BufferType::VALIDITY:
-        vector_type = flatbuf::VectorType_VALIDITY;
-        break;
-      case BufferType::TYPE:
-        vector_type = flatbuf::VectorType_TYPE;
-        break;
-      default:
-        vector_type = flatbuf::VectorType_DATA;
-        break;
-    }
-    auto offset = flatbuf::CreateVectorLayout(
-        fbb, static_cast<int16_t>(descr.bit_width()), vector_type);
-    layout->push_back(offset);
-  }
-
   const DataType* value_type = &type;
 
   if (type.id() == Type::DICTIONARY) {
@@ -543,14 +516,11 @@ static Status FieldToFlatbuffer(FBB& fbb, const Field& field,
 
   flatbuf::Type type_enum;
   Offset type_offset;
-  Offset type_layout;
   std::vector<FieldOffset> children;
-  std::vector<VectorLayoutOffset> layout;
 
-  RETURN_NOT_OK(TypeToFlatbuffer(fbb, *field.type(), &children, &layout, &type_enum,
+  RETURN_NOT_OK(TypeToFlatbuffer(fbb, *field.type(), &children, &type_enum,
                                  dictionary_memo, &type_offset));
   auto fb_children = fbb.CreateVector(children);
-  auto fb_layout = fbb.CreateVector(layout);
 
   DictionaryOffset dictionary = 0;
   if (field.type()->id() == Type::DICTIONARY) {
@@ -560,7 +530,7 @@ static Status FieldToFlatbuffer(FBB& fbb, const Field& field,
 
   // TODO: produce the list of VectorTypes
   *offset = flatbuf::CreateField(fbb, fb_name, field.nullable(), type_enum, type_offset,
-                                 dictionary, fb_children, fb_layout);
+                                 dictionary, fb_children);
 
   return Status::OK();
 }
@@ -945,8 +915,7 @@ Status WriteMessage(const Buffer& message, io::OutputStream* file,
 
   // Write the flatbuffer size prefix including padding
   int32_t flatbuffer_size = padded_message_length - 4;
-  RETURN_NOT_OK(
-      file->Write(reinterpret_cast<const uint8_t*>(&flatbuffer_size), sizeof(int32_t)));
+  RETURN_NOT_OK(file->Write(&flatbuffer_size, sizeof(int32_t)));
 
   // Write the flatbuffer
   RETURN_NOT_OK(file->Write(message.data(), message.size()));

cpp/src/arrow/ipc/reader.cc

@@ -480,14 +480,12 @@ Status RecordBatchStreamReader::Open(std::unique_ptr<MessageReader> message_read
 
 Status RecordBatchStreamReader::Open(io::InputStream* stream,
                                      std::shared_ptr<RecordBatchReader>* out) {
-  std::unique_ptr<MessageReader> message_reader(new InputStreamMessageReader(stream));
-  return Open(std::move(message_reader), out);
+  return Open(MessageReader::Open(stream), out);
 }
 
 Status RecordBatchStreamReader::Open(const std::shared_ptr<io::InputStream>& stream,
                                      std::shared_ptr<RecordBatchReader>* out) {
-  std::unique_ptr<MessageReader> message_reader(new InputStreamMessageReader(stream));
-  return Open(std::move(message_reader), out);
+  return Open(MessageReader::Open(stream), out);
 }
 
 std::shared_ptr<Schema> RecordBatchStreamReader::schema() const {
@@ -717,14 +715,17 @@ Status ReadTensor(int64_t offset, io::RandomAccessFile* file,
 
   std::unique_ptr<Message> message;
   RETURN_NOT_OK(ReadContiguousPayload(file, &message));
+  return ReadTensor(*message, out);
+}
 
+Status ReadTensor(const Message& message, std::shared_ptr<Tensor>* out) {
   std::shared_ptr<DataType> type;
   std::vector<int64_t> shape;
   std::vector<int64_t> strides;
   std::vector<std::string> dim_names;
-  RETURN_NOT_OK(internal::GetTensorMetadata(*message->metadata(), &type, &shape, &strides,
+  RETURN_NOT_OK(internal::GetTensorMetadata(*message.metadata(), &type, &shape, &strides,
                                             &dim_names));
-  *out = std::make_shared<Tensor>(type, message->body(), shape, strides, dim_names);
+  *out = std::make_shared<Tensor>(type, message.body(), shape, strides, dim_names);
   return Status::OK();
 }
 

cpp/src/arrow/ipc/reader.h

@@ -219,7 +219,7 @@ Status ReadRecordBatch(const Buffer& metadata, const std::shared_ptr<Schema>& sc
                        int max_recursion_depth, io::RandomAccessFile* file,
                        std::shared_ptr<RecordBatch>* out);
 
-/// EXPERIMENTAL: Read arrow::Tensor as encapsulated IPC message in file
+/// \brief EXPERIMENTAL: Read arrow::Tensor as encapsulated IPC message in file
 ///
 /// \param[in] offset the file location of the start of the message
 /// \param[in] file the file where the batch is located
@@ -229,6 +229,14 @@ ARROW_EXPORT
 Status ReadTensor(int64_t offset, io::RandomAccessFile* file,
                   std::shared_ptr<Tensor>* out);
 
+/// \brief EXPERIMENTAL: Read arrow::Tensor from IPC message
+///
+/// \param[in] message a Message containing the tensor metadata and body
+/// \param[out] out the read tensor
+/// \return Status
+ARROW_EXPORT
+Status ReadTensor(const Message& message, std::shared_ptr<Tensor>* out);
+
 }  // namespace ipc
 }  // namespace arrow
 

cpp/src/arrow/ipc/writer.cc

@@ -560,9 +560,18 @@ Status WriteLargeRecordBatch(const RecordBatch& batch, int64_t buffer_start_offs
                           pool, kMaxNestingDepth, true);
 }
 
-static Status WriteStridedTensorData(int dim_index, int64_t offset, int elem_size,
-                                     const Tensor& tensor, uint8_t* scratch_space,
-                                     io::OutputStream* dst) {
+namespace {
+
+Status WriteTensorHeader(const Tensor& tensor, io::OutputStream* dst,
+                         int32_t* metadata_length, int64_t* body_length) {
+  std::shared_ptr<Buffer> metadata;
+  RETURN_NOT_OK(internal::WriteTensorMessage(tensor, 0, &metadata));
+  return internal::WriteMessage(*metadata, dst, metadata_length);
+}
+
+Status WriteStridedTensorData(int dim_index, int64_t offset, int elem_size,
+                              const Tensor& tensor, uint8_t* scratch_space,
+                              io::OutputStream* dst) {
   if (dim_index == tensor.ndim() - 1) {
     const uint8_t* data_ptr = tensor.raw_data() + offset;
     const int64_t stride = tensor.strides()[dim_index];
@@ -580,16 +589,37 @@ static Status WriteStridedTensorData(int dim_index, int64_t offset, int elem_siz
   return Status::OK();
 }
 
-Status WriteTensorHeader(const Tensor& tensor, io::OutputStream* dst,
-                         int32_t* metadata_length, int64_t* body_length) {
-  RETURN_NOT_OK(AlignStreamPosition(dst));
-  std::shared_ptr<Buffer> metadata;
-  RETURN_NOT_OK(internal::WriteTensorMessage(tensor, 0, &metadata));
-  return internal::WriteMessage(*metadata, dst, metadata_length);
+Status GetContiguousTensor(const Tensor& tensor, MemoryPool* pool,
+                           std::unique_ptr<Tensor>* out) {
+  const auto& type = static_cast<const FixedWidthType&>(*tensor.type());
+  const int elem_size = type.bit_width() / 8;
+
+  // TODO(wesm): Do we care enough about this temporary allocation to pass in
+  // a MemoryPool to this function?
+  std::shared_ptr<Buffer> scratch_space;
+  RETURN_NOT_OK(AllocateBuffer(default_memory_pool(),
+                               tensor.shape()[tensor.ndim() - 1] * elem_size,
+                               &scratch_space));
+
+  std::shared_ptr<ResizableBuffer> contiguous_data;
+  RETURN_NOT_OK(
+      AllocateResizableBuffer(pool, tensor.size() * elem_size, &contiguous_data));
+
+  io::BufferOutputStream stream(contiguous_data);
+  RETURN_NOT_OK(WriteStridedTensorData(0, 0, elem_size, tensor,
+                                       scratch_space->mutable_data(), &stream));
+
+  out->reset(new Tensor(tensor.type(), contiguous_data, tensor.shape()));
+
+  return Status::OK();
 }
 
+}  // namespace
+
 Status WriteTensor(const Tensor& tensor, io::OutputStream* dst, int32_t* metadata_length,
                    int64_t* body_length) {
+  RETURN_NOT_OK(AlignStreamPosition(dst));
+
   if (tensor.is_contiguous()) {
     RETURN_NOT_OK(WriteTensorHeader(tensor, dst, metadata_length, body_length));
     auto data = tensor.data();
@@ -619,6 +649,22 @@ Status WriteTensor(const Tensor& tensor, io::OutputStream* dst, int32_t* metadat
   }
 }
 
+Status GetTensorMessage(const Tensor& tensor, MemoryPool* pool,
+                        std::unique_ptr<Message>* out) {
+  const Tensor* tensor_to_write = &tensor;
+  std::unique_ptr<Tensor> temp_tensor;
+
+  if (!tensor.is_contiguous()) {
+    RETURN_NOT_OK(GetContiguousTensor(tensor, pool, &temp_tensor));
+    tensor_to_write = temp_tensor.get();
+  }
+
+  std::shared_ptr<Buffer> metadata;
+  RETURN_NOT_OK(internal::WriteTensorMessage(*tensor_to_write, 0, &metadata));
+  out->reset(new Message(metadata, tensor_to_write->data()));
+  return Status::OK();
+}
+
 Status WriteDictionary(int64_t dictionary_id, const std::shared_ptr<Array>& dictionary,
                        int64_t buffer_start_offset, io::OutputStream* dst,
                        int32_t* metadata_length, int64_t* body_length, MemoryPool* pool) {
@@ -651,9 +697,13 @@ Status GetTensorSize(const Tensor& tensor, int64_t* size) {
 
 RecordBatchWriter::~RecordBatchWriter() {}
 
-Status RecordBatchWriter::WriteTable(const Table& table) {
+Status RecordBatchWriter::WriteTable(const Table& table, int64_t max_chunksize) {
   TableBatchReader reader(table);
 
+  if (max_chunksize > 0) {
+    reader.set_chunksize(max_chunksize);
+  }
+
   std::shared_ptr<RecordBatch> batch;
   while (true) {
     RETURN_NOT_OK(reader.ReadNext(&batch));
@@ -666,6 +716,8 @@ Status RecordBatchWriter::WriteTable(const Table& table) {
   return Status::OK();
 }
 
+Status RecordBatchWriter::WriteTable(const Table& table) { return WriteTable(table, -1); }
+
 // ----------------------------------------------------------------------
 // Stream writer implementation
 
@@ -687,7 +739,7 @@ class StreamBookKeeper {
   }
 
   // Write data and update position
-  Status Write(const uint8_t* data, int64_t nbytes) {
+  Status Write(const void* data, int64_t nbytes) {
     RETURN_NOT_OK(sink_->Write(data, nbytes));
     position_ += nbytes;
     return Status::OK();
@@ -776,7 +828,7 @@ class RecordBatchStreamWriter::RecordBatchStreamWriterImpl : public StreamBookKe
 
     // Write 0 EOS message
     const int32_t kEos = 0;
-    return Write(reinterpret_cast<const uint8_t*>(&kEos), sizeof(int32_t));
+    return Write(&kEos, sizeof(int32_t));
   }
 
   Status CheckStarted() {
@@ -864,8 +916,7 @@ class RecordBatchFileWriter::RecordBatchFileWriterImpl
 
   Status Start() override {
     // It is only necessary to align to 8-byte boundary at the start of the file
-    RETURN_NOT_OK(Write(reinterpret_cast<const uint8_t*>(kArrowMagicBytes),
-                        strlen(kArrowMagicBytes)));
+    RETURN_NOT_OK(Write(kArrowMagicBytes, strlen(kArrowMagicBytes)));
     RETURN_NOT_OK(Align());
 
     // We write the schema at the start of the file (and the end). This also
@@ -889,12 +940,10 @@ class RecordBatchFileWriter::RecordBatchFileWriterImpl
       return Status::Invalid("Invalid file footer");
     }
 
-    RETURN_NOT_OK(
-        Write(reinterpret_cast<const uint8_t*>(&footer_length), sizeof(int32_t)));
+    RETURN_NOT_OK(Write(&footer_length, sizeof(int32_t)));
 
     // Write magic bytes to end file
-    return Write(reinterpret_cast<const uint8_t*>(kArrowMagicBytes),
-                 strlen(kArrowMagicBytes));
+    return Write(kArrowMagicBytes, strlen(kArrowMagicBytes));
   }
 };
 

cpp/src/arrow/ipc/writer.h

@@ -65,6 +65,12 @@ class ARROW_EXPORT RecordBatchWriter {
   /// \return Status
   Status WriteTable(const Table& table);
 
+  /// \brief Write Table with a particular chunksize
+  /// \param[in] table table to write
+  /// \param[in] max_chunksize maximum chunk size for table chunks
+  /// \return Status
+  Status WriteTable(const Table& table, int64_t max_chunksize);
+
   /// \brief Perform any logic necessary to finish the stream
   ///
   /// \return Status
@@ -239,6 +245,17 @@ Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size);
 ARROW_EXPORT
 Status GetTensorSize(const Tensor& tensor, int64_t* size);
 
+/// \brief EXPERIMENTAL: Convert arrow::Tensor to a Message with minimal memory
+/// allocation
+///
+/// \param[in] tensor the Tensor to write
+/// \param[in] pool MemoryPool to allocate space for metadata
+/// \param[out] out the resulting Message
+/// \return Status
+ARROW_EXPORT
+Status GetTensorMessage(const Tensor& tensor, MemoryPool* pool,
+                        std::unique_ptr<Message>* out);
+
 /// \brief EXPERIMENTAL: Write arrow::Tensor as a contiguous message
 ///
 /// \param[in] tensor the Tensor to write

cpp/src/arrow/python/arrow_to_pandas.cc

@@ -100,6 +100,7 @@ static inline bool ListTypeSupported(const DataType& type) {
     case Type::UINT64:
     case Type::FLOAT:
     case Type::DOUBLE:
+    case Type::BINARY:
     case Type::STRING:
     case Type::TIMESTAMP:
       // The above types are all supported.
@@ -239,19 +240,14 @@ class PandasBlock {
       block_arr = PyArray_SimpleNewFromDescr(1, block_dims, descr);
     }
 
-    if (block_arr == NULL) {
-      // TODO(wesm): propagating Python exception
-      return Status::OK();
-    }
+    RETURN_IF_PYERROR();
 
     PyArray_ENABLEFLAGS(reinterpret_cast<PyArrayObject*>(block_arr), NPY_ARRAY_OWNDATA);
 
     npy_intp placement_dims[1] = {num_columns_};
     PyObject* placement_arr = PyArray_SimpleNew(1, placement_dims, NPY_INT64);
-    if (placement_arr == NULL) {
-      // TODO(wesm): propagating Python exception
-      return Status::OK();
-    }
+
+    RETURN_IF_PYERROR();
 
     block_arr_.reset(block_arr);
     placement_arr_.reset(placement_arr);
@@ -281,12 +277,19 @@ class PandasBlock {
   ARROW_DISALLOW_COPY_AND_ASSIGN(PandasBlock);
 };
 
+template <typename T>
+inline const T* GetPrimitiveValues(const Array& arr) {
+  const auto& prim_arr = static_cast<const PrimitiveArray&>(arr);
+  const T* raw_values = reinterpret_cast<const T*>(prim_arr.values()->data());
+  return raw_values + arr.offset();
+}
+
 template <typename T>
 inline void ConvertIntegerWithNulls(PandasOptions options, const ChunkedArray& data,
                                     double* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
-    const auto& arr = static_cast<const PrimitiveArray&>(*data.chunk(c));
-    auto in_values = reinterpret_cast<const T*>(arr.raw_values());
+    const auto& arr = *data.chunk(c);
+    const T* in_values = GetPrimitiveValues<T>(arr);
     // Upcast to double, set NaN as appropriate
 
     for (int i = 0; i < arr.length(); ++i) {
@@ -299,8 +302,8 @@ template <typename T>
 inline void ConvertIntegerNoNullsSameType(PandasOptions options, const ChunkedArray& data,
                                           T* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
-    const auto& arr = static_cast<const PrimitiveArray&>(*data.chunk(c));
-    auto in_values = reinterpret_cast<const T*>(arr.raw_values());
+    const auto& arr = *data.chunk(c);
+    const T* in_values = GetPrimitiveValues<T>(arr);
     memcpy(out_values, in_values, sizeof(T) * arr.length());
     out_values += arr.length();
   }
@@ -310,8 +313,8 @@ template <typename InType, typename OutType>
 inline void ConvertIntegerNoNullsCast(PandasOptions options, const ChunkedArray& data,
                                       OutType* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
-    const auto& arr = static_cast<const PrimitiveArray&>(*data.chunk(c));
-    auto in_values = reinterpret_cast<const InType*>(arr.raw_values());
+    const auto& arr = *data.chunk(c);
+    const InType* in_values = GetPrimitiveValues<InType>(arr);
     for (int64_t i = 0; i < arr.length(); ++i) {
       *out_values = in_values[i];
     }
@@ -322,14 +325,13 @@ static Status ConvertBooleanWithNulls(PandasOptions options, const ChunkedArray&
                                       PyObject** out_values) {
   PyAcquireGIL lock;
   for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto bool_arr = static_cast<BooleanArray*>(arr.get());
+    const auto& arr = static_cast<const BooleanArray&>(*data.chunk(c));
 
-    for (int64_t i = 0; i < arr->length(); ++i) {
-      if (bool_arr->IsNull(i)) {
+    for (int64_t i = 0; i < arr.length(); ++i) {
+      if (arr.IsNull(i)) {
         Py_INCREF(Py_None);
         *out_values++ = Py_None;
-      } else if (bool_arr->Value(i)) {
+      } else if (arr.Value(i)) {
         // True
         Py_INCREF(Py_True);
         *out_values++ = Py_True;
@@ -346,10 +348,9 @@ static Status ConvertBooleanWithNulls(PandasOptions options, const ChunkedArray&
 static void ConvertBooleanNoNulls(PandasOptions options, const ChunkedArray& data,
                                   uint8_t* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto bool_arr = static_cast<BooleanArray*>(arr.get());
-    for (int64_t i = 0; i < arr->length(); ++i) {
-      *out_values++ = static_cast<uint8_t>(bool_arr->Value(i));
+    const auto& arr = static_cast<const BooleanArray&>(*data.chunk(c));
+    for (int64_t i = 0; i < arr.length(); ++i) {
+      *out_values++ = static_cast<uint8_t>(arr.Value(i));
     }
   }
 }
@@ -360,17 +361,17 @@ inline Status ConvertBinaryLike(PandasOptions options, const ChunkedArray& data,
   using ArrayType = typename TypeTraits<Type>::ArrayType;
   PyAcquireGIL lock;
   for (int c = 0; c < data.num_chunks(); c++) {
-    auto arr = static_cast<ArrayType*>(data.chunk(c).get());
+    const auto& arr = static_cast<const ArrayType&>(*data.chunk(c));
 
     const uint8_t* data_ptr;
     int32_t length;
     const bool has_nulls = data.null_count() > 0;
-    for (int64_t i = 0; i < arr->length(); ++i) {
-      if (has_nulls && arr->IsNull(i)) {
+    for (int64_t i = 0; i < arr.length(); ++i) {
+      if (has_nulls && arr.IsNull(i)) {
         Py_INCREF(Py_None);
         *out_values = Py_None;
       } else {
-        data_ptr = arr->GetValue(i, &length);
+        data_ptr = arr.GetValue(i, &length);
         *out_values = WrapBytes<ArrayType>::Wrap(data_ptr, length);
         if (*out_values == nullptr) {
           PyErr_Clear();
@@ -480,7 +481,7 @@ inline Status ConvertStruct(PandasOptions options, const ChunkedArray& data,
             Py_INCREF(Py_None);
             field_value.reset(Py_None);
           }
-          // PyDict_SetItemString does not steal the value reference
+          // PyDict_SetItemString increments reference count
           auto setitem_result =
               PyDict_SetItemString(dict_item.obj(), name.c_str(), field_value.obj());
           RETURN_IF_PYERROR();
@@ -529,13 +530,25 @@ inline Status ConvertListsLike(PandasOptions options, const std::shared_ptr<Colu
         PyObject* start = PyLong_FromLongLong(arr->value_offset(i) + chunk_offset);
         PyObject* end = PyLong_FromLongLong(arr->value_offset(i + 1) + chunk_offset);
         PyObject* slice = PySlice_New(start, end, NULL);
+        Py_XDECREF(start);
+        Py_XDECREF(end);
+
+        if (ARROW_PREDICT_FALSE(slice == nullptr)) {
+          // Fall out of loop, will return from RETURN_IF_PYERROR
+          break;
+        }
         *out_values = PyObject_GetItem(numpy_array, slice);
-        Py_DECREF(start);
-        Py_DECREF(end);
-        Py_DECREF(slice);
+
+        if (*out_values == nullptr) {
+          // Fall out of loop, will return from RETURN_IF_PYERROR
+          break;
+        }
+
+        Py_XDECREF(slice);
       }
       ++out_values;
     }
+    RETURN_IF_PYERROR();
 
     chunk_offset += arr->values()->length();
   }
@@ -547,14 +560,12 @@ inline Status ConvertListsLike(PandasOptions options, const std::shared_ptr<Colu
 template <typename T>
 inline void ConvertNumericNullable(const ChunkedArray& data, T na_value, T* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
-    const auto& arr = static_cast<const PrimitiveArray&>(*data.chunk(c));
-    auto in_values = reinterpret_cast<const T*>(arr.raw_values());
-
-    const uint8_t* valid_bits = arr.null_bitmap_data();
+    const auto& arr = *data.chunk(c);
+    const T* in_values = GetPrimitiveValues<T>(arr);
 
     if (arr.null_count() > 0) {
       for (int64_t i = 0; i < arr.length(); ++i) {
-        *out_values++ = BitUtil::BitNotSet(valid_bits, i) ? na_value : in_values[i];
+        *out_values++ = arr.IsNull(i) ? na_value : in_values[i];
       }
     } else {
       memcpy(out_values, in_values, sizeof(T) * arr.length());
@@ -567,8 +578,8 @@ template <typename InType, typename OutType>
 inline void ConvertNumericNullableCast(const ChunkedArray& data, OutType na_value,
                                        OutType* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
-    const auto& arr = static_cast<const PrimitiveArray&>(*data.chunk(c));
-    auto in_values = reinterpret_cast<const InType*>(arr.raw_values());
+    const auto& arr = *data.chunk(c);
+    const InType* in_values = GetPrimitiveValues<InType>(arr);
 
     for (int64_t i = 0; i < arr.length(); ++i) {
       *out_values++ = arr.IsNull(i) ? na_value : static_cast<OutType>(in_values[i]);
@@ -576,11 +587,11 @@ inline void ConvertNumericNullableCast(const ChunkedArray& data, OutType na_valu
   }
 }
 
-template <typename InType, int64_t SHIFT>
+template <typename T, int64_t SHIFT>
 inline void ConvertDatetimeNanos(const ChunkedArray& data, int64_t* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
-    const auto& arr = static_cast<const PrimitiveArray&>(*data.chunk(c));
-    auto in_values = reinterpret_cast<const InType*>(arr.raw_values());
+    const auto& arr = *data.chunk(c);
+    const T* in_values = GetPrimitiveValues<T>(arr);
 
     for (int64_t i = 0; i < arr.length(); ++i) {
       *out_values++ = arr.IsNull(i) ? kPandasTimestampNull
@@ -620,13 +631,6 @@ static Status ConvertTimes(PandasOptions options, const ChunkedArray& data,
   return Status::OK();
 }
 
-static Status RawDecimalToString(const uint8_t* bytes, int scale, std::string* result) {
-  DCHECK_NE(result, nullptr);
-  Decimal128 decimal(bytes);
-  *result = decimal.ToString(scale);
-  return Status::OK();
-}
-
 static Status ConvertDecimals(PandasOptions options, const ChunkedArray& data,
                               PyObject** out_values) {
   PyAcquireGIL lock;
@@ -637,19 +641,14 @@ static Status ConvertDecimals(PandasOptions options, const ChunkedArray& data,
   PyObject* Decimal = Decimal_ref.obj();
 
   for (int c = 0; c < data.num_chunks(); c++) {
-    auto* arr(static_cast<arrow::Decimal128Array*>(data.chunk(c).get()));
-    auto type(std::dynamic_pointer_cast<arrow::Decimal128Type>(arr->type()));
-    const int scale = type->scale();
+    const auto& arr = static_cast<const arrow::Decimal128Array&>(*data.chunk(c));
 
-    for (int64_t i = 0; i < arr->length(); ++i) {
-      if (arr->IsNull(i)) {
+    for (int64_t i = 0; i < arr.length(); ++i) {
+      if (arr.IsNull(i)) {
         Py_INCREF(Py_None);
         *out_values++ = Py_None;
       } else {
-        const uint8_t* raw_value = arr->GetValue(i);
-        std::string decimal_string;
-        RETURN_NOT_OK(RawDecimalToString(raw_value, scale, &decimal_string));
-        *out_values++ = internal::DecimalFromString(Decimal, decimal_string);
+        *out_values++ = internal::DecimalFromString(Decimal, arr.FormatValue(i));
         RETURN_IF_PYERROR();
       }
     }
@@ -707,6 +706,7 @@ class ObjectBlock : public PandasBlock {
         CONVERTLISTSLIKE_CASE(TimestampType, TIMESTAMP)
         CONVERTLISTSLIKE_CASE(FloatType, FLOAT)
         CONVERTLISTSLIKE_CASE(DoubleType, DOUBLE)
+        CONVERTLISTSLIKE_CASE(BinaryType, BINARY)
         CONVERTLISTSLIKE_CASE(StringType, STRING)
         CONVERTLISTSLIKE_CASE(ListType, LIST)
         default: {
@@ -866,7 +866,7 @@ class BoolBlock : public PandasBlock {
     uint8_t* out_buffer =
         reinterpret_cast<uint8_t*>(block_data_) + rel_placement * num_rows_;
 
-    ConvertBooleanNoNulls(options_, *col->data().get(), out_buffer);
+    ConvertBooleanNoNulls(options_, *col->data(), out_buffer);
     placement_data_[rel_placement] = abs_placement;
     return Status::OK();
   }
@@ -894,7 +894,7 @@ class DatetimeBlock : public PandasBlock {
     int64_t* out_buffer =
         reinterpret_cast<int64_t*>(block_data_) + rel_placement * num_rows_;
 
-    const ChunkedArray& data = *col.get()->data();
+    const ChunkedArray& data = *col->data();
 
     if (type == Type::DATE32) {
       // Convert from days since epoch to datetime64[ns]
@@ -1099,7 +1099,7 @@ class CategoricalBlock : public PandasBlock {
   Status AllocateNDArrayFromIndices(int npy_type, const PrimitiveArray& indices) {
     npy_intp block_dims[1] = {num_rows_};
 
-    auto in_values = reinterpret_cast<const T*>(indices.raw_values());
+    const T* in_values = GetPrimitiveValues<T>(indices);
     void* data = const_cast<T*>(in_values);
 
     PyAcquireGIL lock;
@@ -1430,6 +1430,7 @@ class ArrowDeserializer {
     PyAcquireGIL lock;
 
     result_ = NewArray1DFromType(col_->type().get(), type, col_->length(), nullptr);
+    RETURN_IF_PYERROR();
     arr_ = reinterpret_cast<PyArrayObject*>(result_);
     return Status::OK();
   }
@@ -1439,8 +1440,7 @@ class ArrowDeserializer {
                                std::shared_ptr<Array> arr) {
     typedef typename internal::arrow_traits<TYPE>::T T;
 
-    const auto& prim_arr = static_cast<const PrimitiveArray&>(*arr);
-    auto in_values = reinterpret_cast<const T*>(prim_arr.raw_values());
+    const T* in_values = GetPrimitiveValues<T>(*arr);
 
     // Zero-Copy. We can pass the data pointer directly to NumPy.
     void* data = const_cast<T*>(in_values);
@@ -1538,8 +1538,8 @@ class ArrowDeserializer {
     constexpr int64_t kShift = traits::npy_shift;
 
     for (int c = 0; c < data_.num_chunks(); c++) {
-      const auto& arr = static_cast<const PrimitiveArray&>(*data_.chunk(c));
-      auto in_values = reinterpret_cast<const c_type*>(arr.raw_values());
+      const auto& arr = *data_.chunk(c);
+      const c_type* in_values = GetPrimitiveValues<c_type>(arr);
 
       for (int64_t i = 0; i < arr.length(); ++i) {
         *out_values++ = arr.IsNull(i) ? na_value : static_cast<T>(in_values[i]) / kShift;
@@ -1654,6 +1654,7 @@ class ArrowDeserializer {
       CONVERTVALUES_LISTSLIKE_CASE(TimestampType, TIMESTAMP)
       CONVERTVALUES_LISTSLIKE_CASE(FloatType, FLOAT)
       CONVERTVALUES_LISTSLIKE_CASE(DoubleType, DOUBLE)
+      CONVERTVALUES_LISTSLIKE_CASE(BinaryType, BINARY)
       CONVERTVALUES_LISTSLIKE_CASE(StringType, STRING)
       CONVERTVALUES_LISTSLIKE_CASE(Decimal128Type, DECIMAL)
       CONVERTVALUES_LISTSLIKE_CASE(ListType, LIST)

cpp/src/arrow/python/arrow_to_python.cc

@@ -29,15 +29,17 @@
 
 #include "arrow/array.h"
 #include "arrow/io/interfaces.h"
+#include "arrow/io/memory.h"
 #include "arrow/ipc/reader.h"
+#include "arrow/table.h"
+#include "arrow/util/logging.h"
+
 #include "arrow/python/common.h"
 #include "arrow/python/helpers.h"
 #include "arrow/python/numpy_convert.h"
 #include "arrow/python/pyarrow.h"
 #include "arrow/python/python_to_arrow.h"
 #include "arrow/python/util/datetime.h"
-#include "arrow/table.h"
-#include "arrow/util/logging.h"
 
 namespace arrow {
 namespace py {
@@ -286,5 +288,59 @@ Status DeserializeObject(PyObject* context, const SerializedPyObject& obj, PyObj
                          obj, out);
 }
 
+Status GetSerializedFromComponents(int num_tensors, int num_buffers, PyObject* data,
+                                   SerializedPyObject* out) {
+  PyAcquireGIL gil;
+  const Py_ssize_t data_length = PyList_Size(data);
+  RETURN_IF_PYERROR();
+
+  const Py_ssize_t expected_data_length = 1 + num_tensors * 2 + num_buffers;
+  if (data_length != expected_data_length) {
+    return Status::Invalid("Invalid number of buffers in data");
+  }
+
+  auto GetBuffer = [&data](Py_ssize_t index, std::shared_ptr<Buffer>* out) {
+    PyObject* py_buf = PyList_GET_ITEM(data, index);
+    return unwrap_buffer(py_buf, out);
+  };
+
+  Py_ssize_t buffer_index = 0;
+
+  // Read the union batch describing object structure
+  {
+    std::shared_ptr<Buffer> data_buffer;
+    RETURN_NOT_OK(GetBuffer(buffer_index++, &data_buffer));
+    gil.release();
+    io::BufferReader buf_reader(data_buffer);
+    std::shared_ptr<RecordBatchReader> reader;
+    RETURN_NOT_OK(ipc::RecordBatchStreamReader::Open(&buf_reader, &reader));
+    RETURN_NOT_OK(reader->ReadNext(&out->batch));
+    gil.acquire();
+  }
+
+  // Zero-copy reconstruct tensors
+  for (int i = 0; i < num_tensors; ++i) {
+    std::shared_ptr<Buffer> metadata;
+    std::shared_ptr<Buffer> body;
+    std::shared_ptr<Tensor> tensor;
+    RETURN_NOT_OK(GetBuffer(buffer_index++, &metadata));
+    RETURN_NOT_OK(GetBuffer(buffer_index++, &body));
+
+    ipc::Message message(metadata, body);
+
+    RETURN_NOT_OK(ReadTensor(message, &tensor));
+    out->tensors.emplace_back(std::move(tensor));
+  }
+
+  // Unwrap and append buffers
+  for (int i = 0; i < num_buffers; ++i) {
+    std::shared_ptr<Buffer> buffer;
+    RETURN_NOT_OK(GetBuffer(buffer_index++, &buffer));
+    out->buffers.emplace_back(std::move(buffer));
+  }
+
+  return Status::OK();
+}
+
 }  // namespace py
 }  // namespace arrow

cpp/src/arrow/python/arrow_to_python.h

@@ -48,6 +48,19 @@ namespace py {
 ARROW_EXPORT
 Status ReadSerializedObject(io::RandomAccessFile* src, SerializedPyObject* out);
 
+/// \brief Reconstruct SerializedPyObject from representation produced by
+/// SerializedPyObject::GetComponents.
+///
+/// \param[in] num_tensors number of tensors in the object
+/// \param[in] num_buffers number of buffers in the object
+/// \param[in] data a list containing pyarrow.Buffer instances. Must be 1 +
+/// num_tensors * 2 + num_buffers in length
+/// \param[out] out the reconstructed object
+/// \return Status
+ARROW_EXPORT
+Status GetSerializedFromComponents(int num_tensors, int num_buffers, PyObject* data,
+                                   SerializedPyObject* out);
+
 /// \brief Reconstruct Python object from Arrow-serialized representation
 /// \param[in] context Serialization context which contains custom serialization
 /// and deserialization callbacks. Can be any Python object with a

cpp/src/arrow/python/builtin_convert.cc

@@ -37,14 +37,6 @@
 namespace arrow {
 namespace py {
 
-static inline bool IsPyInteger(PyObject* obj) {
-#if PYARROW_IS_PY2
-  return PyLong_Check(obj) || PyInt_Check(obj);
-#else
-  return PyLong_Check(obj);
-#endif
-}
-
 Status InvalidConversion(PyObject* obj, const std::string& expected_types,
                          std::ostream* out) {
   OwnedRef type(PyObject_Type(obj));
@@ -91,7 +83,7 @@ class ScalarVisitor {
       ++bool_count_;
     } else if (PyFloat_Check(obj)) {
       ++float_count_;
-    } else if (IsPyInteger(obj)) {
+    } else if (internal::IsPyInteger(obj)) {
       ++int_count_;
     } else if (PyDate_CheckExact(obj)) {
       ++date_count_;
@@ -156,15 +148,14 @@ static constexpr int MAX_NESTING_LEVELS = 32;
 // SeqVisitor is used to infer the type.
 class SeqVisitor {
  public:
-  SeqVisitor() : max_nesting_level_(0), max_observed_level_(0) {
-    memset(nesting_histogram_, 0, MAX_NESTING_LEVELS * sizeof(int));
+  SeqVisitor() : max_nesting_level_(0), max_observed_level_(0), nesting_histogram_() {
+    std::fill(nesting_histogram_, nesting_histogram_ + MAX_NESTING_LEVELS, 0);
   }
 
   // co-recursive with VisitElem
   Status Visit(PyObject* obj, int level = 0) {
-    if (level > max_nesting_level_) {
-      max_nesting_level_ = level;
-    }
+    max_nesting_level_ = std::max(max_nesting_level_, level);
+
     // Loop through either a sequence or an iterator.
     if (PySequence_Check(obj)) {
       Py_ssize_t size = PySequence_Size(obj);
@@ -173,18 +164,26 @@ class SeqVisitor {
         if (PyArray_Check(obj)) {
           auto array = reinterpret_cast<PyArrayObject*>(obj);
           auto ptr = reinterpret_cast<const char*>(PyArray_GETPTR1(array, i));
+
           ref.reset(PyArray_GETITEM(array, ptr));
+          RETURN_IF_PYERROR();
+
           RETURN_NOT_OK(VisitElem(ref, level));
         } else {
           ref.reset(PySequence_GetItem(obj, i));
+          RETURN_IF_PYERROR();
           RETURN_NOT_OK(VisitElem(ref, level));
         }
       }
     } else if (PyObject_HasAttrString(obj, "__iter__")) {
-      OwnedRef iter = OwnedRef(PyObject_GetIter(obj));
-      PyObject* item;
+      OwnedRef iter(PyObject_GetIter(obj));
+      RETURN_IF_PYERROR();
+
+      PyObject* item = NULLPTR;
       while ((item = PyIter_Next(iter.obj()))) {
-        OwnedRef ref = OwnedRef(item);
+        RETURN_IF_PYERROR();
+
+        OwnedRef ref(item);
         RETURN_NOT_OK(VisitElem(ref, level));
       }
     } else {
@@ -250,6 +249,7 @@ class SeqVisitor {
 
   // Visits a specific element (inner part of the loop).
   Status VisitElem(const OwnedRef& item_ref, int level) {
+    DCHECK_NE(item_ref.obj(), NULLPTR);
     if (PyList_Check(item_ref.obj())) {
       RETURN_NOT_OK(Visit(item_ref.obj(), level + 1));
     } else if (PyDict_Check(item_ref.obj())) {
@@ -331,7 +331,7 @@ class SeqConverter {
 
   virtual Status AppendData(PyObject* seq, int64_t size) = 0;
 
-  virtual ~SeqConverter() {}
+  virtual ~SeqConverter() = default;
 
  protected:
   ArrayBuilder* builder_;
@@ -390,28 +390,26 @@ class TypedConverterVisitor : public TypedConverter<BuilderType> {
     }
     return Status::OK();
   }
-
-  virtual Status AppendItem(const OwnedRef& item) = 0;
 };
 
 class NullConverter : public TypedConverterVisitor<NullBuilder, NullConverter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
+  Status AppendItem(const OwnedRef& item) {
     return Status::Invalid("NullConverter: passed non-None value");
   }
 };
 
 class BoolConverter : public TypedConverterVisitor<BooleanBuilder, BoolConverter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
+  Status AppendItem(const OwnedRef& item) {
     return typed_builder_->Append(item.obj() == Py_True);
   }
 };
 
 class Int8Converter : public TypedConverterVisitor<Int8Builder, Int8Converter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
-    int64_t val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+  Status AppendItem(const OwnedRef& item) {
+    const auto val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
 
     if (ARROW_PREDICT_FALSE(val > std::numeric_limits<int8_t>::max() ||
                             val < std::numeric_limits<int8_t>::min())) {
@@ -426,8 +424,8 @@ class Int8Converter : public TypedConverterVisitor<Int8Builder, Int8Converter> {
 
 class Int16Converter : public TypedConverterVisitor<Int16Builder, Int16Converter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
-    int64_t val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+  Status AppendItem(const OwnedRef& item) {
+    const auto val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
 
     if (ARROW_PREDICT_FALSE(val > std::numeric_limits<int16_t>::max() ||
                             val < std::numeric_limits<int16_t>::min())) {
@@ -442,8 +440,8 @@ class Int16Converter : public TypedConverterVisitor<Int16Builder, Int16Converter
 
 class Int32Converter : public TypedConverterVisitor<Int32Builder, Int32Converter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
-    int64_t val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+  Status AppendItem(const OwnedRef& item) {
+    const auto val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
 
     if (ARROW_PREDICT_FALSE(val > std::numeric_limits<int32_t>::max() ||
                             val < std::numeric_limits<int32_t>::min())) {
@@ -458,8 +456,8 @@ class Int32Converter : public TypedConverterVisitor<Int32Builder, Int32Converter
 
 class Int64Converter : public TypedConverterVisitor<Int64Builder, Int64Converter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
-    int64_t val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+  Status AppendItem(const OwnedRef& item) {
+    const auto val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
     RETURN_IF_PYERROR();
     return typed_builder_->Append(val);
   }
@@ -467,53 +465,53 @@ class Int64Converter : public TypedConverterVisitor<Int64Builder, Int64Converter
 
 class UInt8Converter : public TypedConverterVisitor<UInt8Builder, UInt8Converter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
-    uint64_t val = static_cast<uint64_t>(PyLong_AsLongLong(item.obj()));
+  Status AppendItem(const OwnedRef& item) {
+    const auto val = static_cast<uint64_t>(PyLong_AsLongLong(item.obj()));
+    RETURN_IF_PYERROR();
 
     if (ARROW_PREDICT_FALSE(val > std::numeric_limits<uint8_t>::max())) {
       return Status::Invalid(
           "Cannot coerce values to array type that would "
           "lose data");
     }
-    RETURN_IF_PYERROR();
     return typed_builder_->Append(static_cast<uint8_t>(val));
   }
 };
 
 class UInt16Converter : public TypedConverterVisitor<UInt16Builder, UInt16Converter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
-    uint64_t val = static_cast<uint64_t>(PyLong_AsLongLong(item.obj()));
+  Status AppendItem(const OwnedRef& item) {
+    const auto val = static_cast<uint64_t>(PyLong_AsLongLong(item.obj()));
+    RETURN_IF_PYERROR();
 
     if (ARROW_PREDICT_FALSE(val > std::numeric_limits<uint16_t>::max())) {
       return Status::Invalid(
           "Cannot coerce values to array type that would "
           "lose data");
     }
-    RETURN_IF_PYERROR();
     return typed_builder_->Append(static_cast<uint16_t>(val));
   }
 };
 
 class UInt32Converter : public TypedConverterVisitor<UInt32Builder, UInt32Converter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
-    uint64_t val = static_cast<uint64_t>(PyLong_AsLongLong(item.obj()));
+  Status AppendItem(const OwnedRef& item) {
+    const auto val = static_cast<uint64_t>(PyLong_AsLongLong(item.obj()));
+    RETURN_IF_PYERROR();
 
     if (ARROW_PREDICT_FALSE(val > std::numeric_limits<uint32_t>::max())) {
       return Status::Invalid(
           "Cannot coerce values to array type that would "
           "lose data");
     }
-    RETURN_IF_PYERROR();
     return typed_builder_->Append(static_cast<uint32_t>(val));
   }
 };
 
 class UInt64Converter : public TypedConverterVisitor<UInt64Builder, UInt64Converter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
-    int64_t val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+  Status AppendItem(const OwnedRef& item) {
+    const auto val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
     RETURN_IF_PYERROR();
     return typed_builder_->Append(val);
   }
@@ -521,13 +519,13 @@ class UInt64Converter : public TypedConverterVisitor<UInt64Builder, UInt64Conver
 
 class Date32Converter : public TypedConverterVisitor<Date32Builder, Date32Converter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
+  Status AppendItem(const OwnedRef& item) {
     int32_t t;
     if (PyDate_Check(item.obj())) {
       auto pydate = reinterpret_cast<PyDateTime_Date*>(item.obj());
       t = static_cast<int32_t>(PyDate_to_s(pydate));
     } else {
-      int64_t casted_val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+      const auto casted_val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
       RETURN_IF_PYERROR();
       if (casted_val > std::numeric_limits<int32_t>::max()) {
         return Status::Invalid("Integer as date32 larger than INT32_MAX");
@@ -540,7 +538,7 @@ class Date32Converter : public TypedConverterVisitor<Date32Builder, Date32Conver
 
 class Date64Converter : public TypedConverterVisitor<Date64Builder, Date64Converter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
+  Status AppendItem(const OwnedRef& item) {
     int64_t t;
     if (PyDate_Check(item.obj())) {
       auto pydate = reinterpret_cast<PyDateTime_Date*>(item.obj());
@@ -558,7 +556,7 @@ class TimestampConverter
  public:
   explicit TimestampConverter(TimeUnit::type unit) : unit_(unit) {}
 
-  inline Status AppendItem(const OwnedRef& item) {
+  Status AppendItem(const OwnedRef& item) {
     int64_t t;
     if (PyDateTime_Check(item.obj())) {
       auto pydatetime = reinterpret_cast<PyDateTime_DateTime*>(item.obj());
@@ -590,7 +588,7 @@ class TimestampConverter
 
 class DoubleConverter : public TypedConverterVisitor<DoubleBuilder, DoubleConverter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
+  Status AppendItem(const OwnedRef& item) {
     double val = PyFloat_AsDouble(item.obj());
     RETURN_IF_PYERROR();
     return typed_builder_->Append(val);
@@ -599,7 +597,7 @@ class DoubleConverter : public TypedConverterVisitor<DoubleBuilder, DoubleConver
 
 class BytesConverter : public TypedConverterVisitor<BinaryBuilder, BytesConverter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
+  Status AppendItem(const OwnedRef& item) {
     PyObject* bytes_obj;
     const char* bytes;
     Py_ssize_t length;
@@ -627,7 +625,7 @@ class BytesConverter : public TypedConverterVisitor<BinaryBuilder, BytesConverte
 class FixedWidthBytesConverter
     : public TypedConverterVisitor<FixedSizeBinaryBuilder, FixedWidthBytesConverter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
+  Status AppendItem(const OwnedRef& item) {
     PyObject* bytes_obj;
     OwnedRef tmp;
     Py_ssize_t expected_length =
@@ -654,7 +652,7 @@ class FixedWidthBytesConverter
 
 class UTF8Converter : public TypedConverterVisitor<StringBuilder, UTF8Converter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
+  Status AppendItem(const OwnedRef& item) {
     PyObject* bytes_obj;
     OwnedRef tmp;
     const char* bytes;
@@ -667,7 +665,8 @@ class UTF8Converter : public TypedConverterVisitor<StringBuilder, UTF8Converter>
       RETURN_IF_PYERROR();
       bytes_obj = obj;
     } else if (!PyUnicode_Check(obj)) {
-      PyObjectStringify stringified(obj);
+      OwnedRef repr(PyObject_Repr(obj));
+      PyObjectStringify stringified(repr.obj());
       std::stringstream ss;
       ss << "Non bytes/unicode value encountered: " << stringified.bytes;
       return Status::Invalid(ss.str());
@@ -688,10 +687,10 @@ class ListConverter : public TypedConverterVisitor<ListBuilder, ListConverter> {
  public:
   Status Init(ArrayBuilder* builder) override;
 
-  inline Status AppendItem(const OwnedRef& item) override {
+  Status AppendItem(const OwnedRef& item) {
     RETURN_NOT_OK(typed_builder_->Append());
     PyObject* item_obj = item.obj();
-    int64_t list_size = static_cast<int64_t>(PySequence_Size(item_obj));
+    const auto list_size = static_cast<int64_t>(PySequence_Size(item_obj));
     return value_converter_->AppendData(item_obj, list_size);
   }
 
@@ -702,13 +701,11 @@ class ListConverter : public TypedConverterVisitor<ListBuilder, ListConverter> {
 class DecimalConverter
     : public TypedConverterVisitor<arrow::Decimal128Builder, DecimalConverter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
+  Status AppendItem(const OwnedRef& item) {
     /// TODO(phillipc): Check for nan?
-    std::string string;
-    RETURN_NOT_OK(internal::PythonDecimalToString(item.obj(), &string));
-
     Decimal128 value;
-    RETURN_NOT_OK(Decimal128::FromString(string, &value));
+    const auto& type = static_cast<const DecimalType&>(*typed_builder_->type());
+    RETURN_NOT_OK(internal::DecimalFromPythonDecimal(item.obj(), type, &value));
     return typed_builder_->Append(value);
   }
 };

cpp/src/arrow/python/helpers.cc

@@ -15,8 +15,10 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/python/helpers.h"
+#include <sstream>
+
 #include "arrow/python/common.h"
+#include "arrow/python/helpers.h"
 #include "arrow/util/decimal.h"
 #include "arrow/util/logging.h"
 
@@ -91,20 +93,33 @@ Status PythonDecimalToString(PyObject* python_decimal, std::string* out) {
   return Status::OK();
 }
 
-Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int* precision,
-                                     int* scale) {
-  // Call Python's str(decimal_object)
-  OwnedRef str_obj(PyObject_Str(python_decimal));
+Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int32_t* precision,
+                                     int32_t* scale) {
+  DCHECK_NE(python_decimal, NULLPTR);
+  DCHECK_NE(precision, NULLPTR);
+  DCHECK_NE(scale, NULLPTR);
+
+  OwnedRef as_tuple(PyObject_CallMethod(python_decimal, "as_tuple", "()"));
   RETURN_IF_PYERROR();
-  PyObjectStringify str(str_obj.obj());
+  DCHECK(PyTuple_Check(as_tuple.obj()));
 
-  const char* bytes = str.bytes;
-  DCHECK_NE(bytes, nullptr);
+  OwnedRef digits(PyObject_GetAttrString(as_tuple.obj(), "digits"));
+  RETURN_IF_PYERROR();
+  DCHECK(PyTuple_Check(digits.obj()));
 
-  auto size = str.size;
+  const auto num_digits = static_cast<int32_t>(PyTuple_Size(digits.obj()));
+  RETURN_IF_PYERROR();
 
-  std::string c_string(bytes, size);
-  return Decimal128::FromString(c_string, nullptr, precision, scale);
+  OwnedRef py_exponent(PyObject_GetAttrString(as_tuple.obj(), "exponent"));
+  RETURN_IF_PYERROR();
+  DCHECK(IsPyInteger(py_exponent.obj()));
+
+  const auto exponent = static_cast<int32_t>(PyLong_AsLong(py_exponent.obj()));
+  RETURN_IF_PYERROR();
+
+  *precision = num_digits;
+  *scale = -exponent;
+  return Status::OK();
 }
 
 PyObject* DecimalFromString(PyObject* decimal_constructor,
@@ -121,6 +136,46 @@ PyObject* DecimalFromString(PyObject* decimal_constructor,
                                string_size);
 }
 
+Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arrow_type,
+                                Decimal128* out) {
+  DCHECK_NE(python_decimal, NULLPTR);
+  DCHECK_NE(out, NULLPTR);
+
+  std::string string;
+  RETURN_NOT_OK(PythonDecimalToString(python_decimal, &string));
+
+  int32_t inferred_precision;
+  int32_t inferred_scale;
+
+  RETURN_NOT_OK(
+      Decimal128::FromString(string, out, &inferred_precision, &inferred_scale));
+
+  const int32_t precision = arrow_type.precision();
+  const int32_t scale = arrow_type.scale();
+
+  if (ARROW_PREDICT_FALSE(inferred_precision > precision)) {
+    std::stringstream buf;
+    buf << "Decimal type with precision " << inferred_precision
+        << " does not fit into precision inferred from first array element: "
+        << precision;
+    return Status::Invalid(buf.str());
+  }
+
+  if (scale != inferred_scale) {
+    DCHECK_NE(out, NULLPTR);
+    RETURN_NOT_OK(out->Rescale(inferred_scale, scale, out));
+  }
+  return Status::OK();
+}
+
+bool IsPyInteger(PyObject* obj) {
+#if PYARROW_IS_PY2
+  return PyLong_Check(obj) || PyInt_Check(obj);
+#else
+  return PyLong_Check(obj);
+#endif
+}
+
 }  // namespace internal
 }  // namespace py
 }  // namespace arrow

cpp/src/arrow/python/helpers.h

@@ -29,6 +29,9 @@
 #include "arrow/util/visibility.h"
 
 namespace arrow {
+
+class Decimal128;
+
 namespace py {
 
 class OwnedRef;
@@ -44,11 +47,15 @@ Status ImportFromModule(const OwnedRef& module, const std::string& module_name,
 
 Status PythonDecimalToString(PyObject* python_decimal, std::string* out);
 
-Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int* precision = NULLPTR,
-                                     int* scale = NULLPTR);
+Status InferDecimalPrecisionAndScale(PyObject* python_decimal,
+                                     int32_t* precision = NULLPTR,
+                                     int32_t* scale = NULLPTR);
 
 PyObject* DecimalFromString(PyObject* decimal_constructor,
                             const std::string& decimal_string);
+Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arrow_type,
+                                Decimal128* out);
+bool IsPyInteger(PyObject* obj);
 
 }  // namespace internal
 }  // namespace py

cpp/src/arrow/python/io.cc

@@ -76,7 +76,7 @@ class PythonFile {
     return Status::OK();
   }
 
-  Status Write(const uint8_t* data, int64_t nbytes) {
+  Status Write(const void* data, int64_t nbytes) {
     PyObject* py_data =
         PyBytes_FromStringAndSize(reinterpret_cast<const char*>(data), nbytes);
     PY_RETURN_IF_ERROR(StatusCode::IOError);
@@ -130,7 +130,7 @@ Status PyReadableFile::Tell(int64_t* position) const {
   return file_->Tell(position);
 }
 
-Status PyReadableFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
+Status PyReadableFile::Read(int64_t nbytes, int64_t* bytes_read, void* out) {
   PyAcquireGIL lock;
   PyObject* bytes_obj;
   ARROW_RETURN_NOT_OK(file_->Read(nbytes, &bytes_obj));
@@ -155,7 +155,7 @@ Status PyReadableFile::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
 }
 
 Status PyReadableFile::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
-                              uint8_t* out) {
+                              void* out) {
   std::lock_guard<std::mutex> guard(file_->lock());
   RETURN_NOT_OK(Seek(position));
   return Read(nbytes, bytes_read, out);
@@ -208,7 +208,7 @@ Status PyOutputStream::Tell(int64_t* position) const {
   return Status::OK();
 }
 
-Status PyOutputStream::Write(const uint8_t* data, int64_t nbytes) {
+Status PyOutputStream::Write(const void* data, int64_t nbytes) {
   PyAcquireGIL lock;
   position_ += nbytes;
   return file_->Write(data, nbytes);

cpp/src/arrow/python/io.h

@@ -41,12 +41,12 @@ class ARROW_EXPORT PyReadableFile : public io::RandomAccessFile {
 
   Status Close() override;
 
-  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) override;
+  Status Read(int64_t nbytes, int64_t* bytes_read, void* out) override;
   Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
 
   // Thread-safe version
   Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
-                uint8_t* out) override;
+                void* out) override;
 
   // Thread-safe version
   Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) override;
@@ -70,7 +70,7 @@ class ARROW_EXPORT PyOutputStream : public io::OutputStream {
 
   Status Close() override;
   Status Tell(int64_t* position) const override;
-  Status Write(const uint8_t* data, int64_t nbytes) override;
+  Status Write(const void* data, int64_t nbytes) override;
 
  private:
   std::unique_ptr<PythonFile> file_;

cpp/src/arrow/python/numpy-internal.h

@@ -56,8 +56,8 @@ class Ndarray1DIndexer {
 
   bool is_strided() const { return stride_ == 1; }
 
-  T& operator[](size_type index) { return *(data_ + index * stride_); }
-  T& operator[](size_type index) const { return *(data_ + index * stride_); }
+  T& operator[](size_type index) { return data_[index * stride_]; }
+  T& operator[](size_type index) const { return data_[index * stride_]; }
 
  private:
   PyArrayObject* arr_;

cpp/src/arrow/python/numpy_to_arrow.cc

@@ -120,6 +120,7 @@ int64_t MaskToBitmap(PyArrayObject* mask, int64_t length, uint8_t* bitmap) {
   for (int i = 0; i < length; ++i) {
     if (mask_values[i]) {
       ++null_count;
+      BitUtil::ClearBit(bitmap, i);
     } else {
       BitUtil::SetBit(bitmap, i);
     }
@@ -145,6 +146,52 @@ Status CheckFlatNumpyArray(PyArrayObject* numpy_array, int np_type) {
 
 }  // namespace
 
+/// Append as many string objects from NumPy arrays to a `StringBuilder` as we
+/// can fit
+///
+/// \param[in] offset starting offset for appending
+/// \param[out] end_offset ending offset where we stopped appending. Will
+/// be length of arr if fully consumed
+/// \param[out] have_bytes true if we encountered any PyBytes object
+static Status AppendObjectBinaries(PyArrayObject* arr, PyArrayObject* mask,
+                                   int64_t offset, BinaryBuilder* builder,
+                                   int64_t* end_offset, bool* have_bytes) {
+  PyObject* obj;
+
+  Ndarray1DIndexer<PyObject*> objects(arr);
+  Ndarray1DIndexer<uint8_t> mask_values;
+
+  bool have_mask = false;
+  if (mask != nullptr) {
+    mask_values.Init(mask);
+    have_mask = true;
+  }
+
+  for (; offset < objects.size(); ++offset) {
+    OwnedRef tmp_obj;
+    obj = objects[offset];
+    if ((have_mask && mask_values[offset]) || PandasObjectIsNull(obj)) {
+      RETURN_NOT_OK(builder->AppendNull());
+      continue;
+    } else if (!PyBytes_Check(obj)) {
+      std::stringstream ss;
+      ss << "Error converting to Python objects to bytes: ";
+      RETURN_NOT_OK(InvalidConversion(obj, "str, bytes", &ss));
+      return Status::Invalid(ss.str());
+    }
+
+    const int32_t length = static_cast<int32_t>(PyBytes_GET_SIZE(obj));
+    if (ARROW_PREDICT_FALSE(builder->value_data_length() + length > kBinaryMemoryLimit)) {
+      break;
+    }
+    RETURN_NOT_OK(builder->Append(PyBytes_AS_STRING(obj), length));
+  }
+
+  // If we consumed the whole array, this will be the length of arr
+  *end_offset = offset;
+  return Status::OK();
+}
+
 /// Append as many string objects from NumPy arrays to a `StringBuilder` as we
 /// can fit
 ///
@@ -374,7 +421,11 @@ class NumPyConverter {
     using traits = internal::arrow_traits<ArrowType::type_id>;
 
     const bool null_sentinels_possible =
-        (use_pandas_null_sentinels_ && traits::supports_nulls);
+        // NumPy has a NaT type
+        (ArrowType::type_id == Type::TIMESTAMP || ArrowType::type_id == Type::DATE32) ||
+
+        // Observing pandas's null sentinels
+        ((use_pandas_null_sentinels_ && traits::supports_nulls));
 
     if (mask_ != nullptr || null_sentinels_possible) {
       RETURN_NOT_OK(InitNullBitmap());
@@ -392,9 +443,7 @@ class NumPyConverter {
       null_count = ValuesToBitmap<traits::npy_type>(arr_, null_bitmap_data_);
     }
 
-    BufferVector buffers = {null_bitmap_, data};
-    auto arr_data =
-        std::make_shared<ArrayData>(type_, length_, std::move(buffers), null_count, 0);
+    auto arr_data = ArrayData::Make(type_, length_, {null_bitmap_, data}, null_count, 0);
     return PushArray(arr_data);
   }
 
@@ -472,8 +521,7 @@ Status CastBuffer(const std::shared_ptr<DataType>& in_type,
                   const std::shared_ptr<DataType>& out_type, MemoryPool* pool,
                   std::shared_ptr<Buffer>* out) {
   // Must cast
-  std::vector<std::shared_ptr<Buffer>> buffers = {valid_bitmap, input};
-  auto tmp_data = std::make_shared<ArrayData>(in_type, length, buffers, null_count);
+  auto tmp_data = ArrayData::Make(in_type, length, {valid_bitmap, input}, null_count);
 
   std::shared_ptr<Array> tmp_array = MakeArray(tmp_data);
   std::shared_ptr<Array> casted_array;
@@ -587,8 +635,6 @@ inline Status NumPyConverter::ConvertData<Date32Type>(std::shared_ptr<Buffer>* d
 
   auto date_dtype = reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(dtype_->c_metadata);
   if (dtype_->type_num == NPY_DATETIME) {
-    const int64_t null_count = ValuesToBitmap<NPY_DATETIME>(arr_, null_bitmap_data_);
-
     // If we have inbound datetime64[D] data, this needs to be downcasted
     // separately here from int64_t to int32_t, because this data is not
     // supported in compute::Cast
@@ -598,6 +644,9 @@ inline Status NumPyConverter::ConvertData<Date32Type>(std::shared_ptr<Buffer>* d
       Status s = StaticCastBuffer<int64_t, int32_t>(**data, length_, pool_, data);
       RETURN_NOT_OK(s);
     } else {
+      // TODO(wesm): This is redundant, and recomputed in VisitNative()
+      const int64_t null_count = ValuesToBitmap<NPY_DATETIME>(arr_, null_bitmap_data_);
+
       RETURN_NOT_OK(NumPyDtypeToArrow(reinterpret_cast<PyObject*>(dtype_), &input_type));
       if (!input_type->Equals(*type_)) {
         RETURN_NOT_OK(CastBuffer(input_type, *data, length_, null_bitmap_, null_count,
@@ -685,24 +734,41 @@ Status NumPyConverter::ConvertDecimals() {
   Ndarray1DIndexer<PyObject*> objects(arr_);
   PyObject* object = objects[0];
 
-  int precision;
-  int scale;
+  if (type_ == NULLPTR) {
+    int32_t precision;
+    int32_t desired_scale;
 
-  RETURN_NOT_OK(internal::InferDecimalPrecisionAndScale(object, &precision, &scale));
+    int32_t tmp_precision;
+    int32_t tmp_scale;
 
-  type_ = std::make_shared<Decimal128Type>(precision, scale);
+    RETURN_NOT_OK(
+        internal::InferDecimalPrecisionAndScale(objects[0], &precision, &desired_scale));
+
+    for (int64_t i = 1; i < length_; ++i) {
+      RETURN_NOT_OK(internal::InferDecimalPrecisionAndScale(objects[i], &tmp_precision,
+                                                            &tmp_scale));
+      precision = std::max(precision, tmp_precision);
+
+      if (std::abs(desired_scale) < std::abs(tmp_scale)) {
+        desired_scale = tmp_scale;
+      }
+    }
+
+    type_ = ::arrow::decimal(precision, desired_scale);
+  }
 
   Decimal128Builder builder(type_, pool_);
   RETURN_NOT_OK(builder.Resize(length_));
 
+  const auto& decimal_type = static_cast<const DecimalType&>(*type_);
+  PyObject* Decimal_type_object = Decimal.obj();
+
   for (int64_t i = 0; i < length_; ++i) {
     object = objects[i];
-    if (PyObject_IsInstance(object, Decimal.obj())) {
-      std::string string;
-      RETURN_NOT_OK(internal::PythonDecimalToString(object, &string));
 
+    if (PyObject_IsInstance(object, Decimal_type_object)) {
       Decimal128 value;
-      RETURN_NOT_OK(Decimal128::FromString(string, &value));
+      RETURN_NOT_OK(internal::DecimalFromPythonDecimal(object, decimal_type, &value));
       RETURN_NOT_OK(builder.Append(value));
     } else if (PandasObjectIsNull(object)) {
       RETURN_NOT_OK(builder.AppendNull());
@@ -727,7 +793,7 @@ Status NumPyConverter::ConvertTimes() {
   Time64Builder builder(::arrow::time64(TimeUnit::MICRO), pool_);
   RETURN_NOT_OK(builder.Resize(length_));
 
-  PyObject* obj;
+  PyObject* obj = NULLPTR;
   for (int64_t i = 0; i < length_; ++i) {
     obj = objects[i];
     if (PyTime_Check(obj)) {
@@ -768,7 +834,7 @@ Status NumPyConverter::ConvertObjectStrings() {
   // If we saw PyBytes, convert everything to BinaryArray
   if (global_have_bytes) {
     for (size_t i = 0; i < out_arrays_.size(); ++i) {
-      auto binary_data = out_arrays_[i]->data()->ShallowCopy();
+      auto binary_data = out_arrays_[i]->data()->Copy();
       binary_data->type = ::arrow::binary();
       out_arrays_[i] = std::make_shared<BinaryArray>(binary_data);
     }
@@ -1178,6 +1244,59 @@ inline Status NumPyConverter::ConvertTypedLists<NPY_OBJECT, NullType>(
   return LoopPySequenceWithMasks(list, mask_values, have_mask, foreach_item);
 }
 
+template <>
+inline Status NumPyConverter::ConvertTypedLists<NPY_OBJECT, BinaryType>(
+    const std::shared_ptr<DataType>& type, ListBuilder* builder, PyObject* list) {
+  PyAcquireGIL lock;
+  // TODO: If there are bytes involed, convert to Binary representation
+  bool have_bytes = false;
+
+  Ndarray1DIndexer<uint8_t> mask_values;
+
+  bool have_mask = false;
+  if (mask_ != nullptr) {
+    mask_values.Init(mask_);
+    have_mask = true;
+  }
+
+  auto value_builder = static_cast<BinaryBuilder*>(builder->value_builder());
+
+  auto foreach_item = [&](PyObject* object, bool mask) {
+    if (mask || PandasObjectIsNull(object)) {
+      return builder->AppendNull();
+    } else if (PyArray_Check(object)) {
+      auto numpy_array = reinterpret_cast<PyArrayObject*>(object);
+      RETURN_NOT_OK(builder->Append(true));
+
+      // TODO(uwe): Support more complex numpy array structures
+      RETURN_NOT_OK(CheckFlatNumpyArray(numpy_array, NPY_OBJECT));
+
+      int64_t offset = 0;
+      RETURN_NOT_OK(AppendObjectBinaries(numpy_array, nullptr, 0, value_builder, &offset,
+                                         &have_bytes));
+      if (offset < PyArray_SIZE(numpy_array)) {
+        return Status::Invalid("Array cell value exceeded 2GB");
+      }
+      return Status::OK();
+    } else if (PyList_Check(object)) {
+      int64_t size;
+      std::shared_ptr<DataType> inferred_type;
+      RETURN_NOT_OK(builder->Append(true));
+      RETURN_NOT_OK(InferArrowTypeAndSize(object, &size, &inferred_type));
+      if (inferred_type->id() != Type::NA && inferred_type->id() != Type::BINARY) {
+        std::stringstream ss;
+        ss << inferred_type->ToString() << " cannot be converted to BINARY.";
+        return Status::TypeError(ss.str());
+      }
+      return AppendPySequence(object, size, inferred_type, value_builder);
+    } else {
+      return Status::TypeError("Unsupported Python type for list items");
+    }
+  };
+
+  return LoopPySequenceWithMasks(list, mask_values, have_mask, foreach_item);
+}
+
 template <>
 inline Status NumPyConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
     const std::shared_ptr<DataType>& type, ListBuilder* builder, PyObject* list) {
@@ -1252,6 +1371,7 @@ Status NumPyConverter::ConvertLists(const std::shared_ptr<DataType>& type,
     LIST_CASE(HALF_FLOAT, NPY_FLOAT16, HalfFloatType)
     LIST_CASE(FLOAT, NPY_FLOAT, FloatType)
     LIST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
+    LIST_CASE(BINARY, NPY_OBJECT, BinaryType)
     LIST_CASE(STRING, NPY_OBJECT, StringType)
     case Type::LIST: {
       const auto& list_type = static_cast<const ListType&>(*type);

cpp/src/arrow/python/python-test.cc

@@ -35,34 +35,73 @@ namespace py {
 
 TEST(PyBuffer, InvalidInputObject) { PyBuffer buffer(Py_None); }
 
-TEST(DecimalTest, TestPythonDecimalToString) {
-  PyAcquireGIL lock;
+class DecimalTest : public ::testing::Test {
+ public:
+  DecimalTest() : lock_(), decimal_module_(), decimal_constructor_() {
+    auto s = internal::ImportModule("decimal", &decimal_module_);
+    DCHECK(s.ok()) << s.message();
+    DCHECK_NE(decimal_module_.obj(), NULLPTR);
 
-  OwnedRef decimal;
-  OwnedRef Decimal;
-  ASSERT_OK(internal::ImportModule("decimal", &decimal));
-  ASSERT_NE(decimal.obj(), nullptr);
+    s = internal::ImportFromModule(decimal_module_, "Decimal", &decimal_constructor_);
+    DCHECK(s.ok()) << s.message();
 
-  ASSERT_OK(internal::ImportFromModule(decimal, "Decimal", &Decimal));
-  ASSERT_NE(Decimal.obj(), nullptr);
+    DCHECK_NE(decimal_constructor_.obj(), NULLPTR);
+  }
 
+  OwnedRef CreatePythonDecimal(const std::string& string_value) {
+    OwnedRef ref(internal::DecimalFromString(decimal_constructor_.obj(), string_value));
+    return ref;
+  }
+
+ private:
+  PyAcquireGIL lock_;
+  OwnedRef decimal_module_;
+  OwnedRef decimal_constructor_;
+};
+
+TEST_F(DecimalTest, TestPythonDecimalToString) {
   std::string decimal_string("-39402950693754869342983");
-  const char* format = "s#";
-  auto c_string = decimal_string.c_str();
-  ASSERT_NE(c_string, nullptr);
 
-  auto c_string_size = decimal_string.size();
-  ASSERT_GT(c_string_size, 0);
-  OwnedRef pydecimal(PyObject_CallFunction(Decimal.obj(), const_cast<char*>(format),
-                                           c_string, c_string_size));
-  ASSERT_NE(pydecimal.obj(), nullptr);
-  ASSERT_EQ(PyErr_Occurred(), nullptr);
-
-  PyObject* python_object = pydecimal.obj();
-  ASSERT_NE(python_object, nullptr);
+  OwnedRef python_object = this->CreatePythonDecimal(decimal_string);
+  ASSERT_NE(python_object.obj(), nullptr);
 
   std::string string_result;
-  ASSERT_OK(internal::PythonDecimalToString(python_object, &string_result));
+  ASSERT_OK(internal::PythonDecimalToString(python_object.obj(), &string_result));
+}
+
+TEST_F(DecimalTest, TestInferPrecisionAndScale) {
+  std::string decimal_string("-394029506937548693.42983");
+  OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));
+
+  int32_t precision;
+  int32_t scale;
+
+  ASSERT_OK(
+      internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));
+
+  const auto expected_precision =
+      static_cast<int32_t>(decimal_string.size() - 2);  // 1 for -, 1 for .
+  const int32_t expected_scale = 5;
+
+  ASSERT_EQ(expected_precision, precision);
+  ASSERT_EQ(expected_scale, scale);
+}
+
+TEST_F(DecimalTest, TestInferPrecisionAndNegativeScale) {
+  std::string decimal_string("-3.94042983E+10");
+  OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));
+
+  int32_t precision;
+  int32_t scale;
+
+  ASSERT_OK(
+      internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));
+
+  const auto expected_precision = 9;
+  const int32_t expected_scale = -2;
+
+  ASSERT_EQ(expected_precision, precision);
+  ASSERT_EQ(expected_scale, scale);
 }
 
 TEST(PandasConversionTest, TestObjectBlockWriteFails) {

cpp/src/arrow/python/python_to_arrow.cc

@@ -31,7 +31,9 @@
 #include "arrow/array.h"
 #include "arrow/builder.h"
 #include "arrow/io/interfaces.h"
+#include "arrow/io/memory.h"
 #include "arrow/ipc/writer.h"
+#include "arrow/memory_pool.h"
 #include "arrow/record_batch.h"
 #include "arrow/tensor.h"
 #include "arrow/util/logging.h"
@@ -710,27 +712,89 @@ Status SerializeObject(PyObject* context, PyObject* sequence, SerializedPyObject
   return Status::OK();
 }
 
-Status WriteSerializedObject(const SerializedPyObject& obj, io::OutputStream* dst) {
-  int32_t num_tensors = static_cast<int32_t>(obj.tensors.size());
-  int32_t num_buffers = static_cast<int32_t>(obj.buffers.size());
-  RETURN_NOT_OK(dst->Write(reinterpret_cast<uint8_t*>(&num_tensors), sizeof(int32_t)));
-  RETURN_NOT_OK(dst->Write(reinterpret_cast<uint8_t*>(&num_buffers), sizeof(int32_t)));
-  RETURN_NOT_OK(ipc::WriteRecordBatchStream({obj.batch}, dst));
+Status SerializedPyObject::WriteTo(io::OutputStream* dst) {
+  int32_t num_tensors = static_cast<int32_t>(this->tensors.size());
+  int32_t num_buffers = static_cast<int32_t>(this->buffers.size());
+  RETURN_NOT_OK(
+      dst->Write(reinterpret_cast<const uint8_t*>(&num_tensors), sizeof(int32_t)));
+  RETURN_NOT_OK(
+      dst->Write(reinterpret_cast<const uint8_t*>(&num_buffers), sizeof(int32_t)));
+  RETURN_NOT_OK(ipc::WriteRecordBatchStream({this->batch}, dst));
 
   int32_t metadata_length;
   int64_t body_length;
-  for (const auto& tensor : obj.tensors) {
+  for (const auto& tensor : this->tensors) {
     RETURN_NOT_OK(ipc::WriteTensor(*tensor, dst, &metadata_length, &body_length));
   }
 
-  for (const auto& buffer : obj.buffers) {
+  for (const auto& buffer : this->buffers) {
     int64_t size = buffer->size();
-    RETURN_NOT_OK(dst->Write(reinterpret_cast<uint8_t*>(&size), sizeof(int64_t)));
+    RETURN_NOT_OK(dst->Write(reinterpret_cast<const uint8_t*>(&size), sizeof(int64_t)));
     RETURN_NOT_OK(dst->Write(buffer->data(), size));
   }
 
   return Status::OK();
 }
 
+Status SerializedPyObject::GetComponents(MemoryPool* memory_pool, PyObject** out) {
+  PyAcquireGIL py_gil;
+
+  ScopedRef result(PyDict_New());
+  PyObject* buffers = PyList_New(0);
+
+  // TODO(wesm): Not sure how pedantic we need to be about checking the return
+  // values of these functions. There are other places where we do not check
+  // PyDict_SetItem/SetItemString return value, but these failures would be
+  // quite esoteric
+  PyDict_SetItemString(result.get(), "num_tensors",
+                       PyLong_FromSize_t(this->tensors.size()));
+  PyDict_SetItemString(result.get(), "num_buffers",
+                       PyLong_FromSize_t(this->buffers.size()));
+  PyDict_SetItemString(result.get(), "data", buffers);
+  RETURN_IF_PYERROR();
+
+  Py_DECREF(buffers);
+
+  auto PushBuffer = [&buffers](const std::shared_ptr<Buffer>& buffer) {
+    PyObject* wrapped_buffer = wrap_buffer(buffer);
+    RETURN_IF_PYERROR();
+    if (PyList_Append(buffers, wrapped_buffer) < 0) {
+      Py_DECREF(wrapped_buffer);
+      RETURN_IF_PYERROR();
+    }
+    Py_DECREF(wrapped_buffer);
+    return Status::OK();
+  };
+
+  constexpr int64_t kInitialCapacity = 1024;
+
+  // Write the record batch describing the object structure
+  std::shared_ptr<io::BufferOutputStream> stream;
+  std::shared_ptr<Buffer> buffer;
+
+  py_gil.release();
+  RETURN_NOT_OK(io::BufferOutputStream::Create(kInitialCapacity, memory_pool, &stream));
+  RETURN_NOT_OK(ipc::WriteRecordBatchStream({this->batch}, stream.get()));
+  RETURN_NOT_OK(stream->Finish(&buffer));
+  py_gil.acquire();
+
+  RETURN_NOT_OK(PushBuffer(buffer));
+
+  // For each tensor, get a metadata buffer and a buffer for the body
+  for (const auto& tensor : this->tensors) {
+    std::unique_ptr<ipc::Message> message;
+    RETURN_NOT_OK(ipc::GetTensorMessage(*tensor, memory_pool, &message));
+    RETURN_NOT_OK(PushBuffer(message->metadata()));
+    RETURN_NOT_OK(PushBuffer(message->body()));
+  }
+
+  for (const auto& buf : this->buffers) {
+    RETURN_NOT_OK(PushBuffer(buf));
+  }
+
+  *out = result.release();
+  return Status::OK();
+}
+
 }  // namespace py
 }  // namespace arrow

cpp/src/arrow/python/python_to_arrow.h

@@ -30,6 +30,7 @@
 
 namespace arrow {
 
+class MemoryPool;
 class RecordBatch;
 class Tensor;
 
@@ -45,6 +46,26 @@ struct ARROW_EXPORT SerializedPyObject {
   std::shared_ptr<RecordBatch> batch;
   std::vector<std::shared_ptr<Tensor>> tensors;
   std::vector<std::shared_ptr<Buffer>> buffers;
+
+  /// \brief Write serialized Python object to OutputStream
+  /// \param[in,out] dst an OutputStream
+  /// \return Status
+  Status WriteTo(io::OutputStream* dst);
+
+  /// \brief Convert SerializedPyObject to a dict containing the message
+  /// components as Buffer instances with minimal memory allocation
+  ///
+  /// {
+  ///   'num_tensors': N,
+  ///   'num_buffers': K,
+  ///   'data': [Buffer]
+  /// }
+  ///
+  /// Each tensor is written as two buffers, one for the metadata and one for
+  /// the body. Therefore, the number of buffers in 'data' is 2 * N + K + 1,
+  /// with the first buffer containing the serialized record batch containing
+  /// the UnionArray that describes the whole object
+  Status GetComponents(MemoryPool* pool, PyObject** out);
 };
 
 /// \brief Serialize Python sequence as a RecordBatch plus
@@ -62,13 +83,6 @@ struct ARROW_EXPORT SerializedPyObject {
 ARROW_EXPORT
 Status SerializeObject(PyObject* context, PyObject* sequence, SerializedPyObject* out);
 
-/// \brief Write serialized Python object to OutputStream
-/// \param[in] object a serialized Python object to write out
-/// \param[out] dst an OutputStream
-/// \return Status
-ARROW_EXPORT
-Status WriteSerializedObject(const SerializedPyObject& object, io::OutputStream* dst);
-
 }  // namespace py
 }  // namespace arrow
 

cpp/src/arrow/table-test.cc

@@ -404,6 +404,26 @@ TEST_F(TestTable, RemoveColumn) {
   ASSERT_TRUE(result->Equals(*expected));
 }
 
+TEST_F(TestTable, RemoveColumnEmpty) {
+  // ARROW-1865
+  const int64_t length = 10;
+
+  auto f0 = field("f0", int32());
+  auto schema = ::arrow::schema({f0});
+  auto a0 = MakeRandomArray<Int32Array>(length);
+
+  auto table = Table::Make(schema, {std::make_shared<Column>(f0, a0)});
+
+  std::shared_ptr<Table> empty;
+  ASSERT_OK(table->RemoveColumn(0, &empty));
+
+  ASSERT_EQ(table->num_rows(), empty->num_rows());
+
+  std::shared_ptr<Table> added;
+  ASSERT_OK(empty->AddColumn(0, table->column(0), &added));
+  ASSERT_EQ(table->num_rows(), added->num_rows());
+}
+
 TEST_F(TestTable, AddColumn) {
   const int64_t length = 10;
   MakeExample1(length);
@@ -586,4 +606,37 @@ TEST_F(TestTableBatchReader, ReadNext) {
   ASSERT_EQ(nullptr, batch);
 }
 
+TEST_F(TestTableBatchReader, Chunksize) {
+  auto a1 = MakeRandomArray<Int32Array>(10);
+  auto a2 = MakeRandomArray<Int32Array>(20);
+  auto a3 = MakeRandomArray<Int32Array>(10);
+
+  auto sch1 = arrow::schema({field("f1", int32())});
+  auto t1 = Table::Make(sch1, {column(sch1->field(0), {a1, a2, a3})});
+
+  TableBatchReader i1(*t1);
+
+  i1.set_chunksize(15);
+
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_OK(i1.ReadNext(&batch));
+  ASSERT_OK(batch->Validate());
+  ASSERT_EQ(10, batch->num_rows());
+
+  ASSERT_OK(i1.ReadNext(&batch));
+  ASSERT_OK(batch->Validate());
+  ASSERT_EQ(15, batch->num_rows());
+
+  ASSERT_OK(i1.ReadNext(&batch));
+  ASSERT_OK(batch->Validate());
+  ASSERT_EQ(5, batch->num_rows());
+
+  ASSERT_OK(i1.ReadNext(&batch));
+  ASSERT_OK(batch->Validate());
+  ASSERT_EQ(10, batch->num_rows());
+
+  ASSERT_OK(i1.ReadNext(&batch));
+  ASSERT_EQ(nullptr, batch);
+}
+
 }  // namespace arrow

cpp/src/arrow/table.cc

@@ -19,6 +19,7 @@
 
 #include <algorithm>
 #include <cstdlib>
+#include <limits>
 #include <memory>
 #include <sstream>
 
@@ -201,7 +202,8 @@ class SimpleTable : public Table {
     std::shared_ptr<Schema> new_schema;
     RETURN_NOT_OK(schema_->RemoveField(i, &new_schema));
 
-    *out = Table::Make(new_schema, internal::DeleteVectorElement(columns_, i));
+    *out = Table::Make(new_schema, internal::DeleteVectorElement(columns_, i),
+                       this->num_rows());
     return Status::OK();
   }
 
@@ -403,7 +405,8 @@ class TableBatchReader::TableBatchReaderImpl {
         column_data_(table.num_columns()),
         chunk_numbers_(table.num_columns(), 0),
         chunk_offsets_(table.num_columns(), 0),
-        absolute_row_position_(0) {
+        absolute_row_position_(0),
+        max_chunksize_(std::numeric_limits<int64_t>::max()) {
     for (int i = 0; i < table.num_columns(); ++i) {
       column_data_[i] = table.column(i)->data().get();
     }
@@ -416,7 +419,7 @@ class TableBatchReader::TableBatchReaderImpl {
     }
 
     // Determine the minimum contiguous slice across all columns
-    int64_t chunksize = table_.num_rows();
+    int64_t chunksize = std::min(table_.num_rows(), max_chunksize_);
     std::vector<const Array*> chunks(table_.num_columns());
     for (int i = 0; i < table_.num_columns(); ++i) {
       auto chunk = column_data_[i]->chunk(chunk_numbers_[i]).get();
@@ -430,8 +433,7 @@ class TableBatchReader::TableBatchReaderImpl {
     }
 
     // Slice chunks and advance chunk index as appropriate
-    std::vector<std::shared_ptr<ArrayData>> batch_data;
-    batch_data.reserve(table_.num_columns());
+    std::vector<std::shared_ptr<ArrayData>> batch_data(table_.num_columns());
 
     for (int i = 0; i < table_.num_columns(); ++i) {
       // Exhausted chunk
@@ -441,7 +443,7 @@ class TableBatchReader::TableBatchReaderImpl {
       if ((chunk->length() - offset) == chunksize) {
         ++chunk_numbers_[i];
         chunk_offsets_[i] = 0;
-        if (chunk_offsets_[i] > 0) {
+        if (offset > 0) {
           // Need to slice
           slice_data = chunk->Slice(offset, chunksize)->data();
         } else {
@@ -449,9 +451,10 @@ class TableBatchReader::TableBatchReaderImpl {
           slice_data = chunk->data();
         }
       } else {
+        chunk_offsets_[i] += chunksize;
         slice_data = chunk->Slice(offset, chunksize)->data();
       }
-      batch_data.emplace_back(std::move(slice_data));
+      batch_data[i] = std::move(slice_data);
     }
 
     absolute_row_position_ += chunksize;
@@ -462,12 +465,15 @@ class TableBatchReader::TableBatchReaderImpl {
 
   std::shared_ptr<Schema> schema() const { return table_.schema(); }
 
+  void set_chunksize(int64_t chunksize) { max_chunksize_ = chunksize; }
+
  private:
   const Table& table_;
   std::vector<ChunkedArray*> column_data_;
   std::vector<int> chunk_numbers_;
   std::vector<int64_t> chunk_offsets_;
   int64_t absolute_row_position_;
+  int64_t max_chunksize_;
 };
 
 TableBatchReader::TableBatchReader(const Table& table) {
@@ -478,6 +484,10 @@ TableBatchReader::~TableBatchReader() {}
 
 std::shared_ptr<Schema> TableBatchReader::schema() const { return impl_->schema(); }
 
+void TableBatchReader::set_chunksize(int64_t chunksize) {
+  impl_->set_chunksize(chunksize);
+}
+
 Status TableBatchReader::ReadNext(std::shared_ptr<RecordBatch>* out) {
   return impl_->ReadNext(out);
 }

cpp/src/arrow/table.h

@@ -197,6 +197,8 @@ class ARROW_EXPORT TableBatchReader : public RecordBatchReader {
 
   Status ReadNext(std::shared_ptr<RecordBatch>* out) override;
 
+  void set_chunksize(int64_t chunksize);
+
  private:
   class TableBatchReaderImpl;
   std::unique_ptr<TableBatchReaderImpl> impl_;

cpp/src/arrow/type.cc

@@ -474,43 +474,6 @@ std::shared_ptr<DataType> decimal(int32_t precision, int32_t scale) {
   return std::make_shared<Decimal128Type>(precision, scale);
 }
 
-static const BufferDescr kValidityBuffer(BufferType::VALIDITY, 1);
-static const BufferDescr kOffsetBuffer(BufferType::OFFSET, 32);
-static const BufferDescr kTypeBuffer(BufferType::TYPE, 32);
-static const BufferDescr kBooleanBuffer(BufferType::DATA, 1);
-static const BufferDescr kValues64(BufferType::DATA, 64);
-static const BufferDescr kValues32(BufferType::DATA, 32);
-static const BufferDescr kValues16(BufferType::DATA, 16);
-static const BufferDescr kValues8(BufferType::DATA, 8);
-
-std::vector<BufferDescr> FixedWidthType::GetBufferLayout() const {
-  return {kValidityBuffer, BufferDescr(BufferType::DATA, bit_width())};
-}
-
-std::vector<BufferDescr> NullType::GetBufferLayout() const { return {}; }
-
-std::vector<BufferDescr> BinaryType::GetBufferLayout() const {
-  return {kValidityBuffer, kOffsetBuffer, kValues8};
-}
-
-std::vector<BufferDescr> FixedSizeBinaryType::GetBufferLayout() const {
-  return {kValidityBuffer, BufferDescr(BufferType::DATA, bit_width())};
-}
-
-std::vector<BufferDescr> ListType::GetBufferLayout() const {
-  return {kValidityBuffer, kOffsetBuffer};
-}
-
-std::vector<BufferDescr> StructType::GetBufferLayout() const { return {kValidityBuffer}; }
-
-std::vector<BufferDescr> UnionType::GetBufferLayout() const {
-  if (mode_ == UnionMode::SPARSE) {
-    return {kValidityBuffer, kTypeBuffer};
-  } else {
-    return {kValidityBuffer, kTypeBuffer, kOffsetBuffer};
-  }
-}
-
 std::string Decimal128Type::ToString() const {
   std::stringstream s;
   s << "decimal(" << precision_ << ", " << scale_ << ")";

cpp/src/arrow/type.h

@@ -133,20 +133,6 @@ struct Type {
   };
 };
 
-enum class BufferType : char { DATA, OFFSET, TYPE, VALIDITY };
-
-class BufferDescr {
- public:
-  BufferDescr(BufferType type, int bit_width) : type_(type), bit_width_(bit_width) {}
-
-  BufferType type() const { return type_; }
-  int bit_width() const { return bit_width_; }
-
- private:
-  BufferType type_;
-  int bit_width_;
-};
-
 class ARROW_EXPORT DataType {
  public:
   explicit DataType(Type::type id) : id_(id) {}
@@ -176,8 +162,6 @@ class ARROW_EXPORT DataType {
   /// \since 0.7.0
   virtual std::string name() const = 0;
 
-  virtual std::vector<BufferDescr> GetBufferLayout() const = 0;
-
   Type::type id() const { return id_; }
 
  protected:
@@ -201,8 +185,6 @@ class ARROW_EXPORT FixedWidthType : public DataType {
   using DataType::DataType;
 
   virtual int bit_width() const = 0;
-
-  std::vector<BufferDescr> GetBufferLayout() const override;
 };
 
 class ARROW_EXPORT PrimitiveCType : public FixedWidthType {
@@ -319,8 +301,6 @@ class ARROW_EXPORT NullType : public DataType, public NoExtraMeta {
   std::string ToString() const override;
 
   std::string name() const override { return "null"; }
-
-  std::vector<BufferDescr> GetBufferLayout() const override;
 };
 
 class ARROW_EXPORT BooleanType : public FixedWidthType, public NoExtraMeta {
@@ -425,8 +405,6 @@ class ARROW_EXPORT ListType : public NestedType {
   std::string ToString() const override;
 
   std::string name() const override { return "list"; }
-
-  std::vector<BufferDescr> GetBufferLayout() const override;
 };
 
 // BinaryType type is represents lists of 1-byte values.
@@ -440,8 +418,6 @@ class ARROW_EXPORT BinaryType : public DataType, public NoExtraMeta {
   std::string ToString() const override;
   std::string name() const override { return "binary"; }
 
-  std::vector<BufferDescr> GetBufferLayout() const override;
-
  protected:
   // Allow subclasses to change the logical type.
   explicit BinaryType(Type::type logical_type) : DataType(logical_type) {}
@@ -461,8 +437,6 @@ class ARROW_EXPORT FixedSizeBinaryType : public FixedWidthType, public Parametri
   std::string ToString() const override;
   std::string name() const override { return "fixed_size_binary"; }
 
-  std::vector<BufferDescr> GetBufferLayout() const override;
-
   int32_t byte_width() const { return byte_width_; }
   int bit_width() const override;
 
@@ -494,8 +468,6 @@ class ARROW_EXPORT StructType : public NestedType {
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
   std::string name() const override { return "struct"; }
-
-  std::vector<BufferDescr> GetBufferLayout() const override;
 };
 
 class ARROW_EXPORT DecimalType : public FixedSizeBinaryType {
@@ -541,8 +513,6 @@ class ARROW_EXPORT UnionType : public NestedType {
   std::string name() const override { return "union"; }
   Status Accept(TypeVisitor* visitor) const override;
 
-  std::vector<BufferDescr> GetBufferLayout() const override;
-
   const std::vector<uint8_t>& type_codes() const { return type_codes_; }
 
   UnionMode::type mode() const { return mode_; }

cpp/src/arrow/util/CMakeLists.txt

@@ -34,6 +34,7 @@ install(FILES
   cpu-info.h
   decimal.h
   hash-util.h
+  hash.h
   key_value_metadata.h
   logging.h
   macros.h