First working iteration of string deduplication when calling to_pandas

cpp/src/arrow/python/arrow_to_pandas.cc

@@ -36,9 +36,11 @@
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/checked_cast.h"
+#include "arrow/util/hashing.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/parallel.h"
+#include "arrow/util/string_view.h"
 #include "arrow/visitor_inline.h"
 
 #include "arrow/compute/api.h"
@@ -75,21 +77,21 @@ template <typename T>
 struct WrapBytes {};
 
 template <>
-struct WrapBytes<StringArray> {
+struct WrapBytes<StringType> {
   static inline PyObject* Wrap(const char* data, int64_t length) {
     return PyUnicode_FromStringAndSize(data, length);
   }
 };
 
 template <>
-struct WrapBytes<BinaryArray> {
+struct WrapBytes<BinaryType> {
   static inline PyObject* Wrap(const char* data, int64_t length) {
     return PyBytes_FromStringAndSize(data, length);
   }
 };
 
 template <>
-struct WrapBytes<FixedSizeBinaryArray> {
+struct WrapBytes<FixedSizeBinaryType> {
   static inline PyObject* Wrap(const char* data, int64_t length) {
     return PyBytes_FromStringAndSize(data, length);
   }
@@ -216,7 +218,7 @@ class PandasBlock {
     CATEGORICAL
   };
 
-  PandasBlock(PandasOptions options, int64_t num_rows, int num_columns)
+  PandasBlock(const PandasOptions& options, int64_t num_rows, int num_columns)
       : num_rows_(num_rows), num_columns_(num_columns), options_(options) {}
   virtual ~PandasBlock() {}
 
@@ -301,8 +303,8 @@ inline const T* GetPrimitiveValues(const Array& arr) {
 }
 
 template <typename T>
-inline void ConvertIntegerWithNulls(PandasOptions options, const ChunkedArray& data,
-                                    double* out_values) {
+inline void ConvertIntegerWithNulls(const PandasOptions& options,
+                                    const ChunkedArray& data, double* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
     const auto& arr = *data.chunk(c);
     const T* in_values = GetPrimitiveValues<T>(arr);
@@ -315,8 +317,8 @@ inline void ConvertIntegerWithNulls(PandasOptions options, const ChunkedArray& d
 }
 
 template <typename T>
-inline void ConvertIntegerNoNullsSameType(PandasOptions options, const ChunkedArray& data,
-                                          T* out_values) {
+inline void ConvertIntegerNoNullsSameType(const PandasOptions& options,
+                                          const ChunkedArray& data, T* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
     const auto& arr = *data.chunk(c);
     if (arr.length() > 0) {
@@ -328,8 +330,8 @@ inline void ConvertIntegerNoNullsSameType(PandasOptions options, const ChunkedAr
 }
 
 template <typename InType, typename OutType>
-inline void ConvertIntegerNoNullsCast(PandasOptions options, const ChunkedArray& data,
-                                      OutType* out_values) {
+inline void ConvertIntegerNoNullsCast(const PandasOptions& options,
+                                      const ChunkedArray& data, OutType* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
     const auto& arr = *data.chunk(c);
     const InType* in_values = GetPrimitiveValues<InType>(arr);
@@ -339,8 +341,8 @@ inline void ConvertIntegerNoNullsCast(PandasOptions options, const ChunkedArray&
   }
 }
 
-static Status ConvertBooleanWithNulls(PandasOptions options, const ChunkedArray& data,
-                                      PyObject** out_values) {
+static Status ConvertBooleanWithNulls(const PandasOptions& options,
+                                      const ChunkedArray& data, PyObject** out_values) {
   PyAcquireGIL lock;
   for (int c = 0; c < data.num_chunks(); c++) {
     const auto& arr = checked_cast<const BooleanArray&>(*data.chunk(c));
@@ -363,7 +365,7 @@ static Status ConvertBooleanWithNulls(PandasOptions options, const ChunkedArray&
   return Status::OK();
 }
 
-static void ConvertBooleanNoNulls(PandasOptions options, const ChunkedArray& data,
+static void ConvertBooleanNoNulls(const PandasOptions& options, const ChunkedArray& data,
                                   uint8_t* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
     const auto& arr = checked_cast<const BooleanArray&>(*data.chunk(c));
@@ -373,57 +375,106 @@ static void ConvertBooleanNoNulls(PandasOptions options, const ChunkedArray& dat
   }
 }
 
-template <typename T>
-static Status ConvertIntegerObjects(PandasOptions options, const ChunkedArray& data,
-                                    PyObject** out_values) {
-  PyAcquireGIL lock;
-  constexpr bool is_signed = std::is_signed<T>::value;
-  for (int c = 0; c < data.num_chunks(); c++) {
-    const auto& arr = *data.chunk(c);
-    const auto* in_values = GetPrimitiveValues<T>(arr);
-
-    for (int i = 0; i < arr.length(); ++i) {
-      if (arr.IsNull(i)) {
-        Py_INCREF(Py_None);
-        *out_values++ = Py_None;
-      } else {
-        *out_values++ = is_signed ? PyLong_FromLongLong(in_values[i])
-                                  : PyLong_FromUnsignedLongLong(in_values[i]);
-        RETURN_IF_PYERROR();
-      }
+// Generic Array -> PyObject** converter that handles object deduplication, if
+// requested
+template <typename ArrayType, typename WriteValue>
+inline Status WriteArrayObjects(const ArrayType& arr, WriteValue&& write_func,
+                                PyObject** out_values) {
+  const bool has_nulls = arr.null_count() > 0;
+  for (int64_t i = 0; i < arr.length(); ++i) {
+    if (has_nulls && arr.IsNull(i)) {
+      Py_INCREF(Py_None);
+      *out_values = Py_None;
+    } else {
+      RETURN_NOT_OK(write_func(arr.GetView(i), out_values));
     }
+    ++out_values;
   }
   return Status::OK();
 }
 
+template <typename T, typename Enable = void>
+struct MemoizationTraits {
+  using Scalar = typename T::c_type;
+};
+
+template <typename T>
+struct MemoizationTraits<T, enable_if_binary_like<T>> {
+  // For binary, we memoize string_view as a scalar value to avoid having to
+  // unnecessarily copy the memory into the memo table data structure
+  using Scalar = util::string_view;
+};
+
+template <typename Type, typename WrapFunction>
+inline Status ConvertAsPyObjects(const PandasOptions& options, const ChunkedArray& data,
+                                 WrapFunction&& wrap_func, PyObject** out_values) {
+  using ArrayType = typename TypeTraits<Type>::ArrayType;
+  using Scalar = typename MemoizationTraits<Type>::Scalar;
+
+  PyAcquireGIL lock;
+  ::arrow::internal::ScalarMemoTable<Scalar> memo_table;
+  std::vector<PyObject*> unique_values;
+  int32_t memo_size = 0;
+
+  auto WrapMemoized = [&](const Scalar& value, PyObject** out_values) {
+    int32_t memo_index = memo_table.GetOrInsert(value);
+    if (memo_index == memo_size) {
+      // New entry
+      RETURN_NOT_OK(wrap_func(value, out_values));
+      unique_values.push_back(*out_values);
+      ++memo_size;
+    } else {
+      // Duplicate entry
+      Py_INCREF(unique_values[memo_index]);
+      *out_values = unique_values[memo_index];
+    }
+    return Status::OK();
+  };
+
+  auto WrapUnmemoized = [&](const Scalar& value, PyObject** out_values) {
+    return wrap_func(value, out_values);
+  };
+
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const auto& arr = checked_cast<const ArrayType&>(*data.chunk(c));
+    if (options.deduplicate_objects) {
+      RETURN_NOT_OK(WriteArrayObjects(arr, WrapMemoized, out_values));
+    } else {
+      RETURN_NOT_OK(WriteArrayObjects(arr, WrapUnmemoized, out_values));
+    }
+    out_values += arr.length();
+  }
+  return Status::OK();
+}
+
+template <typename T>
+static Status ConvertIntegerObjects(const PandasOptions& options,
+                                    const ChunkedArray& data, PyObject** out_values) {
+  auto WrapValue = [](int64_t value, PyObject** out) {
+    constexpr bool is_signed = std::is_signed<T>::value;
+    *out = is_signed ? PyLong_FromLongLong(in_values[i])
+    : PyLong_FromUnsignedLongLong(in_values[i]);
+    RETURN_IF_PYERROR();
+    return Status::OK();
+  };
+  return ConvertAsPyObjects<Type>(options, data, WrapValue, out_values);
+}
+
 template <typename Type>
-inline Status ConvertBinaryLike(PandasOptions options, const ChunkedArray& data,
+inline Status ConvertBinaryLike(const PandasOptions& options, const ChunkedArray& data,
                                 PyObject** out_values) {
-  using ArrayType = typename TypeTraits<Type>::ArrayType;
-  PyAcquireGIL lock;
-  for (int c = 0; c < data.num_chunks(); c++) {
-    const auto& arr = checked_cast<const ArrayType&>(*data.chunk(c));
-
-    const bool has_nulls = data.null_count() > 0;
-    for (int64_t i = 0; i < arr.length(); ++i) {
-      if (has_nulls && arr.IsNull(i)) {
-        Py_INCREF(Py_None);
-        *out_values = Py_None;
-      } else {
-        auto view = arr.GetView(i);
-        *out_values = WrapBytes<ArrayType>::Wrap(view.data(), view.length());
-        if (*out_values == nullptr) {
-          PyErr_Clear();
-          return Status::UnknownError("Wrapping ", view, " failed");
-        }
-      }
-      ++out_values;
+  auto WrapValue = [](const util::string_view& view, PyObject** out) {
+    *out = WrapBytes<Type>::Wrap(view.data(), view.length());
+    if (*out == nullptr) {
+      PyErr_Clear();
+      return Status::UnknownError("Wrapping ", view, " failed");
     }
-  }
-  return Status::OK();
+    return Status::OK();
+  };
+  return ConvertAsPyObjects<Type>(options, data, WrapValue, out_values);
 }
 
-inline Status ConvertNulls(PandasOptions options, const ChunkedArray& data,
+inline Status ConvertNulls(const PandasOptions& options, const ChunkedArray& data,
                            PyObject** out_values) {
   PyAcquireGIL lock;
   for (int c = 0; c < data.num_chunks(); c++) {
@@ -439,7 +490,7 @@ inline Status ConvertNulls(PandasOptions options, const ChunkedArray& data,
   return Status::OK();
 }
 
-inline Status ConvertStruct(PandasOptions options, const ChunkedArray& data,
+inline Status ConvertStruct(const PandasOptions& options, const ChunkedArray& data,
                             PyObject** out_values) {
   PyAcquireGIL lock;
   if (data.num_chunks() <= 0) {
@@ -503,7 +554,8 @@ inline Status ConvertStruct(PandasOptions options, const ChunkedArray& data,
 }
 
 template <typename ArrowType>
-inline Status ConvertListsLike(PandasOptions options, const std::shared_ptr<Column>& col,
+inline Status ConvertListsLike(const PandasOptions& options,
+                               const std::shared_ptr<Column>& col,
                                PyObject** out_values) {
   const ChunkedArray& data = *col->data().get();
   const auto& list_type = checked_cast<const ListType&>(*col->type());
@@ -604,39 +656,23 @@ inline void ConvertDatetimeNanos(const ChunkedArray& data, int64_t* out_values)
   }
 }
 
-template <typename TYPE>
-static Status ConvertDates(PandasOptions options, const ChunkedArray& data,
+template <typename Type>
+static Status ConvertDates(const PandasOptions& options, const ChunkedArray& data,
                            PyObject** out_values) {
-  using ArrayType = typename TypeTraits<TYPE>::ArrayType;
+  using ArrayType = typename TypeTraits<Type>::ArrayType;
+  using T = typename
 
   PyAcquireGIL lock;
-  OwnedRef date_ref;
-
   PyDateTime_IMPORT;
 
-  for (int c = 0; c < data.num_chunks(); c++) {
-    const auto& arr = checked_cast<const ArrayType&>(*data.chunk(c));
-    auto type = std::dynamic_pointer_cast<TYPE>(arr.type());
-    DCHECK(type);
-
-    const DateUnit unit = type->unit();
-
-    for (int64_t i = 0; i < arr.length(); ++i) {
-      if (arr.IsNull(i)) {
-        Py_INCREF(Py_None);
-        *out_values++ = Py_None;
-      } else {
-        RETURN_NOT_OK(PyDate_from_int(arr.Value(i), unit, out_values++));
-        RETURN_IF_PYERROR();
-      }
-    }
-  }
-
-  return Status::OK();
+  auto WrapValue = [](T value, PyObject** out) {
+    return PyDate_from_int(value, Type::Unit, out);
+  };
+  return ConvertAsPyObjects<Type>(options, data, WrapValue, out_values);
 }
 
 template <typename TYPE>
-static Status ConvertTimes(PandasOptions options, const ChunkedArray& data,
+static Status ConvertTimes(const PandasOptions& options, const ChunkedArray& data,
                            PyObject** out_values) {
   using ArrayType = typename TypeTraits<TYPE>::ArrayType;
 
@@ -666,7 +702,7 @@ static Status ConvertTimes(PandasOptions options, const ChunkedArray& data,
   return Status::OK();
 }
 
-static Status ConvertDecimals(PandasOptions options, const ChunkedArray& data,
+static Status ConvertDecimals(const PandasOptions& options, const ChunkedArray& data,
                               PyObject** out_values) {
   PyAcquireGIL lock;
   OwnedRef decimal;
@@ -1009,7 +1045,8 @@ class DatetimeBlock : public PandasBlock {
 
 class DatetimeTZBlock : public DatetimeBlock {
  public:
-  DatetimeTZBlock(PandasOptions options, const std::string& timezone, int64_t num_rows)
+  DatetimeTZBlock(const PandasOptions& options, const std::string& timezone,
+                  int64_t num_rows)
       : DatetimeBlock(options, num_rows, 1), timezone_(timezone) {}
 
   // Like Categorical, the internal ndarray is 1-dimensional
@@ -1038,7 +1075,8 @@ class DatetimeTZBlock : public DatetimeBlock {
 
 class CategoricalBlock : public PandasBlock {
  public:
-  explicit CategoricalBlock(PandasOptions options, MemoryPool* pool, int64_t num_rows)
+  explicit CategoricalBlock(const PandasOptions& options, MemoryPool* pool,
+                            int64_t num_rows)
       : PandasBlock(options, num_rows, 1), pool_(pool), needs_copy_(false) {}
 
   Status Allocate() override {
@@ -1235,7 +1273,7 @@ class CategoricalBlock : public PandasBlock {
   bool needs_copy_;
 };
 
-Status MakeBlock(PandasOptions options, PandasBlock::type type, int64_t num_rows,
+Status MakeBlock(const PandasOptions& options, PandasBlock::type type, int64_t num_rows,
                  int num_columns, std::shared_ptr<PandasBlock>* block) {
 #define BLOCK_CASE(NAME, TYPE)                                       \
   case PandasBlock::NAME:                                            \
@@ -1518,7 +1556,7 @@ class DataFrameBlockCreator {
 
 class ArrowDeserializer {
  public:
-  ArrowDeserializer(PandasOptions options, const std::shared_ptr<Column>& col,
+  ArrowDeserializer(const PandasOptions& options, const std::shared_ptr<Column>& col,
                     PyObject* py_ref)
       : col_(col), data_(*col->data().get()), options_(options), py_ref_(py_ref) {}
 
@@ -1532,7 +1570,7 @@ class ArrowDeserializer {
   }
 
   template <int TYPE>
-  Status ConvertValuesZeroCopy(PandasOptions options, int npy_type,
+  Status ConvertValuesZeroCopy(const PandasOptions& options, int npy_type,
                                const std::shared_ptr<Array>& arr) {
     typedef typename internal::arrow_traits<TYPE>::T T;
 
@@ -1878,15 +1916,16 @@ class ArrowDeserializer {
   PyObject* result_;
 };
 
-Status ConvertArrayToPandas(PandasOptions options, const std::shared_ptr<Array>& arr,
-                            PyObject* py_ref, PyObject** out) {
+Status ConvertArrayToPandas(const PandasOptions& options,
+                            const std::shared_ptr<Array>& arr, PyObject* py_ref,
+                            PyObject** out) {
   static std::string dummy_name = "dummy";
   auto field = std::make_shared<Field>(dummy_name, arr->type());
   auto col = std::make_shared<Column>(field, arr);
   return ConvertColumnToPandas(options, col, py_ref, out);
 }
 
-Status ConvertChunkedArrayToPandas(PandasOptions options,
+Status ConvertChunkedArrayToPandas(const PandasOptions& options,
                                    const std::shared_ptr<ChunkedArray>& ca,
                                    PyObject* py_ref, PyObject** out) {
   static std::string dummy_name = "dummy";
@@ -1895,19 +1934,21 @@ Status ConvertChunkedArrayToPandas(PandasOptions options,
   return ConvertColumnToPandas(options, col, py_ref, out);
 }
 
-Status ConvertColumnToPandas(PandasOptions options, const std::shared_ptr<Column>& col,
-                             PyObject* py_ref, PyObject** out) {
+Status ConvertColumnToPandas(const PandasOptions& options,
+                             const std::shared_ptr<Column>& col, PyObject* py_ref,
+                             PyObject** out) {
   ArrowDeserializer converter(options, col, py_ref);
   return converter.Convert(out);
 }
 
-Status ConvertTableToPandas(PandasOptions options, const std::shared_ptr<Table>& table,
-                            MemoryPool* pool, PyObject** out) {
+Status ConvertTableToPandas(const PandasOptions& options,
+                            const std::shared_ptr<Table>& table, MemoryPool* pool,
+                            PyObject** out) {
   return ConvertTableToPandas(options, std::unordered_set<std::string>(), table, pool,
                               out);
 }
 
-Status ConvertTableToPandas(PandasOptions options,
+Status ConvertTableToPandas(const PandasOptions& options,
                             const std::unordered_set<std::string>& categorical_columns,
                             const std::shared_ptr<Table>& table, MemoryPool* pool,
                             PyObject** out) {

cpp/src/arrow/python/arrow_to_pandas.h

@@ -43,32 +43,32 @@ namespace py {
 
 struct PandasOptions {
   /// If true, we will convert all string columns to categoricals
-  bool strings_to_categorical;
-  bool zero_copy_only;
-  bool integer_object_nulls;
-  bool date_as_object;
-  bool use_threads;
+  bool strings_to_categorical = false;
+  bool zero_copy_only = false;
+  bool integer_object_nulls = false;
+  bool date_as_object = false;
+  bool use_threads = false;
 
-  PandasOptions()
-      : strings_to_categorical(false),
-        zero_copy_only(false),
-        integer_object_nulls(false),
-        date_as_object(false),
-        use_threads(false) {}
+  /// \brief If true, do not create duplicate PyObject versions of equal
+  /// objects. This only applies to immutable objects like strings or datetime
+  /// objects
+  bool deduplicate_objects = false;
 };
 
 ARROW_PYTHON_EXPORT
-Status ConvertArrayToPandas(PandasOptions options, const std::shared_ptr<Array>& arr,
-                            PyObject* py_ref, PyObject** out);
+Status ConvertArrayToPandas(const PandasOptions& options,
+                            const std::shared_ptr<Array>& arr, PyObject* py_ref,
+                            PyObject** out);
 
 ARROW_PYTHON_EXPORT
-Status ConvertChunkedArrayToPandas(PandasOptions options,
+Status ConvertChunkedArrayToPandas(const PandasOptions& options,
                                    const std::shared_ptr<ChunkedArray>& col,
                                    PyObject* py_ref, PyObject** out);
 
 ARROW_PYTHON_EXPORT
-Status ConvertColumnToPandas(PandasOptions options, const std::shared_ptr<Column>& col,
-                             PyObject* py_ref, PyObject** out);
+Status ConvertColumnToPandas(const PandasOptions& options,
+                             const std::shared_ptr<Column>& col, PyObject* py_ref,
+                             PyObject** out);
 
 // Convert a whole table as efficiently as possible to a pandas.DataFrame.
 //
@@ -77,15 +77,16 @@ Status ConvertColumnToPandas(PandasOptions options, const std::shared_ptr<Column
 //
 // tuple item: (indices: ndarray[int32], block: ndarray[TYPE, ndim=2])
 ARROW_PYTHON_EXPORT
-Status ConvertTableToPandas(PandasOptions options, const std::shared_ptr<Table>& table,
-                            MemoryPool* pool, PyObject** out);
+Status ConvertTableToPandas(const PandasOptions& options,
+                            const std::shared_ptr<Table>& table, MemoryPool* pool,
+                            PyObject** out);
 
 /// Convert a whole table as efficiently as possible to a pandas.DataFrame.
 ///
 /// Explicitly name columns that should be a categorical
 /// This option is only used on conversions that are applied to a table.
 ARROW_PYTHON_EXPORT
-Status ConvertTableToPandas(PandasOptions options,
+Status ConvertTableToPandas(const PandasOptions& options,
                             const std::unordered_set<std::string>& categorical_columns,
                             const std::shared_ptr<Table>& table, MemoryPool* pool,
                             PyObject** out);

cpp/src/arrow/type.h

@@ -613,6 +613,7 @@ class ARROW_EXPORT Date32Type : public DateType {
   static constexpr Type::type type_id = Type::DATE32;
 
   using c_type = int32_t;
+  constexpr DateUnit Unit = DateUnit::DAY;
 
   Date32Type();
 
@@ -630,6 +631,7 @@ class ARROW_EXPORT Date64Type : public DateType {
   static constexpr Type::type type_id = Type::DATE64;
 
   using c_type = int64_t;
+  constexpr DateUnit Unit = DateUnit::MILLI;
 
   Date64Type();
 

cpp/src/arrow/type_traits.h

@@ -371,6 +371,11 @@ template <typename T>
 using enable_if_boolean =
     typename std::enable_if<std::is_same<BooleanType, T>::value>::type;
 
+template <typename T>
+using enable_if_binary_like =
+    typename std::enable_if<std::is_base_of<BinaryType, T>::value ||
+                            std::is_base_of<FixedSizeBinaryType, T>::value>::type;
+
 template <typename T>
 using enable_if_fixed_size_binary =
     typename std::enable_if<std::is_base_of<FixedSizeBinaryType, T>::value>::type;

cpp/src/arrow/util/hashing.h

@@ -102,6 +102,18 @@ struct ScalarHelper<Scalar, AlgNum,
   }
 };
 
+template <typename Scalar, uint64_t AlgNum>
+struct ScalarHelper<
+    Scalar, AlgNum,
+    typename std::enable_if<std::is_same<util::string_view, Scalar>::value>::type>
+    : public ScalarHelperBase<Scalar, AlgNum> {
+  // ScalarHelper specialization for util::string_view
+
+  static hash_t ComputeHash(const util::string_view& value) {
+    return ComputeStringHash<AlgNum>(value.data(), static_cast<int64_t>(value.size()));
+  }
+};
+
 template <typename Scalar, uint64_t AlgNum>
 struct ScalarHelper<Scalar, AlgNum,
                     typename std::enable_if<std::is_floating_point<Scalar>::value>::type>
@@ -332,7 +344,7 @@ class ScalarMemoTable {
   explicit ScalarMemoTable(int64_t entries = 0)
       : hash_table_(static_cast<uint64_t>(entries)) {}
 
-  int32_t Get(const Scalar value) const {
+  int32_t Get(const Scalar& value) const {
     auto cmp_func = [value](const Payload* payload) -> bool {
       return ScalarHelper<Scalar, 0>::CompareScalars(payload->value, value);
     };
@@ -346,7 +358,7 @@ class ScalarMemoTable {
   }
 
   template <typename Func1, typename Func2>
-  int32_t GetOrInsert(const Scalar value, Func1&& on_found, Func2&& on_not_found) {
+  int32_t GetOrInsert(const Scalar& value, Func1&& on_found, Func2&& on_not_found) {
     auto cmp_func = [value](const Payload* payload) -> bool {
       return ScalarHelper<Scalar, 0>::CompareScalars(value, payload->value);
     };
@@ -364,7 +376,7 @@ class ScalarMemoTable {
     return memo_index;
   }
 
-  int32_t GetOrInsert(const Scalar value) {
+  int32_t GetOrInsert(const Scalar& value) {
     return GetOrInsert(value, [](int32_t i) {}, [](int32_t i) {});
   }
 
@@ -389,6 +401,7 @@ class ScalarMemoTable {
     Scalar value;
     int32_t memo_index;
   };
+
   using HashTableType = HashTableTemplateType<Payload>;
   using HashTableEntry = typename HashTableType::Entry;
   HashTableType hash_table_;
@@ -621,9 +634,11 @@ class BinaryMemoTable {
   struct Payload {
     int32_t memo_index;
   };
+
   using HashTableType = HashTable<Payload>;
   using HashTableEntry = typename HashTable<Payload>::Entry;
   HashTableType hash_table_;
+
   std::vector<int32_t> offsets_;
   std::string values_;
 

python/pyarrow/array.pxi

@@ -339,7 +339,61 @@ def _restore_array(data):
     return pyarrow_wrap_array(MakeArray(ad))
 
 
-cdef class Array:
+cdef class _PandasConvertible:
+
+    def to_pandas(self, categories=None, bint strings_to_categorical=False,
+                  bint zero_copy_only=False, bint integer_object_nulls=False,
+                  bint date_as_object=False,
+                  bint use_threads=True,
+                  bint deduplicate_objects=True,
+                  bint ignore_metadata=False):
+        """
+        Convert to a pandas-compatible NumPy array or DataFrame, as appropriate
+
+        Parameters
+        ----------
+        strings_to_categorical : boolean, default False
+            Encode string (UTF8) and binary types to pandas.Categorical
+        categories: list, default empty
+            List of fields that should be returned as pandas.Categorical. Only
+            applies to table-like data structures
+        zero_copy_only : boolean, default False
+            Raise an ArrowException if this function call would require copying
+            the underlying data
+        integer_object_nulls : boolean, default False
+            Cast integers with nulls to objects
+        date_as_object : boolean, default False
+            Cast dates to objects
+        use_threads: boolean, default True
+            Whether to parallelize the conversion using multiple threads
+        deduplicate_objects : boolean, default False
+            Do not create multiple copies Python objects when created, to save
+            on memory use. Conversion will be slower
+        ignore_metadata : boolean, default False
+            If True, do not use the 'pandas' metadata to reconstruct the
+            DataFrame index, if present
+
+        Returns
+        -------
+        NumPy array or DataFrame depending on type of object
+        """
+        cdef:
+            PyObject* out
+            PandasOptions options
+
+        options = PandasOptions(
+            strings_to_categorical=strings_to_categorical,
+            zero_copy_only=zero_copy_only,
+            integer_object_nulls=integer_object_nulls,
+            date_as_object=date_as_object,
+            use_threads=use_threads,
+            deduplicate_objects=deduplicate_objects)
+
+        return self._to_pandas(options, categories=categories,
+                               ignore_metadata=ignore_metadata)
+
+
+cdef class Array(_PandasConvertible):
 
     def __init__(self):
         raise TypeError("Do not call {}'s constructor directly, use one of "
@@ -602,42 +656,13 @@ cdef class Array:
 
         return pyarrow_wrap_array(result)
 
-    def to_pandas(self, bint strings_to_categorical=False,
-                  bint zero_copy_only=False, bint integer_object_nulls=False,
-                  bint date_as_object=False):
-        """
-        Convert to a NumPy array object suitable for use in pandas.
-
-        Parameters
-        ----------
-        strings_to_categorical : boolean, default False
-            Encode string (UTF8) and binary types to pandas.Categorical
-        zero_copy_only : boolean, default False
-            Raise an ArrowException if this function call would require copying
-            the underlying data
-        integer_object_nulls : boolean, default False
-            Cast integers with nulls to objects
-        date_as_object : boolean, default False
-            Cast dates to objects
-
-        See also
-        --------
-        Column.to_pandas
-        Table.to_pandas
-        RecordBatch.to_pandas
-        """
+    def _to_pandas(self, options, **kwargs):
         cdef:
             PyObject* out
-            PandasOptions options
+            PandasOptions c_options = options
 
-        options = PandasOptions(
-            strings_to_categorical=strings_to_categorical,
-            zero_copy_only=zero_copy_only,
-            integer_object_nulls=integer_object_nulls,
-            date_as_object=date_as_object,
-            use_threads=False)
         with nogil:
-            check_status(ConvertArrayToPandas(options, self.sp_array,
+            check_status(ConvertArrayToPandas(c_options, self.sp_array,
                                               self, &out))
         return wrap_array_output(out)
 

python/pyarrow/includes/libarrow.pxd

@@ -1064,20 +1064,20 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
     CStatus TensorToNdarray(const shared_ptr[CTensor]& tensor, object base,
                             PyObject** out)
 
-    CStatus ConvertArrayToPandas(PandasOptions options,
+    CStatus ConvertArrayToPandas(const PandasOptions& options,
                                  const shared_ptr[CArray]& arr,
                                  object py_ref, PyObject** out)
 
-    CStatus ConvertChunkedArrayToPandas(PandasOptions options,
+    CStatus ConvertChunkedArrayToPandas(const PandasOptions& options,
                                         const shared_ptr[CChunkedArray]& arr,
                                         object py_ref, PyObject** out)
 
-    CStatus ConvertColumnToPandas(PandasOptions options,
+    CStatus ConvertColumnToPandas(const PandasOptions& options,
                                   const shared_ptr[CColumn]& arr,
                                   object py_ref, PyObject** out)
 
     CStatus ConvertTableToPandas(
-        PandasOptions options,
+        const PandasOptions& options,
         const unordered_set[c_string]& categorical_columns,
         const shared_ptr[CTable]& table,
         CMemoryPool* pool,
@@ -1110,6 +1110,7 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
         c_bool integer_object_nulls
         c_bool date_as_object
         c_bool use_threads
+        c_bool deduplicate_objects
 
 cdef extern from "arrow/python/api.h" namespace 'arrow::py' nogil:
 

python/pyarrow/lib.pxd

@@ -179,7 +179,11 @@ cdef class FixedSizeBinaryValue(ArrayValue):
     pass
 
 
-cdef class Array:
+cdef class _PandasConvertible:
+    pass
+
+
+cdef class Array(_PandasConvertible):
     cdef:
         shared_ptr[CArray] sp_array
         CArray* ap
@@ -306,7 +310,7 @@ cdef object box_scalar(DataType type,
                        int64_t index)
 
 
-cdef class ChunkedArray:
+cdef class ChunkedArray(_PandasConvertible):
     cdef:
         shared_ptr[CChunkedArray] sp_chunked_array
         CChunkedArray* chunked_array
@@ -315,7 +319,7 @@ cdef class ChunkedArray:
     cdef getitem(self, int64_t i)
 
 
-cdef class Column:
+cdef class Column(_PandasConvertible):
     cdef:
         shared_ptr[CColumn] sp_column
         CColumn* column
@@ -323,7 +327,7 @@ cdef class Column:
     cdef void init(self, const shared_ptr[CColumn]& column)
 
 
-cdef class Table:
+cdef class Table(_PandasConvertible):
     cdef:
         shared_ptr[CTable] sp_table
         CTable* table
@@ -331,7 +335,7 @@ cdef class Table:
     cdef void init(self, const shared_ptr[CTable]& table)
 
 
-cdef class RecordBatch:
+cdef class RecordBatch(_PandasConvertible):
     cdef:
         shared_ptr[CRecordBatch] sp_batch
         CRecordBatch* batch

python/pyarrow/pandas_compat.py

@@ -548,7 +548,7 @@ def _make_datetimetz(tz):
 # Converting pyarrow.Table efficiently to pandas.DataFrame
 
 
-def table_to_blockmanager(options, table, memory_pool, categories=None,
+def table_to_blockmanager(options, table, categories=None,
                           ignore_metadata=False):
     from pyarrow.compat import DatetimeTZDtype
 
@@ -624,7 +624,8 @@ def table_to_blockmanager(options, table, memory_pool, categories=None,
                 block_table.schema.get_field_index(raw_name)
             )
 
-    blocks = _table_to_blocks(options, block_table, memory_pool, categories)
+    blocks = _table_to_blocks(options, block_table, pa.default_memory_pool(),
+                              categories)
 
     # Construct the row index
     if len(index_arrays) > 1:

python/pyarrow/table.pxi

@@ -28,7 +28,7 @@ else:
     import pyarrow.pandas_compat as pdcompat
 
 
-cdef class ChunkedArray:
+cdef class ChunkedArray(_PandasConvertible):
     """
     Array backed via one or more memory chunks.
 
@@ -145,43 +145,14 @@ cdef class ChunkedArray:
 
         return result
 
-    def to_pandas(self, bint strings_to_categorical=False,
-                  bint zero_copy_only=False, bint integer_object_nulls=False,
-                  bint date_as_object=False):
-        """
-        Convert the arrow::ChunkedArray to an array object suitable for use
-        in pandas
-
-        Parameters
-        ----------
-        strings_to_categorical : boolean, default False
-            Encode string (UTF8) and binary types to pandas.Categorical
-        zero_copy_only : boolean, default False
-            Raise an ArrowException if this function call would require copying
-            the underlying data
-        integer_object_nulls : boolean, default False
-            Cast integers with nulls to objects
-        date_as_object : boolean, default False
-            Cast dates to objects
-
-        See also
-        --------
-        Column.to_pandas
-        """
+    def _to_pandas(self, options, **kwargs):
         cdef:
             PyObject* out
-            PandasOptions options
-
-        options = PandasOptions(
-            strings_to_categorical=strings_to_categorical,
-            zero_copy_only=zero_copy_only,
-            integer_object_nulls=integer_object_nulls,
-            date_as_object=date_as_object,
-            use_threads=False)
+            PandasOptions c_options = options
 
         with nogil:
             check_status(libarrow.ConvertChunkedArrayToPandas(
-                options,
+                c_options,
                 self.sp_chunked_array,
                 self, &out))
 
@@ -385,7 +356,7 @@ def column(object field_or_name, arr):
     return pyarrow_wrap_column(sp_column)
 
 
-cdef class Column:
+cdef class Column(_PandasConvertible):
     """
     Named vector of elements of equal type.
 
@@ -497,33 +468,8 @@ cdef class Column:
 
         return [pyarrow_wrap_column(col) for col in flattened]
 
-    def to_pandas(self, bint strings_to_categorical=False,
-                  bint zero_copy_only=False, bint integer_object_nulls=False,
-                  bint date_as_object=False):
-        """
-        Convert the arrow::Column to a pandas.Series
-
-        Parameters
-        ----------
-        strings_to_categorical : boolean, default False
-            Encode string (UTF8) and binary types to pandas.Categorical
-        zero_copy_only : boolean, default False
-            Raise an ArrowException if this function call would require copying
-            the underlying data
-        integer_object_nulls : boolean, default False
-            Cast integers with nulls to objects
-        date_as_object : boolean, default False
-            Cast dates to objects
-
-        Returns
-        -------
-        pandas.Series
-        """
-        values = self.data.to_pandas(
-            strings_to_categorical=strings_to_categorical,
-            zero_copy_only=zero_copy_only,
-            date_as_object=date_as_object,
-            integer_object_nulls=integer_object_nulls)
+    def _to_pandas(self, options, **kwargs):
+        values = self.data._to_pandas(options)
         result = pd.Series(values, name=self.name)
 
         if isinstance(self.type, TimestampType):
@@ -685,7 +631,7 @@ cdef _schema_from_arrays(arrays, names, metadata, shared_ptr[CSchema]* schema):
     schema.reset(new CSchema(c_fields, c_meta))
 
 
-cdef class RecordBatch:
+cdef class RecordBatch(_PandasConvertible):
     """
     Batch of rows of columns of equal length
 
@@ -887,46 +833,8 @@ cdef class RecordBatch:
             entries.append((name, column))
         return OrderedDict(entries)
 
-    def to_pandas(self, MemoryPool memory_pool=None, categories=None,
-                  bint strings_to_categorical=False, bint zero_copy_only=False,
-                  bint integer_object_nulls=False, bint date_as_object=False,
-                  bint use_threads=True, bint ignore_metadata=False):
-        """
-        Convert the arrow::RecordBatch to a pandas DataFrame
-
-        Parameters
-        ----------
-        memory_pool: MemoryPool, optional
-            Specific memory pool to use to allocate casted columns
-        categories: list, default empty
-            List of columns that should be returned as pandas.Categorical
-        strings_to_categorical : boolean, default False
-            Encode string (UTF8) and binary types to pandas.Categorical
-        zero_copy_only : boolean, default False
-            Raise an ArrowException if this function call would require copying
-            the underlying data
-        integer_object_nulls : boolean, default False
-            Cast integers with nulls to objects
-        date_as_object : boolean, default False
-            Cast dates to objects
-        use_threads: boolean, default True
-            Whether to parallelize the conversion using multiple threads
-        ignore_metadata : boolean, default False
-            If True, do not use the 'pandas' metadata to reconstruct the
-            DataFrame index, if present
-
-        Returns
-        -------
-        pandas.DataFrame
-        """
-        return Table.from_batches([self]).to_pandas(
-            memory_pool=memory_pool, categories=categories,
-            strings_to_categorical=strings_to_categorical,
-            zero_copy_only=zero_copy_only,
-            integer_object_nulls=integer_object_nulls,
-            date_as_object=date_as_object, use_threads=use_threads,
-            ignore_metadata=ignore_metadata
-        )
+    def _to_pandas(self, options, **kwargs):
+        return Table.from_batches([self])._to_pandas(options, **kwargs)
 
     @classmethod
     def from_pandas(cls, df, Schema schema=None, bint preserve_index=True,
@@ -1031,7 +939,7 @@ def table_to_blocks(PandasOptions options, Table table,
     return PyObject_to_object(result_obj)
 
 
-cdef class Table:
+cdef class Table(_PandasConvertible):
     """
     A collection of top-level named, equal length Arrow arrays.
 
@@ -1386,50 +1294,8 @@ cdef class Table:
 
         return result
 
-    def to_pandas(self, MemoryPool memory_pool=None, categories=None,
-                  bint strings_to_categorical=False, bint zero_copy_only=False,
-                  bint integer_object_nulls=False, bint date_as_object=False,
-                  bint use_threads=True, bint ignore_metadata=False):
-        """
-        Convert the arrow::Table to a pandas DataFrame
-
-        Parameters
-        ----------
-        memory_pool: MemoryPool, optional
-            Specific memory pool to use to allocate casted columns
-        categories: list, default empty
-            List of columns that should be returned as pandas.Categorical
-        strings_to_categorical : boolean, default False
-            Encode string (UTF8) and binary types to pandas.Categorical
-        zero_copy_only : boolean, default False
-            Raise an ArrowException if this function call would require copying
-            the underlying data
-        integer_object_nulls : boolean, default False
-            Cast integers with nulls to objects
-        date_as_object : boolean, default False
-            Cast dates to objects
-        use_threads: boolean, default True
-            Whether to parallelize the conversion using multiple threads
-        ignore_metadata : boolean, default False
-            If True, do not use the 'pandas' metadata to reconstruct the
-            DataFrame index, if present
-
-        Returns
-        -------
-        pandas.DataFrame
-        """
-        cdef:
-            PandasOptions options
-
-        options = PandasOptions(
-            strings_to_categorical=strings_to_categorical,
-            zero_copy_only=zero_copy_only,
-            integer_object_nulls=integer_object_nulls,
-            date_as_object=date_as_object,
-            use_threads=use_threads)
-
-        mgr = pdcompat.table_to_blockmanager(options, self, memory_pool,
-                                             categories,
+    def _to_pandas(self, options, categories=None, ignore_metadata=False):
+        mgr = pdcompat.table_to_blockmanager(options, self, categories,
                                              ignore_metadata=ignore_metadata)
         return pd.DataFrame(mgr)