Templatize integer conversions

cpp/src/arrow/python/builtin_convert.cc

@@ -429,83 +429,15 @@ class BoolConverter : public TypedConverterVisitor<BooleanBuilder, BoolConverter
   }
 };
 
-class Int8Converter : public TypedConverterVisitor<Int8Builder, Int8Converter> {
+template <typename IntType>
+class TypedIntConverter
+    : public TypedConverterVisitor<NumericBuilder<IntType>, TypedIntConverter<IntType>> {
  public:
   // Append a non-missing item
   Status AppendItem(PyObject* obj) {
-    int8_t value;
-    RETURN_NOT_OK(internal::Int8FromPythonInt(obj, &value));
-    return typed_builder_->Append(value);
-  }
-};
-
-class Int16Converter : public TypedConverterVisitor<Int16Builder, Int16Converter> {
- public:
-  // Append a non-missing item
-  Status AppendItem(PyObject* obj) {
-    int16_t value;
-    RETURN_NOT_OK(internal::Int16FromPythonInt(obj, &value));
-    return typed_builder_->Append(value);
-  }
-};
-
-class Int32Converter : public TypedConverterVisitor<Int32Builder, Int32Converter> {
- public:
-  // Append a non-missing item
-  Status AppendItem(PyObject* obj) {
-    int32_t value;
-    RETURN_NOT_OK(internal::Int32FromPythonInt(obj, &value));
-    return typed_builder_->Append(value);
-  }
-};
-
-class Int64Converter : public TypedConverterVisitor<Int64Builder, Int64Converter> {
- public:
-  // Append a non-missing item
-  Status AppendItem(PyObject* obj) {
-    int64_t value;
-    RETURN_NOT_OK(internal::Int64FromPythonInt(obj, &value));
-    return typed_builder_->Append(value);
-  }
-};
-
-class UInt8Converter : public TypedConverterVisitor<UInt8Builder, UInt8Converter> {
- public:
-  // Append a non-missing item
-  Status AppendItem(PyObject* obj) {
-    uint8_t value;
-    RETURN_NOT_OK(internal::UInt8FromPythonInt(obj, &value));
-    return typed_builder_->Append(value);
-  }
-};
-
-class UInt16Converter : public TypedConverterVisitor<UInt16Builder, UInt16Converter> {
- public:
-  // Append a non-missing item
-  Status AppendItem(PyObject* obj) {
-    uint16_t value;
-    RETURN_NOT_OK(internal::UInt16FromPythonInt(obj, &value));
-    return typed_builder_->Append(value);
-  }
-};
-
-class UInt32Converter : public TypedConverterVisitor<UInt32Builder, UInt32Converter> {
- public:
-  // Append a non-missing item
-  Status AppendItem(PyObject* obj) {
-    uint32_t value;
-    RETURN_NOT_OK(internal::UInt32FromPythonInt(obj, &value));
-    return typed_builder_->Append(value);
-  }
-};
-
-class UInt64Converter : public TypedConverterVisitor<UInt64Builder, UInt64Converter> {
- public:
-  // Append a non-missing item
-  Status AppendItem(PyObject* obj) {
-    uint64_t val;
-    RETURN_NOT_OK(internal::UInt64FromPythonInt(obj, &val));
-    return typed_builder_->Append(val);
+    typename IntType::c_type value;
+    RETURN_NOT_OK(internal::CIntFromPython(obj, &value));
+    return this->typed_builder_->Append(value);
   }
 };
 
@@ -518,12 +450,7 @@ class Date32Converter : public TypedConverterVisitor<Date32Builder, Date32Conver
       auto pydate = reinterpret_cast<PyDateTime_Date*>(obj);
       t = static_cast<int32_t>(PyDate_to_s(pydate));
     } else {
-      const auto casted_val = static_cast<int64_t>(PyLong_AsLongLong(obj));
-      RETURN_IF_PYERROR();
-      if (casted_val > std::numeric_limits<int32_t>::max()) {
-        return Status::Invalid("Integer as date32 larger than INT32_MAX");
-      }
-      t = static_cast<int32_t>(casted_val);
+      RETURN_NOT_OK(internal::CIntFromPython(obj, &t, "Integer too large for date32"));
     }
     return typed_builder_->Append(t);
   }
@@ -538,8 +465,7 @@ class Date64Converter : public TypedConverterVisitor<Date64Builder, Date64Conver
       auto pydate = reinterpret_cast<PyDateTime_Date*>(obj);
       t = PyDate_to_ms(pydate);
     } else {
-      t = static_cast<int64_t>(PyLong_AsLongLong(obj));
-      RETURN_IF_PYERROR();
+      RETURN_NOT_OK(internal::CIntFromPython(obj, &t, "Integer too large for date64"));
     }
     return typed_builder_->Append(t);
   }
@@ -590,8 +516,7 @@ class TimestampConverter
 
       t = reinterpret_cast<PyDatetimeScalarObject*>(obj)->obval;
     } else {
-      t = static_cast<int64_t>(PyLong_AsLongLong(obj));
-      RETURN_IF_PYERROR();
+      RETURN_NOT_OK(internal::CIntFromPython(obj, &t));
     }
     return typed_builder_->Append(t);
   }
@@ -756,21 +681,21 @@ std::unique_ptr<SeqConverter> GetConverter(const std::shared_ptr<DataType>& type
     case Type::BOOL:
       return std::unique_ptr<SeqConverter>(new BoolConverter);
     case Type::INT8:
-      return std::unique_ptr<SeqConverter>(new Int8Converter);
+      return std::unique_ptr<SeqConverter>(new TypedIntConverter<Int8Type>);
     case Type::INT16:
-      return std::unique_ptr<SeqConverter>(new Int16Converter);
+      return std::unique_ptr<SeqConverter>(new TypedIntConverter<Int16Type>);
     case Type::INT32:
-      return std::unique_ptr<SeqConverter>(new Int32Converter);
+      return std::unique_ptr<SeqConverter>(new TypedIntConverter<Int32Type>);
     case Type::INT64:
-      return std::unique_ptr<SeqConverter>(new Int64Converter);
+      return std::unique_ptr<SeqConverter>(new TypedIntConverter<Int64Type>);
     case Type::UINT8:
-      return std::unique_ptr<SeqConverter>(new UInt8Converter);
+      return std::unique_ptr<SeqConverter>(new TypedIntConverter<UInt8Type>);
     case Type::UINT16:
-      return std::unique_ptr<SeqConverter>(new UInt16Converter);
+      return std::unique_ptr<SeqConverter>(new TypedIntConverter<UInt16Type>);
     case Type::UINT32:
-      return std::unique_ptr<SeqConverter>(new UInt32Converter);
+      return std::unique_ptr<SeqConverter>(new TypedIntConverter<UInt32Type>);
     case Type::UINT64:
-      return std::unique_ptr<SeqConverter>(new UInt64Converter);
+      return std::unique_ptr<SeqConverter>(new TypedIntConverter<UInt64Type>);
     case Type::DATE32:
       return std::unique_ptr<SeqConverter>(new Date32Converter);
     case Type::DATE64:

cpp/src/arrow/python/common.h

@@ -35,9 +35,9 @@ class MemoryPool;
 
 namespace py {
 
-Status CheckPyError(StatusCode code = StatusCode::UnknownError);
+ARROW_EXPORT Status CheckPyError(StatusCode code = StatusCode::UnknownError);
 
-Status PassPyError();
+ARROW_EXPORT Status PassPyError();
 
 // TODO(wesm): We can just let errors pass through. To be explored later
 #define RETURN_IF_PYERROR() RETURN_NOT_OK(CheckPyError());

cpp/src/arrow/python/helpers.cc

@@ -17,6 +17,8 @@
 
 #include <limits>
 #include <sstream>
+#include <type_traits>
+#include <typeinfo>
 
 #include "arrow/python/common.h"
 #include "arrow/python/helpers.h"
@@ -99,6 +101,7 @@ Status PyUnicode_AsStdString(PyObject* obj, std::string* out) {
 }
 
 std::string PyObject_StdStringRepr(PyObject* obj) {
+#if PY_MAJOR_VERSION >= 3
   OwnedRef unicode_ref(PyObject_Repr(obj));
   OwnedRef bytes_ref;
 
@@ -106,18 +109,27 @@ std::string PyObject_StdStringRepr(PyObject* obj) {
     bytes_ref.reset(
         PyUnicode_AsEncodedString(unicode_ref.obj(), "utf8", "backslashreplace"));
   }
+#else
+  OwnedRef bytes_ref(PyObject_Repr(obj));
   if (!bytes_ref) {
     PyErr_Clear();
     std::stringstream ss;
     ss << "<object of type '" << Py_TYPE(obj)->tp_name << "' repr() failed>";
     return ss.str();
   }
+#endif
   return PyBytes_AsStdString(bytes_ref.obj());
 }
 
 Status PyObject_StdStringStr(PyObject* obj, std::string* out) {
-  OwnedRef unicode_ref(PyObject_Str(obj));
-  return PyUnicode_AsStdString(unicode_ref.obj(), out);
+  OwnedRef string_ref(PyObject_Str(obj));
+  RETURN_IF_PYERROR();
+#if PY_MAJOR_VERSION >= 3
+  return PyUnicode_AsStdString(string_ref.obj(), out);
+#else
+  *out = PyBytes_AsStdString(string_ref.obj());
+  return Status::OK();
+#endif
 }
 
 Status ImportModule(const std::string& module_name, OwnedRef* ref) {
@@ -214,10 +226,50 @@ Status BuilderAppend(StringBuilder* builder, PyObject* obj, bool check_valid,
 
 namespace {
 
-template <typename UInt>
-Status UnsignedIntFromPython(PyObject* obj, UInt* out) {
-  static_assert(std::is_unsigned<UInt>::value, "requires unsigned integer type");
-  static_assert(sizeof(UInt) <= sizeof(unsigned long long),  // NOLINT
+Status IntegerOverflowStatus(const std::string& overflow_message) {
+  if (overflow_message.empty()) {
+    return Status::Invalid("Value too large to fit in C integer type");
+  } else {
+    return Status::Invalid(overflow_message);
+  }
+}
+
+// Extract C signed int from Python object
+template <typename Int,
+          typename std::enable_if<std::is_signed<Int>::value, Int>::type = 0>
+Status CIntFromPythonImpl(PyObject* obj, Int* out, const std::string& overflow_message) {
+  static_assert(sizeof(Int) <= sizeof(long long),  // NOLINT
+                "integer type larger than long long");
+
+  if (sizeof(Int) > sizeof(long)) {  // NOLINT
+    const auto value = PyLong_AsLongLong(obj);
+    if (ARROW_PREDICT_FALSE(value == -1)) {
+      RETURN_IF_PYERROR();
+    }
+    if (ARROW_PREDICT_FALSE(value < std::numeric_limits<Int>::min() ||
+                            value > std::numeric_limits<Int>::max())) {
+      return IntegerOverflowStatus(overflow_message);
+    }
+    *out = static_cast<Int>(value);
+  } else {
+    const auto value = PyLong_AsLong(obj);
+    if (ARROW_PREDICT_FALSE(value == -1)) {
+      RETURN_IF_PYERROR();
+    }
+    if (ARROW_PREDICT_FALSE(value < std::numeric_limits<Int>::min() ||
+                            value > std::numeric_limits<Int>::max())) {
+      return IntegerOverflowStatus(overflow_message);
+    }
+    *out = static_cast<Int>(value);
+  }
+  return Status::OK();
+}
+
+// Extract C unsigned int from Python object
+template <typename Int,
+          typename std::enable_if<std::is_unsigned<Int>::value, Int>::type = 0>
+Status CIntFromPythonImpl(PyObject* obj, Int* out, const std::string& overflow_message) {
+  static_assert(sizeof(Int) <= sizeof(unsigned long long),  // NOLINT
                 "integer type larger than unsigned long long");
 
   OwnedRef ref;
@@ -230,60 +282,22 @@ Status UnsignedIntFromPython(PyObject* obj, UInt* out) {
     }
     obj = ref.obj();
   }
-  if (sizeof(UInt) > sizeof(unsigned long)) {  // NOLINT
+  if (sizeof(Int) > sizeof(unsigned long)) {  // NOLINT
     const auto value = PyLong_AsUnsignedLongLong(obj);
     if (ARROW_PREDICT_FALSE(value == static_cast<decltype(value)>(-1))) {
       RETURN_IF_PYERROR();
     }
-    if (ARROW_PREDICT_FALSE(value > std::numeric_limits<UInt>::max())) {
-      std::stringstream ss;
-      ss << "value too large to fit in " << typeid(UInt).name();
-      return Status::Invalid(ss.str());
+    if (ARROW_PREDICT_FALSE(value > std::numeric_limits<Int>::max())) {
+      return IntegerOverflowStatus(overflow_message);
     }
-    *out = static_cast<UInt>(value);
+    *out = static_cast<Int>(value);
   } else {
     const auto value = PyLong_AsUnsignedLong(obj);
     if (ARROW_PREDICT_FALSE(value == static_cast<decltype(value)>(-1))) {
       RETURN_IF_PYERROR();
     }
-    if (ARROW_PREDICT_FALSE(value > std::numeric_limits<UInt>::max())) {
-      std::stringstream ss;
-      ss << "value too large to fit in " << typeid(UInt).name();
-      return Status::Invalid(ss.str());
-    }
-    *out = static_cast<UInt>(value);
-  }
-  return Status::OK();
-}
-
-template <typename Int>
-Status SignedIntFromPython(PyObject* obj, Int* out) {
-  static_assert(std::is_signed<Int>::value, "requires signed integer type");
-  static_assert(sizeof(Int) <= sizeof(long long),  // NOLINT
-                "integer type larger than long long");
-
-  if (sizeof(Int) > sizeof(long)) {  // NOLINT
-    const auto value = PyLong_AsLongLong(obj);
-    if (ARROW_PREDICT_FALSE(value == -1)) {
-      RETURN_IF_PYERROR();
-    }
-    if (ARROW_PREDICT_FALSE(value < std::numeric_limits<Int>::min() ||
-                            value > std::numeric_limits<Int>::max())) {
-      std::stringstream ss;
-      ss << "value too large to fit in " << typeid(Int).name();
-      return Status::Invalid(ss.str());
-    }
-    *out = static_cast<Int>(value);
-  } else {
-    const auto value = PyLong_AsLong(obj);
-    if (ARROW_PREDICT_FALSE(value == -1)) {
-      RETURN_IF_PYERROR();
-    }
-    if (ARROW_PREDICT_FALSE(value < std::numeric_limits<Int>::min() ||
-                            value > std::numeric_limits<Int>::max())) {
-      std::stringstream ss;
-      ss << "value too large to fit in " << typeid(Int).name();
-      return Status::Invalid(ss.str());
+    if (ARROW_PREDICT_FALSE(value > std::numeric_limits<Int>::max())) {
+      return IntegerOverflowStatus(overflow_message);
     }
     *out = static_cast<Int>(value);
   }
@@ -292,37 +306,22 @@ Status SignedIntFromPython(PyObject* obj, Int* out) {
 
 }  // namespace
 
-Status UInt8FromPythonInt(PyObject* obj, uint8_t* out) {
-  return UnsignedIntFromPython(obj, out);
+template <typename Int>
+Status CIntFromPython(PyObject* obj, Int* out, const std::string& overflow_message) {
+  if (PyBool_Check(obj)) {
+    return Status::TypeError("Expected integer, got bool");
+  }
+  return CIntFromPythonImpl(obj, out, overflow_message);
 }
 
-Status UInt16FromPythonInt(PyObject* obj, uint16_t* out) {
-  return UnsignedIntFromPython(obj, out);
-}
-
-Status UInt32FromPythonInt(PyObject* obj, uint32_t* out) {
-  return UnsignedIntFromPython(obj, out);
-}
-
-Status UInt64FromPythonInt(PyObject* obj, uint64_t* out) {
-  return UnsignedIntFromPython(obj, out);
-}
-
-Status Int8FromPythonInt(PyObject* obj, int8_t* out) {
-  return SignedIntFromPython(obj, out);
-}
-
-Status Int16FromPythonInt(PyObject* obj, int16_t* out) {
-  return SignedIntFromPython(obj, out);
-}
-
-Status Int32FromPythonInt(PyObject* obj, int32_t* out) {
-  return SignedIntFromPython(obj, out);
-}
-
-Status Int64FromPythonInt(PyObject* obj, int64_t* out) {
-  return SignedIntFromPython(obj, out);
-}
+template Status CIntFromPython(PyObject*, int8_t*, const std::string&);
+template Status CIntFromPython(PyObject*, int16_t*, const std::string&);
+template Status CIntFromPython(PyObject*, int32_t*, const std::string&);
+template Status CIntFromPython(PyObject*, int64_t*, const std::string&);
+template Status CIntFromPython(PyObject*, uint8_t*, const std::string&);
+template Status CIntFromPython(PyObject*, uint16_t*, const std::string&);
+template Status CIntFromPython(PyObject*, uint32_t*, const std::string&);
+template Status CIntFromPython(PyObject*, uint64_t*, const std::string&);
 
 }  // namespace internal
 }  // namespace py

cpp/src/arrow/python/helpers.h

@@ -85,14 +85,8 @@ inline bool IsPyBinary(PyObject* obj) {
 // \param[in] obj A Python integer
 // \param[out] out A pointer to a C integer to hold the result of the conversion
 // \return The status of the operation
-Status Int8FromPythonInt(PyObject* obj, int8_t* out);
-Status Int16FromPythonInt(PyObject* obj, int16_t* out);
-Status Int32FromPythonInt(PyObject* obj, int32_t* out);
-Status Int64FromPythonInt(PyObject* obj, int64_t* out);
-Status UInt8FromPythonInt(PyObject* obj, uint8_t* out);
-Status UInt16FromPythonInt(PyObject* obj, uint16_t* out);
-Status UInt32FromPythonInt(PyObject* obj, uint32_t* out);
-Status UInt64FromPythonInt(PyObject* obj, uint64_t* out);
+template <typename Int>
+Status CIntFromPython(PyObject* obj, Int* out, const std::string& overflow_message = "");
 
 // \brief Convert a Python unicode string to a std::string
 Status PyUnicode_AsStdString(PyObject* obj, std::string* out);

cpp/src/arrow/python/numpy_to_arrow.cc

@@ -930,15 +930,10 @@ Status NumPyConverter::ConvertObjectIntegers() {
     obj = objects[i];
     if ((have_mask && mask_values[i]) || internal::PandasObjectIsNull(obj)) {
       RETURN_NOT_OK(builder.AppendNull());
-    } else if (PyObject_is_integer(obj)) {
-      const int64_t val = static_cast<int64_t>(PyLong_AsLong(obj));
-      RETURN_IF_PYERROR();
-      RETURN_NOT_OK(builder.Append(val));
     } else {
-      std::stringstream ss;
-      ss << "Error converting from Python objects to Int64: ";
-      RETURN_NOT_OK(InvalidConversion(obj, "integer", &ss));
-      return Status::Invalid(ss.str());
+      int64_t val;
+      RETURN_NOT_OK(internal::CIntFromPython(obj, &val));
+      RETURN_NOT_OK(builder.Append(val));
     }
   }