Implement casts from null to numbers. Try to cast for types where we do not have an inference rule when converting from arrays of Python objects

Change-Id: Ife0da474c9c6dc679395911757e948ceb90757fa

cpp/src/arrow/compute/cast.cc

@@ -80,10 +80,12 @@ struct CastFunctor<O, I, typename std::enable_if<is_zero_copy_cast<O, I>::value>
 
 template <typename T>
 struct CastFunctor<T, NullType,
-                   typename std::enable_if<!std::is_same<T, NullType>::value>::type> {
+                   typename std::enable_if<std::is_base_of<FixedWidthType, T>::value>::type> {
   void operator()(FunctionContext* ctx, const CastOptions& options,
                   const ArrayData& input, ArrayData* output) {
-    ctx->SetStatus(Status::NotImplemented("NullType"));
+    // Simply initialize data to 0
+    auto buf = output->buffers[1];
+    memset(buf->mutable_data(), 0, buf->size());
   }
 };
 
@@ -234,7 +236,15 @@ static Status AllocateLike(FunctionContext* ctx, const ArrayData& in_data,
   // Propagate null bitmap
   // TODO(wesm): handling null bitmap when input type is NullType
   out->buffers.clear();
-  out->buffers.push_back(in_data.buffers[0]);
+
+  // Propagate bitmap unless we are null type
+  std::shared_ptr<Buffer> validity_bitmap = in_data.buffers[0];
+  if (in_data.type->id() == Type::NA) {
+    int64_t bitmap_size = BitUtil::BytesForBits(in_data.length);
+    RETURN_NOT_OK(ctx->Allocate(bitmap_size, &validity_bitmap));
+    memset(validity_bitmap->mutable_data(), 0, bitmap_size);
+  }
+  out->buffers.push_back(validity_bitmap);
 
   std::shared_ptr<Buffer> out_data;
 
@@ -292,6 +302,14 @@ void CastKernel::operator()(FunctionContext* ctx, const ArrayData& input,
   FN(IN_TYPE, FloatType);          \
   FN(IN_TYPE, DoubleType);
 
+#define NULL_CASES(FN, IN_TYPE)                 \
+  NUMERIC_CASES(FN, IN_TYPE)                    \
+  FN(NullType, Time32Type);                     \
+  FN(NullType, Date32Type);                     \
+  FN(NullType, TimestampType);                  \
+  FN(NullType, Time64Type);                     \
+  FN(NullType, Date64Type);
+
 #define INT32_CASES(FN, IN_TYPE) \
   NUMERIC_CASES(FN, IN_TYPE)     \
   FN(Int32Type, Time32Type);     \
@@ -329,6 +347,7 @@ void CastKernel::operator()(FunctionContext* ctx, const ArrayData& input,
     return nullptr;                                                                    \
   }
 
+GET_CAST_FUNCTION(NULL_CASES, NullType);
 GET_CAST_FUNCTION(NUMERIC_CASES, BooleanType);
 GET_CAST_FUNCTION(NUMERIC_CASES, UInt8Type);
 GET_CAST_FUNCTION(NUMERIC_CASES, Int8Type);
@@ -354,6 +373,7 @@ GET_CAST_FUNCTION(TIMESTAMP_CASES, TimestampType);
 Status GetCastFunction(const DataType& in_type, const std::shared_ptr<DataType>& out_type,
                        const CastOptions& options, std::unique_ptr<CastKernel>* kernel) {
   switch (in_type.id()) {
+    CAST_FUNCTION_CASE(NullType);
     CAST_FUNCTION_CASE(BooleanType);
     CAST_FUNCTION_CASE(UInt8Type);
     CAST_FUNCTION_CASE(Int8Type);

cpp/src/arrow/compute/compute-test.cc

@@ -341,5 +341,21 @@ TEST_F(TestCast, DateTimeZeroCopy) {
   CheckZeroCopy(*arr, timestamp(TimeUnit::NANO));
 }
 
+TEST_F(TestCast, FromNull) {
+  // Null casts to everything
+  const int length = 10;
+
+  NullArray arr(length);
+
+  std::shared_ptr<Array> result;
+  ASSERT_OK(Cast(&ctx_, arr, int32(), {}, &result));
+
+  ASSERT_EQ(length, result->length());
+  ASSERT_EQ(length, result->null_count());
+
+  // OK to look at bitmaps
+  AssertArraysEqual(*result, *result);
+}
+
 }  // namespace compute
 }  // namespace arrow

cpp/src/arrow/python/pandas_to_arrow.cc

@@ -422,9 +422,11 @@ class PandasConverter {
   Status ConvertLists(const std::shared_ptr<DataType>& type);
   Status ConvertLists(const std::shared_ptr<DataType>& type, ListBuilder* builder,
                       PyObject* list);
-  Status ConvertObjects();
   Status ConvertDecimals();
   Status ConvertTimes();
+  Status ConvertObjects();
+  Status ConvertObjectsInfer();
+  Status ConvertObjectsInferAndCast();
 
  protected:
   MemoryPool* pool_;
@@ -867,6 +869,74 @@ Status PandasConverter::ConvertBooleans() {
   return Status::OK();
 }
 
+Status PandasConverter::ConvertObjectsInfer() {
+  Ndarray1DIndexer<PyObject*> objects;
+
+  PyAcquireGIL lock;
+  objects.Init(arr_);
+  PyDateTime_IMPORT;
+
+  OwnedRef decimal;
+  OwnedRef Decimal;
+  RETURN_NOT_OK(ImportModule("decimal", &decimal));
+  RETURN_NOT_OK(ImportFromModule(decimal, "Decimal", &Decimal));
+
+  for (int64_t i = 0; i < length_; ++i) {
+    PyObject* obj = objects[i];
+    if (PandasObjectIsNull(obj)) {
+      continue;
+    } else if (PyObject_is_string(obj)) {
+      return ConvertObjectStrings();
+    } else if (PyObject_is_float(obj)) {
+      return ConvertObjectFloats();
+    } else if (PyBool_Check(obj)) {
+      return ConvertBooleans();
+    } else if (PyObject_is_integer(obj)) {
+      return ConvertObjectIntegers();
+    } else if (PyDate_CheckExact(obj)) {
+      // We could choose Date32 or Date64
+      return ConvertDates<Date32Type>();
+    } else if (PyTime_Check(obj)) {
+      return ConvertTimes();
+    } else if (PyObject_IsInstance(const_cast<PyObject*>(obj), Decimal.obj())) {
+      return ConvertDecimals();
+    } else if (PyList_Check(obj) || PyArray_Check(obj)) {
+      std::shared_ptr<DataType> inferred_type;
+      RETURN_NOT_OK(InferArrowType(obj, &inferred_type));
+      return ConvertLists(inferred_type);
+    } else {
+      const std::string supported_types =
+        "string, bool, float, int, date, time, decimal, list, array";
+      std::stringstream ss;
+      ss << "Error inferring Arrow type for Python object array. ";
+      RETURN_NOT_OK(InvalidConversion(obj, supported_types, &ss));
+      return Status::Invalid(ss.str());
+    }
+  }
+  out_arrays_.push_back(std::make_shared<NullArray>(length_));
+  return Status::OK();
+}
+
+Status PandasConverter::ConvertObjectsInferAndCast() {
+  size_t position = out_arrays_.size();
+  RETURN_NOT_OK(ConvertObjectsInfer());
+
+  std::shared_ptr<Array> arr = out_arrays_[position];
+
+  // Perform cast
+  compute::FunctionContext context(pool_);
+  compute::CastOptions options;
+  options.allow_int_overflow = false;
+
+  std::shared_ptr<Array> casted;
+  RETURN_NOT_OK(compute::Cast(&context, *arr, type_, options, &casted));
+
+  // Replace with casted values
+  out_arrays_[position] = casted;
+
+  return Status::OK();
+}
+
 Status PandasConverter::ConvertObjects() {
   // Python object arrays are annoying, since we could have one of:
   //
@@ -880,13 +950,6 @@ Status PandasConverter::ConvertObjects() {
 
   RETURN_NOT_OK(InitNullBitmap());
 
-  Ndarray1DIndexer<PyObject*> objects;
-
-  PyAcquireGIL lock;
-  objects.Init(arr_);
-  PyDateTime_IMPORT;
-  lock.release();
-
   // This means we received an explicit type from the user
   if (type_) {
     switch (type_->id()) {
@@ -907,53 +970,12 @@ Status PandasConverter::ConvertObjects() {
       case Type::DECIMAL:
         return ConvertDecimals();
       default:
-        return Status::TypeError("No known conversion to Arrow type");
+        return ConvertObjectsInferAndCast();
     }
   } else {
     // Re-acquire GIL
-    lock.acquire();
-
-    OwnedRef decimal;
-    OwnedRef Decimal;
-    RETURN_NOT_OK(ImportModule("decimal", &decimal));
-    RETURN_NOT_OK(ImportFromModule(decimal, "Decimal", &Decimal));
-
-    for (int64_t i = 0; i < length_; ++i) {
-      PyObject* obj = objects[i];
-      if (PandasObjectIsNull(obj)) {
-        continue;
-      } else if (PyObject_is_string(obj)) {
-        return ConvertObjectStrings();
-      } else if (PyObject_is_float(obj)) {
-        return ConvertObjectFloats();
-      } else if (PyBool_Check(obj)) {
-        return ConvertBooleans();
-      } else if (PyObject_is_integer(obj)) {
-        return ConvertObjectIntegers();
-      } else if (PyDate_CheckExact(obj)) {
-        // We could choose Date32 or Date64
-        return ConvertDates<Date32Type>();
-      } else if (PyTime_Check(obj)) {
-        return ConvertTimes();
-      } else if (PyObject_IsInstance(const_cast<PyObject*>(obj), Decimal.obj())) {
-        return ConvertDecimals();
-      } else if (PyList_Check(obj) || PyArray_Check(obj)) {
-        std::shared_ptr<DataType> inferred_type;
-        RETURN_NOT_OK(InferArrowType(obj, &inferred_type));
-        return ConvertLists(inferred_type);
-      } else {
-        const std::string supported_types =
-            "string, bool, float, int, date, time, decimal, list, array";
-        std::stringstream ss;
-        ss << "Error inferring Arrow type for Python object array. ";
-        RETURN_NOT_OK(InvalidConversion(obj, supported_types, &ss));
-        return Status::Invalid(ss.str());
-      }
-    }
+    return ConvertObjectsInfer();
   }
-
-  out_arrays_.push_back(std::make_shared<NullArray>(length_));
-  return Status::OK();
 }
 
 template <typename T>

python/pyarrow/tests/test_convert_pandas.py

@@ -288,6 +288,17 @@ class TestPandasConversion(unittest.TestCase):
         schema = pa.schema([field])
         self._check_pandas_roundtrip(df, expected_schema=schema)
 
+    def test_all_nulls_to_numeric(self):
+        arr = np.array([None], dtype=object)
+
+        def _check_type(t):
+            a2 = pa.Array.from_pandas(arr, type=t)
+            assert a2.type == t
+            assert a2[0].as_py() is None
+
+        _check_type(pa.int32())
+        _check_type(pa.float64())
+
     def test_unicode(self):
         repeats = 1000
         values = [u'foo', None, u'bar', u'mañana', np.nan]