Add datetime.date parsing for numpy conversion

Change-Id: Ice6cc2a8bf082edf20ab7950fae4bd10114f002e

python/pyarrow/tests/test_convert_pandas.py

@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import datetime
 import unittest
 
 import numpy as np
@@ -204,6 +205,20 @@ class TestPandasConversion(unittest.TestCase):
             })
         self._check_pandas_roundtrip(df, timestamps_to_ms=False)
 
+    def test_date(self):
+        df = pd.DataFrame({
+            'date': [
+                datetime.date(2000, 1, 1),
+                None,
+                datetime.date(1970, 1, 1),
+                datetime.date(2040, 2, 26)
+        ]})
+        table = A.from_pandas_dataframe(df)
+        result = table.to_pandas()
+        expected = df.copy()
+        expected['date'] = pd.to_datetime(df['date'])
+        tm.assert_frame_equal(result, expected)
+
     # def test_category(self):
     #     repeats = 1000
     #     values = [b'foo', None, u'bar', 'qux', np.nan]

python/src/pyarrow/adapters/builtin.cc

@@ -25,6 +25,7 @@
 #include "arrow/status.h"
 
 #include "pyarrow/helpers.h"
+#include "pyarrow/util/datetime.h"
 
 using arrow::ArrayBuilder;
 using arrow::DataType;
@@ -313,7 +314,6 @@ class Int64Converter : public TypedConverter<arrow::Int64Builder> {
 class DateConverter : public TypedConverter<arrow::DateBuilder> {
  public:
   Status AppendData(PyObject* seq) override {
-    int64_t val;
     Py_ssize_t size = PySequence_Size(seq);
     RETURN_NOT_OK(typed_builder_->Reserve(size));
     for (int64_t i = 0; i < size; ++i) {
@@ -322,17 +322,7 @@ class DateConverter : public TypedConverter<arrow::DateBuilder> {
         typed_builder_->AppendNull();
       } else {
         PyDateTime_Date* pydate = reinterpret_cast<PyDateTime_Date*>(item.obj());
-        struct tm date = {0};
-        date.tm_year = PyDateTime_GET_YEAR(pydate) - 1900;
-        date.tm_mon = PyDateTime_GET_MONTH(pydate) - 1;
-        date.tm_mday = PyDateTime_GET_DAY(pydate);
-        RETURN_IF_PYERROR();
-        struct tm epoch = {0};
-        epoch.tm_year = 70;
-        epoch.tm_mday = 1;
-        // Milliseconds since the epoch
-        int64_t val = lrint(difftime(mktime(&date), mktime(&epoch)) * 1000);
-        typed_builder_->Append(val);
+        typed_builder_->Append(PyDate_to_ms(pydate));
       }
     }
     return Status::OK();
@@ -342,7 +332,6 @@ class DateConverter : public TypedConverter<arrow::DateBuilder> {
 class TimestampConverter : public TypedConverter<arrow::TimestampBuilder> {
  public:
   Status AppendData(PyObject* seq) override {
-    int64_t val;
     Py_ssize_t size = PySequence_Size(seq);
     RETURN_NOT_OK(typed_builder_->Reserve(size));
     for (int64_t i = 0; i < size; ++i) {

python/src/pyarrow/adapters/pandas.cc

@@ -35,6 +35,7 @@
 
 #include "pyarrow/common.h"
 #include "pyarrow/config.h"
+#include "pyarrow/util/datetime.h"
 
 namespace pyarrow {
 
@@ -167,6 +168,28 @@ class ArrowSerializer {
  private:
   Status ConvertData();
 
+  Status ConvertDates(std::shared_ptr<Array>* out) {
+    PyAcquireGIL lock;
+
+    PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+    arrow::TypePtr string_type(new arrow::DateType());
+    arrow::DateBuilder date_builder(pool_, string_type);
+    RETURN_NOT_OK(date_builder.Resize(length_));
+
+    Status s;
+    PyObject* obj;
+    for (int64_t i = 0; i < length_; ++i) {
+      obj = objects[i];
+      if (PyDate_CheckExact(obj)) {
+        PyDateTime_Date* pydate = reinterpret_cast<PyDateTime_Date*>(obj);
+        date_builder.Append(PyDate_to_ms(pydate));
+      } else {
+        date_builder.AppendNull();
+      }
+    }
+    return date_builder.Finish(out);
+  }
+
   Status ConvertObjectStrings(std::shared_ptr<Array>* out) {
     PyAcquireGIL lock;
 
@@ -369,6 +392,10 @@ inline Status ArrowSerializer<NPY_OBJECT>::Convert(std::shared_ptr<Array>* out)
 
   // TODO: mask not supported here
   const PyObject** objects = reinterpret_cast<const PyObject**>(PyArray_DATA(arr_));
+  {
+    PyAcquireGIL lock;
+    PyDateTime_IMPORT;
+  }
 
   for (int64_t i = 0; i < length_; ++i) {
     if (PyObject_is_null(objects[i])) {
@@ -377,6 +404,8 @@ inline Status ArrowSerializer<NPY_OBJECT>::Convert(std::shared_ptr<Array>* out)
       return ConvertObjectStrings(out);
     } else if (PyBool_Check(objects[i])) {
       return ConvertBooleans(out);
+    } else if (PyDate_CheckExact(objects[i])) {
+      return ConvertDates(out);
     } else {
       return Status::TypeError("unhandled python type");
     }

python/src/pyarrow/util/datetime.h

@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PYARROW_UTIL_DATETIME_H
+#define PYARROW_UTIL_DATETIME_H
+
+#include <Python.h>
+#include <datetime.h>
+
+namespace pyarrow {
+  
+inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) { 
+  struct tm date = {0};
+  date.tm_year = PyDateTime_GET_YEAR(pydate) - 1900;
+  date.tm_mon = PyDateTime_GET_MONTH(pydate) - 1;
+  date.tm_mday = PyDateTime_GET_DAY(pydate);
+  struct tm epoch = {0};
+  epoch.tm_year = 70;
+  epoch.tm_mday = 1;
+  // Milliseconds since the epoch
+  return lrint(difftime(mktime(&date), mktime(&epoch)) * 1000);
+}
+
+} // namespace pyarrow
+
+#endif // PYARROW_UTIL_DATETIME_H