implement python conversion

cpp/src/arrow/python/datetime.h

@@ -113,6 +113,34 @@ inline int64_t PyDateTime_to_ns(PyDateTime_DateTime* pydatetime) {
   return PyDateTime_to_us(pydatetime) * 1000;
 }
 
+ARROW_PYTHON_EXPORT
+inline int64_t PyDelta_to_s(PyDateTime_Delta* pytimedelta) {
+  int64_t total_seconds = 0;
+  total_seconds += PyDateTime_DELTA_GET_SECONDS(pytimedelta);
+  total_seconds += PyDateTime_DELTA_GET_DAYS(pytimedelta) * 24 * 3600;
+  return total_seconds;
+}
+
+ARROW_PYTHON_EXPORT
+inline int64_t PyDelta_to_ms(PyDateTime_Delta* pytimedelta) {
+  int64_t total_ms = PyDelta_to_s(pytimedelta) * 1000;
+  total_ms += PyDateTime_DELTA_GET_MICROSECONDS(pytimedelta) / 1000;
+  return total_ms;
+}
+
+ARROW_PYTHON_EXPORT
+inline int64_t PyDelta_to_us(PyDateTime_Delta* pytimedelta) {
+  int64_t total_us = 0;
+  total_us += PyDelta_to_s(pytimedelta) * 1000 * 1000;
+  total_us += PyDateTime_DELTA_GET_MICROSECONDS(pytimedelta);
+  return total_us;
+}
+
+ARROW_PYTHON_EXPORT
+inline int64_t PyDelta_to_ns(PyDateTime_Delta* pytimedelta) {
+  return PyDelta_to_us(pytimedelta) * 1000;
+}
+
 }  // namespace internal
 }  // namespace py
 }  // namespace arrow

cpp/src/arrow/python/inference.cc

@@ -299,6 +299,7 @@ class TypeInferrer {
         timestamp_milli_count_(0),
         timestamp_micro_count_(0),
         timestamp_nano_count_(0),
+        duration_count_(0),
         float_count_(0),
         binary_count_(0),
         unicode_count_(0),
@@ -332,6 +333,9 @@ class TypeInferrer {
     } else if (PyDateTime_Check(obj)) {
       ++timestamp_micro_count_;
       *keep_going = make_unions_;
+    } else if (PyDelta_Check(obj)) {
+      ++duration_count_;
+      *keep_going = make_unions_;
     } else if (PyDate_Check(obj)) {
       ++date_count_;
       *keep_going = make_unions_;
@@ -460,6 +464,8 @@ class TypeInferrer {
       *out = timestamp(TimeUnit::MILLI);
     } else if (timestamp_second_count_) {
       *out = timestamp(TimeUnit::SECOND);
+    } else if (duration_count_) {
+      *out = duration(TimeUnit::MICRO);
     } else if (bool_count_) {
       *out = boolean();
     } else if (binary_count_) {
@@ -593,6 +599,7 @@ class TypeInferrer {
   int64_t timestamp_milli_count_;
   int64_t timestamp_micro_count_;
   int64_t timestamp_nano_count_;
+  int64_t duration_count_;
   int64_t float_count_;
   int64_t binary_count_;
   int64_t unicode_count_;

cpp/src/arrow/python/python_to_arrow.cc

@@ -449,6 +449,63 @@ class TimestampConverter
   TimeUnit::type unit_;
 };
 
+template <NullCoding null_coding>
+class DurationConverter
+    : public TypedConverter<DurationType, DurationConverter<null_coding>, null_coding> {
+ public:
+  explicit DurationConverter(TimeUnit::type unit) : unit_(unit) {}
+
+  Status AppendItem(PyObject* obj) {
+    int64_t t;
+    if (PyDelta_Check(obj)) {
+      auto pytimedelta = reinterpret_cast<PyDateTime_Delta*>(obj);
+
+      switch (unit_) {
+        case TimeUnit::SECOND:
+          t = internal::PyDelta_to_s(pytimedelta);
+          break;
+        case TimeUnit::MILLI:
+          t = internal::PyDelta_to_ms(pytimedelta);
+          break;
+        case TimeUnit::MICRO:
+          t = internal::PyDelta_to_us(pytimedelta);
+          break;
+        case TimeUnit::NANO:
+          t = internal::PyDelta_to_ns(pytimedelta);
+          break;
+        default:
+          return Status::UnknownError("Invalid time unit");
+      }
+    } else if (PyArray_CheckAnyScalarExact(obj)) {
+      // numpy.datetime64
+      using traits = internal::npy_traits<NPY_TIMEDELTA>;
+
+      std::shared_ptr<DataType> type;
+      RETURN_NOT_OK(NumPyDtypeToArrow(PyArray_DescrFromScalar(obj), &type));
+      if (type->id() != Type::DURATION) {
+        return Status::Invalid("Expected np.timedelta64 but got: ", type->ToString());
+      }
+      const DurationType& ttype = checked_cast<const DurationType&>(*type);
+      if (unit_ != ttype.unit()) {
+        return Status::NotImplemented(
+            "Cannot convert NumPy timedelta64 objects with differing unit");
+      }
+
+      t = reinterpret_cast<PyTimedeltaScalarObject*>(obj)->obval;
+      if (traits::isnull(t)) {
+        // checks numpy NaT sentinel after conversion
+        return this->typed_builder_->AppendNull();
+      }
+    } else {
+      RETURN_NOT_OK(internal::CIntFromPython(obj, &t));
+    }
+    return this->typed_builder_->Append(t);
+  }
+
+ private:
+  TimeUnit::type unit_;
+};
+
 // ----------------------------------------------------------------------
 // Sequence converters for Binary, FixedSizeBinary, String
 
@@ -652,7 +709,7 @@ class ListConverter
 
     const bool null_sentinels_possible =
         // Always treat Numpy's NaT as null
-        NUMPY_TYPE == NPY_DATETIME ||
+        NUMPY_TYPE == NPY_DATETIME || NUMPY_TYPE == NPY_TIMEDELTA ||
         // Observing pandas's null sentinels
         (from_pandas_ && traits::supports_nulls);
 
@@ -700,6 +757,7 @@ class ListConverter
       LIST_SLOW_CASE(TIME32)
       LIST_SLOW_CASE(TIME64)
       LIST_FAST_CASE(TIMESTAMP, NPY_DATETIME, TimestampType)
+      LIST_FAST_CASE(DURATION, NPY_TIMEDELTA, DurationType)
       LIST_FAST_CASE(HALF_FLOAT, NPY_FLOAT16, HalfFloatType)
       LIST_FAST_CASE(FLOAT, NPY_FLOAT, FloatType)
       LIST_FAST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
@@ -951,6 +1009,11 @@ Status GetConverterFlat(const std::shared_ptr<DataType>& type, bool strict_conve
           checked_cast<const TimestampType&>(*type).unit()));
       break;
     }
+    case Type::DURATION: {
+      *out = std::unique_ptr<SeqConverter>(new DurationConverter<null_coding>(
+          checked_cast<const DurationType&>(*type).unit()));
+      break;
+    }
     default:
       return Status::NotImplemented("Sequence converter for type ", type->ToString(),
                                     " not implemented");

python/pyarrow/lib.pyx

@@ -85,6 +85,7 @@ Type_DATE64 = _Type_DATE64
 Type_TIMESTAMP = _Type_TIMESTAMP
 Type_TIME32 = _Type_TIME32
 Type_TIME64 = _Type_TIME64
+Type_DURATION = _Type_DURATION
 Type_BINARY = _Type_BINARY
 Type_STRING = _Type_STRING
 Type_LARGE_BINARY = _Type_LARGE_BINARY

python/pyarrow/scalar.pxi

@@ -464,16 +464,6 @@ cdef class DurationValue(ArrayValue):
         cdef CDurationType* dtype = <CDurationType*> ap.type().get()
 
         cdef int64_t value = ap.Value(self.index)
-
-        # if dtype.unit() == TimeUnit_SECOND:
-        #     return datetime.timedelta(seconds=val)
-        # elif dtype.unit() == TimeUnit_MILLI:
-        #     return datetime.timedelta(milliseconds=val)
-        # elif dtype.unit() == TimeUnit_MICRO:
-        #     return datetime.timedelta(microseconds=val)
-        # elif dtype.unit() == TimeUnit_NANO:
-        #     raise NotImplementedError
-
         converter = _timedelta_conversion_functions()[dtype.unit()]
         return converter(value)
 

python/pyarrow/tests/strategies.py

@@ -82,7 +82,12 @@ timestamp_types = st.builds(
     unit=st.sampled_from(['s', 'ms', 'us', 'ns']),
     tz=tzst.timezones()
 )
-temporal_types = st.one_of(date_types, time_types, timestamp_types)
+duration_types = st.builds(
+    pa.duration,
+    unit=st.sampled_from(['s', 'ms', 'us', 'ns'])
+)
+temporal_types = st.one_of(
+    date_types, time_types, timestamp_types, duration_types)
 
 primitive_types = st.one_of(
     null_type,
@@ -198,6 +203,8 @@ def arrays(draw, type, size=None):
     elif pa.types.is_timestamp(type):
         tz = pytz.timezone(type.tz) if type.tz is not None else None
         value = st.datetimes(timezones=st.just(tz))
+    elif pa.types.is_duration(type):
+        value = st.timedeltas()
     elif pa.types.is_binary(type) or pa.types.is_large_binary(type):
         value = st.binary()
     elif pa.types.is_string(type) or pa.types.is_large_string(type):

python/pyarrow/tests/test_array.py

@@ -300,7 +300,11 @@ def test_struct_array_slice():
 
 
 def test_array_factory_invalid_type():
-    arr = np.array([datetime.timedelta(1), datetime.timedelta(2)])
+
+    class MyObject:
+        pass
+
+    arr = np.array([MyObject()])
     with pytest.raises(ValueError):
         pa.array(arr)
 
@@ -823,7 +827,7 @@ def test_cast_from_null():
         pa.timestamp('us'),
         pa.timestamp('us', tz='UTC'),
         pa.timestamp('us', tz='Europe/Paris'),
-        # pa.duration('us'),
+        # pa.duration('us'),  # TODO implement duration cast
         pa.struct([pa.field('a', pa.int32()),
                    pa.field('b', pa.list_(pa.int8())),
                    pa.field('c', pa.string())]),
@@ -1160,7 +1164,7 @@ def test_pandas_null_sentinels_raise_error():
         ([0, np.nan], pa.time32('s')),
         ([0, np.nan], pa.time64('us')),
         ([0, np.nan], pa.timestamp('us')),
-        # ([0, np.nan], pa.duration('us')),
+        ([0, np.nan], pa.duration('us')),
     ]
     for case, ty in cases:
         # Both types of exceptions are raised. May want to clean that up
@@ -1259,14 +1263,12 @@ def test_array_from_numpy_timedelta(dtype, type):
     arr = pa.array(np_arr)
     assert isinstance(arr, pa.DurationArray)
     assert arr.type == type
-    # TODO implement python conversion
-    # expected = pa.array(data, type=type)
-    # assert arr.equals(expected)
-    np.testing.assert_array_equal(np.array(arr), np_arr)
+    expected = pa.array(data, type=type)
+    assert arr.equals(expected)
 
-    # # from list of numpy scalars
-    # arr = pa.array(list(np.array(data, dtype=dtype)))
-    # assert arr.equals(expected)
+    # from list of numpy scalars
+    arr = pa.array(list(np.array(data, dtype=dtype)))
+    assert arr.equals(expected)
 
 
 def test_array_from_numpy_ascii():

python/pyarrow/tests/test_convert_builtin.py

@@ -827,6 +827,10 @@ class MyDatetime(datetime.datetime):
     pass
 
 
+class MyTimedelta(datetime.timedelta):
+    pass
+
+
 def test_datetime_subclassing():
     data = [
         MyDate(2007, 7, 13),
@@ -863,6 +867,34 @@ def test_datetime_subclassing():
     assert arr_us[0].as_py() == datetime.datetime(2007, 7, 13, 1,
                                                   23, 34, 123456)
 
+    data = [
+        MyTimedelta(1, 1, 1000),
+    ]
+
+    s = pa.duration('s')
+    ms = pa.duration('ms')
+    us = pa.duration('us')
+
+    arr_s = pa.array(data)
+    assert len(arr_s) == 1
+    assert arr_s.type == us
+    assert arr_s[0].as_py() == datetime.timedelta(1, 1, 1000)
+
+    arr_s = pa.array(data, type=s)
+    assert len(arr_s) == 1
+    assert arr_s.type == s
+    assert arr_s[0].as_py() == datetime.timedelta(1, 1)
+
+    arr_ms = pa.array(data, type=ms)
+    assert len(arr_ms) == 1
+    assert arr_ms.type == ms
+    assert arr_ms[0].as_py() == datetime.timedelta(1, 1, 1000)
+
+    arr_us = pa.array(data, type=us)
+    assert len(arr_us) == 1
+    assert arr_us.type == us
+    assert arr_us[0].as_py() == datetime.timedelta(1, 1, 1000)
+
 
 @pytest.mark.xfail(not _pandas_api.have_pandas,
                    reason="pandas required for nanosecond conversion")
@@ -932,6 +964,93 @@ def test_sequence_timestamp_from_int_with_unit():
             pa.array([1, CustomClass()], type=ty)
 
 
+@pytest.mark.parametrize('np_scalar', [True, False])
+def test_sequence_duration(np_scalar):
+    td1 = datetime.timedelta(1, 1, 1)
+    td2 = datetime.timedelta(1, 100)
+    if np_scalar:
+        data = [np.timedelta64(td1), None, np.timedelta64(td2)]
+    else:
+        data = [td1, None, td2]
+
+    arr = pa.array(data)
+    assert len(arr) == 3
+    assert arr.type == pa.duration('us')
+    assert arr.null_count == 1
+    assert arr[0].as_py() == td1
+    assert arr[1].as_py() is None
+    assert arr[2].as_py() == td2
+
+
+@pytest.mark.parametrize('unit', ['s', 'ms', 'us', 'ns'])
+def test_sequence_duration_with_unit(unit):
+    data = [
+        datetime.timedelta(1, 1, 1001),
+    ]
+    expected = {'s': datetime.timedelta(1, 1),
+                'ms': datetime.timedelta(1, 1, 1000),
+                'us': datetime.timedelta(1, 1, 1001),
+                'ns': datetime.timedelta(1, 1, 1001)}
+
+    ty = pa.duration(unit)
+
+    arr_s = pa.array(data, type=ty)
+    assert len(arr_s) == 1
+    assert arr_s.type == ty
+    assert arr_s[0].as_py() == expected[unit]
+
+
+@pytest.mark.parametrize('unit', ['s', 'ms', 'us', 'ns'])
+def test_sequence_duration_from_int_with_unit(unit):
+    data = [1]
+
+    ty = pa.timestamp(unit)
+
+    arr = pa.array(data, type=ty)
+    assert len(arr) == 1
+    assert arr.type == ty
+    assert arr[0].value == 1
+    assert arr[0].as_py()
+
+
+def test_sequence_duration_nested_lists():
+    td1 = datetime.timedelta(1, 1, 1000)
+    td2 = datetime.timedelta(1, 100)
+
+    data = [[td1, None], [td1, td2]]
+
+    arr = pa.array(data)
+    assert len(arr) == 2
+    assert arr.type == pa.list_(pa.duration('us'))
+    assert arr.to_pylist() == data
+
+    arr = pa.array(data, type=pa.list_(pa.duration('ms')))
+    assert len(arr) == 2
+    assert arr.type == pa.list_(pa.duration('ms'))
+    assert arr.to_pylist() == data
+
+
+def test_sequence_duration_nested_lists_numpy():
+    td1 = datetime.timedelta(1, 1, 1000)
+    td2 = datetime.timedelta(1, 100)
+
+    data = [[np.timedelta64(td1), None],
+            [np.timedelta64(td1), np.timedelta64(td2)]]
+
+    arr = pa.array(data)
+    assert len(arr) == 2
+    assert arr.type == pa.list_(pa.duration('us'))
+    assert arr.to_pylist() == [[td1, None], [td1, td2]]
+
+    data = [np.array([np.timedelta64(td1), None], dtype='timedelta64[us]'),
+            np.array([np.timedelta64(td1), np.timedelta64(td2)])]
+
+    arr = pa.array(data)
+    assert len(arr) == 2
+    assert arr.type == pa.list_(pa.duration('us'))
+    assert arr.to_pylist() == [[td1, None], [td1, td2]]
+
+
 def test_sequence_nesting_levels():
     data = [1, 2, None]
     arr = pa.array(data)

python/pyarrow/tests/test_scalars.py

@@ -316,28 +316,26 @@ class TestScalars(unittest.TestCase):
     @pytest.mark.pandas
     def test_duration_nanos_pandas(self):
         import pandas as pd
-        arr = np.array([0, 3600000000000], dtype='timedelta64[ns]')
-        arrow_arr = pa.array(arr)
+        arr = pa.array([0, 3600000000000], type=pa.duration('ns'))
         expected = pd.Timedelta('1 hour')
-        assert isinstance(arrow_arr[1].as_py(), pd.Timedelta)
-        assert arrow_arr[1].as_py() == expected
-        assert arrow_arr[1].value == expected.value
+        assert isinstance(arr[1].as_py(), pd.Timedelta)
+        assert arr[1].as_py() == expected
+        assert arr[1].value == expected.value
 
         # Non-zero nanos work fine
-        arr = pa.array(np.array([946684800000000001], dtype='timedelta64[ns]'))
+        arr = pa.array([946684800000000001], type=pa.duration('ns'))
         arr[0].as_py() == pd.Timedelta(946684800000000001, unit='ns')
 
     @pytest.mark.nopandas
     def test_duration_nanos_nopandas(self):
-        arr = np.array([0, 3600000000000], dtype='timedelta64[ns]')
-        arrow_arr = pa.array(arr)
+        arr = pa.array([0, 3600000000000], pa.duration('ns'))
         expected = datetime.timedelta(seconds=60*60)
-        assert isinstance(arrow_arr[1].as_py(), datetime.timedelta)
-        assert arrow_arr[1].as_py() == expected
-        assert arrow_arr[1].value == expected.total_seconds() * 1e9
+        assert isinstance(arr[1].as_py(), datetime.timedelta)
+        assert arr[1].as_py() == expected
+        assert arr[1].value == expected.total_seconds() * 1e9
 
         # Non-zero nanos yields ValueError
-        arr = pa.array(np.array([946684800000000001], dtype='timedelta64[ns]'))
+        arr = pa.array([946684800000000001], type=pa.duration('ns'))
         with pytest.raises(ValueError):
             arr[0].as_py()
 

python/pyarrow/tests/test_schema.py

@@ -120,6 +120,10 @@ def test_type_for_alias():
         ('timestamp[ms]', pa.timestamp('ms')),
         ('timestamp[us]', pa.timestamp('us')),
         ('timestamp[ns]', pa.timestamp('ns')),
+        ('duration[s]', pa.duration('s')),
+        ('duration[ms]', pa.duration('ms')),
+        ('duration[us]', pa.duration('us')),
+        ('duration[ns]', pa.duration('ns')),
     ]
 
     for val, expected in cases:
@@ -177,7 +181,11 @@ def test_from_numpy_dtype():
         (np.dtype('datetime64[s]'), pa.timestamp('s')),
         (np.dtype('datetime64[ms]'), pa.timestamp('ms')),
         (np.dtype('datetime64[us]'), pa.timestamp('us')),
-        (np.dtype('datetime64[ns]'), pa.timestamp('ns'))
+        (np.dtype('datetime64[ns]'), pa.timestamp('ns')),
+        (np.dtype('timedelta64[s]'), pa.duration('s')),
+        (np.dtype('timedelta64[ms]'), pa.duration('ms')),
+        (np.dtype('timedelta64[us]'), pa.duration('us')),
+        (np.dtype('timedelta64[ns]'), pa.duration('ns')),
     ]
 
     for dt, pt in cases:

python/pyarrow/tests/test_types.py

@@ -184,24 +184,34 @@ def test_is_temporal_date_time_timestamp():
     date_types = [pa.date32(), pa.date64()]
     time_types = [pa.time32('s'), pa.time64('ns')]
     timestamp_types = [pa.timestamp('ms')]
+    duration_types = [pa.duration('ms')]
 
-    for case in date_types + time_types + timestamp_types:
+    for case in date_types + time_types + timestamp_types + duration_types:
         assert types.is_temporal(case)
 
     for case in date_types:
         assert types.is_date(case)
         assert not types.is_time(case)
         assert not types.is_timestamp(case)
+        assert not types.is_duration(case)
 
     for case in time_types:
         assert types.is_time(case)
         assert not types.is_date(case)
         assert not types.is_timestamp(case)
+        assert not types.is_duration(case)
 
     for case in timestamp_types:
         assert types.is_timestamp(case)
         assert not types.is_date(case)
         assert not types.is_time(case)
+        assert not types.is_duration(case)
+
+    for case in duration_types:
+        assert types.is_duration(case)
+        assert not types.is_date(case)
+        assert not types.is_time(case)
+        assert not types.is_timestamp(case)
 
     assert not types.is_temporal(pa.int32())
 

python/pyarrow/types.py

@@ -34,7 +34,8 @@ _INTEGER_TYPES = _SIGNED_INTEGER_TYPES | _UNSIGNED_INTEGER_TYPES
 _FLOATING_TYPES = {lib.Type_HALF_FLOAT, lib.Type_FLOAT, lib.Type_DOUBLE}
 _DATE_TYPES = {lib.Type_DATE32, lib.Type_DATE64}
 _TIME_TYPES = {lib.Type_TIME32, lib.Type_TIME64}
-_TEMPORAL_TYPES = {lib.Type_TIMESTAMP} | _TIME_TYPES | _DATE_TYPES
+_TEMPORAL_TYPES = {lib.Type_TIMESTAMP,
+                   lib.Type_DURATION} | _TIME_TYPES | _DATE_TYPES
 _NESTED_TYPES = {lib.Type_LIST, lib.Type_LARGE_LIST, lib.Type_STRUCT,
                  lib.Type_UNION, lib.Type_MAP}
 
@@ -208,6 +209,13 @@ def is_timestamp(t):
     return t.id == lib.Type_TIMESTAMP
 
 
+def is_duration(t):
+    """
+    Return True if value is an instance of a duration type
+    """
+    return t.id == lib.Type_DURATION
+
+
 def is_time(t):
     """
     Return True if value is an instance of a time type