mirrors/arrow
0
0 Fork 0
mirror of https://github.com/apache/arrow.git synced 2026-10-28 16:47:19 +08:00
Code Activity

Set date_as_objects=True as default in to_pandas methods

Browse Source
This commit is contained in:
Wes McKinney committed 2018-12-31 17:49:48 -06:00
1 parent 71ccba9b21
commit f77c29674c
2 files changed
+35 -65
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

python/pyarrow/array.pxi
+2 -4
View File
@@ -343,10 +343,8 @@ cdef class _PandasConvertible:
def to_pandas(self, categories=None, bint strings_to_categorical=False,
bint zero_copy_only=False, bint integer_object_nulls=False,
bint date_as_object=False,
bint use_threads=True,
bint deduplicate_objects=True,
bint ignore_metadata=False):
bint date_as_object=True, bint use_threads=True,
bint deduplicate_objects=True, bint ignore_metadata=False):
"""
Convert to a pandas-compatible NumPy array or DataFrame, as appropriate
python/pyarrow/tests/test_convert_pandas.py
+33 -61
View File
@@ -912,7 +912,7 @@ class TestConvertDateTimeLikeTypes(object):
result = table.to_pandas()
expected_df = pd.DataFrame(
{"date": np.array(["2000-01-01"], dtype="datetime64[ns]")}
{"date": np.array([date(2000, 1, 1)], dtype=object)}
)
tm.assert_frame_equal(expected_df, result)
@@ -962,7 +962,7 @@ class TestConvertDateTimeLikeTypes(object):
with pytest.raises(pa.ArrowInvalid, match=expected_msg):
pa.Array.from_pandas(s, type=pa.date64(), mask=mask)
def test_array_date_as_object(self):
def test_array_types_date_as_object(self):
data = [date(2000, 1, 1),
None,
date(1970, 1, 1),
@@ -972,58 +972,23 @@ class TestConvertDateTimeLikeTypes(object):
'1970-01-01',
'2040-02-26'], dtype='datetime64')
arr = pa.array(data)
assert arr.equals(pa.array(expected))
objects = [
# The second value is the expected value for date_as_object=False
(pa.array(data), expected),
(pa.chunked_array([data]), expected),
(pa.column('date', [data]), expected.astype('M8[ns]'))]
result = arr.to_pandas()
assert result.dtype == expected.dtype
npt.assert_array_equal(arr.to_pandas(), expected)
assert objects[0][0].equals(pa.array(expected))
result = arr.to_pandas(date_as_object=True)
expected = expected.astype(object)
assert result.dtype == expected.dtype
npt.assert_array_equal(result, expected)
for obj, expected_datetime64 in objects:
result = obj.to_pandas()
expected_obj = expected.astype(object)
assert result.dtype == expected_obj.dtype
npt.assert_array_equal(result, expected_obj)
def test_chunked_array_convert_date_as_object(self):
data = [date(2000, 1, 1),
None,
date(1970, 1, 1),
date(2040, 2, 26)]
expected = np.array(['2000-01-01',
None,
'1970-01-01',
'2040-02-26'], dtype='datetime64')
carr = pa.chunked_array([data])
result = carr.to_pandas()
assert result.dtype == expected.dtype
npt.assert_array_equal(carr.to_pandas(), expected)
result = carr.to_pandas(date_as_object=True)
expected = expected.astype(object)
assert result.dtype == expected.dtype
npt.assert_array_equal(result, expected)
def test_column_convert_date_as_object(self):
data = [date(2000, 1, 1),
None,
date(1970, 1, 1),
date(2040, 2, 26)]
expected = np.array(['2000-01-01',
None,
'1970-01-01',
'2040-02-26'], dtype='datetime64')
arr = pa.array(data)
column = pa.column('date', arr)
result = column.to_pandas()
npt.assert_array_equal(column.to_pandas(), expected)
result = column.to_pandas(date_as_object=True)
expected = expected.astype(object)
assert result.dtype == expected.dtype
npt.assert_array_equal(result, expected)
result = obj.to_pandas(date_as_object=False)
assert result.dtype == expected_datetime64.dtype
npt.assert_array_equal(result, expected_datetime64)
def test_table_convert_date_as_object(self):
df = pd.DataFrame({
@@ -1034,8 +999,8 @@ class TestConvertDateTimeLikeTypes(object):
table = pa.Table.from_pandas(df, preserve_index=False)
df_datetime = table.to_pandas()
df_object = table.to_pandas(date_as_object=True)
df_datetime = table.to_pandas(date_as_object=False)
df_object = table.to_pandas()
tm.assert_frame_equal(df.astype('datetime64[ns]'), df_datetime,
check_dtype=True)
@@ -1055,9 +1020,7 @@ class TestConvertDateTimeLikeTypes(object):
assert table.schema.equals(expected_schema)
result = table.to_pandas()
expected = df.copy()
expected['date'] = pd.to_datetime(df['date'])
tm.assert_frame_equal(result, expected)
tm.assert_frame_equal(result, df)
def test_date_mask(self):
arr = np.array([date(2017, 4, 3), date(2017, 4, 4)],
@@ -1094,18 +1057,27 @@ class TestConvertDateTimeLikeTypes(object):
# Test converting back to pandas
colnames = ['date32', 'date64']
table = pa.Table.from_arrays([a32, a64], colnames)
table_pandas = table.to_pandas()
ex_values = (np.array(['2017-04-03', '2017-04-04', '2017-04-04',
'2017-04-05'],
dtype='datetime64[D]')
.astype('datetime64[ns]'))
dtype='datetime64[D]'))
ex_values[1] = pd.NaT.value
expected_pandas = pd.DataFrame({'date32': ex_values,
'date64': ex_values},
ex_datetime64ns = ex_values.astype('datetime64[ns]')
expected_pandas = pd.DataFrame({'date32': ex_datetime64ns,
'date64': ex_datetime64ns},
columns=colnames)
table_pandas = table.to_pandas(date_as_object=False)
tm.assert_frame_equal(table_pandas, expected_pandas)
table_pandas_objects = table.to_pandas()
ex_objects = ex_values.astype('object')
expected_pandas_objects = pd.DataFrame({'date32': ex_objects,
'date64': ex_objects},
columns=colnames)
tm.assert_frame_equal(table_pandas_objects,
expected_pandas_objects)
def test_dates_from_integers(self):
t1 = pa.date32()
t2 = pa.date64()