Do not require pandas to run unit tests

python/pyarrow/pandas_compat.py

@@ -33,6 +33,8 @@ from pyarrow.compat import (builtin_pickle, PY2, zip_longest)  # noqa
 def _imported_property(f):
     @property
     def wrapper(self):
+        if self._have_pandas:
+            return f(self)
         self._check_import()
         return f(self)
     return wrapper
@@ -53,7 +55,7 @@ class _PandasAPI(object):
 
     def __init__(self):
         self._imported_pandas = False
-        self._have_pandas = False
+        self._have_pandas = None
         self._pd = None
         self._loose_version = None
         self._version = None
@@ -75,7 +77,8 @@ class _PandasAPI(object):
 
     @property
     def have_pandas(self):
-        self._check_import(raise_=False)
+        if self._have_pandas is None:
+            self._check_import(raise_=False)
         return self._have_pandas
 
     @_imported_property
@@ -119,6 +122,8 @@ class _PandasAPI(object):
         except ImportError:
             if raise_:
                 raise
+            else:
+                return
 
         self._pd = pd
         self._compat_module = pdcompat
@@ -146,16 +151,28 @@ class _PandasAPI(object):
                                 self._categorical_type))
 
     def is_categorical(self, obj):
-        self._check_import()
-        return isinstance(obj, self._categorical_type)
+        if self.have_pandas:
+            return isinstance(obj, self._categorical_type)
+        else:
+            return False
 
     def is_datetimetz(self, obj):
-        self._check_import()
-        return isinstance(obj, self._datetimetz_type)
+        if self.have_pandas:
+            return isinstance(obj, self._datetimetz_type)
+        else:
+            return False
+
+    def is_data_frame(self, obj):
+        if self.have_pandas:
+            return isinstance(obj, self._data_frame)
+        else:
+            return False
 
     def is_series(self, obj):
-        self._check_import()
-        return isinstance(obj, self._series)
+        if self.have_pandas:
+            return isinstance(obj, self._series)
+        else:
+            return False
 
     def dataframe_to_arrays(self, *args, **kwargs):
         return _dataframe_to_arrays(*args, **kwargs)
@@ -169,6 +186,10 @@ class _PandasAPI(object):
     def make_datetimetz(self, tz):
         return _make_datetimetz(tz)
 
+    def assert_frame_equal(self, *args, **kwargs):
+        self._check_import()
+        return self._pd.util.testing.assert_frame_equal
+
 
 _pandas_api = _PandasAPI.get_instance()
 

python/pyarrow/tests/conftest.py

@@ -43,6 +43,7 @@ groups = [
     'hdfs',
     'large_memory',
     'orc',
+    'pandas',
     'parquet',
     'plasma',
     's3',
@@ -56,6 +57,7 @@ defaults = {
     'hdfs': False,
     'large_memory': False,
     'orc': False,
+    'pandas': False,
     'parquet': False,
     'plasma': False,
     's3': False,
@@ -74,6 +76,14 @@ try:
 except ImportError:
     pass
 
+
+try:
+    import pandas  # noqa
+    defaults['pandas'] = True
+except ImportError:
+    pass
+
+
 try:
     import pyarrow.parquet  # noqa
     defaults['parquet'] = True

python/pyarrow/tests/test_array.py

@@ -27,8 +27,6 @@ import struct
 import sys
 
 import numpy as np
-import pandas as pd
-import pandas.util.testing as tm
 try:
     import pickle5
 except ImportError:
@@ -36,7 +34,6 @@ except ImportError:
 
 import pyarrow as pa
 import pyarrow.tests.strategies as past
-from pyarrow.pandas_compat import get_logical_type
 
 
 def test_total_bytes_allocated():
@@ -362,26 +359,6 @@ def test_dictionary_from_arrays_boundscheck():
     pa.DictionaryArray.from_arrays(indices2, dictionary, safe=False)
 
 
-def test_dictionary_with_pandas():
-    indices = np.repeat([0, 1, 2], 2)
-    dictionary = np.array(['foo', 'bar', 'baz'], dtype=object)
-    mask = np.array([False, False, True, False, False, False])
-
-    d1 = pa.DictionaryArray.from_arrays(indices, dictionary)
-    d2 = pa.DictionaryArray.from_arrays(indices, dictionary, mask=mask)
-
-    pandas1 = d1.to_pandas()
-    ex_pandas1 = pd.Categorical.from_codes(indices, categories=dictionary)
-
-    tm.assert_series_equal(pd.Series(pandas1), pd.Series(ex_pandas1))
-
-    pandas2 = d2.to_pandas()
-    ex_pandas2 = pd.Categorical.from_codes(np.where(mask, -1, indices),
-                                           categories=dictionary)
-
-    tm.assert_series_equal(pd.Series(pandas2), pd.Series(ex_pandas2))
-
-
 def test_list_from_arrays():
     offsets_arr = np.array([0, 2, 5, 8], dtype='i4')
     offsets = pa.array(offsets_arr, type='int32')
@@ -605,52 +582,6 @@ def test_safe_cast_nan_to_int_raises():
         arr.cast(pa.int64(), safe=True)
 
 
-def test_cast_timestamp_unit():
-    # ARROW-1680
-    val = datetime.datetime.now()
-    s = pd.Series([val])
-    s_nyc = s.dt.tz_localize('tzlocal()').dt.tz_convert('America/New_York')
-
-    us_with_tz = pa.timestamp('us', tz='America/New_York')
-
-    arr = pa.Array.from_pandas(s_nyc, type=us_with_tz)
-
-    # ARROW-1906
-    assert arr.type == us_with_tz
-
-    arr2 = pa.Array.from_pandas(s, type=pa.timestamp('us'))
-
-    assert arr[0].as_py() == s_nyc[0]
-    assert arr2[0].as_py() == s[0]
-
-    # Disallow truncation
-    arr = pa.array([123123], type='int64').cast(pa.timestamp('ms'))
-    expected = pa.array([123], type='int64').cast(pa.timestamp('s'))
-
-    target = pa.timestamp('s')
-    with pytest.raises(ValueError):
-        arr.cast(target)
-
-    result = arr.cast(target, safe=False)
-    assert result.equals(expected)
-
-    # ARROW-1949
-    series = pd.Series([pd.Timestamp(1), pd.Timestamp(10), pd.Timestamp(1000)])
-    expected = pa.array([0, 0, 1], type=pa.timestamp('us'))
-
-    with pytest.raises(ValueError):
-        pa.array(series, type=pa.timestamp('us'))
-
-    with pytest.raises(ValueError):
-        pa.Array.from_pandas(series, type=pa.timestamp('us'))
-
-    result = pa.Array.from_pandas(series, type=pa.timestamp('us'), safe=False)
-    assert result.equals(expected)
-
-    result = pa.array(series, type=pa.timestamp('us'), safe=False)
-    assert result.equals(expected)
-
-
 def test_cast_signed_to_unsigned():
     safe_cases = [
         (np.array([0, 1, 2, 3], dtype='i1'), pa.uint8(),
@@ -877,39 +808,6 @@ def test_to_numpy_roundtrip(narr):
     np.testing.assert_array_equal(narr[2:6], arr[2:6].to_numpy())
 
 
-@pytest.mark.parametrize(
-    ('type', 'expected'),
-    [
-        (pa.null(), 'empty'),
-        (pa.bool_(), 'bool'),
-        (pa.int8(), 'int8'),
-        (pa.int16(), 'int16'),
-        (pa.int32(), 'int32'),
-        (pa.int64(), 'int64'),
-        (pa.uint8(), 'uint8'),
-        (pa.uint16(), 'uint16'),
-        (pa.uint32(), 'uint32'),
-        (pa.uint64(), 'uint64'),
-        (pa.float16(), 'float16'),
-        (pa.float32(), 'float32'),
-        (pa.float64(), 'float64'),
-        (pa.date32(), 'date'),
-        (pa.date64(), 'date'),
-        (pa.binary(), 'bytes'),
-        (pa.binary(length=4), 'bytes'),
-        (pa.string(), 'unicode'),
-        (pa.list_(pa.list_(pa.int16())), 'list[list[int16]]'),
-        (pa.decimal128(18, 3), 'decimal'),
-        (pa.timestamp('ms'), 'datetime'),
-        (pa.timestamp('us', 'UTC'), 'datetimetz'),
-        (pa.time32('s'), 'time'),
-        (pa.time64('us'), 'time')
-    ]
-)
-def test_logical_type(type, expected):
-    assert get_logical_type(type) == expected
-
-
 def test_array_uint64_from_py_over_range():
     arr = pa.array([2 ** 63], type=pa.uint64())
     expected = pa.array(np.array([2 ** 63], dtype='u8'))
@@ -999,23 +897,6 @@ def test_array_from_timestamp_with_generic_unit():
         pa.array([n, x, y])
 
 
-def test_array_from_py_float32():
-    data = [[1.2, 3.4], [9.0, 42.0]]
-
-    t = pa.float32()
-
-    arr1 = pa.array(data[0], type=t)
-    arr2 = pa.array(data, type=pa.list_(t))
-
-    expected1 = np.array(data[0], dtype=np.float32)
-    expected2 = pd.Series([np.array(data[0], dtype=np.float32),
-                           np.array(data[1], dtype=np.float32)])
-
-    assert arr1.type == t
-    assert arr1.equals(pa.array(expected1))
-    assert arr2.equals(pa.array(expected2))
-
-
 def test_array_from_numpy_ascii():
     arr = np.array(['abcde', 'abc', ''], dtype='|S5')
 

python/pyarrow/tests/test_convert_builtin.py

@@ -19,6 +19,7 @@
 import pytest
 
 from pyarrow.compat import unittest, u  # noqa
+from pyarrow.pandas_compat import _pandas_api  # noqa
 import pyarrow as pa
 
 import collections
@@ -643,7 +644,6 @@ def test_sequence_timestamp_with_unit():
     s = pa.timestamp('s')
     ms = pa.timestamp('ms')
     us = pa.timestamp('us')
-    ns = pa.timestamp('ns')
 
     arr_s = pa.array(data, type=s)
     assert len(arr_s) == 1
@@ -663,16 +663,16 @@ def test_sequence_timestamp_with_unit():
     assert arr_us[0].as_py() == datetime.datetime(2007, 7, 13, 1,
                                                   23, 34, 123456)
 
-    arr_ns = pa.array(data, type=ns)
-    assert len(arr_ns) == 1
-    assert arr_ns.type == ns
-    assert arr_ns[0].as_py() == datetime.datetime(2007, 7, 13, 1,
-                                                  23, 34, 123456)
+
+class MyDate(datetime.date):
+    pass
+
+
+class MyDatetime(datetime.datetime):
+    pass
 
 
 def test_datetime_subclassing():
-    class MyDate(datetime.date):
-        pass
     data = [
         MyDate(2007, 7, 13),
     ]
@@ -682,9 +682,6 @@ def test_datetime_subclassing():
     assert arr_date.type == date_type
     assert arr_date[0].as_py() == datetime.date(2007, 7, 13)
 
-    class MyDatetime(datetime.datetime):
-        pass
-
     data = [
         MyDatetime(2007, 7, 13, 1, 23, 34, 123456),
     ]
@@ -692,7 +689,6 @@ def test_datetime_subclassing():
     s = pa.timestamp('s')
     ms = pa.timestamp('ms')
     us = pa.timestamp('us')
-    ns = pa.timestamp('ns')
 
     arr_s = pa.array(data, type=s)
     assert len(arr_s) == 1
@@ -712,14 +708,29 @@ def test_datetime_subclassing():
     assert arr_us[0].as_py() == datetime.datetime(2007, 7, 13, 1,
                                                   23, 34, 123456)
 
-    arr_ns = pa.array(data, type=ns)
-    assert len(arr_ns) == 1
-    assert arr_ns.type == ns
-    assert arr_ns[0].as_py() == datetime.datetime(2007, 7, 13, 1,
-                                                  23, 34, 123456)
+
+@pytest.mark.xfail(not _pandas_api.have_pandas,
+                   reason="pandas required for nanosecond conversion")
+def test_sequence_timestamp_nanoseconds():
+    inputs = [
+        [datetime.datetime(2007, 7, 13, 1, 23, 34, 123456)],
+        [MyDatetime(2007, 7, 13, 1, 23, 34, 123456)]
+    ]
+
+    for data in inputs:
+        ns = pa.timestamp('ns')
+        arr_ns = pa.array(data, type=ns)
+        assert len(arr_ns) == 1
+        assert arr_ns.type == ns
+        assert arr_ns[0].as_py() == datetime.datetime(2007, 7, 13, 1,
+                                                      23, 34, 123456)
 
 
+@pytest.mark.pandas
 def test_sequence_timestamp_from_int_with_unit():
+    # TODO(wesm): This test might be rewritten to assert the actual behavior
+    # when pandas is not installed
+
     data = [1]
 
     s = pa.timestamp('s')

python/pyarrow/tests/test_feather.py

@@ -25,15 +25,25 @@ import pytest
 from numpy.testing import assert_array_equal
 import numpy as np
 
-from pandas.util.testing import assert_frame_equal
-import pandas as pd
-
 import pyarrow as pa
 from pyarrow.feather import (read_feather, write_feather,
                              read_table, FeatherReader, FeatherDataset)
 from pyarrow.lib import FeatherWriter
 
 
+try:
+    from pandas.util.testing import assert_frame_equal
+    import pandas as pd
+except ImportError:
+    pass
+
+
+# TODO(wesm): The Feather tests currently are tangled with pandas
+# dependency. We should isolate the pandas-depending parts and mark those with
+# pytest.mark.pandas
+pytestmark = pytest.mark.pandas
+
+
 def random_path(prefix='feather_'):
     return tempfile.mktemp(prefix=prefix)
 

python/pyarrow/tests/test_gandiva.py

@@ -19,7 +19,6 @@ import datetime
 import pytest
 
 import pyarrow as pa
-import pandas as pd
 
 
 @pytest.mark.gandiva
@@ -60,8 +59,8 @@ def test_tree_exp_builder():
 def test_table():
     import pyarrow.gandiva as gandiva
 
-    df = pd.DataFrame({"a": [1.0, 2.0], "b": [3.0, 4.0]})
-    table = pa.Table.from_pandas(df)
+    table = pa.Table.from_arrays([pa.array([1.0, 2.0]), pa.array([3.0, 4.0])],
+                                 ['a', 'b'])
 
     builder = gandiva.TreeExprBuilder()
     node_a = builder.make_field(table.schema.field_by_name("a"))
@@ -79,7 +78,7 @@ def test_table():
     # RecordBatches
     r, = projector.evaluate(table.to_batches()[0])
 
-    e = pa.Array.from_pandas(df["a"] + df["b"])
+    e = pa.array([4.0, 6.0])
     assert r.equals(e)
 
 
@@ -87,8 +86,8 @@ def test_table():
 def test_filter():
     import pyarrow.gandiva as gandiva
 
-    df = pd.DataFrame({"a": [1.0 * i for i in range(10000)]})
-    table = pa.Table.from_pandas(df)
+    table = pa.Table.from_arrays([pa.array([1.0 * i for i in range(10000)])],
+                                 ['a'])
 
     builder = gandiva.TreeExprBuilder()
     node_a = builder.make_field(table.schema.field_by_name("a"))
@@ -216,9 +215,10 @@ def test_in_expr_todo():
 def test_boolean():
     import pyarrow.gandiva as gandiva
 
-    df = pd.DataFrame({"a": [1., 31., 46., 3., 57., 44., 22.],
-                       "b": [5., 45., 36., 73., 83., 23., 76.]})
-    table = pa.Table.from_pandas(df)
+    table = pa.Table.from_arrays([pa.array([1., 31., 46., 3., 57., 44., 22.]),
+                                  pa.array([5., 45., 36., 73.,
+                                            83., 23., 76.])],
+                                 ['a', 'b'])
 
     builder = gandiva.TreeExprBuilder()
     node_a = builder.make_field(table.schema.field_by_name("a"))

python/pyarrow/tests/test_hdfs.py

@@ -20,7 +20,6 @@ import pickle
 import pytest
 import random
 import unittest
-import pandas.util.testing as pdt
 
 from io import BytesIO
 from os.path import join as pjoin
@@ -30,6 +29,7 @@ import pyarrow as pa
 import pyarrow.tests.test_parquet as test_parquet
 
 from pyarrow.compat import guid
+from pyarrow.pandas_compat import _pandas_api
 
 
 # ----------------------------------------------------------------------
@@ -297,6 +297,7 @@ class HdfsTestCases(object):
         expected = pa.concat_tables(test_data)
         return expected
 
+    @pytest.mark.pandas
     @pytest.mark.parquet
     def test_read_multiple_parquet_files(self):
 
@@ -307,10 +308,12 @@ class HdfsTestCases(object):
         expected = self._write_multiple_hdfs_pq_files(tmpdir)
         result = self.hdfs.read_parquet(tmpdir)
 
-        pdt.assert_frame_equal(result.to_pandas()
-                               .sort_values(by='index').reset_index(drop=True),
-                               expected.to_pandas())
+        _pandas_api.assert_frame_equal(result.to_pandas()
+                                       .sort_values(by='index')
+                                       .reset_index(drop=True),
+                                       expected.to_pandas())
 
+    @pytest.mark.pandas
     @pytest.mark.parquet
     def test_read_multiple_parquet_files_with_uri(self):
         import pyarrow.parquet as pq
@@ -323,10 +326,12 @@ class HdfsTestCases(object):
         path = _get_hdfs_uri(tmpdir)
         result = pq.read_table(path)
 
-        pdt.assert_frame_equal(result.to_pandas()
-                               .sort_values(by='index').reset_index(drop=True),
-                               expected.to_pandas())
+        _pandas_api.assert_frame_equal(result.to_pandas()
+                                       .sort_values(by='index')
+                                       .reset_index(drop=True),
+                                       expected.to_pandas())
 
+    @pytest.mark.pandas
     @pytest.mark.parquet
     def test_read_write_parquet_files_with_uri(self):
         import pyarrow.parquet as pq
@@ -345,7 +350,7 @@ class HdfsTestCases(object):
 
         result = pq.read_table(path, filesystem=self.hdfs).to_pandas()
 
-        pdt.assert_frame_equal(result, df)
+        _pandas_api.assert_frame_equal(result, df)
 
     @pytest.mark.parquet
     def test_read_common_metadata_files(self):
@@ -409,10 +414,13 @@ def _get_hdfs_uri(path):
     return uri
 
 
+@pytest.mark.pandas
 @pytest.mark.parquet
 @pytest.mark.fastparquet
 @pytest.mark.parametrize('client', ['libhdfs', 'libhdfs3'])
 def test_fastparquet_read_with_hdfs(client):
+    from pandas.util.testing import assert_frame_equal, makeDataFrame
+
     try:
         import snappy  # noqa
     except ImportError:
@@ -423,7 +431,7 @@ def test_fastparquet_read_with_hdfs(client):
 
     fs = hdfs_test_client(client)
 
-    df = pdt.makeDataFrame()
+    df = makeDataFrame()
     table = pa.Table.from_pandas(df)
 
     path = '/tmp/testing.parquet'
@@ -433,4 +441,4 @@ def test_fastparquet_read_with_hdfs(client):
     parquet_file = fastparquet.ParquetFile(path, open_with=fs.open)
 
     result = parquet_file.to_pandas()
-    pdt.assert_frame_equal(result, df)
+    assert_frame_equal(result, df)

python/pyarrow/tests/test_io.py

@@ -33,8 +33,6 @@ except ImportError:
 
 import numpy as np
 
-import pandas as pd
-
 from pyarrow.compat import u, guid, PY2
 import pyarrow as pa
 
@@ -628,9 +626,8 @@ def test_mock_output_stream():
 
     assert f1.size() == len(f2.getvalue())
 
-    # Do the same test with a pandas DataFrame
-    val = pd.DataFrame({'a': [1, 2, 3]})
-    record_batch = pa.RecordBatch.from_pandas(val)
+    # Do the same test with a table
+    record_batch = pa.RecordBatch.from_arrays([pa.array([1, 2, 3])], ['a'])
 
     f1 = pa.MockOutputStream()
     f2 = pa.BufferOutputStream()

python/pyarrow/tests/test_ipc.py

@@ -23,13 +23,22 @@ import threading
 
 import numpy as np
 
-from pandas.util.testing import (assert_frame_equal,
-                                 assert_series_equal)
-import pandas as pd
-
 import pyarrow as pa
 
 
+try:
+    from pandas.util.testing import (assert_frame_equal,
+                                     assert_series_equal)
+    import pandas as pd
+except ImportError:
+    pass
+
+
+# TODO(wesm): The IPC tests depend a lot on pandas currently, so all excluded
+# when it is not installed
+pytestmark = pytest.mark.pandas
+
+
 class IpcFixture(object):
 
     def __init__(self, sink_factory=lambda: io.BytesIO()):

python/pyarrow/tests/test_orc.py

@@ -19,16 +19,25 @@ import pytest
 import decimal
 import datetime
 
-import pandas as pd
 import pyarrow as pa
 
-from pandas.util.testing import assert_frame_equal
-
 # Marks all of the tests in this module
 # Ignore these with pytest ... -m 'not orc'
 pytestmark = pytest.mark.orc
 
 
+try:
+    from pandas.util.testing import assert_frame_equal
+    import pandas as pd
+except ImportError:
+    pass
+
+
+# TODO(wesm): The ORC tests depend a lot on pandas currently, so all excluded
+# when it is not installed
+pytestmark = pytest.mark.pandas
+
+
 @pytest.fixture(scope='module')
 def datadir(datadir):
     return datadir / 'orc'

python/pyarrow/tests/test_pandas.py

@@ -30,13 +30,22 @@ import hypothesis.extra.pytz as tzst
 import hypothesis.strategies as st
 import numpy as np
 import numpy.testing as npt
-import pandas as pd
-import pandas.util.testing as tm
 import pytest
 import pytz
 
+from pyarrow.pandas_compat import get_logical_type
 import pyarrow as pa
-from .pandas_examples import dataframe_with_arrays, dataframe_with_lists
+
+try:
+    import pandas as pd
+    import pandas.util.testing as tm
+    from .pandas_examples import dataframe_with_arrays, dataframe_with_lists
+except ImportError:
+    pass
+
+
+# Marks all of the tests in this module
+pytestmark = pytest.mark.pandas
 
 
 def _alltypes_example(size=100):
@@ -2531,6 +2540,180 @@ def test_table_from_pandas_columns_and_schema_are_mutually_exclusive():
         pa.Table.from_pandas(df, schema=schema, columns=columns)
 
 
+# ----------------------------------------------------------------------
+# RecordBatch, Table
+
+
+def test_recordbatch_from_to_pandas():
+    data = pd.DataFrame({
+        'c1': np.array([1, 2, 3, 4, 5], dtype='int64'),
+        'c2': np.array([1, 2, 3, 4, 5], dtype='uint32'),
+        'c3': np.random.randn(5),
+        'c4': ['foo', 'bar', None, 'baz', 'qux'],
+        'c5': [False, True, False, True, False]
+    })
+
+    batch = pa.RecordBatch.from_pandas(data)
+    result = batch.to_pandas()
+    tm.assert_frame_equal(data, result)
+
+
+def test_recordbatchlist_to_pandas():
+    data1 = pd.DataFrame({
+        'c1': np.array([1, 1, 2], dtype='uint32'),
+        'c2': np.array([1.0, 2.0, 3.0], dtype='float64'),
+        'c3': [True, None, False],
+        'c4': ['foo', 'bar', None]
+    })
+
+    data2 = pd.DataFrame({
+        'c1': np.array([3, 5], dtype='uint32'),
+        'c2': np.array([4.0, 5.0], dtype='float64'),
+        'c3': [True, True],
+        'c4': ['baz', 'qux']
+    })
+
+    batch1 = pa.RecordBatch.from_pandas(data1)
+    batch2 = pa.RecordBatch.from_pandas(data2)
+
+    table = pa.Table.from_batches([batch1, batch2])
+    result = table.to_pandas()
+    data = pd.concat([data1, data2]).reset_index(drop=True)
+    tm.assert_frame_equal(data, result)
+
+
+# ----------------------------------------------------------------------
+# Metadata serialization
+
+
+@pytest.mark.parametrize(
+    ('type', 'expected'),
+    [
+        (pa.null(), 'empty'),
+        (pa.bool_(), 'bool'),
+        (pa.int8(), 'int8'),
+        (pa.int16(), 'int16'),
+        (pa.int32(), 'int32'),
+        (pa.int64(), 'int64'),
+        (pa.uint8(), 'uint8'),
+        (pa.uint16(), 'uint16'),
+        (pa.uint32(), 'uint32'),
+        (pa.uint64(), 'uint64'),
+        (pa.float16(), 'float16'),
+        (pa.float32(), 'float32'),
+        (pa.float64(), 'float64'),
+        (pa.date32(), 'date'),
+        (pa.date64(), 'date'),
+        (pa.binary(), 'bytes'),
+        (pa.binary(length=4), 'bytes'),
+        (pa.string(), 'unicode'),
+        (pa.list_(pa.list_(pa.int16())), 'list[list[int16]]'),
+        (pa.decimal128(18, 3), 'decimal'),
+        (pa.timestamp('ms'), 'datetime'),
+        (pa.timestamp('us', 'UTC'), 'datetimetz'),
+        (pa.time32('s'), 'time'),
+        (pa.time64('us'), 'time')
+    ]
+)
+def test_logical_type(type, expected):
+    assert get_logical_type(type) == expected
+
+
+# ----------------------------------------------------------------------
+# Some nested array tests array tests
+
+
+def test_array_from_py_float32():
+    data = [[1.2, 3.4], [9.0, 42.0]]
+
+    t = pa.float32()
+
+    arr1 = pa.array(data[0], type=t)
+    arr2 = pa.array(data, type=pa.list_(t))
+
+    expected1 = np.array(data[0], dtype=np.float32)
+    expected2 = pd.Series([np.array(data[0], dtype=np.float32),
+                           np.array(data[1], dtype=np.float32)])
+
+    assert arr1.type == t
+    assert arr1.equals(pa.array(expected1))
+    assert arr2.equals(pa.array(expected2))
+
+
+# ----------------------------------------------------------------------
+# Timestamp tests
+
+
+def test_cast_timestamp_unit():
+    # ARROW-1680
+    val = datetime.now()
+    s = pd.Series([val])
+    s_nyc = s.dt.tz_localize('tzlocal()').dt.tz_convert('America/New_York')
+
+    us_with_tz = pa.timestamp('us', tz='America/New_York')
+
+    arr = pa.Array.from_pandas(s_nyc, type=us_with_tz)
+
+    # ARROW-1906
+    assert arr.type == us_with_tz
+
+    arr2 = pa.Array.from_pandas(s, type=pa.timestamp('us'))
+
+    assert arr[0].as_py() == s_nyc[0]
+    assert arr2[0].as_py() == s[0]
+
+    # Disallow truncation
+    arr = pa.array([123123], type='int64').cast(pa.timestamp('ms'))
+    expected = pa.array([123], type='int64').cast(pa.timestamp('s'))
+
+    target = pa.timestamp('s')
+    with pytest.raises(ValueError):
+        arr.cast(target)
+
+    result = arr.cast(target, safe=False)
+    assert result.equals(expected)
+
+    # ARROW-1949
+    series = pd.Series([pd.Timestamp(1), pd.Timestamp(10), pd.Timestamp(1000)])
+    expected = pa.array([0, 0, 1], type=pa.timestamp('us'))
+
+    with pytest.raises(ValueError):
+        pa.array(series, type=pa.timestamp('us'))
+
+    with pytest.raises(ValueError):
+        pa.Array.from_pandas(series, type=pa.timestamp('us'))
+
+    result = pa.Array.from_pandas(series, type=pa.timestamp('us'), safe=False)
+    assert result.equals(expected)
+
+    result = pa.array(series, type=pa.timestamp('us'), safe=False)
+    assert result.equals(expected)
+
+
+# ----------------------------------------------------------------------
+# DictionaryArray tests
+
+
+def test_dictionary_with_pandas():
+    indices = np.repeat([0, 1, 2], 2)
+    dictionary = np.array(['foo', 'bar', 'baz'], dtype=object)
+    mask = np.array([False, False, True, False, False, False])
+
+    d1 = pa.DictionaryArray.from_arrays(indices, dictionary)
+    d2 = pa.DictionaryArray.from_arrays(indices, dictionary, mask=mask)
+
+    pandas1 = d1.to_pandas()
+    ex_pandas1 = pd.Categorical.from_codes(indices, categories=dictionary)
+
+    tm.assert_series_equal(pd.Series(pandas1), pd.Series(ex_pandas1))
+
+    pandas2 = d2.to_pandas()
+    ex_pandas2 = pd.Categorical.from_codes(np.where(mask, -1, indices),
+                                           categories=dictionary)
+
+    tm.assert_series_equal(pd.Series(pandas2), pd.Series(ex_pandas2))
+
+
 # ----------------------------------------------------------------------
 # Legacy metadata compatibility tests
 

python/pyarrow/tests/test_parquet.py

@@ -25,14 +25,12 @@ import six
 import pytest
 
 import numpy as np
-import pandas as pd
-import pandas.util.testing as tm
 
 import pyarrow as pa
 from pyarrow.compat import guid, u, BytesIO, unichar, PY2
+from pyarrow.pandas_compat import _pandas_api
 from pyarrow.tests import util
 from pyarrow.filesystem import LocalFileSystem, FileSystem
-from .pandas_examples import dataframe_with_arrays, dataframe_with_lists
 
 try:
     import pyarrow.parquet as pq
@@ -40,6 +38,14 @@ except ImportError:
     pq = None
 
 
+try:
+    import pandas as pd
+    import pandas.util.testing as tm
+    from .pandas_examples import dataframe_with_arrays, dataframe_with_lists
+except ImportError:
+    pd = tm = None
+
+
 # Marks all of the tests in this module
 # Ignore these with pytest ... -m 'not parquet'
 pytestmark = pytest.mark.parquet
@@ -54,7 +60,7 @@ def _write_table(table, path, **kwargs):
     # So we see the ImportError somewhere
     import pyarrow.parquet as pq
 
-    if isinstance(table, pd.DataFrame):
+    if _pandas_api.is_data_frame(table):
         table = pa.Table.from_pandas(table)
 
     pq.write_table(table, path, **kwargs)
@@ -147,6 +153,7 @@ def alltypes_sample(size=10000, seed=0, categorical=False):
     return pd.DataFrame(arrays)
 
 
+@pytest.mark.pandas
 @pytest.mark.parametrize('chunk_size', [None, 1000])
 def test_pandas_parquet_2_0_rountrip(tempdir, chunk_size):
     df = alltypes_sample(size=10000, categorical=True)
@@ -166,6 +173,7 @@ def test_pandas_parquet_2_0_rountrip(tempdir, chunk_size):
     tm.assert_frame_equal(df, df_read, check_categorical=False)
 
 
+@pytest.mark.pandas
 def test_chunked_table_write():
     # ARROW-232
     df = alltypes_sample(size=10)
@@ -183,6 +191,7 @@ def test_chunked_table_write():
     _check_roundtrip(table, version='2.0')
 
 
+@pytest.mark.pandas
 def test_no_memory_map(tempdir):
     df = alltypes_sample(size=10)
     # The nanosecond->us conversion is a nuisance, so we just avoid it here
@@ -210,6 +219,7 @@ def test_special_chars_filename(tempdir):
     assert table_read.equals(table)
 
 
+@pytest.mark.pandas
 def test_empty_table_roundtrip():
     df = alltypes_sample(size=10)
     # The nanosecond->us conversion is a nuisance, so we just avoid it here
@@ -233,6 +243,7 @@ def test_empty_lists_table_roundtrip():
     _check_roundtrip(table)
 
 
+@pytest.mark.pandas
 def test_pandas_parquet_datetime_tz():
     s = pd.Series([datetime.datetime(2017, 9, 6)])
     s = s.dt.tz_localize('utc')
@@ -257,6 +268,7 @@ def test_pandas_parquet_datetime_tz():
     tm.assert_frame_equal(df, df_read)
 
 
+@pytest.mark.pandas
 @pytest.mark.skipif(six.PY2, reason='datetime.timezone is available since '
                                     'python version 3.2')
 def test_datetime_timezone_tzinfo():
@@ -267,6 +279,7 @@ def test_datetime_timezone_tzinfo():
     _roundtrip_pandas_dataframe(df, write_kwargs={})
 
 
+@pytest.mark.pandas
 def test_pandas_parquet_custom_metadata(tempdir):
     df = alltypes_sample(size=10000)
 
@@ -286,6 +299,7 @@ def test_pandas_parquet_custom_metadata(tempdir):
                                     'step': 1}]
 
 
+@pytest.mark.pandas
 def test_pandas_parquet_column_multiindex(tempdir):
     df = alltypes_sample(size=10)
     df.columns = pd.MultiIndex.from_tuples(
@@ -304,6 +318,7 @@ def test_pandas_parquet_column_multiindex(tempdir):
     tm.assert_frame_equal(df, df_read)
 
 
+@pytest.mark.pandas
 def test_pandas_parquet_2_0_rountrip_read_pandas_no_index_written(tempdir):
     df = alltypes_sample(size=10000)
 
@@ -327,6 +342,7 @@ def test_pandas_parquet_2_0_rountrip_read_pandas_no_index_written(tempdir):
     tm.assert_frame_equal(df, df_read)
 
 
+@pytest.mark.pandas
 def test_pandas_parquet_1_0_rountrip(tempdir):
     size = 10000
     np.random.seed(0)
@@ -358,6 +374,7 @@ def test_pandas_parquet_1_0_rountrip(tempdir):
     tm.assert_frame_equal(df, df_read)
 
 
+@pytest.mark.pandas
 def test_multiple_path_types(tempdir):
     # Test compatibility with PEP 519 path-like objects
     path = tempdir / 'zzz.parquet'
@@ -376,6 +393,7 @@ def test_multiple_path_types(tempdir):
     tm.assert_frame_equal(df, df_read)
 
 
+@pytest.mark.pandas
 def test_pandas_column_selection(tempdir):
     size = 10000
     np.random.seed(0)
@@ -431,6 +449,7 @@ def _test_dataframe(size=10000, seed=0):
     return df
 
 
+@pytest.mark.pandas
 def test_pandas_parquet_native_file_roundtrip(tempdir):
     df = _test_dataframe(10000)
     arrow_table = pa.Table.from_pandas(df)
@@ -442,6 +461,7 @@ def test_pandas_parquet_native_file_roundtrip(tempdir):
     tm.assert_frame_equal(df, df_read)
 
 
+@pytest.mark.pandas
 def test_parquet_incremental_file_build(tempdir):
     df = _test_dataframe(100)
     df['unique_id'] = 0
@@ -468,6 +488,7 @@ def test_parquet_incremental_file_build(tempdir):
     tm.assert_frame_equal(result.to_pandas(), expected)
 
 
+@pytest.mark.pandas
 def test_read_pandas_column_subset(tempdir):
     df = _test_dataframe(10000)
     arrow_table = pa.Table.from_pandas(df)
@@ -479,6 +500,7 @@ def test_read_pandas_column_subset(tempdir):
     tm.assert_frame_equal(df[['strings', 'uint8']], df_read)
 
 
+@pytest.mark.pandas
 def test_pandas_parquet_empty_roundtrip(tempdir):
     df = _test_dataframe(0)
     arrow_table = pa.Table.from_pandas(df)
@@ -490,6 +512,7 @@ def test_pandas_parquet_empty_roundtrip(tempdir):
     tm.assert_frame_equal(df, df_read)
 
 
+@pytest.mark.pandas
 def test_pandas_parquet_pyfile_roundtrip(tempdir):
     filename = tempdir / 'pandas_pyfile_roundtrip.parquet'
     size = 5
@@ -513,6 +536,7 @@ def test_pandas_parquet_pyfile_roundtrip(tempdir):
     tm.assert_frame_equal(df, df_read)
 
 
+@pytest.mark.pandas
 def test_pandas_parquet_configuration_options(tempdir):
     size = 10000
     np.random.seed(0)
@@ -561,6 +585,7 @@ def make_sample_file(table_or_df):
     return pq.ParquetFile(buf)
 
 
+@pytest.mark.pandas
 def test_parquet_metadata_api():
     df = alltypes_sample(size=10000)
     df = df.reindex(columns=sorted(df.columns))
@@ -639,6 +664,7 @@ def test_parquet_metadata_api():
         col_meta.index_page_offset
 
 
+@pytest.mark.pandas
 @pytest.mark.parametrize(
     (
         'data',
@@ -706,6 +732,7 @@ def test_parquet_column_statistics_api(data, type, physical_type, min_value,
     assert stat.physical_type == physical_type
 
 
+@pytest.mark.pandas
 def test_compare_schemas():
     df = alltypes_sample(size=10000)
 
@@ -754,6 +781,7 @@ def test_validate_schema_write_table(tempdir):
             w.write_table(simple_table)
 
 
+@pytest.mark.pandas
 def test_column_of_arrays(tempdir):
     df, schema = dataframe_with_arrays()
 
@@ -765,6 +793,7 @@ def test_column_of_arrays(tempdir):
     tm.assert_frame_equal(df, df_read)
 
 
+@pytest.mark.pandas
 def test_coerce_timestamps(tempdir):
     from collections import OrderedDict
     # ARROW-622
@@ -806,6 +835,7 @@ def test_coerce_timestamps(tempdir):
                      coerce_timestamps='unknown')
 
 
+@pytest.mark.pandas
 def test_coerce_timestamps_truncated(tempdir):
     """
     ARROW-2555: Test that we can truncate timestamps when coercing if
@@ -835,6 +865,7 @@ def test_coerce_timestamps_truncated(tempdir):
     tm.assert_frame_equal(df_expected, df_ms)
 
 
+@pytest.mark.pandas
 def test_column_of_lists(tempdir):
     df, schema = dataframe_with_lists(parquet_compatible=True)
 
@@ -856,6 +887,7 @@ def test_column_of_lists(tempdir):
     tm.assert_frame_equal(df, df_read)
 
 
+@pytest.mark.pandas
 def test_date_time_types():
     t1 = pa.date32()
     data1 = np.array([17259, 17260, 17261], dtype='int32')
@@ -943,6 +975,7 @@ def test_date_time_types():
     _assert_unsupported(a7)
 
 
+@pytest.mark.pandas
 def test_list_of_datetime_time_roundtrip():
     # ARROW-4135
     times = pd.to_datetime(['09:00', '09:30', '10:00', '10:30', '11:00',
@@ -978,6 +1011,7 @@ def test_sanitized_spark_field_names():
     assert result.schema[0].name == expected_name
 
 
+@pytest.mark.pandas
 def test_spark_flavor_preserves_pandas_metadata():
     df = _test_dataframe(size=100)
     df.index = np.arange(0, 10 * len(df), 10)
@@ -998,6 +1032,7 @@ def test_fixed_size_binary():
     _check_roundtrip(table)
 
 
+@pytest.mark.pandas
 def test_multithreaded_read():
     df = alltypes_sample(size=10000)
 
@@ -1015,6 +1050,7 @@ def test_multithreaded_read():
     assert table1.equals(table2)
 
 
+@pytest.mark.pandas
 def test_min_chunksize():
     data = pd.DataFrame([np.arange(4)], columns=['A', 'B', 'C', 'D'])
     table = pa.Table.from_pandas(data.reset_index())
@@ -1031,6 +1067,7 @@ def test_min_chunksize():
         _write_table(table, buf, chunk_size=0)
 
 
+@pytest.mark.pandas
 def test_pass_separate_metadata():
     # ARROW-471
     df = alltypes_sample(size=10000)
@@ -1050,6 +1087,7 @@ def test_pass_separate_metadata():
     tm.assert_frame_equal(df, fileh.read().to_pandas())
 
 
+@pytest.mark.pandas
 def test_read_single_row_group():
     # ARROW-471
     N, K = 10000, 4
@@ -1072,6 +1110,7 @@ def test_read_single_row_group():
     tm.assert_frame_equal(df, result.to_pandas())
 
 
+@pytest.mark.pandas
 def test_read_single_row_group_with_column_subset():
     N, K = 10000, 4
     df = alltypes_sample(size=N)
@@ -1096,6 +1135,7 @@ def test_read_single_row_group_with_column_subset():
     tm.assert_frame_equal(df[cols], result.to_pandas())
 
 
+@pytest.mark.pandas
 def test_scan_contents():
     N, K = 10000, 4
     df = alltypes_sample(size=N)
@@ -1112,6 +1152,7 @@ def test_scan_contents():
     assert pf.scan_contents(df.columns[:4]) == 10000
 
 
+@pytest.mark.pandas
 def test_parquet_piece_read(tempdir):
     df = _test_dataframe(1000)
     table = pa.Table.from_pandas(df)
@@ -1155,11 +1196,13 @@ def test_partition_set_dictionary_type():
         set3.dictionary
 
 
+@pytest.mark.pandas
 def test_read_partitioned_directory(tempdir):
     fs = LocalFileSystem.get_instance()
     _partition_test_for_filesystem(fs, tempdir)
 
 
+@pytest.mark.pandas
 def test_create_parquet_dataset_multi_threaded(tempdir):
     fs = LocalFileSystem.get_instance()
     base_path = tempdir
@@ -1176,6 +1219,7 @@ def test_create_parquet_dataset_multi_threaded(tempdir):
     assert len(partitions.levels) == len(manifest.partitions.levels)
 
 
+@pytest.mark.pandas
 def test_equivalency(tempdir):
     fs = LocalFileSystem.get_instance()
     base_path = tempdir
@@ -1251,6 +1295,7 @@ def test_equivalency(tempdir):
         pq.ParquetDataset(base_path, filesystem=fs, filters=filters)
 
 
+@pytest.mark.pandas
 def test_cutoff_exclusive_integer(tempdir):
     fs = LocalFileSystem.get_instance()
     base_path = tempdir
@@ -1284,6 +1329,7 @@ def test_cutoff_exclusive_integer(tempdir):
     assert result_list == [2, 3]
 
 
+@pytest.mark.pandas
 @pytest.mark.xfail(
     raises=TypeError,
     reason='Loss of type information in creation of categoricals.'
@@ -1330,6 +1376,7 @@ def test_cutoff_exclusive_datetime(tempdir):
     assert result_df['dates'].values == expected
 
 
+@pytest.mark.pandas
 def test_inclusive_integer(tempdir):
     fs = LocalFileSystem.get_instance()
     base_path = tempdir
@@ -1356,13 +1403,14 @@ def test_inclusive_integer(tempdir):
     )
     table = dataset.read()
     result_df = (table.to_pandas()
-                      .sort_values(by='index')
-                      .reset_index(drop=True))
+                 .sort_values(by='index')
+                 .reset_index(drop=True))
 
     result_list = [int(x) for x in map(int, result_df['integers'].values)]
     assert result_list == [2, 3]
 
 
+@pytest.mark.pandas
 def test_inclusive_set(tempdir):
     fs = LocalFileSystem.get_instance()
     base_path = tempdir
@@ -1398,6 +1446,7 @@ def test_inclusive_set(tempdir):
     assert False not in result_df['boolean'].values
 
 
+@pytest.mark.pandas
 def test_invalid_pred_op(tempdir):
     fs = LocalFileSystem.get_instance()
     base_path = tempdir
@@ -1454,6 +1503,7 @@ def s3_example():
     fs.rm(bucket_uri, recursive=True)
 
 
+@pytest.mark.pandas
 @pytest.mark.s3
 def test_read_partitioned_directory_s3fs(s3_example):
     from pyarrow.filesystem import S3FSWrapper
@@ -1565,11 +1615,13 @@ def _test_read_common_metadata_files(fs, base_path):
     assert dataset2.schema.equals(dataset.schema)
 
 
+@pytest.mark.pandas
 def test_read_common_metadata_files(tempdir):
     fs = LocalFileSystem.get_instance()
     _test_read_common_metadata_files(fs, tempdir)
 
 
+@pytest.mark.pandas
 def test_read_metadata_files(tempdir):
     fs = LocalFileSystem.get_instance()
 
@@ -1598,6 +1650,7 @@ def test_read_metadata_files(tempdir):
     assert dataset.schema.equals(metadata_schema)
 
 
+@pytest.mark.pandas
 def test_read_schema(tempdir):
     N = 100
     df = pd.DataFrame({
@@ -1630,6 +1683,7 @@ def _filter_partition(df, part_keys):
     return df[predicate].drop(to_drop, axis=1)
 
 
+@pytest.mark.pandas
 def test_read_multiple_files(tempdir):
     nfiles = 10
     size = 5
@@ -1709,6 +1763,7 @@ def test_read_multiple_files(tempdir):
         read_multiple_files(mixed_paths)
 
 
+@pytest.mark.pandas
 def test_dataset_read_pandas(tempdir):
     nfiles = 5
     size = 5
@@ -1740,6 +1795,7 @@ def test_dataset_read_pandas(tempdir):
     tm.assert_frame_equal(result, expected)
 
 
+@pytest.mark.pandas
 def test_dataset_no_memory_map(tempdir):
     # ARROW-2627: Check that we can use ParquetDataset without memory-mapping
     dirpath = tempdir / guid()
@@ -1756,6 +1812,7 @@ def test_dataset_no_memory_map(tempdir):
     assert dataset.pieces[0].read().equals(table)
 
 
+@pytest.mark.pandas
 @pytest.mark.parametrize('preserve_index', [True, False])
 def test_dataset_read_pandas_common_metadata(tempdir, preserve_index):
     # ARROW-1103
@@ -1811,6 +1868,7 @@ def _make_example_multifile_dataset(base_path, nfiles=10, file_nrows=5):
     return paths
 
 
+@pytest.mark.pandas
 def test_ignore_private_directories(tempdir):
     dirpath = tempdir / guid()
     dirpath.mkdir()
@@ -1825,6 +1883,7 @@ def test_ignore_private_directories(tempdir):
     assert set(map(str, paths)) == set(x.path for x in dataset.pieces)
 
 
+@pytest.mark.pandas
 def test_ignore_hidden_files(tempdir):
     dirpath = tempdir / guid()
     dirpath.mkdir()
@@ -1842,6 +1901,7 @@ def test_ignore_hidden_files(tempdir):
     assert set(map(str, paths)) == set(x.path for x in dataset.pieces)
 
 
+@pytest.mark.pandas
 def test_multiindex_duplicate_values(tempdir):
     num_rows = 3
     numbers = list(range(num_rows))
@@ -1863,6 +1923,7 @@ def test_multiindex_duplicate_values(tempdir):
     tm.assert_frame_equal(result_df, df)
 
 
+@pytest.mark.pandas
 def test_write_error_deletes_incomplete_file(tempdir):
     # ARROW-1285
     df = pd.DataFrame({'a': list('abc'),
@@ -1984,10 +2045,12 @@ def _test_write_to_dataset_no_partitions(base_path, filesystem=None):
     assert output_df.equals(input_df)
 
 
+@pytest.mark.pandas
 def test_write_to_dataset_with_partitions(tempdir):
     _test_write_to_dataset_with_partitions(str(tempdir))
 
 
+@pytest.mark.pandas
 def test_write_to_dataset_with_partitions_and_schema(tempdir):
     schema = pa.schema([pa.field('group1', type=pa.string()),
                         pa.field('group2', type=pa.string()),
@@ -1997,11 +2060,13 @@ def test_write_to_dataset_with_partitions_and_schema(tempdir):
     _test_write_to_dataset_with_partitions(str(tempdir), schema=schema)
 
 
+@pytest.mark.pandas
 def test_write_to_dataset_with_partitions_and_index_name(tempdir):
     _test_write_to_dataset_with_partitions(str(tempdir),
                                            index_name='index_name')
 
 
+@pytest.mark.pandas
 def test_write_to_dataset_no_partitions(tempdir):
     _test_write_to_dataset_no_partitions(str(tempdir))
 
@@ -2019,6 +2084,7 @@ def test_large_table_int32_overflow():
     _write_table(table, f)
 
 
+@pytest.mark.pandas
 @pytest.mark.large_memory
 def test_binary_array_overflow_to_chunked():
     # ARROW-3762
@@ -2046,6 +2112,7 @@ def test_binary_array_overflow_to_chunked():
     assert tbl.equals(read_tbl)
 
 
+@pytest.mark.pandas
 def test_index_column_name_duplicate(tempdir):
     data = {
         'close': {
@@ -2070,6 +2137,7 @@ def test_index_column_name_duplicate(tempdir):
     tm.assert_frame_equal(result_df, dfx)
 
 
+@pytest.mark.pandas
 def test_parquet_nested_convenience(tempdir):
     # ARROW-1684
     df = pd.DataFrame({
@@ -2089,6 +2157,7 @@ def test_parquet_nested_convenience(tempdir):
     tm.assert_frame_equal(read.to_pandas(), df)
 
 
+@pytest.mark.pandas
 def test_backwards_compatible_index_naming(datadir):
     expected_string = b"""\
 carat        cut  color  clarity  depth  table  price     x     y     z
@@ -2109,6 +2178,7 @@ carat        cut  color  clarity  depth  table  price     x     y     z
     tm.assert_frame_equal(result, expected)
 
 
+@pytest.mark.pandas
 def test_backwards_compatible_index_multi_level_named(datadir):
     expected_string = b"""\
 carat        cut  color  clarity  depth  table  price     x     y     z
@@ -2133,6 +2203,7 @@ carat        cut  color  clarity  depth  table  price     x     y     z
     tm.assert_frame_equal(result, expected)
 
 
+@pytest.mark.pandas
 def test_backwards_compatible_index_multi_level_some_named(datadir):
     expected_string = b"""\
 carat        cut  color  clarity  depth  table  price     x     y     z
@@ -2158,6 +2229,7 @@ carat        cut  color  clarity  depth  table  price     x     y     z
     tm.assert_frame_equal(result, expected)
 
 
+@pytest.mark.pandas
 def test_backwards_compatible_column_metadata_handling(datadir):
     expected = pd.DataFrame(
         {'a': [1, 2, 3], 'b': [.1, .2, .3],
@@ -2177,6 +2249,7 @@ def test_backwards_compatible_column_metadata_handling(datadir):
     tm.assert_frame_equal(result, expected[['a']].reset_index(drop=True))
 
 
+@pytest.mark.pandas
 def test_decimal_roundtrip(tempdir):
     num_values = 10
 
@@ -2202,6 +2275,7 @@ def test_decimal_roundtrip(tempdir):
     tm.assert_frame_equal(result, expected)
 
 
+@pytest.mark.pandas
 @pytest.mark.xfail(
     raises=pa.ArrowException, reason='Parquet does not support negative scale'
 )
@@ -2216,6 +2290,7 @@ def test_decimal_roundtrip_negative_scale(tempdir):
     tm.assert_frame_equal(result, expected)
 
 
+@pytest.mark.pandas
 def test_parquet_writer_context_obj(tempdir):
     df = _test_dataframe(100)
     df['unique_id'] = 0
@@ -2240,6 +2315,7 @@ def test_parquet_writer_context_obj(tempdir):
     tm.assert_frame_equal(result.to_pandas(), expected)
 
 
+@pytest.mark.pandas
 def test_parquet_writer_context_obj_with_exception(tempdir):
     df = _test_dataframe(100)
     df['unique_id'] = 0
@@ -2271,6 +2347,7 @@ def test_parquet_writer_context_obj_with_exception(tempdir):
     tm.assert_frame_equal(result.to_pandas(), expected)
 
 
+@pytest.mark.pandas
 def test_zlib_compression_bug():
     # ARROW-3514: "zlib deflate failed, output buffer too small"
     table = pa.Table.from_arrays([pa.array(['abc', 'def'])], ['some_col'])
@@ -2282,6 +2359,7 @@ def test_zlib_compression_bug():
     tm.assert_frame_equal(roundtrip.to_pandas(), table.to_pandas())
 
 
+@pytest.mark.pandas
 def test_merging_parquet_tables_with_different_pandas_metadata(tempdir):
     # ARROW-3728: Merging Parquet Files - Pandas Meta in Schema Mismatch
     schema = pa.schema([
@@ -2328,6 +2406,7 @@ def test_empty_row_groups(tempdir):
         assert reader.read_row_group(i).equals(table)
 
 
+@pytest.mark.pandas
 def test_parquet_writer_with_caller_provided_filesystem():
     out = pa.BufferOutputStream()
 

python/pyarrow/tests/test_plasma.py

@@ -32,7 +32,6 @@ import time
 
 import numpy as np
 import pyarrow as pa
-import pandas as pd
 
 
 DEFAULT_PLASMA_STORE_MEMORY = 10 ** 8
@@ -408,7 +407,9 @@ class TestPlasmaClient(object):
         # Assert that they are equal.
         np.testing.assert_equal(data, array)
 
+    @pytest.mark.pandas
     def test_store_pandas_dataframe(self):
+        import pandas as pd
         import pyarrow.plasma as plasma
         d = {'one': pd.Series([1., 2., 3.], index=['a', 'b', 'c']),
              'two': pd.Series([1., 2., 3., 4.], index=['a', 'b', 'c', 'd'])}

python/pyarrow/tests/test_scalars.py

@@ -19,7 +19,6 @@
 import pytest
 
 import numpy as np
-import pandas as pd
 
 from pyarrow.compat import unittest, u, unicode_type
 import pyarrow as pa
@@ -159,7 +158,9 @@ class TestScalars(unittest.TestCase):
         v = arr[3]
         assert len(v) == 0
 
+    @pytest.mark.pandas
     def test_timestamp(self):
+        import pandas as pd
         arr = pd.date_range('2000-01-01 12:34:56', periods=10).values
 
         units = ['ns', 'us', 'ms', 's']
@@ -185,7 +186,9 @@ class TestScalars(unittest.TestCase):
             assert arrow_arr[0].as_py() == expected
             assert arrow_arr[0].value * 1000**i == expected.value
 
+    @pytest.mark.pandas
     def test_dictionary(self):
+        import pandas as pd
         colors = ['red', 'green', 'blue']
         colors_dict = {'red': 0, 'green': 1, 'blue': 2}
         values = pd.Series(colors * 4)

python/pyarrow/tests/test_schema.py

@@ -20,8 +20,6 @@ import pickle
 
 import pytest
 import numpy as np
-import pandas as pd
-
 import pyarrow as pa
 
 
@@ -493,18 +491,21 @@ def test_empty_table():
     assert table.schema == schema
 
 
-@pytest.mark.parametrize('data', [
-    list(range(10)),
-    pd.Categorical(list(range(10))),
-    ['foo', 'bar', None, 'baz', 'qux'],
-    np.array([
-        '2007-07-13T01:23:34.123456789',
-        '2006-01-13T12:34:56.432539784',
-        '2010-08-13T05:46:57.437699912'
-    ], dtype='datetime64[ns]')
-])
-def test_schema_from_pandas(data):
-    df = pd.DataFrame({'a': data})
-    schema = pa.Schema.from_pandas(df)
-    expected = pa.Table.from_pandas(df).schema
-    assert schema == expected
+@pytest.mark.pandas
+def test_schema_from_pandas():
+    import pandas as pd
+    inputs = [
+        list(range(10)),
+        pd.Categorical(list(range(10))),
+        ['foo', 'bar', None, 'baz', 'qux'],
+        np.array([
+            '2007-07-13T01:23:34.123456789',
+            '2006-01-13T12:34:56.432539784',
+            '2010-08-13T05:46:57.437699912'
+        ], dtype='datetime64[ns]')
+    ]
+    for data in inputs:
+        df = pd.DataFrame({'a': data})
+        schema = pa.Schema.from_pandas(df)
+        expected = pa.Table.from_pandas(df).schema
+        assert schema == expected

python/pyarrow/tests/test_table.py

@@ -19,10 +19,7 @@ from collections import OrderedDict, Iterable
 import pickle
 
 import numpy as np
-from pandas.util.testing import assert_frame_equal
-import pandas as pd
 import pytest
-
 import pyarrow as pa
 
 
@@ -176,6 +173,7 @@ def test_chunked_array_pickle(data, typ):
     assert result.equals(array)
 
 
+@pytest.mark.pandas
 def test_chunked_array_to_pandas():
     data = [
         pa.array([-10, -5, 0, 5, 10])
@@ -269,6 +267,7 @@ def test_column_pickle():
     assert result.field == field
 
 
+@pytest.mark.pandas
 def test_column_to_pandas():
     data = [
         pa.array([-10, -5, 0, 5, 10])
@@ -459,50 +458,11 @@ def test_recordbatch_slice_getitem():
     assert batch.slice(len(batch) - 4, 2).equals(batch[-4:-2])
 
 
-def test_recordbatch_from_to_pandas():
-    data = pd.DataFrame({
-        'c1': np.array([1, 2, 3, 4, 5], dtype='int64'),
-        'c2': np.array([1, 2, 3, 4, 5], dtype='uint32'),
-        'c3': np.random.randn(5),
-        'c4': ['foo', 'bar', None, 'baz', 'qux'],
-        'c5': [False, True, False, True, False]
-    })
-
-    batch = pa.RecordBatch.from_pandas(data)
-    result = batch.to_pandas()
-    assert_frame_equal(data, result)
-
-
-def test_recordbatchlist_to_pandas():
-    data1 = pd.DataFrame({
-        'c1': np.array([1, 1, 2], dtype='uint32'),
-        'c2': np.array([1.0, 2.0, 3.0], dtype='float64'),
-        'c3': [True, None, False],
-        'c4': ['foo', 'bar', None]
-    })
-
-    data2 = pd.DataFrame({
-        'c1': np.array([3, 5], dtype='uint32'),
-        'c2': np.array([4.0, 5.0], dtype='float64'),
-        'c3': [True, True],
-        'c4': ['baz', 'qux']
-    })
-
-    batch1 = pa.RecordBatch.from_pandas(data1)
-    batch2 = pa.RecordBatch.from_pandas(data2)
-
-    table = pa.Table.from_batches([batch1, batch2])
-    result = table.to_pandas()
-    data = pd.concat([data1, data2]).reset_index(drop=True)
-    assert_frame_equal(data, result)
-
-
 def test_recordbatchlist_schema_equals():
-    data1 = pd.DataFrame({'c1': np.array([1], dtype='uint32')})
-    data2 = pd.DataFrame({'c1': np.array([4.0, 5.0], dtype='float64')})
-
-    batch1 = pa.RecordBatch.from_pandas(data1)
-    batch2 = pa.RecordBatch.from_pandas(data2)
+    a1 = np.array([1], dtype='uint32')
+    a2 = np.array([4.0, 5.0], dtype='float64')
+    batch1 = pa.RecordBatch.from_arrays([pa.array(a1)], ['c1'])
+    batch2 = pa.RecordBatch.from_arrays([pa.array(a2)], ['c1'])
 
     with pytest.raises(pa.ArrowInvalid):
         pa.Table.from_batches([batch1, batch2])
@@ -537,7 +497,11 @@ def test_table_from_batches_and_schema():
         pa.Table.from_batches([incompatible_batch], schema)
 
 
+@pytest.mark.pandas
 def test_table_to_batches():
+    from pandas.util.testing import assert_frame_equal
+    import pandas as pd
+
     df1 = pd.DataFrame({'a': list(range(10))})
     df2 = pd.DataFrame({'a': list(range(10, 30))})
 
@@ -809,7 +773,10 @@ def test_concat_tables():
     assert result.equals(expected)
 
 
+@pytest.mark.pandas
 def test_concat_tables_with_different_schema_metadata():
+    import pandas as pd
+
     schema = pa.schema([
         pa.field('a', pa.string()),
         pa.field('b', pa.string()),

python/requirements-test.txt

@@ -1,4 +1,5 @@
 pandas
 pytest
 hypothesis
+pytz
 pathlib2; python_version < "3.4"