Begin to refactor to make references to pandas more centralized and lazy

python/pyarrow/array.pxi

@@ -15,10 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from pyarrow.compat import HAVE_PANDAS
-
-if HAVE_PANDAS:
-    import pyarrow.pandas_compat as pdcompat
+from pyarrow.compat import PandasAPI
+_pandas_api = PandasAPI.get_instance()
 
 
 cdef _sequence_to_array(object sequence, object mask, object size,
@@ -47,11 +45,10 @@ cdef _sequence_to_array(object sequence, object mask, object size,
 
 
 cdef _is_array_like(obj):
-    try:
-        import pandas
-        return isinstance(obj, (np.ndarray, pd.Series, pd.Index, Categorical))
-    except ImportError:
-        return isinstance(obj, np.ndarray)
+    is_ndarray = isinstance(obj, np.ndarray)
+    if _pandas_api.have_pandas:
+        return is_ndarray or _pandas_api.is_array_like(obj)
+    return is_ndarray
 
 
 def _ndarray_to_arrow_type(object values, DataType type):
@@ -163,15 +160,15 @@ def array(object obj, type=None, mask=None, size=None, bint from_pandas=False,
 
         values = get_series_values(obj)
 
-        if isinstance(values, Categorical):
+        if _pandas_api.is_categorical(values):
             return DictionaryArray.from_arrays(
                 values.codes, values.categories.values,
                 mask=mask, ordered=values.ordered,
                 from_pandas=True, safe=safe,
                 memory_pool=memory_pool)
         else:
-            if HAVE_PANDAS:
-                values, type = pdcompat.get_datetimetz_type(
+            if _pandas_api.have_pandas:
+                values, type = _pandas_api.compat.get_datetimetz_type(
                     values, obj.dtype, type)
             return _ndarray_to_array(values, mask, type, from_pandas, safe,
                                      pool)
@@ -852,9 +849,10 @@ cdef wrap_array_output(PyObject* output):
     cdef object obj = PyObject_to_object(output)
 
     if isinstance(obj, dict):
-        return Categorical(obj['indices'],
-                           categories=obj['dictionary'],
-                           ordered=obj['ordered'], fastpath=True)
+        return _pandas_api.categorical_type(obj['indices'],
+                                            categories=obj['dictionary'],
+                                            ordered=obj['ordered'],
+                                            fastpath=True)
     else:
         return obj
 
@@ -1385,11 +1383,11 @@ cdef dict _array_classes = {
 
 
 cdef object get_series_values(object obj):
-    if isinstance(obj, PandasSeries):
+    if _pandas_api.is_series(obj):
         result = obj.values
     elif isinstance(obj, np.ndarray):
         result = obj
     else:
-        result = PandasSeries(obj).values
+        result = _pandas_api.make_series(obj).values
 
     return result

python/pyarrow/compat.py

@@ -31,37 +31,102 @@ import socket
 PY26 = sys.version_info[:2] == (2, 6)
 PY2 = sys.version_info[0] == 2
 
-try:
-    import pandas as pd
-    pdver = LooseVersion(pd.__version__)
-    if pdver >= '0.20.0':
-        from pandas.api.types import DatetimeTZDtype
-        pdapi = pd.api.types
-    elif pdver >= '0.19.0':
-        from pandas.types.dtypes import DatetimeTZDtype
-        pdapi = pd.api.types
-    else:
-        from pandas.types.dtypes import DatetimeTZDtype
-        pdapi = pd.core.common
 
-    PandasSeries = pd.Series
-    Categorical = pd.Categorical
-    HAVE_PANDAS = True
-except:
-    HAVE_PANDAS = False
-    class DatetimeTZDtype(object):
-        pass
+class PandasAPI(object):
+    """
 
-    class ClassPlaceholder(object):
+    """
+    _instance = None
 
-        def __init__(self, *args, **kwargs):
-            raise NotImplementedError
+    @classmethod
+    def get_instance(cls):
+        if cls._instance is None:
+            cls._instance = PandasAPI()
+        return cls._instance
 
-    class PandasSeries(ClassPlaceholder):
-        pass
+    def __init__(self):
+        self._imported_pandas = False
+        self._have_pandas = False
+        self._pd = None
+        self._compat_module = None
+        self._types_api = None
+        self._series = None
+        self._categorical_type = None
+        self._datetimetz_type = None
+        self._get_datetimetz_type = None
 
-    class Categorical(ClassPlaceholder):
-        pass
+    @property
+    def make_series(self, *args, **kwargs):
+        self._check_import()
+        return self._series(*args, **kwargs)
+
+    @property
+    def have_pandas(self):
+        self._check_import(raise_=False)
+        return self._have_pandas
+
+    @property
+    def compat(self):
+        self._check_import()
+        return self._compat_module
+
+    def infer_dtype(self, obj):
+        try:
+            return self._types_api.infer_dtype(obj, skipna=False)
+        except AttributeError:
+            return self._pd.lib.infer_dtype(obj)
+
+    @property
+    def version(self):
+        self._check_import()
+        return self._version
+
+    @property
+    def categorical_type(self):
+        self._check_import()
+        return self._categorical_type
+
+    def _check_import(self, raise_=True):
+        if self._imported_pandas:
+            return
+
+        try:
+            import pandas as pd
+            import pyarrow.pandas_compat as pdcompat
+            self._pd = pd
+            self._compat_module = pdcompat
+            self._series = pd.Series
+            self._have_pandas = True
+
+            self._version = LooseVersion(pd.__version__)
+            if self._version >= '0.20.0':
+                from pandas.api.types import DatetimeTZDtype
+                self._types_api = pd.api.types
+            elif pdver >= '0.19.0':
+                from pandas.types.dtypes import DatetimeTZDtype
+                self._types_api = pd.api.types
+            else:
+                from pandas.types.dtypes import DatetimeTZDtype
+                self._types_api = pd.core.common
+
+            self._datetimetz_type = DatetimeTZDtype
+            self._categorical_type = pd.Categorical
+        except ImportError:
+            if raise_:
+                raise
+
+    def is_array_like(self, obj):
+        self._check_import()
+        return isinstance(obj, (self._pd.Series, self._pd.Index,
+                                self._categorical_type))
+
+    def is_categorical(self, obj):
+        self._check_import()
+        return isinstance(obj, self._categorical_type)
+
+    def is_series(self, obj):
+        self._check_import()
+        return isinstance(obj, self._series)
 
 
 if PY26:

python/pyarrow/feather.py

@@ -15,31 +15,29 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from distutils.version import LooseVersion
 import os
 
 import six
 import pandas as pd
 
-from pyarrow.compat import pdapi
+from pyarrow.compat import PandasAPI as _PandasAPI  # noqa
 from pyarrow.lib import FeatherError  # noqa
 from pyarrow.lib import Table, concat_tables
 import pyarrow.lib as ext
 
 
-try:
-    infer_dtype = pdapi.infer_dtype
-except AttributeError:
-    infer_dtype = pd.lib.infer_dtype
+_pandas_api = _PandasAPI.get_instance()
 
 
-if LooseVersion(pd.__version__) < '0.17.0':
-    raise ImportError("feather requires pandas >= 0.17.0")
+def _check_pandas_version():
+    if _pandas_api.version < '0.17.0':
+        raise ImportError("feather requires pandas >= 0.17.0")
 
 
 class FeatherReader(ext.FeatherReader):
 
     def __init__(self, source):
+        _check_pandas_version()
         self.source = source
         self.open(source)
 
@@ -80,6 +78,7 @@ def check_chunked_overflow(col):
 class FeatherWriter(object):
 
     def __init__(self, dest):
+        _check_pandas_version()
         self.dest = dest
         self.writer = ext.FeatherWriter()
         self.writer.open(dest)
@@ -114,6 +113,7 @@ class FeatherDataset(object):
         Check that individual file schemas are all the same / compatible
     """
     def __init__(self, path_or_paths, validate_schema=True):
+        _check_pandas_version()
         self.paths = path_or_paths
         self.validate_schema = validate_schema
 

python/pyarrow/lib.pyx

@@ -25,7 +25,7 @@ import multiprocessing
 import numpy as np
 import os
 import six
-from pyarrow.compat import frombytes, tobytes, PandasSeries, Categorical
+from pyarrow.compat import frombytes, tobytes
 
 from cython.operator cimport dereference as deref
 from pyarrow.includes.libarrow cimport *

python/pyarrow/pandas_compat.py

@@ -21,22 +21,13 @@ import json
 import operator
 import re
 
-import pandas.core.internals as _int
 import numpy as np
-import pandas as pd
 
 import six
 
 import pyarrow as pa
-from pyarrow.compat import (builtin_pickle, DatetimeTZDtype,  # noqa
-                            PY2, zip_longest)
-
-
-def infer_dtype(column):
-    try:
-        return pd.api.types.infer_dtype(column, skipna=False)
-    except AttributeError:
-        return pd.lib.infer_dtype(column)
+from pyarrow.compat import (builtin_pickle, PY2, zip_longest,
+                            _pandas_api)  # noqa
 
 
 _logical_type_map = {}
@@ -515,6 +506,7 @@ def get_datetimetz_type(values, dtype, type_):
 
 
 def dataframe_to_serialized_dict(frame):
+    import pandas.core.internals as _int
     block_manager = frame._data
 
     blocks = []
@@ -554,11 +546,12 @@ def dataframe_to_serialized_dict(frame):
 
 
 def serialized_dict_to_dataframe(data):
+    import pandas.core.internals as _int
     reconstructed_blocks = [_reconstruct_block(block)
                             for block in data['blocks']]
 
     block_mgr = _int.BlockManager(reconstructed_blocks, data['axes'])
-    return pd.DataFrame(block_mgr)
+    return _pandas_api.data_frame(block_mgr)
 
 
 def _reconstruct_block(item):

python/pyarrow/scalar.pxi

@@ -284,37 +284,45 @@ cdef class Time64Value(ArrayValue):
                     datetime.timedelta(microseconds=val / 1000)).time()
 
 
-cdef dict DATETIME_CONVERSION_FUNCTIONS
+cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
+cdef c_bool _datetime_conversion_initialized = False
 
-try:
-    import pandas as pd
-except ImportError:
-    DATETIME_CONVERSION_FUNCTIONS = {
-        TimeUnit_SECOND: lambda x, tzinfo: (
-            datetime.datetime.utcfromtimestamp(x).replace(tzinfo=tzinfo)
-        ),
-        TimeUnit_MILLI: lambda x, tzinfo: (
-            datetime.datetime.utcfromtimestamp(x / 1e3).replace(tzinfo=tzinfo)
-        ),
-        TimeUnit_MICRO: lambda x, tzinfo: (
-            datetime.datetime.utcfromtimestamp(x / 1e6).replace(tzinfo=tzinfo)
-        ),
-    }
-else:
-    DATETIME_CONVERSION_FUNCTIONS = {
-        TimeUnit_SECOND: lambda x, tzinfo: pd.Timestamp(
-            x * 1000000000, tz=tzinfo, unit='ns',
-        ),
-        TimeUnit_MILLI: lambda x, tzinfo: pd.Timestamp(
-            x * 1000000, tz=tzinfo, unit='ns',
-        ),
-        TimeUnit_MICRO: lambda x, tzinfo: pd.Timestamp(
-            x * 1000, tz=tzinfo, unit='ns',
-        ),
-        TimeUnit_NANO: lambda x, tzinfo: pd.Timestamp(
-            x, tz=tzinfo, unit='ns',
-        )
-    }
+cdef _datetime_conversion_functions():
+    global _datetime_conversion_initialized
+    if _datetime_conversion_initialized:
+        return _DATETIME_CONVERSION_FUNCTIONS
+
+    try:
+        import pandas as pd
+    except ImportError:
+        _DATETIME_CONVERSION_FUNCTIONS.update({
+            TimeUnit_SECOND: lambda x, tzinfo: (
+                datetime.datetime.utcfromtimestamp(x).replace(tzinfo=tzinfo)
+            ),
+            TimeUnit_MILLI: lambda x, tzinfo: (
+                datetime.datetime.utcfromtimestamp(x / 1e3).replace(tzinfo=tzinfo)
+            ),
+            TimeUnit_MICRO: lambda x, tzinfo: (
+                datetime.datetime.utcfromtimestamp(x / 1e6).replace(tzinfo=tzinfo)
+            ),
+        })
+    else:
+        _DATETIME_CONVERSION_FUNCTIONS.update({
+            TimeUnit_SECOND: lambda x, tzinfo: pd.Timestamp(
+                x * 1000000000, tz=tzinfo, unit='ns',
+            ),
+            TimeUnit_MILLI: lambda x, tzinfo: pd.Timestamp(
+                x * 1000000, tz=tzinfo, unit='ns',
+            ),
+            TimeUnit_MICRO: lambda x, tzinfo: pd.Timestamp(
+                x * 1000, tz=tzinfo, unit='ns',
+            ),
+            TimeUnit_NANO: lambda x, tzinfo: pd.Timestamp(
+                x, tz=tzinfo, unit='ns',
+            )
+        })
+    _datetime_conversion_initialized = True
+    return _DATETIME_CONVERSION_FUNCTIONS
 
 
 cdef class TimestampValue(ArrayValue):
@@ -344,7 +352,7 @@ cdef class TimestampValue(ArrayValue):
             tzinfo = None
 
         try:
-            converter = DATETIME_CONVERSION_FUNCTIONS[dtype.unit()]
+            converter = _datetime_conversion_functions()[dtype.unit()]
         except KeyError:
             raise ValueError(
                 'Cannot convert nanosecond timestamps without pandas'