Cythonize pandas API shim for better performance

python/pyarrow/array.pxi

@@ -41,11 +41,10 @@ cdef _sequence_to_array(object sequence, object mask, object size,
         return pyarrow_wrap_chunked_array(out)
 
 
-cdef _is_array_like(obj):
-    is_ndarray = isinstance(obj, np.ndarray)
-    if _pandas_api.have_pandas:
-        return is_ndarray or _pandas_api.is_array_like(obj)
-    return is_ndarray
+cdef inline _is_array_like(obj):
+    if isinstance(obj, np.ndarray):
+        return True
+    return pandas_api._have_pandas_internal() and pandas_api.is_array_like(obj)
 
 
 def _ndarray_to_arrow_type(object values, DataType type):
@@ -157,15 +156,15 @@ def array(object obj, type=None, mask=None, size=None, bint from_pandas=False,
 
         values = get_series_values(obj)
 
-        if _pandas_api.is_categorical(values):
+        if pandas_api.is_categorical(values):
             return DictionaryArray.from_arrays(
                 values.codes, values.categories.values,
                 mask=mask, ordered=values.ordered,
                 from_pandas=True, safe=safe,
                 memory_pool=memory_pool)
         else:
-            if _pandas_api.have_pandas:
-                values, type = _pandas_api.compat.get_datetimetz_type(
+            if pandas_api.have_pandas:
+                values, type = pandas_api.compat.get_datetimetz_type(
                     values, obj.dtype, type)
             return _ndarray_to_array(values, mask, type, from_pandas, safe,
                                      pool)
@@ -846,10 +845,10 @@ cdef wrap_array_output(PyObject* output):
     cdef object obj = PyObject_to_object(output)
 
     if isinstance(obj, dict):
-        return _pandas_api.categorical_type(obj['indices'],
-                                            categories=obj['dictionary'],
-                                            ordered=obj['ordered'],
-                                            fastpath=True)
+        return pandas_api.categorical_type(obj['indices'],
+                                           categories=obj['dictionary'],
+                                           ordered=obj['ordered'],
+                                           fastpath=True)
     else:
         return obj
 
@@ -1380,11 +1379,11 @@ cdef dict _array_classes = {
 
 
 cdef object get_series_values(object obj):
-    if _pandas_api.is_series(obj):
+    if pandas_api.is_series(obj):
         result = obj.values
     elif isinstance(obj, np.ndarray):
         result = obj
     else:
-        result = _pandas_api.make_series(obj).values
+        result = pandas_api.make_series(obj).values
 
     return result

python/pyarrow/lib.pyx

@@ -19,14 +19,15 @@
 # distutils: language = c++
 # cython: embedsignature = True
 
+from collections import OrderedDict
 import datetime
 import decimal as _pydecimal
+import json
 import multiprocessing
 import numpy as np
 import os
 import six
 from pyarrow.compat import frombytes, tobytes
-from pyarrow.pandas_compat import _pandas_api
 
 from cython.operator cimport dereference as deref
 from pyarrow.includes.libarrow cimport *
@@ -91,6 +92,9 @@ Type_MAP = _Type_MAP
 UnionMode_SPARSE = _UnionMode_SPARSE
 UnionMode_DENSE = _UnionMode_DENSE
 
+# pandas API shim
+include "pandas-shim.pxi"
+
 # Exception types
 include "error.pxi"
 

python/pyarrow/pandas-shim.pxi

@@ -0,0 +1,172 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# pandas lazy-loading API shim that reduces API call and import overhead
+
+cdef class _PandasAPIShim(object):
+    """
+    Lazy pandas importer that isolates usages of pandas APIs and avoids
+    importing pandas until it's actually needed
+    """
+    cdef:
+        bint _tried_importing_pandas
+        bint _have_pandas
+
+    cdef readonly:
+        object _loose_version, _version
+        object _pd, _types_api, _compat_module
+        object _data_frame, _index, _series, _categorical_type
+        object _datetimetz_type
+        object _array_like_types
+
+    def __init__(self):
+        self._tried_importing_pandas = False
+        self._have_pandas = 0
+
+    cdef _import_pandas(self, bint raise_):
+        try:
+            import pandas as pd
+            import pyarrow.pandas_compat as pdcompat
+        except ImportError:
+            self._have_pandas = False
+            if raise_:
+                raise
+            else:
+                return
+
+        self._pd = pd
+        self._compat_module = pdcompat
+        self._data_frame = pd.DataFrame
+        self._index = pd.Index
+        self._categorical_type = pd.Categorical
+        self._series = pd.Series
+
+        self._array_like_types = (self._series, self._index,
+                                  self._categorical_type)
+
+        from distutils.version import LooseVersion
+        self._loose_version = LooseVersion(pd.__version__)
+        if self._loose_version >= '0.20.0':
+            from pandas.api.types import DatetimeTZDtype
+            self._types_api = pd.api.types
+        elif self._loose_version >= '0.19.0':
+            from pandas.types.dtypes import DatetimeTZDtype
+            self._types_api = pd.api.types
+        else:
+            from pandas.types.dtypes import DatetimeTZDtype
+            self._types_api = pd.core.common
+
+        self._datetimetz_type = DatetimeTZDtype
+        self._have_pandas = True
+
+    cdef inline _check_import(self, bint raise_=True):
+        if self._tried_importing_pandas:
+            if not self._have_pandas and raise_:
+                self._import_pandas(raise_)
+            return
+
+        self._tried_importing_pandas = True
+        self._import_pandas(raise_)
+
+    def make_series(self, *args, **kwargs):
+        self._check_import()
+        return self._series(*args, **kwargs)
+
+    def data_frame(self, *args, **kwargs):
+        self._check_import()
+        return self._data_frame(*args, **kwargs)
+
+    cdef inline bint _have_pandas_internal(self):
+        if not self._tried_importing_pandas:
+            self._check_import(raise_=False)
+        return self._have_pandas
+
+    @property
+    def have_pandas(self):
+        return self._have_pandas_internal()
+
+    @property
+    def compat(self):
+        self._check_import()
+        return self._compat_module
+
+    @property
+    def pd(self):
+        self._check_import()
+        return self._pd
+
+    cpdef infer_dtype(self, obj):
+        self._check_import()
+        try:
+            return self._types_api.infer_dtype(obj, skipna=False)
+        except AttributeError:
+            return self._pd.lib.infer_dtype(obj)
+
+    @property
+    def loose_version(self):
+        self._check_import()
+        return self._loose_version
+
+    @property
+    def version(self):
+        self._check_import()
+        return self._version
+
+    @property
+    def categorical_type(self):
+        self._check_import()
+        return self._categorical_type
+
+    @property
+    def datetimetz_type(self):
+        return self._datetimetz_type
+
+    cpdef is_array_like(self, obj):
+        self._check_import()
+        return isinstance(obj, self._array_like_types)
+
+    cpdef is_categorical(self, obj):
+        if self._have_pandas_internal():
+            return isinstance(obj, self._categorical_type)
+        else:
+            return False
+
+    cpdef is_datetimetz(self, obj):
+        if self._have_pandas_internal():
+            return isinstance(obj, self._datetimetz_type)
+        else:
+            return False
+
+    cpdef is_data_frame(self, obj):
+        if self._have_pandas_internal():
+            return isinstance(obj, self._data_frame)
+        else:
+            return False
+
+    cpdef is_series(self, obj):
+        if self._have_pandas_internal():
+            return isinstance(obj, self._series)
+        else:
+            return False
+
+    def assert_frame_equal(self, *args, **kwargs):
+        self._check_import()
+        return self._pd.util.testing.assert_frame_equal
+
+
+cdef _PandasAPIShim pandas_api = _PandasAPIShim()
+_pandas_api = pandas_api

python/pyarrow/pandas_compat.py

@@ -15,7 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from distutils.version import LooseVersion
 import ast
 import collections
 import json
@@ -27,173 +26,10 @@ import numpy as np
 import six
 
 import pyarrow as pa
+from pyarrow.lib import _pandas_api
 from pyarrow.compat import (builtin_pickle, PY2, zip_longest)  # noqa
 
 
-def _imported_property(f):
-    @property
-    def wrapper(self):
-        if self._have_pandas:
-            return f(self)
-        self._check_import()
-        return f(self)
-    return wrapper
-
-
-class _PandasAPI(object):
-    """
-    Lazy pandas importer that isolates usages of pandas APIs and avoids
-    importing pandas until it's actually needed
-    """
-    _instance = None
-
-    @classmethod
-    def get_instance(cls):
-        if cls._instance is None:
-            cls._instance = _PandasAPI()
-        return cls._instance
-
-    def __init__(self):
-        self._imported_pandas = False
-        self._have_pandas = None
-        self._pd = None
-        self._loose_version = None
-        self._version = None
-        self._compat_module = None
-        self._types_api = None
-        self._data_frame = None
-        self._series = None
-        self._categorical_type = None
-        self._datetimetz_type = None
-        self._get_datetimetz_type = None
-
-    def make_series(self, *args, **kwargs):
-        self._check_import()
-        return self._series(*args, **kwargs)
-
-    def data_frame(self, *args, **kwargs):
-        self._check_import()
-        return self._data_frame(*args, **kwargs)
-
-    @property
-    def have_pandas(self):
-        if self._have_pandas is None:
-            self._check_import(raise_=False)
-        return self._have_pandas
-
-    @_imported_property
-    def compat(self):
-        return self._compat_module
-
-    @_imported_property
-    def pd(self):
-        return self._pd
-
-    def infer_dtype(self, obj):
-        self._check_import()
-        try:
-            return self._types_api.infer_dtype(obj, skipna=False)
-        except AttributeError:
-            return self._pd.lib.infer_dtype(obj)
-
-    @_imported_property
-    def loose_version(self):
-        return self._loose_version
-
-    @_imported_property
-    def version(self):
-        return self._version
-
-    @_imported_property
-    def categorical_type(self):
-        return self._categorical_type
-
-    @_imported_property
-    def datetimetz_type(self):
-        return self._datetimetz_type
-
-    def _check_import(self, raise_=True):
-        if self._imported_pandas:
-            return
-
-        try:
-            import pandas as pd
-            import pyarrow.pandas_compat as pdcompat
-        except ImportError:
-            if raise_:
-                raise
-            else:
-                return
-
-        self._pd = pd
-        self._compat_module = pdcompat
-        self._data_frame = pd.DataFrame
-        self._series = pd.Series
-        self._have_pandas = True
-
-        self._loose_version = LooseVersion(pd.__version__)
-        if self._loose_version >= '0.20.0':
-            from pandas.api.types import DatetimeTZDtype
-            self._types_api = pd.api.types
-        elif self._loose_version >= '0.19.0':
-            from pandas.types.dtypes import DatetimeTZDtype
-            self._types_api = pd.api.types
-        else:
-            from pandas.types.dtypes import DatetimeTZDtype
-            self._types_api = pd.core.common
-
-        self._datetimetz_type = DatetimeTZDtype
-        self._categorical_type = pd.Categorical
-
-    def is_array_like(self, obj):
-        self._check_import()
-        return isinstance(obj, (self._pd.Series, self._pd.Index,
-                                self._categorical_type))
-
-    def is_categorical(self, obj):
-        if self.have_pandas:
-            return isinstance(obj, self._categorical_type)
-        else:
-            return False
-
-    def is_datetimetz(self, obj):
-        if self.have_pandas:
-            return isinstance(obj, self._datetimetz_type)
-        else:
-            return False
-
-    def is_data_frame(self, obj):
-        if self.have_pandas:
-            return isinstance(obj, self._data_frame)
-        else:
-            return False
-
-    def is_series(self, obj):
-        if self.have_pandas:
-            return isinstance(obj, self._series)
-        else:
-            return False
-
-    def dataframe_to_arrays(self, *args, **kwargs):
-        return _dataframe_to_arrays(*args, **kwargs)
-
-    def dataframe_to_types(self, *args, **kwargs):
-        return _dataframe_to_types(*args, **kwargs)
-
-    def table_to_blockmanager(self, *args, **kwargs):
-        return _table_to_blockmanager(*args, **kwargs)
-
-    def make_datetimetz(self, tz):
-        return _make_datetimetz(tz)
-
-    def assert_frame_equal(self, *args, **kwargs):
-        self._check_import()
-        return self._pd.util.testing.assert_frame_equal
-
-
-_pandas_api = _PandasAPI.get_instance()
-
-
 _logical_type_map = {}
 
 
@@ -572,7 +408,7 @@ def _resolve_columns_of_interest(df, schema, columns):
     return columns
 
 
-def _dataframe_to_types(df, preserve_index, columns=None):
+def dataframe_to_types(df, preserve_index, columns=None):
     (all_names,
      column_names,
      index_descriptors,
@@ -599,8 +435,8 @@ def _dataframe_to_types(df, preserve_index, columns=None):
     return all_names, types, metadata
 
 
-def _dataframe_to_arrays(df, schema, preserve_index, nthreads=1, columns=None,
-                         safe=True):
+def dataframe_to_arrays(df, schema, preserve_index, nthreads=1, columns=None,
+                        safe=True):
     (all_names,
      column_names,
      index_descriptors,
@@ -732,7 +568,7 @@ def _reconstruct_block(item):
         block = _int.make_block(cat, placement=placement,
                                 klass=_int.CategoricalBlock)
     elif 'timezone' in item:
-        dtype = _make_datetimetz(item['timezone'])
+        dtype = make_datetimetz(item['timezone'])
         block = _int.make_block(block_arr, placement=placement,
                                 klass=_int.DatetimeTZBlock,
                                 dtype=dtype)
@@ -745,7 +581,7 @@ def _reconstruct_block(item):
     return block
 
 
-def _make_datetimetz(tz):
+def make_datetimetz(tz):
     tz = pa.lib.string_to_tzinfo(tz)
     return _pandas_api.datetimetz_type('ns', tz=tz)
 
@@ -754,8 +590,8 @@ def _make_datetimetz(tz):
 # Converting pyarrow.Table efficiently to pandas.DataFrame
 
 
-def _table_to_blockmanager(options, table, categories=None,
-                           ignore_metadata=False):
+def table_to_blockmanager(options, table, categories=None,
+                          ignore_metadata=False):
     from pandas.core.internals import BlockManager
 
     all_columns = []

python/pyarrow/table.pxi

@@ -15,11 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import json
-
-from collections import OrderedDict
-
-
 cdef class ChunkedArray(_PandasConvertible):
     """
     Array backed via one or more memory chunks.
@@ -465,7 +460,7 @@ cdef class Column(_PandasConvertible):
 
     def _to_pandas(self, options, **kwargs):
         values = self.data._to_pandas(options)
-        result = _pandas_api.make_series(values, name=self.name)
+        result = pandas_api.make_series(values, name=self.name)
 
         if isinstance(self.type, TimestampType):
             tz = self.type.tz
@@ -859,7 +854,8 @@ cdef class RecordBatch(_PandasConvertible):
         -------
         pyarrow.RecordBatch
         """
-        names, arrays, metadata = _pandas_api.dataframe_to_arrays(
+        from pyarrow.pandas_compat import dataframe_to_arrays
+        names, arrays, metadata = dataframe_to_arrays(
             df, schema, preserve_index, nthreads=nthreads, columns=columns
         )
         return cls.from_arrays(arrays, names, metadata)
@@ -1139,7 +1135,8 @@ cdef class Table(_PandasConvertible):
         >>> pa.Table.from_pandas(df)
         <pyarrow.lib.Table object at 0x7f05d1fb1b40>
         """
-        names, arrays, metadata = _pandas_api.dataframe_to_arrays(
+        from pyarrow.pandas_compat import dataframe_to_arrays
+        names, arrays, metadata = dataframe_to_arrays(
             df,
             schema=schema,
             preserve_index=preserve_index,
@@ -1295,10 +1292,11 @@ cdef class Table(_PandasConvertible):
         return result
 
     def _to_pandas(self, options, categories=None, ignore_metadata=False):
-        mgr = _pandas_api.table_to_blockmanager(
+        from pyarrow.pandas_compat import table_to_blockmanager
+        mgr = table_to_blockmanager(
             options, self, categories,
             ignore_metadata=ignore_metadata)
-        return _pandas_api.data_frame(mgr)
+        return pandas_api.data_frame(mgr)
 
     def to_pydict(self):
         """

python/pyarrow/types.pxi

@@ -388,7 +388,8 @@ cdef class TimestampType(DataType):
             return _pandas_type_map[_Type_TIMESTAMP]
         else:
             # Return DatetimeTZ
-            return _pandas_api.make_datetimetz(self.tz)
+            from pyarrow.pandas_compat import make_datetimetz
+            return make_datetimetz(self.tz)
 
     def __reduce__(self):
         return timestamp, (self.unit, self.tz)
@@ -745,7 +746,8 @@ cdef class Schema:
         str: string
         __index_level_0__: int64
         """
-        names, types, metadata = _pandas_api.dataframe_to_types(
+        from pyarrow.pandas_compat import dataframe_to_types
+        names, types, metadata = dataframe_to_types(
             df,
             preserve_index=preserve_index
         )