compatibility with py35

python/pyarrow/filesystem.py

@@ -16,26 +16,14 @@
 # under the License.
 
 import os
-import six
 import inspect
 import posixpath
 
 from os.path import join as pjoin
 from six.moves.urllib.parse import urlparse
 
-try:
-    # pathlib might not be available
-    try:
-        import pathlib
-    except ImportError:
-        # python 2 backport
-        import pathlib2 as pathlib
-    _has_pathlib = True
-except ImportError:
-    _has_pathlib = False
-
 import pyarrow as pa
-from pyarrow.util import implements
+from pyarrow.util import implements, _stringify_path
 
 
 class FileSystem(object):
@@ -84,7 +72,7 @@ class FileSystem(object):
         -------
         usage : int
         """
-        path = _ensure_str_path(path)
+        path = _stringify_path(path)
         path_info = self.stat(path)
         if path_info['kind'] == 'file':
             return path_info['size']
@@ -216,12 +204,12 @@ class LocalFileSystem(FileSystem):
 
     @implements(FileSystem.ls)
     def ls(self, path):
-        path = _ensure_str_path(path)
+        path = _stringify_path(path)
         return sorted(pjoin(path, x) for x in os.listdir(path))
 
     @implements(FileSystem.mkdir)
     def mkdir(self, path, create_parents=True):
-        path = _ensure_str_path(path)
+        path = _stringify_path(path)
         if create_parents:
             os.makedirs(path)
         else:
@@ -229,12 +217,12 @@ class LocalFileSystem(FileSystem):
 
     @implements(FileSystem.isdir)
     def isdir(self, path):
-        path = _ensure_str_path(path)
+        path = _stringify_path(path)
         return os.path.isdir(path)
 
     @implements(FileSystem.isfile)
     def isfile(self, path):
-        path = _ensure_str_path(path)
+        path = _stringify_path(path)
         return os.path.isfile(path)
 
     @implements(FileSystem._isfilestore)
@@ -243,7 +231,7 @@ class LocalFileSystem(FileSystem):
 
     @implements(FileSystem.exists)
     def exists(self, path):
-        path = _ensure_str_path(path)
+        path = _stringify_path(path)
         return os.path.exists(path)
 
     @implements(FileSystem.open)
@@ -251,7 +239,7 @@ class LocalFileSystem(FileSystem):
         """
         Open file for reading or writing
         """
-        path = _ensure_str_path(path)
+        path = _stringify_path(path)
         return open(path, mode=mode)
 
     @property
@@ -262,7 +250,7 @@ class LocalFileSystem(FileSystem):
         """
         Directory tree generator, see os.walk
         """
-        path = _ensure_str_path(path)
+        path = _stringify_path(path)
         return os.walk(path)
 
 
@@ -292,17 +280,17 @@ class DaskFileSystem(FileSystem):
 
     @implements(FileSystem.delete)
     def delete(self, path, recursive=False):
-        path = _ensure_str_path(path)
+        path = _stringify_path(path)
         return self.fs.rm(path, recursive=recursive)
 
     @implements(FileSystem.exists)
     def exists(self, path):
-        path = _ensure_str_path(path)
+        path = _stringify_path(path)
         return self.fs.exists(path)
 
     @implements(FileSystem.mkdir)
     def mkdir(self, path, create_parents=True):
-        path = _ensure_str_path(path)
+        path = _stringify_path(path)
         if create_parents:
             return self.fs.mkdirs(path)
         else:
@@ -313,18 +301,18 @@ class DaskFileSystem(FileSystem):
         """
         Open file for reading or writing
         """
-        path = _ensure_str_path(path)
+        path = _stringify_path(path)
         return self.fs.open(path, mode=mode)
 
     def ls(self, path, detail=False):
-        path = _ensure_str_path(path)
+        path = _stringify_path(path)
         return self.fs.ls(path, detail=detail)
 
     def walk(self, path):
         """
         Directory tree generator, like os.walk
         """
-        path = _ensure_str_path(path)
+        path = _stringify_path(path)
         return self.fs.walk(path)
 
 
@@ -332,7 +320,7 @@ class S3FSWrapper(DaskFileSystem):
 
     @implements(FileSystem.isdir)
     def isdir(self, path):
-        path = _ensure_str_path(path)
+        path = _stringify_path(path)
         try:
             contents = self.fs.ls(path)
             if len(contents) == 1 and contents[0] == path:
@@ -344,7 +332,7 @@ class S3FSWrapper(DaskFileSystem):
 
     @implements(FileSystem.isfile)
     def isfile(self, path):
-        path = _ensure_str_path(path)
+        path = _stringify_path(path)
         try:
             contents = self.fs.ls(path)
             return len(contents) == 1 and contents[0] == path
@@ -358,7 +346,7 @@ class S3FSWrapper(DaskFileSystem):
         Generator version of what is in s3fs, which yields a flattened list of
         files
         """
-        path = _ensure_str_path(path).replace('s3://', '')
+        path = _stringify_path(path).replace('s3://', '')
         directories = set()
         files = set()
 
@@ -384,21 +372,6 @@ class S3FSWrapper(DaskFileSystem):
                 yield tup
 
 
-def is_path(x):
-    return (isinstance(x, six.string_types)
-            or (_has_pathlib and isinstance(x, pathlib.Path)))
-
-
-def _ensure_str_path(path):
-    if isinstance(path, six.string_types):
-        return path
-    elif _has_pathlib and isinstance(path, pathlib.Path):
-        return str(path)
-    else:
-        raise TypeError('Path must be a string or an instance of pathlib.Path,'
-                        ' got object with type: {}'.format(type(path)))
-
-
 def _ensure_filesystem(fs):
     fs_type = type(fs)
 
@@ -423,7 +396,7 @@ def _get_fs_from_path(path):
     return filesystem from path which could be an HDFS URI
     """
     # input can be hdfs URI such as hdfs://host:port/myfile.parquet
-    path = _ensure_str_path(path)
+    path = _stringify_path(path)
     # if _has_pathlib and isinstance(path, pathlib.Path):
     #     path = str(path)
     parsed_uri = urlparse(path)

python/pyarrow/io.pxi

@@ -19,8 +19,6 @@
 # arrow::ipc
 
 from libc.stdlib cimport malloc, free
-from pyarrow.compat import builtin_pickle, frombytes, tobytes, encode_file_path
-from io import BufferedIOBase, UnsupportedOperation
 
 import re
 import six
@@ -28,6 +26,10 @@ import sys
 import threading
 import time
 import warnings
+from io import BufferedIOBase, UnsupportedOperation
+
+from pyarrow.util import _stringify_path
+from pyarrow.compat import builtin_pickle, frombytes, tobytes, encode_file_path
 
 
 # 64K
@@ -40,18 +42,6 @@ cdef extern from "Python.h":
         char *v, Py_ssize_t len) except NULL
 
 
-def _stringify_path(path):
-    """
-    Convert *path* to a string or unicode path if possible.
-    """
-    if isinstance(path, six.string_types):
-        return path
-    try:
-        return path.__fspath__()
-    except AttributeError:
-        raise TypeError("not a path-like object")
-
-
 cdef class NativeFile:
 
     def __cinit__(self):

python/pyarrow/parquet.py

@@ -23,15 +23,16 @@ import re
 
 import numpy as np
 
-from pyarrow.filesystem import (LocalFileSystem, is_path,  _ensure_str_path,
-                                _ensure_filesystem, _get_fs_from_path)
+import pyarrow as pa
+import pyarrow._parquet as _parquet
+import pyarrow.lib as lib
 from pyarrow._parquet import (ParquetReader, RowGroupStatistics,  # noqa
                               FileMetaData, RowGroupMetaData,
                               ColumnChunkMetaData,
                               ParquetSchema, ColumnSchema)
-import pyarrow._parquet as _parquet
-import pyarrow.lib as lib
-import pyarrow as pa
+from pyarrow.filesystem import (LocalFileSystem, _ensure_filesystem,
+                                _get_fs_from_path)
+from pyarrow.util import _is_path_like, _stringify_path
 
 
 # ----------------------------------------------------------------------
@@ -44,7 +45,7 @@ class ParquetFile(object):
 
     Parameters
     ----------
-    source : str, pyarrow.NativeFile, or file-like object
+    source : str, pathlib.Path, pyarrow.NativeFile, or file-like object
         Readable source. For passing bytes or buffer-like file containing a
         Parquet file, use pyarorw.BufferReader
     metadata : ParquetFileMetadata, default None
@@ -288,7 +289,7 @@ schema : arrow Schema
         # to be closed
         self.file_handle = None
 
-        if is_path(where):
+        if _is_path_like(where):
             fs = _get_fs_from_path(where)
             sink = self.file_handle = fs.open(where, 'wb')
         else:
@@ -363,7 +364,7 @@ class ParquetDatasetPiece(object):
     """
 
     def __init__(self, path, row_group=None, partition_keys=None):
-        self.path = _ensure_str_path(path)
+        self.path = _stringify_path(path)
         self.row_group = row_group
         self.partition_keys = partition_keys or []
 
@@ -634,7 +635,7 @@ class ParquetManifest(object):
                  partition_scheme='hive', metadata_nthreads=1):
         self.filesystem = filesystem or _get_fs_from_path(dirpath)
         self.pathsep = pathsep
-        self.dirpath = _ensure_str_path(dirpath)
+        self.dirpath = _stringify_path(dirpath)
         self.partition_scheme = partition_scheme
         self.partitions = ParquetPartitions()
         self.pieces = []
@@ -953,7 +954,7 @@ def _make_manifest(path_or_paths, fs, pathsep='/', metadata_nthreads=1):
         # Dask passes a directory as a list of length 1
         path_or_paths = path_or_paths[0]
 
-    if is_path(path_or_paths) and fs.isdir(path_or_paths):
+    if _is_path_like(path_or_paths) and fs.isdir(path_or_paths):
         manifest = ParquetManifest(path_or_paths, filesystem=fs,
                                    pathsep=fs.pathsep,
                                    metadata_nthreads=metadata_nthreads)
@@ -1008,7 +1009,7 @@ Returns
 
 def read_table(source, columns=None, nthreads=1, metadata=None,
                use_pandas_metadata=False):
-    if is_path(source):
+    if _is_path_like(source):
         fs = _get_fs_from_path(source)
         return fs.read_parquet(source, columns=columns, metadata=metadata,
                                use_pandas_metadata=use_pandas_metadata)
@@ -1060,9 +1061,9 @@ def write_table(table, where, row_group_size=None, version='1.0',
                 **kwargs) as writer:
             writer.write_table(table, row_group_size=row_group_size)
     except Exception:
-        if is_path(where):
+        if _is_path_like(where):
             try:
-                os.remove(_ensure_str_path(where))
+                os.remove(_stringify_path(where))
             except os.error:
                 pass
         raise

python/pyarrow/tests/conftest.py

@@ -126,7 +126,7 @@ def pytest_runtest_setup(item):
 @pytest.fixture
 def tempdir(tmpdir):
     # convert pytest's LocalPath to pathlib.Path
-    return pathlib.Path(tmpdir)
+    return pathlib.Path(tmpdir.strpath)
 
 
 @pytest.fixture(scope='session')

python/pyarrow/util.py

@@ -15,9 +15,20 @@
 # specific language governing permissions and limitations
 # under the License.
 
+# Miscellaneous utility code
+
+import six
 import warnings
 
-# Miscellaneous utility code
+try:
+    # pathlib might not be available
+    try:
+        import pathlib
+    except ImportError:
+        import pathlib2 as pathlib  # python 2 backport
+    _has_pathlib = True
+except ImportError:
+    _has_pathlib = False
 
 
 def implements(f):
@@ -35,3 +46,26 @@ def _deprecate_api(old_name, new_name, api, next_version):
         warnings.warn(msg, FutureWarning)
         return api(*args)
     return wrapper
+
+
+def _is_path_like(path):
+    # PEP519 filesystem path protocol is available from python 3.6, so pathlib
+    # doesn't implement __fspath__ for earlier versions
+    return (isinstance(path, six.string_types) or
+            isinstance(path, pathlib.Path) or
+            hasattr(path, '__fspath__'))
+
+
+def _stringify_path(path):
+    """
+    Convert *path* to a string or unicode path if possible.
+    """
+    if isinstance(path, six.string_types):
+        return path
+    elif isinstance(path, pathlib.Path):
+        return str(path)
+    else:
+        try:
+            return path.__fspath__()  # new in python 3.6
+        except AttributeError:
+            raise TypeError("not a path-like object")